From 1f33860bfcd353c472881a8db513ec956ed43193 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 18 Jun 2024 10:07:40 +0200 Subject: [PATCH 01/15] update mongo sink adding ssl options --- .../backends/mongo/MongoBackendImpl.java | 18 ++++++++++-- .../backends/mongo/MongoBackendImplTest.java | 14 ++++----- .../iot/cygnus/sinks/NGSIMongoBaseSink.java | 29 ++++++++++++++++++- 3 files changed, 50 insertions(+), 11 deletions(-) diff --git a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java index 9b263e01e..d5de16376 100644 --- a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java +++ b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java @@ -60,6 +60,8 @@ public enum Resolution { SECOND, MINUTE, HOUR, DAY, MONTH } private final String mongoPassword; private final String mongoAuthSource; private final String mongoReplicaSet; + private final Boolean sslEnabled; + private final Boolean sslInvalidHostNameAllowed; private final DataModel dataModel; private static final CygnusLogger LOGGER = new CygnusLogger(MongoBackendImpl.class); @@ -73,13 +75,16 @@ public enum Resolution { SECOND, MINUTE, HOUR, DAY, MONTH } * @param dataModel */ public MongoBackendImpl(String mongoHosts, String mongoUsername, String mongoPassword, - String mongoAuthSource, String mongoReplicaSet, DataModel dataModel) { + String mongoAuthSource, String mongoReplicaSet, DataModel dataModel, + Boolean sslEnabled, Boolean sslInvalidHostNameAllowed) { client = null; this.mongoHosts = mongoHosts; this.mongoUsername = mongoUsername; this.mongoPassword = mongoPassword; this.mongoAuthSource = mongoAuthSource; this.mongoReplicaSet = mongoReplicaSet; + this.sslEnabled = sslEnabled; + this.sslInvalidHostNameAllowed = sslInvalidHostNameAllowed; this.dataModel = dataModel; } // MongoBackendImpl @@ -592,6 +597,7 @@ private MongoDatabase getDatabase(String dbName) { } MongoCredential credential = MongoCredential.createCredential(mongoUsername, authSource, mongoPassword.toCharArray()); + /**** // This constructor is deprecated see Mongo Client API documentation // @deprecated Prefer {@link #MongoClient(List, MongoCredential, MongoClientOptions)} @@ -599,9 +605,15 @@ private MongoDatabase getDatabase(String dbName) { ****/ if ((mongoReplicaSet!= null) && !mongoReplicaSet.isEmpty()) { client = new MongoClient(servers, credential, new MongoClientOptions.Builder(). - requiredReplicaSetName(mongoReplicaSet).build()); + requiredReplicaSetName(mongoReplicaSet). + sslEnabled(sslEnabled). + sslInvalidHostNameAllowed(sslInvalidHostNameAllowed). + build()); } else { - client = new MongoClient(servers, credential, new MongoClientOptions.Builder().build()); + client = new MongoClient(servers, credential, new MongoClientOptions.Builder(). + sslEnabled(sslEnabled). + sslInvalidHostNameAllowed(sslInvalidHostNameAllowed). + build()); } } else { client = new MongoClient(servers); diff --git a/cygnus-common/src/test/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImplTest.java b/cygnus-common/src/test/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImplTest.java index 594c986cc..435c9a761 100644 --- a/cygnus-common/src/test/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImplTest.java +++ b/cygnus-common/src/test/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImplTest.java @@ -53,7 +53,7 @@ public MongoBackendImplTest() { public void testGetRange() { System.out.println(getTestTraceHead("[MongoBackendImpl.getRange]") + "-------- Given a resolution, its related range is correctly returned"); - MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null); + MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, false, false); try { @@ -115,7 +115,7 @@ public void testGetRange() { public void testGetOrigin() { System.out.println(getTestTraceHead("[MongoBackendImpl.getOrigin]") + "-------- Given a calendar and a resolution, its related origin is correctly returned"); - MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null); + MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, false, false); GregorianCalendar calendar = new GregorianCalendar(2017, 4, 5, 11, 46, 13); try { @@ -198,7 +198,7 @@ public void testGetOrigin() { public void testGetOffset() { System.out.println(getTestTraceHead("[MongoBackendImpl.getOffset]") + "-------- Given a calendar and a resolution, its related offset is correctly returned"); - MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null); + MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, false, false); GregorianCalendar calendar = new GregorianCalendar(2017, 3, 5, 11, 46, 13); // month 3 is April try { @@ -266,7 +266,7 @@ public void testBuildQueryForInsertAggregated() { String entityType = "someType"; String attrName = "someName"; GregorianCalendar calendar = new GregorianCalendar(2017, 3, 5, 11, 46, 13); // month 3 is April - MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, DataModel.DMBYSERVICEPATH); + MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, DataModel.DMBYSERVICEPATH, false, false); String queryForInsertAggregated = "{\"_id\": {\"entityId\": \"someId\", \"entityType\": \"someType\", " + "\"attrName\": \"someName\", \"origin\": {\"$date\": 1491392760000}, " + "\"resolution\": \"second\", \"range\": \"minute\"}, \"points.offset\": 13}"; @@ -353,7 +353,7 @@ public void testBuildQueryForInsertAggregated() { throw e; } // try catch - backend = new MongoBackendImpl(null, null, null, null, null, DataModel.DMBYENTITY); + backend = new MongoBackendImpl(null, null, null, null, null, DataModel.DMBYENTITY, false, false); queryForInsertAggregated = "{\"_id\": {\"attrName\": \"someName\", " + "\"origin\": {\"$date\": 1491392760000}, \"resolution\": \"second\", " @@ -457,7 +457,7 @@ public void testBuildUpdateForUpdateNumerical() { double sum2 = 200; int numSamples = 2; GregorianCalendar calendar = new GregorianCalendar(2017, 3, 5, 11, 46, 13); // month 3 is April - MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null,null); + MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, false, false); String updateForUpdate = "{\"$set\": {\"attrType\": \"someType\"}, " + "\"$inc\": {\"points.$.samples\": 2, \"points.$.sum\": 20.0, \"points.$.sum2\": 200.0}, " + "\"$min\": {\"points.$.min\": 0.0}, \"$max\": {\"points.$.max\": 10.0}}"; @@ -489,7 +489,7 @@ public void testBuildUpdateForUpdateString() { String value = "someString"; int count = 2; GregorianCalendar calendar = new GregorianCalendar(2017, 3, 5, 11, 46, 13); // month 3 is April - MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null); + MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, false, false); String updateForUpdate = "{\"$set\": {\"attrType\": \"someType\"}, " + "\"$inc\": {\"points.13.samples\": 2, \"points.13.occur.someString\": 2}}"; diff --git a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSIMongoBaseSink.java b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSIMongoBaseSink.java index deb1995ab..a3f115fbb 100644 --- a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSIMongoBaseSink.java +++ b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSIMongoBaseSink.java @@ -39,6 +39,8 @@ public abstract class NGSIMongoBaseSink extends NGSISink { protected String mongoPassword; protected String mongoAuthSource; protected String mongoReplicaSet; + protected Boolean sslEnabled; + protected Boolean sslInvalidHostNameAllowed; protected String dbPrefix; protected String collectionPrefix; protected MongoBackendImpl backend; @@ -169,12 +171,37 @@ public void configure(Context context) { LOGGER.warn("[" + this.getName() + "] Invalid configuration (ignore_white_spaces=" + ignoreWhiteSpacesStr + ") -- Must be 'true' or 'false'"); } // if else + + String sslEnabledStr = context.getString("sslEnabled", "true"); + if (sslEnabledStr.equals("true") || sslEnabledStr.equals("false")) { + sslEnabled = Boolean.valueOf(sslEnabledStr); + LOGGER.debug("[" + this.getName() + "] Reading configuration (sslEnabled=" + + sslEnabledStr + ")"); + } else { + invalidConfiguration = true; + LOGGER.warn("[" + this.getName() + "] Invalid configuration (sslEnabled=" + + sslEnabledStr + ") -- Must be 'true' or 'false'"); + } // if else + + String sslInvalidHostNameAllowedStr = context.getString("sslInvalidHostNameAllowed", "true"); + if (sslInvalidHostNameAllowedStr.equals("true") || sslInvalidHostNameAllowedStr.equals("false")) { + sslInvalidHostNameAllowed = Boolean.valueOf(sslInvalidHostNameAllowedStr); + LOGGER.debug("[" + this.getName() + "] Reading configuration (sslInvalidHostNameAllowed=" + + sslInvalidHostNameAllowedStr + ")"); + } else { + invalidConfiguration = true; + LOGGER.warn("[" + this.getName() + "] Invalid configuration (sslInvalidHostNameAllowed=" + + sslInvalidHostNameAllowedStr + ") -- Must be 'true' or 'false'"); + } // if else + } // configure @Override public void start() { try { - backend = new MongoBackendImpl(mongoHosts, mongoUsername, mongoPassword, mongoAuthSource, mongoReplicaSet, dataModel); + backend = new MongoBackendImpl(mongoHosts, mongoUsername, mongoPassword, + mongoAuthSource, mongoReplicaSet, dataModel, + sslEnabled, sslInvalidHostNameAllowed); LOGGER.debug("[" + this.getName() + "] MongoDB persistence backend created"); } catch (Exception e) { LOGGER.error("Error while creating the MongoDB persistence backend. Details=" From aa3f058b1ea0ad0d553fda33bedac2ea72b310d2 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 18 Jun 2024 10:27:49 +0200 Subject: [PATCH 02/15] update doc --- .../iot/cygnus/sinks/NGSIMongoBaseSink.java | 12 ++++++------ .../flume_extensions_catalogue/ngsi_mongo_sink.md | 2 ++ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSIMongoBaseSink.java b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSIMongoBaseSink.java index a3f115fbb..0bf9045e3 100644 --- a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSIMongoBaseSink.java +++ b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSIMongoBaseSink.java @@ -172,25 +172,25 @@ public void configure(Context context) { + ignoreWhiteSpacesStr + ") -- Must be 'true' or 'false'"); } // if else - String sslEnabledStr = context.getString("sslEnabled", "true"); + String sslEnabledStr = context.getString("mongo_ssl", "false"); if (sslEnabledStr.equals("true") || sslEnabledStr.equals("false")) { sslEnabled = Boolean.valueOf(sslEnabledStr); - LOGGER.debug("[" + this.getName() + "] Reading configuration (sslEnabled=" + LOGGER.debug("[" + this.getName() + "] Reading configuration (mongo_ssl=" + sslEnabledStr + ")"); } else { invalidConfiguration = true; - LOGGER.warn("[" + this.getName() + "] Invalid configuration (sslEnabled=" + LOGGER.warn("[" + this.getName() + "] Invalid configuration (mongo_ssl=" + sslEnabledStr + ") -- Must be 'true' or 'false'"); } // if else - String sslInvalidHostNameAllowedStr = context.getString("sslInvalidHostNameAllowed", "true"); + String sslInvalidHostNameAllowedStr = context.getString("mongo_ssl_invalid_host_allowed", "false"); if (sslInvalidHostNameAllowedStr.equals("true") || sslInvalidHostNameAllowedStr.equals("false")) { sslInvalidHostNameAllowed = Boolean.valueOf(sslInvalidHostNameAllowedStr); - LOGGER.debug("[" + this.getName() + "] Reading configuration (sslInvalidHostNameAllowed=" + LOGGER.debug("[" + this.getName() + "] Reading configuration (mongo_ssl_invalid_host_allowed=" + sslInvalidHostNameAllowedStr + ")"); } else { invalidConfiguration = true; - LOGGER.warn("[" + this.getName() + "] Invalid configuration (sslInvalidHostNameAllowed=" + LOGGER.warn("[" + this.getName() + "] Invalid configuration (mongo_ssl_invalid_host_allowed=" + sslInvalidHostNameAllowedStr + ") -- Must be 'true' or 'false'"); } // if else diff --git a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md index 6f99b5948..189576133 100644 --- a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md +++ b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md @@ -320,6 +320,8 @@ When datamodel changes Cygnus tries to recreate index (delete current and create | mongo\_auth_source | no | empty | Auth source database use to authenticate the user. Usually could be `admin`. | | mongo\_replica_set | no | empty | Replica Set name. Note that this parameter is optional because Cygnus is able to connect to a MongoDB replica set without needing to specify its name.| | db\_prefix | no | sth_ || +| mongo\_ssl | no | false | Enable SSL in mongodb connection | +| mongo\_ssl\_invalid\_host\_allowed | no | false | Allow invalid host name in mongo SSL connections | | collection\_prefix | no | sth_ | `system.` is not accepted. | | batch\_size | no | 1 | Number of events accumulated before persistence. | | batch\_timeout | no | 30 | Number of seconds the batch will be building before it is persisted as it is. | From 489c578840d45fd89f0bb66a2ae25ab7dcb30a50 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 18 Jun 2024 10:30:27 +0200 Subject: [PATCH 03/15] update CNR --- CHANGES_NEXT_RELEASE | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index e69de29bb..b6aedbd95 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -0,0 +1,2 @@ +[cygnus-ngsi] [mongo-sink] Add mongo_ssl and mongo_ssl_invalid_host_allowed options for mongoDB connections +[cygnus-common] [mongo-backend] Use sslEnabled and sslInvalidHostNameAllowed options for mongoDB connections From 6b46e17c20e4489ae667b10e8d31474d100e6aa1 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 18 Jun 2024 11:36:55 +0200 Subject: [PATCH 04/15] add missed MongoClientOptions --- .../iot/cygnus/backends/mongo/MongoBackendImpl.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java index d5de16376..6e24e0793 100644 --- a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java +++ b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java @@ -616,7 +616,11 @@ private MongoDatabase getDatabase(String dbName) { build()); } } else { - client = new MongoClient(servers); + MongoClientOptions options = MongoClientOptions.builder() + .sslEnabled(sslEnabled) + .sslInvalidHostNameAllowed(sslInvalidHostNameAllowed) + .build(); + client = new MongoClient(servers, options); } // if else } // if From 171b7c14820529f39e7c541f554de7a2283853f8 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 18 Jun 2024 13:22:27 +0200 Subject: [PATCH 05/15] add ssl context to mongoClientOptions --- .../iot/cygnus/backends/mongo/MongoBackendImpl.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java index 6e24e0793..9c8a0c8bc 100644 --- a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java +++ b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java @@ -40,6 +40,8 @@ import java.util.List; import java.util.TimeZone; import java.util.concurrent.TimeUnit; +import javax.net.ssl.SSLContext; +import java.security.NoSuchAlgorithmException; import org.bson.Document; /** @@ -588,6 +590,14 @@ private MongoDatabase getDatabase(String dbName) { // create a Mongo client if (client == null) { + SSLContext sslContext = null; + if (sslEnabled) { + try { + sslContext = SSLContext.getInstance("TLS"); + } catch (NoSuchAlgorithmException e) { + LOGGER.warn("Error with TLS algorithm " + e.getMessage()); + } + } if (mongoUsername.length() != 0) { String authSource; if ((mongoAuthSource != null) && !mongoAuthSource.isEmpty()) { @@ -608,17 +618,20 @@ private MongoDatabase getDatabase(String dbName) { requiredReplicaSetName(mongoReplicaSet). sslEnabled(sslEnabled). sslInvalidHostNameAllowed(sslInvalidHostNameAllowed). + sslContext(sslContext). build()); } else { client = new MongoClient(servers, credential, new MongoClientOptions.Builder(). sslEnabled(sslEnabled). sslInvalidHostNameAllowed(sslInvalidHostNameAllowed). + sslContext(sslContext). build()); } } else { MongoClientOptions options = MongoClientOptions.builder() .sslEnabled(sslEnabled) .sslInvalidHostNameAllowed(sslInvalidHostNameAllowed) + .sslContext(sslContext) .build(); client = new MongoClient(servers, options); } // if else From c72263a7cd1b9a49b9e24923e1e35209048f8b9f Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 19 Jun 2024 08:21:42 +0200 Subject: [PATCH 06/15] init SSLContext with a keystore based on a file --- .../backends/mongo/MongoBackendImpl.java | 19 +++++++++++++++++-- .../backends/mongo/MongoBackendImplTest.java | 14 +++++++------- .../iot/cygnus/sinks/NGSIMongoBaseSink.java | 6 +++++- 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java index 9c8a0c8bc..f846a05b9 100644 --- a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java +++ b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java @@ -42,6 +42,11 @@ import java.util.concurrent.TimeUnit; import javax.net.ssl.SSLContext; import java.security.NoSuchAlgorithmException; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManagerFactory; +import java.security.KeyStore; +import java.io.FileInputStream; +import java.io.InputStream; import org.bson.Document; /** @@ -64,6 +69,7 @@ public enum Resolution { SECOND, MINUTE, HOUR, DAY, MONTH } private final String mongoReplicaSet; private final Boolean sslEnabled; private final Boolean sslInvalidHostNameAllowed; + private final String sslKeystorePathFile; private final DataModel dataModel; private static final CygnusLogger LOGGER = new CygnusLogger(MongoBackendImpl.class); @@ -78,7 +84,7 @@ public enum Resolution { SECOND, MINUTE, HOUR, DAY, MONTH } */ public MongoBackendImpl(String mongoHosts, String mongoUsername, String mongoPassword, String mongoAuthSource, String mongoReplicaSet, DataModel dataModel, - Boolean sslEnabled, Boolean sslInvalidHostNameAllowed) { + Boolean sslEnabled, Boolean sslInvalidHostNameAllowed, String sslKeystorePathFile) { client = null; this.mongoHosts = mongoHosts; this.mongoUsername = mongoUsername; @@ -87,6 +93,7 @@ public MongoBackendImpl(String mongoHosts, String mongoUsername, String mongoPas this.mongoReplicaSet = mongoReplicaSet; this.sslEnabled = sslEnabled; this.sslInvalidHostNameAllowed = sslInvalidHostNameAllowed; + this.sslKeystorePathFile = sslKeystorePathFile; this.dataModel = dataModel; } // MongoBackendImpl @@ -590,11 +597,19 @@ private MongoDatabase getDatabase(String dbName) { // create a Mongo client if (client == null) { + SSLContext sslContext = null; if (sslEnabled) { try { + // Init TrustManager to init SSL Context + KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType()); + try (InputStream keyStoreStream = new FileInputStream(sslKeystorePathFile)) { + keyStore.load(keyStoreStream, "keystore-password".toCharArray()); + } + TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + trustManagerFactory.init(keyStore); sslContext = SSLContext.getInstance("TLS"); - } catch (NoSuchAlgorithmException e) { + } catch (Exception e) { LOGGER.warn("Error with TLS algorithm " + e.getMessage()); } } diff --git a/cygnus-common/src/test/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImplTest.java b/cygnus-common/src/test/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImplTest.java index 435c9a761..eb057ea39 100644 --- a/cygnus-common/src/test/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImplTest.java +++ b/cygnus-common/src/test/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImplTest.java @@ -53,7 +53,7 @@ public MongoBackendImplTest() { public void testGetRange() { System.out.println(getTestTraceHead("[MongoBackendImpl.getRange]") + "-------- Given a resolution, its related range is correctly returned"); - MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, false, false); + MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, false, false, null); try { @@ -115,7 +115,7 @@ public void testGetRange() { public void testGetOrigin() { System.out.println(getTestTraceHead("[MongoBackendImpl.getOrigin]") + "-------- Given a calendar and a resolution, its related origin is correctly returned"); - MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, false, false); + MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, false, false, null); GregorianCalendar calendar = new GregorianCalendar(2017, 4, 5, 11, 46, 13); try { @@ -198,7 +198,7 @@ public void testGetOrigin() { public void testGetOffset() { System.out.println(getTestTraceHead("[MongoBackendImpl.getOffset]") + "-------- Given a calendar and a resolution, its related offset is correctly returned"); - MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, false, false); + MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, false, false, null); GregorianCalendar calendar = new GregorianCalendar(2017, 3, 5, 11, 46, 13); // month 3 is April try { @@ -266,7 +266,7 @@ public void testBuildQueryForInsertAggregated() { String entityType = "someType"; String attrName = "someName"; GregorianCalendar calendar = new GregorianCalendar(2017, 3, 5, 11, 46, 13); // month 3 is April - MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, DataModel.DMBYSERVICEPATH, false, false); + MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, DataModel.DMBYSERVICEPATH, false, false, null); String queryForInsertAggregated = "{\"_id\": {\"entityId\": \"someId\", \"entityType\": \"someType\", " + "\"attrName\": \"someName\", \"origin\": {\"$date\": 1491392760000}, " + "\"resolution\": \"second\", \"range\": \"minute\"}, \"points.offset\": 13}"; @@ -353,7 +353,7 @@ public void testBuildQueryForInsertAggregated() { throw e; } // try catch - backend = new MongoBackendImpl(null, null, null, null, null, DataModel.DMBYENTITY, false, false); + backend = new MongoBackendImpl(null, null, null, null, null, DataModel.DMBYENTITY, false, false, null); queryForInsertAggregated = "{\"_id\": {\"attrName\": \"someName\", " + "\"origin\": {\"$date\": 1491392760000}, \"resolution\": \"second\", " @@ -457,7 +457,7 @@ public void testBuildUpdateForUpdateNumerical() { double sum2 = 200; int numSamples = 2; GregorianCalendar calendar = new GregorianCalendar(2017, 3, 5, 11, 46, 13); // month 3 is April - MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, false, false); + MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, false, false, null); String updateForUpdate = "{\"$set\": {\"attrType\": \"someType\"}, " + "\"$inc\": {\"points.$.samples\": 2, \"points.$.sum\": 20.0, \"points.$.sum2\": 200.0}, " + "\"$min\": {\"points.$.min\": 0.0}, \"$max\": {\"points.$.max\": 10.0}}"; @@ -489,7 +489,7 @@ public void testBuildUpdateForUpdateString() { String value = "someString"; int count = 2; GregorianCalendar calendar = new GregorianCalendar(2017, 3, 5, 11, 46, 13); // month 3 is April - MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, false, false); + MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, false, false, null); String updateForUpdate = "{\"$set\": {\"attrType\": \"someType\"}, " + "\"$inc\": {\"points.13.samples\": 2, \"points.13.occur.someString\": 2}}"; diff --git a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSIMongoBaseSink.java b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSIMongoBaseSink.java index 0bf9045e3..f8f87da46 100644 --- a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSIMongoBaseSink.java +++ b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSIMongoBaseSink.java @@ -41,6 +41,7 @@ public abstract class NGSIMongoBaseSink extends NGSISink { protected String mongoReplicaSet; protected Boolean sslEnabled; protected Boolean sslInvalidHostNameAllowed; + protected String sslKeystorePathFile; protected String dbPrefix; protected String collectionPrefix; protected MongoBackendImpl backend; @@ -194,6 +195,9 @@ public void configure(Context context) { + sslInvalidHostNameAllowedStr + ") -- Must be 'true' or 'false'"); } // if else + sslKeystorePathFile = context.getString("mongo_ssl_keystone_path_file", ""); + LOGGER.debug("[" + this.getName() + "] Reading configuration (mongo_ssl_keystone_path_file=" + sslKeystorePathFile + ")"); + } // configure @Override @@ -201,7 +205,7 @@ public void start() { try { backend = new MongoBackendImpl(mongoHosts, mongoUsername, mongoPassword, mongoAuthSource, mongoReplicaSet, dataModel, - sslEnabled, sslInvalidHostNameAllowed); + sslEnabled, sslInvalidHostNameAllowed, sslKeystorePathFile); LOGGER.debug("[" + this.getName() + "] MongoDB persistence backend created"); } catch (Exception e) { LOGGER.error("Error while creating the MongoDB persistence backend. Details=" From e025dade7513d7b47ba53ec04ade78d74dfb0d60 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 19 Jun 2024 08:33:26 +0200 Subject: [PATCH 07/15] update doc --- doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md index 189576133..6e792ba18 100644 --- a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md +++ b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md @@ -322,6 +322,7 @@ When datamodel changes Cygnus tries to recreate index (delete current and create | db\_prefix | no | sth_ || | mongo\_ssl | no | false | Enable SSL in mongodb connection | | mongo\_ssl\_invalid\_host\_allowed | no | false | Allow invalid host name in mongo SSL connections | +| mongo\_ssl\_keystore\_path\_File | no | empty | Java SSL KeyStore path file (JKS file) | | collection\_prefix | no | sth_ | `system.` is not accepted. | | batch\_size | no | 1 | Number of events accumulated before persistence. | | batch\_timeout | no | 30 | Number of seconds the batch will be building before it is persisted as it is. | From 781f30212e7e00a4eb71c4cc85c3610e91fce6c3 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 19 Jun 2024 08:38:53 +0200 Subject: [PATCH 08/15] add check sslKeystore --- .../telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java index f846a05b9..e1adf1085 100644 --- a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java +++ b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java @@ -599,7 +599,7 @@ private MongoDatabase getDatabase(String dbName) { if (client == null) { SSLContext sslContext = null; - if (sslEnabled) { + if (sslEnabled && sslKeystorePathFile && !sslKeystorePathFile.isEmpty()) { try { // Init TrustManager to init SSL Context KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType()); From 6cfe346d7aa82414c18f476d2f0f4af563bed6a6 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 19 Jun 2024 09:08:50 +0200 Subject: [PATCH 09/15] fix check --- .../telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java index e1adf1085..c5eebc552 100644 --- a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java +++ b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java @@ -599,7 +599,7 @@ private MongoDatabase getDatabase(String dbName) { if (client == null) { SSLContext sslContext = null; - if (sslEnabled && sslKeystorePathFile && !sslKeystorePathFile.isEmpty()) { + if (sslEnabled && (sslKeystorePathFile != null) && !sslKeystorePathFile.isEmpty()) { try { // Init TrustManager to init SSL Context KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType()); From c592177750ef57a8843acafaa214b2dfd564cfd7 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 19 Jun 2024 09:15:08 +0200 Subject: [PATCH 10/15] update doc --- doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md index 6e792ba18..9746a791a 100644 --- a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md +++ b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md @@ -322,7 +322,7 @@ When datamodel changes Cygnus tries to recreate index (delete current and create | db\_prefix | no | sth_ || | mongo\_ssl | no | false | Enable SSL in mongodb connection | | mongo\_ssl\_invalid\_host\_allowed | no | false | Allow invalid host name in mongo SSL connections | -| mongo\_ssl\_keystore\_path\_File | no | empty | Java SSL KeyStore path file (JKS file) | +| mongo\_ssl\_keystore\_path\_File | no | empty | Java SSL KeyStore path file (JKS file). A JKS file could be create from a certificate file using keytool: ```keytool -importkeystore -srckeystore certificate.p12 -srcstoretype pkcs12 -destkeystore mongo_ssl_keystore.jks``` | | collection\_prefix | no | sth_ | `system.` is not accepted. | | batch\_size | no | 1 | Number of events accumulated before persistence. | | batch\_timeout | no | 30 | Number of seconds the batch will be building before it is persisted as it is. | From a318d312d1eaaa78b063cb66726d312b43b6e0c1 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 19 Jun 2024 09:27:47 +0200 Subject: [PATCH 11/15] update CNR --- CHANGES_NEXT_RELEASE | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index b6aedbd95..ec02c0f8c 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,2 +1,2 @@ -[cygnus-ngsi] [mongo-sink] Add mongo_ssl and mongo_ssl_invalid_host_allowed options for mongoDB connections -[cygnus-common] [mongo-backend] Use sslEnabled and sslInvalidHostNameAllowed options for mongoDB connections +[cygnus-ngsi] [mongo-sink] Add mongo_ssl, mongo_ssl_invalid_host_allowed, mongo_ssl_keystore_path_file options for mongoDB connections +[cygnus-common] [mongo-backend] Use sslEnabled, sslInvalidHostNameAllowed and sslKeystorePathFile options for mongoDB connections From bb3d96832d0d164b64427722797d7db7e5dbd26c Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 19 Jun 2024 09:51:20 +0200 Subject: [PATCH 12/15] init sslContext --- .../telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java index c5eebc552..6d5619225 100644 --- a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java +++ b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java @@ -609,6 +609,7 @@ private MongoDatabase getDatabase(String dbName) { TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); trustManagerFactory.init(keyStore); sslContext = SSLContext.getInstance("TLS"); + sslContext.init(null, trustManagerFactory.getTrustManagers(), new java.security.SecureRandom()); } catch (Exception e) { LOGGER.warn("Error with TLS algorithm " + e.getMessage()); } From e18f58dc219408602e8bcd09f18baa733dc4dacb Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 19 Jun 2024 09:56:13 +0200 Subject: [PATCH 13/15] Update error log about init SSL context --- .../telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java index 6d5619225..7825ab1cc 100644 --- a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java +++ b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java @@ -611,7 +611,7 @@ private MongoDatabase getDatabase(String dbName) { sslContext = SSLContext.getInstance("TLS"); sslContext.init(null, trustManagerFactory.getTrustManagers(), new java.security.SecureRandom()); } catch (Exception e) { - LOGGER.warn("Error with TLS algorithm " + e.getMessage()); + LOGGER.warn("Error when init SSL Context: " + e.getMessage()); } } if (mongoUsername.length() != 0) { From d0d92ceaa7202091f9b6f95d7b40296baf7b88dc Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 19 Jun 2024 11:08:56 +0200 Subject: [PATCH 14/15] add keystore password --- .../backends/mongo/MongoBackendImpl.java | 42 +++++++++---------- .../backends/mongo/MongoBackendImplTest.java | 14 +++---- .../iot/cygnus/sinks/NGSIMongoBaseSink.java | 11 +++-- .../ngsi_mongo_sink.md | 3 +- 4 files changed, 38 insertions(+), 32 deletions(-) diff --git a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java index 7825ab1cc..9d1b29b8e 100644 --- a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java +++ b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java @@ -70,6 +70,7 @@ public enum Resolution { SECOND, MINUTE, HOUR, DAY, MONTH } private final Boolean sslEnabled; private final Boolean sslInvalidHostNameAllowed; private final String sslKeystorePathFile; + private final String sslKeystorePassword; private final DataModel dataModel; private static final CygnusLogger LOGGER = new CygnusLogger(MongoBackendImpl.class); @@ -84,7 +85,8 @@ public enum Resolution { SECOND, MINUTE, HOUR, DAY, MONTH } */ public MongoBackendImpl(String mongoHosts, String mongoUsername, String mongoPassword, String mongoAuthSource, String mongoReplicaSet, DataModel dataModel, - Boolean sslEnabled, Boolean sslInvalidHostNameAllowed, String sslKeystorePathFile) { + Boolean sslEnabled, Boolean sslInvalidHostNameAllowed, + String sslKeystorePathFile, String sslKeystorePassword) { client = null; this.mongoHosts = mongoHosts; this.mongoUsername = mongoUsername; @@ -94,6 +96,7 @@ public MongoBackendImpl(String mongoHosts, String mongoUsername, String mongoPas this.sslEnabled = sslEnabled; this.sslInvalidHostNameAllowed = sslInvalidHostNameAllowed; this.sslKeystorePathFile = sslKeystorePathFile; + this.sslKeystorePassword = sslKeystorePassword; this.dataModel = dataModel; } // MongoBackendImpl @@ -604,7 +607,7 @@ private MongoDatabase getDatabase(String dbName) { // Init TrustManager to init SSL Context KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType()); try (InputStream keyStoreStream = new FileInputStream(sslKeystorePathFile)) { - keyStore.load(keyStoreStream, "keystore-password".toCharArray()); + keyStore.load(keyStoreStream, sslKeystorePassword.toCharArray()); } TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); trustManagerFactory.init(keyStore); @@ -614,6 +617,13 @@ private MongoDatabase getDatabase(String dbName) { LOGGER.warn("Error when init SSL Context: " + e.getMessage()); } } + + MongoClientOptions options = MongoClientOptions.builder() + .sslEnabled(sslEnabled) + .sslInvalidHostNameAllowed(sslInvalidHostNameAllowed) + .sslContext(sslContext) + .build(); + if (mongoUsername.length() != 0) { String authSource; if ((mongoAuthSource != null) && !mongoAuthSource.isEmpty()) { @@ -622,7 +632,7 @@ private MongoDatabase getDatabase(String dbName) { authSource = dbName; } MongoCredential credential = MongoCredential.createCredential(mongoUsername, authSource, - mongoPassword.toCharArray()); + mongoPassword.toCharArray()); /**** // This constructor is deprecated see Mongo Client API documentation @@ -630,26 +640,16 @@ private MongoDatabase getDatabase(String dbName) { client = new MongoClient(servers, Arrays.asList(credential)); ****/ if ((mongoReplicaSet!= null) && !mongoReplicaSet.isEmpty()) { - client = new MongoClient(servers, credential, new MongoClientOptions.Builder(). - requiredReplicaSetName(mongoReplicaSet). - sslEnabled(sslEnabled). - sslInvalidHostNameAllowed(sslInvalidHostNameAllowed). - sslContext(sslContext). - build()); - } else { - client = new MongoClient(servers, credential, new MongoClientOptions.Builder(). - sslEnabled(sslEnabled). - sslInvalidHostNameAllowed(sslInvalidHostNameAllowed). - sslContext(sslContext). - build()); + options = MongoClientOptions.builder() + .requiredReplicaSetName(mongoReplicaSet) + .sslEnabled(sslEnabled) + .sslInvalidHostNameAllowed(sslInvalidHostNameAllowed) + .sslContext(sslContext) + .build(); } + client = new MongoClient(servers, credential, options); } else { - MongoClientOptions options = MongoClientOptions.builder() - .sslEnabled(sslEnabled) - .sslInvalidHostNameAllowed(sslInvalidHostNameAllowed) - .sslContext(sslContext) - .build(); - client = new MongoClient(servers, options); + client = new MongoClient(servers, options); } // if else } // if diff --git a/cygnus-common/src/test/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImplTest.java b/cygnus-common/src/test/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImplTest.java index eb057ea39..0625507b9 100644 --- a/cygnus-common/src/test/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImplTest.java +++ b/cygnus-common/src/test/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImplTest.java @@ -53,7 +53,7 @@ public MongoBackendImplTest() { public void testGetRange() { System.out.println(getTestTraceHead("[MongoBackendImpl.getRange]") + "-------- Given a resolution, its related range is correctly returned"); - MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, false, false, null); + MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, false, false, null, null); try { @@ -115,7 +115,7 @@ public void testGetRange() { public void testGetOrigin() { System.out.println(getTestTraceHead("[MongoBackendImpl.getOrigin]") + "-------- Given a calendar and a resolution, its related origin is correctly returned"); - MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, false, false, null); + MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, false, false, null, null); GregorianCalendar calendar = new GregorianCalendar(2017, 4, 5, 11, 46, 13); try { @@ -198,7 +198,7 @@ public void testGetOrigin() { public void testGetOffset() { System.out.println(getTestTraceHead("[MongoBackendImpl.getOffset]") + "-------- Given a calendar and a resolution, its related offset is correctly returned"); - MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, false, false, null); + MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, false, false, null, null); GregorianCalendar calendar = new GregorianCalendar(2017, 3, 5, 11, 46, 13); // month 3 is April try { @@ -266,7 +266,7 @@ public void testBuildQueryForInsertAggregated() { String entityType = "someType"; String attrName = "someName"; GregorianCalendar calendar = new GregorianCalendar(2017, 3, 5, 11, 46, 13); // month 3 is April - MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, DataModel.DMBYSERVICEPATH, false, false, null); + MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, DataModel.DMBYSERVICEPATH, false, false, null, null); String queryForInsertAggregated = "{\"_id\": {\"entityId\": \"someId\", \"entityType\": \"someType\", " + "\"attrName\": \"someName\", \"origin\": {\"$date\": 1491392760000}, " + "\"resolution\": \"second\", \"range\": \"minute\"}, \"points.offset\": 13}"; @@ -353,7 +353,7 @@ public void testBuildQueryForInsertAggregated() { throw e; } // try catch - backend = new MongoBackendImpl(null, null, null, null, null, DataModel.DMBYENTITY, false, false, null); + backend = new MongoBackendImpl(null, null, null, null, null, DataModel.DMBYENTITY, false, false, null, null); queryForInsertAggregated = "{\"_id\": {\"attrName\": \"someName\", " + "\"origin\": {\"$date\": 1491392760000}, \"resolution\": \"second\", " @@ -457,7 +457,7 @@ public void testBuildUpdateForUpdateNumerical() { double sum2 = 200; int numSamples = 2; GregorianCalendar calendar = new GregorianCalendar(2017, 3, 5, 11, 46, 13); // month 3 is April - MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, false, false, null); + MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, false, false, null, null); String updateForUpdate = "{\"$set\": {\"attrType\": \"someType\"}, " + "\"$inc\": {\"points.$.samples\": 2, \"points.$.sum\": 20.0, \"points.$.sum2\": 200.0}, " + "\"$min\": {\"points.$.min\": 0.0}, \"$max\": {\"points.$.max\": 10.0}}"; @@ -489,7 +489,7 @@ public void testBuildUpdateForUpdateString() { String value = "someString"; int count = 2; GregorianCalendar calendar = new GregorianCalendar(2017, 3, 5, 11, 46, 13); // month 3 is April - MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, false, false, null); + MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, false, false, null, null); String updateForUpdate = "{\"$set\": {\"attrType\": \"someType\"}, " + "\"$inc\": {\"points.13.samples\": 2, \"points.13.occur.someString\": 2}}"; diff --git a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSIMongoBaseSink.java b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSIMongoBaseSink.java index f8f87da46..aef37af7d 100644 --- a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSIMongoBaseSink.java +++ b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSIMongoBaseSink.java @@ -42,6 +42,7 @@ public abstract class NGSIMongoBaseSink extends NGSISink { protected Boolean sslEnabled; protected Boolean sslInvalidHostNameAllowed; protected String sslKeystorePathFile; + protected String sslKeystorePassword; protected String dbPrefix; protected String collectionPrefix; protected MongoBackendImpl backend; @@ -195,8 +196,11 @@ public void configure(Context context) { + sslInvalidHostNameAllowedStr + ") -- Must be 'true' or 'false'"); } // if else - sslKeystorePathFile = context.getString("mongo_ssl_keystone_path_file", ""); - LOGGER.debug("[" + this.getName() + "] Reading configuration (mongo_ssl_keystone_path_file=" + sslKeystorePathFile + ")"); + sslKeystorePathFile = context.getString("mongo_ssl_keystore_path_file", ""); + LOGGER.debug("[" + this.getName() + "] Reading configuration (mongo_ssl_keystore_path_file=" + sslKeystorePathFile + ")"); + + sslKeystorePassword = context.getString("mongo_ssl_keystore_password", ""); + LOGGER.debug("[" + this.getName() + "] Reading configuration (mongo_ssl_keystore_password=" + sslKeystorePassword + ")"); } // configure @@ -205,7 +209,8 @@ public void start() { try { backend = new MongoBackendImpl(mongoHosts, mongoUsername, mongoPassword, mongoAuthSource, mongoReplicaSet, dataModel, - sslEnabled, sslInvalidHostNameAllowed, sslKeystorePathFile); + sslEnabled, sslInvalidHostNameAllowed, + sslKeystorePathFile, sslKeystorePassword); LOGGER.debug("[" + this.getName() + "] MongoDB persistence backend created"); } catch (Exception e) { LOGGER.error("Error while creating the MongoDB persistence backend. Details=" diff --git a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md index 9746a791a..d1e7fda1f 100644 --- a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md +++ b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md @@ -322,7 +322,8 @@ When datamodel changes Cygnus tries to recreate index (delete current and create | db\_prefix | no | sth_ || | mongo\_ssl | no | false | Enable SSL in mongodb connection | | mongo\_ssl\_invalid\_host\_allowed | no | false | Allow invalid host name in mongo SSL connections | -| mongo\_ssl\_keystore\_path\_File | no | empty | Java SSL KeyStore path file (JKS file). A JKS file could be create from a certificate file using keytool: ```keytool -importkeystore -srckeystore certificate.p12 -srcstoretype pkcs12 -destkeystore mongo_ssl_keystore.jks``` | +| mongo\_ssl\_keystore\_path\_file | no | empty | Java SSL KeyStore path file (JKS file). A JKS file could be create from a certificate file using keytool: ```keytool -importkeystore -srckeystore certificate.p12 -srcstoretype pkcs12 -destkeystore mongo_ssl_keystore.jks``` | +| mongo\_ssl\_keystore\_password | no | empty | Java SSL KeyStore password for keystore file (JKS file). | | collection\_prefix | no | sth_ | `system.` is not accepted. | | batch\_size | no | 1 | Number of events accumulated before persistence. | | batch\_timeout | no | 30 | Number of seconds the batch will be building before it is persisted as it is. | From ccef1ff8bcae2021c8a450e51b98399683c4d3fc Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 19 Jun 2024 11:10:05 +0200 Subject: [PATCH 15/15] Update CNR --- CHANGES_NEXT_RELEASE | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index ec02c0f8c..78dc1831a 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,2 +1,2 @@ -[cygnus-ngsi] [mongo-sink] Add mongo_ssl, mongo_ssl_invalid_host_allowed, mongo_ssl_keystore_path_file options for mongoDB connections -[cygnus-common] [mongo-backend] Use sslEnabled, sslInvalidHostNameAllowed and sslKeystorePathFile options for mongoDB connections +[cygnus-ngsi] [mongo-sink] Add mongo_ssl, mongo_ssl_invalid_host_allowed, mongo_ssl_keystore_path_file and mongo_ssl_keystore_password options for mongoDB connections +[cygnus-common] [mongo-backend] Use sslEnabled, sslInvalidHostNameAllowed, sslKeystorePathFile and sslKeystorePassword options for mongoDB connections