From 6e5a5b7b78047af724ef5e7c307a49787950e9e8 Mon Sep 17 00:00:00 2001 From: kumardeepak5 Date: Wed, 21 Aug 2024 13:44:11 +0000 Subject: [PATCH 1/6] changes regarding issue 2387 --- .../backends/mongo/MongoBackendImpl.java | 139 ++++++------------ .../backends/mongo/MongoBackendImplTest.java | 14 +- .../iot/cygnus/sinks/NGSIMongoBaseSink.java | 62 +------- .../ngsi_mongo_sink.md | 6 +- 4 files changed, 59 insertions(+), 162 deletions(-) diff --git a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java index d4021859d..d04c6648e 100644 --- a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java +++ b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java @@ -17,7 +17,14 @@ */ package com.telefonica.iot.cygnus.backends.mongo; -import com.mongodb.*; +import com.mongodb.BasicDBList; +import com.mongodb.BasicDBObject; +import com.mongodb.ConnectionString; +import com.mongodb.ErrorCategory; +import com.mongodb.MongoClientSettings; +import com.mongodb.MongoException; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; import com.mongodb.client.model.CreateCollectionOptions; @@ -26,6 +33,7 @@ import com.mongodb.client.model.WriteModel; import com.mongodb.client.model.UpdateOneModel; import com.mongodb.client.result.UpdateResult; +import com.mongodb.connection.SslSettings; import com.telefonica.iot.cygnus.log.CygnusLogger; import com.telefonica.iot.cygnus.sinks.Enums.DataModel; import static com.telefonica.iot.cygnus.sinks.Enums.DataModel.DMBYATTRIBUTE; @@ -64,11 +72,7 @@ public class MongoBackendImpl implements MongoBackend { public enum Resolution { SECOND, MINUTE, HOUR, DAY, MONTH } private MongoClient client; - private final String mongoHosts; - private final String mongoUsername; - private final String mongoPassword; - private final String mongoAuthSource; - private final String mongoReplicaSet; + private final String mongoURI; private final Boolean sslEnabled; private final Boolean sslInvalidHostNameAllowed; private final String sslKeystorePathFile; @@ -87,17 +91,12 @@ public enum Resolution { SECOND, MINUTE, HOUR, DAY, MONTH } * @param mongoReplicaSet * @param dataModel */ - public MongoBackendImpl(String mongoHosts, String mongoUsername, String mongoPassword, - String mongoAuthSource, String mongoReplicaSet, DataModel dataModel, + public MongoBackendImpl(String mongoURI, DataModel dataModel, Boolean sslEnabled, Boolean sslInvalidHostNameAllowed, String sslKeystorePathFile, String sslKeystorePassword, String sslTruststorePathFile, String sslTruststorePassword) { client = null; - this.mongoHosts = mongoHosts; - this.mongoUsername = mongoUsername; - this.mongoPassword = mongoPassword; - this.mongoAuthSource = mongoAuthSource; - this.mongoReplicaSet = mongoReplicaSet; + this.mongoURI = mongoURI; this.sslEnabled = sslEnabled; this.sslInvalidHostNameAllowed = sslInvalidHostNameAllowed; this.sslKeystorePathFile = sslKeystorePathFile; @@ -589,94 +588,42 @@ protected ArrayList buildUpdateForUpdate(String attrType, boolean * @return */ private MongoDatabase getDatabase(String dbName) { - // create a ServerAddress object for each configured URI - List servers = new ArrayList<>(); - String[] uris = mongoHosts.split(","); - - for (String uri: uris) { - String[] uriParts = uri.split(":"); - if (uriParts.length == 2) { - LOGGER.debug("Adding 2-part Mongo ServerAddress: Host=" + uriParts[0] + " Port=" + uriParts[1]); - servers.add(new ServerAddress(uriParts[0], new Integer(uriParts[1]))); - } else { - LOGGER.debug("Adding 1-part Mongo ServerAddress: Host=" + uri); - servers.add(new ServerAddress(uri)); - } // if else - } // for - - // create a Mongo client - if (client == null) { - - SSLContext sslContext = null; - - if (sslEnabled) { - try { - KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType()); - if ((sslKeystorePathFile != null) && !sslKeystorePathFile.isEmpty()) { - InputStream keyStoreStream = new FileInputStream(sslKeystorePathFile); - keyStore.load(keyStoreStream, sslKeystorePassword.toCharArray()); - } else { - keyStore.load(null); - } - if ((sslTruststorePathFile != null) && !sslTruststorePathFile.isEmpty()) { - InputStream trustStoreStream = new FileInputStream(sslTruststorePathFile); - keyStore.load(trustStoreStream, sslTruststorePassword.toCharArray()); - } - TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); - trustManagerFactory.init(keyStore); - sslContext = SSLContext.getInstance("TLS"); - sslContext.init(null, trustManagerFactory.getTrustManagers(), new java.security.SecureRandom()); - } catch (Exception e) { - LOGGER.warn("Error when init SSL Context: " + e.getMessage()); + SSLContext sslContext = null; + + if (sslEnabled) { + try { + KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType()); + if ((sslKeystorePathFile != null) && !sslKeystorePathFile.isEmpty()) { + InputStream keyStoreStream = new FileInputStream(sslKeystorePathFile); + keyStore.load(keyStoreStream, sslKeystorePassword.toCharArray()); + } else { + keyStore.load(null); + } + if ((sslTruststorePathFile != null) && !sslTruststorePathFile.isEmpty()) { + InputStream trustStoreStream = new FileInputStream(sslTruststorePathFile); + keyStore.load(trustStoreStream, sslTruststorePassword.toCharArray()); + } + TrustManagerFactory trustManagerFactory = TrustManagerFactory + .getInstance(TrustManagerFactory.getDefaultAlgorithm()); + trustManagerFactory.init(keyStore); + sslContext = SSLContext.getInstance("TLS"); + sslContext.init(null, trustManagerFactory.getTrustManagers(), new java.security.SecureRandom()); + } catch (Exception e) { + LOGGER.warn("Error when init SSL Context: " + e.getMessage()); + } } - } - MongoClientOptions options = MongoClientOptions.builder() - .sslEnabled(sslEnabled) - .sslInvalidHostNameAllowed(sslInvalidHostNameAllowed) - .sslContext(sslContext) - .build(); - - if (mongoUsername.length() != 0) { - String authSource; - if ((mongoAuthSource != null) && !mongoAuthSource.isEmpty()) { - authSource = mongoAuthSource; - } else { - authSource = dbName; - } - MongoCredential credential = MongoCredential.createCredential(mongoUsername, authSource, - mongoPassword.toCharArray()); - - /**** - // This constructor is deprecated see Mongo Client API documentation - // @deprecated Prefer {@link #MongoClient(List, MongoCredential, MongoClientOptions)} - client = new MongoClient(servers, Arrays.asList(credential)); - ****/ - if ((mongoReplicaSet!= null) && !mongoReplicaSet.isEmpty()) { - options = MongoClientOptions.builder() - .requiredReplicaSetName(mongoReplicaSet) - .sslEnabled(sslEnabled) - .sslInvalidHostNameAllowed(sslInvalidHostNameAllowed) - .sslContext(sslContext) - .build(); - } - if (servers.size() == 1) { // allow auto-discover when just one endpoint is provided - client = new MongoClient(servers.get(0), credential, options); - } else { - client = new MongoClient(servers, credential, options); - } - } else { - if (servers.size() == 1) { // allow auto-discover when just one endpoint is provided - client = new MongoClient(servers.get(0), options); - } else { - client = new MongoClient(servers, options); - } - } // if else - } // if + SslSettings sslSetting = SslSettings.builder().enabled(sslEnabled) + .invalidHostNameAllowed(sslInvalidHostNameAllowed).context(sslContext).build(); + MongoClientSettings settings = MongoClientSettings.builder() + .applyConnectionString(new ConnectionString(mongoURI)) + .applyToSslSettings(builder -> builder.applySettings(sslSetting).build()).build(); - // get the database + client = MongoClients.create(settings); + } return client.getDatabase(dbName); + } // getDatabase /** diff --git a/cygnus-common/src/test/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImplTest.java b/cygnus-common/src/test/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImplTest.java index f4bb076f9..8e039949e 100644 --- a/cygnus-common/src/test/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImplTest.java +++ b/cygnus-common/src/test/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImplTest.java @@ -53,7 +53,7 @@ public MongoBackendImplTest() { public void testGetRange() { System.out.println(getTestTraceHead("[MongoBackendImpl.getRange]") + "-------- Given a resolution, its related range is correctly returned"); - MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, false, false, null, null, null, null); + MongoBackendImpl backend = new MongoBackendImpl(null, null, false, false, null, null, null, null); try { @@ -115,7 +115,7 @@ public void testGetRange() { public void testGetOrigin() { System.out.println(getTestTraceHead("[MongoBackendImpl.getOrigin]") + "-------- Given a calendar and a resolution, its related origin is correctly returned"); - MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, false, false, null, null, null, null); + MongoBackendImpl backend = new MongoBackendImpl(null, null, false, false, null, null, null, null); GregorianCalendar calendar = new GregorianCalendar(2017, 4, 5, 11, 46, 13); try { @@ -198,7 +198,7 @@ public void testGetOrigin() { public void testGetOffset() { System.out.println(getTestTraceHead("[MongoBackendImpl.getOffset]") + "-------- Given a calendar and a resolution, its related offset is correctly returned"); - MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, false, false, null, null, null, null); + MongoBackendImpl backend = new MongoBackendImpl(null, null, false, false, null, null, null, null); GregorianCalendar calendar = new GregorianCalendar(2017, 3, 5, 11, 46, 13); // month 3 is April try { @@ -266,7 +266,7 @@ public void testBuildQueryForInsertAggregated() { String entityType = "someType"; String attrName = "someName"; GregorianCalendar calendar = new GregorianCalendar(2017, 3, 5, 11, 46, 13); // month 3 is April - MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, DataModel.DMBYSERVICEPATH, false, false, null, null, null, null); + MongoBackendImpl backend = new MongoBackendImpl(null, DataModel.DMBYSERVICEPATH, false, false, null, null, null, null); String queryForInsertAggregated = "{\"_id\": {\"entityId\": \"someId\", \"entityType\": \"someType\", " + "\"attrName\": \"someName\", \"origin\": {\"$date\": 1491392760000}, " + "\"resolution\": \"second\", \"range\": \"minute\"}, \"points.offset\": 13}"; @@ -353,7 +353,7 @@ public void testBuildQueryForInsertAggregated() { throw e; } // try catch - backend = new MongoBackendImpl(null, null, null, null, null, DataModel.DMBYENTITY, false, false, null, null, null, null); + backend = new MongoBackendImpl(null, DataModel.DMBYENTITY, false, false, null, null, null, null); queryForInsertAggregated = "{\"_id\": {\"attrName\": \"someName\", " + "\"origin\": {\"$date\": 1491392760000}, \"resolution\": \"second\", " @@ -457,7 +457,7 @@ public void testBuildUpdateForUpdateNumerical() { double sum2 = 200; int numSamples = 2; GregorianCalendar calendar = new GregorianCalendar(2017, 3, 5, 11, 46, 13); // month 3 is April - MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, false, false, null, null, null, null); + MongoBackendImpl backend = new MongoBackendImpl(null, null, false, false, null, null, null, null); String updateForUpdate = "{\"$set\": {\"attrType\": \"someType\"}, " + "\"$inc\": {\"points.$.samples\": 2, \"points.$.sum\": 20.0, \"points.$.sum2\": 200.0}, " + "\"$min\": {\"points.$.min\": 0.0}, \"$max\": {\"points.$.max\": 10.0}}"; @@ -489,7 +489,7 @@ public void testBuildUpdateForUpdateString() { String value = "someString"; int count = 2; GregorianCalendar calendar = new GregorianCalendar(2017, 3, 5, 11, 46, 13); // month 3 is April - MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, false, false, null, null, null, null); + MongoBackendImpl backend = new MongoBackendImpl(null, null, false, false, null, null, null, null); String updateForUpdate = "{\"$set\": {\"attrType\": \"someType\"}, " + "\"$inc\": {\"points.13.samples\": 2, \"points.13.occur.someString\": 2}}"; diff --git a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSIMongoBaseSink.java b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSIMongoBaseSink.java index bf355d64f..6038053bd 100644 --- a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSIMongoBaseSink.java +++ b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSIMongoBaseSink.java @@ -34,11 +34,7 @@ public abstract class NGSIMongoBaseSink extends NGSISink { protected static final CygnusLogger LOGGER = new CygnusLogger(NGSIMongoBaseSink.class); - protected String mongoHosts; - protected String mongoUsername; - protected String mongoPassword; - protected String mongoAuthSource; - protected String mongoReplicaSet; + protected String mongoURI; protected Boolean sslEnabled; protected Boolean sslInvalidHostNameAllowed; protected String sslKeystorePathFile; @@ -52,44 +48,12 @@ public abstract class NGSIMongoBaseSink extends NGSISink { protected boolean ignoreWhiteSpaces; /** - * Gets the mongo hosts. It is protected since it is used by the tests. + * Gets the mongo uri. It is protected since it is used by the tests. * @return */ - protected String getMongoHosts() { - return mongoHosts; - } // getMongoHosts - - /** - * Gets the mongo username. It is protected since it is used by the tests. - * @return - */ - protected String getUsername() { - return mongoUsername; - } // getUsername - - /** - * Gets the mongo password. It is protected since it is used by the tests. - * @return - */ - protected String getPassword() { - return mongoPassword; - } // getPassword - - /** - * Gets the mongo auth_source. It is protected since it is used by the tests. - * @return - */ - protected String getAuthSource() { - return mongoAuthSource; - } // getAuthSource - - /** - * Gets the mongo replica_set. It is protected since it is used by the tests. - * @return - */ - protected String getReplicaSet() { - return mongoReplicaSet; - } // getReplicaSet + protected String getMongoURI() { + return mongoURI; + } // getMongoURI /** * Gets the database prefix. It is protected since it is used by the tests. @@ -127,17 +91,8 @@ protected MongoBackendImpl getBackend() { public void configure(Context context) { super.configure(context); - mongoHosts = context.getString("mongo_hosts", "localhost:27017"); - LOGGER.debug("[" + this.getName() + "] Reading configuration (mongo_hosts=" + mongoHosts + ")"); - mongoUsername = context.getString("mongo_username", ""); - LOGGER.debug("[" + this.getName() + "] Reading configuration (mongo_username=" + mongoUsername + ")"); - // FIXME: mongoPassword should be read as a SHA1 and decoded here - mongoPassword = context.getString("mongo_password", ""); - LOGGER.debug("[" + this.getName() + "] Reading configuration (mongo_password=" + mongoPassword + ")"); - mongoAuthSource = context.getString("mongo_auth_source", ""); - LOGGER.debug("[" + this.getName() + "] Reading configuration (mongo_auth_source=" + mongoAuthSource + ")"); - mongoReplicaSet = context.getString("mongo_replica_set", ""); - LOGGER.debug("[" + this.getName() + "] Reading configuration (mongo_replica_set=" + mongoReplicaSet + ")"); + mongoURI = context.getString("mongo_uri", "mongodb://localhost:27017"); + LOGGER.info("[" + this.getName() + "] Reading configuration (mongo_uri=" + mongoURI + ")"); if (enableEncoding) { dbPrefix = NGSICharsets.encodeMongoDBDatabase(context.getString("db_prefix", "sth_")); @@ -215,8 +170,7 @@ public void configure(Context context) { @Override public void start() { try { - backend = new MongoBackendImpl(mongoHosts, mongoUsername, mongoPassword, - mongoAuthSource, mongoReplicaSet, dataModel, + backend = new MongoBackendImpl(mongoURI, dataModel, sslEnabled, sslInvalidHostNameAllowed, sslKeystorePathFile, sslKeystorePassword, sslTruststorePathFile, sslTruststorePassword); diff --git a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md index 575ceb4d3..ce912888d 100644 --- a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md +++ b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md @@ -314,11 +314,7 @@ When datamodel changes Cygnus tries to recreate index (delete current and create | data\_model | no | dm-by-entity | dm-by-service-path, dm-by-entity or . dm-by-service is not currently supported. | | attr\_persistence | no | row | row or column. | | attr\_metadata\_store | no | false | true or false. | -| mongo\_hosts | no | localhost:27017 | FQDN/IP:port where the MongoDB server runs (standalone case) or comma-separated list of FQDN/IP:port pairs where the MongoDB replica set members run. | -| mongo\_username | no | empty | If empty, no authentication is done. | -| mongo\_password | no | empty | If empty, no authentication is done. | -| mongo\_auth_source | no | empty | Auth source database use to authenticate the user. Usually could be `admin`. | -| mongo\_replica_set | no | empty | Replica Set name. Note that this parameter is optional because Cygnus is able to connect to a MongoDB replica set without needing to specify its name.| +| mongo\_uri | no | mongodb://localhost:27017 | Mongo DB Connection String. | | db\_prefix | no | sth_ || | mongo\_ssl | no | false | Enable SSL in mongodb connection | | mongo\_ssl\_invalid\_host\_allowed | no | false | Allow invalid host name in mongo SSL connections | From 77ef501af972a37a19b3c63b3066d3d8f0aa0fc5 Mon Sep 17 00:00:00 2001 From: kumardeepak5 Date: Wed, 21 Aug 2024 14:03:41 +0000 Subject: [PATCH 2/6] removed unsued imports --- .../iot/cygnus/backends/mongo/MongoBackendImpl.java | 9 --------- 1 file changed, 9 deletions(-) diff --git a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java index d04c6648e..8713d0fab 100644 --- a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java +++ b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java @@ -32,15 +32,10 @@ import com.mongodb.client.model.UpdateOptions; import com.mongodb.client.model.WriteModel; import com.mongodb.client.model.UpdateOneModel; -import com.mongodb.client.result.UpdateResult; import com.mongodb.connection.SslSettings; import com.telefonica.iot.cygnus.log.CygnusLogger; import com.telefonica.iot.cygnus.sinks.Enums.DataModel; -import static com.telefonica.iot.cygnus.sinks.Enums.DataModel.DMBYATTRIBUTE; -import static com.telefonica.iot.cygnus.sinks.Enums.DataModel.DMBYENTITY; -import static com.telefonica.iot.cygnus.sinks.Enums.DataModel.DMBYSERVICEPATH; import java.util.ArrayList; -import java.util.Arrays; import java.util.Calendar; import java.util.Date; import java.util.GregorianCalendar; @@ -49,12 +44,8 @@ import java.util.TimeZone; import java.util.concurrent.TimeUnit; import javax.net.ssl.SSLContext; -import java.security.NoSuchAlgorithmException; -import javax.net.ssl.SSLContext; import javax.net.ssl.TrustManagerFactory; import java.security.KeyStore; -import java.security.cert.CertificateFactory; -import java.security.cert.X509Certificate; import java.io.FileInputStream; import java.io.InputStream; import org.bson.Document; From 3f1a9d82aff2f811e3a8f0fa5c541d90afc60193 Mon Sep 17 00:00:00 2001 From: kumardeepak5 Date: Thu, 22 Aug 2024 10:58:28 +0000 Subject: [PATCH 3/6] reverting commit -6e5a5b7b changes regarding issue 2387 --- .../backends/mongo/MongoBackendImpl.java | 139 ++++++++++++------ .../backends/mongo/MongoBackendImplTest.java | 14 +- .../iot/cygnus/sinks/NGSIMongoBaseSink.java | 62 +++++++- .../ngsi_mongo_sink.md | 6 +- 4 files changed, 163 insertions(+), 58 deletions(-) diff --git a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java index 8713d0fab..c857b67a6 100644 --- a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java +++ b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java @@ -17,14 +17,7 @@ */ package com.telefonica.iot.cygnus.backends.mongo; -import com.mongodb.BasicDBList; -import com.mongodb.BasicDBObject; -import com.mongodb.ConnectionString; -import com.mongodb.ErrorCategory; -import com.mongodb.MongoClientSettings; -import com.mongodb.MongoException; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; +import com.mongodb.*; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; import com.mongodb.client.model.CreateCollectionOptions; @@ -33,6 +26,7 @@ import com.mongodb.client.model.WriteModel; import com.mongodb.client.model.UpdateOneModel; import com.mongodb.connection.SslSettings; +import com.mongodb.client.result.UpdateResult; import com.telefonica.iot.cygnus.log.CygnusLogger; import com.telefonica.iot.cygnus.sinks.Enums.DataModel; import java.util.ArrayList; @@ -63,7 +57,11 @@ public class MongoBackendImpl implements MongoBackend { public enum Resolution { SECOND, MINUTE, HOUR, DAY, MONTH } private MongoClient client; - private final String mongoURI; + private final String mongoHosts; + private final String mongoUsername; + private final String mongoPassword; + private final String mongoAuthSource; + private final String mongoReplicaSet; private final Boolean sslEnabled; private final Boolean sslInvalidHostNameAllowed; private final String sslKeystorePathFile; @@ -82,12 +80,17 @@ public enum Resolution { SECOND, MINUTE, HOUR, DAY, MONTH } * @param mongoReplicaSet * @param dataModel */ - public MongoBackendImpl(String mongoURI, DataModel dataModel, + public MongoBackendImpl(String mongoHosts, String mongoUsername, String mongoPassword, + String mongoAuthSource, String mongoReplicaSet, DataModel dataModel, Boolean sslEnabled, Boolean sslInvalidHostNameAllowed, String sslKeystorePathFile, String sslKeystorePassword, String sslTruststorePathFile, String sslTruststorePassword) { client = null; - this.mongoURI = mongoURI; + this.mongoHosts = mongoHosts; + this.mongoUsername = mongoUsername; + this.mongoPassword = mongoPassword; + this.mongoAuthSource = mongoAuthSource; + this.mongoReplicaSet = mongoReplicaSet; this.sslEnabled = sslEnabled; this.sslInvalidHostNameAllowed = sslInvalidHostNameAllowed; this.sslKeystorePathFile = sslKeystorePathFile; @@ -579,42 +582,94 @@ protected ArrayList buildUpdateForUpdate(String attrType, boolean * @return */ private MongoDatabase getDatabase(String dbName) { + // create a ServerAddress object for each configured URI + List servers = new ArrayList<>(); + String[] uris = mongoHosts.split(","); + + for (String uri: uris) { + String[] uriParts = uri.split(":"); + if (uriParts.length == 2) { + LOGGER.debug("Adding 2-part Mongo ServerAddress: Host=" + uriParts[0] + " Port=" + uriParts[1]); + servers.add(new ServerAddress(uriParts[0], new Integer(uriParts[1]))); + } else { + LOGGER.debug("Adding 1-part Mongo ServerAddress: Host=" + uri); + servers.add(new ServerAddress(uri)); + } // if else + } // for + + // create a Mongo client + if (client == null) { - SSLContext sslContext = null; - - if (sslEnabled) { - try { - KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType()); - if ((sslKeystorePathFile != null) && !sslKeystorePathFile.isEmpty()) { - InputStream keyStoreStream = new FileInputStream(sslKeystorePathFile); - keyStore.load(keyStoreStream, sslKeystorePassword.toCharArray()); - } else { - keyStore.load(null); - } - if ((sslTruststorePathFile != null) && !sslTruststorePathFile.isEmpty()) { - InputStream trustStoreStream = new FileInputStream(sslTruststorePathFile); - keyStore.load(trustStoreStream, sslTruststorePassword.toCharArray()); - } - TrustManagerFactory trustManagerFactory = TrustManagerFactory - .getInstance(TrustManagerFactory.getDefaultAlgorithm()); - trustManagerFactory.init(keyStore); - sslContext = SSLContext.getInstance("TLS"); - sslContext.init(null, trustManagerFactory.getTrustManagers(), new java.security.SecureRandom()); - } catch (Exception e) { - LOGGER.warn("Error when init SSL Context: " + e.getMessage()); - } + + SSLContext sslContext = null; + + if (sslEnabled) { + try { + KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType()); + if ((sslKeystorePathFile != null) && !sslKeystorePathFile.isEmpty()) { + InputStream keyStoreStream = new FileInputStream(sslKeystorePathFile); + keyStore.load(keyStoreStream, sslKeystorePassword.toCharArray()); + } else { + keyStore.load(null); + } + if ((sslTruststorePathFile != null) && !sslTruststorePathFile.isEmpty()) { + InputStream trustStoreStream = new FileInputStream(sslTruststorePathFile); + keyStore.load(trustStoreStream, sslTruststorePassword.toCharArray()); + } + TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + trustManagerFactory.init(keyStore); + sslContext = SSLContext.getInstance("TLS"); + sslContext.init(null, trustManagerFactory.getTrustManagers(), new java.security.SecureRandom()); + } catch (Exception e) { + LOGGER.warn("Error when init SSL Context: " + e.getMessage()); } + } - SslSettings sslSetting = SslSettings.builder().enabled(sslEnabled) - .invalidHostNameAllowed(sslInvalidHostNameAllowed).context(sslContext).build(); - MongoClientSettings settings = MongoClientSettings.builder() - .applyConnectionString(new ConnectionString(mongoURI)) - .applyToSslSettings(builder -> builder.applySettings(sslSetting).build()).build(); + MongoClientOptions options = MongoClientOptions.builder() + .sslEnabled(sslEnabled) + .sslInvalidHostNameAllowed(sslInvalidHostNameAllowed) + .sslContext(sslContext) + .build(); + + if (mongoUsername.length() != 0) { + String authSource; + if ((mongoAuthSource != null) && !mongoAuthSource.isEmpty()) { + authSource = mongoAuthSource; + } else { + authSource = dbName; + } + MongoCredential credential = MongoCredential.createCredential(mongoUsername, authSource, + mongoPassword.toCharArray()); + + /**** + // This constructor is deprecated see Mongo Client API documentation + // @deprecated Prefer {@link #MongoClient(List, MongoCredential, MongoClientOptions)} + client = new MongoClient(servers, Arrays.asList(credential)); + ****/ + if ((mongoReplicaSet!= null) && !mongoReplicaSet.isEmpty()) { + options = MongoClientOptions.builder() + .requiredReplicaSetName(mongoReplicaSet) + .sslEnabled(sslEnabled) + .sslInvalidHostNameAllowed(sslInvalidHostNameAllowed) + .sslContext(sslContext) + .build(); + } + if (servers.size() == 1) { // allow auto-discover when just one endpoint is provided + client = new MongoClient(servers.get(0), credential, options); + } else { + client = new MongoClient(servers, credential, options); + } + } else { + if (servers.size() == 1) { // allow auto-discover when just one endpoint is provided + client = new MongoClient(servers.get(0), options); + } else { + client = new MongoClient(servers, options); + } + } // if else + } // if - client = MongoClients.create(settings); - } + // get the database return client.getDatabase(dbName); - } // getDatabase /** diff --git a/cygnus-common/src/test/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImplTest.java b/cygnus-common/src/test/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImplTest.java index 8e039949e..f4bb076f9 100644 --- a/cygnus-common/src/test/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImplTest.java +++ b/cygnus-common/src/test/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImplTest.java @@ -53,7 +53,7 @@ public MongoBackendImplTest() { public void testGetRange() { System.out.println(getTestTraceHead("[MongoBackendImpl.getRange]") + "-------- Given a resolution, its related range is correctly returned"); - MongoBackendImpl backend = new MongoBackendImpl(null, null, false, false, null, null, null, null); + MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, false, false, null, null, null, null); try { @@ -115,7 +115,7 @@ public void testGetRange() { public void testGetOrigin() { System.out.println(getTestTraceHead("[MongoBackendImpl.getOrigin]") + "-------- Given a calendar and a resolution, its related origin is correctly returned"); - MongoBackendImpl backend = new MongoBackendImpl(null, null, false, false, null, null, null, null); + MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, false, false, null, null, null, null); GregorianCalendar calendar = new GregorianCalendar(2017, 4, 5, 11, 46, 13); try { @@ -198,7 +198,7 @@ public void testGetOrigin() { public void testGetOffset() { System.out.println(getTestTraceHead("[MongoBackendImpl.getOffset]") + "-------- Given a calendar and a resolution, its related offset is correctly returned"); - MongoBackendImpl backend = new MongoBackendImpl(null, null, false, false, null, null, null, null); + MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, false, false, null, null, null, null); GregorianCalendar calendar = new GregorianCalendar(2017, 3, 5, 11, 46, 13); // month 3 is April try { @@ -266,7 +266,7 @@ public void testBuildQueryForInsertAggregated() { String entityType = "someType"; String attrName = "someName"; GregorianCalendar calendar = new GregorianCalendar(2017, 3, 5, 11, 46, 13); // month 3 is April - MongoBackendImpl backend = new MongoBackendImpl(null, DataModel.DMBYSERVICEPATH, false, false, null, null, null, null); + MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, DataModel.DMBYSERVICEPATH, false, false, null, null, null, null); String queryForInsertAggregated = "{\"_id\": {\"entityId\": \"someId\", \"entityType\": \"someType\", " + "\"attrName\": \"someName\", \"origin\": {\"$date\": 1491392760000}, " + "\"resolution\": \"second\", \"range\": \"minute\"}, \"points.offset\": 13}"; @@ -353,7 +353,7 @@ public void testBuildQueryForInsertAggregated() { throw e; } // try catch - backend = new MongoBackendImpl(null, DataModel.DMBYENTITY, false, false, null, null, null, null); + backend = new MongoBackendImpl(null, null, null, null, null, DataModel.DMBYENTITY, false, false, null, null, null, null); queryForInsertAggregated = "{\"_id\": {\"attrName\": \"someName\", " + "\"origin\": {\"$date\": 1491392760000}, \"resolution\": \"second\", " @@ -457,7 +457,7 @@ public void testBuildUpdateForUpdateNumerical() { double sum2 = 200; int numSamples = 2; GregorianCalendar calendar = new GregorianCalendar(2017, 3, 5, 11, 46, 13); // month 3 is April - MongoBackendImpl backend = new MongoBackendImpl(null, null, false, false, null, null, null, null); + MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, false, false, null, null, null, null); String updateForUpdate = "{\"$set\": {\"attrType\": \"someType\"}, " + "\"$inc\": {\"points.$.samples\": 2, \"points.$.sum\": 20.0, \"points.$.sum2\": 200.0}, " + "\"$min\": {\"points.$.min\": 0.0}, \"$max\": {\"points.$.max\": 10.0}}"; @@ -489,7 +489,7 @@ public void testBuildUpdateForUpdateString() { String value = "someString"; int count = 2; GregorianCalendar calendar = new GregorianCalendar(2017, 3, 5, 11, 46, 13); // month 3 is April - MongoBackendImpl backend = new MongoBackendImpl(null, null, false, false, null, null, null, null); + MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, false, false, null, null, null, null); String updateForUpdate = "{\"$set\": {\"attrType\": \"someType\"}, " + "\"$inc\": {\"points.13.samples\": 2, \"points.13.occur.someString\": 2}}"; diff --git a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSIMongoBaseSink.java b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSIMongoBaseSink.java index 6038053bd..bf355d64f 100644 --- a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSIMongoBaseSink.java +++ b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSIMongoBaseSink.java @@ -34,7 +34,11 @@ public abstract class NGSIMongoBaseSink extends NGSISink { protected static final CygnusLogger LOGGER = new CygnusLogger(NGSIMongoBaseSink.class); - protected String mongoURI; + protected String mongoHosts; + protected String mongoUsername; + protected String mongoPassword; + protected String mongoAuthSource; + protected String mongoReplicaSet; protected Boolean sslEnabled; protected Boolean sslInvalidHostNameAllowed; protected String sslKeystorePathFile; @@ -48,12 +52,44 @@ public abstract class NGSIMongoBaseSink extends NGSISink { protected boolean ignoreWhiteSpaces; /** - * Gets the mongo uri. It is protected since it is used by the tests. + * Gets the mongo hosts. It is protected since it is used by the tests. * @return */ - protected String getMongoURI() { - return mongoURI; - } // getMongoURI + protected String getMongoHosts() { + return mongoHosts; + } // getMongoHosts + + /** + * Gets the mongo username. It is protected since it is used by the tests. + * @return + */ + protected String getUsername() { + return mongoUsername; + } // getUsername + + /** + * Gets the mongo password. It is protected since it is used by the tests. + * @return + */ + protected String getPassword() { + return mongoPassword; + } // getPassword + + /** + * Gets the mongo auth_source. It is protected since it is used by the tests. + * @return + */ + protected String getAuthSource() { + return mongoAuthSource; + } // getAuthSource + + /** + * Gets the mongo replica_set. It is protected since it is used by the tests. + * @return + */ + protected String getReplicaSet() { + return mongoReplicaSet; + } // getReplicaSet /** * Gets the database prefix. It is protected since it is used by the tests. @@ -91,8 +127,17 @@ protected MongoBackendImpl getBackend() { public void configure(Context context) { super.configure(context); - mongoURI = context.getString("mongo_uri", "mongodb://localhost:27017"); - LOGGER.info("[" + this.getName() + "] Reading configuration (mongo_uri=" + mongoURI + ")"); + mongoHosts = context.getString("mongo_hosts", "localhost:27017"); + LOGGER.debug("[" + this.getName() + "] Reading configuration (mongo_hosts=" + mongoHosts + ")"); + mongoUsername = context.getString("mongo_username", ""); + LOGGER.debug("[" + this.getName() + "] Reading configuration (mongo_username=" + mongoUsername + ")"); + // FIXME: mongoPassword should be read as a SHA1 and decoded here + mongoPassword = context.getString("mongo_password", ""); + LOGGER.debug("[" + this.getName() + "] Reading configuration (mongo_password=" + mongoPassword + ")"); + mongoAuthSource = context.getString("mongo_auth_source", ""); + LOGGER.debug("[" + this.getName() + "] Reading configuration (mongo_auth_source=" + mongoAuthSource + ")"); + mongoReplicaSet = context.getString("mongo_replica_set", ""); + LOGGER.debug("[" + this.getName() + "] Reading configuration (mongo_replica_set=" + mongoReplicaSet + ")"); if (enableEncoding) { dbPrefix = NGSICharsets.encodeMongoDBDatabase(context.getString("db_prefix", "sth_")); @@ -170,7 +215,8 @@ public void configure(Context context) { @Override public void start() { try { - backend = new MongoBackendImpl(mongoURI, dataModel, + backend = new MongoBackendImpl(mongoHosts, mongoUsername, mongoPassword, + mongoAuthSource, mongoReplicaSet, dataModel, sslEnabled, sslInvalidHostNameAllowed, sslKeystorePathFile, sslKeystorePassword, sslTruststorePathFile, sslTruststorePassword); diff --git a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md index ce912888d..575ceb4d3 100644 --- a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md +++ b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md @@ -314,7 +314,11 @@ When datamodel changes Cygnus tries to recreate index (delete current and create | data\_model | no | dm-by-entity | dm-by-service-path, dm-by-entity or . dm-by-service is not currently supported. | | attr\_persistence | no | row | row or column. | | attr\_metadata\_store | no | false | true or false. | -| mongo\_uri | no | mongodb://localhost:27017 | Mongo DB Connection String. | +| mongo\_hosts | no | localhost:27017 | FQDN/IP:port where the MongoDB server runs (standalone case) or comma-separated list of FQDN/IP:port pairs where the MongoDB replica set members run. | +| mongo\_username | no | empty | If empty, no authentication is done. | +| mongo\_password | no | empty | If empty, no authentication is done. | +| mongo\_auth_source | no | empty | Auth source database use to authenticate the user. Usually could be `admin`. | +| mongo\_replica_set | no | empty | Replica Set name. Note that this parameter is optional because Cygnus is able to connect to a MongoDB replica set without needing to specify its name.| | db\_prefix | no | sth_ || | mongo\_ssl | no | false | Enable SSL in mongodb connection | | mongo\_ssl\_invalid\_host\_allowed | no | false | Allow invalid host name in mongo SSL connections | From b1f41b241262f9e68c2ebbc139600ee63e11b14a Mon Sep 17 00:00:00 2001 From: kumardeepak5 Date: Thu, 22 Aug 2024 12:33:14 +0000 Subject: [PATCH 4/6] changes as review comment on PR-2402 --- CHANGES_NEXT_RELEASE | 2 + .../backends/mongo/MongoBackendImpl.java | 55 ++++++++++++++++++- .../backends/mongo/MongoBackendImplTest.java | 14 ++--- .../iot/cygnus/sinks/NGSIMongoBaseSink.java | 13 ++++- .../ngsi_mongo_sink.md | 1 + 5 files changed, 75 insertions(+), 10 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 7112c6dfe..0aa8fabf7 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1 +1,3 @@ - [cygnus-ngsi] Upgrade Debian version from 12.5 to 12.6 in Dockerfile +- [cygnus-common][cygnus-ngsi] New setting mongo_uri (#2402) +- [cygnus-common][cygnus-ngsi] Deprecate (mongo_hosts, mongo_username, mongo_password, mongo_auth_source, mongo_replica_set) (use mongo_uri instead) diff --git a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java index c857b67a6..88d4790b3 100644 --- a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java +++ b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImpl.java @@ -18,6 +18,7 @@ package com.telefonica.iot.cygnus.backends.mongo; import com.mongodb.*; +import com.mongodb.client.MongoClients; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; import com.mongodb.client.model.CreateCollectionOptions; @@ -26,7 +27,6 @@ import com.mongodb.client.model.WriteModel; import com.mongodb.client.model.UpdateOneModel; import com.mongodb.connection.SslSettings; -import com.mongodb.client.result.UpdateResult; import com.telefonica.iot.cygnus.log.CygnusLogger; import com.telefonica.iot.cygnus.sinks.Enums.DataModel; import java.util.ArrayList; @@ -42,6 +42,8 @@ import java.security.KeyStore; import java.io.FileInputStream; import java.io.InputStream; + +import org.apache.commons.lang.StringUtils; import org.bson.Document; /** @@ -57,6 +59,8 @@ public class MongoBackendImpl implements MongoBackend { public enum Resolution { SECOND, MINUTE, HOUR, DAY, MONTH } private MongoClient client; + private com.mongodb.client.MongoClient clientI; + private final String mongoURI; private final String mongoHosts; private final String mongoUsername; private final String mongoPassword; @@ -80,12 +84,14 @@ public enum Resolution { SECOND, MINUTE, HOUR, DAY, MONTH } * @param mongoReplicaSet * @param dataModel */ - public MongoBackendImpl(String mongoHosts, String mongoUsername, String mongoPassword, + public MongoBackendImpl(String mongoURI, String mongoHosts, String mongoUsername, String mongoPassword, String mongoAuthSource, String mongoReplicaSet, DataModel dataModel, Boolean sslEnabled, Boolean sslInvalidHostNameAllowed, String sslKeystorePathFile, String sslKeystorePassword, String sslTruststorePathFile, String sslTruststorePassword) { client = null; + clientI = null; + this.mongoURI = mongoURI; this.mongoHosts = mongoHosts; this.mongoUsername = mongoUsername; this.mongoPassword = mongoPassword; @@ -582,6 +588,9 @@ protected ArrayList buildUpdateForUpdate(String attrType, boolean * @return */ private MongoDatabase getDatabase(String dbName) { + if(StringUtils.isNotEmpty(mongoURI)) { + return getDatabaseByUsingMongoURI(dbName); + } // create a ServerAddress object for each configured URI List servers = new ArrayList<>(); String[] uris = mongoHosts.split(","); @@ -672,6 +681,48 @@ private MongoDatabase getDatabase(String dbName) { return client.getDatabase(dbName); } // getDatabase + /** + * Gets a Mongo database by using mongouri. + * @param dbName + * @return + */ + private MongoDatabase getDatabaseByUsingMongoURI(String dbName) { + if (clientI == null) { + SSLContext sslContext = null; + if (sslEnabled) { + try { + KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType()); + if ((sslKeystorePathFile != null) && !sslKeystorePathFile.isEmpty()) { + InputStream keyStoreStream = new FileInputStream(sslKeystorePathFile); + keyStore.load(keyStoreStream, sslKeystorePassword.toCharArray()); + } else { + keyStore.load(null); + } + if ((sslTruststorePathFile != null) && !sslTruststorePathFile.isEmpty()) { + InputStream trustStoreStream = new FileInputStream(sslTruststorePathFile); + keyStore.load(trustStoreStream, sslTruststorePassword.toCharArray()); + } + TrustManagerFactory trustManagerFactory = TrustManagerFactory + .getInstance(TrustManagerFactory.getDefaultAlgorithm()); + trustManagerFactory.init(keyStore); + sslContext = SSLContext.getInstance("TLS"); + sslContext.init(null, trustManagerFactory.getTrustManagers(), new java.security.SecureRandom()); + } catch (Exception e) { + LOGGER.warn("Error when init SSL Context: " + e.getMessage()); + } + } + + SslSettings sslSetting = SslSettings.builder().enabled(sslEnabled) + .invalidHostNameAllowed(sslInvalidHostNameAllowed).context(sslContext).build(); + MongoClientSettings settings = MongoClientSettings.builder() + .applyConnectionString(new ConnectionString(mongoURI)) + .applyToSslSettings(builder -> builder.applySettings(sslSetting).build()).build(); + + clientI = MongoClients.create(settings); + } + return clientI.getDatabase(dbName); + } + /** * Given a resolution, gets the range. It is protected for testing purposes. * @param resolution diff --git a/cygnus-common/src/test/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImplTest.java b/cygnus-common/src/test/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImplTest.java index f4bb076f9..831e70473 100644 --- a/cygnus-common/src/test/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImplTest.java +++ b/cygnus-common/src/test/java/com/telefonica/iot/cygnus/backends/mongo/MongoBackendImplTest.java @@ -53,7 +53,7 @@ public MongoBackendImplTest() { public void testGetRange() { System.out.println(getTestTraceHead("[MongoBackendImpl.getRange]") + "-------- Given a resolution, its related range is correctly returned"); - MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, false, false, null, null, null, null); + MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, null, false, false, null, null, null, null); try { @@ -115,7 +115,7 @@ public void testGetRange() { public void testGetOrigin() { System.out.println(getTestTraceHead("[MongoBackendImpl.getOrigin]") + "-------- Given a calendar and a resolution, its related origin is correctly returned"); - MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, false, false, null, null, null, null); + MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, null, false, false, null, null, null, null); GregorianCalendar calendar = new GregorianCalendar(2017, 4, 5, 11, 46, 13); try { @@ -198,7 +198,7 @@ public void testGetOrigin() { public void testGetOffset() { System.out.println(getTestTraceHead("[MongoBackendImpl.getOffset]") + "-------- Given a calendar and a resolution, its related offset is correctly returned"); - MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, false, false, null, null, null, null); + MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, null, false, false, null, null, null, null); GregorianCalendar calendar = new GregorianCalendar(2017, 3, 5, 11, 46, 13); // month 3 is April try { @@ -266,7 +266,7 @@ public void testBuildQueryForInsertAggregated() { String entityType = "someType"; String attrName = "someName"; GregorianCalendar calendar = new GregorianCalendar(2017, 3, 5, 11, 46, 13); // month 3 is April - MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, DataModel.DMBYSERVICEPATH, false, false, null, null, null, null); + MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, DataModel.DMBYSERVICEPATH, false, false, null, null, null, null); String queryForInsertAggregated = "{\"_id\": {\"entityId\": \"someId\", \"entityType\": \"someType\", " + "\"attrName\": \"someName\", \"origin\": {\"$date\": 1491392760000}, " + "\"resolution\": \"second\", \"range\": \"minute\"}, \"points.offset\": 13}"; @@ -353,7 +353,7 @@ public void testBuildQueryForInsertAggregated() { throw e; } // try catch - backend = new MongoBackendImpl(null, null, null, null, null, DataModel.DMBYENTITY, false, false, null, null, null, null); + backend = new MongoBackendImpl(null, null, null, null, null, null, DataModel.DMBYENTITY, false, false, null, null, null, null); queryForInsertAggregated = "{\"_id\": {\"attrName\": \"someName\", " + "\"origin\": {\"$date\": 1491392760000}, \"resolution\": \"second\", " @@ -457,7 +457,7 @@ public void testBuildUpdateForUpdateNumerical() { double sum2 = 200; int numSamples = 2; GregorianCalendar calendar = new GregorianCalendar(2017, 3, 5, 11, 46, 13); // month 3 is April - MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, false, false, null, null, null, null); + MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, null, false, false, null, null, null, null); String updateForUpdate = "{\"$set\": {\"attrType\": \"someType\"}, " + "\"$inc\": {\"points.$.samples\": 2, \"points.$.sum\": 20.0, \"points.$.sum2\": 200.0}, " + "\"$min\": {\"points.$.min\": 0.0}, \"$max\": {\"points.$.max\": 10.0}}"; @@ -489,7 +489,7 @@ public void testBuildUpdateForUpdateString() { String value = "someString"; int count = 2; GregorianCalendar calendar = new GregorianCalendar(2017, 3, 5, 11, 46, 13); // month 3 is April - MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, false, false, null, null, null, null); + MongoBackendImpl backend = new MongoBackendImpl(null, null, null, null, null, null, null, false, false, null, null, null, null); String updateForUpdate = "{\"$set\": {\"attrType\": \"someType\"}, " + "\"$inc\": {\"points.13.samples\": 2, \"points.13.occur.someString\": 2}}"; diff --git a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSIMongoBaseSink.java b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSIMongoBaseSink.java index bf355d64f..65c64320d 100644 --- a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSIMongoBaseSink.java +++ b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSIMongoBaseSink.java @@ -34,6 +34,7 @@ public abstract class NGSIMongoBaseSink extends NGSISink { protected static final CygnusLogger LOGGER = new CygnusLogger(NGSIMongoBaseSink.class); + protected String mongoURI; protected String mongoHosts; protected String mongoUsername; protected String mongoPassword; @@ -50,6 +51,14 @@ public abstract class NGSIMongoBaseSink extends NGSISink { protected MongoBackendImpl backend; protected long dataExpiration; protected boolean ignoreWhiteSpaces; + + /** + * Gets the mongo uri. It is protected since it is used by the tests. + * @return + */ + protected String getMongoURI() { + return mongoURI; + } // getMongoURI /** * Gets the mongo hosts. It is protected since it is used by the tests. @@ -127,6 +136,8 @@ protected MongoBackendImpl getBackend() { public void configure(Context context) { super.configure(context); + mongoURI = context.getString("mongo_uri", ""); + LOGGER.debug("[" + this.getName() + "] Reading configuration (mongo_uri=" + mongoURI + ")"); mongoHosts = context.getString("mongo_hosts", "localhost:27017"); LOGGER.debug("[" + this.getName() + "] Reading configuration (mongo_hosts=" + mongoHosts + ")"); mongoUsername = context.getString("mongo_username", ""); @@ -215,7 +226,7 @@ public void configure(Context context) { @Override public void start() { try { - backend = new MongoBackendImpl(mongoHosts, mongoUsername, mongoPassword, + backend = new MongoBackendImpl(mongoURI, mongoHosts, mongoUsername, mongoPassword, mongoAuthSource, mongoReplicaSet, dataModel, sslEnabled, sslInvalidHostNameAllowed, sslKeystorePathFile, sslKeystorePassword, diff --git a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md index 575ceb4d3..bc7ce72f0 100644 --- a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md +++ b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md @@ -314,6 +314,7 @@ When datamodel changes Cygnus tries to recreate index (delete current and create | data\_model | no | dm-by-entity | dm-by-service-path, dm-by-entity or . dm-by-service is not currently supported. | | attr\_persistence | no | row | row or column. | | attr\_metadata\_store | no | false | true or false. | +| mongo\_uri | no | empty | Mongo DB Connection String. In case of non empty mongo\_uri parameters (mongo\_hosts, mongo\_username, mongo\_password, mongo\_auth_source, mongo\_replica_set) would be ignored. | | mongo\_hosts | no | localhost:27017 | FQDN/IP:port where the MongoDB server runs (standalone case) or comma-separated list of FQDN/IP:port pairs where the MongoDB replica set members run. | | mongo\_username | no | empty | If empty, no authentication is done. | | mongo\_password | no | empty | If empty, no authentication is done. | From 85e58fb8eaafe564f4241242748f74d29b9267b3 Mon Sep 17 00:00:00 2001 From: kumardeepak5 <122004784+kumardeepak5@users.noreply.github.com> Date: Fri, 30 Aug 2024 14:43:17 +0530 Subject: [PATCH 5/6] Update CHANGES_NEXT_RELEASE Modified CNR Entry As per Review Comment --- CHANGES_NEXT_RELEASE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 0aa8fabf7..75bdbe803 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,3 +1,3 @@ - [cygnus-ngsi] Upgrade Debian version from 12.5 to 12.6 in Dockerfile -- [cygnus-common][cygnus-ngsi] New setting mongo_uri (#2402) +- [cygnus-common][cygnus-ngsi] New setting mongo_uri (#2387) - [cygnus-common][cygnus-ngsi] Deprecate (mongo_hosts, mongo_username, mongo_password, mongo_auth_source, mongo_replica_set) (use mongo_uri instead) From 016d5ebe4429590759df1ae6be5785a14ed89627 Mon Sep 17 00:00:00 2001 From: kumardeepak5 <122004784+kumardeepak5@users.noreply.github.com> Date: Fri, 30 Aug 2024 15:03:02 +0530 Subject: [PATCH 6/6] Updated ngsi_mongo_sink doc --- .../flume_extensions_catalogue/ngsi_mongo_sink.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md index bc7ce72f0..6ae02f385 100644 --- a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md +++ b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_mongo_sink.md @@ -315,11 +315,11 @@ When datamodel changes Cygnus tries to recreate index (delete current and create | attr\_persistence | no | row | row or column. | | attr\_metadata\_store | no | false | true or false. | | mongo\_uri | no | empty | Mongo DB Connection String. In case of non empty mongo\_uri parameters (mongo\_hosts, mongo\_username, mongo\_password, mongo\_auth_source, mongo\_replica_set) would be ignored. | -| mongo\_hosts | no | localhost:27017 | FQDN/IP:port where the MongoDB server runs (standalone case) or comma-separated list of FQDN/IP:port pairs where the MongoDB replica set members run. | -| mongo\_username | no | empty | If empty, no authentication is done. | -| mongo\_password | no | empty | If empty, no authentication is done. | -| mongo\_auth_source | no | empty | Auth source database use to authenticate the user. Usually could be `admin`. | -| mongo\_replica_set | no | empty | Replica Set name. Note that this parameter is optional because Cygnus is able to connect to a MongoDB replica set without needing to specify its name.| +| mongo\_hosts | no | localhost:27017 | **DEPRECATED** (use mongo_uri instead). FQDN/IP:port where the MongoDB server runs (standalone case) or comma-separated list of FQDN/IP:port pairs where the MongoDB replica set members run. | +| mongo\_username | no | empty | **DEPRECATED** (use mongo_uri instead). If empty, no authentication is done. | +| mongo\_password | no | empty | **DEPRECATED** (use mongo_uri instead). If empty, no authentication is done. | +| mongo\_auth_source | no | empty | **DEPRECATED** (use mongo_uri instead). Auth source database use to authenticate the user. Usually could be `admin`. | +| mongo\_replica_set | no | empty | **DEPRECATED** (use mongo_uri instead). Replica Set name. Note that this parameter is optional because Cygnus is able to connect to a MongoDB replica set without needing to specify its name.| | db\_prefix | no | sth_ || | mongo\_ssl | no | false | Enable SSL in mongodb connection | | mongo\_ssl\_invalid\_host\_allowed | no | false | Allow invalid host name in mongo SSL connections |