From 2e5b123d46dac8e1e8062b1fac0e3f2a448a405a Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 2 Aug 2023 09:03:20 +0200 Subject: [PATCH 01/10] ensure tableName already not contains schema --- .../com/telefonica/iot/cygnus/backends/sql/SQLBackendImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/sql/SQLBackendImpl.java b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/sql/SQLBackendImpl.java index ad697b4da..c4d933253 100644 --- a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/sql/SQLBackendImpl.java +++ b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/sql/SQLBackendImpl.java @@ -310,7 +310,7 @@ private CachedRowSet select(String dataBase, String schema, String tableName, St if (sqlInstance == SQLInstance.MYSQL) { query = "select " + selection + " from `" + tableName + "` order by recvTime"; } else if (sqlInstance == SQLInstance.POSTGRESQL) { - if (schema != null) { + if (schema != null && (!tableName.startsWith(schema))) { tableName = schema + '.' + tableName; } query = "select " + selection + " from " + tableName + " order by recvTime"; From a1e973b786a11ffc0b46b338d1e21e442db1da84 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 2 Aug 2023 09:32:18 +0200 Subject: [PATCH 02/10] check in delete --- .../com/telefonica/iot/cygnus/backends/sql/SQLBackendImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/sql/SQLBackendImpl.java b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/sql/SQLBackendImpl.java index c4d933253..f352aab92 100644 --- a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/sql/SQLBackendImpl.java +++ b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/sql/SQLBackendImpl.java @@ -356,7 +356,7 @@ private void delete(String dataBase, String schema, String tableName, String fil if (sqlInstance == SQLInstance.MYSQL) { query = "delete from `" + tableName + "` where " + filters; } else if (sqlInstance == SQLInstance.POSTGRESQL) { - if (schema != null) { + if (schema != null && (!tableName.startsWith(schema))) { tableName = schema + '.' + tableName; } query = "delete from " + tableName + " where " + filters; From e0f98755f811fd04bcbee2672186f43527d83e4a Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 2 Aug 2023 10:36:10 +0200 Subject: [PATCH 03/10] close resultSet explicitly --- .../com/telefonica/iot/cygnus/backends/sql/SQLBackendImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/sql/SQLBackendImpl.java b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/sql/SQLBackendImpl.java index f352aab92..05edb9c7f 100644 --- a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/sql/SQLBackendImpl.java +++ b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/sql/SQLBackendImpl.java @@ -336,6 +336,7 @@ private CachedRowSet select(String dataBase, String schema, String tableName, St CachedRowSet crs = new CachedRowSetImpl(); crs.populate(rs); // FIXME: close Resultset Objects?? closeSQLObjects(con, stmt); + rs.close(); return crs; } catch (SQLTimeoutException e) { throw new CygnusPersistenceError(sqlInstance.toString().toUpperCase() + " Data select error. Query " + query, "SQLTimeoutException", e.getMessage()); From d81b0132f9db29ae56181277e37c357b5a5cb2bf Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 2 Aug 2023 11:37:45 +0200 Subject: [PATCH 04/10] limit select for exp records --- .../iot/cygnus/backends/sql/SQLBackendImpl.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/sql/SQLBackendImpl.java b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/sql/SQLBackendImpl.java index 05edb9c7f..e60f5d8fa 100644 --- a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/sql/SQLBackendImpl.java +++ b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/sql/SQLBackendImpl.java @@ -53,6 +53,7 @@ public class SQLBackendImpl implements SQLBackend{ private final int maxLatestErrors; private static final String DEFAULT_ERROR_TABLE_SUFFIX = "_error_log"; private static final int DEFAULT_MAX_LATEST_ERRORS = 100; + private static final String DEFAULT_LIMIT_SELECT_EXP_RECORDS = "32768"; private String nlsTimestampFormat; private String nlsTimestampTzFormat; @@ -308,14 +309,14 @@ private CachedRowSet select(String dataBase, String schema, String tableName, St Connection con = driver.getConnection(dataBase); String query = ""; if (sqlInstance == SQLInstance.MYSQL) { - query = "select " + selection + " from `" + tableName + "` order by recvTime"; + query = "select " + selection + " from `" + tableName + "` order by recvTime desc limit " + DEFAULT_LIMIT_SELECT_EXP_RECORDS; } else if (sqlInstance == SQLInstance.POSTGRESQL) { if (schema != null && (!tableName.startsWith(schema))) { tableName = schema + '.' + tableName; } - query = "select " + selection + " from " + tableName + " order by recvTime"; + query = "select " + selection + " from " + tableName + " order by recvTime desc limit " + DEFAULT_LIMIT_SELECT_EXP_RECORDS; } else { - query = "select " + selection + " from " + tableName + " order by recvTime"; + query = "select " + selection + " from " + tableName + " order by recvTime desc limit " + DEFAULT_LIMIT_SELECT_EXP_RECORDS; } try { @@ -377,7 +378,7 @@ private void delete(String dataBase, String schema, String tableName, String fil stmt.executeUpdate(query); } catch (SQLTimeoutException e) { throw new CygnusPersistenceError(sqlInstance.toString().toUpperCase() + " Data delete error. Query " + query, "SQLTimeoutException", e.getMessage()); - }catch (SQLException e) { + } catch (SQLException e) { closeSQLObjects(con, stmt); persistError(dataBase, schema, query, e); throw new CygnusPersistenceError(sqlInstance.toString().toUpperCase() + " Deleting error", "SQLException", e.getMessage()); From 91a8b4df2497f26ee989d16a8ceff5fbd0d7bfd0 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 2 Aug 2023 12:41:39 +0200 Subject: [PATCH 05/10] reduce limit --- .../com/telefonica/iot/cygnus/backends/sql/SQLBackendImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/sql/SQLBackendImpl.java b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/sql/SQLBackendImpl.java index e60f5d8fa..4b4318399 100644 --- a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/sql/SQLBackendImpl.java +++ b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/sql/SQLBackendImpl.java @@ -53,7 +53,7 @@ public class SQLBackendImpl implements SQLBackend{ private final int maxLatestErrors; private static final String DEFAULT_ERROR_TABLE_SUFFIX = "_error_log"; private static final int DEFAULT_MAX_LATEST_ERRORS = 100; - private static final String DEFAULT_LIMIT_SELECT_EXP_RECORDS = "32768"; + private static final String DEFAULT_LIMIT_SELECT_EXP_RECORDS = "256"; private String nlsTimestampFormat; private String nlsTimestampTzFormat; From b3bb8d0fdd19266ebff37fcb274914a40ba09f02 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 2 Aug 2023 13:16:55 +0200 Subject: [PATCH 06/10] restore limit --- .../com/telefonica/iot/cygnus/backends/sql/SQLBackendImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/sql/SQLBackendImpl.java b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/sql/SQLBackendImpl.java index 4b4318399..f73f25ba6 100644 --- a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/sql/SQLBackendImpl.java +++ b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/sql/SQLBackendImpl.java @@ -53,7 +53,7 @@ public class SQLBackendImpl implements SQLBackend{ private final int maxLatestErrors; private static final String DEFAULT_ERROR_TABLE_SUFFIX = "_error_log"; private static final int DEFAULT_MAX_LATEST_ERRORS = 100; - private static final String DEFAULT_LIMIT_SELECT_EXP_RECORDS = "256"; + private static final String DEFAULT_LIMIT_SELECT_EXP_RECORDS = "16384"; private String nlsTimestampFormat; private String nlsTimestampTzFormat; From e1e7f7d1736abb6b726c2c9a0e30d84567d5084b Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 2 Aug 2023 13:46:56 +0200 Subject: [PATCH 07/10] restore limit --- .../com/telefonica/iot/cygnus/backends/sql/SQLBackendImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/sql/SQLBackendImpl.java b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/sql/SQLBackendImpl.java index f73f25ba6..b01674d68 100644 --- a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/sql/SQLBackendImpl.java +++ b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/sql/SQLBackendImpl.java @@ -53,7 +53,7 @@ public class SQLBackendImpl implements SQLBackend{ private final int maxLatestErrors; private static final String DEFAULT_ERROR_TABLE_SUFFIX = "_error_log"; private static final int DEFAULT_MAX_LATEST_ERRORS = 100; - private static final String DEFAULT_LIMIT_SELECT_EXP_RECORDS = "16384"; + private static final String DEFAULT_LIMIT_SELECT_EXP_RECORDS = "8192"; private String nlsTimestampFormat; private String nlsTimestampTzFormat; From d464c010459e8da7f4691119f26e3bcaeb68566b Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 2 Aug 2023 13:47:48 +0200 Subject: [PATCH 08/10] restore limit --- .../com/telefonica/iot/cygnus/backends/sql/SQLBackendImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/sql/SQLBackendImpl.java b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/sql/SQLBackendImpl.java index b01674d68..d1aec7ae3 100644 --- a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/sql/SQLBackendImpl.java +++ b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/backends/sql/SQLBackendImpl.java @@ -53,7 +53,7 @@ public class SQLBackendImpl implements SQLBackend{ private final int maxLatestErrors; private static final String DEFAULT_ERROR_TABLE_SUFFIX = "_error_log"; private static final int DEFAULT_MAX_LATEST_ERRORS = 100; - private static final String DEFAULT_LIMIT_SELECT_EXP_RECORDS = "8192"; + private static final String DEFAULT_LIMIT_SELECT_EXP_RECORDS = "4096"; private String nlsTimestampFormat; private String nlsTimestampTzFormat; From e5e0cd1797830baf95437489ff303b43234f615b Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 2 Aug 2023 15:38:05 +0200 Subject: [PATCH 09/10] update CNR --- CHANGES_NEXT_RELEASE | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 1491716db..855af4fa0 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,2 +1,4 @@ -- [cygnus-ngsi] Removes "_" in schema name for DM -schema family (#2270, #2201 reopens for Postgis) - +- [cygnus-common][SQL] Fix expiration records tablename used by delete and select (#2265) +- [cygnus-common][SQL] Fix expiraton records select with a limit to avoid java out of memory error (#2273) +- [cygnus-ngsi] Removes "_" in schema name for DM -schema family (#2270, #2201 reopens for Postgis) + From b27fa8506c8e77d46ea53e2a922f23959abd3e54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 4 Aug 2023 13:33:12 +0200 Subject: [PATCH 10/10] Update CHANGES_NEXT_RELEASE --- CHANGES_NEXT_RELEASE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 855af4fa0..4d695a238 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,4 +1,4 @@ - [cygnus-common][SQL] Fix expiration records tablename used by delete and select (#2265) -- [cygnus-common][SQL] Fix expiraton records select with a limit to avoid java out of memory error (#2273) +- [cygnus-common][SQL] Fix expiration records select with a limit to avoid java out of memory error (#2273) - [cygnus-ngsi] Removes "_" in schema name for DM -schema family (#2270, #2201 reopens for Postgis)