update to 0.9.58

bersler · Dec 7, 2022 · d198bd8 · d198bd8
1 parent 4a4a189
commit d198bd8
Show file tree

Hide file tree

Showing 14 changed files with 33 additions and 29 deletions.
diff --git a/CHANGELOG b/CHANGELOG
@@ -1,3 +1,8 @@
+0.9.58
+- added support for Oracle 19.17 and 21.7
+- bug fixes
+- code cleanup
+
 0.9.57
 - fixed compilation warnings for Release
 

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,5 +1,5 @@
 cmake_minimum_required(VERSION 3.16)
-project(OpenLogReplicator VERSION 0.9.57)
+project(OpenLogReplicator VERSION 0.9.58)
 
 set(CMAKE_CXX_STANDARD_REQUIRED TRUE)
 set(CMAKE_CXX_STANDARD 17)

diff --git a/src/OpenLogReplicator.cpp b/src/OpenLogReplicator.cpp
@@ -237,13 +237,19 @@ namespace OpenLogReplicator {
 
             if (sourceJson.HasMember("flags")) {
                 ctx->flags = Ctx::getJsonFieldU64(fileName, sourceJson, "flags");
-                if (ctx->flags > 131071)
+                if (ctx->flags > 65535)
                     throw ConfigurationException("bad JSON, invalid 'flags' value: " + std::to_string(ctx->flags) +
-                            ", expected one of: {0 .. 131071}");
+                            ", expected one of: {0 .. 65535}");
                 if (FLAG(REDO_FLAGS_DIRECT_DISABLE))
                     ctx->redoVerifyDelayUs = 500000;
             }
 
+            if (sourceJson.HasMember("experimental-lobs")) {
+                uint64_t experimentalLobs = Ctx::getJsonFieldU64(fileName, sourceJson, "experimental-lobs");
+                if (experimentalLobs > 0)
+                    ctx->experimentalLobs = true;
+            }
+
             if (readerJson.HasMember("disable-checks")) {
                 ctx->disableChecks = Ctx::getJsonFieldU64(fileName, readerJson, "disable-checks");
                 if (ctx->disableChecks > 7)

diff --git a/src/builder/Builder.cpp b/src/builder/Builder.cpp
@@ -161,7 +161,7 @@ namespace OpenLogReplicator {
             break;
 
         case SYS_COL_TYPE_BLOB:
-            if (FLAG(REDO_FLAGS_EXPERIMENTAL_LOBS)) {
+            if (ctx->experimentalLobs) {
                 if (after) {
                     parseLob(lobCtx, data, length, after, table->dataObj, false);
                     columnRaw(column->name, reinterpret_cast<uint8_t*>(valueBuffer), valueLength);
@@ -172,7 +172,7 @@ namespace OpenLogReplicator {
             break;
 
         case SYS_COL_TYPE_CLOB:
-            if (FLAG(REDO_FLAGS_EXPERIMENTAL_LOBS)) {
+            if (ctx->experimentalLobs) {
                 if (after) {
                     parseLob(lobCtx, data, length, column->charsetId, table->dataObj, true);
                     columnString(column->name);
@@ -595,7 +595,7 @@ namespace OpenLogReplicator {
             }
 
             if (redoLogRecord2p == nullptr) {
-                WARNING("incomplete row for table (OBJID: " << std::dec << redoLogRecord1->obj<< "), probably IOT offset: " << std::dec <<
+                WARNING("incomplete row for table (OBJID: " << std::dec << redoLogRecord1->obj << "), probably IOT offset: " << std::dec <<
                         redoLogRecord1->dataOffset << " xid: " << lastXid)
                 obj = 0;
                 dataObj = 0;

diff --git a/src/builder/Builder.h b/src/builder/Builder.h
@@ -702,7 +702,8 @@ namespace OpenLogReplicator {
                                 pageCnt = ctx->read16Big(data + dataOffset);
                                 dataOffset += 2;
                             } else {
-                                WARNING("incorrect LOB (new in-value) xid: " << lastXid << " data-obj: " << std::dec << dataObj)
+                                WARNING("incorrect LOB (new in-value) xid: " << lastXid << " data-obj: " << std::dec << dataObj << " page: 0x" <<
+                                        std::hex << page << " offset: " << std::dec << dataOffset)
                                 WARNING("dump LOB: " << lobId.upper() << " data: " << dumpLob(data, length))
                                 return;
                             }

diff --git a/src/builder/SystemTransaction.cpp b/src/builder/SystemTransaction.cpp
@@ -909,7 +909,7 @@ namespace OpenLogReplicator {
                                             typeSlot slot __attribute__((unused)), typeRowId& rowId) {
         if (metadata->schema->dictSysTsFind(rowId))
             throw RuntimeException(std::string("DDL: duplicate SYS.TS$: (rowid: ") + rowId.toString() + ") for insert");
-        sysTs = new SysTs(rowId, 0, 0, 0, true);
+        sysTs = new SysTs(rowId, 0, "", 0, true);
 
         uint64_t baseMax = builder->valuesMax >> 6;
         for (uint64_t base = 0; base <= baseMax; ++base) {

diff --git a/src/common/Ctx.cpp b/src/common/Ctx.cpp
@@ -75,6 +75,7 @@ namespace OpenLogReplicator {
             trace2(0),
             flags(0),
             disableChecks(0),
+            experimentalLobs(false),
             hardShutdown(false),
             softShutdown(false),
             replicatorFinished(false),

diff --git a/src/common/Ctx.h b/src/common/Ctx.h
@@ -189,8 +189,8 @@ along with OpenLogReplicator; see the file LICENSE;  If not see
 #define REDO_FLAGS_HIDE_CHECKPOINT              0x00001000
 #define REDO_FLAGS_CHECKPOINT_KEEP              0x00002000
 #define REDO_FLAGS_VERIFY_SCHEMA                0x00004000
-#define REDO_FLAGS_EXPERIMENTAL_LOBS            0x00008000
-#define REDO_FLAGS_RAW_COLUMN_DATA              0x00010000
+#define REDO_FLAGS_RAW_COLUMN_DATA              0x00008000
+
 #define FLAG(x)                                 ((ctx->flags&(x))!=0)
 
 #define DISABLE_CHECKS_GRANTS                   0x00000001
@@ -268,6 +268,7 @@ namespace OpenLogReplicator {
         std::atomic<uint64_t> trace2;
         std::atomic<uint64_t> flags;
         std::atomic<uint64_t> disableChecks;
+        std::atomic<bool> experimentalLobs;
         std::atomic<bool> hardShutdown;
         std::atomic<bool> softShutdown;
         std::atomic<bool> replicatorFinished;

diff --git a/src/main.cpp b/src/main.cpp
@@ -24,7 +24,6 @@ along with OpenLogReplicator; see the file LICENSE;  If not see
 #include <sys/utsname.h>
 #include <thread>
 #include <unistd.h>
-#include <execinfo.h>
 
 #include "OpenLogReplicator.h"
 #include "common/Ctx.h"

diff --git a/src/metadata/SerializerJson.h b/src/metadata/SerializerJson.h
@@ -33,7 +33,6 @@ namespace OpenLogReplicator {
 
     class SerializerJson: public Serializer {
     protected:
-        std::string escape(const std::string& name);
         void deserializeSysCCol(Metadata* metadata, const std::string& name, const rapidjson::Value& sysCColJson);
         void deserializeSysCDef(Metadata* metadata, const std::string& name, const rapidjson::Value& sysCDefJson);
         void deserializeSysCol(Metadata* metadata, const std::string& name, const rapidjson::Value& sysColJson);

diff --git a/src/parser/Parser.cpp b/src/parser/Parser.cpp
@@ -314,7 +314,7 @@ namespace OpenLogReplicator {
 
                 // REDO: Insert leaf row
                 case 0x0A02:
-                    if (FLAG(REDO_FLAGS_EXPERIMENTAL_LOBS)) {
+                    if (ctx->experimentalLobs) {
                         if (vectorPrev != -1 && redoLogRecord[vectorPrev].opCode == 0x0501) {
                             redoLogRecord[vectorCur].recordDataObj = redoLogRecord[vectorPrev].dataObj;
                             redoLogRecord[vectorCur].recordObj = redoLogRecord[vectorPrev].obj;
@@ -325,7 +325,7 @@ namespace OpenLogReplicator {
 
                 // REDO: Init header
                 case 0x0A08:
-                    if (FLAG(REDO_FLAGS_EXPERIMENTAL_LOBS)) {
+                    if (ctx->experimentalLobs) {
                         if (vectorPrev != -1 && redoLogRecord[vectorPrev].opCode == 0x0501) {
                             redoLogRecord[vectorCur].recordDataObj = redoLogRecord[vectorPrev].dataObj;
                             redoLogRecord[vectorCur].recordObj = redoLogRecord[vectorPrev].obj;
@@ -336,7 +336,7 @@ namespace OpenLogReplicator {
 
                 // REDO: Update key data in row
                 case 0x0A12:
-                    if (FLAG(REDO_FLAGS_EXPERIMENTAL_LOBS)) {
+                    if (ctx->experimentalLobs) {
                         if (vectorPrev != -1 && redoLogRecord[vectorPrev].opCode == 0x0501) {
                             redoLogRecord[vectorCur].recordDataObj = redoLogRecord[vectorPrev].dataObj;
                             redoLogRecord[vectorCur].recordObj = redoLogRecord[vectorPrev].obj;
@@ -437,14 +437,14 @@ namespace OpenLogReplicator {
 
                 // LOB
                 case 0x1301:
-                    if (FLAG(REDO_FLAGS_EXPERIMENTAL_LOBS)) {
+                    if (ctx->experimentalLobs) {
                         if (vectorPrev == -1)
                             OpCode1301::process(ctx, &redoLogRecord[vectorCur]);
                     }
                     break;
 
                 case 0x1A06:
-                    if (FLAG(REDO_FLAGS_EXPERIMENTAL_LOBS)) {
+                    if (ctx->experimentalLobs) {
                         if (vectorPrev == -1)
                             OpCode1A06::process(ctx, &redoLogRecord[vectorCur]);
                     }
@@ -581,7 +581,7 @@ namespace OpenLogReplicator {
     }
 
     void Parser::appendToTransactionLob(RedoLogRecord* redoLogRecord1) {
-        if (!FLAG(REDO_FLAGS_EXPERIMENTAL_LOBS))
+        if (!ctx->experimentalLobs)
             return;
 
         OracleLob* lob = metadata->schema->checkLobDict(redoLogRecord1->dataObj);
@@ -950,7 +950,7 @@ namespace OpenLogReplicator {
     }
 
     void Parser::appendToTransactionIndex(RedoLogRecord* redoLogRecord1, RedoLogRecord* redoLogRecord2) {
-        if (!FLAG(REDO_FLAGS_EXPERIMENTAL_LOBS))
+        if (!ctx->experimentalLobs)
             return;
 
         // Skip other PDB vectors

diff --git a/src/reader/Reader.cpp b/src/reader/Reader.cpp
@@ -280,8 +280,8 @@ namespace OpenLogReplicator {
             || (compatVsn >= 0x0C100000 && compatVsn <= 0x0C100200)  // 12.1.0.0 - 12.1.0.2
             || (compatVsn >= 0x0C200000 && compatVsn <= 0x0C200100)  // 12.2.0.0 - 12.2.0.1
             || (compatVsn >= 0x12000000 && compatVsn <= 0x120E0000)  // 18.0.0.0 - 18.14.0.0
-            || (compatVsn >= 0x13000000 && compatVsn <= 0x13100000)  // 19.0.0.0 - 19.16.0.0
-            || (compatVsn >= 0x15000000 && compatVsn <= 0x15070000)) // 21.0.0.0 - 21.7.0.0
+            || (compatVsn >= 0x13000000 && compatVsn <= 0x13110000)  // 19.0.0.0 - 19.17.0.0
+            || (compatVsn >= 0x15000000 && compatVsn <= 0x15080000)) // 21.0.0.0 - 21.8.0.0
             version = compatVsn;
         else {
             ERROR("invalid database version (found: 0x" << std::setfill('0') << std::setw(8) << std::hex << compatVsn << "): " << fileName)

diff --git a/src/replicator/Replicator.cpp b/src/replicator/Replicator.cpp
@@ -573,10 +573,6 @@ namespace OpenLogReplicator {
         return p1->sequence > p2->sequence;
     }
 
-    bool parserCompareReverse::operator()(Parser* const& p1, Parser* const& p2) {
-        return p1->sequence < p2->sequence;
-    }
-
     void Replicator::updateResetlogs() {
         for (OracleIncarnation* oi : metadata->oracleIncarnations) {
             if (oi->resetlogs == metadata->resetlogs) {

diff --git a/src/replicator/Replicator.h b/src/replicator/Replicator.h
@@ -45,10 +45,6 @@ namespace OpenLogReplicator {
         bool operator()(Parser* const& p1, Parser* const& p2);
     };
 
-    struct parserCompareReverse {
-        bool operator()(Parser* const& p1, Parser* const& p2);
-    };
-
     class Replicator : public Thread {
     protected:
         void (*archGetLog)(Replicator* replicator);