unstable features

CMakeLists.txt

@@ -432,8 +432,8 @@ if (NOT DEBUG)
     SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DNDEBUG")
 endif ()
 
-SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 ${DEBUG_SYMBOL} -O2 -pipe -m64 -fopenmp -Wall -W -fPIC -Wno-unused-parameter -Wno-strict-aliasing -Wno-parentheses -fno-omit-frame-pointer ")
-SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 ${DEBUG_SYMBOL} -O2 -pipe -m64 -fopenmp -Wall -W -fPIC -Wno-unused-parameter -Wno-strict-aliasing -Wno-parentheses -fno-omit-frame-pointer")
+SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 ${DEBUG_SYMBOL} -O2 -pipe -m64 -fopenmp -Wall -g -W -fPIC -Wno-unused-parameter -Wno-strict-aliasing -Wno-parentheses -fno-omit-frame-pointer ")
+SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 ${DEBUG_SYMBOL} -O2 -pipe -m64 -fopenmp -Wall -g -W -fPIC -Wno-unused-parameter -Wno-strict-aliasing -Wno-parentheses -fno-omit-frame-pointer")
 
 add_definitions(
         -D_GNU_SOURCE

include/common/common.h

@@ -1387,6 +1387,16 @@ inline bool float_equal(double value, double compare, double epsilon = 1e-9) {
     return std::fabs(value - compare) < epsilon;
 }
 
+inline std::string redis_encode(std::string str) {
+    //\n<length>\n<data>
+    std::string tmp;
+    tmp += "\n";
+    tmp += std::to_string(str.length());
+    tmp += "\n";
+    tmp += str;
+    return tmp;
+}
+
 //set double buffer
 template<typename T>
 using DoubleBufferSet = butil::DoublyBufferedData<std::unordered_set<T>>;

include/common/expr_value.h

@@ -167,6 +167,7 @@ struct ExprValue {
             case pb::HLL:
             case pb::HEX:
             case pb::TDIGEST:
+            case pb::JSON:
                 str_val = value.string_val();
                 break;
             case pb::BITMAP: {
@@ -190,6 +191,7 @@ struct ExprValue {
         float_precision_len = -1;
         str_val = value_str;
         if (primitive_type == pb::STRING 
+            || primitive_type == pb::JSON
             || primitive_type == pb::HEX 
             || primitive_type == pb::BITMAP 
             || primitive_type == pb::HLL 
@@ -323,6 +325,7 @@ struct ExprValue {
             case pb::STRING:
             case pb::HEX:
             case pb::BITMAP:
+            case pb::JSON:
             case pb::TDIGEST:
                 value->set_string_val(str_val);
                 break;
@@ -422,6 +425,7 @@ struct ExprValue {
             case pb::STRING:
             case pb::HEX:
             case pb::HLL:
+            case pb::JSON:
             case pb::TDIGEST:
                 return str_val.length();
             case pb::DATETIME:
@@ -522,6 +526,9 @@ struct ExprValue {
             case pb::STRING:
                 str_val = get_string();
                 break;
+            case pb::JSON:
+                str_val = get_string();
+                break;
             case pb::BITMAP: {
                 _u.bitmap = new(std::nothrow) Roaring();
                 if (str_val.size() > 0) {
@@ -576,6 +583,7 @@ struct ExprValue {
                 butil::MurmurHash3_x64_128(&_u, 8, seed, out);
                 return out[0];
             case pb::STRING: 
+            case pb::JSON:
             case pb::HEX: {
                 butil::MurmurHash3_x64_128(str_val.c_str(), str_val.size(), seed, out);
                 return out[0];
@@ -620,6 +628,7 @@ struct ExprValue {
             }
             case pb::STRING:
             case pb::HEX:
+            case pb::JSON:
             case pb::HLL:
             case pb::TDIGEST:
                 return str_val;
@@ -734,6 +743,7 @@ struct ExprValue {
                     (_u.double_val < other._u.double_val ? -1 : 0);
             case pb::STRING:
             case pb::HEX:
+            case pb::JSON:
                 return str_val.compare(other.str_val);
             case pb::NULL_TYPE:
                 return -1;
@@ -788,7 +798,7 @@ struct ExprValue {
     }
 
     bool is_string() const {
-        return type == pb::STRING || type == pb::HEX || type == pb::BITMAP || type == pb::HLL || type == pb::TDIGEST;
+        return type == pb::STRING || type == pb::HEX || type == pb::BITMAP || type == pb::HLL || type == pb::TDIGEST || type == pb::JSON;
     }
 
     bool is_double() const {
@@ -862,15 +872,16 @@ struct ExprValue {
     // 默认不带us
     static ExprValue Now(int32_t precision = 0) {
         ExprValue tmp(pb::TIMESTAMP);
-        tmp._u.uint32_val = time(NULL);
-        tmp.cast_to(pb::DATETIME);
-
         if (precision > 0 and precision <= 6) {
             timeval tv;
             gettimeofday(&tv, NULL);
+            tmp._u.uint32_val = tv.tv_sec;
+            tmp.cast_to(pb::DATETIME);
             tmp._u.uint64_val |= tv.tv_usec;
             tmp.set_precision_len(precision);
         } else {
+            tmp._u.uint32_val = time(NULL);
+            tmp.cast_to(pb::DATETIME);
             tmp.set_precision_len(0);
         }
         return tmp;
@@ -895,14 +906,16 @@ struct ExprValue {
         long offset = timeinfo.tm_gmtoff;
 
         ExprValue tmp(pb::TIMESTAMP);
-        tmp._u.uint32_val = time(NULL) - offset;
-        tmp.cast_to(pb::DATETIME);
-        timeval tv;
-        gettimeofday(&tv, NULL);
-        tmp._u.uint64_val |= tv.tv_usec;
         if (precision >=0 && precision <= 6) {
+            timeval tv;
+            gettimeofday(&tv, NULL);
+            tmp._u.uint32_val = tv.tv_sec - offset;
+            tmp.cast_to(pb::DATETIME);
+            tmp._u.uint64_val |= tv.tv_usec;
             tmp.set_precision_len(precision);
         } else {
+            tmp._u.uint32_val = time(NULL) - offset;
+            tmp.cast_to(pb::DATETIME);
             tmp.set_precision_len(0);
         }
         return tmp;
@@ -927,7 +940,7 @@ struct ExprValue {
 
     struct HashFunction {
         size_t operator()(const ExprValue& ev) const {
-            if (ev.type == pb::STRING || ev.type == pb::HEX) {
+            if (ev.type == pb::STRING || ev.type == pb::HEX || ev.type == pb::JSON) {
                 return ev.hash();
             }
             return ev._u.uint64_val;

include/common/histogram.h

@@ -302,7 +302,9 @@ class PacketSample {
                     sample_sorter.insert_row(batch->get_row().get());
                 }
                 batch->reset();
-                _batch_vector.push_back(batch);
+                if (batch->size() > 0) {
+                    _batch_vector.push_back(batch);
+                }
             } while (!eos);
 
             sample_sorter.insert_done();

include/common/schema_factory.h

@@ -60,6 +60,7 @@ static const std::string TABLE_BINLOG_BACKUP_DAYS = "binlog_backup_days";     //
 
 struct UserInfo;
 class TableRecord;
+class MutTableKey;
 typedef std::shared_ptr<TableRecord> SmartRecord;
 typedef std::map<std::string, int64_t> StrInt64Map;
 
@@ -975,6 +976,13 @@ typedef ::google::protobuf::RepeatedPtrField<pb::Statistics> StatisticsVec;
                               const std::vector<SmartRecord>&  records,
                               std::vector<int64_t>& region_ids);
 
+    int get_region_by_primary_key(int64_t main_table_id,
+                          IndexInfo& index,
+                          MutTableKey &primary_key,
+                          int partition_id,
+                          pb::RegionInfo &region_info
+                          );
+
     bool exist_tableid(int64_t table_id);
     void get_all_table_by_db(const std::string& namespace_, const std::string& db_name, std::vector<SmartTable>& table_ptrs);
     void get_all_table_version(std::unordered_map<int64_t, int64_t>& table_id_version);

include/common/type_utils.h

@@ -348,6 +348,8 @@ inline uint8_t to_mysql_type(pb::PrimitiveType type) {
         case pb::BITMAP:
         case pb::TDIGEST:
             return MYSQL_TYPE_STRING;
+        case pb::JSON:
+            return MYSQL_TYPE_JSON;
         default:
             return MYSQL_TYPE_STRING;
     }
@@ -444,6 +446,8 @@ inline std::string to_mysql_type_full_string(pb::PrimitiveType type,
             return "BITMAP";
         case pb::TDIGEST:
             return "TDIGEST";
+        case pb::JSON:
+            return "json";
         default:
             return "";
     }

include/engine/table_iterator.h

@@ -57,27 +57,6 @@ struct IndexRange {
 
     IndexRange() {}
 
-    IndexRange(TableRecord* _left, 
-        TableRecord* _right,
-        IndexInfo*  _index_info,
-        IndexInfo*  _pri_info,
-        pb::RegionInfo* _region_info,
-        int left_cnt,
-        int right_cnt,
-        bool _l_open,
-        bool _r_open,
-        bool _like_prefix) :
-        left(_left),
-        right(_right),
-        index_info(_index_info),
-        pri_info(_pri_info),
-        region_info(_region_info),
-        left_field_cnt(left_cnt),
-        right_field_cnt(right_cnt),
-        left_open(_l_open),
-        right_open(_r_open),
-        like_prefix(_like_prefix) {}
-
     IndexRange(const MutTableKey& _left,
         const MutTableKey& _right,
         IndexInfo*  _index_info,

include/exec/access_path.h

@@ -194,6 +194,7 @@ enum IndexHint {
     bool _in_pred = false;
     bool _is_eq_or_in = true;
     bool _need_filter = false;
+    uint32_t prefix_ratio_index_score = UINT32_MAX;
 };
 typedef std::shared_ptr<AccessPath> SmartPath;
 }

include/exec/dml_node.h

@@ -28,7 +28,7 @@ class DMLNode : public ExecNode {
     virtual void find_place_holder(std::unordered_multimap<int, ExprNode*>& placeholders);
     int insert_row(RuntimeState* state, SmartRecord record, bool is_update = false);
     int delete_row(RuntimeState* state, SmartRecord record, MemRow* row);
-    int get_lock_row(RuntimeState* state, SmartRecord record, std::string* pk_str, MemRow* row);
+    int get_lock_row(RuntimeState* state, SmartRecord record, std::string* pk_str, MemRow* row, int64_t& ttl_ts);
     int remove_row(RuntimeState* state, SmartRecord record, 
             const std::string& pk_str, bool delete_primary = true);
     int update_row(RuntimeState* state, SmartRecord record, MemRow* row);
@@ -81,6 +81,7 @@ class DMLNode : public ExecNode {
         return _table_info;
     }
 
+
 protected:
     int init_schema_info(RuntimeState* state);
     void add_delete_conditon_fields();
@@ -128,6 +129,7 @@ class DMLNode : public ExecNode {
     int64_t _ttl_timestamp_us = 0; //ttl写入时间，0表示无ttl
     bool  _ddl_need_write = false;
     int64_t  _ddl_index_id = -1;
+    ExprNode* _last_value_expr = nullptr; // not own it
 };
 }