From 951d5ba459125be0c1f87d1f65fa8fb8d58b1bb6 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Tue, 26 Apr 2022 12:21:13 +0800 Subject: [PATCH 01/29] add metric for kvstore data size --- dbms/src/Common/TiFlashMetrics.h | 1 + .../Storages/Transaction/BackgroundService.cpp | 17 +++++++++++++++++ .../Storages/Transaction/BackgroundService.h | 1 + 3 files changed, 19 insertions(+) diff --git a/dbms/src/Common/TiFlashMetrics.h b/dbms/src/Common/TiFlashMetrics.h index b67b263a617..a679659cad5 100644 --- a/dbms/src/Common/TiFlashMetrics.h +++ b/dbms/src/Common/TiFlashMetrics.h @@ -80,6 +80,7 @@ namespace DB F(type_cancel_mpp_task, {{"type", "cancel_mpp_task"}}, ExpBuckets{0.0005, 2, 30}), \ F(type_run_mpp_task, {{"type", "run_mpp_task"}}, ExpBuckets{0.0005, 2, 30})) \ M(tiflash_coprocessor_response_bytes, "Total bytes of response body", Counter) \ + M(tiflash_kvstore_region_data_memory_size, "Total region data memory size of tiflash kvstore", Gauge) \ M(tiflash_schema_version, "Current version of tiflash cached schema", Gauge) \ M(tiflash_schema_applying, "Whether the schema is applying or not (holding lock)", Gauge) \ M(tiflash_schema_apply_count, "Total number of each kinds of apply", Counter, F(type_diff, {"type", "diff"}), \ diff --git a/dbms/src/Storages/Transaction/BackgroundService.cpp b/dbms/src/Storages/Transaction/BackgroundService.cpp index e751575575b..e51abbe94c7 100644 --- a/dbms/src/Storages/Transaction/BackgroundService.cpp +++ b/dbms/src/Storages/Transaction/BackgroundService.cpp @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include #include @@ -36,6 +37,17 @@ BackgroundService::BackgroundService(TMTContext & tmt_) }, false); + kvstore_size_metric_handle = background_pool.addTask( + [this] { + size_t data_size = 0; + tmt.getKVStore()->traverseRegions([&data_size](RegionID, const RegionPtr & region) { + data_size += region->dataSize(); + }); + GET_METRIC(tiflash_kvstore_region_data_memory_size).Set(data_size); + return false; + }, + false); + if (!tmt.isBgFlushDisabled()) { table_flush_handle = background_pool.addTask([this] { @@ -104,6 +116,11 @@ BackgroundService::~BackgroundService() background_pool.removeTask(single_thread_task_handle); single_thread_task_handle = nullptr; } + if (kvstore_size_metric_handle) + { + background_pool.removeTask(kvstore_size_metric_handle); + kvstore_size_metric_handle = nullptr; + } if (table_flush_handle) { background_pool.removeTask(table_flush_handle); diff --git a/dbms/src/Storages/Transaction/BackgroundService.h b/dbms/src/Storages/Transaction/BackgroundService.h index 18a0f04206f..3ac288d5ccc 100644 --- a/dbms/src/Storages/Transaction/BackgroundService.h +++ b/dbms/src/Storages/Transaction/BackgroundService.h @@ -51,6 +51,7 @@ class BackgroundService : boost::noncopyable RegionMap regions_to_flush; BackgroundProcessingPool::TaskHandle single_thread_task_handle; + BackgroundProcessingPool::TaskHandle kvstore_size_metric_handle; BackgroundProcessingPool::TaskHandle table_flush_handle; BackgroundProcessingPool::TaskHandle region_handle; BackgroundProcessingPool::TaskHandle storage_gc_handle; From 3fc42b7cc2d6693ed16b2c00a4e5d51a6e295593 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Tue, 26 Apr 2022 15:21:19 +0800 Subject: [PATCH 02/29] add log for metric --- dbms/src/Storages/Transaction/BackgroundService.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dbms/src/Storages/Transaction/BackgroundService.cpp b/dbms/src/Storages/Transaction/BackgroundService.cpp index e51abbe94c7..5f456a9490d 100644 --- a/dbms/src/Storages/Transaction/BackgroundService.cpp +++ b/dbms/src/Storages/Transaction/BackgroundService.cpp @@ -44,7 +44,8 @@ BackgroundService::BackgroundService(TMTContext & tmt_) data_size += region->dataSize(); }); GET_METRIC(tiflash_kvstore_region_data_memory_size).Set(data_size); - return false; + LOG_FMT_INFO(log, "KVStore region data memory size is {}.", data_size); + return true; }, false); From be1e11ae951e8992abbd700c99a921184c59c79f Mon Sep 17 00:00:00 2001 From: lidezhu Date: Tue, 26 Apr 2022 15:47:38 +0800 Subject: [PATCH 03/29] small fix --- dbms/src/Storages/Transaction/BackgroundService.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/Transaction/BackgroundService.cpp b/dbms/src/Storages/Transaction/BackgroundService.cpp index 5f456a9490d..9e4ad8f3910 100644 --- a/dbms/src/Storages/Transaction/BackgroundService.cpp +++ b/dbms/src/Storages/Transaction/BackgroundService.cpp @@ -45,7 +45,7 @@ BackgroundService::BackgroundService(TMTContext & tmt_) }); GET_METRIC(tiflash_kvstore_region_data_memory_size).Set(data_size); LOG_FMT_INFO(log, "KVStore region data memory size is {}.", data_size); - return true; + return false; }, false); From 487962960e23368f6e4d897ffa45bca7ab2887a4 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Tue, 26 Apr 2022 16:51:08 +0800 Subject: [PATCH 04/29] add more metric --- dbms/src/Common/TiFlashMetrics.h | 1 + dbms/src/Storages/Transaction/BackgroundService.cpp | 13 ++++++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/dbms/src/Common/TiFlashMetrics.h b/dbms/src/Common/TiFlashMetrics.h index a679659cad5..bced3ead0b8 100644 --- a/dbms/src/Common/TiFlashMetrics.h +++ b/dbms/src/Common/TiFlashMetrics.h @@ -81,6 +81,7 @@ namespace DB F(type_run_mpp_task, {{"type", "run_mpp_task"}}, ExpBuckets{0.0005, 2, 30})) \ M(tiflash_coprocessor_response_bytes, "Total bytes of response body", Counter) \ M(tiflash_kvstore_region_data_memory_size, "Total region data memory size of tiflash kvstore", Gauge) \ + M(tiflash_kvstore_data_approx_cache_memory_size, "Total region data memory size of tiflash memory data", Gauge) \ M(tiflash_schema_version, "Current version of tiflash cached schema", Gauge) \ M(tiflash_schema_applying, "Whether the schema is applying or not (holding lock)", Gauge) \ M(tiflash_schema_apply_count, "Total number of each kinds of apply", Counter, F(type_diff, {"type", "diff"}), \ diff --git a/dbms/src/Storages/Transaction/BackgroundService.cpp b/dbms/src/Storages/Transaction/BackgroundService.cpp index 9e4ad8f3910..b644bbe9bb6 100644 --- a/dbms/src/Storages/Transaction/BackgroundService.cpp +++ b/dbms/src/Storages/Transaction/BackgroundService.cpp @@ -39,12 +39,15 @@ BackgroundService::BackgroundService(TMTContext & tmt_) kvstore_size_metric_handle = background_pool.addTask( [this] { - size_t data_size = 0; - tmt.getKVStore()->traverseRegions([&data_size](RegionID, const RegionPtr & region) { - data_size += region->dataSize(); + size_t region_data_size = 0; + size_t memory_cache_size = 0; + tmt.getKVStore()->traverseRegions([®ion_data_size, &memory_cache_size](RegionID, const RegionPtr & region) { + region_data_size += region->dataSize(); + memory_cache_size += region->getApproxMemCacheInfo().second; }); - GET_METRIC(tiflash_kvstore_region_data_memory_size).Set(data_size); - LOG_FMT_INFO(log, "KVStore region data memory size is {}.", data_size); + GET_METRIC(tiflash_kvstore_region_data_memory_size).Set(region_data_size); + GET_METRIC(tiflash_kvstore_data_approx_cache_memory_size).Set(memory_cache_size); + LOG_FMT_INFO(log, "KVStore region data memory size is {}, memory cache size is {}", region_data_size, memory_cache_size); return false; }, false); From 4a28c6479a7e797af96c7ea3b317c4a35e293d5a Mon Sep 17 00:00:00 2001 From: lidezhu Date: Wed, 27 Apr 2022 16:42:22 +0800 Subject: [PATCH 05/29] modify flush threhold --- dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp index 80b1d81f817..3d8073dcf33 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp +++ b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp @@ -1240,7 +1240,7 @@ void DeltaMergeStore::checkSegmentUpdate(const DMContextPtr & dm_context, const bool should_background_flush = (unsaved_rows >= delta_cache_limit_rows || unsaved_bytes >= delta_cache_limit_bytes) // && (delta_rows - delta_last_try_flush_rows >= delta_cache_limit_rows || delta_bytes - delta_last_try_flush_bytes >= delta_cache_limit_bytes); - bool should_foreground_flush = unsaved_rows >= delta_cache_limit_rows * 3 || unsaved_bytes >= delta_cache_limit_bytes * 3; + bool should_foreground_flush = unsaved_rows >= delta_cache_limit_rows * 100 || unsaved_bytes >= delta_cache_limit_bytes * 100; bool should_background_merge_delta = ((delta_check_rows >= delta_limit_rows || delta_check_bytes >= delta_limit_bytes) // && (delta_rows - delta_last_try_merge_delta_rows >= delta_cache_limit_rows From b3b2833a58a75912d48650dfa5930707e2ddab84 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Wed, 27 Apr 2022 22:31:53 +0800 Subject: [PATCH 06/29] avoid flush region in some scenario --- dbms/src/Storages/Transaction/KVStore.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/dbms/src/Storages/Transaction/KVStore.cpp b/dbms/src/Storages/Transaction/KVStore.cpp index 8a7df489f20..cc1d949884e 100644 --- a/dbms/src/Storages/Transaction/KVStore.cpp +++ b/dbms/src/Storages/Transaction/KVStore.cpp @@ -446,14 +446,14 @@ EngineStoreApplyRes KVStore::handleAdminRaftCmd(raft_cmdpb::AdminRequest && requ const auto try_to_flush_region = [&tmt](const RegionPtr & region) { if (tmt.isBgFlushDisabled()) { - try - { - tmt.getRegionTable().tryFlushRegion(region, false); - } - catch (const Exception & e) - { - tryLogCurrentException(__PRETTY_FUNCTION__); - } +// try +// { +// tmt.getRegionTable().tryFlushRegion(region, false); +// } +// catch (const Exception & e) +// { +// tryLogCurrentException(__PRETTY_FUNCTION__); +// } } else { @@ -463,7 +463,7 @@ EngineStoreApplyRes KVStore::handleAdminRaftCmd(raft_cmdpb::AdminRequest && requ }; const auto persist_and_sync = [&](const Region & region) { - tryFlushRegionCacheInStorage(tmt, region, log); +// tryFlushRegionCacheInStorage(tmt, region, log); persistRegion(region, region_task_lock, "admin raft cmd"); }; From 674fcd6b71485ce51ec91a6071b1fe83cbe11397 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Thu, 28 Apr 2022 15:53:35 +0800 Subject: [PATCH 07/29] add raft command size metric --- dbms/src/Common/TiFlashMetrics.h | 1 + dbms/src/Storages/Transaction/KVStore.cpp | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/dbms/src/Common/TiFlashMetrics.h b/dbms/src/Common/TiFlashMetrics.h index bced3ead0b8..79704a91504 100644 --- a/dbms/src/Common/TiFlashMetrics.h +++ b/dbms/src/Common/TiFlashMetrics.h @@ -81,6 +81,7 @@ namespace DB F(type_run_mpp_task, {{"type", "run_mpp_task"}}, ExpBuckets{0.0005, 2, 30})) \ M(tiflash_coprocessor_response_bytes, "Total bytes of response body", Counter) \ M(tiflash_kvstore_region_data_memory_size, "Total region data memory size of tiflash kvstore", Gauge) \ + M(tiflash_kvstore_raft_command_size, "Total region data memory size of tiflash kvstore", Counter) \ M(tiflash_kvstore_data_approx_cache_memory_size, "Total region data memory size of tiflash memory data", Gauge) \ M(tiflash_schema_version, "Current version of tiflash cached schema", Gauge) \ M(tiflash_schema_applying, "Whether the schema is applying or not (holding lock)", Gauge) \ diff --git a/dbms/src/Storages/Transaction/KVStore.cpp b/dbms/src/Storages/Transaction/KVStore.cpp index cc1d949884e..8a513378343 100644 --- a/dbms/src/Storages/Transaction/KVStore.cpp +++ b/dbms/src/Storages/Transaction/KVStore.cpp @@ -243,6 +243,8 @@ EngineStoreApplyRes KVStore::handleWriteRaftCmd( cmd_types.reserve(request.requests_size()); cmd_cf.reserve(request.requests_size()); + size_t cmd_size = 0; + for (const auto & req : request.requests()) { auto type = req.cmd_type(); @@ -254,17 +256,23 @@ EngineStoreApplyRes KVStore::handleWriteRaftCmd( vals.push_back({req.put().value().data(), req.put().value().size()}); cmd_types.push_back(WriteCmdType::Put); cmd_cf.push_back(NameToCF(req.put().cf())); + cmd_size += req.put().key().size() + 16; + cmd_size += req.put().value().size() + 16; break; case raft_cmdpb::CmdType::Delete: keys.push_back({req.delete_().key().data(), req.delete_().key().size()}); vals.push_back({nullptr, 0}); cmd_types.push_back(WriteCmdType::Del); cmd_cf.push_back(NameToCF(req.delete_().cf())); + cmd_size += req.put().key().size() + 16; break; default: throw Exception(fmt::format("Unsupport raft cmd {}", raft_cmdpb::CmdType_Name(type)), ErrorCodes::LOGICAL_ERROR); } } + cmd_size += cmd_types.size() * sizeof(WriteCmdType); + cmd_size += cmd_cf.size() * sizeof(ColumnFamilyType); + GET_METRIC(tiflash_kvstore_raft_command_size).Increment(cmd_size); return handleWriteRaftCmd( WriteCmdsView{.keys = keys.data(), .vals = vals.data(), .cmd_types = cmd_types.data(), .cmd_cf = cmd_cf.data(), .len = keys.size()}, region_id, From 8a4f221cbe4bf9cbbd507ff5b7eff44c6a0b42df Mon Sep 17 00:00:00 2001 From: lidezhu Date: Thu, 28 Apr 2022 16:15:11 +0800 Subject: [PATCH 08/29] fix raft command size metric --- dbms/src/Storages/Transaction/KVStore.cpp | 29 ++++++++++++++++------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/dbms/src/Storages/Transaction/KVStore.cpp b/dbms/src/Storages/Transaction/KVStore.cpp index 8a513378343..af63c614a3c 100644 --- a/dbms/src/Storages/Transaction/KVStore.cpp +++ b/dbms/src/Storages/Transaction/KVStore.cpp @@ -243,8 +243,6 @@ EngineStoreApplyRes KVStore::handleWriteRaftCmd( cmd_types.reserve(request.requests_size()); cmd_cf.reserve(request.requests_size()); - size_t cmd_size = 0; - for (const auto & req : request.requests()) { auto type = req.cmd_type(); @@ -256,23 +254,17 @@ EngineStoreApplyRes KVStore::handleWriteRaftCmd( vals.push_back({req.put().value().data(), req.put().value().size()}); cmd_types.push_back(WriteCmdType::Put); cmd_cf.push_back(NameToCF(req.put().cf())); - cmd_size += req.put().key().size() + 16; - cmd_size += req.put().value().size() + 16; break; case raft_cmdpb::CmdType::Delete: keys.push_back({req.delete_().key().data(), req.delete_().key().size()}); vals.push_back({nullptr, 0}); cmd_types.push_back(WriteCmdType::Del); cmd_cf.push_back(NameToCF(req.delete_().cf())); - cmd_size += req.put().key().size() + 16; break; default: throw Exception(fmt::format("Unsupport raft cmd {}", raft_cmdpb::CmdType_Name(type)), ErrorCodes::LOGICAL_ERROR); } } - cmd_size += cmd_types.size() * sizeof(WriteCmdType); - cmd_size += cmd_cf.size() * sizeof(ColumnFamilyType); - GET_METRIC(tiflash_kvstore_raft_command_size).Increment(cmd_size); return handleWriteRaftCmd( WriteCmdsView{.keys = keys.data(), .vals = vals.data(), .cmd_types = cmd_types.data(), .cmd_cf = cmd_cf.data(), .len = keys.size()}, region_id, @@ -283,6 +275,27 @@ EngineStoreApplyRes KVStore::handleWriteRaftCmd( EngineStoreApplyRes KVStore::handleWriteRaftCmd(const WriteCmdsView & cmds, UInt64 region_id, UInt64 index, UInt64 term, TMTContext & tmt) { + size_t cmd_size = 0; + cmd_size += sizeof(WriteCmdsView); + for (size_t i = 0; i < cmds.len; i++) + { + auto type = cmds.cmd_types[i]; + auto cf = cmds.cmd_cf[i]; + switch (type) + { + case WriteCmdType::Put: + { + cmd_size += cmds.keys[i].len + sizeof(BaseBuffView); + cmd_size += cmds.vals[i].len + sizeof(BaseBuffView); + break; + } + case WriteCmdType::Del: + { + cmd_size += cmds.keys[i].len + sizeof(BaseBuffView); + break; + } + } + GET_METRIC(tiflash_kvstore_raft_command_size).Increment(cmd_size); auto region_persist_lock = region_manager.genRegionTaskLock(region_id); const RegionPtr region = getRegion(region_id); From 185f490f3e48864e6bef6c910b2021acbd1e0c2c Mon Sep 17 00:00:00 2001 From: lidezhu Date: Thu, 28 Apr 2022 16:17:59 +0800 Subject: [PATCH 09/29] fix build --- dbms/src/Storages/Transaction/KVStore.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/dbms/src/Storages/Transaction/KVStore.cpp b/dbms/src/Storages/Transaction/KVStore.cpp index af63c614a3c..bbdbdc07375 100644 --- a/dbms/src/Storages/Transaction/KVStore.cpp +++ b/dbms/src/Storages/Transaction/KVStore.cpp @@ -294,6 +294,7 @@ EngineStoreApplyRes KVStore::handleWriteRaftCmd(const WriteCmdsView & cmds, UInt cmd_size += cmds.keys[i].len + sizeof(BaseBuffView); break; } + } } GET_METRIC(tiflash_kvstore_raft_command_size).Increment(cmd_size); auto region_persist_lock = region_manager.genRegionTaskLock(region_id); From c662515a7eac75363b46561925e3c2c2e398ae24 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Thu, 28 Apr 2022 16:22:34 +0800 Subject: [PATCH 10/29] fix build --- dbms/src/Storages/Transaction/KVStore.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/dbms/src/Storages/Transaction/KVStore.cpp b/dbms/src/Storages/Transaction/KVStore.cpp index bbdbdc07375..ae550b8dbdc 100644 --- a/dbms/src/Storages/Transaction/KVStore.cpp +++ b/dbms/src/Storages/Transaction/KVStore.cpp @@ -280,7 +280,6 @@ EngineStoreApplyRes KVStore::handleWriteRaftCmd(const WriteCmdsView & cmds, UInt for (size_t i = 0; i < cmds.len; i++) { auto type = cmds.cmd_types[i]; - auto cf = cmds.cmd_cf[i]; switch (type) { case WriteCmdType::Put: From 620117e5f5c69df19fa2490a70742f291dd0e528 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Thu, 28 Apr 2022 17:02:16 +0800 Subject: [PATCH 11/29] don't write any data --- dbms/src/Common/TiFlashMetrics.h | 2 +- dbms/src/Storages/Transaction/KVStore.cpp | 24 +++++++++++++---------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/dbms/src/Common/TiFlashMetrics.h b/dbms/src/Common/TiFlashMetrics.h index 79704a91504..bd875db09a6 100644 --- a/dbms/src/Common/TiFlashMetrics.h +++ b/dbms/src/Common/TiFlashMetrics.h @@ -81,7 +81,7 @@ namespace DB F(type_run_mpp_task, {{"type", "run_mpp_task"}}, ExpBuckets{0.0005, 2, 30})) \ M(tiflash_coprocessor_response_bytes, "Total bytes of response body", Counter) \ M(tiflash_kvstore_region_data_memory_size, "Total region data memory size of tiflash kvstore", Gauge) \ - M(tiflash_kvstore_raft_command_size, "Total region data memory size of tiflash kvstore", Counter) \ + M(tiflash_kvstore_raft_command_size, "Total region data memory size of tiflash kvstore", Gauge) \ M(tiflash_kvstore_data_approx_cache_memory_size, "Total region data memory size of tiflash memory data", Gauge) \ M(tiflash_schema_version, "Current version of tiflash cached schema", Gauge) \ M(tiflash_schema_applying, "Whether the schema is applying or not (holding lock)", Gauge) \ diff --git a/dbms/src/Storages/Transaction/KVStore.cpp b/dbms/src/Storages/Transaction/KVStore.cpp index ae550b8dbdc..933ae02c598 100644 --- a/dbms/src/Storages/Transaction/KVStore.cpp +++ b/dbms/src/Storages/Transaction/KVStore.cpp @@ -296,16 +296,20 @@ EngineStoreApplyRes KVStore::handleWriteRaftCmd(const WriteCmdsView & cmds, UInt } } GET_METRIC(tiflash_kvstore_raft_command_size).Increment(cmd_size); - auto region_persist_lock = region_manager.genRegionTaskLock(region_id); - - const RegionPtr region = getRegion(region_id); - if (region == nullptr) - { - return EngineStoreApplyRes::NotFound; - } - - auto res = region->handleWriteRaftCmd(cmds, index, term, tmt); - return res; + SCOPE_EXIT({ + GET_METRIC(tiflash_kvstore_raft_command_size).Decrement(cmd_size); + }); + return EngineStoreApplyRes::None; +// auto region_persist_lock = region_manager.genRegionTaskLock(region_id); +// +// const RegionPtr region = getRegion(region_id); +// if (region == nullptr) +// { +// return EngineStoreApplyRes::NotFound; +// } +// +// auto res = region->handleWriteRaftCmd(cmds, index, term, tmt); +// return res; } void KVStore::handleDestroy(UInt64 region_id, TMTContext & tmt) From 59f1b9fdcbc234733315ac3e20761accc548e8a4 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Thu, 28 Apr 2022 17:13:08 +0800 Subject: [PATCH 12/29] don't write data --- dbms/src/Storages/Transaction/KVStore.cpp | 21 ++++++++++---------- dbms/src/Storages/Transaction/Region.cpp | 24 +++++++++++------------ 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/dbms/src/Storages/Transaction/KVStore.cpp b/dbms/src/Storages/Transaction/KVStore.cpp index 933ae02c598..df0e7bd0ed2 100644 --- a/dbms/src/Storages/Transaction/KVStore.cpp +++ b/dbms/src/Storages/Transaction/KVStore.cpp @@ -299,17 +299,16 @@ EngineStoreApplyRes KVStore::handleWriteRaftCmd(const WriteCmdsView & cmds, UInt SCOPE_EXIT({ GET_METRIC(tiflash_kvstore_raft_command_size).Decrement(cmd_size); }); - return EngineStoreApplyRes::None; -// auto region_persist_lock = region_manager.genRegionTaskLock(region_id); -// -// const RegionPtr region = getRegion(region_id); -// if (region == nullptr) -// { -// return EngineStoreApplyRes::NotFound; -// } -// -// auto res = region->handleWriteRaftCmd(cmds, index, term, tmt); -// return res; + auto region_persist_lock = region_manager.genRegionTaskLock(region_id); + + const RegionPtr region = getRegion(region_id); + if (region == nullptr) + { + return EngineStoreApplyRes::NotFound; + } + + auto res = region->handleWriteRaftCmd(cmds, index, term, tmt); + return res; } void KVStore::handleDestroy(UInt64 region_id, TMTContext & tmt) diff --git a/dbms/src/Storages/Transaction/Region.cpp b/dbms/src/Storages/Transaction/Region.cpp index 4ee2bb7963a..8b34e6c604c 100644 --- a/dbms/src/Storages/Transaction/Region.cpp +++ b/dbms/src/Storages/Transaction/Region.cpp @@ -702,18 +702,18 @@ EngineStoreApplyRes Region::handleWriteRaftCmd(const WriteCmdsView & cmds, UInt6 { std::unique_lock lock(mutex); - handle_write_cmd_func(); - - // If transfer-leader happened during ingest-sst, there might be illegal data. - if (0 != cmds.len) - { - if (tmt.isBgFlushDisabled()) - { - /// Flush data right after they are committed. - RegionDataReadInfoList data_list_to_remove; - RegionTable::writeBlockByRegion(context, shared_from_this(), data_list_to_remove, log, false); - } - } +// handle_write_cmd_func(); +// +// // If transfer-leader happened during ingest-sst, there might be illegal data. +// if (0 != cmds.len) +// { +// if (tmt.isBgFlushDisabled()) +// { +// /// Flush data right after they are committed. +// RegionDataReadInfoList data_list_to_remove; +// RegionTable::writeBlockByRegion(context, shared_from_this(), data_list_to_remove, log, false); +// } +// } meta.setApplied(index, term); } From ff88f369f94a562c0b690c69087238049b07cbc9 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Thu, 28 Apr 2022 17:32:01 +0800 Subject: [PATCH 13/29] avoid write to storage --- .../Storages/Transaction/PartitionStreams.cpp | 2 +- dbms/src/Storages/Transaction/Region.cpp | 24 +++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/dbms/src/Storages/Transaction/PartitionStreams.cpp b/dbms/src/Storages/Transaction/PartitionStreams.cpp index 9142aad5358..661245d5ba8 100644 --- a/dbms/src/Storages/Transaction/PartitionStreams.cpp +++ b/dbms/src/Storages/Transaction/PartitionStreams.cpp @@ -345,7 +345,7 @@ void RegionTable::writeBlockByRegion( return; reportUpstreamLatency(*data_list_read); - writeRegionDataToStorage(context, region, *data_list_read, log); +// writeRegionDataToStorage(context, region, *data_list_read, log); RemoveRegionCommitCache(region, *data_list_read, lock_region); diff --git a/dbms/src/Storages/Transaction/Region.cpp b/dbms/src/Storages/Transaction/Region.cpp index 8b34e6c604c..4ee2bb7963a 100644 --- a/dbms/src/Storages/Transaction/Region.cpp +++ b/dbms/src/Storages/Transaction/Region.cpp @@ -702,18 +702,18 @@ EngineStoreApplyRes Region::handleWriteRaftCmd(const WriteCmdsView & cmds, UInt6 { std::unique_lock lock(mutex); -// handle_write_cmd_func(); -// -// // If transfer-leader happened during ingest-sst, there might be illegal data. -// if (0 != cmds.len) -// { -// if (tmt.isBgFlushDisabled()) -// { -// /// Flush data right after they are committed. -// RegionDataReadInfoList data_list_to_remove; -// RegionTable::writeBlockByRegion(context, shared_from_this(), data_list_to_remove, log, false); -// } -// } + handle_write_cmd_func(); + + // If transfer-leader happened during ingest-sst, there might be illegal data. + if (0 != cmds.len) + { + if (tmt.isBgFlushDisabled()) + { + /// Flush data right after they are committed. + RegionDataReadInfoList data_list_to_remove; + RegionTable::writeBlockByRegion(context, shared_from_this(), data_list_to_remove, log, false); + } + } meta.setApplied(index, term); } From 5a195f327b58cc3397d06b7644767b897537cd57 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Thu, 28 Apr 2022 22:52:05 +0800 Subject: [PATCH 14/29] try to calculate all memory --- dbms/src/Common/TiFlashMetrics.h | 1 + .../Transaction/BackgroundService.cpp | 9 ++- dbms/src/Storages/Transaction/KVStore.cpp | 18 ++--- .../Storages/Transaction/PartitionStreams.cpp | 2 +- dbms/src/Storages/Transaction/Region.cpp | 5 ++ dbms/src/Storages/Transaction/Region.h | 1 + .../Storages/Transaction/RegionCFDataBase.cpp | 10 +-- .../Storages/Transaction/RegionCFDataBase.h | 4 +- .../Storages/Transaction/RegionCFDataTrait.h | 44 +++++++++-- dbms/src/Storages/Transaction/RegionData.cpp | 77 ++++++++++++++++++- dbms/src/Storages/Transaction/RegionData.h | 7 ++ 11 files changed, 150 insertions(+), 28 deletions(-) diff --git a/dbms/src/Common/TiFlashMetrics.h b/dbms/src/Common/TiFlashMetrics.h index bd875db09a6..52518084e2e 100644 --- a/dbms/src/Common/TiFlashMetrics.h +++ b/dbms/src/Common/TiFlashMetrics.h @@ -81,6 +81,7 @@ namespace DB F(type_run_mpp_task, {{"type", "run_mpp_task"}}, ExpBuckets{0.0005, 2, 30})) \ M(tiflash_coprocessor_response_bytes, "Total bytes of response body", Counter) \ M(tiflash_kvstore_region_data_memory_size, "Total region data memory size of tiflash kvstore", Gauge) \ + M(tiflash_kvstore_region_data_consume_memory_size, "Total region data memory size of tiflash kvstore", Gauge) \ M(tiflash_kvstore_raft_command_size, "Total region data memory size of tiflash kvstore", Gauge) \ M(tiflash_kvstore_data_approx_cache_memory_size, "Total region data memory size of tiflash memory data", Gauge) \ M(tiflash_schema_version, "Current version of tiflash cached schema", Gauge) \ diff --git a/dbms/src/Storages/Transaction/BackgroundService.cpp b/dbms/src/Storages/Transaction/BackgroundService.cpp index b644bbe9bb6..2e13fdd90bb 100644 --- a/dbms/src/Storages/Transaction/BackgroundService.cpp +++ b/dbms/src/Storages/Transaction/BackgroundService.cpp @@ -40,14 +40,19 @@ BackgroundService::BackgroundService(TMTContext & tmt_) kvstore_size_metric_handle = background_pool.addTask( [this] { size_t region_data_size = 0; + size_t region_data_memory_size = 0; size_t memory_cache_size = 0; - tmt.getKVStore()->traverseRegions([®ion_data_size, &memory_cache_size](RegionID, const RegionPtr & region) { + size_t region_count = 0; + tmt.getKVStore()->traverseRegions([®ion_data_size, ®ion_data_memory_size, &memory_cache_size, ®ion_count](RegionID, const RegionPtr & region) { region_data_size += region->dataSize(); + region_data_memory_size += region->memorySize(); memory_cache_size += region->getApproxMemCacheInfo().second; + region_count += 1; }); GET_METRIC(tiflash_kvstore_region_data_memory_size).Set(region_data_size); + GET_METRIC(tiflash_kvstore_region_data_consume_memory_size).Set(region_data_memory_size); GET_METRIC(tiflash_kvstore_data_approx_cache_memory_size).Set(memory_cache_size); - LOG_FMT_INFO(log, "KVStore region data memory size is {}, memory cache size is {}", region_data_size, memory_cache_size); + LOG_FMT_INFO(log, "KVStore region data memory size is {}, memory cache size is {}, region count {}", region_data_size, memory_cache_size, region_count); return false; }, false); diff --git a/dbms/src/Storages/Transaction/KVStore.cpp b/dbms/src/Storages/Transaction/KVStore.cpp index df0e7bd0ed2..065c4f1b140 100644 --- a/dbms/src/Storages/Transaction/KVStore.cpp +++ b/dbms/src/Storages/Transaction/KVStore.cpp @@ -470,14 +470,14 @@ EngineStoreApplyRes KVStore::handleAdminRaftCmd(raft_cmdpb::AdminRequest && requ const auto try_to_flush_region = [&tmt](const RegionPtr & region) { if (tmt.isBgFlushDisabled()) { -// try -// { -// tmt.getRegionTable().tryFlushRegion(region, false); -// } -// catch (const Exception & e) -// { -// tryLogCurrentException(__PRETTY_FUNCTION__); -// } + try + { + tmt.getRegionTable().tryFlushRegion(region, false); + } + catch (const Exception & e) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } } else { @@ -487,7 +487,7 @@ EngineStoreApplyRes KVStore::handleAdminRaftCmd(raft_cmdpb::AdminRequest && requ }; const auto persist_and_sync = [&](const Region & region) { -// tryFlushRegionCacheInStorage(tmt, region, log); + tryFlushRegionCacheInStorage(tmt, region, log); persistRegion(region, region_task_lock, "admin raft cmd"); }; diff --git a/dbms/src/Storages/Transaction/PartitionStreams.cpp b/dbms/src/Storages/Transaction/PartitionStreams.cpp index 661245d5ba8..9142aad5358 100644 --- a/dbms/src/Storages/Transaction/PartitionStreams.cpp +++ b/dbms/src/Storages/Transaction/PartitionStreams.cpp @@ -345,7 +345,7 @@ void RegionTable::writeBlockByRegion( return; reportUpstreamLatency(*data_list_read); -// writeRegionDataToStorage(context, region, *data_list_read, log); + writeRegionDataToStorage(context, region, *data_list_read, log); RemoveRegionCommitCache(region, *data_list_read, lock_region); diff --git a/dbms/src/Storages/Transaction/Region.cpp b/dbms/src/Storages/Transaction/Region.cpp index 4ee2bb7963a..5603c83ac08 100644 --- a/dbms/src/Storages/Transaction/Region.cpp +++ b/dbms/src/Storages/Transaction/Region.cpp @@ -434,6 +434,11 @@ size_t Region::dataSize() const return data.dataSize(); } +size_t Region::memorySize() const +{ + return data.memorySize(); +} + size_t Region::writeCFCount() const { std::shared_lock lock(mutex); diff --git a/dbms/src/Storages/Transaction/Region.h b/dbms/src/Storages/Transaction/Region.h index b31ae0cdc49..c886bdc2522 100644 --- a/dbms/src/Storages/Transaction/Region.h +++ b/dbms/src/Storages/Transaction/Region.h @@ -147,6 +147,7 @@ class Region : public std::enable_shared_from_this void setStateApplying(); size_t dataSize() const; + size_t memorySize() const; size_t writeCFCount() const; std::string dataInfo() const; diff --git a/dbms/src/Storages/Transaction/RegionCFDataBase.cpp b/dbms/src/Storages/Transaction/RegionCFDataBase.cpp index dacb982692c..81cb42deaf0 100644 --- a/dbms/src/Storages/Transaction/RegionCFDataBase.cpp +++ b/dbms/src/Storages/Transaction/RegionCFDataBase.cpp @@ -43,11 +43,11 @@ template RegionDataRes RegionCFDataBase::insert(TiKVKey && key, TiKVValue && value) { const auto & raw_key = RecordKVFormat::decodeTiKVKey(key); - auto kv_pair = Trait::genKVPair(std::move(key), raw_key, std::move(value)); + auto [kv_pair, memory_size] = Trait::genKVPair(std::move(key), raw_key, std::move(value)); if (!kv_pair) - return 0; + return std::pair(0, 0); - return insert(std::move(*kv_pair)); + return std::make_pair(insert(std::move(*kv_pair)), memory_size); } template <> @@ -56,11 +56,11 @@ RegionDataRes RegionCFDataBase::insert(TiKVKey && key, Ti Pair kv_pair = RegionLockCFDataTrait::genKVPair(std::move(key), std::move(value)); // according to the process of pessimistic lock, just overwrite. data.insert_or_assign(std::move(kv_pair.first), std::move(kv_pair.second)); - return 0; + return std::pair(0, 0); } template -RegionDataRes RegionCFDataBase::insert(std::pair && kv_pair) +size_t RegionCFDataBase::insert(std::pair && kv_pair) { auto & map = data; auto [it, ok] = map.emplace(std::move(kv_pair)); diff --git a/dbms/src/Storages/Transaction/RegionCFDataBase.h b/dbms/src/Storages/Transaction/RegionCFDataBase.h index 09a43d5a288..1705b4d4eb5 100644 --- a/dbms/src/Storages/Transaction/RegionCFDataBase.h +++ b/dbms/src/Storages/Transaction/RegionCFDataBase.h @@ -23,7 +23,7 @@ namespace DB struct TiKVRangeKey; using RegionRange = std::pair; -using RegionDataRes = size_t; +using RegionDataRes = std::pair; template struct RegionCFDataBase @@ -70,7 +70,7 @@ struct RegionCFDataBase private: static bool shouldIgnoreRemove(const Value & value); - RegionDataRes insert(std::pair && kv_pair); + size_t insert(std::pair && kv_pair); private: Data data; diff --git a/dbms/src/Storages/Transaction/RegionCFDataTrait.h b/dbms/src/Storages/Transaction/RegionCFDataTrait.h index bc85adcb21e..6f3b07a8c2c 100644 --- a/dbms/src/Storages/Transaction/RegionCFDataTrait.h +++ b/dbms/src/Storages/Transaction/RegionCFDataTrait.h @@ -38,17 +38,35 @@ struct RegionWriteCFDataTrait using Value = std::tuple, std::shared_ptr, DecodedWriteCFValue>; using Map = std::map; - static std::optional genKVPair(TiKVKey && key, const DecodedTiKVKey & raw_key, TiKVValue && value) + static std::pair, size_t> genKVPair(TiKVKey && key, const DecodedTiKVKey & raw_key, TiKVValue && value) { auto decoded_val = RecordKVFormat::decodeWriteCfValue(value); if (!decoded_val) - return std::nullopt; + return std::pair(std::nullopt, 0); RawTiDBPK tidb_pk = RecordKVFormat::getRawTiDBPK(raw_key); Timestamp ts = RecordKVFormat::getTs(key); - return Map::value_type(Key(std::move(tidb_pk), ts), + + size_t memory_size = 0; + memory_size += key.capacity(); + memory_size += raw_key.capacity(); + memory_size += value.capacity(); + memory_size += sizeof(decoded_val); + memory_size += sizeof(RecordKVFormat::InnerDecodedWriteCFValue); + memory_size += sizeof(decoded_val->short_value); + memory_size += decoded_val->short_value->capacity(); + memory_size += sizeof(tidb_pk); + memory_size += tidb_pk->capacity(); + memory_size += sizeof(ts); + memory_size += sizeof(Key); + memory_size += sizeof(Value); + memory_size += sizeof(std::shared_ptr); + memory_size += sizeof(std::shared_ptr); + memory_size += sizeof(Map::value_type); + + return std::make_pair(Map::value_type(Key(std::move(tidb_pk), ts), Value(std::make_shared(std::move(key)), std::make_shared(std::move(value)), - std::move(*decoded_val))); + std::move(*decoded_val))), memory_size); } static const std::shared_ptr & getRecordRawValuePtr(const Value & value) { return std::get<2>(value).short_value; } @@ -63,12 +81,24 @@ struct RegionDefaultCFDataTrait using Value = std::tuple, std::shared_ptr>; using Map = std::map; - static std::optional genKVPair(TiKVKey && key, const DecodedTiKVKey & raw_key, TiKVValue && value) + static std::pair, size_t> genKVPair(TiKVKey && key, const DecodedTiKVKey & raw_key, TiKVValue && value) { RawTiDBPK tidb_pk = RecordKVFormat::getRawTiDBPK(raw_key); Timestamp ts = RecordKVFormat::getTs(key); - return Map::value_type(Key(std::move(tidb_pk), ts), - Value(std::make_shared(std::move(key)), std::make_shared(std::move(value)))); + size_t memory_size = 0; + memory_size += key.capacity(); + memory_size += raw_key.capacity(); + memory_size += value.capacity(); + memory_size += sizeof(tidb_pk); + memory_size += tidb_pk->capacity(); + memory_size += sizeof(ts); + memory_size += sizeof(Key); + memory_size += sizeof(Value); + memory_size += sizeof(std::shared_ptr); + memory_size += sizeof(std::shared_ptr); + memory_size += sizeof(Map::value_type); + return std::make_pair(Map::value_type(Key(std::move(tidb_pk), ts), + Value(std::make_shared(std::move(key)), std::make_shared(std::move(value)))), memory_size); } static std::shared_ptr getTiKVValue(const Map::const_iterator & it) { return std::get<1>(it->second); } diff --git a/dbms/src/Storages/Transaction/RegionData.cpp b/dbms/src/Storages/Transaction/RegionData.cpp index 26bd2e93fdf..2581ae7ae3e 100644 --- a/dbms/src/Storages/Transaction/RegionData.cpp +++ b/dbms/src/Storages/Transaction/RegionData.cpp @@ -17,6 +17,7 @@ #include #include #include +#include namespace DB { @@ -38,12 +39,20 @@ void RegionData::insert(ColumnFamilyType cf, TiKVKey && key, TiKVValue && value) { case ColumnFamilyType::Write: { - cf_data_size += write_cf.insert(std::move(key), std::move(value)); + auto size_key = TiKVKey::copyFrom(key); + auto [data_size, memory_size] = write_cf.insert(std::move(key), std::move(value)); + cf_data_size += data_size; + total_memory_size += memory_size; + write_cf_memory_size.emplace(size_key, memory_size); return; } case ColumnFamilyType::Default: { - cf_data_size += default_cf.insert(std::move(key), std::move(value)); + auto size_key = TiKVKey::copyFrom(key); + auto [data_size, memory_size] = default_cf.insert(std::move(key), std::move(value)); + cf_data_size += data_size; + total_memory_size += memory_size; + default_cf_memory_size.emplace(size_key, memory_size); return; } case ColumnFamilyType::Lock: @@ -65,6 +74,11 @@ void RegionData::remove(ColumnFamilyType cf, const TiKVKey & key) Timestamp ts = RecordKVFormat::getTs(key); // removed by gc, may not exist. cf_data_size -= write_cf.remove(RegionWriteCFData::Key{pk, ts}, true); + if (write_cf_memory_size.find(key) != write_cf_memory_size.end()) + { + total_memory_size -= write_cf_memory_size[key]; + write_cf_memory_size.erase(key); + } return; } case ColumnFamilyType::Default: @@ -74,6 +88,11 @@ void RegionData::remove(ColumnFamilyType cf, const TiKVKey & key) Timestamp ts = RecordKVFormat::getTs(key); // removed by gc, may not exist. cf_data_size -= default_cf.remove(RegionDefaultCFData::Key{pk, ts}, true); + if (default_cf_memory_size.find(key) != default_cf_memory_size.end()) + { + total_memory_size -= default_cf_memory_size[key]; + default_cf_memory_size.erase(key); + } return; } case ColumnFamilyType::Lock: @@ -100,11 +119,15 @@ RegionData::WriteCFIter RegionData::removeDataByWriteIt(const WriteCFIter & writ if (auto data_it = map.find({pk, decoded_val.prewrite_ts}); data_it != map.end()) { cf_data_size -= RegionDefaultCFData::calcTiKVKeyValueSize(data_it->second); + total_memory_size -= default_cf_memory_size[*key]; + default_cf_memory_size.erase(*key); map.erase(data_it); } } cf_data_size -= RegionWriteCFData::calcTiKVKeyValueSize(write_it->second); + total_memory_size -= write_cf_memory_size[*key]; + write_cf_memory_size.erase(*key); return write_cf.getDataMut().erase(write_it); } @@ -161,6 +184,7 @@ DecodedLockCFValuePtr RegionData::getLockInfo(const RegionLockReadQuery & query) return nullptr; } +using RegionRange = std::pair; void RegionData::splitInto(const RegionRange & range, RegionData & new_region_data) { size_t size_changed = 0; @@ -169,6 +193,35 @@ void RegionData::splitInto(const RegionRange & range, RegionData & new_region_da size_changed += lock_cf.splitInto(range, new_region_data.lock_cf); cf_data_size -= size_changed; new_region_data.cf_data_size += size_changed; + + const auto & [start_key, end_key] = range; + size_t memory_size_changed = 0; + for (auto iter = default_cf_memory_size.begin(); iter != default_cf_memory_size.end();) + { + auto & key = iter->first; + if (start_key.compare(key) <= 0 && end_key.compare(key) > 0) + { + memory_size_changed += iter->second; + new_region_data.default_cf_memory_size.emplace(iter->first, iter->second); + iter = default_cf_memory_size.erase(iter); + } + else + ++iter; + } + for (auto iter = write_cf_memory_size.begin(); iter != write_cf_memory_size.end();) + { + auto & key = iter->first; + if (start_key.compare(key) <= 0 && end_key.compare(key) > 0) + { + memory_size_changed += iter->second; + new_region_data.write_cf_memory_size.emplace(iter->first, iter->second); + iter = write_cf_memory_size.erase(iter); + } + else + ++iter; + } + total_memory_size -= memory_size_changed; + new_region_data.total_memory_size = memory_size_changed; } void RegionData::mergeFrom(const RegionData & ori_region_data) @@ -178,6 +231,16 @@ void RegionData::mergeFrom(const RegionData & ori_region_data) size_changed += write_cf.mergeFrom(ori_region_data.write_cf); size_changed += lock_cf.mergeFrom(ori_region_data.lock_cf); cf_data_size += size_changed; + for (auto iter = ori_region_data.default_cf_memory_size.begin(); iter != ori_region_data.default_cf_memory_size.end(); iter++) + { + default_cf_memory_size.emplace(iter->first, iter->second); + total_memory_size += iter->second; + } + for (auto iter = ori_region_data.write_cf_memory_size.begin(); iter != ori_region_data.write_cf_memory_size.end();) + { + write_cf_memory_size.emplace(iter->first, iter->second); + total_memory_size += iter->second; + } } size_t RegionData::dataSize() const @@ -185,13 +248,21 @@ size_t RegionData::dataSize() const return cf_data_size; } +size_t RegionData::memorySize() const +{ + return total_memory_size; +} + void RegionData::assignRegionData(RegionData && new_region_data) { default_cf = std::move(new_region_data.default_cf); write_cf = std::move(new_region_data.write_cf); lock_cf = std::move(new_region_data.lock_cf); + default_cf_memory_size = std::move(new_region_data.default_cf_memory_size); + write_cf_memory_size = std::move(new_region_data.write_cf_memory_size); cf_data_size = new_region_data.cf_data_size.load(); + total_memory_size = new_region_data.total_memory_size.load(); } size_t RegionData::serialize(WriteBuffer & buf) const @@ -247,6 +318,7 @@ RegionData::RegionData(RegionData && data) , default_cf(std::move(data.default_cf)) , lock_cf(std::move(data.lock_cf)) , cf_data_size(data.cf_data_size.load()) + , total_memory_size(data.total_memory_size.load()) {} RegionData & RegionData::operator=(RegionData && rhs) @@ -255,6 +327,7 @@ RegionData & RegionData::operator=(RegionData && rhs) default_cf = std::move(rhs.default_cf); lock_cf = std::move(rhs.lock_cf); cf_data_size = rhs.cf_data_size.load(); + total_memory_size = rhs.total_memory_size.load(); return *this; } diff --git a/dbms/src/Storages/Transaction/RegionData.h b/dbms/src/Storages/Transaction/RegionData.h index d1ae0cf7f9c..fe339df88c7 100644 --- a/dbms/src/Storages/Transaction/RegionData.h +++ b/dbms/src/Storages/Transaction/RegionData.h @@ -51,6 +51,8 @@ class RegionData size_t dataSize() const; + size_t memorySize() const; + void assignRegionData(RegionData && new_region_data); size_t serialize(WriteBuffer & buf) const; @@ -81,8 +83,13 @@ class RegionData RegionDefaultCFData default_cf; RegionLockCFData lock_cf; + std::map write_cf_memory_size; + std::map default_cf_memory_size; + // Size of data cf & write cf, without lock cf. std::atomic cf_data_size = 0; + + std::atomic total_memory_size = 0; }; } // namespace DB From 804747b3b51891ad44787e2aa04465d18c67fc7f Mon Sep 17 00:00:00 2001 From: lidezhu Date: Thu, 28 Apr 2022 22:56:56 +0800 Subject: [PATCH 15/29] fix build --- dbms/src/Storages/Transaction/RegionCFDataBase.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dbms/src/Storages/Transaction/RegionCFDataBase.cpp b/dbms/src/Storages/Transaction/RegionCFDataBase.cpp index 81cb42deaf0..d6826010b84 100644 --- a/dbms/src/Storages/Transaction/RegionCFDataBase.cpp +++ b/dbms/src/Storages/Transaction/RegionCFDataBase.cpp @@ -241,7 +241,9 @@ size_t RegionCFDataBase::deserialize(ReadBuffer & buf, RegionCFDataBase & { auto key = TiKVKey::deserialize(buf); auto value = TiKVValue::deserialize(buf); - cf_data_size += new_region_data.insert(std::move(key), std::move(value)); + auto [data_size, memory_size] = new_region_data.insert(std::move(key), std::move(value)); + cf_data_size += data_size; + std::ignore = memory_size; } return cf_data_size; } From a4056c8d0ad5bfc0bfe05a1a41a95b55f0452c49 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Thu, 28 Apr 2022 23:16:55 +0800 Subject: [PATCH 16/29] fix build --- dbms/src/Storages/Transaction/RegionData.cpp | 28 ++++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/dbms/src/Storages/Transaction/RegionData.cpp b/dbms/src/Storages/Transaction/RegionData.cpp index 2581ae7ae3e..5cb19345f82 100644 --- a/dbms/src/Storages/Transaction/RegionData.cpp +++ b/dbms/src/Storages/Transaction/RegionData.cpp @@ -39,20 +39,20 @@ void RegionData::insert(ColumnFamilyType cf, TiKVKey && key, TiKVValue && value) { case ColumnFamilyType::Write: { - auto size_key = TiKVKey::copyFrom(key); + TiKVKey size_key = TiKVKey::copyFrom(key); auto [data_size, memory_size] = write_cf.insert(std::move(key), std::move(value)); cf_data_size += data_size; total_memory_size += memory_size; - write_cf_memory_size.emplace(size_key, memory_size); + write_cf_memory_size.emplace(std::move(size_key), memory_size); return; } case ColumnFamilyType::Default: { - auto size_key = TiKVKey::copyFrom(key); + TiKVKey size_key = TiKVKey::copyFrom(key); auto [data_size, memory_size] = default_cf.insert(std::move(key), std::move(value)); cf_data_size += data_size; total_memory_size += memory_size; - default_cf_memory_size.emplace(size_key, memory_size); + default_cf_memory_size.emplace(std::move(size_key), memory_size); return; } case ColumnFamilyType::Lock: @@ -76,7 +76,7 @@ void RegionData::remove(ColumnFamilyType cf, const TiKVKey & key) cf_data_size -= write_cf.remove(RegionWriteCFData::Key{pk, ts}, true); if (write_cf_memory_size.find(key) != write_cf_memory_size.end()) { - total_memory_size -= write_cf_memory_size[key]; + total_memory_size -= write_cf_memory_size.at(key); write_cf_memory_size.erase(key); } return; @@ -90,7 +90,7 @@ void RegionData::remove(ColumnFamilyType cf, const TiKVKey & key) cf_data_size -= default_cf.remove(RegionDefaultCFData::Key{pk, ts}, true); if (default_cf_memory_size.find(key) != default_cf_memory_size.end()) { - total_memory_size -= default_cf_memory_size[key]; + total_memory_size -= default_cf_memory_size.at(key); default_cf_memory_size.erase(key); } return; @@ -119,14 +119,14 @@ RegionData::WriteCFIter RegionData::removeDataByWriteIt(const WriteCFIter & writ if (auto data_it = map.find({pk, decoded_val.prewrite_ts}); data_it != map.end()) { cf_data_size -= RegionDefaultCFData::calcTiKVKeyValueSize(data_it->second); - total_memory_size -= default_cf_memory_size[*key]; + total_memory_size -= default_cf_memory_size.at(*key); default_cf_memory_size.erase(*key); map.erase(data_it); } } cf_data_size -= RegionWriteCFData::calcTiKVKeyValueSize(write_it->second); - total_memory_size -= write_cf_memory_size[*key]; + total_memory_size -= write_cf_memory_size.at(*key); write_cf_memory_size.erase(*key); return write_cf.getDataMut().erase(write_it); @@ -198,11 +198,11 @@ void RegionData::splitInto(const RegionRange & range, RegionData & new_region_da size_t memory_size_changed = 0; for (auto iter = default_cf_memory_size.begin(); iter != default_cf_memory_size.end();) { - auto & key = iter->first; + const auto & key = iter->first; if (start_key.compare(key) <= 0 && end_key.compare(key) > 0) { memory_size_changed += iter->second; - new_region_data.default_cf_memory_size.emplace(iter->first, iter->second); + new_region_data.default_cf_memory_size.emplace(TiKVKey::copyFrom(iter->first), iter->second); iter = default_cf_memory_size.erase(iter); } else @@ -210,11 +210,11 @@ void RegionData::splitInto(const RegionRange & range, RegionData & new_region_da } for (auto iter = write_cf_memory_size.begin(); iter != write_cf_memory_size.end();) { - auto & key = iter->first; + const auto & key = iter->first; if (start_key.compare(key) <= 0 && end_key.compare(key) > 0) { memory_size_changed += iter->second; - new_region_data.write_cf_memory_size.emplace(iter->first, iter->second); + new_region_data.write_cf_memory_size.emplace(TiKVKey::copyFrom(iter->first), iter->second); iter = write_cf_memory_size.erase(iter); } else @@ -233,12 +233,12 @@ void RegionData::mergeFrom(const RegionData & ori_region_data) cf_data_size += size_changed; for (auto iter = ori_region_data.default_cf_memory_size.begin(); iter != ori_region_data.default_cf_memory_size.end(); iter++) { - default_cf_memory_size.emplace(iter->first, iter->second); + default_cf_memory_size.emplace(TiKVKey::copyFrom(iter->first), iter->second); total_memory_size += iter->second; } for (auto iter = ori_region_data.write_cf_memory_size.begin(); iter != ori_region_data.write_cf_memory_size.end();) { - write_cf_memory_size.emplace(iter->first, iter->second); + write_cf_memory_size.emplace(TiKVKey::copyFrom(iter->first), iter->second); total_memory_size += iter->second; } } From 08241a58fc8f6902bb210ae7c66f13b9baab3f1f Mon Sep 17 00:00:00 2001 From: lidezhu Date: Thu, 28 Apr 2022 23:53:40 +0800 Subject: [PATCH 17/29] add some log --- dbms/src/Storages/Transaction/KVStore.cpp | 8 ++++++++ dbms/src/Storages/Transaction/PartitionStreams.cpp | 1 + 2 files changed, 9 insertions(+) diff --git a/dbms/src/Storages/Transaction/KVStore.cpp b/dbms/src/Storages/Transaction/KVStore.cpp index 065c4f1b140..66c7b20f561 100644 --- a/dbms/src/Storages/Transaction/KVStore.cpp +++ b/dbms/src/Storages/Transaction/KVStore.cpp @@ -308,6 +308,14 @@ EngineStoreApplyRes KVStore::handleWriteRaftCmd(const WriteCmdsView & cmds, UInt } auto res = region->handleWriteRaftCmd(cmds, index, term, tmt); + size_t region_data_size = 0; + size_t region_data_memory_size = 0; + tmt.getKVStore()->traverseRegions([®ion_data_size, ®ion_data_memory_size](RegionID, const RegionPtr & region) { + region_data_size += region->dataSize(); + region_data_memory_size += region->memorySize(); + }); + GET_METRIC(tiflash_kvstore_region_data_memory_size).Set(region_data_size); + GET_METRIC(tiflash_kvstore_region_data_consume_memory_size).Set(region_data_memory_size); return res; } diff --git a/dbms/src/Storages/Transaction/PartitionStreams.cpp b/dbms/src/Storages/Transaction/PartitionStreams.cpp index 9142aad5358..fd708f2e24b 100644 --- a/dbms/src/Storages/Transaction/PartitionStreams.cpp +++ b/dbms/src/Storages/Transaction/PartitionStreams.cpp @@ -344,6 +344,7 @@ void RegionTable::writeBlockByRegion( if (!data_list_read) return; + LOG_FMT_DEBUG(&Poco::Logger::get(__PRETTY_FUNCTION__), "write committed data rows {}", data_list_read->size()); reportUpstreamLatency(*data_list_read); writeRegionDataToStorage(context, region, *data_list_read, log); From 5140b6e2a86f0c1a8c02fdc78816f9e7e6732bab Mon Sep 17 00:00:00 2001 From: lidezhu Date: Fri, 29 Apr 2022 00:28:41 +0800 Subject: [PATCH 18/29] add more log --- dbms/src/Common/TiFlashMetrics.h | 1 + dbms/src/Storages/Transaction/KVStore.cpp | 9 +++++---- dbms/src/Storages/Transaction/PartitionStreams.cpp | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/dbms/src/Common/TiFlashMetrics.h b/dbms/src/Common/TiFlashMetrics.h index 52518084e2e..69ae5ac390d 100644 --- a/dbms/src/Common/TiFlashMetrics.h +++ b/dbms/src/Common/TiFlashMetrics.h @@ -82,6 +82,7 @@ namespace DB M(tiflash_coprocessor_response_bytes, "Total bytes of response body", Counter) \ M(tiflash_kvstore_region_data_memory_size, "Total region data memory size of tiflash kvstore", Gauge) \ M(tiflash_kvstore_region_data_consume_memory_size, "Total region data memory size of tiflash kvstore", Gauge) \ + M(tiflash_kvstore_region_cf_size, "Total region data memory size of tiflash kvstore", Gauge) \ M(tiflash_kvstore_raft_command_size, "Total region data memory size of tiflash kvstore", Gauge) \ M(tiflash_kvstore_data_approx_cache_memory_size, "Total region data memory size of tiflash memory data", Gauge) \ M(tiflash_schema_version, "Current version of tiflash cached schema", Gauge) \ diff --git a/dbms/src/Storages/Transaction/KVStore.cpp b/dbms/src/Storages/Transaction/KVStore.cpp index 66c7b20f561..c5d66c03ac4 100644 --- a/dbms/src/Storages/Transaction/KVStore.cpp +++ b/dbms/src/Storages/Transaction/KVStore.cpp @@ -296,9 +296,6 @@ EngineStoreApplyRes KVStore::handleWriteRaftCmd(const WriteCmdsView & cmds, UInt } } GET_METRIC(tiflash_kvstore_raft_command_size).Increment(cmd_size); - SCOPE_EXIT({ - GET_METRIC(tiflash_kvstore_raft_command_size).Decrement(cmd_size); - }); auto region_persist_lock = region_manager.genRegionTaskLock(region_id); const RegionPtr region = getRegion(region_id); @@ -310,12 +307,16 @@ EngineStoreApplyRes KVStore::handleWriteRaftCmd(const WriteCmdsView & cmds, UInt auto res = region->handleWriteRaftCmd(cmds, index, term, tmt); size_t region_data_size = 0; size_t region_data_memory_size = 0; - tmt.getKVStore()->traverseRegions([®ion_data_size, ®ion_data_memory_size](RegionID, const RegionPtr & region) { + size_t region_write_cf_num = 0; + tmt.getKVStore()->traverseRegions([®ion_data_size, ®ion_data_memory_size, ®ion_write_cf_num](RegionID, const RegionPtr & region) { region_data_size += region->dataSize(); region_data_memory_size += region->memorySize(); + region_write_cf_num += region->writeCFCount(); }); GET_METRIC(tiflash_kvstore_region_data_memory_size).Set(region_data_size); GET_METRIC(tiflash_kvstore_region_data_consume_memory_size).Set(region_data_memory_size); + GET_METRIC(tiflash_kvstore_region_cf_size).Set(region_write_cf_num); + LOG_FMT_DEBUG(&Poco::Logger::get(__PRETTY_FUNCTION__), "region total cf count {}", region_write_cf_num); return res; } diff --git a/dbms/src/Storages/Transaction/PartitionStreams.cpp b/dbms/src/Storages/Transaction/PartitionStreams.cpp index fd708f2e24b..e59c4f9671e 100644 --- a/dbms/src/Storages/Transaction/PartitionStreams.cpp +++ b/dbms/src/Storages/Transaction/PartitionStreams.cpp @@ -346,7 +346,7 @@ void RegionTable::writeBlockByRegion( LOG_FMT_DEBUG(&Poco::Logger::get(__PRETTY_FUNCTION__), "write committed data rows {}", data_list_read->size()); reportUpstreamLatency(*data_list_read); - writeRegionDataToStorage(context, region, *data_list_read, log); +// writeRegionDataToStorage(context, region, *data_list_read, log); RemoveRegionCommitCache(region, *data_list_read, lock_region); From aa329faa3f327aca1f9a59fd2971a9b0948b864b Mon Sep 17 00:00:00 2001 From: lidezhu Date: Fri, 29 Apr 2022 00:45:24 +0800 Subject: [PATCH 19/29] more log --- dbms/src/Storages/Transaction/KVStore.cpp | 10 +++++----- dbms/src/Storages/Transaction/Region.cpp | 6 ++++++ dbms/src/Storages/Transaction/Region.h | 1 + 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/dbms/src/Storages/Transaction/KVStore.cpp b/dbms/src/Storages/Transaction/KVStore.cpp index c5d66c03ac4..0842b0e5508 100644 --- a/dbms/src/Storages/Transaction/KVStore.cpp +++ b/dbms/src/Storages/Transaction/KVStore.cpp @@ -307,16 +307,16 @@ EngineStoreApplyRes KVStore::handleWriteRaftCmd(const WriteCmdsView & cmds, UInt auto res = region->handleWriteRaftCmd(cmds, index, term, tmt); size_t region_data_size = 0; size_t region_data_memory_size = 0; - size_t region_write_cf_num = 0; - tmt.getKVStore()->traverseRegions([®ion_data_size, ®ion_data_memory_size, ®ion_write_cf_num](RegionID, const RegionPtr & region) { + size_t region_default_cf_num = 0; + tmt.getKVStore()->traverseRegions([®ion_data_size, ®ion_data_memory_size, ®ion_default_cf_num](RegionID, const RegionPtr & region) { region_data_size += region->dataSize(); region_data_memory_size += region->memorySize(); - region_write_cf_num += region->writeCFCount(); + region_default_cf_num += region->defaultCFCount(); }); GET_METRIC(tiflash_kvstore_region_data_memory_size).Set(region_data_size); GET_METRIC(tiflash_kvstore_region_data_consume_memory_size).Set(region_data_memory_size); - GET_METRIC(tiflash_kvstore_region_cf_size).Set(region_write_cf_num); - LOG_FMT_DEBUG(&Poco::Logger::get(__PRETTY_FUNCTION__), "region total cf count {}", region_write_cf_num); + GET_METRIC(tiflash_kvstore_region_cf_size).Set(region_default_cf_num); + LOG_FMT_DEBUG(&Poco::Logger::get(__PRETTY_FUNCTION__), "region total default cf count {}", region_default_cf_num); return res; } diff --git a/dbms/src/Storages/Transaction/Region.cpp b/dbms/src/Storages/Transaction/Region.cpp index 5603c83ac08..7e6e2e1c236 100644 --- a/dbms/src/Storages/Transaction/Region.cpp +++ b/dbms/src/Storages/Transaction/Region.cpp @@ -445,6 +445,12 @@ size_t Region::writeCFCount() const return data.writeCF().getSize(); } +size_t Region::defaultCFCount() const +{ + std::shared_lock lock(mutex); + return data.defaultCF().getSize(); +} + std::string Region::dataInfo() const { std::shared_lock lock(mutex); diff --git a/dbms/src/Storages/Transaction/Region.h b/dbms/src/Storages/Transaction/Region.h index c886bdc2522..5d886e6d639 100644 --- a/dbms/src/Storages/Transaction/Region.h +++ b/dbms/src/Storages/Transaction/Region.h @@ -149,6 +149,7 @@ class Region : public std::enable_shared_from_this size_t dataSize() const; size_t memorySize() const; size_t writeCFCount() const; + size_t defaultCFCount() const; std::string dataInfo() const; void markCompactLog() const; From b2e865614528b0014dcc991abc4f11366dc8248d Mon Sep 17 00:00:00 2001 From: lidezhu Date: Fri, 29 Apr 2022 11:16:41 +0800 Subject: [PATCH 20/29] add lock cf count --- .../Transaction/BackgroundService.cpp | 26 ------------------- .../Storages/Transaction/BackgroundService.h | 1 - dbms/src/Storages/Transaction/KVStore.cpp | 6 +++-- .../Storages/Transaction/PartitionStreams.cpp | 2 +- dbms/src/Storages/Transaction/Region.cpp | 6 +++++ dbms/src/Storages/Transaction/Region.h | 1 + 6 files changed, 12 insertions(+), 30 deletions(-) diff --git a/dbms/src/Storages/Transaction/BackgroundService.cpp b/dbms/src/Storages/Transaction/BackgroundService.cpp index 2e13fdd90bb..e751575575b 100644 --- a/dbms/src/Storages/Transaction/BackgroundService.cpp +++ b/dbms/src/Storages/Transaction/BackgroundService.cpp @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include #include #include #include @@ -37,26 +36,6 @@ BackgroundService::BackgroundService(TMTContext & tmt_) }, false); - kvstore_size_metric_handle = background_pool.addTask( - [this] { - size_t region_data_size = 0; - size_t region_data_memory_size = 0; - size_t memory_cache_size = 0; - size_t region_count = 0; - tmt.getKVStore()->traverseRegions([®ion_data_size, ®ion_data_memory_size, &memory_cache_size, ®ion_count](RegionID, const RegionPtr & region) { - region_data_size += region->dataSize(); - region_data_memory_size += region->memorySize(); - memory_cache_size += region->getApproxMemCacheInfo().second; - region_count += 1; - }); - GET_METRIC(tiflash_kvstore_region_data_memory_size).Set(region_data_size); - GET_METRIC(tiflash_kvstore_region_data_consume_memory_size).Set(region_data_memory_size); - GET_METRIC(tiflash_kvstore_data_approx_cache_memory_size).Set(memory_cache_size); - LOG_FMT_INFO(log, "KVStore region data memory size is {}, memory cache size is {}, region count {}", region_data_size, memory_cache_size, region_count); - return false; - }, - false); - if (!tmt.isBgFlushDisabled()) { table_flush_handle = background_pool.addTask([this] { @@ -125,11 +104,6 @@ BackgroundService::~BackgroundService() background_pool.removeTask(single_thread_task_handle); single_thread_task_handle = nullptr; } - if (kvstore_size_metric_handle) - { - background_pool.removeTask(kvstore_size_metric_handle); - kvstore_size_metric_handle = nullptr; - } if (table_flush_handle) { background_pool.removeTask(table_flush_handle); diff --git a/dbms/src/Storages/Transaction/BackgroundService.h b/dbms/src/Storages/Transaction/BackgroundService.h index 3ac288d5ccc..18a0f04206f 100644 --- a/dbms/src/Storages/Transaction/BackgroundService.h +++ b/dbms/src/Storages/Transaction/BackgroundService.h @@ -51,7 +51,6 @@ class BackgroundService : boost::noncopyable RegionMap regions_to_flush; BackgroundProcessingPool::TaskHandle single_thread_task_handle; - BackgroundProcessingPool::TaskHandle kvstore_size_metric_handle; BackgroundProcessingPool::TaskHandle table_flush_handle; BackgroundProcessingPool::TaskHandle region_handle; BackgroundProcessingPool::TaskHandle storage_gc_handle; diff --git a/dbms/src/Storages/Transaction/KVStore.cpp b/dbms/src/Storages/Transaction/KVStore.cpp index 0842b0e5508..38ade9bcaf3 100644 --- a/dbms/src/Storages/Transaction/KVStore.cpp +++ b/dbms/src/Storages/Transaction/KVStore.cpp @@ -308,15 +308,17 @@ EngineStoreApplyRes KVStore::handleWriteRaftCmd(const WriteCmdsView & cmds, UInt size_t region_data_size = 0; size_t region_data_memory_size = 0; size_t region_default_cf_num = 0; - tmt.getKVStore()->traverseRegions([®ion_data_size, ®ion_data_memory_size, ®ion_default_cf_num](RegionID, const RegionPtr & region) { + size_t region_lock_cf_num = 0; + traverseRegions([®ion_data_size, ®ion_data_memory_size, ®ion_default_cf_num, ®ion_lock_cf_num](RegionID, const RegionPtr & region) { region_data_size += region->dataSize(); region_data_memory_size += region->memorySize(); region_default_cf_num += region->defaultCFCount(); + region_lock_cf_num += region->lockCFCount(); }); GET_METRIC(tiflash_kvstore_region_data_memory_size).Set(region_data_size); GET_METRIC(tiflash_kvstore_region_data_consume_memory_size).Set(region_data_memory_size); GET_METRIC(tiflash_kvstore_region_cf_size).Set(region_default_cf_num); - LOG_FMT_DEBUG(&Poco::Logger::get(__PRETTY_FUNCTION__), "region total default cf count {}", region_default_cf_num); + LOG_FMT_DEBUG(&Poco::Logger::get(__PRETTY_FUNCTION__), "region total default cf count {} lock cf count", region_default_cf_num, region_lock_cf_num); return res; } diff --git a/dbms/src/Storages/Transaction/PartitionStreams.cpp b/dbms/src/Storages/Transaction/PartitionStreams.cpp index e59c4f9671e..fd708f2e24b 100644 --- a/dbms/src/Storages/Transaction/PartitionStreams.cpp +++ b/dbms/src/Storages/Transaction/PartitionStreams.cpp @@ -346,7 +346,7 @@ void RegionTable::writeBlockByRegion( LOG_FMT_DEBUG(&Poco::Logger::get(__PRETTY_FUNCTION__), "write committed data rows {}", data_list_read->size()); reportUpstreamLatency(*data_list_read); -// writeRegionDataToStorage(context, region, *data_list_read, log); + writeRegionDataToStorage(context, region, *data_list_read, log); RemoveRegionCommitCache(region, *data_list_read, lock_region); diff --git a/dbms/src/Storages/Transaction/Region.cpp b/dbms/src/Storages/Transaction/Region.cpp index 7e6e2e1c236..cbf7e973da4 100644 --- a/dbms/src/Storages/Transaction/Region.cpp +++ b/dbms/src/Storages/Transaction/Region.cpp @@ -451,6 +451,12 @@ size_t Region::defaultCFCount() const return data.defaultCF().getSize(); } +size_t Region::lockCFCount() const +{ + std::shared_lock lock(mutex); + return data.lockCF().getSize(); +} + std::string Region::dataInfo() const { std::shared_lock lock(mutex); diff --git a/dbms/src/Storages/Transaction/Region.h b/dbms/src/Storages/Transaction/Region.h index 5d886e6d639..bb6e5c91b5e 100644 --- a/dbms/src/Storages/Transaction/Region.h +++ b/dbms/src/Storages/Transaction/Region.h @@ -150,6 +150,7 @@ class Region : public std::enable_shared_from_this size_t memorySize() const; size_t writeCFCount() const; size_t defaultCFCount() const; + size_t lockCFCount() const; std::string dataInfo() const; void markCompactLog() const; From d612c68d27f0f4b1e64bcd6f42df586e0a1e2f9e Mon Sep 17 00:00:00 2001 From: lidezhu Date: Fri, 29 Apr 2022 11:51:41 +0800 Subject: [PATCH 21/29] small fix --- dbms/src/Storages/Transaction/KVStore.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/Transaction/KVStore.cpp b/dbms/src/Storages/Transaction/KVStore.cpp index 38ade9bcaf3..f45c7545686 100644 --- a/dbms/src/Storages/Transaction/KVStore.cpp +++ b/dbms/src/Storages/Transaction/KVStore.cpp @@ -318,7 +318,7 @@ EngineStoreApplyRes KVStore::handleWriteRaftCmd(const WriteCmdsView & cmds, UInt GET_METRIC(tiflash_kvstore_region_data_memory_size).Set(region_data_size); GET_METRIC(tiflash_kvstore_region_data_consume_memory_size).Set(region_data_memory_size); GET_METRIC(tiflash_kvstore_region_cf_size).Set(region_default_cf_num); - LOG_FMT_DEBUG(&Poco::Logger::get(__PRETTY_FUNCTION__), "region total default cf count {} lock cf count", region_default_cf_num, region_lock_cf_num); + LOG_FMT_DEBUG(&Poco::Logger::get(__PRETTY_FUNCTION__), "region total default cf count {} lock cf count {}", region_default_cf_num, region_lock_cf_num); return res; } From 02cbbf31dc0e6f9b8ee8c09a4ea6d6cfcdd88081 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Fri, 29 Apr 2022 18:40:31 +0800 Subject: [PATCH 22/29] count lock cf size --- .../Storages/Transaction/RegionCFDataBase.cpp | 13 +- .../Storages/Transaction/RegionCFDataBase.h | 4 +- dbms/src/Storages/Transaction/RegionData.cpp | 123 +++++++++--------- 3 files changed, 68 insertions(+), 72 deletions(-) diff --git a/dbms/src/Storages/Transaction/RegionCFDataBase.cpp b/dbms/src/Storages/Transaction/RegionCFDataBase.cpp index d6826010b84..30799fc39d8 100644 --- a/dbms/src/Storages/Transaction/RegionCFDataBase.cpp +++ b/dbms/src/Storages/Transaction/RegionCFDataBase.cpp @@ -44,10 +44,11 @@ RegionDataRes RegionCFDataBase::insert(TiKVKey && key, TiKVValue && value { const auto & raw_key = RecordKVFormat::decodeTiKVKey(key); auto [kv_pair, memory_size] = Trait::genKVPair(std::move(key), raw_key, std::move(value)); + std::ignore = memory_size; if (!kv_pair) - return std::pair(0, 0); + return 0; - return std::make_pair(insert(std::move(*kv_pair)), memory_size); + return insert(std::move(*kv_pair)); } template <> @@ -55,12 +56,14 @@ RegionDataRes RegionCFDataBase::insert(TiKVKey && key, Ti { Pair kv_pair = RegionLockCFDataTrait::genKVPair(std::move(key), std::move(value)); // according to the process of pessimistic lock, just overwrite. - data.insert_or_assign(std::move(kv_pair.first), std::move(kv_pair.second)); - return std::pair(0, 0); + auto [it, ok] = data.insert_or_assign(std::move(kv_pair.first), std::move(kv_pair.second)); + std::ignore = ok; + return calcTiKVKeyValueSize(it->second); + // return 0; } template -size_t RegionCFDataBase::insert(std::pair && kv_pair) +RegionDataRes RegionCFDataBase::insert(std::pair && kv_pair) { auto & map = data; auto [it, ok] = map.emplace(std::move(kv_pair)); diff --git a/dbms/src/Storages/Transaction/RegionCFDataBase.h b/dbms/src/Storages/Transaction/RegionCFDataBase.h index 1705b4d4eb5..09a43d5a288 100644 --- a/dbms/src/Storages/Transaction/RegionCFDataBase.h +++ b/dbms/src/Storages/Transaction/RegionCFDataBase.h @@ -23,7 +23,7 @@ namespace DB struct TiKVRangeKey; using RegionRange = std::pair; -using RegionDataRes = std::pair; +using RegionDataRes = size_t; template struct RegionCFDataBase @@ -70,7 +70,7 @@ struct RegionCFDataBase private: static bool shouldIgnoreRemove(const Value & value); - size_t insert(std::pair && kv_pair); + RegionDataRes insert(std::pair && kv_pair); private: Data data; diff --git a/dbms/src/Storages/Transaction/RegionData.cpp b/dbms/src/Storages/Transaction/RegionData.cpp index 5cb19345f82..7a36d838004 100644 --- a/dbms/src/Storages/Transaction/RegionData.cpp +++ b/dbms/src/Storages/Transaction/RegionData.cpp @@ -39,25 +39,17 @@ void RegionData::insert(ColumnFamilyType cf, TiKVKey && key, TiKVValue && value) { case ColumnFamilyType::Write: { - TiKVKey size_key = TiKVKey::copyFrom(key); - auto [data_size, memory_size] = write_cf.insert(std::move(key), std::move(value)); - cf_data_size += data_size; - total_memory_size += memory_size; - write_cf_memory_size.emplace(std::move(size_key), memory_size); + cf_data_size += write_cf.insert(std::move(key), std::move(value)); return; } case ColumnFamilyType::Default: { - TiKVKey size_key = TiKVKey::copyFrom(key); - auto [data_size, memory_size] = default_cf.insert(std::move(key), std::move(value)); - cf_data_size += data_size; - total_memory_size += memory_size; - default_cf_memory_size.emplace(std::move(size_key), memory_size); + cf_data_size += default_cf.insert(std::move(key), std::move(value)); return; } case ColumnFamilyType::Lock: { - lock_cf.insert(std::move(key), std::move(value)); + total_memory_size += lock_cf.insert(std::move(key), std::move(value)); return; } } @@ -74,11 +66,11 @@ void RegionData::remove(ColumnFamilyType cf, const TiKVKey & key) Timestamp ts = RecordKVFormat::getTs(key); // removed by gc, may not exist. cf_data_size -= write_cf.remove(RegionWriteCFData::Key{pk, ts}, true); - if (write_cf_memory_size.find(key) != write_cf_memory_size.end()) - { - total_memory_size -= write_cf_memory_size.at(key); - write_cf_memory_size.erase(key); - } +// if (write_cf_memory_size.find(key) != write_cf_memory_size.end()) +// { +// total_memory_size -= write_cf_memory_size.at(key); +// write_cf_memory_size.erase(key); +// } return; } case ColumnFamilyType::Default: @@ -88,16 +80,16 @@ void RegionData::remove(ColumnFamilyType cf, const TiKVKey & key) Timestamp ts = RecordKVFormat::getTs(key); // removed by gc, may not exist. cf_data_size -= default_cf.remove(RegionDefaultCFData::Key{pk, ts}, true); - if (default_cf_memory_size.find(key) != default_cf_memory_size.end()) - { - total_memory_size -= default_cf_memory_size.at(key); - default_cf_memory_size.erase(key); - } +// if (default_cf_memory_size.find(key) != default_cf_memory_size.end()) +// { +// total_memory_size -= default_cf_memory_size.at(key); +// default_cf_memory_size.erase(key); +// } return; } case ColumnFamilyType::Lock: { - lock_cf.remove(RegionLockCFDataTrait::Key{nullptr, std::string_view(key.data(), key.dataSize())}, true); + total_memory_size -= lock_cf.remove(RegionLockCFDataTrait::Key{nullptr, std::string_view(key.data(), key.dataSize())}, true); return; } } @@ -119,15 +111,15 @@ RegionData::WriteCFIter RegionData::removeDataByWriteIt(const WriteCFIter & writ if (auto data_it = map.find({pk, decoded_val.prewrite_ts}); data_it != map.end()) { cf_data_size -= RegionDefaultCFData::calcTiKVKeyValueSize(data_it->second); - total_memory_size -= default_cf_memory_size.at(*key); - default_cf_memory_size.erase(*key); +// total_memory_size -= default_cf_memory_size.at(*key); +// default_cf_memory_size.erase(*key); map.erase(data_it); } } cf_data_size -= RegionWriteCFData::calcTiKVKeyValueSize(write_it->second); - total_memory_size -= write_cf_memory_size.at(*key); - write_cf_memory_size.erase(*key); +// total_memory_size -= write_cf_memory_size.at(*key); +// write_cf_memory_size.erase(*key); return write_cf.getDataMut().erase(write_it); } @@ -190,38 +182,39 @@ void RegionData::splitInto(const RegionRange & range, RegionData & new_region_da size_t size_changed = 0; size_changed += default_cf.splitInto(range, new_region_data.default_cf); size_changed += write_cf.splitInto(range, new_region_data.write_cf); - size_changed += lock_cf.splitInto(range, new_region_data.lock_cf); + size_t memory_size_changed = 0; + memory_size_changed += lock_cf.splitInto(range, new_region_data.lock_cf); cf_data_size -= size_changed; new_region_data.cf_data_size += size_changed; - - const auto & [start_key, end_key] = range; - size_t memory_size_changed = 0; - for (auto iter = default_cf_memory_size.begin(); iter != default_cf_memory_size.end();) - { - const auto & key = iter->first; - if (start_key.compare(key) <= 0 && end_key.compare(key) > 0) - { - memory_size_changed += iter->second; - new_region_data.default_cf_memory_size.emplace(TiKVKey::copyFrom(iter->first), iter->second); - iter = default_cf_memory_size.erase(iter); - } - else - ++iter; - } - for (auto iter = write_cf_memory_size.begin(); iter != write_cf_memory_size.end();) - { - const auto & key = iter->first; - if (start_key.compare(key) <= 0 && end_key.compare(key) > 0) - { - memory_size_changed += iter->second; - new_region_data.write_cf_memory_size.emplace(TiKVKey::copyFrom(iter->first), iter->second); - iter = write_cf_memory_size.erase(iter); - } - else - ++iter; - } total_memory_size -= memory_size_changed; new_region_data.total_memory_size = memory_size_changed; + +// const auto & [start_key, end_key] = range; +// size_t memory_size_changed = 0; +// for (auto iter = default_cf_memory_size.begin(); iter != default_cf_memory_size.end();) +// { +// const auto & key = iter->first; +// if (start_key.compare(key) <= 0 && end_key.compare(key) > 0) +// { +// memory_size_changed += iter->second; +// new_region_data.default_cf_memory_size.emplace(TiKVKey::copyFrom(iter->first), iter->second); +// iter = default_cf_memory_size.erase(iter); +// } +// else +// ++iter; +// } +// for (auto iter = write_cf_memory_size.begin(); iter != write_cf_memory_size.end();) +// { +// const auto & key = iter->first; +// if (start_key.compare(key) <= 0 && end_key.compare(key) > 0) +// { +// memory_size_changed += iter->second; +// new_region_data.write_cf_memory_size.emplace(TiKVKey::copyFrom(iter->first), iter->second); +// iter = write_cf_memory_size.erase(iter); +// } +// else +// ++iter; +// } } void RegionData::mergeFrom(const RegionData & ori_region_data) @@ -229,18 +222,18 @@ void RegionData::mergeFrom(const RegionData & ori_region_data) size_t size_changed = 0; size_changed += default_cf.mergeFrom(ori_region_data.default_cf); size_changed += write_cf.mergeFrom(ori_region_data.write_cf); - size_changed += lock_cf.mergeFrom(ori_region_data.lock_cf); + total_memory_size += lock_cf.mergeFrom(ori_region_data.lock_cf); cf_data_size += size_changed; - for (auto iter = ori_region_data.default_cf_memory_size.begin(); iter != ori_region_data.default_cf_memory_size.end(); iter++) - { - default_cf_memory_size.emplace(TiKVKey::copyFrom(iter->first), iter->second); - total_memory_size += iter->second; - } - for (auto iter = ori_region_data.write_cf_memory_size.begin(); iter != ori_region_data.write_cf_memory_size.end();) - { - write_cf_memory_size.emplace(TiKVKey::copyFrom(iter->first), iter->second); - total_memory_size += iter->second; - } +// for (auto iter = ori_region_data.default_cf_memory_size.begin(); iter != ori_region_data.default_cf_memory_size.end(); iter++) +// { +// default_cf_memory_size.emplace(TiKVKey::copyFrom(iter->first), iter->second); +// total_memory_size += iter->second; +// } +// for (auto iter = ori_region_data.write_cf_memory_size.begin(); iter != ori_region_data.write_cf_memory_size.end();) +// { +// write_cf_memory_size.emplace(TiKVKey::copyFrom(iter->first), iter->second); +// total_memory_size += iter->second; +// } } size_t RegionData::dataSize() const From b5588852726f6c58abdbdbaa4e14fd98e1d3c424 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Fri, 29 Apr 2022 18:53:51 +0800 Subject: [PATCH 23/29] fix build --- dbms/src/Storages/Transaction/RegionCFDataBase.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dbms/src/Storages/Transaction/RegionCFDataBase.cpp b/dbms/src/Storages/Transaction/RegionCFDataBase.cpp index 30799fc39d8..a2264541fbd 100644 --- a/dbms/src/Storages/Transaction/RegionCFDataBase.cpp +++ b/dbms/src/Storages/Transaction/RegionCFDataBase.cpp @@ -244,9 +244,8 @@ size_t RegionCFDataBase::deserialize(ReadBuffer & buf, RegionCFDataBase & { auto key = TiKVKey::deserialize(buf); auto value = TiKVValue::deserialize(buf); - auto [data_size, memory_size] = new_region_data.insert(std::move(key), std::move(value)); + auto data_size = new_region_data.insert(std::move(key), std::move(value)); cf_data_size += data_size; - std::ignore = memory_size; } return cf_data_size; } From da6dac699eb71e216961ba35a1a876176c0ad162 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Fri, 29 Apr 2022 19:36:47 +0800 Subject: [PATCH 24/29] fix lock cf size --- dbms/src/Storages/Transaction/RegionCFDataBase.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dbms/src/Storages/Transaction/RegionCFDataBase.cpp b/dbms/src/Storages/Transaction/RegionCFDataBase.cpp index a2264541fbd..6f1377b173e 100644 --- a/dbms/src/Storages/Transaction/RegionCFDataBase.cpp +++ b/dbms/src/Storages/Transaction/RegionCFDataBase.cpp @@ -84,9 +84,7 @@ size_t RegionCFDataBase::calcTiKVKeyValueSize(const TiKVKey & key, const { if constexpr (std::is_same::value) { - std::ignore = key; - std::ignore = value; - return 0; + return key.dataSize() + value.dataSize(); } else return key.dataSize() + value.dataSize(); From c1fdea87eff60e5f3033d97c68328c6e41525fbb Mon Sep 17 00:00:00 2001 From: lidezhu Date: Fri, 29 Apr 2022 20:20:27 +0800 Subject: [PATCH 25/29] try fix metric --- .../Storages/Transaction/RegionCFDataBase.cpp | 11 +++++++--- dbms/src/Storages/Transaction/RegionData.cpp | 20 +++++-------------- 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/dbms/src/Storages/Transaction/RegionCFDataBase.cpp b/dbms/src/Storages/Transaction/RegionCFDataBase.cpp index 6f1377b173e..e40fb4b8984 100644 --- a/dbms/src/Storages/Transaction/RegionCFDataBase.cpp +++ b/dbms/src/Storages/Transaction/RegionCFDataBase.cpp @@ -57,9 +57,14 @@ RegionDataRes RegionCFDataBase::insert(TiKVKey && key, Ti Pair kv_pair = RegionLockCFDataTrait::genKVPair(std::move(key), std::move(value)); // according to the process of pessimistic lock, just overwrite. auto [it, ok] = data.insert_or_assign(std::move(kv_pair.first), std::move(kv_pair.second)); - std::ignore = ok; - return calcTiKVKeyValueSize(it->second); - // return 0; + if (ok) + { + return calcTiKVKeyValueSize(it->second); + } + else + { + return 0; + } } template diff --git a/dbms/src/Storages/Transaction/RegionData.cpp b/dbms/src/Storages/Transaction/RegionData.cpp index 7a36d838004..44ca5e7d3cc 100644 --- a/dbms/src/Storages/Transaction/RegionData.cpp +++ b/dbms/src/Storages/Transaction/RegionData.cpp @@ -66,11 +66,6 @@ void RegionData::remove(ColumnFamilyType cf, const TiKVKey & key) Timestamp ts = RecordKVFormat::getTs(key); // removed by gc, may not exist. cf_data_size -= write_cf.remove(RegionWriteCFData::Key{pk, ts}, true); -// if (write_cf_memory_size.find(key) != write_cf_memory_size.end()) -// { -// total_memory_size -= write_cf_memory_size.at(key); -// write_cf_memory_size.erase(key); -// } return; } case ColumnFamilyType::Default: @@ -80,16 +75,15 @@ void RegionData::remove(ColumnFamilyType cf, const TiKVKey & key) Timestamp ts = RecordKVFormat::getTs(key); // removed by gc, may not exist. cf_data_size -= default_cf.remove(RegionDefaultCFData::Key{pk, ts}, true); -// if (default_cf_memory_size.find(key) != default_cf_memory_size.end()) -// { -// total_memory_size -= default_cf_memory_size.at(key); -// default_cf_memory_size.erase(key); -// } return; } case ColumnFamilyType::Lock: { - total_memory_size -= lock_cf.remove(RegionLockCFDataTrait::Key{nullptr, std::string_view(key.data(), key.dataSize())}, true); + size_t size_removed = lock_cf.remove(RegionLockCFDataTrait::Key{nullptr, std::string_view(key.data(), key.dataSize())}, true); + if (total_memory_size <= size_removed) + total_memory_size = 0; + else + total_memory_size -= size_removed; return; } } @@ -111,15 +105,11 @@ RegionData::WriteCFIter RegionData::removeDataByWriteIt(const WriteCFIter & writ if (auto data_it = map.find({pk, decoded_val.prewrite_ts}); data_it != map.end()) { cf_data_size -= RegionDefaultCFData::calcTiKVKeyValueSize(data_it->second); -// total_memory_size -= default_cf_memory_size.at(*key); -// default_cf_memory_size.erase(*key); map.erase(data_it); } } cf_data_size -= RegionWriteCFData::calcTiKVKeyValueSize(write_it->second); -// total_memory_size -= write_cf_memory_size.at(*key); -// write_cf_memory_size.erase(*key); return write_cf.getDataMut().erase(write_it); } From be441bf0b25c24c901de1890bb7b95108aa848f7 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Thu, 5 May 2022 14:42:28 +0800 Subject: [PATCH 26/29] add a cusom allocator --- .../Storages/Transaction/RegionCFDataBase.cpp | 33 +++++++++++++++---- .../Storages/Transaction/RegionCFDataTrait.h | 32 +++++++++++++++++- 2 files changed, 57 insertions(+), 8 deletions(-) diff --git a/dbms/src/Storages/Transaction/RegionCFDataBase.cpp b/dbms/src/Storages/Transaction/RegionCFDataBase.cpp index e40fb4b8984..fa7b07cd420 100644 --- a/dbms/src/Storages/Transaction/RegionCFDataBase.cpp +++ b/dbms/src/Storages/Transaction/RegionCFDataBase.cpp @@ -16,6 +16,7 @@ #include #include #include +#include namespace DB { @@ -54,15 +55,19 @@ RegionDataRes RegionCFDataBase::insert(TiKVKey && key, TiKVValue && value template <> RegionDataRes RegionCFDataBase::insert(TiKVKey && key, TiKVValue && value) { + auto allocator = data.get_allocator(); + auto old_size = allocator.getMemoryAllocatedSize(); Pair kv_pair = RegionLockCFDataTrait::genKVPair(std::move(key), std::move(value)); // according to the process of pessimistic lock, just overwrite. auto [it, ok] = data.insert_or_assign(std::move(kv_pair.first), std::move(kv_pair.second)); if (ok) { - return calcTiKVKeyValueSize(it->second); + auto new_size = allocator.getMemoryAllocatedSize(); + return new_size - old_size + calcTiKVKeyValueSize(it->second); } else { + LOG_FMT_DEBUG(&Poco::Logger::get(__PRETTY_FUNCTION__), "RegionCFDataBase override lock"); return 0; } } @@ -87,12 +92,13 @@ size_t RegionCFDataBase::calcTiKVKeyValueSize(const Value & value) template size_t RegionCFDataBase::calcTiKVKeyValueSize(const TiKVKey & key, const TiKVValue & value) { - if constexpr (std::is_same::value) - { - return key.dataSize() + value.dataSize(); - } - else - return key.dataSize() + value.dataSize(); +// if constexpr (std::is_same::value) +// { +// return key.dataSize() + value.dataSize(); +// } +// else +// return key.dataSize() + value.dataSize(); + return key.dataSize() + value.dataSize(); } @@ -121,7 +127,20 @@ size_t RegionCFDataBase::remove(const Key & key, bool quiet) return 0; size_t size = calcTiKVKeyValueSize(value); + size_t old_map_size = 0; + size_t new_map_size = 0; + if constexpr (std::is_same::value) + { + old_map_size = map.get_allocator().getMemoryAllocatedSize(); + } map.erase(it); + if constexpr (std::is_same::value) + { + new_map_size = map.get_allocator().getMemoryAllocatedSize(); + assert(new_map_size <= old_map_size); + size += old_map_size - new_map_size; + } + return size; } else if (!quiet) diff --git a/dbms/src/Storages/Transaction/RegionCFDataTrait.h b/dbms/src/Storages/Transaction/RegionCFDataTrait.h index 6f3b07a8c2c..0fbd6922a22 100644 --- a/dbms/src/Storages/Transaction/RegionCFDataTrait.h +++ b/dbms/src/Storages/Transaction/RegionCFDataTrait.h @@ -104,6 +104,36 @@ struct RegionDefaultCFDataTrait static std::shared_ptr getTiKVValue(const Map::const_iterator & it) { return std::get<1>(it->second); } }; +template +class MyAlloc { +public: + typedef T value_type; + + MyAlloc() =default; + + T* allocate (std::size_t num) + { + memory_allocated += num * sizeof(T); + return static_cast(::operator new(num*sizeof(T))); + } + void deallocate (T* p, std::size_t num) + { + memory_allocated -= num * sizeof(T); + ::operator delete(p); + } + + size_t getMemoryAllocatedSize() noexcept { return memory_allocated; } + +private: + size_t memory_allocated = 0; +}; + +template +bool operator== (const MyAlloc&, const MyAlloc&) noexcept { return false; } + +template +bool operator!= (const MyAlloc&, const MyAlloc&) noexcept { return true; } + struct RegionLockCFDataTrait { struct Key @@ -118,7 +148,7 @@ struct RegionLockCFDataTrait }; using DecodedLockCFValue = RecordKVFormat::DecodedLockCFValue; using Value = std::tuple, std::shared_ptr, std::shared_ptr>; - using Map = std::unordered_map; + using Map = std::unordered_map, MyAlloc>>; static Map::value_type genKVPair(TiKVKey && key_, TiKVValue && value_) { From ad481bef988a35a193737ac03dbdf98e9afa81f0 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Thu, 5 May 2022 15:05:32 +0800 Subject: [PATCH 27/29] fix build --- dbms/src/Storages/Transaction/RegionCFDataTrait.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dbms/src/Storages/Transaction/RegionCFDataTrait.h b/dbms/src/Storages/Transaction/RegionCFDataTrait.h index 0fbd6922a22..8413603f1a6 100644 --- a/dbms/src/Storages/Transaction/RegionCFDataTrait.h +++ b/dbms/src/Storages/Transaction/RegionCFDataTrait.h @@ -110,6 +110,10 @@ class MyAlloc { typedef T value_type; MyAlloc() =default; + template + MyAlloc (const MyAlloc& other) noexcept { + memory_allocated = other.getMemoryAllocatedSize(); + } T* allocate (std::size_t num) { @@ -122,7 +126,7 @@ class MyAlloc { ::operator delete(p); } - size_t getMemoryAllocatedSize() noexcept { return memory_allocated; } + size_t getMemoryAllocatedSize() const noexcept { return memory_allocated; } private: size_t memory_allocated = 0; From 5e8e85260e66b9a3e90c4096aebf101a83152baa Mon Sep 17 00:00:00 2001 From: lidezhu Date: Thu, 5 May 2022 15:56:08 +0800 Subject: [PATCH 28/29] try not decode --- dbms/src/Storages/Transaction/RegionCFDataTrait.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/Transaction/RegionCFDataTrait.h b/dbms/src/Storages/Transaction/RegionCFDataTrait.h index 8413603f1a6..3df133b1c95 100644 --- a/dbms/src/Storages/Transaction/RegionCFDataTrait.h +++ b/dbms/src/Storages/Transaction/RegionCFDataTrait.h @@ -159,7 +159,7 @@ struct RegionLockCFDataTrait auto key = std::make_shared(std::move(key_)); auto value = std::make_shared(std::move(value_)); return {{key, std::string_view(key->data(), key->dataSize())}, - Value{key, value, std::make_shared(key, value)}}; + Value{key, value, nullptr}}; } }; From 00f7a8dd28921884f5605088b158a62e5bbf1686 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Thu, 5 May 2022 17:14:10 +0800 Subject: [PATCH 29/29] add more log --- dbms/src/Storages/Transaction/RegionCFDataBase.cpp | 2 ++ dbms/src/Storages/Transaction/RegionData.cpp | 2 ++ 2 files changed, 4 insertions(+) diff --git a/dbms/src/Storages/Transaction/RegionCFDataBase.cpp b/dbms/src/Storages/Transaction/RegionCFDataBase.cpp index fa7b07cd420..f6c4a7d4e57 100644 --- a/dbms/src/Storages/Transaction/RegionCFDataBase.cpp +++ b/dbms/src/Storages/Transaction/RegionCFDataBase.cpp @@ -60,6 +60,8 @@ RegionDataRes RegionCFDataBase::insert(TiKVKey && key, Ti Pair kv_pair = RegionLockCFDataTrait::genKVPair(std::move(key), std::move(value)); // according to the process of pessimistic lock, just overwrite. auto [it, ok] = data.insert_or_assign(std::move(kv_pair.first), std::move(kv_pair.second)); + LOG_FMT_DEBUG(&Poco::Logger::get(__PRETTY_FUNCTION__), "Lock cf size {} bucket count {} allocator size {} entry size {}", + data.size(), data.bucket_count(), allocator.getMemoryAllocatedSize(), sizeof(RegionLockCFDataTrait::Map::value_type)); if (ok) { auto new_size = allocator.getMemoryAllocatedSize(); diff --git a/dbms/src/Storages/Transaction/RegionData.cpp b/dbms/src/Storages/Transaction/RegionData.cpp index 44ca5e7d3cc..fad358d9d2d 100644 --- a/dbms/src/Storages/Transaction/RegionData.cpp +++ b/dbms/src/Storages/Transaction/RegionData.cpp @@ -18,6 +18,7 @@ #include #include #include +#include namespace DB { @@ -45,6 +46,7 @@ void RegionData::insert(ColumnFamilyType cf, TiKVKey && key, TiKVValue && value) case ColumnFamilyType::Default: { cf_data_size += default_cf.insert(std::move(key), std::move(value)); + LOG_FMT_DEBUG(&Poco::Logger::get(__PRETTY_FUNCTION__), "default cf size {}", default_cf.getSize()); return; } case ColumnFamilyType::Lock: