Skip to content

KVStore: reduce log #10813

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ti-chi-bot merged 2 commits into from
Apr 24, 2026
+26 −19
Merged

KVStore: reduce log #10813

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8e26c69
a
CalvinNeo Apr 24, 2026
41c3060
reformat
CalvinNeo Apr 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions dbms/src/Storages/KVStore/KVStore.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@

#pragma once

#include <string_view>

#include <Interpreters/Context_fwd.h>
#include <Parsers/IAST_fwd.h>
#include <Storages/DeltaMerge/DeltaMergeInterfaces.h>
Expand Down Expand Up @@ -372,13 +374,14 @@ class KVStore final : private boost::noncopyable
TMTContext & tmt,
const RegionTaskLock & region_task_lock,
UInt64 index,
UInt64 term) const;
UInt64 term,
std::string_view persist_extra_msg) const;

void persistRegion(
const Region & region,
const RegionTaskLock & region_task_lock,
PersistRegionReason reason,
const char * extra_msg) const;
std::string_view extra_msg) const;

bool tryRegisterEagerRaftLogGCTask(const RegionPtr & region, RegionTaskLock &);

Expand Down
35 changes: 20 additions & 15 deletions dbms/src/Storages/KVStore/MultiRaft/Persistence.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,15 +28,17 @@ void KVStore::persistRegion(
const Region & region,
const RegionTaskLock & region_task_lock,
PersistRegionReason reason,
const char * extra_msg) const
std::string_view extra_msg) const
Comment thread
JaySon-Huang marked this conversation as resolved.
{
RUNTIME_CHECK_MSG(
region_persister,
"try access to region_persister without initialization, stack={}",
StackTrace().toString());

auto reason_id = magic_enum::enum_underlying(reason);
std::string caller = fmt::format("{} {}", PersistRegionReasonMap[reason_id], extra_msg);
std::string caller = fmt::format("{}", PersistRegionReasonMap[reason_id]);
if (!extra_msg.empty())
caller = fmt::format("{} {}", caller, extra_msg);
LOG_INFO(
log,
"Start to persist {}, cache size: {} bytes for `{}`",
Expand Down Expand Up @@ -130,13 +132,13 @@ bool KVStore::tryFlushRegionData(

// force persist
auto & curr_region = *curr_region_ptr;
const auto force_persist_msg = fmt::format("by force, term={} index={}", term, index);
LOG_DEBUG(
log,
"flush region due to tryFlushRegionData by force, region_id={} term={} index={}",
curr_region.id(),
term,
index);
if (!forceFlushRegionDataImpl(curr_region, try_until_succeed, tmt, region_task_lock, index, term))
"{} flush region due to tryFlushRegionData {}",
curr_region.toString(false),
force_persist_msg);
if (!forceFlushRegionDataImpl(curr_region, try_until_succeed, tmt, region_task_lock, index, term, force_persist_msg))
{
throw Exception(ErrorCodes::LOGICAL_ERROR, "Force flush region failed, region_id={}", region_id);
}
Expand Down Expand Up @@ -215,11 +217,8 @@ bool KVStore::canFlushRegionDataImpl(
if (can_flush && flush_if_possible)
{
// This rarely happens when there are too may raft logs, which don't trigger a proactive flush.
LOG_INFO(
log,
"{} flush region due to tryFlushRegionData, index {} term {} truncated_index {} truncated_term {}"
" gap {}/{} table_in_mem_size={} table_id={} keyspace={}",
curr_region.toString(false),
const auto flush_msg = fmt::format(
"index {} term {} truncated_index {} truncated_term {} gap {}/{} table_in_mem_size={} table_id={} keyspace={}",
index,
term,
truncated_index,
Expand All @@ -229,8 +228,13 @@ bool KVStore::canFlushRegionDataImpl(
curr_region_ptr->getRegionTableSize(),
curr_region_ptr->getMappedTableID(),
curr_region_ptr->getKeyspaceID());
LOG_DEBUG(
log,
"{} flush region due to tryFlushRegionData, {}",
curr_region.toString(false),
flush_msg);
GET_METRIC(tiflash_raft_region_flush_bytes, type_flushed).Observe(size_bytes);
return forceFlushRegionDataImpl(curr_region, try_until_succeed, tmt, region_task_lock, index, term);
return forceFlushRegionDataImpl(curr_region, try_until_succeed, tmt, region_task_lock, index, term, flush_msg);
}
else
{
Expand All @@ -246,7 +250,8 @@ bool KVStore::forceFlushRegionDataImpl(
TMTContext & tmt,
const RegionTaskLock & region_task_lock,
UInt64 index,
UInt64 term) const
UInt64 term,
std::string_view persist_extra_msg) const
{
Stopwatch watch;
if (index)
Expand All @@ -261,7 +266,7 @@ bool KVStore::forceFlushRegionDataImpl(
}

// flush cache in storage level is done, persist the region info
persistRegion(curr_region, region_task_lock, PersistRegionReason::Flush, "");
persistRegion(curr_region, region_task_lock, PersistRegionReason::Flush, persist_extra_msg);
// CompactLog will be done in proxy soon, we advance the eager truncate index in TiFlash
curr_region.updateRaftLogEagerIndex(index);
curr_region.cleanApproxMemCacheInfo();
Expand Down