Skip to content

[fix](recycler) Add recycle state for rs meta to avoid data loss #58459

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

[fix](recycler) Add recycle state for rs meta to avoid data loss #58459

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions be/src/cloud/pb_convert.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,9 @@ void doris_rowset_meta_to_cloud(RowsetMetaCloudPB* out, const RowsetMetaPB& in)
if (in.has_reference_instance_id()) {
out->set_reference_instance_id(in.reference_instance_id());
}
if (in.has_recycled_marked()) {
out->set_recycled_marked(in.recycled_marked());
}
}

void doris_rowset_meta_to_cloud(RowsetMetaCloudPB* out, RowsetMetaPB&& in) {
Expand Down Expand Up @@ -171,6 +174,9 @@ void doris_rowset_meta_to_cloud(RowsetMetaCloudPB* out, RowsetMetaPB&& in) {
if (in.has_reference_instance_id()) {
out->set_reference_instance_id(in.reference_instance_id());
}
if (in.has_recycled_marked()) {
out->set_recycled_marked(in.recycled_marked());
}
}

RowsetMetaPB cloud_rowset_meta_to_doris(const RowsetMetaCloudPB& in) {
Expand Down Expand Up @@ -250,6 +256,9 @@ void cloud_rowset_meta_to_doris(RowsetMetaPB* out, const RowsetMetaCloudPB& in)
if (in.has_reference_instance_id()) {
out->set_reference_instance_id(in.reference_instance_id());
}
if (in.has_recycled_marked()) {
out->set_recycled_marked(in.recycled_marked());
}
}

void cloud_rowset_meta_to_doris(RowsetMetaPB* out, RowsetMetaCloudPB&& in) {
Expand Down Expand Up @@ -318,6 +327,9 @@ void cloud_rowset_meta_to_doris(RowsetMetaPB* out, RowsetMetaCloudPB&& in) {
if (in.has_reference_instance_id()) {
out->set_reference_instance_id(in.reference_instance_id());
}
if (in.has_recycled_marked()) {
out->set_recycled_marked(in.recycled_marked());
}
}

TabletSchemaCloudPB doris_tablet_schema_to_cloud(const TabletSchemaPB& in) {
Expand Down
2 changes: 2 additions & 0 deletions cloud/src/common/config.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -253,6 +253,8 @@ CONF_mBool(enable_load_txn_status_check, "true");

CONF_mBool(enable_tablet_job_check, "true");

CONF_mBool(enable_recycle_rowset_key_check, "true");

// Declare a selection strategy for those servers have many ips.
// Note that there should at most one ip match this list.
// this is a list in semicolon-delimited format, in CIDR notation,
Expand Down
58 changes: 34 additions & 24 deletions cloud/src/meta-service/meta_service.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2567,29 +2567,8 @@ void MetaServiceImpl::commit_rowset(::google::protobuf::RpcController* controlle
return;
}

// Check if the compaction/sc tablet job has finished
// // Check if the compaction/sc tablet job has finished
bool is_versioned_read = is_version_read_enabled(instance_id);
if (config::enable_tablet_job_check && request->has_tablet_job_id() &&
!request->tablet_job_id().empty()) {
if (!check_job_existed(txn.get(), code, msg, instance_id, tablet_id, rowset_id,
request->tablet_job_id(), is_versioned_read, resource_mgr_.get())) {
return;
}
}

// Check if the commit rowset request is invalid.
// If the transaction has been finished, it means this commit rowset is a timeout retry request.
// In this case, do not write the recycle key again, otherwise it may cause data loss.
// If the rowset has load id, it means it is a load request, otherwise it is a
// compaction/sc request.
if (config::enable_load_txn_status_check && rowset_meta.has_load_id() &&
!check_transaction_status(TxnStatusPB::TXN_STATUS_PREPARED, txn.get(), instance_id,
rowset_meta.txn_id(), code, msg)) {
LOG(WARNING) << "commit rowset failed, txn_id=" << rowset_meta.txn_id()
<< ", tablet_id=" << tablet_id << ", rowset_id=" << rowset_id
<< ", rowset_state=" << rowset_meta.rowset_state() << ", msg=" << msg;
return;
}

// Check if commit key already exists.
std::string existed_commit_val;
Expand Down Expand Up @@ -2697,6 +2676,39 @@ void MetaServiceImpl::commit_rowset(::google::protobuf::RpcController* controlle
rowset_meta.set_allocated_tablet_schema(nullptr);
}

auto recycle_rs_key = recycle_rowset_key({instance_id, tablet_id, rowset_id});

if (config::enable_recycle_rowset_key_check) {
std::string recycle_rs_val;
err = txn->get(recycle_rs_key, &recycle_rs_val);
if (err == TxnErrorCode::TXN_OK) {
RecycleRowsetPB recycle_rowset;
if (!recycle_rowset.ParseFromString(recycle_rs_val)) {
code = MetaServiceCode::PROTOBUF_PARSE_ERR;
msg = fmt::format("malformed recycle rowset value. key={}", hex(recycle_rs_key));
return;
}
auto rs_meta = recycle_rowset.rowset_meta();
if (rs_meta.has_recycled_marked() && rs_meta.recycled_marked()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = fmt::format("rowset has already been marked as recycled, key={}, rs_meta={}",
hex(recycle_rs_key), rs_meta.ShortDebugString());
LOG(WARNING) << msg;
return;
}
} else if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = fmt::format("recycle rowset key not found, key={}", hex(recycle_rs_key));
LOG(WARNING) << msg;
return;
} else {
code = cast_as<ErrCategory::READ>(err);
msg = fmt::format("failed to get recycle rowset, err={}", err);
LOG(WARNING) << msg;
return;
}
}

if (is_version_write_enabled(instance_id)) {
std::string rowset_ref_count_key =
versioned::data_rowset_ref_count_key({instance_id, tablet_id, rowset_id});
Expand All @@ -2705,9 +2717,7 @@ void MetaServiceImpl::commit_rowset(::google::protobuf::RpcController* controlle
txn->atomic_add(rowset_ref_count_key, 1);
}

auto recycle_rs_key = recycle_rowset_key({instance_id, tablet_id, rowset_id});
txn->remove(recycle_rs_key);

DCHECK_GT(rowset_meta.txn_expiration(), 0);
auto tmp_rs_val = rowset_meta.SerializeAsString();
txn->put(tmp_rs_key, tmp_rs_val);
Expand Down
17 changes: 17 additions & 0 deletions cloud/src/meta-service/meta_service_job.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1215,6 +1215,14 @@ void process_compaction_job(MetaServiceCode& code, std::string& msg, std::string
return;
}

if (rs_meta.has_recycled_marked() && rs_meta.recycled_marked()) {
SS << "rowset has already been marked as recycled, tablet_id=" << tablet_id
<< " txn_id=" << rs_meta.txn_id();
msg = ss.str();
code = MetaServiceCode::TXN_ALREADY_ABORTED;
return;
}

txn->remove(tmp_rowset_key);
INSTANCE_LOG(INFO) << "remove tmp rowset meta, tablet_id=" << tablet_id
<< " tmp_rowset_key=" << hex(tmp_rowset_key);
Expand Down Expand Up @@ -1805,6 +1813,15 @@ void process_schema_change_job(MetaServiceCode& code, std::string& msg, std::str
msg = ss.str();
return;
}

if (tmp_rowset_meta.has_recycled_marked() && tmp_rowset_meta.recycled_marked()) {
SS << "rowset has already been marked as recycled, tablet_id=" << new_tablet_id
<< " txn_id=" << tmp_rowset_meta.txn_id();
msg = ss.str();
code = MetaServiceCode::TXN_ALREADY_ABORTED;
return;
}

using namespace std::chrono;
auto rowset_visible_time =
duration_cast<milliseconds>(system_clock::now().time_since_epoch()).count();
Expand Down
15 changes: 10 additions & 5 deletions cloud/src/meta-service/meta_service_txn.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1153,17 +1153,22 @@ void scan_tmp_rowset(
while (it->has_next()) {
auto [k, v] = it->next();
LOG(INFO) << "range_get rowset_tmp_key=" << hex(k) << " txn_id=" << txn_id;
tmp_rowsets_meta->emplace_back();
if (!tmp_rowsets_meta->back().second.ParseFromArray(v.data(), v.size())) {
RowsetMetaCloudPB rs_meta;
if (!rs_meta.ParseFromArray(v.data(), v.size())) {
code = MetaServiceCode::PROTOBUF_PARSE_ERR;
ss << "malformed rowset meta, unable to initialize, txn_id=" << txn_id
<< " key=" << hex(k);
<< " key=" << hex(k) << " err=" << err;
msg = ss.str();
LOG(WARNING) << msg;
return;
}
// Save keys that will be removed later
tmp_rowsets_meta->back().first = std::string(k.data(), k.size());
if (rs_meta.has_recycled_marked() && rs_meta.recycled_marked()) {
code = MetaServiceCode::TXN_ALREADY_ABORTED;
msg = "rowset has already been marked as recycled";
LOG(WARNING) << msg;
continue;
}
tmp_rowsets_meta->emplace_back(std::string(k.data(), k.size()), std::move(rs_meta));
++num_rowsets;
if (!it->has_next()) rs_tmp_key0 = k;
}
Expand Down
65 changes: 54 additions & 11 deletions cloud/src/recycler/recycler.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1105,6 +1105,39 @@ int64_t calculate_restore_job_expired_time(
return final_expiration;
}

int mark_rowset_as_recycled(TxnKv* txn_kv, const std::string& instance_id, std::string_view key,
Copy link
Contributor

@gavinchou gavinchou Dec 3, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

add comment for behavior param and return values

RowsetMetaCloudPB& rs_meta) {
bool need_write_back = false;
if (!rs_meta.has_recycled_marked()) {
rs_meta.set_recycled_marked(false);
need_write_back = true;
} else if (!rs_meta.recycled_marked()) {
rs_meta.set_recycled_marked(true);
need_write_back = true;
}

if (need_write_back) {
std::unique_ptr<Transaction> txn;
TxnErrorCode err = txn_kv->create_txn(&txn);
if (err != TxnErrorCode::TXN_OK) {
LOG(WARNING) << "failed to create txn, instance_id=" << instance_id;
return -1;
}
std::string val;
if (!rs_meta.SerializeToString(&val)) {
LOG(WARNING) << "failed to serialize rs_meta, instance_id=" << instance_id;
return -1;
}
txn->put(key, val);
err = txn->commit();
if (err != TxnErrorCode::TXN_OK) {
LOG(WARNING) << "failed to commit txn, instance_id=" << instance_id;
return -1;
}
}
return need_write_back ? 1 : 0;
}

int InstanceRecycler::recycle_indexes() {
const std::string task_name = "recycle_indexes";
int64_t num_scanned = 0;
Expand Down Expand Up @@ -3072,6 +3105,7 @@ int InstanceRecycler::recycle_rowsets() {
return -1;
}

auto* rowset_meta = rowset.mutable_rowset_meta();
int64_t current_time = ::time(nullptr);
int64_t expiration = calculate_rowset_expired_time(instance_id_, rowset, &earlest_ts);

Expand All @@ -3083,6 +3117,18 @@ int InstanceRecycler::recycle_rowsets() {
}
++num_expired;
expired_rowset_size += v.size();

int mark_ret = mark_rowset_as_recycled(txn_kv_.get(), instance_id_, k, *rowset_meta);
if (mark_ret == -1) {
LOG(WARNING) << "failed to mark rowset as recycled, instance_id=" << instance_id_
<< " key=" << hex(k);
return -1;
} else if (mark_ret == 1) {
LOG(INFO) << "rowset already marked as recycled, instance_id=" << instance_id_
<< " key=" << hex(k);
return 0;
}

if (!rowset.has_type()) { // old version `RecycleRowsetPB`
if (!rowset.has_resource_id()) [[unlikely]] { // impossible
// in old version, keep this key-value pair and it needs to be checked manually
Expand Down Expand Up @@ -3120,7 +3166,6 @@ int InstanceRecycler::recycle_rowsets() {
return 0;
}
// TODO(plat1ko): check rowset not referenced
auto rowset_meta = rowset.mutable_rowset_meta();
if (!rowset_meta->has_resource_id()) [[unlikely]] { // impossible
if (rowset.type() != RecycleRowsetPB::PREPARE && rowset_meta->num_segments() == 0) {
LOG_INFO("recycle rowset that has empty resource id");
Expand Down Expand Up @@ -3795,16 +3840,14 @@ int InstanceRecycler::recycle_tmp_rowsets() {
return 0;
}

DCHECK_GT(rowset.txn_id(), 0)
<< "txn_id=" << rowset.txn_id() << " rowset=" << rowset.ShortDebugString();
if (!is_txn_finished(txn_kv_, instance_id_, rowset.txn_id())) {
LOG(INFO) << "txn is not finished, skip recycle tmp rowset, instance_id="
<< instance_id_ << " tablet_id=" << rowset.tablet_id()
<< " rowset_id=" << rowset.rowset_id_v2() << " version=["
<< rowset.start_version() << '-' << rowset.end_version()
<< "] txn_id=" << rowset.txn_id()
<< " creation_time=" << rowset.creation_time() << " expiration=" << expiration
<< " txn_expiration=" << rowset.txn_expiration();
int mark_ret = mark_rowset_as_recycled(txn_kv_.get(), instance_id_, k, rowset);
if (mark_ret == -1) {
LOG(WARNING) << "failed to mark rowset as recycled, instance_id=" << instance_id_
<< " key=" << hex(k);
return -1;
} else if (mark_ret == 1) {
LOG(INFO) << "rowset already marked as recycled, instance_id=" << instance_id_
<< " key=" << hex(k);
return 0;
}

Expand Down
2 changes: 1 addition & 1 deletion cloud/test/meta_service_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10592,7 +10592,7 @@ TEST(MetaServiceTest, StaleCommitRowset) {

commit_txn(meta_service.get(), db_id, txn_id, label);
ASSERT_NO_FATAL_FAILURE(commit_rowset(meta_service.get(), rowset, res));
ASSERT_TRUE(res.status().msg().find("txn is not in") != std::string::npos)
ASSERT_TRUE(res.status().msg().find("key not found") != std::string::npos)
<< res.status().msg();
ASSERT_EQ(res.status().code(), MetaServiceCode::INVALID_ARGUMENT) << res.status().code();
}
Expand Down
2 changes: 2 additions & 0 deletions gensrc/proto/olap_file.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,7 @@ message RowsetMetaPB {

optional int64 visible_ts_ms = 1010;
optional string reference_instance_id = 1011; // For cluster snapshot
optional bool recycled_marked = 1012;
}

message SchemaDictKeyList {
Expand Down Expand Up @@ -249,6 +250,7 @@ message RowsetMetaCloudPB {

optional int64 visible_ts_ms = 109;
optional string reference_instance_id = 110; // For cluster snapshot
optional bool recycled_marked = 111;
}

message SegmentStatisticsPB {
Expand Down