Skip to content

Commit 9538ef4

Browse files
iuwqyiriuwqyir
authored
improve delete performance for clickhouse (#132)
### TL;DR Improved the deletion logic in case of reorgs in ClickHouse by implementing partition-aware deletes and disabling lightweight delete synchronization. ### What changed? - Added `lightweight_deletes_sync` setting (value 0) to ClickHouse configuration. This will make the deletes async, so processing does not need to wait for it to finish - Split generic `deleteBatch` function into specific delete functions for blocks, logs, transactions, and traces, which will improve performance due to having all of the order by keys defined - Implemented two-step deletion process: 1. Query for existing records that need deletion 2. Execute targeted deletes using partition values and primary keys - Added proper error handling and batch processing for deletions ### Why make this change? The previous deletion method was inefficient and could cause performance issues in ClickHouse.
2 parents bcaf1ca + 7e42585 commit 9538ef4

File tree

6 files changed

+265
-107
lines changed

6 files changed

+265
-107
lines changed

‎internal/common/block.go‎

Lines changed: 22 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -5,28 +5,28 @@ import (
55
)
66

77
type Block struct {
8-
ChainId *big.Int `json:"chain_id"`
9-
Number *big.Int `json:"number"`
10-
Hash string `json:"hash"`
11-
ParentHash string `json:"parent_hash"`
12-
Timestamp uint64 `json:"timestamp"`
13-
Nonce string `json:"nonce"`
14-
Sha3Uncles string `json:"sha3_uncles"`
15-
MixHash string `json:"mix_hash"`
16-
Miner string `json:"miner"`
17-
StateRoot string `json:"state_root"`
18-
TransactionsRoot string `json:"transactions_root"`
19-
ReceiptsRoot string `json:"receipts_root"`
20-
LogsBloom string `json:"logs_bloom"`
21-
Size uint64 `json:"size"`
22-
ExtraData string `json:"extra_data"`
23-
Difficulty *big.Int `json:"difficulty"`
24-
TotalDifficulty *big.Int `json:"total_difficulty"`
25-
TransactionCount uint64 `json:"transaction_count"`
26-
GasLimit *big.Int `json:"gas_limit"`
27-
GasUsed *big.Int `json:"gas_used"`
28-
WithdrawalsRoot string `json:"withdrawals_root"`
29-
BaseFeePerGas uint64 `json:"base_fee_per_gas"`
8+
ChainId *big.Int `json:"chain_id" ch:"chain_id"`
9+
Number *big.Int `json:"number" ch:"number"`
10+
Hash string `json:"hash" ch:"hash"`
11+
ParentHash string `json:"parent_hash" ch:"parent_hash"`
12+
Timestamp uint64 `json:"timestamp" ch:"timestamp"`
13+
Nonce string `json:"nonce" ch:"nonce"`
14+
Sha3Uncles string `json:"sha3_uncles" ch:"sha3_uncles"`
15+
MixHash string `json:"mix_hash" ch:"mix_hash"`
16+
Miner string `json:"miner" ch:"miner"`
17+
StateRoot string `json:"state_root" ch:"state_root"`
18+
TransactionsRoot string `json:"transactions_root" ch:"transactions_root"`
19+
ReceiptsRoot string `json:"receipts_root" ch:"receipts_root"`
20+
LogsBloom string `json:"logs_bloom" ch:"logs_bloom"`
21+
Size uint64 `json:"size" ch:"size"`
22+
ExtraData string `json:"extra_data" ch:"extra_data"`
23+
Difficulty *big.Int `json:"difficulty" ch:"difficulty"`
24+
TotalDifficulty *big.Int `json:"total_difficulty" ch:"total_difficulty"`
25+
TransactionCount uint64 `json:"transaction_count" ch:"transaction_count"`
26+
GasLimit *big.Int `json:"gas_limit" ch:"gas_limit"`
27+
GasUsed *big.Int `json:"gas_used" ch:"gas_used"`
28+
WithdrawalsRoot string `json:"withdrawals_root" ch:"withdrawals_root"`
29+
BaseFeePerGas uint64 `json:"base_fee_per_gas" ch:"base_fee_per_gas"`
3030
}
3131

3232
type BlockData struct {

‎internal/common/log.go‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -11,15 +11,15 @@ import (
1111
)
1212

1313
type Log struct {
14-
ChainId *big.Int `json:"chain_id" swaggertype:"string"`
15-
BlockNumber *big.Int `json:"block_number" swaggertype:"string"`
16-
BlockHash string `json:"block_hash"`
17-
BlockTimestamp uint64 `json:"block_timestamp"`
18-
TransactionHash string `json:"transaction_hash"`
19-
TransactionIndex uint64 `json:"transaction_index"`
20-
LogIndex uint64 `json:"log_index"`
21-
Address string `json:"address"`
22-
Data string `json:"data"`
14+
ChainId *big.Int `json:"chain_id" ch:"chain_id" swaggertype:"string"`
15+
BlockNumber *big.Int `json:"block_number" ch:"block_number" swaggertype:"string"`
16+
BlockHash string `json:"block_hash" ch:"block_hash"`
17+
BlockTimestamp uint64 `json:"block_timestamp" ch:"block_timestamp"`
18+
TransactionHash string `json:"transaction_hash" ch:"transaction_hash"`
19+
TransactionIndex uint64 `json:"transaction_index" ch:"transaction_index"`
20+
LogIndex uint64 `json:"log_index" ch:"log_index"`
21+
Address string `json:"address" ch:"address"`
22+
Data string `json:"data" ch:"data"`
2323
Topics []string `json:"topics"`
2424
}
2525

‎internal/common/set.go‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
package common
2+
3+
type Set[T comparable] struct {
4+
elements map[T]struct{}
5+
}
6+
7+
// NewSet creates a new set
8+
func NewSet[T comparable]() *Set[T] {
9+
return &Set[T]{
10+
elements: make(map[T]struct{}),
11+
}
12+
}
13+
14+
// Add inserts an element into the set
15+
func (s *Set[T]) Add(value T) {
16+
s.elements[value] = struct{}{}
17+
}
18+
19+
// Remove deletes an element from the set
20+
func (s *Set[T]) Remove(value T) {
21+
delete(s.elements, value)
22+
}
23+
24+
// Contains checks if an element is in the set
25+
func (s *Set[T]) Contains(value T) bool {
26+
_, found := s.elements[value]
27+
return found
28+
}
29+
30+
// Size returns the number of elements in the set
31+
func (s *Set[T]) Size() int {
32+
return len(s.elements)
33+
}
34+
35+
// List returns all elements in the set as a slice
36+
func (s *Set[T]) List() []T {
37+
keys := make([]T, 0, len(s.elements))
38+
for key := range s.elements {
39+
keys = append(keys, key)
40+
}
41+
return keys
42+
}

‎internal/common/trace.go‎

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -5,27 +5,27 @@ import (
55
)
66

77
type Trace struct {
8-
ChainID *big.Int `json:"chain_id"`
9-
BlockNumber *big.Int `json:"block_number"`
10-
BlockHash string `json:"block_hash"`
11-
BlockTimestamp uint64 `json:"block_timestamp"`
12-
TransactionHash string `json:"transaction_hash"`
13-
TransactionIndex uint64 `json:"transaction_index"`
14-
Subtraces int64 `json:"subtraces"`
15-
TraceAddress []uint64 `json:"trace_address"`
16-
TraceType string `json:"trace_type"`
17-
CallType string `json:"call_type"`
18-
Error string `json:"error"`
19-
FromAddress string `json:"from_address"`
20-
ToAddress string `json:"to_address"`
21-
Gas *big.Int `json:"gas"`
22-
GasUsed *big.Int `json:"gas_used"`
23-
Input string `json:"input"`
24-
Output string `json:"output"`
25-
Value *big.Int `json:"value"`
26-
Author string `json:"author"`
27-
RewardType string `json:"reward_type"`
28-
RefundAddress string `json:"refund_address"`
8+
ChainID *big.Int `json:"chain_id" ch:"chain_id"`
9+
BlockNumber *big.Int `json:"block_number" ch:"block_number"`
10+
BlockHash string `json:"block_hash" ch:"block_hash"`
11+
BlockTimestamp uint64 `json:"block_timestamp" ch:"block_timestamp"`
12+
TransactionHash string `json:"transaction_hash" ch:"transaction_hash"`
13+
TransactionIndex uint64 `json:"transaction_index" ch:"transaction_index"`
14+
Subtraces int64 `json:"subtraces" ch:"subtraces"`
15+
TraceAddress []uint64 `json:"trace_address" ch:"trace_address"`
16+
TraceType string `json:"trace_type" ch:"type"`
17+
CallType string `json:"call_type" ch:"call_type"`
18+
Error string `json:"error" ch:"error"`
19+
FromAddress string `json:"from_address" ch:"from_address"`
20+
ToAddress string `json:"to_address" ch:"to_address"`
21+
Gas *big.Int `json:"gas" ch:"gas"`
22+
GasUsed *big.Int `json:"gas_used" ch:"gas_used"`
23+
Input string `json:"input" ch:"input"`
24+
Output string `json:"output" ch:"output"`
25+
Value *big.Int `json:"value" ch:"value"`
26+
Author string `json:"author" ch:"author"`
27+
RewardType string `json:"reward_type" ch:"reward_type"`
28+
RefundAddress string `json:"refund_address" ch:"refund_address"`
2929
}
3030

3131
type RawTraces = []map[string]interface{}

‎internal/common/transaction.go‎

Lines changed: 29 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -10,35 +10,35 @@ import (
1010
)
1111

1212
type Transaction struct {
13-
ChainId *big.Int `json:"chain_id" swaggertype:"string"`
14-
Hash string `json:"hash"`
15-
Nonce uint64 `json:"nonce"`
16-
BlockHash string `json:"block_hash"`
17-
BlockNumber *big.Int `json:"block_number" swaggertype:"string"`
18-
BlockTimestamp uint64 `json:"block_timestamp"`
19-
TransactionIndex uint64 `json:"transaction_index"`
20-
FromAddress string `json:"from_address"`
21-
ToAddress string `json:"to_address"`
22-
Value *big.Int `json:"value" swaggertype:"string"`
23-
Gas uint64 `json:"gas"`
24-
GasPrice *big.Int `json:"gas_price" swaggertype:"string"`
25-
Data string `json:"data"`
26-
FunctionSelector string `json:"function_selector"`
27-
MaxFeePerGas *big.Int `json:"max_fee_per_gas" swaggertype:"string"`
28-
MaxPriorityFeePerGas *big.Int `json:"max_priority_fee_per_gas" swaggertype:"string"`
29-
TransactionType uint8 `json:"transaction_type"`
30-
R *big.Int `json:"r" swaggertype:"string"`
31-
S *big.Int `json:"s" swaggertype:"string"`
32-
V *big.Int `json:"v" swaggertype:"string"`
33-
AccessListJson *string `json:"access_list_json"`
34-
ContractAddress *string `json:"contract_address"`
35-
GasUsed *uint64 `json:"gas_used"`
36-
CumulativeGasUsed *uint64 `json:"cumulative_gas_used"`
37-
EffectiveGasPrice *big.Int `json:"effective_gas_price" swaggertype:"string"`
38-
BlobGasUsed *uint64 `json:"blob_gas_used"`
39-
BlobGasPrice *big.Int `json:"blob_gas_price" swaggertype:"string"`
40-
LogsBloom *string `json:"logs_bloom"`
41-
Status *uint64 `json:"status"`
13+
ChainId *big.Int `json:"chain_id" ch:"chain_id" swaggertype:"string"`
14+
Hash string `json:"hash" ch:"hash"`
15+
Nonce uint64 `json:"nonce" ch:"nonce"`
16+
BlockHash string `json:"block_hash" ch:"block_hash"`
17+
BlockNumber *big.Int `json:"block_number" ch:"block_number" swaggertype:"string"`
18+
BlockTimestamp uint64 `json:"block_timestamp" ch:"block_timestamp"`
19+
TransactionIndex uint64 `json:"transaction_index" ch:"transaction_index"`
20+
FromAddress string `json:"from_address" ch:"from_address"`
21+
ToAddress string `json:"to_address" ch:"to_address"`
22+
Value *big.Int `json:"value" ch:"value" swaggertype:"string"`
23+
Gas uint64 `json:"gas" ch:"gas"`
24+
GasPrice *big.Int `json:"gas_price" ch:"gas_price" swaggertype:"string"`
25+
Data string `json:"data" ch:"data"`
26+
FunctionSelector string `json:"function_selector" ch:"function_selector"`
27+
MaxFeePerGas *big.Int `json:"max_fee_per_gas" ch:"max_fee_per_gas" swaggertype:"string"`
28+
MaxPriorityFeePerGas *big.Int `json:"max_priority_fee_per_gas" ch:"max_priority_fee_per_gas" swaggertype:"string"`
29+
TransactionType uint8 `json:"transaction_type" ch:"transaction_type"`
30+
R *big.Int `json:"r" ch:"r" swaggertype:"string"`
31+
S *big.Int `json:"s" ch:"s" swaggertype:"string"`
32+
V *big.Int `json:"v" ch:"v" swaggertype:"string"`
33+
AccessListJson *string `json:"access_list_json" ch:"access_list_json"`
34+
ContractAddress *string `json:"contract_address" ch:"contract_address"`
35+
GasUsed *uint64 `json:"gas_used" ch:"gas_used"`
36+
CumulativeGasUsed *uint64 `json:"cumulative_gas_used" ch:"cumulative_gas_used"`
37+
EffectiveGasPrice *big.Int `json:"effective_gas_price" ch:"effective_gas_price" swaggertype:"string"`
38+
BlobGasUsed *uint64 `json:"blob_gas_used" ch:"blob_gas_used"`
39+
BlobGasPrice *big.Int `json:"blob_gas_price" ch:"blob_gas_price" swaggertype:"string"`
40+
LogsBloom *string `json:"logs_bloom" ch:"logs_bloom"`
41+
Status *uint64 `json:"status" ch:"status"`
4242
}
4343

4444
type DecodedTransactionData struct {

0 commit comments

Comments
 (0)