1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
|
# frozen_string_literal: true
module LooseForeignKeys
class BatchCleanerService
CLEANUP_ATTEMPTS_BEFORE_RESCHEDULE = 3
CONSUME_AFTER_RESCHEDULE = 5.minutes
def initialize(parent_table:, loose_foreign_key_definitions:, deleted_parent_records:, modification_tracker: LooseForeignKeys::ModificationTracker.new)
@parent_table = parent_table
@loose_foreign_key_definitions = loose_foreign_key_definitions
@deleted_parent_records = deleted_parent_records
@modification_tracker = modification_tracker
@deleted_records_counter = Gitlab::Metrics.counter(
:loose_foreign_key_processed_deleted_records,
'The number of processed loose foreign key deleted records'
)
@deleted_records_rescheduled_count = Gitlab::Metrics.counter(
:loose_foreign_key_rescheduled_deleted_records,
'The number of rescheduled loose foreign key deleted records'
)
@deleted_records_incremented_count = Gitlab::Metrics.counter(
:loose_foreign_key_incremented_deleted_records,
'The number of loose foreign key deleted records with incremented cleanup_attempts'
)
end
def execute
loose_foreign_key_definitions.each do |loose_foreign_key_definition|
run_cleaner_service(loose_foreign_key_definition, with_skip_locked: true)
if modification_tracker.over_limit?
handle_over_limit
break
end
run_cleaner_service(loose_foreign_key_definition, with_skip_locked: false)
if modification_tracker.over_limit?
handle_over_limit
break
end
end
return if modification_tracker.over_limit?
# At this point, all associations are cleaned up, we can update the status of the parent records
update_count = LooseForeignKeys::DeletedRecord.mark_records_processed(deleted_parent_records)
deleted_records_counter.increment({ table: parent_table, db_config_name: db_config_name }, update_count)
end
private
attr_reader :parent_table, :loose_foreign_key_definitions, :deleted_parent_records, :modification_tracker, :deleted_records_counter, :deleted_records_rescheduled_count, :deleted_records_incremented_count
def handle_over_limit
return if Feature.disabled?(:lfk_fair_queueing)
records_to_reschedule = []
records_to_increment = []
deleted_parent_records.each do |deleted_record|
if deleted_record.cleanup_attempts >= CLEANUP_ATTEMPTS_BEFORE_RESCHEDULE
records_to_reschedule << deleted_record
else
records_to_increment << deleted_record
end
end
reschedule_count = LooseForeignKeys::DeletedRecord.reschedule(records_to_reschedule, CONSUME_AFTER_RESCHEDULE.from_now)
deleted_records_rescheduled_count.increment({ table: parent_table, db_config_name: db_config_name }, reschedule_count)
increment_count = LooseForeignKeys::DeletedRecord.increment_attempts(records_to_increment)
deleted_records_incremented_count.increment({ table: parent_table, db_config_name: db_config_name }, increment_count)
end
def record_result(cleaner, result)
if cleaner.async_delete?
modification_tracker.add_deletions(result[:table], result[:affected_rows])
elsif cleaner.async_nullify?
modification_tracker.add_updates(result[:table], result[:affected_rows])
end
end
def run_cleaner_service(loose_foreign_key_definition, with_skip_locked:)
base_models_for_gitlab_schema = Gitlab::Database.schemas_to_base_models.fetch(loose_foreign_key_definition.options[:gitlab_schema])
base_models_for_gitlab_schema.each do |base_model|
cleaner = CleanerService.new(
loose_foreign_key_definition: loose_foreign_key_definition,
connection: base_model.connection,
deleted_parent_records: deleted_parent_records,
with_skip_locked: with_skip_locked
)
loop do
result = cleaner.execute
record_result(cleaner, result)
break if modification_tracker.over_limit? || result[:affected_rows] == 0
end
end
end
def db_config_name
LooseForeignKeys::DeletedRecord.connection.pool.db_config.name
end
end
end
|
