1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
|
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe ClickHouse::EventsSyncWorker, feature_category: :value_stream_management do
let(:worker) { described_class.new }
specify do
expect(worker.class.click_house_worker_attrs).to match(
a_hash_including(migration_lock_ttl: ClickHouse::MigrationSupport::ExclusiveLock::DEFAULT_CLICKHOUSE_WORKER_TTL)
)
end
it_behaves_like 'an idempotent worker' do
context 'when the event_sync_worker_for_click_house feature flag is on', :click_house do
before do
stub_feature_flags(event_sync_worker_for_click_house: true)
end
context 'when there is nothing to sync' do
it 'adds metadata for the worker' do
expect(worker).to receive(:log_extra_metadata_on_done).with(:result,
{ status: :processed, records_inserted: 0, reached_end_of_table: true })
worker.perform
events = ClickHouse::Client.select('SELECT * FROM events', :main)
expect(events).to be_empty
end
end
context 'when syncing records' do
let_it_be(:group) { create(:group) }
let_it_be(:project) { create(:project, group: group) }
let_it_be(:issue) { create(:issue, project: project) }
let_it_be(:project_event2) { create(:event, :closed, project: project, target: issue) }
let_it_be(:event_without_parent) { create(:event, :joined, project: nil, group: nil) }
let_it_be(:group_event) { create(:event, :created, group: group, project: nil) }
let_it_be(:project_event1) { create(:event, :created, project: project, target: issue) }
# looks invalid but we have some records like this on PRD
it 'inserts all records' do
expect(worker).to receive(:log_extra_metadata_on_done).with(:result,
{ status: :processed, records_inserted: 4, reached_end_of_table: true })
worker.perform
expected_records = [
hash_including('id' => project_event2.id, 'path' => "#{group.id}/#{project.project_namespace.id}/",
'target_type' => 'Issue'),
hash_including('id' => event_without_parent.id, 'path' => '', 'target_type' => ''),
hash_including('id' => group_event.id, 'path' => "#{group.id}/", 'target_type' => ''),
hash_including('id' => project_event1.id, 'path' => "#{group.id}/#{project.project_namespace.id}/",
'target_type' => 'Issue')
]
events = ClickHouse::Client.select('SELECT * FROM events ORDER BY id', :main)
expect(events).to match(expected_records)
last_processed_id = ClickHouse::SyncCursor.cursor_for(:events)
expect(last_processed_id).to eq(project_event1.id)
end
context 'when multiple batches are needed' do
before do
stub_const("#{described_class}::BATCH_SIZE", 1)
stub_const("#{described_class}::INSERT_BATCH_SIZE", 1)
end
it 'inserts all records' do
expect(worker).to receive(:log_extra_metadata_on_done).with(:result,
{ status: :processed, records_inserted: 4, reached_end_of_table: true })
worker.perform
events = ClickHouse::Client.select('SELECT * FROM events', :main)
expect(events.size).to eq(4)
end
context 'when new records are inserted while processing' do
it 'does not process new records created during the iteration' do
expect(worker).to receive(:log_extra_metadata_on_done).with(:result,
{ status: :processed, records_inserted: 4,
reached_end_of_table: true })
# Simulating the case when there is an insert during the iteration
call_count = 0
allow(worker).to receive(:next_batch).and_wrap_original do |method|
call_count += 1
create(:event) if call_count == 3
method.call
end
worker.perform
end
end
end
context 'when time limit is reached' do
before do
stub_const("#{described_class}::BATCH_SIZE", 1)
end
it 'stops the processing' do
allow_next_instance_of(Analytics::CycleAnalytics::RuntimeLimiter) do |runtime_limiter|
allow(runtime_limiter).to receive(:over_time?).and_return(false, true)
end
expect(worker).to receive(:log_extra_metadata_on_done).with(:result,
{ status: :processed, records_inserted: 2, reached_end_of_table: false })
worker.perform
last_processed_id = ClickHouse::SyncCursor.cursor_for(:events)
expect(last_processed_id).to eq(event_without_parent.id)
end
end
context 'when syncing from a certain point' do
before do
ClickHouse::SyncCursor.update_cursor_for(:events, project_event2.id)
end
it 'syncs records after the cursor' do
expect(worker).to receive(:log_extra_metadata_on_done).with(:result,
{ status: :processed, records_inserted: 3, reached_end_of_table: true })
worker.perform
events = ClickHouse::Client.select('SELECT id FROM events ORDER BY id', :main)
expect(events).to eq([{ 'id' => event_without_parent.id }, { 'id' => group_event.id },
{ 'id' => project_event1.id }])
end
context 'when there is nothing to sync' do
it 'does nothing' do
expect(worker).to receive(:log_extra_metadata_on_done).with(:result,
{ status: :processed, records_inserted: 0, reached_end_of_table: true })
ClickHouse::SyncCursor.update_cursor_for(:events, project_event1.id)
worker.perform
events = ClickHouse::Client.select('SELECT id FROM events ORDER BY id', :main)
expect(events).to be_empty
end
end
end
end
end
context 'when clickhouse is not configured' do
before do
allow(ClickHouse::Client).to receive(:database_configured?).and_return(false)
end
it 'skips execution' do
expect(worker).to receive(:log_extra_metadata_on_done).with(:result, { status: :disabled })
worker.perform
end
end
end
context 'when exclusive lease error happens' do
it 'skips execution' do
stub_feature_flags(event_sync_worker_for_click_house: true)
allow(ClickHouse::Client).to receive(:database_configured?).with(:main).and_return(true)
expect(worker).to receive(:in_lock).and_raise(Gitlab::ExclusiveLeaseHelpers::FailedToObtainLockError)
expect(worker).to receive(:log_extra_metadata_on_done).with(:result, { status: :skipped })
worker.perform
end
end
context 'when the event_sync_worker_for_click_house feature flag is off' do
before do
stub_feature_flags(event_sync_worker_for_click_house: false)
end
it 'skips execution' do
expect(worker).to receive(:log_extra_metadata_on_done).with(:result, { status: :disabled })
worker.perform
end
end
end
|
