1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
|
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::BackfillPartitionedTable, feature_category: :database do
subject(:backfill_job) do
described_class.new(
start_id: 1,
end_id: 3,
batch_table: source_table,
batch_column: :id,
sub_batch_size: 2,
pause_ms: 0,
job_arguments: [destination_table],
connection: connection
)
end
let(:connection) { ApplicationRecord.connection }
let(:source_table) { '_test_source_table' }
let(:destination_table) { "#{source_table}_partitioned" }
let(:source_model) { Class.new(ApplicationRecord) }
let(:destination_model) { Class.new(ApplicationRecord) }
describe '#perform' do
context 'without the destination table' do
let(:expected_error_message) do
"exiting backfill migration because partitioned table #{destination_table} does not exist. " \
"This could be due to rollback of the migration which created the partitioned table."
end
it 'raises an exception' do
expect { backfill_job.perform }.to raise_error(expected_error_message)
end
end
context 'with destination table being not partitioned' do
before do
connection.execute(<<~SQL)
CREATE TABLE #{destination_table} (
id serial NOT NULL,
col1 int NOT NULL,
col2 text NOT NULL,
created_at timestamptz NOT NULL,
PRIMARY KEY (id, created_at)
)
SQL
end
after do
connection.drop_table destination_table
end
let(:expected_error_message) do
"exiting backfill migration because the given destination table is not partitioned."
end
it 'raises an exception' do
expect { backfill_job.perform }.to raise_error(expected_error_message)
end
end
context 'when the destination table exists' do
before do
connection.execute(<<~SQL)
CREATE TABLE #{source_table} (
id serial NOT NULL PRIMARY KEY,
col1 int NOT NULL,
col2 text NOT NULL,
created_at timestamptz NOT NULL
)
SQL
connection.execute(<<~SQL)
CREATE TABLE #{destination_table} (
id serial NOT NULL,
col1 int NOT NULL,
col2 text NOT NULL,
created_at timestamptz NOT NULL,
PRIMARY KEY (id, created_at)
) PARTITION BY RANGE (created_at)
SQL
connection.execute(<<~SQL)
CREATE TABLE #{destination_table}_202001 PARTITION OF #{destination_table}
FOR VALUES FROM ('2020-01-01') TO ('2020-02-01')
SQL
connection.execute(<<~SQL)
CREATE TABLE #{destination_table}_202002 PARTITION OF #{destination_table}
FOR VALUES FROM ('2020-02-01') TO ('2020-03-01')
SQL
source_model.table_name = source_table
destination_model.table_name = destination_table
end
after do
connection.drop_table source_table
connection.drop_table destination_table
end
let(:timestamp) { Time.utc(2020, 1, 2).round }
let!(:source1) { create_source_record(timestamp) }
let!(:source2) { create_source_record(timestamp + 1.day) }
let!(:source3) { create_source_record(timestamp + 1.month) }
it 'copies data into the destination table idempotently' do
expect(destination_model.count).to eq(0)
backfill_job.perform
expect(destination_model.count).to eq(3)
source_model.find_each do |source_record|
destination_record = destination_model.find_by_id(source_record.id)
expect(destination_record.attributes).to eq(source_record.attributes)
end
backfill_job.perform
expect(destination_model.count).to eq(3)
end
it 'breaks the assigned batch into smaller sub batches' do
expect_next_instance_of(Gitlab::Database::PartitioningMigrationHelpers::BulkCopy) do |bulk_copy|
expect(bulk_copy).to receive(:copy_between).with(source1.id, source2.id)
expect(bulk_copy).to receive(:copy_between).with(source3.id, source3.id)
end
backfill_job.perform
end
end
end
def create_source_record(timestamp)
source_model.create!(col1: 123, col2: 'original value', created_at: timestamp)
end
end
|
