1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
|
# frozen_string_literal: true
module ProjectAuthorizations
# How to use this class
# authorizations_to_add:
# Rows to insert in the form `[{ user_id: user_id, project_id: project_id, access_level: access_level}, ...]
#
# ProjectAuthorizations::Changes.new do |changes|
# changes.add(authorizations_to_add)
# changes.remove_users_in_project(project, user_ids)
# changes.remove_projects_for_user(user, project_ids)
# end.apply!
class Changes
attr_reader :projects_to_remove, :users_to_remove_in_project, :authorizations_to_add
BATCH_SIZE = 1000
EVENT_USER_BATCH_SIZE = 100
SLEEP_DELAY = 0.1
def initialize
@authorizations_to_add = []
@affected_project_ids = Set.new
@removed_user_ids = Set.new
@added_user_ids = Set.new
yield self
end
def add(authorizations_to_add)
@authorizations_to_add += authorizations_to_add
end
def remove_users_in_project(project, user_ids)
@users_to_remove_in_project = { user_ids: user_ids, scope: project }
end
def remove_projects_for_user(user, project_ids)
@projects_to_remove = { project_ids: project_ids, scope: user }
end
def apply!
delete_authorizations_for_user if should_delete_authorizations_for_user?
delete_authorizations_for_project if should_delete_authorizations_for_project?
add_authorizations if should_add_authorization?
publish_events
end
private
def should_add_authorization?
authorizations_to_add.present?
end
def should_delete_authorizations_for_user?
user && project_ids.present?
end
def should_delete_authorizations_for_project?
project && user_ids.present?
end
def add_authorizations
insert_all_in_batches(authorizations_to_add)
@affected_project_ids += authorizations_to_add.pluck(:project_id)
@added_user_ids += authorizations_to_add.pluck(:user_id)
end
def delete_authorizations_for_user
delete_all_in_batches(resource: user,
ids_to_remove: project_ids,
column_name_of_ids_to_remove: :project_id)
@affected_project_ids += project_ids
@removed_user_ids.add(user.id)
end
def delete_authorizations_for_project
delete_all_in_batches(resource: project,
ids_to_remove: user_ids,
column_name_of_ids_to_remove: :user_id)
@affected_project_ids << project.id
@removed_user_ids += user_ids
end
def delete_all_in_batches(resource:, ids_to_remove:, column_name_of_ids_to_remove:)
add_delay = add_delay_between_batches?(entire_size: ids_to_remove.size, batch_size: BATCH_SIZE)
log_details(entire_size: ids_to_remove.size, batch_size: BATCH_SIZE) if add_delay
ids_to_remove.each_slice(BATCH_SIZE) do |ids_batch|
resource.project_authorizations.where(column_name_of_ids_to_remove => ids_batch).delete_all
perform_delay if add_delay
end
end
def insert_all_in_batches(attributes)
add_delay = add_delay_between_batches?(entire_size: attributes.size, batch_size: BATCH_SIZE)
log_details(entire_size: attributes.size, batch_size: BATCH_SIZE) if add_delay
attributes.each_slice(BATCH_SIZE) do |attributes_batch|
attributes_batch.each { |attrs| attrs[:is_unique] = true }
ProjectAuthorization.insert_all(attributes_batch)
perform_delay if add_delay
end
end
def add_delay_between_batches?(entire_size:, batch_size:)
# The reason for adding a delay is to give the replica database enough time to
# catch up with the primary when large batches of records are being added/removed.
# Hence, we add a delay only if the GitLab installation has a replica database configured.
entire_size > batch_size &&
!::Gitlab::Database::LoadBalancing.primary_only?
end
def log_details(entire_size:, batch_size:)
Gitlab::AppLogger.info(
entire_size: entire_size,
total_delay: (entire_size / batch_size.to_f).ceil * SLEEP_DELAY,
message: 'Project authorizations refresh performed with delay',
**Gitlab::ApplicationContext.current
)
end
def perform_delay
sleep(SLEEP_DELAY)
end
def user
projects_to_remove&.[](:scope)
end
def project_ids
projects_to_remove&.[](:project_ids)
end
def project
users_to_remove_in_project&.[](:scope)
end
def user_ids
users_to_remove_in_project&.[](:user_ids)
end
def publish_events
publish_changed_event
publish_removed_event
publish_added_event
end
def publish_changed_event
# This event is used to add policy approvers to approval rules by re-syncing all project policies which is costly.
# If the feature flag below is enabled, the policies won't be re-synced and
# the approvers will be added via `AuthorizationsAddedEvent`.
return if ::Feature.enabled?(:add_policy_approvers_to_rules)
@affected_project_ids.each do |project_id|
::Gitlab::EventStore.publish(
::ProjectAuthorizations::AuthorizationsChangedEvent.new(data: { project_id: project_id })
)
end
end
def publish_removed_event
return if @removed_user_ids.none?
events = @affected_project_ids.flat_map do |project_id|
@removed_user_ids.to_a.each_slice(EVENT_USER_BATCH_SIZE).map do |user_ids_batch|
::ProjectAuthorizations::AuthorizationsRemovedEvent.new(data: {
project_id: project_id,
user_ids: user_ids_batch
})
end
end
::Gitlab::EventStore.publish_group(events)
end
def publish_added_event
return if ::Feature.disabled?(:add_policy_approvers_to_rules)
return if @added_user_ids.none?
events = @affected_project_ids.flat_map do |project_id|
@added_user_ids.to_a.each_slice(EVENT_USER_BATCH_SIZE).map do |user_ids_batch|
::ProjectAuthorizations::AuthorizationsAddedEvent.new(data: {
project_id: project_id,
user_ids: user_ids_batch
})
end
end
::Gitlab::EventStore.publish_group(events)
end
end
end
|
