1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
|
# frozen_string_literal: true
class RemoteMirror < ApplicationRecord
include AfterCommitQueue
include MirrorAuthentication
PROTECTED_BACKOFF_DELAY = 1.minute
UNPROTECTED_BACKOFF_DELAY = 5.minutes
attr_encrypted :credentials,
key: Settings.attr_encrypted_db_key_base,
marshal: true,
encode: true,
mode: :per_attribute_iv_and_salt,
insecure_mode: true,
algorithm: 'aes-256-cbc'
belongs_to :project, inverse_of: :remote_mirrors
validates :url, presence: true, public_url: { schemes: %w(ssh git http https), allow_blank: true, enforce_user: true }
before_save :set_new_remote_name, if: :mirror_url_changed?
after_save :set_override_remote_mirror_available, unless: -> { Gitlab::CurrentSettings.current_application_settings.mirror_available }
after_save :refresh_remote, if: :saved_change_to_mirror_url?
after_update :reset_fields, if: :saved_change_to_mirror_url?
after_commit :remove_remote, on: :destroy
before_validation :store_credentials
scope :enabled, -> { where(enabled: true) }
scope :started, -> { with_update_status(:started) }
scope :stuck, -> { started.where('last_update_at < ? OR (last_update_at IS NULL AND updated_at < ?)', 1.day.ago, 1.day.ago) }
state_machine :update_status, initial: :none do
event :update_start do
transition [:none, :finished, :failed] => :started
end
event :update_finish do
transition started: :finished
end
event :update_fail do
transition started: :failed
end
state :started
state :finished
state :failed
after_transition any => :started do |remote_mirror, _|
Gitlab::Metrics.add_event(:remote_mirrors_running)
remote_mirror.update(last_update_started_at: Time.now)
end
after_transition started: :finished do |remote_mirror, _|
Gitlab::Metrics.add_event(:remote_mirrors_finished)
timestamp = Time.now
remote_mirror.update!(
last_update_at: timestamp,
last_successful_update_at: timestamp,
last_error: nil,
error_notification_sent: false
)
end
after_transition started: :failed do |remote_mirror|
Gitlab::Metrics.add_event(:remote_mirrors_failed)
remote_mirror.update(last_update_at: Time.now)
remote_mirror.run_after_commit do
RemoteMirrorNotificationWorker.perform_async(remote_mirror.id)
end
end
end
def remote_name
super || fallback_remote_name
end
def update_failed?
update_status == 'failed'
end
def update_in_progress?
update_status == 'started'
end
def update_repository(options)
if ssh_mirror_url?
if ssh_key_auth? && ssh_private_key.present?
options[:ssh_key] = ssh_private_key
end
if ssh_known_hosts.present?
options[:known_hosts] = ssh_known_hosts
end
end
Gitlab::Git::RemoteMirror.new(
project.repository.raw,
remote_name,
**options
).update
end
def sync?
enabled?
end
def sync
return unless sync?
if recently_scheduled?
RepositoryUpdateRemoteMirrorWorker.perform_in(backoff_delay, self.id, Time.now)
else
RepositoryUpdateRemoteMirrorWorker.perform_async(self.id, Time.now)
end
end
def enabled
return false unless project && super
return false unless project.remote_mirror_available?
return false unless project.repository_exists?
return false if project.pending_delete?
true
end
alias_method :enabled?, :enabled
def updated_since?(timestamp)
last_update_started_at && last_update_started_at > timestamp && !update_failed?
end
def mark_for_delete_if_blank_url
mark_for_destruction if url.blank?
end
def mark_as_failed(error_message)
update_column(:last_error, Gitlab::UrlSanitizer.sanitize(error_message))
update_fail
end
def url=(value)
super(value) && return unless Gitlab::UrlSanitizer.valid?(value)
mirror_url = Gitlab::UrlSanitizer.new(value)
self.credentials ||= {}
self.credentials = self.credentials.merge(mirror_url.credentials)
super(mirror_url.sanitized_url)
end
def url
if super
Gitlab::UrlSanitizer.new(super, credentials: credentials).full_url
end
rescue
super
end
def safe_url
return if url.nil?
result = URI.parse(url)
result.password = '*****' if result.password
result.user = '*****' if result.user && result.user != "git" # tokens or other data may be saved as user
result.to_s
end
def ensure_remote!
return unless project
return unless remote_name && remote_url
# If this fails or the remote already exists, we won't know due to
# https://gitlab.com/gitlab-org/gitaly/issues/1317
project.repository.add_remote(remote_name, remote_url)
end
def after_sent_notification
update_column(:error_notification_sent, true)
end
private
def store_credentials
# This is a necessary workaround for attr_encrypted, which doesn't otherwise
# notice that the credentials have changed
self.credentials = self.credentials
end
# The remote URL omits any password if SSH public-key authentication is in use
def remote_url
return url unless ssh_key_auth? && password.present?
Gitlab::UrlSanitizer.new(read_attribute(:url), credentials: { user: user }).full_url
rescue
super
end
def fallback_remote_name
return unless id
"remote_mirror_#{id}"
end
def recently_scheduled?
return false unless self.last_update_started_at
self.last_update_started_at >= Time.now - backoff_delay
end
def backoff_delay
if self.only_protected_branches
PROTECTED_BACKOFF_DELAY
else
UNPROTECTED_BACKOFF_DELAY
end
end
def reset_fields
update_columns(
last_error: nil,
last_update_at: nil,
last_successful_update_at: nil,
update_status: 'finished',
error_notification_sent: false
)
end
def set_override_remote_mirror_available
enabled = read_attribute(:enabled)
project.update(remote_mirror_available_overridden: enabled)
end
def set_new_remote_name
self.remote_name = "remote_mirror_#{SecureRandom.hex}"
end
def refresh_remote
return unless project
# Before adding a new remote we have to delete the data from
# the previous remote name
prev_remote_name = remote_name_before_last_save || fallback_remote_name
run_after_commit do
project.repository.async_remove_remote(prev_remote_name)
end
project.repository.add_remote(remote_name, remote_url)
end
def remove_remote
return unless project # could be pending to delete so don't need to touch the git repository
project.repository.async_remove_remote(remote_name)
end
def mirror_url_changed?
url_changed? || credentials_changed?
end
def saved_change_to_mirror_url?
saved_change_to_url? || saved_change_to_credentials?
end
end
|