diff options
author | Douwe Maan <douwe@gitlab.com> | 2018-05-07 18:15:57 +0300 |
---|---|---|
committer | Douwe Maan <douwe@gitlab.com> | 2018-05-07 18:15:57 +0300 |
commit | d0cbef7e8f4faf76b9a528a4629517f29b8c265b (patch) | |
tree | 90610e3e78f17389bcd601f43b9fbe29723b6034 /app | |
parent | 32009764527a89708e89fb547ce4086e75cd04d0 (diff) | |
parent | 2a279143ebbccdbc180619cb29936d35d11b226d (diff) |
Merge branch '42099-port-push-mirroring-to-ce-ce-port-v-2' into 'master'
CE backport of Backports Push Mirrors to CE
Closes #42099
See merge request gitlab-org/gitlab-ce!18715
Diffstat (limited to 'app')
22 files changed, 651 insertions, 3 deletions
diff --git a/app/controllers/projects/mirrors_controller.rb b/app/controllers/projects/mirrors_controller.rb new file mode 100644 index 00000000000..5698ff4e706 --- /dev/null +++ b/app/controllers/projects/mirrors_controller.rb @@ -0,0 +1,67 @@ +class Projects::MirrorsController < Projects::ApplicationController + include RepositorySettingsRedirect + + # Authorize + before_action :remote_mirror, only: [:update] + before_action :check_mirror_available! + before_action :authorize_admin_project! + + layout "project_settings" + + def show + redirect_to_repository_settings(project) + end + + def update + if project.update_attributes(mirror_params) + flash[:notice] = 'Mirroring settings were successfully updated.' + else + flash[:alert] = project.errors.full_messages.join(', ').html_safe + end + + respond_to do |format| + format.html { redirect_to_repository_settings(project) } + format.json do + if project.errors.present? + render json: project.errors, status: :unprocessable_entity + else + render json: ProjectMirrorSerializer.new.represent(project) + end + end + end + end + + def update_now + if params[:sync_remote] + project.update_remote_mirrors + flash[:notice] = "The remote repository is being updated..." + end + + redirect_to_repository_settings(project) + end + + private + + def remote_mirror + @remote_mirror = project.remote_mirrors.first_or_initialize + end + + def check_mirror_available! + Gitlab::CurrentSettings.current_application_settings.mirror_available || current_user&.admin? + end + + def mirror_params_attributes + [ + remote_mirrors_attributes: %i[ + url + id + enabled + only_protected_branches + ] + ] + end + + def mirror_params + params.require(:project).permit(mirror_params_attributes) + end +end diff --git a/app/controllers/projects/settings/repository_controller.rb b/app/controllers/projects/settings/repository_controller.rb index f17056f13e0..4697af4f26a 100644 --- a/app/controllers/projects/settings/repository_controller.rb +++ b/app/controllers/projects/settings/repository_controller.rb @@ -2,6 +2,7 @@ module Projects module Settings class RepositoryController < Projects::ApplicationController before_action :authorize_admin_project! + before_action :remote_mirror, only: [:show] def show render_show @@ -25,6 +26,7 @@ module Projects define_deploy_token define_protected_refs + remote_mirror render 'show' end @@ -41,6 +43,10 @@ module Projects load_gon_index end + def remote_mirror + @remote_mirror = project.remote_mirrors.first_or_initialize + end + def access_levels_options { create_access_levels: levels_for_dropdown, diff --git a/app/helpers/application_settings_helper.rb b/app/helpers/application_settings_helper.rb index 1bf98d550b0..b948e431882 100644 --- a/app/helpers/application_settings_helper.rb +++ b/app/helpers/application_settings_helper.rb @@ -250,7 +250,8 @@ module ApplicationSettingsHelper :version_check_enabled, :allow_local_requests_from_hooks_and_services, :enforce_terms, - :terms + :terms, + :mirror_available ] end end diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index a734cc7a26b..451e512aef7 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -334,7 +334,8 @@ class ApplicationSetting < ActiveRecord::Base gitaly_timeout_fast: 10, gitaly_timeout_medium: 30, gitaly_timeout_default: 55, - allow_local_requests_from_hooks_and_services: false + allow_local_requests_from_hooks_and_services: false, + mirror_available: true } end diff --git a/app/models/project.rb b/app/models/project.rb index 37f1fcea280..0a549d843f3 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -65,6 +65,9 @@ class Project < ActiveRecord::Base default_value_for :only_allow_merge_if_all_discussions_are_resolved, false add_authentication_token_field :runners_token + + before_validation :mark_remote_mirrors_for_removal, if: -> { ActiveRecord::Base.connection.table_exists?(:remote_mirrors) } + before_save :ensure_runners_token after_save :update_project_statistics, if: :namespace_id_changed? @@ -246,11 +249,17 @@ class Project < ActiveRecord::Base has_many :project_badges, class_name: 'ProjectBadge' has_one :ci_cd_settings, class_name: 'ProjectCiCdSetting', inverse_of: :project, autosave: true, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :remote_mirrors, inverse_of: :project + accepts_nested_attributes_for :variables, allow_destroy: true accepts_nested_attributes_for :project_feature, update_only: true accepts_nested_attributes_for :import_data accepts_nested_attributes_for :auto_devops, update_only: true + accepts_nested_attributes_for :remote_mirrors, + allow_destroy: true, + reject_if: ->(attrs) { attrs[:id].blank? && attrs[:url].blank? } + delegate :name, to: :owner, allow_nil: true, prefix: true delegate :members, to: :team, prefix: true delegate :add_user, :add_users, to: :team @@ -340,6 +349,7 @@ class Project < ActiveRecord::Base scope :with_issues_enabled, -> { with_feature_enabled(:issues) } scope :with_issues_available_for_user, ->(current_user) { with_feature_available_for_user(:issues, current_user) } scope :with_merge_requests_enabled, -> { with_feature_enabled(:merge_requests) } + scope :with_remote_mirrors, -> { joins(:remote_mirrors).where(remote_mirrors: { enabled: true }).distinct } scope :with_group_runners_enabled, -> do joins(:ci_cd_settings) @@ -759,6 +769,37 @@ class Project < ActiveRecord::Base import_type == 'gitea' end + def has_remote_mirror? + remote_mirror_available? && remote_mirrors.enabled.exists? + end + + def updating_remote_mirror? + remote_mirrors.enabled.started.exists? + end + + def update_remote_mirrors + return unless remote_mirror_available? + + remote_mirrors.enabled.each(&:sync) + end + + def mark_stuck_remote_mirrors_as_failed! + remote_mirrors.stuck.update_all( + update_status: :failed, + last_error: 'The remote mirror took to long to complete.', + last_update_at: Time.now + ) + end + + def mark_remote_mirrors_for_removal + remote_mirrors.each(&:mark_for_delete_if_blank_url) + end + + def remote_mirror_available? + remote_mirror_available_overridden || + ::Gitlab::CurrentSettings.mirror_available + end + def check_limit unless creator.can_create_project? || namespace.kind == 'group' projects_limit = creator.projects_limit diff --git a/app/models/remote_mirror.rb b/app/models/remote_mirror.rb new file mode 100644 index 00000000000..bbf8fd9c6a7 --- /dev/null +++ b/app/models/remote_mirror.rb @@ -0,0 +1,219 @@ +class RemoteMirror < ActiveRecord::Base + include AfterCommitQueue + + PROTECTED_BACKOFF_DELAY = 1.minute + UNPROTECTED_BACKOFF_DELAY = 5.minutes + + attr_encrypted :credentials, + key: Gitlab::Application.secrets.db_key_base, + marshal: true, + encode: true, + mode: :per_attribute_iv_and_salt, + insecure_mode: true, + algorithm: 'aes-256-cbc' + + default_value_for :only_protected_branches, true + + belongs_to :project, inverse_of: :remote_mirrors + + validates :url, presence: true, url: { protocols: %w(ssh git http https), allow_blank: true } + validates :url, addressable_url: true, if: :url_changed? + + before_save :set_new_remote_name, if: :mirror_url_changed? + + after_save :set_override_remote_mirror_available, unless: -> { Gitlab::CurrentSettings.current_application_settings.mirror_available } + after_save :refresh_remote, if: :mirror_url_changed? + after_update :reset_fields, if: :mirror_url_changed? + + after_commit :remove_remote, on: :destroy + + scope :enabled, -> { where(enabled: true) } + scope :started, -> { with_update_status(:started) } + scope :stuck, -> { started.where('last_update_at < ? OR (last_update_at IS NULL AND updated_at < ?)', 1.day.ago, 1.day.ago) } + + state_machine :update_status, initial: :none do + event :update_start do + transition [:none, :finished, :failed] => :started + end + + event :update_finish do + transition started: :finished + end + + event :update_fail do + transition started: :failed + end + + state :started + state :finished + state :failed + + after_transition any => :started do |remote_mirror, _| + Gitlab::Metrics.add_event(:remote_mirrors_running, path: remote_mirror.project.full_path) + + remote_mirror.update(last_update_started_at: Time.now) + end + + after_transition started: :finished do |remote_mirror, _| + Gitlab::Metrics.add_event(:remote_mirrors_finished, path: remote_mirror.project.full_path) + + timestamp = Time.now + remote_mirror.update_attributes!( + last_update_at: timestamp, last_successful_update_at: timestamp, last_error: nil + ) + end + + after_transition started: :failed do |remote_mirror, _| + Gitlab::Metrics.add_event(:remote_mirrors_failed, path: remote_mirror.project.full_path) + + remote_mirror.update(last_update_at: Time.now) + end + end + + def remote_name + super || fallback_remote_name + end + + def update_failed? + update_status == 'failed' + end + + def update_in_progress? + update_status == 'started' + end + + def update_repository(options) + raw.update(options) + end + + def sync? + enabled? + end + + def sync + return unless sync? + + if recently_scheduled? + RepositoryUpdateRemoteMirrorWorker.perform_in(backoff_delay, self.id, Time.now) + else + RepositoryUpdateRemoteMirrorWorker.perform_async(self.id, Time.now) + end + end + + def enabled + return false unless project && super + return false unless project.remote_mirror_available? + return false unless project.repository_exists? + return false if project.pending_delete? + + true + end + alias_method :enabled?, :enabled + + def updated_since?(timestamp) + last_update_started_at && last_update_started_at > timestamp && !update_failed? + end + + def mark_for_delete_if_blank_url + mark_for_destruction if url.blank? + end + + def mark_as_failed(error_message) + update_fail + update_column(:last_error, Gitlab::UrlSanitizer.sanitize(error_message)) + end + + def url=(value) + super(value) && return unless Gitlab::UrlSanitizer.valid?(value) + + mirror_url = Gitlab::UrlSanitizer.new(value) + self.credentials = mirror_url.credentials + + super(mirror_url.sanitized_url) + end + + def url + if super + Gitlab::UrlSanitizer.new(super, credentials: credentials).full_url + end + rescue + super + end + + def safe_url + return if url.nil? + + result = URI.parse(url) + result.password = '*****' if result.password + result.user = '*****' if result.user && result.user != "git" # tokens or other data may be saved as user + result.to_s + end + + private + + def raw + @raw ||= Gitlab::Git::RemoteMirror.new(project.repository.raw, remote_name) + end + + def fallback_remote_name + return unless id + + "remote_mirror_#{id}" + end + + def recently_scheduled? + return false unless self.last_update_started_at + + self.last_update_started_at >= Time.now - backoff_delay + end + + def backoff_delay + if self.only_protected_branches + PROTECTED_BACKOFF_DELAY + else + UNPROTECTED_BACKOFF_DELAY + end + end + + def reset_fields + update_columns( + last_error: nil, + last_update_at: nil, + last_successful_update_at: nil, + update_status: 'finished' + ) + end + + def set_override_remote_mirror_available + enabled = read_attribute(:enabled) + + project.update(remote_mirror_available_overridden: enabled) + end + + def set_new_remote_name + self.remote_name = "remote_mirror_#{SecureRandom.hex}" + end + + def refresh_remote + return unless project + + # Before adding a new remote we have to delete the data from + # the previous remote name + prev_remote_name = remote_name_was || fallback_remote_name + run_after_commit do + project.repository.async_remove_remote(prev_remote_name) + end + + project.repository.add_remote(remote_name, url) + end + + def remove_remote + return unless project # could be pending to delete so don't need to touch the git repository + + project.repository.async_remove_remote(remote_name) + end + + def mirror_url_changed? + url_changed? || encrypted_credentials_changed? + end +end diff --git a/app/models/repository.rb b/app/models/repository.rb index 6831305fb93..b75c4aca982 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -854,13 +854,27 @@ class Repository add_remote(remote_name, url, mirror_refmap: refmap) fetch_remote(remote_name, forced: forced, prune: prune) ensure - remove_remote(remote_name) if tmp_remote_name + async_remove_remote(remote_name) if tmp_remote_name end def fetch_remote(remote, forced: false, ssh_auth: nil, no_tags: false, prune: true) gitlab_shell.fetch_remote(raw_repository, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, prune: prune) end + def async_remove_remote(remote_name) + return unless remote_name + + job_id = RepositoryRemoveRemoteWorker.perform_async(project.id, remote_name) + + if job_id + Rails.logger.info("Remove remote job scheduled for #{project.id} with remote name: #{remote_name} job ID #{job_id}.") + else + Rails.logger.info("Remove remote job failed to create for #{project.id} with remote name #{remote_name}.") + end + + job_id + end + def fetch_source_branch!(source_repository, source_branch, local_ref) raw_repository.fetch_source_branch!(source_repository.raw_repository, source_branch, local_ref) end diff --git a/app/policies/project_policy.rb b/app/policies/project_policy.rb index 3529d0aa60c..5759b1a376f 100644 --- a/app/policies/project_policy.rb +++ b/app/policies/project_policy.rb @@ -80,6 +80,11 @@ class ProjectPolicy < BasePolicy project.merge_requests_allowing_push_to_user(user).any? end + with_scope :global + condition(:mirror_available, score: 0) do + ::Gitlab::CurrentSettings.current_application_settings.mirror_available + end + # We aren't checking `:read_issue` or `:read_merge_request` in this case # because it could be possible for a user to see an issuable-iid # (`:read_issue_iid` or `:read_merge_request_iid`) but then wouldn't be @@ -246,6 +251,8 @@ class ProjectPolicy < BasePolicy enable :create_cluster end + rule { (mirror_available & can?(:admin_project)) | admin }.enable :admin_remote_mirror + rule { archived }.policy do prevent :push_code prevent :push_to_delete_protected_branch diff --git a/app/serializers/project_mirror_entity.rb b/app/serializers/project_mirror_entity.rb new file mode 100644 index 00000000000..a9c08ac021a --- /dev/null +++ b/app/serializers/project_mirror_entity.rb @@ -0,0 +1,11 @@ +class ProjectMirrorEntity < Grape::Entity + expose :id + + expose :remote_mirrors_attributes do |project| + next [] unless project.remote_mirrors.present? + + project.remote_mirrors.map do |remote| + remote.as_json(only: %i[id url enabled]) + end + end +end diff --git a/app/services/concerns/exclusive_lease_guard.rb b/app/services/concerns/exclusive_lease_guard.rb new file mode 100644 index 00000000000..30be6accc32 --- /dev/null +++ b/app/services/concerns/exclusive_lease_guard.rb @@ -0,0 +1,52 @@ +# +# Concern that helps with getting an exclusive lease for running a block +# of code. +# +# `#try_obtain_lease` takes a block which will be run if it was able to +# obtain the lease. Implement `#lease_timeout` to configure the timeout +# for the exclusive lease. Optionally override `#lease_key` to set the +# lease key, it defaults to the class name with underscores. +# +module ExclusiveLeaseGuard + extend ActiveSupport::Concern + + def try_obtain_lease + lease = exclusive_lease.try_obtain + + unless lease + log_error('Cannot obtain an exclusive lease. There must be another instance already in execution.') + return + end + + begin + yield lease + ensure + release_lease(lease) + end + end + + def exclusive_lease + @lease ||= Gitlab::ExclusiveLease.new(lease_key, timeout: lease_timeout) + end + + def lease_key + @lease_key ||= self.class.name.underscore + end + + def lease_timeout + raise NotImplementedError, + "#{self.class.name} does not implement #{__method__}" + end + + def release_lease(uuid) + Gitlab::ExclusiveLease.cancel(lease_key, uuid) + end + + def renew_lease! + exclusive_lease.renew + end + + def log_error(message, extra_args = {}) + logger.error(message) + end +end diff --git a/app/services/git_push_service.rb b/app/services/git_push_service.rb index c037141fcde..f3bfc53dcd3 100644 --- a/app/services/git_push_service.rb +++ b/app/services/git_push_service.rb @@ -55,6 +55,7 @@ class GitPushService < BaseService execute_related_hooks perform_housekeeping + update_remote_mirrors update_caches update_signatures @@ -119,6 +120,13 @@ class GitPushService < BaseService protected + def update_remote_mirrors + return unless @project.has_remote_mirror? + + @project.mark_stuck_remote_mirrors_as_failed! + @project.update_remote_mirrors + end + def execute_related_hooks # Update merge requests that may be affected by this push. A new branch # could cause the last commit of a merge request to change. diff --git a/app/services/projects/update_remote_mirror_service.rb b/app/services/projects/update_remote_mirror_service.rb new file mode 100644 index 00000000000..8183a2f26d7 --- /dev/null +++ b/app/services/projects/update_remote_mirror_service.rb @@ -0,0 +1,30 @@ +module Projects + class UpdateRemoteMirrorService < BaseService + attr_reader :errors + + def execute(remote_mirror) + @errors = [] + + return success unless remote_mirror.enabled? + + begin + repository.fetch_remote(remote_mirror.remote_name, no_tags: true) + + opts = {} + if remote_mirror.only_protected_branches? + opts[:only_branches_matching] = project.protected_branches.select(:name).map(&:name) + end + + remote_mirror.update_repository(opts) + rescue => e + errors << e.message.strip + end + + if errors.present? + error(errors.join("\n\n")) + else + success + end + end + end +end diff --git a/app/views/admin/application_settings/_repository_mirrors_form.html.haml b/app/views/admin/application_settings/_repository_mirrors_form.html.haml new file mode 100644 index 00000000000..09183ec6260 --- /dev/null +++ b/app/views/admin/application_settings/_repository_mirrors_form.html.haml @@ -0,0 +1,16 @@ += form_for @application_setting, url: admin_application_settings_path, html: { class: 'form-horizontal fieldset-form' } do |f| + = form_errors(@application_setting) + + %fieldset + .form-group + = f.label :mirror_available, 'Enable mirror configuration', class: 'control-label col-sm-2' + .col-sm-10 + .checkbox + = f.label :mirror_available do + = f.check_box :mirror_available + Allow mirrors to be setup for projects + %span.help-block + If disabled, only admins will be able to setup mirrors in projects. + = link_to icon('question-circle'), help_page_path('workflow/repository_mirroring') + + = f.submit 'Save changes', class: "btn btn-success" diff --git a/app/views/admin/application_settings/show.html.haml b/app/views/admin/application_settings/show.html.haml index 3c00e3c8fc4..3f440c76ee0 100644 --- a/app/views/admin/application_settings/show.html.haml +++ b/app/views/admin/application_settings/show.html.haml @@ -313,3 +313,14 @@ = _('Allow requests to the local network from hooks and services.') .settings-content = render 'outbound' + +%section.settings.as-mirror.no-animate#js-mirror-settings{ class: ('expanded' if expanded) } + .settings-header + %h4 + = _('Repository mirror settings') + %button.btn.js-settings-toggle{ type: 'button' } + = expanded ? 'Collapse' : 'Expand' + %p + = _('Configure push mirrors.') + .settings-content + = render partial: 'repository_mirrors_form' diff --git a/app/views/projects/mirrors/_instructions.html.haml b/app/views/projects/mirrors/_instructions.html.haml new file mode 100644 index 00000000000..64f0fde30cf --- /dev/null +++ b/app/views/projects/mirrors/_instructions.html.haml @@ -0,0 +1,10 @@ +.account-well.prepend-top-default.append-bottom-default + %ul + %li + The repository must be accessible over <code>http://</code>, <code>https://</code>, <code>ssh://</code> or <code>git://</code>. + %li + Include the username in the URL if required: <code>https://username@gitlab.company.com/group/project.git</code>. + %li + The update action will time out after 10 minutes. For big repositories, use a clone/push combination. + %li + The Git LFS objects will <strong>not</strong> be synced. diff --git a/app/views/projects/mirrors/_push.html.haml b/app/views/projects/mirrors/_push.html.haml new file mode 100644 index 00000000000..4a6aefce351 --- /dev/null +++ b/app/views/projects/mirrors/_push.html.haml @@ -0,0 +1,50 @@ +- expanded = Rails.env.test? +%section.settings.no-animate{ class: ('expanded' if expanded) } + .settings-header + %h4 + Push to a remote repository + %button.btn.js-settings-toggle + = expanded ? 'Collapse' : 'Expand' + %p + Set up the remote repository that you want to update with the content of the current repository + every time someone pushes to it. + = link_to 'Read more', help_page_path('workflow/repository_mirroring', anchor: 'pushing-to-a-remote-repository'), target: '_blank' + .settings-content + = form_for @project, url: project_mirror_path(@project) do |f| + %div + = form_errors(@project) + = render "shared/remote_mirror_update_button", remote_mirror: @remote_mirror + - if @remote_mirror.last_error.present? + .panel.panel-danger + .panel-heading + - if @remote_mirror.last_update_at + The remote repository failed to update #{time_ago_with_tooltip(@remote_mirror.last_update_at)}. + - else + The remote repository failed to update. + + - if @remote_mirror.last_successful_update_at + Last successful update #{time_ago_with_tooltip(@remote_mirror.last_successful_update_at)}. + .panel-body + %pre + :preserve + #{h(@remote_mirror.last_error.strip)} + = f.fields_for :remote_mirrors, @remote_mirror do |rm_form| + .form-group + = rm_form.check_box :enabled, class: "pull-left" + .prepend-left-20 + = rm_form.label :enabled, "Remote mirror repository", class: "label-light append-bottom-0" + %p.light.append-bottom-0 + Automatically update the remote mirror's branches, tags, and commits from this repository every time someone pushes to it. + .form-group.has-feedback + = rm_form.label :url, "Git repository URL", class: "label-light" + = rm_form.text_field :url, class: "form-control", placeholder: 'https://username:password@gitlab.company.com/group/project.git' + + = render "projects/mirrors/instructions" + + .form-group + = rm_form.check_box :only_protected_branches, class: 'pull-left' + .prepend-left-20 + = rm_form.label :only_protected_branches, class: 'label-light' + = link_to icon('question-circle'), help_page_path('user/project/protected_branches') + + = f.submit 'Save changes', class: 'btn btn-create', name: 'update_remote_mirror' diff --git a/app/views/projects/mirrors/_show.html.haml b/app/views/projects/mirrors/_show.html.haml new file mode 100644 index 00000000000..de77701a373 --- /dev/null +++ b/app/views/projects/mirrors/_show.html.haml @@ -0,0 +1,3 @@ +- if can?(current_user, :admin_remote_mirror, @project) + = render 'projects/mirrors/push' + diff --git a/app/views/projects/settings/repository/show.html.haml b/app/views/projects/settings/repository/show.html.haml index f57590a908f..5dda2ec28b4 100644 --- a/app/views/projects/settings/repository/show.html.haml +++ b/app/views/projects/settings/repository/show.html.haml @@ -2,6 +2,8 @@ - page_title "Repository" - @content_class = "limit-container-width" unless fluid_layout += render "projects/mirrors/show" + -# Protected branches & tags use a lot of nested partials. -# The shared parts of the views can be found in the `shared` directory. -# Those are used throughout the actual views. These `shared` views are then diff --git a/app/views/shared/_remote_mirror_update_button.html.haml b/app/views/shared/_remote_mirror_update_button.html.haml new file mode 100644 index 00000000000..34de1c0695f --- /dev/null +++ b/app/views/shared/_remote_mirror_update_button.html.haml @@ -0,0 +1,13 @@ +- if @project.has_remote_mirror? + .append-bottom-default + - if remote_mirror.update_in_progress? + %span.btn.disabled + = icon("refresh spin") + Updating… + - else + = link_to update_now_project_mirror_path(@project, sync_remote: true), method: :post, class: "btn" do + = icon("refresh") + Update Now + - if @remote_mirror.last_successful_update_at + %p.inline.prepend-left-10 + Successfully updated #{time_ago_with_tooltip(@remote_mirror.last_successful_update_at)}. diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 5d9ec6142d7..b6433eb3eff 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -107,9 +107,11 @@ - rebase - repository_fork - repository_import +- repository_remove_remote - storage_migrator - system_hook_push - update_merge_requests - update_user_activity - upload_checksum - web_hook +- repository_update_remote_mirror diff --git a/app/workers/repository_remove_remote_worker.rb b/app/workers/repository_remove_remote_worker.rb new file mode 100644 index 00000000000..1c19b604b77 --- /dev/null +++ b/app/workers/repository_remove_remote_worker.rb @@ -0,0 +1,35 @@ +class RepositoryRemoveRemoteWorker + include ApplicationWorker + include ExclusiveLeaseGuard + + LEASE_TIMEOUT = 1.hour + + attr_reader :project, :remote_name + + def perform(project_id, remote_name) + @remote_name = remote_name + @project = Project.find_by_id(project_id) + + return unless @project + + logger.info("Removing remote #{remote_name} from project #{project.id}") + + try_obtain_lease do + remove_remote = @project.repository.remove_remote(remote_name) + + if remove_remote + logger.info("Remote #{remote_name} was successfully removed from project #{project.id}") + else + logger.error("Could not remove remote #{remote_name} from project #{project.id}") + end + end + end + + def lease_timeout + LEASE_TIMEOUT + end + + def lease_key + "remove_remote_#{project.id}_#{remote_name}" + end +end diff --git a/app/workers/repository_update_remote_mirror_worker.rb b/app/workers/repository_update_remote_mirror_worker.rb new file mode 100644 index 00000000000..bb963979e88 --- /dev/null +++ b/app/workers/repository_update_remote_mirror_worker.rb @@ -0,0 +1,49 @@ +class RepositoryUpdateRemoteMirrorWorker + UpdateAlreadyInProgressError = Class.new(StandardError) + UpdateError = Class.new(StandardError) + + include ApplicationWorker + include Gitlab::ShellAdapter + + sidekiq_options retry: 3, dead: false + + sidekiq_retry_in { |count| 30 * count } + + sidekiq_retries_exhausted do |msg, _| + Sidekiq.logger.warn "Failed #{msg['class']} with #{msg['args']}: #{msg['error_message']}" + end + + def perform(remote_mirror_id, scheduled_time) + remote_mirror = RemoteMirror.find(remote_mirror_id) + return if remote_mirror.updated_since?(scheduled_time) + + raise UpdateAlreadyInProgressError if remote_mirror.update_in_progress? + + remote_mirror.update_start + + project = remote_mirror.project + current_user = project.creator + result = Projects::UpdateRemoteMirrorService.new(project, current_user).execute(remote_mirror) + raise UpdateError, result[:message] if result[:status] == :error + + remote_mirror.update_finish + rescue UpdateAlreadyInProgressError + raise + rescue UpdateError => ex + fail_remote_mirror(remote_mirror, ex.message) + raise + rescue => ex + return unless remote_mirror + + fail_remote_mirror(remote_mirror, ex.message) + raise UpdateError, "#{ex.class}: #{ex.message}" + end + + private + + def fail_remote_mirror(remote_mirror, message) + remote_mirror.mark_as_failed(message) + + Rails.logger.error(message) + end +end |