diff options
53 files changed, 1971 insertions, 3 deletions
diff --git a/app/controllers/projects/mirrors_controller.rb b/app/controllers/projects/mirrors_controller.rb new file mode 100644 index 00000000000..5698ff4e706 --- /dev/null +++ b/app/controllers/projects/mirrors_controller.rb @@ -0,0 +1,67 @@ +class Projects::MirrorsController < Projects::ApplicationController + include RepositorySettingsRedirect + + # Authorize + before_action :remote_mirror, only: [:update] + before_action :check_mirror_available! + before_action :authorize_admin_project! + + layout "project_settings" + + def show + redirect_to_repository_settings(project) + end + + def update + if project.update_attributes(mirror_params) + flash[:notice] = 'Mirroring settings were successfully updated.' + else + flash[:alert] = project.errors.full_messages.join(', ').html_safe + end + + respond_to do |format| + format.html { redirect_to_repository_settings(project) } + format.json do + if project.errors.present? + render json: project.errors, status: :unprocessable_entity + else + render json: ProjectMirrorSerializer.new.represent(project) + end + end + end + end + + def update_now + if params[:sync_remote] + project.update_remote_mirrors + flash[:notice] = "The remote repository is being updated..." + end + + redirect_to_repository_settings(project) + end + + private + + def remote_mirror + @remote_mirror = project.remote_mirrors.first_or_initialize + end + + def check_mirror_available! + Gitlab::CurrentSettings.current_application_settings.mirror_available || current_user&.admin? + end + + def mirror_params_attributes + [ + remote_mirrors_attributes: %i[ + url + id + enabled + only_protected_branches + ] + ] + end + + def mirror_params + params.require(:project).permit(mirror_params_attributes) + end +end diff --git a/app/controllers/projects/settings/repository_controller.rb b/app/controllers/projects/settings/repository_controller.rb index f17056f13e0..4697af4f26a 100644 --- a/app/controllers/projects/settings/repository_controller.rb +++ b/app/controllers/projects/settings/repository_controller.rb @@ -2,6 +2,7 @@ module Projects module Settings class RepositoryController < Projects::ApplicationController before_action :authorize_admin_project! + before_action :remote_mirror, only: [:show] def show render_show @@ -25,6 +26,7 @@ module Projects define_deploy_token define_protected_refs + remote_mirror render 'show' end @@ -41,6 +43,10 @@ module Projects load_gon_index end + def remote_mirror + @remote_mirror = project.remote_mirrors.first_or_initialize + end + def access_levels_options { create_access_levels: levels_for_dropdown, diff --git a/app/helpers/application_settings_helper.rb b/app/helpers/application_settings_helper.rb index 1bf98d550b0..b948e431882 100644 --- a/app/helpers/application_settings_helper.rb +++ b/app/helpers/application_settings_helper.rb @@ -250,7 +250,8 @@ module ApplicationSettingsHelper :version_check_enabled, :allow_local_requests_from_hooks_and_services, :enforce_terms, - :terms + :terms, + :mirror_available ] end end diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index a734cc7a26b..451e512aef7 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -334,7 +334,8 @@ class ApplicationSetting < ActiveRecord::Base gitaly_timeout_fast: 10, gitaly_timeout_medium: 30, gitaly_timeout_default: 55, - allow_local_requests_from_hooks_and_services: false + allow_local_requests_from_hooks_and_services: false, + mirror_available: true } end diff --git a/app/models/project.rb b/app/models/project.rb index 37f1fcea280..0a549d843f3 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -65,6 +65,9 @@ class Project < ActiveRecord::Base default_value_for :only_allow_merge_if_all_discussions_are_resolved, false add_authentication_token_field :runners_token + + before_validation :mark_remote_mirrors_for_removal, if: -> { ActiveRecord::Base.connection.table_exists?(:remote_mirrors) } + before_save :ensure_runners_token after_save :update_project_statistics, if: :namespace_id_changed? @@ -246,11 +249,17 @@ class Project < ActiveRecord::Base has_many :project_badges, class_name: 'ProjectBadge' has_one :ci_cd_settings, class_name: 'ProjectCiCdSetting', inverse_of: :project, autosave: true, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :remote_mirrors, inverse_of: :project + accepts_nested_attributes_for :variables, allow_destroy: true accepts_nested_attributes_for :project_feature, update_only: true accepts_nested_attributes_for :import_data accepts_nested_attributes_for :auto_devops, update_only: true + accepts_nested_attributes_for :remote_mirrors, + allow_destroy: true, + reject_if: ->(attrs) { attrs[:id].blank? && attrs[:url].blank? } + delegate :name, to: :owner, allow_nil: true, prefix: true delegate :members, to: :team, prefix: true delegate :add_user, :add_users, to: :team @@ -340,6 +349,7 @@ class Project < ActiveRecord::Base scope :with_issues_enabled, -> { with_feature_enabled(:issues) } scope :with_issues_available_for_user, ->(current_user) { with_feature_available_for_user(:issues, current_user) } scope :with_merge_requests_enabled, -> { with_feature_enabled(:merge_requests) } + scope :with_remote_mirrors, -> { joins(:remote_mirrors).where(remote_mirrors: { enabled: true }).distinct } scope :with_group_runners_enabled, -> do joins(:ci_cd_settings) @@ -759,6 +769,37 @@ class Project < ActiveRecord::Base import_type == 'gitea' end + def has_remote_mirror? + remote_mirror_available? && remote_mirrors.enabled.exists? + end + + def updating_remote_mirror? + remote_mirrors.enabled.started.exists? + end + + def update_remote_mirrors + return unless remote_mirror_available? + + remote_mirrors.enabled.each(&:sync) + end + + def mark_stuck_remote_mirrors_as_failed! + remote_mirrors.stuck.update_all( + update_status: :failed, + last_error: 'The remote mirror took to long to complete.', + last_update_at: Time.now + ) + end + + def mark_remote_mirrors_for_removal + remote_mirrors.each(&:mark_for_delete_if_blank_url) + end + + def remote_mirror_available? + remote_mirror_available_overridden || + ::Gitlab::CurrentSettings.mirror_available + end + def check_limit unless creator.can_create_project? || namespace.kind == 'group' projects_limit = creator.projects_limit diff --git a/app/models/remote_mirror.rb b/app/models/remote_mirror.rb new file mode 100644 index 00000000000..bbf8fd9c6a7 --- /dev/null +++ b/app/models/remote_mirror.rb @@ -0,0 +1,219 @@ +class RemoteMirror < ActiveRecord::Base + include AfterCommitQueue + + PROTECTED_BACKOFF_DELAY = 1.minute + UNPROTECTED_BACKOFF_DELAY = 5.minutes + + attr_encrypted :credentials, + key: Gitlab::Application.secrets.db_key_base, + marshal: true, + encode: true, + mode: :per_attribute_iv_and_salt, + insecure_mode: true, + algorithm: 'aes-256-cbc' + + default_value_for :only_protected_branches, true + + belongs_to :project, inverse_of: :remote_mirrors + + validates :url, presence: true, url: { protocols: %w(ssh git http https), allow_blank: true } + validates :url, addressable_url: true, if: :url_changed? + + before_save :set_new_remote_name, if: :mirror_url_changed? + + after_save :set_override_remote_mirror_available, unless: -> { Gitlab::CurrentSettings.current_application_settings.mirror_available } + after_save :refresh_remote, if: :mirror_url_changed? + after_update :reset_fields, if: :mirror_url_changed? + + after_commit :remove_remote, on: :destroy + + scope :enabled, -> { where(enabled: true) } + scope :started, -> { with_update_status(:started) } + scope :stuck, -> { started.where('last_update_at < ? OR (last_update_at IS NULL AND updated_at < ?)', 1.day.ago, 1.day.ago) } + + state_machine :update_status, initial: :none do + event :update_start do + transition [:none, :finished, :failed] => :started + end + + event :update_finish do + transition started: :finished + end + + event :update_fail do + transition started: :failed + end + + state :started + state :finished + state :failed + + after_transition any => :started do |remote_mirror, _| + Gitlab::Metrics.add_event(:remote_mirrors_running, path: remote_mirror.project.full_path) + + remote_mirror.update(last_update_started_at: Time.now) + end + + after_transition started: :finished do |remote_mirror, _| + Gitlab::Metrics.add_event(:remote_mirrors_finished, path: remote_mirror.project.full_path) + + timestamp = Time.now + remote_mirror.update_attributes!( + last_update_at: timestamp, last_successful_update_at: timestamp, last_error: nil + ) + end + + after_transition started: :failed do |remote_mirror, _| + Gitlab::Metrics.add_event(:remote_mirrors_failed, path: remote_mirror.project.full_path) + + remote_mirror.update(last_update_at: Time.now) + end + end + + def remote_name + super || fallback_remote_name + end + + def update_failed? + update_status == 'failed' + end + + def update_in_progress? + update_status == 'started' + end + + def update_repository(options) + raw.update(options) + end + + def sync? + enabled? + end + + def sync + return unless sync? + + if recently_scheduled? + RepositoryUpdateRemoteMirrorWorker.perform_in(backoff_delay, self.id, Time.now) + else + RepositoryUpdateRemoteMirrorWorker.perform_async(self.id, Time.now) + end + end + + def enabled + return false unless project && super + return false unless project.remote_mirror_available? + return false unless project.repository_exists? + return false if project.pending_delete? + + true + end + alias_method :enabled?, :enabled + + def updated_since?(timestamp) + last_update_started_at && last_update_started_at > timestamp && !update_failed? + end + + def mark_for_delete_if_blank_url + mark_for_destruction if url.blank? + end + + def mark_as_failed(error_message) + update_fail + update_column(:last_error, Gitlab::UrlSanitizer.sanitize(error_message)) + end + + def url=(value) + super(value) && return unless Gitlab::UrlSanitizer.valid?(value) + + mirror_url = Gitlab::UrlSanitizer.new(value) + self.credentials = mirror_url.credentials + + super(mirror_url.sanitized_url) + end + + def url + if super + Gitlab::UrlSanitizer.new(super, credentials: credentials).full_url + end + rescue + super + end + + def safe_url + return if url.nil? + + result = URI.parse(url) + result.password = '*****' if result.password + result.user = '*****' if result.user && result.user != "git" # tokens or other data may be saved as user + result.to_s + end + + private + + def raw + @raw ||= Gitlab::Git::RemoteMirror.new(project.repository.raw, remote_name) + end + + def fallback_remote_name + return unless id + + "remote_mirror_#{id}" + end + + def recently_scheduled? + return false unless self.last_update_started_at + + self.last_update_started_at >= Time.now - backoff_delay + end + + def backoff_delay + if self.only_protected_branches + PROTECTED_BACKOFF_DELAY + else + UNPROTECTED_BACKOFF_DELAY + end + end + + def reset_fields + update_columns( + last_error: nil, + last_update_at: nil, + last_successful_update_at: nil, + update_status: 'finished' + ) + end + + def set_override_remote_mirror_available + enabled = read_attribute(:enabled) + + project.update(remote_mirror_available_overridden: enabled) + end + + def set_new_remote_name + self.remote_name = "remote_mirror_#{SecureRandom.hex}" + end + + def refresh_remote + return unless project + + # Before adding a new remote we have to delete the data from + # the previous remote name + prev_remote_name = remote_name_was || fallback_remote_name + run_after_commit do + project.repository.async_remove_remote(prev_remote_name) + end + + project.repository.add_remote(remote_name, url) + end + + def remove_remote + return unless project # could be pending to delete so don't need to touch the git repository + + project.repository.async_remove_remote(remote_name) + end + + def mirror_url_changed? + url_changed? || encrypted_credentials_changed? + end +end diff --git a/app/models/repository.rb b/app/models/repository.rb index 6831305fb93..b75c4aca982 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -854,13 +854,27 @@ class Repository add_remote(remote_name, url, mirror_refmap: refmap) fetch_remote(remote_name, forced: forced, prune: prune) ensure - remove_remote(remote_name) if tmp_remote_name + async_remove_remote(remote_name) if tmp_remote_name end def fetch_remote(remote, forced: false, ssh_auth: nil, no_tags: false, prune: true) gitlab_shell.fetch_remote(raw_repository, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, prune: prune) end + def async_remove_remote(remote_name) + return unless remote_name + + job_id = RepositoryRemoveRemoteWorker.perform_async(project.id, remote_name) + + if job_id + Rails.logger.info("Remove remote job scheduled for #{project.id} with remote name: #{remote_name} job ID #{job_id}.") + else + Rails.logger.info("Remove remote job failed to create for #{project.id} with remote name #{remote_name}.") + end + + job_id + end + def fetch_source_branch!(source_repository, source_branch, local_ref) raw_repository.fetch_source_branch!(source_repository.raw_repository, source_branch, local_ref) end diff --git a/app/policies/project_policy.rb b/app/policies/project_policy.rb index 3529d0aa60c..5759b1a376f 100644 --- a/app/policies/project_policy.rb +++ b/app/policies/project_policy.rb @@ -80,6 +80,11 @@ class ProjectPolicy < BasePolicy project.merge_requests_allowing_push_to_user(user).any? end + with_scope :global + condition(:mirror_available, score: 0) do + ::Gitlab::CurrentSettings.current_application_settings.mirror_available + end + # We aren't checking `:read_issue` or `:read_merge_request` in this case # because it could be possible for a user to see an issuable-iid # (`:read_issue_iid` or `:read_merge_request_iid`) but then wouldn't be @@ -246,6 +251,8 @@ class ProjectPolicy < BasePolicy enable :create_cluster end + rule { (mirror_available & can?(:admin_project)) | admin }.enable :admin_remote_mirror + rule { archived }.policy do prevent :push_code prevent :push_to_delete_protected_branch diff --git a/app/serializers/project_mirror_entity.rb b/app/serializers/project_mirror_entity.rb new file mode 100644 index 00000000000..a9c08ac021a --- /dev/null +++ b/app/serializers/project_mirror_entity.rb @@ -0,0 +1,11 @@ +class ProjectMirrorEntity < Grape::Entity + expose :id + + expose :remote_mirrors_attributes do |project| + next [] unless project.remote_mirrors.present? + + project.remote_mirrors.map do |remote| + remote.as_json(only: %i[id url enabled]) + end + end +end diff --git a/app/services/concerns/exclusive_lease_guard.rb b/app/services/concerns/exclusive_lease_guard.rb new file mode 100644 index 00000000000..30be6accc32 --- /dev/null +++ b/app/services/concerns/exclusive_lease_guard.rb @@ -0,0 +1,52 @@ +# +# Concern that helps with getting an exclusive lease for running a block +# of code. +# +# `#try_obtain_lease` takes a block which will be run if it was able to +# obtain the lease. Implement `#lease_timeout` to configure the timeout +# for the exclusive lease. Optionally override `#lease_key` to set the +# lease key, it defaults to the class name with underscores. +# +module ExclusiveLeaseGuard + extend ActiveSupport::Concern + + def try_obtain_lease + lease = exclusive_lease.try_obtain + + unless lease + log_error('Cannot obtain an exclusive lease. There must be another instance already in execution.') + return + end + + begin + yield lease + ensure + release_lease(lease) + end + end + + def exclusive_lease + @lease ||= Gitlab::ExclusiveLease.new(lease_key, timeout: lease_timeout) + end + + def lease_key + @lease_key ||= self.class.name.underscore + end + + def lease_timeout + raise NotImplementedError, + "#{self.class.name} does not implement #{__method__}" + end + + def release_lease(uuid) + Gitlab::ExclusiveLease.cancel(lease_key, uuid) + end + + def renew_lease! + exclusive_lease.renew + end + + def log_error(message, extra_args = {}) + logger.error(message) + end +end diff --git a/app/services/git_push_service.rb b/app/services/git_push_service.rb index c037141fcde..f3bfc53dcd3 100644 --- a/app/services/git_push_service.rb +++ b/app/services/git_push_service.rb @@ -55,6 +55,7 @@ class GitPushService < BaseService execute_related_hooks perform_housekeeping + update_remote_mirrors update_caches update_signatures @@ -119,6 +120,13 @@ class GitPushService < BaseService protected + def update_remote_mirrors + return unless @project.has_remote_mirror? + + @project.mark_stuck_remote_mirrors_as_failed! + @project.update_remote_mirrors + end + def execute_related_hooks # Update merge requests that may be affected by this push. A new branch # could cause the last commit of a merge request to change. diff --git a/app/services/projects/update_remote_mirror_service.rb b/app/services/projects/update_remote_mirror_service.rb new file mode 100644 index 00000000000..8183a2f26d7 --- /dev/null +++ b/app/services/projects/update_remote_mirror_service.rb @@ -0,0 +1,30 @@ +module Projects + class UpdateRemoteMirrorService < BaseService + attr_reader :errors + + def execute(remote_mirror) + @errors = [] + + return success unless remote_mirror.enabled? + + begin + repository.fetch_remote(remote_mirror.remote_name, no_tags: true) + + opts = {} + if remote_mirror.only_protected_branches? + opts[:only_branches_matching] = project.protected_branches.select(:name).map(&:name) + end + + remote_mirror.update_repository(opts) + rescue => e + errors << e.message.strip + end + + if errors.present? + error(errors.join("\n\n")) + else + success + end + end + end +end diff --git a/app/views/admin/application_settings/_repository_mirrors_form.html.haml b/app/views/admin/application_settings/_repository_mirrors_form.html.haml new file mode 100644 index 00000000000..09183ec6260 --- /dev/null +++ b/app/views/admin/application_settings/_repository_mirrors_form.html.haml @@ -0,0 +1,16 @@ += form_for @application_setting, url: admin_application_settings_path, html: { class: 'form-horizontal fieldset-form' } do |f| + = form_errors(@application_setting) + + %fieldset + .form-group + = f.label :mirror_available, 'Enable mirror configuration', class: 'control-label col-sm-2' + .col-sm-10 + .checkbox + = f.label :mirror_available do + = f.check_box :mirror_available + Allow mirrors to be setup for projects + %span.help-block + If disabled, only admins will be able to setup mirrors in projects. + = link_to icon('question-circle'), help_page_path('workflow/repository_mirroring') + + = f.submit 'Save changes', class: "btn btn-success" diff --git a/app/views/admin/application_settings/show.html.haml b/app/views/admin/application_settings/show.html.haml index 3c00e3c8fc4..3f440c76ee0 100644 --- a/app/views/admin/application_settings/show.html.haml +++ b/app/views/admin/application_settings/show.html.haml @@ -313,3 +313,14 @@ = _('Allow requests to the local network from hooks and services.') .settings-content = render 'outbound' + +%section.settings.as-mirror.no-animate#js-mirror-settings{ class: ('expanded' if expanded) } + .settings-header + %h4 + = _('Repository mirror settings') + %button.btn.js-settings-toggle{ type: 'button' } + = expanded ? 'Collapse' : 'Expand' + %p + = _('Configure push mirrors.') + .settings-content + = render partial: 'repository_mirrors_form' diff --git a/app/views/projects/mirrors/_instructions.html.haml b/app/views/projects/mirrors/_instructions.html.haml new file mode 100644 index 00000000000..64f0fde30cf --- /dev/null +++ b/app/views/projects/mirrors/_instructions.html.haml @@ -0,0 +1,10 @@ +.account-well.prepend-top-default.append-bottom-default + %ul + %li + The repository must be accessible over <code>http://</code>, <code>https://</code>, <code>ssh://</code> or <code>git://</code>. + %li + Include the username in the URL if required: <code>https://username@gitlab.company.com/group/project.git</code>. + %li + The update action will time out after 10 minutes. For big repositories, use a clone/push combination. + %li + The Git LFS objects will <strong>not</strong> be synced. diff --git a/app/views/projects/mirrors/_push.html.haml b/app/views/projects/mirrors/_push.html.haml new file mode 100644 index 00000000000..4a6aefce351 --- /dev/null +++ b/app/views/projects/mirrors/_push.html.haml @@ -0,0 +1,50 @@ +- expanded = Rails.env.test? +%section.settings.no-animate{ class: ('expanded' if expanded) } + .settings-header + %h4 + Push to a remote repository + %button.btn.js-settings-toggle + = expanded ? 'Collapse' : 'Expand' + %p + Set up the remote repository that you want to update with the content of the current repository + every time someone pushes to it. + = link_to 'Read more', help_page_path('workflow/repository_mirroring', anchor: 'pushing-to-a-remote-repository'), target: '_blank' + .settings-content + = form_for @project, url: project_mirror_path(@project) do |f| + %div + = form_errors(@project) + = render "shared/remote_mirror_update_button", remote_mirror: @remote_mirror + - if @remote_mirror.last_error.present? + .panel.panel-danger + .panel-heading + - if @remote_mirror.last_update_at + The remote repository failed to update #{time_ago_with_tooltip(@remote_mirror.last_update_at)}. + - else + The remote repository failed to update. + + - if @remote_mirror.last_successful_update_at + Last successful update #{time_ago_with_tooltip(@remote_mirror.last_successful_update_at)}. + .panel-body + %pre + :preserve + #{h(@remote_mirror.last_error.strip)} + = f.fields_for :remote_mirrors, @remote_mirror do |rm_form| + .form-group + = rm_form.check_box :enabled, class: "pull-left" + .prepend-left-20 + = rm_form.label :enabled, "Remote mirror repository", class: "label-light append-bottom-0" + %p.light.append-bottom-0 + Automatically update the remote mirror's branches, tags, and commits from this repository every time someone pushes to it. + .form-group.has-feedback + = rm_form.label :url, "Git repository URL", class: "label-light" + = rm_form.text_field :url, class: "form-control", placeholder: 'https://username:password@gitlab.company.com/group/project.git' + + = render "projects/mirrors/instructions" + + .form-group + = rm_form.check_box :only_protected_branches, class: 'pull-left' + .prepend-left-20 + = rm_form.label :only_protected_branches, class: 'label-light' + = link_to icon('question-circle'), help_page_path('user/project/protected_branches') + + = f.submit 'Save changes', class: 'btn btn-create', name: 'update_remote_mirror' diff --git a/app/views/projects/mirrors/_show.html.haml b/app/views/projects/mirrors/_show.html.haml new file mode 100644 index 00000000000..de77701a373 --- /dev/null +++ b/app/views/projects/mirrors/_show.html.haml @@ -0,0 +1,3 @@ +- if can?(current_user, :admin_remote_mirror, @project) + = render 'projects/mirrors/push' + diff --git a/app/views/projects/settings/repository/show.html.haml b/app/views/projects/settings/repository/show.html.haml index f57590a908f..5dda2ec28b4 100644 --- a/app/views/projects/settings/repository/show.html.haml +++ b/app/views/projects/settings/repository/show.html.haml @@ -2,6 +2,8 @@ - page_title "Repository" - @content_class = "limit-container-width" unless fluid_layout += render "projects/mirrors/show" + -# Protected branches & tags use a lot of nested partials. -# The shared parts of the views can be found in the `shared` directory. -# Those are used throughout the actual views. These `shared` views are then diff --git a/app/views/shared/_remote_mirror_update_button.html.haml b/app/views/shared/_remote_mirror_update_button.html.haml new file mode 100644 index 00000000000..34de1c0695f --- /dev/null +++ b/app/views/shared/_remote_mirror_update_button.html.haml @@ -0,0 +1,13 @@ +- if @project.has_remote_mirror? + .append-bottom-default + - if remote_mirror.update_in_progress? + %span.btn.disabled + = icon("refresh spin") + Updating… + - else + = link_to update_now_project_mirror_path(@project, sync_remote: true), method: :post, class: "btn" do + = icon("refresh") + Update Now + - if @remote_mirror.last_successful_update_at + %p.inline.prepend-left-10 + Successfully updated #{time_ago_with_tooltip(@remote_mirror.last_successful_update_at)}. diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 5d9ec6142d7..b6433eb3eff 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -107,9 +107,11 @@ - rebase - repository_fork - repository_import +- repository_remove_remote - storage_migrator - system_hook_push - update_merge_requests - update_user_activity - upload_checksum - web_hook +- repository_update_remote_mirror diff --git a/app/workers/repository_remove_remote_worker.rb b/app/workers/repository_remove_remote_worker.rb new file mode 100644 index 00000000000..1c19b604b77 --- /dev/null +++ b/app/workers/repository_remove_remote_worker.rb @@ -0,0 +1,35 @@ +class RepositoryRemoveRemoteWorker + include ApplicationWorker + include ExclusiveLeaseGuard + + LEASE_TIMEOUT = 1.hour + + attr_reader :project, :remote_name + + def perform(project_id, remote_name) + @remote_name = remote_name + @project = Project.find_by_id(project_id) + + return unless @project + + logger.info("Removing remote #{remote_name} from project #{project.id}") + + try_obtain_lease do + remove_remote = @project.repository.remove_remote(remote_name) + + if remove_remote + logger.info("Remote #{remote_name} was successfully removed from project #{project.id}") + else + logger.error("Could not remove remote #{remote_name} from project #{project.id}") + end + end + end + + def lease_timeout + LEASE_TIMEOUT + end + + def lease_key + "remove_remote_#{project.id}_#{remote_name}" + end +end diff --git a/app/workers/repository_update_remote_mirror_worker.rb b/app/workers/repository_update_remote_mirror_worker.rb new file mode 100644 index 00000000000..bb963979e88 --- /dev/null +++ b/app/workers/repository_update_remote_mirror_worker.rb @@ -0,0 +1,49 @@ +class RepositoryUpdateRemoteMirrorWorker + UpdateAlreadyInProgressError = Class.new(StandardError) + UpdateError = Class.new(StandardError) + + include ApplicationWorker + include Gitlab::ShellAdapter + + sidekiq_options retry: 3, dead: false + + sidekiq_retry_in { |count| 30 * count } + + sidekiq_retries_exhausted do |msg, _| + Sidekiq.logger.warn "Failed #{msg['class']} with #{msg['args']}: #{msg['error_message']}" + end + + def perform(remote_mirror_id, scheduled_time) + remote_mirror = RemoteMirror.find(remote_mirror_id) + return if remote_mirror.updated_since?(scheduled_time) + + raise UpdateAlreadyInProgressError if remote_mirror.update_in_progress? + + remote_mirror.update_start + + project = remote_mirror.project + current_user = project.creator + result = Projects::UpdateRemoteMirrorService.new(project, current_user).execute(remote_mirror) + raise UpdateError, result[:message] if result[:status] == :error + + remote_mirror.update_finish + rescue UpdateAlreadyInProgressError + raise + rescue UpdateError => ex + fail_remote_mirror(remote_mirror, ex.message) + raise + rescue => ex + return unless remote_mirror + + fail_remote_mirror(remote_mirror, ex.message) + raise UpdateError, "#{ex.class}: #{ex.message}" + end + + private + + def fail_remote_mirror(remote_mirror, message) + remote_mirror.mark_as_failed(message) + + Rails.logger.error(message) + end +end diff --git a/changelogs/unreleased/42099-port-push-mirroring-to-ce-ce-port-v-2.yml b/changelogs/unreleased/42099-port-push-mirroring-to-ce-ce-port-v-2.yml new file mode 100644 index 00000000000..f23521ea416 --- /dev/null +++ b/changelogs/unreleased/42099-port-push-mirroring-to-ce-ce-port-v-2.yml @@ -0,0 +1,5 @@ +--- +title: Adds push mirrors to GitLab Community Edition +merge_request: 18715 +author: +type: changed diff --git a/config/routes/project.rb b/config/routes/project.rb index f36341cdcaf..5a1be1a8b73 100644 --- a/config/routes/project.rb +++ b/config/routes/project.rb @@ -174,6 +174,12 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do end end + resource :mirror, only: [:show, :update] do + member do + post :update_now + end + end + resources :pipelines, only: [:index, :new, :create, :show] do collection do resource :pipelines_settings, path: 'settings', only: [:show, :update] diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index 47fbbed44cf..e1e8f36b663 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -73,3 +73,6 @@ - [object_storage, 1] - [plugin, 1] - [pipeline_background, 1] + - [repository_update_remote_mirror, 1] + - [repository_remove_remote, 1] + diff --git a/db/migrate/20180503131624_create_remote_mirrors.rb b/db/migrate/20180503131624_create_remote_mirrors.rb new file mode 100644 index 00000000000..7800186455f --- /dev/null +++ b/db/migrate/20180503131624_create_remote_mirrors.rb @@ -0,0 +1,33 @@ +class CreateRemoteMirrors < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + return if table_exists?(:remote_mirrors) + + create_table :remote_mirrors do |t| + t.references :project, index: true, foreign_key: { on_delete: :cascade } + t.string :url + t.boolean :enabled, default: true + t.string :update_status + t.datetime :last_update_at + t.datetime :last_successful_update_at + t.datetime :last_update_started_at + t.string :last_error + t.boolean :only_protected_branches, default: false, null: false + t.string :remote_name + t.text :encrypted_credentials + t.string :encrypted_credentials_iv + t.string :encrypted_credentials_salt + + t.timestamps null: false + end + end + + def down + drop_table(:remote_mirrors) if table_exists?(:remote_mirrors) + end +end diff --git a/db/migrate/20180503141722_add_remote_mirror_available_overridden_to_projects.rb b/db/migrate/20180503141722_add_remote_mirror_available_overridden_to_projects.rb new file mode 100644 index 00000000000..841393971f4 --- /dev/null +++ b/db/migrate/20180503141722_add_remote_mirror_available_overridden_to_projects.rb @@ -0,0 +1,15 @@ +class AddRemoteMirrorAvailableOverriddenToProjects < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_column(:projects, :remote_mirror_available_overridden, :boolean) unless column_exists?(:projects, :remote_mirror_available_overridden) + end + + def down + remove_column(:projects, :remote_mirror_available_overridden) if column_exists?(:projects, :remote_mirror_available_overridden) + end +end diff --git a/db/migrate/20180503193542_add_indexes_to_remote_mirror.rb b/db/migrate/20180503193542_add_indexes_to_remote_mirror.rb new file mode 100644 index 00000000000..9a9decffdab --- /dev/null +++ b/db/migrate/20180503193542_add_indexes_to_remote_mirror.rb @@ -0,0 +1,15 @@ +class AddIndexesToRemoteMirror < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index :remote_mirrors, :last_successful_update_at unless index_exists?(:remote_mirrors, :last_successful_update_at) + end + + def down + remove_index :remote_mirrors, :last_successful_update_at if index_exists? :remote_mirrors, :last_successful_update_at + end +end diff --git a/db/migrate/20180503193953_add_mirror_available_to_application_settings.rb b/db/migrate/20180503193953_add_mirror_available_to_application_settings.rb new file mode 100644 index 00000000000..25b9905b1a9 --- /dev/null +++ b/db/migrate/20180503193953_add_mirror_available_to_application_settings.rb @@ -0,0 +1,15 @@ +class AddMirrorAvailableToApplicationSettings < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_column_with_default(:application_settings, :mirror_available, :boolean, default: true, allow_null: false) unless column_exists?(:application_settings, :mirror_available) + end + + def down + remove_column(:application_settings, :mirror_available) if column_exists?(:application_settings, :mirror_available) + end +end diff --git a/db/schema.rb b/db/schema.rb index 9c6caaeb7ef..65e9cc4ea08 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -165,6 +165,7 @@ ActiveRecord::Schema.define(version: 20180503200320) do t.boolean "pages_domain_verification_enabled", default: true, null: false t.boolean "allow_local_requests_from_hooks_and_services", default: false, null: false t.boolean "enforce_terms", default: false + t.boolean "mirror_available", default: true, null: false end create_table "audit_events", force: :cascade do |t| @@ -1602,6 +1603,7 @@ ActiveRecord::Schema.define(version: 20180503200320) do t.boolean "merge_requests_rebase_enabled", default: false, null: false t.integer "jobs_cache_index" t.boolean "pages_https_only", default: true + t.boolean "remote_mirror_available_overridden" end add_index "projects", ["ci_id"], name: "index_projects_on_ci_id", using: :btree @@ -1707,6 +1709,27 @@ ActiveRecord::Schema.define(version: 20180503200320) do add_index "releases", ["project_id", "tag"], name: "index_releases_on_project_id_and_tag", using: :btree add_index "releases", ["project_id"], name: "index_releases_on_project_id", using: :btree + create_table "remote_mirrors", force: :cascade do |t| + t.integer "project_id" + t.string "url" + t.boolean "enabled", default: true + t.string "update_status" + t.datetime "last_update_at" + t.datetime "last_successful_update_at" + t.datetime "last_update_started_at" + t.string "last_error" + t.boolean "only_protected_branches", default: false, null: false + t.string "remote_name" + t.text "encrypted_credentials" + t.string "encrypted_credentials_iv" + t.string "encrypted_credentials_salt" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + end + + add_index "remote_mirrors", ["last_successful_update_at"], name: "index_remote_mirrors_on_last_successful_update_at", using: :btree + add_index "remote_mirrors", ["project_id"], name: "index_remote_mirrors_on_project_id", using: :btree + create_table "routes", force: :cascade do |t| t.integer "source_id", null: false t.string "source_type", null: false @@ -2243,6 +2266,7 @@ ActiveRecord::Schema.define(version: 20180503200320) do add_foreign_key "protected_tags", "projects", name: "fk_8e4af87648", on_delete: :cascade add_foreign_key "push_event_payloads", "events", name: "fk_36c74129da", on_delete: :cascade add_foreign_key "releases", "projects", name: "fk_47fe2a0596", on_delete: :cascade + add_foreign_key "remote_mirrors", "projects", on_delete: :cascade add_foreign_key "services", "projects", name: "fk_71cce407f9", on_delete: :cascade add_foreign_key "snippets", "projects", name: "fk_be41fd4bb7", on_delete: :cascade add_foreign_key "subscriptions", "projects", on_delete: :cascade diff --git a/doc/workflow/repository_mirroring.md b/doc/workflow/repository_mirroring.md new file mode 100644 index 00000000000..dbe63144e38 --- /dev/null +++ b/doc/workflow/repository_mirroring.md @@ -0,0 +1,111 @@ +# Repository mirroring + +Repository Mirroring is a way to mirror repositories from external sources. +It can be used to mirror all branches, tags, and commits that you have +in your repository. + +Your mirror at GitLab will be updated automatically. You can +also manually trigger an update at most once every 5 minutes. + +## Overview + +Repository mirroring is very useful when, for some reason, you must use a +project from another source. + +There are two kinds of repository mirroring features supported by GitLab: +**push** and **pull**, the latter being only available in GitLab Enterprise Edition. +The **push** method mirrors the repository in GitLab to another location. + +Once the mirror repository is updated, all new branches, +tags, and commits will be visible in the project's activity feed. +Users with at least [developer access][perms] to the project can also force an +immediate update with the click of a button. This button will not be available if +the mirror is already being updated or 5 minutes still haven't passed since its last update. + +A few things/limitations to consider: + +- The repository must be accessible over `http://`, `https://`, `ssh://` or `git://`. +- If your HTTP repository is not publicly accessible, add authentication + information to the URL, like: `https://username@gitlab.company.com/group/project.git`. + In some cases, you might need to use a personal access token instead of a + password, e.g., you want to mirror to GitHub and have 2FA enabled. +- The import will time out after 15 minutes. For repositories that take longer + use a clone/push combination. +- The Git LFS objects will not be synced. You'll need to push/pull them + manually. + +## Use-case + +- You have old projects in another source that you don't use actively anymore, + but don't want to remove for archiving purposes. In that case, you can create + a push mirror so that your active GitLab repository can push its changes to the + old location. + +## Pushing to a remote repository **[STARTER]** + +>[Introduced](https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/249) in +GitLab Enterprise Edition 8.7. [Moved to GitLab Community Edition][ce-18715] in 10.8. + +For an existing project, you can set up push mirror from your project's +**Settings âž” Repository** and searching for the "Push to a remote repository" +section. Check the "Remote mirror repository" box and fill in the Git URL of +the repository to push to. Click **Save changes** for the changes to take +effect. + +![Push settings](repository_mirroring/repository_mirroring_push_settings.png) + +When push mirroring is enabled, you are advised not to push commits directly +to the mirrored repository to prevent the mirror diverging. +All changes will end up in the mirrored repository whenever commits +are pushed to GitLab, or when a [forced update](#forcing-an-update) is +initiated. + +Pushes into GitLab are automatically pushed to the remote mirror at least once +every 5 minutes after they are received or once every minute if **push only +protected branches** is enabled. + +In case of a diverged branch, you will see an error indicated at the **Mirror +repository** settings. + +![Diverged branch]( +repository_mirroring/repository_mirroring_diverged_branch_push.png) + +### Push only protected branches + +>[Introduced][ee-3350] in GitLab Enterprise Edition 10.3. [Moved to GitLab Community Edition][ce-18715] in 10.8. + +You can choose to only push your protected branches from GitLab to your remote repository. + +To use this option go to your project's repository settings page under push mirror. + +## Setting up a push mirror from GitLab to GitHub + +To set up a mirror from GitLab to GitHub, you need to follow these steps: + +1. Create a [GitHub personal access token](https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/) with the "public_repo" box checked: + + ![edit personal access token GitHub](repository_mirroring/repository_mirroring_github_edit_personal_access_token.png) + +1. Fill in the "Git repository URL" with the personal access token replacing the password `https://GitHubUsername:GitHubPersonalAccessToken@github.com/group/project.git`: + + ![push to remote repo](repository_mirroring/repository_mirroring_gitlab_push_to_a_remote_repository.png) + +1. Save +1. And either wait or trigger the "Update Now" button: + + ![update now](repository_mirroring/repository_mirroring_gitlab_push_to_a_remote_repository_update_now.png) + +## Forcing an update + +While mirrors are scheduled to update automatically, you can always force an update +by using the **Update now** button which is exposed in various places: + +- in the commits page +- in the branches page +- in the tags page +- in the **Mirror repository** settings page + +[ee-3350]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/3350 +[ce-18715]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/18715 +[perms]: ../user/permissions.md + diff --git a/doc/workflow/repository_mirroring/repository_mirroring_diverged_branch_push.png b/doc/workflow/repository_mirroring/repository_mirroring_diverged_branch_push.png Binary files differnew file mode 100644 index 00000000000..038b05cb31d --- /dev/null +++ b/doc/workflow/repository_mirroring/repository_mirroring_diverged_branch_push.png diff --git a/doc/workflow/repository_mirroring/repository_mirroring_github_edit_personal_access_token.png b/doc/workflow/repository_mirroring/repository_mirroring_github_edit_personal_access_token.png Binary files differnew file mode 100644 index 00000000000..139de42d8db --- /dev/null +++ b/doc/workflow/repository_mirroring/repository_mirroring_github_edit_personal_access_token.png diff --git a/doc/workflow/repository_mirroring/repository_mirroring_gitlab_push_to_a_remote_repository.png b/doc/workflow/repository_mirroring/repository_mirroring_gitlab_push_to_a_remote_repository.png Binary files differnew file mode 100644 index 00000000000..ccbc1d92329 --- /dev/null +++ b/doc/workflow/repository_mirroring/repository_mirroring_gitlab_push_to_a_remote_repository.png diff --git a/doc/workflow/repository_mirroring/repository_mirroring_gitlab_push_to_a_remote_repository_update_now.png b/doc/workflow/repository_mirroring/repository_mirroring_gitlab_push_to_a_remote_repository_update_now.png Binary files differnew file mode 100644 index 00000000000..b16b3d2828e --- /dev/null +++ b/doc/workflow/repository_mirroring/repository_mirroring_gitlab_push_to_a_remote_repository_update_now.png diff --git a/doc/workflow/repository_mirroring/repository_mirroring_push_settings.png b/doc/workflow/repository_mirroring/repository_mirroring_push_settings.png Binary files differnew file mode 100644 index 00000000000..f8199aa7c0f --- /dev/null +++ b/doc/workflow/repository_mirroring/repository_mirroring_push_settings.png diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 0d1c4f73c6e..21ac7f7e0b6 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -106,6 +106,7 @@ excluded_attributes: - :last_repository_updated_at - :last_repository_check_at - :storage_version + - :remote_mirror_available_overridden - :description_html snippets: - :expired_at diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 8c0a4d55ea2..e294f3c4ebc 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -71,6 +71,7 @@ module Gitlab projects_imported_from_github: Project.where(import_type: 'github').count, protected_branches: ProtectedBranch.count, releases: Release.count, + remote_mirrors: RemoteMirror.count, snippets: Snippet.count, todos: Todo.count, uploads: Upload.count, diff --git a/spec/controllers/projects/mirrors_controller_spec.rb b/spec/controllers/projects/mirrors_controller_spec.rb new file mode 100644 index 00000000000..45c1218a39c --- /dev/null +++ b/spec/controllers/projects/mirrors_controller_spec.rb @@ -0,0 +1,72 @@ +require 'spec_helper' + +describe Projects::MirrorsController do + include ReactiveCachingHelpers + + describe 'setting up a remote mirror' do + set(:project) { create(:project, :repository) } + + context 'when the current project is not a mirror' do + it 'allows to create a remote mirror' do + sign_in(project.owner) + + expect do + do_put(project, remote_mirrors_attributes: { '0' => { 'enabled' => 1, 'url' => 'http://foo.com' } }) + end.to change { RemoteMirror.count }.to(1) + end + end + end + + describe '#update' do + let(:project) { create(:project, :repository, :remote_mirror) } + + before do + sign_in(project.owner) + end + + around do |example| + Sidekiq::Testing.fake! { example.run } + end + + context 'With valid URL for a push' do + let(:remote_mirror_attributes) do + { "0" => { "enabled" => "0", url: 'https://updated.example.com' } } + end + + it 'processes a successful update' do + do_put(project, remote_mirrors_attributes: remote_mirror_attributes) + + expect(response).to redirect_to(project_settings_repository_path(project)) + expect(flash[:notice]).to match(/successfully updated/) + end + + it 'should create a RemoteMirror object' do + expect { do_put(project, remote_mirrors_attributes: remote_mirror_attributes) }.to change(RemoteMirror, :count).by(1) + end + end + + context 'With invalid URL for a push' do + let(:remote_mirror_attributes) do + { "0" => { "enabled" => "0", url: 'ftp://invalid.invalid' } } + end + + it 'processes an unsuccessful update' do + do_put(project, remote_mirrors_attributes: remote_mirror_attributes) + + expect(response).to redirect_to(project_settings_repository_path(project)) + expect(flash[:alert]).to match(/must be a valid URL/) + end + + it 'should not create a RemoteMirror object' do + expect { do_put(project, remote_mirrors_attributes: remote_mirror_attributes) }.not_to change(RemoteMirror, :count) + end + end + end + + def do_put(project, options, extra_attrs = {}) + attrs = extra_attrs.merge(namespace_id: project.namespace.to_param, project_id: project.to_param) + attrs[:project] = options + + put :update, attrs + end +end diff --git a/spec/factories/projects.rb b/spec/factories/projects.rb index 9ab57af1c60..d129815aeac 100644 --- a/spec/factories/projects.rb +++ b/spec/factories/projects.rb @@ -183,6 +183,17 @@ FactoryBot.define do end end + trait :remote_mirror do + transient do + remote_name "remote_mirror_#{SecureRandom.hex}" + url "http://foo.com" + enabled true + end + after(:create) do |project, evaluator| + project.remote_mirrors.create!(url: evaluator.url, enabled: evaluator.enabled) + end + end + trait :stubbed_repository do after(:build) do |project| allow(project).to receive(:empty_repo?).and_return(false) diff --git a/spec/factories/remote_mirrors.rb b/spec/factories/remote_mirrors.rb new file mode 100644 index 00000000000..adc7da27522 --- /dev/null +++ b/spec/factories/remote_mirrors.rb @@ -0,0 +1,6 @@ +FactoryBot.define do + factory :remote_mirror, class: 'RemoteMirror' do + association :project, :repository + url "http://foo:bar@test.com" + end +end diff --git a/spec/features/projects/remote_mirror_spec.rb b/spec/features/projects/remote_mirror_spec.rb new file mode 100644 index 00000000000..81a6b613cc8 --- /dev/null +++ b/spec/features/projects/remote_mirror_spec.rb @@ -0,0 +1,34 @@ +require 'spec_helper' + +feature 'Project remote mirror', :feature do + let(:project) { create(:project, :repository, :remote_mirror) } + let(:remote_mirror) { project.remote_mirrors.first } + let(:user) { create(:user) } + + describe 'On a project', :js do + before do + project.add_master(user) + sign_in user + end + + context 'when last_error is present but last_update_at is not' do + it 'renders error message without timstamp' do + remote_mirror.update_attributes(last_error: 'Some new error', last_update_at: nil) + + visit project_mirror_path(project) + + expect(page).to have_content('The remote repository failed to update.') + end + end + + context 'when last_error and last_update_at are present' do + it 'renders error message with timestamp' do + remote_mirror.update_attributes(last_error: 'Some new error', last_update_at: Time.now - 5.minutes) + + visit project_mirror_path(project) + + expect(page).to have_content('The remote repository failed to update 5 minutes ago.') + end + end + end +end diff --git a/spec/features/projects/settings/repository_settings_spec.rb b/spec/features/projects/settings/repository_settings_spec.rb index e1dfe617691..162aee63942 100644 --- a/spec/features/projects/settings/repository_settings_spec.rb +++ b/spec/features/projects/settings/repository_settings_spec.rb @@ -115,5 +115,20 @@ describe 'Projects > Settings > Repository settings' do expect(page).to have_content('Your new project deploy token has been created') end end + + context 'remote mirror settings' do + let(:user2) { create(:user) } + + before do + project.add_master(user2) + + visit project_settings_repository_path(project) + end + + it 'shows push mirror settings' do + expect(page).to have_selector('#project_remote_mirrors_attributes_0_enabled') + expect(page).to have_selector('#project_remote_mirrors_attributes_0_url') + end + end end end diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index a08e0c93df9..ad76adcc2e5 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -268,6 +268,7 @@ project: - pages_domains - authorized_users - project_authorizations +- remote_mirrors - route - redirect_routes - statistics diff --git a/spec/lib/gitlab/usage_data_spec.rb b/spec/lib/gitlab/usage_data_spec.rb index 9e6aa109a4b..a716e6f5434 100644 --- a/spec/lib/gitlab/usage_data_spec.rb +++ b/spec/lib/gitlab/usage_data_spec.rb @@ -96,6 +96,7 @@ describe Gitlab::UsageData do pages_domains protected_branches releases + remote_mirrors snippets todos uploads diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index f3cf21cf279..41622fbbb6f 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -1852,6 +1852,85 @@ describe Project do it { expect(project.gitea_import?).to be true } end + describe '#has_remote_mirror?' do + let(:project) { create(:project, :remote_mirror, :import_started) } + subject { project.has_remote_mirror? } + + before do + allow_any_instance_of(RemoteMirror).to receive(:refresh_remote) + end + + it 'returns true when a remote mirror is enabled' do + is_expected.to be_truthy + end + + it 'returns false when remote mirror is disabled' do + project.remote_mirrors.first.update_attributes(enabled: false) + + is_expected.to be_falsy + end + end + + describe '#update_remote_mirrors' do + let(:project) { create(:project, :remote_mirror, :import_started) } + delegate :update_remote_mirrors, to: :project + + before do + allow_any_instance_of(RemoteMirror).to receive(:refresh_remote) + end + + it 'syncs enabled remote mirror' do + expect_any_instance_of(RemoteMirror).to receive(:sync) + + update_remote_mirrors + end + + # TODO: study if remote_mirror_available_overridden is still a necessary attribute considering that + # it is no longer under any license + it 'does nothing when remote mirror is disabled globally and not overridden' do + stub_application_setting(mirror_available: false) + project.remote_mirror_available_overridden = false + + expect_any_instance_of(RemoteMirror).not_to receive(:sync) + + update_remote_mirrors + end + + it 'does not sync disabled remote mirrors' do + project.remote_mirrors.first.update_attributes(enabled: false) + + expect_any_instance_of(RemoteMirror).not_to receive(:sync) + + update_remote_mirrors + end + end + + describe '#remote_mirror_available?' do + let(:project) { create(:project) } + + context 'when remote mirror global setting is enabled' do + it 'returns true' do + expect(project.remote_mirror_available?).to be(true) + end + end + + context 'when remote mirror global setting is disabled' do + before do + stub_application_setting(mirror_available: false) + end + + it 'returns true when overridden' do + project.remote_mirror_available_overridden = true + + expect(project.remote_mirror_available?).to be(true) + end + + it 'returns false when not overridden' do + expect(project.remote_mirror_available?).to be(false) + end + end + end + describe '#ancestors_upto', :nested_groups do let(:parent) { create(:group) } let(:child) { create(:group, parent: parent) } diff --git a/spec/models/remote_mirror_spec.rb b/spec/models/remote_mirror_spec.rb new file mode 100644 index 00000000000..a80800c6c92 --- /dev/null +++ b/spec/models/remote_mirror_spec.rb @@ -0,0 +1,267 @@ +require 'rails_helper' + +describe RemoteMirror do + describe 'URL validation' do + context 'with a valid URL' do + it 'should be valid' do + remote_mirror = build(:remote_mirror) + expect(remote_mirror).to be_valid + end + end + + context 'with an invalid URL' do + it 'should not be valid' do + remote_mirror = build(:remote_mirror, url: 'ftp://invalid.invalid') + expect(remote_mirror).not_to be_valid + expect(remote_mirror.errors[:url].size).to eq(2) + end + end + end + + describe 'encrypting credentials' do + context 'when setting URL for a first time' do + it 'stores the URL without credentials' do + mirror = create_mirror(url: 'http://foo:bar@test.com') + + expect(mirror.read_attribute(:url)).to eq('http://test.com') + end + + it 'stores the credentials on a separate field' do + mirror = create_mirror(url: 'http://foo:bar@test.com') + + expect(mirror.credentials).to eq({ user: 'foo', password: 'bar' }) + end + + it 'handles credentials with large content' do + mirror = create_mirror(url: 'http://bxnhm8dote33ct932r3xavslj81wxmr7o8yux8do10oozckkif:9ne7fuvjn40qjt35dgt8v86q9m9g9essryxj76sumg2ccl2fg26c0krtz2gzfpyq4hf22h328uhq6npuiq6h53tpagtsj7vsrz75@test.com') + + expect(mirror.credentials).to eq({ + user: 'bxnhm8dote33ct932r3xavslj81wxmr7o8yux8do10oozckkif', + password: '9ne7fuvjn40qjt35dgt8v86q9m9g9essryxj76sumg2ccl2fg26c0krtz2gzfpyq4hf22h328uhq6npuiq6h53tpagtsj7vsrz75' + }) + end + end + + context 'when updating the URL' do + it 'allows a new URL without credentials' do + mirror = create_mirror(url: 'http://foo:bar@test.com') + + mirror.update_attribute(:url, 'http://test.com') + + expect(mirror.url).to eq('http://test.com') + expect(mirror.credentials).to eq({ user: nil, password: nil }) + end + + it 'allows a new URL with credentials' do + mirror = create_mirror(url: 'http://test.com') + + mirror.update_attribute(:url, 'http://foo:bar@test.com') + + expect(mirror.url).to eq('http://foo:bar@test.com') + expect(mirror.credentials).to eq({ user: 'foo', password: 'bar' }) + end + + it 'updates the remote config if credentials changed' do + mirror = create_mirror(url: 'http://foo:bar@test.com') + repo = mirror.project.repository + + mirror.update_attribute(:url, 'http://foo:baz@test.com') + + config = repo.raw_repository.rugged.config + expect(config["remote.#{mirror.remote_name}.url"]).to eq('http://foo:baz@test.com') + end + + it 'removes previous remote' do + mirror = create_mirror(url: 'http://foo:bar@test.com') + + expect(RepositoryRemoveRemoteWorker).to receive(:perform_async).with(mirror.project.id, mirror.remote_name).and_call_original + + mirror.update_attributes(url: 'http://test.com') + end + end + end + + describe '#remote_name' do + context 'when remote name is persisted in the database' do + it 'returns remote name with random value' do + allow(SecureRandom).to receive(:hex).and_return('secret') + + remote_mirror = create(:remote_mirror) + + expect(remote_mirror.remote_name).to eq("remote_mirror_secret") + end + end + + context 'when remote name is not persisted in the database' do + it 'returns remote name with remote mirror id' do + remote_mirror = create(:remote_mirror) + remote_mirror.remote_name = nil + + expect(remote_mirror.remote_name).to eq("remote_mirror_#{remote_mirror.id}") + end + end + + context 'when remote is not persisted in the database' do + it 'returns nil' do + remote_mirror = build(:remote_mirror, remote_name: nil) + + expect(remote_mirror.remote_name).to be_nil + end + end + end + + describe '#safe_url' do + context 'when URL contains credentials' do + it 'masks the credentials' do + mirror = create_mirror(url: 'http://foo:bar@test.com') + + expect(mirror.safe_url).to eq('http://*****:*****@test.com') + end + end + + context 'when URL does not contain credentials' do + it 'shows the full URL' do + mirror = create_mirror(url: 'http://test.com') + + expect(mirror.safe_url).to eq('http://test.com') + end + end + end + + context 'when remote mirror gets destroyed' do + it 'removes remote' do + mirror = create_mirror(url: 'http://foo:bar@test.com') + + expect(RepositoryRemoveRemoteWorker).to receive(:perform_async).with(mirror.project.id, mirror.remote_name).and_call_original + + mirror.destroy! + end + end + + context 'stuck mirrors' do + it 'includes mirrors stuck in started with no last_update_at set' do + mirror = create_mirror(url: 'http://cantbeblank', + update_status: 'started', + last_update_at: nil, + updated_at: 25.hours.ago) + + expect(described_class.stuck.last).to eq(mirror) + end + end + + context '#sync' do + let(:remote_mirror) { create(:project, :repository, :remote_mirror).remote_mirrors.first } + + around do |example| + Timecop.freeze { example.run } + end + + context 'with remote mirroring disabled' do + it 'returns nil' do + remote_mirror.update_attributes(enabled: false) + + expect(remote_mirror.sync).to be_nil + end + end + + context 'with remote mirroring enabled' do + context 'with only protected branches enabled' do + context 'when it did not update in the last minute' do + it 'schedules a RepositoryUpdateRemoteMirrorWorker to run now' do + expect(RepositoryUpdateRemoteMirrorWorker).to receive(:perform_async).with(remote_mirror.id, Time.now) + + remote_mirror.sync + end + end + + context 'when it did update in the last minute' do + it 'schedules a RepositoryUpdateRemoteMirrorWorker to run in the next minute' do + remote_mirror.last_update_started_at = Time.now - 30.seconds + + expect(RepositoryUpdateRemoteMirrorWorker).to receive(:perform_in).with(RemoteMirror::PROTECTED_BACKOFF_DELAY, remote_mirror.id, Time.now) + + remote_mirror.sync + end + end + end + + context 'with only protected branches disabled' do + before do + remote_mirror.only_protected_branches = false + end + + context 'when it did not update in the last 5 minutes' do + it 'schedules a RepositoryUpdateRemoteMirrorWorker to run now' do + expect(RepositoryUpdateRemoteMirrorWorker).to receive(:perform_async).with(remote_mirror.id, Time.now) + + remote_mirror.sync + end + end + + context 'when it did update within the last 5 minutes' do + it 'schedules a RepositoryUpdateRemoteMirrorWorker to run in the next 5 minutes' do + remote_mirror.last_update_started_at = Time.now - 30.seconds + + expect(RepositoryUpdateRemoteMirrorWorker).to receive(:perform_in).with(RemoteMirror::UNPROTECTED_BACKOFF_DELAY, remote_mirror.id, Time.now) + + remote_mirror.sync + end + end + end + end + end + + context '#updated_since?' do + let(:remote_mirror) { create(:project, :repository, :remote_mirror).remote_mirrors.first } + let(:timestamp) { Time.now - 5.minutes } + + around do |example| + Timecop.freeze { example.run } + end + + before do + remote_mirror.update_attributes(last_update_started_at: Time.now) + end + + context 'when remote mirror does not have status failed' do + it 'returns true when last update started after the timestamp' do + expect(remote_mirror.updated_since?(timestamp)).to be true + end + + it 'returns false when last update started before the timestamp' do + expect(remote_mirror.updated_since?(Time.now + 5.minutes)).to be false + end + end + + context 'when remote mirror has status failed' do + it 'returns false when last update started after the timestamp' do + remote_mirror.update_attributes(update_status: 'failed') + + expect(remote_mirror.updated_since?(timestamp)).to be false + end + end + end + + context 'no project' do + it 'includes mirror with a project in pending_delete' do + mirror = create_mirror(url: 'http://cantbeblank', + update_status: 'finished', + enabled: true, + last_update_at: nil, + updated_at: 25.hours.ago) + project = mirror.project + project.pending_delete = true + project.save + mirror.reload + + expect(mirror.sync).to be_nil + expect(mirror.valid?).to be_truthy + expect(mirror.update_status).to eq('finished') + end + end + + def create_mirror(params) + project = FactoryBot.create(:project, :repository) + project.remote_mirrors.create!(params) + end +end diff --git a/spec/models/repository_spec.rb b/spec/models/repository_spec.rb index 630b9e0519f..4b736b02b7d 100644 --- a/spec/models/repository_spec.rb +++ b/spec/models/repository_spec.rb @@ -758,6 +758,38 @@ describe Repository do end end + describe '#async_remove_remote' do + before do + masterrev = repository.find_branch('master').dereferenced_target + create_remote_branch('joe', 'remote_branch', masterrev) + end + + context 'when worker is scheduled successfully' do + before do + masterrev = repository.find_branch('master').dereferenced_target + create_remote_branch('remote_name', 'remote_branch', masterrev) + + allow(RepositoryRemoveRemoteWorker).to receive(:perform_async).and_return('1234') + end + + it 'returns job_id' do + expect(repository.async_remove_remote('joe')).to eq('1234') + end + end + + context 'when worker does not schedule successfully' do + before do + allow(RepositoryRemoveRemoteWorker).to receive(:perform_async).and_return(nil) + end + + it 'returns nil' do + expect(Rails.logger).to receive(:info).with("Remove remote job failed to create for #{project.id} with remote name joe.") + + expect(repository.async_remove_remote('joe')).to be_nil + end + end + end + describe '#fetch_ref' do let(:broken_repository) { create(:project, :broken_storage).repository } @@ -2338,6 +2370,11 @@ describe Repository do end end + def create_remote_branch(remote_name, branch_name, target) + rugged = repository.rugged + rugged.references.create("refs/remotes/#{remote_name}/#{branch_name}", target.id) + end + describe '#ancestor?' do let(:commit) { repository.commit } let(:ancestor) { commit.parents.first } diff --git a/spec/services/git_push_service_spec.rb b/spec/services/git_push_service_spec.rb index 26fdf8d4b24..35826de5814 100644 --- a/spec/services/git_push_service_spec.rb +++ b/spec/services/git_push_service_spec.rb @@ -14,6 +14,72 @@ describe GitPushService, services: true do project.add_master(user) end + describe 'with remote mirrors' do + let(:project) { create(:project, :repository, :remote_mirror) } + + subject do + described_class.new(project, user, oldrev: oldrev, newrev: newrev, ref: ref) + end + + context 'when remote mirror feature is enabled' do + it 'fails stuck remote mirrors' do + allow(project).to receive(:update_remote_mirrors).and_return(project.remote_mirrors) + expect(project).to receive(:mark_stuck_remote_mirrors_as_failed!) + + subject.execute + end + + it 'updates remote mirrors' do + expect(project).to receive(:update_remote_mirrors) + + subject.execute + end + end + + context 'when remote mirror feature is disabled' do + before do + stub_application_setting(mirror_available: false) + end + + context 'with remote mirrors global setting overridden' do + before do + project.remote_mirror_available_overridden = true + end + + it 'fails stuck remote mirrors' do + allow(project).to receive(:update_remote_mirrors).and_return(project.remote_mirrors) + expect(project).to receive(:mark_stuck_remote_mirrors_as_failed!) + + subject.execute + end + + it 'updates remote mirrors' do + expect(project).to receive(:update_remote_mirrors) + + subject.execute + end + end + + context 'without remote mirrors global setting overridden' do + before do + project.remote_mirror_available_overridden = false + end + + it 'does not fails stuck remote mirrors' do + expect(project).not_to receive(:mark_stuck_remote_mirrors_as_failed!) + + subject.execute + end + + it 'does not updates remote mirrors' do + expect(project).not_to receive(:update_remote_mirrors) + + subject.execute + end + end + end + end + describe 'Push branches' do subject do execute_service(project, user, oldrev, newrev, ref) diff --git a/spec/services/projects/destroy_service_spec.rb b/spec/services/projects/destroy_service_spec.rb index b2c52214f48..b63f409579e 100644 --- a/spec/services/projects/destroy_service_spec.rb +++ b/spec/services/projects/destroy_service_spec.rb @@ -65,6 +65,19 @@ describe Projects::DestroyService do Sidekiq::Testing.inline! { destroy_project(project, user, {}) } end + context 'when has remote mirrors' do + let!(:project) do + create(:project, :repository, namespace: user.namespace).tap do |project| + project.remote_mirrors.create(url: 'http://test.com') + end + end + let!(:async) { true } + + it 'destroys them' do + expect(RemoteMirror.count).to eq(0) + end + end + it_behaves_like 'deleting the project' it 'invalidates personal_project_count cache' do diff --git a/spec/services/projects/update_remote_mirror_service_spec.rb b/spec/services/projects/update_remote_mirror_service_spec.rb new file mode 100644 index 00000000000..be09afd9f36 --- /dev/null +++ b/spec/services/projects/update_remote_mirror_service_spec.rb @@ -0,0 +1,355 @@ +require 'spec_helper' + +describe Projects::UpdateRemoteMirrorService do + let(:project) { create(:project, :repository) } + let(:remote_project) { create(:forked_project_with_submodules) } + let(:repository) { project.repository } + let(:raw_repository) { repository.raw } + let(:remote_mirror) { project.remote_mirrors.create!(url: remote_project.http_url_to_repo, enabled: true, only_protected_branches: false) } + + subject { described_class.new(project, project.creator) } + + describe "#execute", :skip_gitaly_mock do + before do + create_branch(repository, 'existing-branch') + allow(raw_repository).to receive(:remote_tags) do + generate_tags(repository, 'v1.0.0', 'v1.1.0') + end + allow(raw_repository).to receive(:push_remote_branches).and_return(true) + end + + it "fetches the remote repository" do + expect(repository).to receive(:fetch_remote).with(remote_mirror.remote_name, no_tags: true) do + sync_remote(repository, remote_mirror.remote_name, local_branch_names) + end + + subject.execute(remote_mirror) + end + + it "succeeds" do + allow(repository).to receive(:fetch_remote) { sync_remote(repository, remote_mirror.remote_name, local_branch_names) } + + result = subject.execute(remote_mirror) + + expect(result[:status]).to eq(:success) + end + + describe 'Syncing branches' do + it "push all the branches the first time" do + allow(repository).to receive(:fetch_remote) + + expect(raw_repository).to receive(:push_remote_branches).with(remote_mirror.remote_name, local_branch_names) + + subject.execute(remote_mirror) + end + + it "does not push anything is remote is up to date" do + allow(repository).to receive(:fetch_remote) { sync_remote(repository, remote_mirror.remote_name, local_branch_names) } + + expect(raw_repository).not_to receive(:push_remote_branches) + + subject.execute(remote_mirror) + end + + it "sync new branches" do + # call local_branch_names early so it is not called after the new branch has been created + current_branches = local_branch_names + allow(repository).to receive(:fetch_remote) { sync_remote(repository, remote_mirror.remote_name, current_branches) } + create_branch(repository, 'my-new-branch') + + expect(raw_repository).to receive(:push_remote_branches).with(remote_mirror.remote_name, ['my-new-branch']) + + subject.execute(remote_mirror) + end + + it "sync updated branches" do + allow(repository).to receive(:fetch_remote) do + sync_remote(repository, remote_mirror.remote_name, local_branch_names) + update_branch(repository, 'existing-branch') + end + + expect(raw_repository).to receive(:push_remote_branches).with(remote_mirror.remote_name, ['existing-branch']) + + subject.execute(remote_mirror) + end + + context 'when push only protected branches option is set' do + let(:unprotected_branch_name) { 'existing-branch' } + let(:protected_branch_name) do + project.repository.branch_names.find { |n| n != unprotected_branch_name } + end + let!(:protected_branch) do + create(:protected_branch, project: project, name: protected_branch_name) + end + + before do + project.reload + remote_mirror.only_protected_branches = true + end + + it "sync updated protected branches" do + allow(repository).to receive(:fetch_remote) do + sync_remote(repository, remote_mirror.remote_name, local_branch_names) + update_branch(repository, protected_branch_name) + end + + expect(raw_repository).to receive(:push_remote_branches).with(remote_mirror.remote_name, [protected_branch_name]) + + subject.execute(remote_mirror) + end + + it 'does not sync unprotected branches' do + allow(repository).to receive(:fetch_remote) do + sync_remote(repository, remote_mirror.remote_name, local_branch_names) + update_branch(repository, unprotected_branch_name) + end + + expect(raw_repository).not_to receive(:push_remote_branches).with(remote_mirror.remote_name, [unprotected_branch_name]) + + subject.execute(remote_mirror) + end + end + + context 'when branch exists in local and remote repo' do + context 'when it has diverged' do + it 'syncs branches' do + allow(repository).to receive(:fetch_remote) do + sync_remote(repository, remote_mirror.remote_name, local_branch_names) + update_remote_branch(repository, remote_mirror.remote_name, 'markdown') + end + + expect(raw_repository).to receive(:push_remote_branches).with(remote_mirror.remote_name, ['markdown']) + + subject.execute(remote_mirror) + end + end + end + + describe 'for delete' do + context 'when branch exists in local and remote repo' do + it 'deletes the branch from remote repo' do + allow(repository).to receive(:fetch_remote) do + sync_remote(repository, remote_mirror.remote_name, local_branch_names) + delete_branch(repository, 'existing-branch') + end + + expect(raw_repository).to receive(:delete_remote_branches).with(remote_mirror.remote_name, ['existing-branch']) + + subject.execute(remote_mirror) + end + end + + context 'when push only protected branches option is set' do + before do + remote_mirror.only_protected_branches = true + end + + context 'when branch exists in local and remote repo' do + let!(:protected_branch_name) { local_branch_names.first } + + before do + create(:protected_branch, project: project, name: protected_branch_name) + project.reload + end + + it 'deletes the protected branch from remote repo' do + allow(repository).to receive(:fetch_remote) do + sync_remote(repository, remote_mirror.remote_name, local_branch_names) + delete_branch(repository, protected_branch_name) + end + + expect(raw_repository).not_to receive(:delete_remote_branches).with(remote_mirror.remote_name, [protected_branch_name]) + + subject.execute(remote_mirror) + end + + it 'does not delete the unprotected branch from remote repo' do + allow(repository).to receive(:fetch_remote) do + sync_remote(repository, remote_mirror.remote_name, local_branch_names) + delete_branch(repository, 'existing-branch') + end + + expect(raw_repository).not_to receive(:delete_remote_branches).with(remote_mirror.remote_name, ['existing-branch']) + + subject.execute(remote_mirror) + end + end + + context 'when branch only exists on remote repo' do + let!(:protected_branch_name) { 'remote-branch' } + + before do + create(:protected_branch, project: project, name: protected_branch_name) + end + + context 'when it has diverged' do + it 'does not delete the remote branch' do + allow(repository).to receive(:fetch_remote) do + sync_remote(repository, remote_mirror.remote_name, local_branch_names) + + rev = repository.find_branch('markdown').dereferenced_target + create_remote_branch(repository, remote_mirror.remote_name, 'remote-branch', rev.id) + end + + expect(raw_repository).not_to receive(:delete_remote_branches) + + subject.execute(remote_mirror) + end + end + + context 'when it has not diverged' do + it 'deletes the remote branch' do + allow(repository).to receive(:fetch_remote) do + sync_remote(repository, remote_mirror.remote_name, local_branch_names) + + masterrev = repository.find_branch('master').dereferenced_target + create_remote_branch(repository, remote_mirror.remote_name, protected_branch_name, masterrev.id) + end + + expect(raw_repository).to receive(:delete_remote_branches).with(remote_mirror.remote_name, [protected_branch_name]) + + subject.execute(remote_mirror) + end + end + end + end + + context 'when branch only exists on remote repo' do + context 'when it has diverged' do + it 'does not delete the remote branch' do + allow(repository).to receive(:fetch_remote) do + sync_remote(repository, remote_mirror.remote_name, local_branch_names) + + rev = repository.find_branch('markdown').dereferenced_target + create_remote_branch(repository, remote_mirror.remote_name, 'remote-branch', rev.id) + end + + expect(raw_repository).not_to receive(:delete_remote_branches) + + subject.execute(remote_mirror) + end + end + + context 'when it has not diverged' do + it 'deletes the remote branch' do + allow(repository).to receive(:fetch_remote) do + sync_remote(repository, remote_mirror.remote_name, local_branch_names) + + masterrev = repository.find_branch('master').dereferenced_target + create_remote_branch(repository, remote_mirror.remote_name, 'remote-branch', masterrev.id) + end + + expect(raw_repository).to receive(:delete_remote_branches).with(remote_mirror.remote_name, ['remote-branch']) + + subject.execute(remote_mirror) + end + end + end + end + end + + describe 'Syncing tags' do + before do + allow(repository).to receive(:fetch_remote) { sync_remote(repository, remote_mirror.remote_name, local_branch_names) } + end + + context 'when there are not tags to push' do + it 'does not try to push tags' do + allow(repository).to receive(:remote_tags) { {} } + allow(repository).to receive(:tags) { [] } + + expect(repository).not_to receive(:push_tags) + + subject.execute(remote_mirror) + end + end + + context 'when there are some tags to push' do + it 'pushes tags to remote' do + allow(raw_repository).to receive(:remote_tags) { {} } + + expect(raw_repository).to receive(:push_remote_branches).with(remote_mirror.remote_name, ['v1.0.0', 'v1.1.0']) + + subject.execute(remote_mirror) + end + end + + context 'when there are some tags to delete' do + it 'deletes tags from remote' do + remote_tags = generate_tags(repository, 'v1.0.0', 'v1.1.0') + allow(raw_repository).to receive(:remote_tags) { remote_tags } + + repository.rm_tag(create(:user), 'v1.0.0') + + expect(raw_repository).to receive(:delete_remote_branches).with(remote_mirror.remote_name, ['v1.0.0']) + + subject.execute(remote_mirror) + end + end + end + end + + def create_branch(repository, branch_name) + rugged = repository.rugged + masterrev = repository.find_branch('master').dereferenced_target + parentrev = repository.commit(masterrev).parent_id + + rugged.references.create("refs/heads/#{branch_name}", parentrev) + + repository.expire_branches_cache + end + + def create_remote_branch(repository, remote_name, branch_name, source_id) + rugged = repository.rugged + + rugged.references.create("refs/remotes/#{remote_name}/#{branch_name}", source_id) + end + + def sync_remote(repository, remote_name, local_branch_names) + rugged = repository.rugged + + local_branch_names.each do |branch| + target = repository.find_branch(branch).try(:dereferenced_target) + rugged.references.create("refs/remotes/#{remote_name}/#{branch}", target.id) if target + end + end + + def update_remote_branch(repository, remote_name, branch) + rugged = repository.rugged + masterrev = repository.find_branch('master').dereferenced_target.id + + rugged.references.create("refs/remotes/#{remote_name}/#{branch}", masterrev, force: true) + repository.expire_branches_cache + end + + def update_branch(repository, branch) + rugged = repository.rugged + masterrev = repository.find_branch('master').dereferenced_target.id + + # Updated existing branch + rugged.references.create("refs/heads/#{branch}", masterrev, force: true) + repository.expire_branches_cache + end + + def delete_branch(repository, branch) + rugged = repository.rugged + + rugged.references.delete("refs/heads/#{branch}") + repository.expire_branches_cache + end + + def generate_tags(repository, *tag_names) + tag_names.each_with_object([]) do |name, tags| + tag = repository.find_tag(name) + target = tag.try(:target) + target_commit = tag.try(:dereferenced_target) + tags << Gitlab::Git::Tag.new(repository.raw_repository, name, target, target_commit) + end + end + + def local_branch_names + branch_names = repository.branches.map(&:name) + # we want the protected branch to be pushed first + branch_names.unshift(branch_names.delete('master')) + end +end diff --git a/spec/workers/repository_remove_remote_worker_spec.rb b/spec/workers/repository_remove_remote_worker_spec.rb new file mode 100644 index 00000000000..f22d7c1d073 --- /dev/null +++ b/spec/workers/repository_remove_remote_worker_spec.rb @@ -0,0 +1,50 @@ +require 'rails_helper' + +describe RepositoryRemoveRemoteWorker do + subject(:worker) { described_class.new } + + describe '#perform' do + let(:remote_name) { 'joe'} + let!(:project) { create(:project, :repository) } + + context 'when it cannot obtain lease' do + it 'logs error' do + allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain) { nil } + + expect_any_instance_of(Repository).not_to receive(:remove_remote) + expect(worker).to receive(:log_error).with('Cannot obtain an exclusive lease. There must be another instance already in execution.') + + worker.perform(project.id, remote_name) + end + end + + context 'when it gets the lease' do + before do + allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain).and_return(true) + end + + context 'when project does not exist' do + it 'returns nil' do + expect(worker.perform(-1, 'remote_name')).to be_nil + end + end + + context 'when project exists' do + it 'removes remote from repository' do + masterrev = project.repository.find_branch('master').dereferenced_target + + create_remote_branch(remote_name, 'remote_branch', masterrev) + + expect_any_instance_of(Repository).to receive(:remove_remote).with(remote_name).and_call_original + + worker.perform(project.id, remote_name) + end + end + end + end + + def create_remote_branch(remote_name, branch_name, target) + rugged = project.repository.rugged + rugged.references.create("refs/remotes/#{remote_name}/#{branch_name}", target.id) + end +end diff --git a/spec/workers/repository_update_remote_mirror_worker_spec.rb b/spec/workers/repository_update_remote_mirror_worker_spec.rb new file mode 100644 index 00000000000..152ba2509b9 --- /dev/null +++ b/spec/workers/repository_update_remote_mirror_worker_spec.rb @@ -0,0 +1,84 @@ +require 'rails_helper' + +describe RepositoryUpdateRemoteMirrorWorker do + subject { described_class.new } + + let(:remote_mirror) { create(:project, :repository, :remote_mirror).remote_mirrors.first } + let(:scheduled_time) { Time.now - 5.minutes } + + around do |example| + Timecop.freeze(Time.now) { example.run } + end + + describe '#perform' do + context 'with status none' do + before do + remote_mirror.update_attributes(update_status: 'none') + end + + it 'sets status as finished when update remote mirror service executes successfully' do + expect_any_instance_of(Projects::UpdateRemoteMirrorService).to receive(:execute).with(remote_mirror).and_return(status: :success) + + expect { subject.perform(remote_mirror.id, Time.now) }.to change { remote_mirror.reload.update_status }.to('finished') + end + + it 'sets status as failed when update remote mirror service executes with errors' do + error_message = 'fail!' + + expect_any_instance_of(Projects::UpdateRemoteMirrorService).to receive(:execute).with(remote_mirror).and_return(status: :error, message: error_message) + expect do + subject.perform(remote_mirror.id, Time.now) + end.to raise_error(RepositoryUpdateRemoteMirrorWorker::UpdateError, error_message) + + expect(remote_mirror.reload.update_status).to eq('failed') + end + + it 'does nothing if last_update_started_at is higher than the time the job was scheduled in' do + remote_mirror.update_attributes(last_update_started_at: Time.now) + + expect_any_instance_of(RemoteMirror).to receive(:updated_since?).with(scheduled_time).and_return(true) + expect_any_instance_of(Projects::UpdateRemoteMirrorService).not_to receive(:execute).with(remote_mirror) + + expect(subject.perform(remote_mirror.id, scheduled_time)).to be_nil + end + end + + context 'with unexpected error' do + it 'marks mirror as failed' do + allow_any_instance_of(Projects::UpdateRemoteMirrorService).to receive(:execute).with(remote_mirror).and_raise(RuntimeError) + + expect do + subject.perform(remote_mirror.id, Time.now) + end.to raise_error(RepositoryUpdateRemoteMirrorWorker::UpdateError) + expect(remote_mirror.reload.update_status).to eq('failed') + end + end + + context 'with another worker already running' do + before do + remote_mirror.update_attributes(update_status: 'started') + end + + it 'raises RemoteMirrorUpdateAlreadyInProgressError' do + expect do + subject.perform(remote_mirror.id, Time.now) + end.to raise_error(RepositoryUpdateRemoteMirrorWorker::UpdateAlreadyInProgressError) + end + end + + context 'with status failed' do + before do + remote_mirror.update_attributes(update_status: 'failed') + end + + it 'sets status as finished if last_update_started_at is higher than the time the job was scheduled in' do + remote_mirror.update_attributes(last_update_started_at: Time.now) + + expect_any_instance_of(RemoteMirror).to receive(:updated_since?).with(scheduled_time).and_return(false) + expect_any_instance_of(Projects::UpdateRemoteMirrorService).to receive(:execute).with(remote_mirror).and_return(status: :success) + + expect { subject.perform(remote_mirror.id, scheduled_time) }.to change { remote_mirror.reload.update_status }.to('finished') + end + end + end +end |