Merge branch '42099-port-push-mirroring-to-ce-ce-port-v-2' into 'master'

CE backport of Backports Push Mirrors to CE

Closes #42099

See merge request gitlab-org/gitlab-ce!18715

15 files changed, 1091 insertions, 0 deletions

diff --git a/spec/controllers/projects/mirrors_controller_spec.rb b/spec/controllers/projects/mirrors_controller_spec.rb
new file mode 100644
index 00000000000..45c1218a39c
--- /dev/null
+++ b/spec/controllers/projects/mirrors_controller_spec.rb
@@ -0,0 +1,72 @@
+require 'spec_helper'
+
+describe Projects::MirrorsController do
+  include ReactiveCachingHelpers
+
+  describe 'setting up a remote mirror' do
+    set(:project) { create(:project, :repository) }
+
+    context 'when the current project is not a mirror' do
+      it 'allows to create a remote mirror' do
+        sign_in(project.owner)
+
+        expect do
+          do_put(project, remote_mirrors_attributes: { '0' => { 'enabled' => 1, 'url' => 'http://foo.com' } })
+        end.to change { RemoteMirror.count }.to(1)
+      end
+    end
+  end
+
+  describe '#update' do
+    let(:project) { create(:project, :repository, :remote_mirror) }
+
+    before do
+      sign_in(project.owner)
+    end
+
+    around do |example|
+      Sidekiq::Testing.fake! { example.run }
+    end
+
+    context 'With valid URL for a push' do
+      let(:remote_mirror_attributes) do
+        { "0" => { "enabled" => "0", url: 'https://updated.example.com' } }
+      end
+
+      it 'processes a successful update' do
+        do_put(project, remote_mirrors_attributes: remote_mirror_attributes)
+
+        expect(response).to redirect_to(project_settings_repository_path(project))
+        expect(flash[:notice]).to match(/successfully updated/)
+      end
+
+      it 'should create a RemoteMirror object' do
+        expect { do_put(project, remote_mirrors_attributes: remote_mirror_attributes) }.to change(RemoteMirror, :count).by(1)
+      end
+    end
+
+    context 'With invalid URL for a push' do
+      let(:remote_mirror_attributes) do
+        { "0" => { "enabled" => "0", url: 'ftp://invalid.invalid' } }
+      end
+
+      it 'processes an unsuccessful update' do
+        do_put(project, remote_mirrors_attributes: remote_mirror_attributes)
+
+        expect(response).to redirect_to(project_settings_repository_path(project))
+        expect(flash[:alert]).to match(/must be a valid URL/)
+      end
+
+      it 'should not create a RemoteMirror object' do
+        expect { do_put(project, remote_mirrors_attributes: remote_mirror_attributes) }.not_to change(RemoteMirror, :count)
+      end
+    end
+  end
+
+  def do_put(project, options, extra_attrs = {})
+    attrs = extra_attrs.merge(namespace_id: project.namespace.to_param, project_id: project.to_param)
+    attrs[:project] = options
+
+    put :update, attrs
+  end
+end
diff --git a/spec/factories/projects.rb b/spec/factories/projects.rb
index 9ab57af1c60..d129815aeac 100644
--- a/spec/factories/projects.rb
+++ b/spec/factories/projects.rb
@@ -183,6 +183,17 @@ FactoryBot.define do
       end
     end
 
+    trait :remote_mirror do
+      transient do
+        remote_name "remote_mirror_#{SecureRandom.hex}"
+        url "http://foo.com"
+        enabled true
+      end
+      after(:create) do |project, evaluator|
+        project.remote_mirrors.create!(url: evaluator.url, enabled: evaluator.enabled)
+      end
+    end
+
     trait :stubbed_repository do
       after(:build) do |project|
         allow(project).to receive(:empty_repo?).and_return(false)
diff --git a/spec/factories/remote_mirrors.rb b/spec/factories/remote_mirrors.rb
new file mode 100644
index 00000000000..adc7da27522
--- /dev/null
+++ b/spec/factories/remote_mirrors.rb
@@ -0,0 +1,6 @@
+FactoryBot.define do
+  factory :remote_mirror, class: 'RemoteMirror' do
+    association :project, :repository
+    url "http://foo:bar@test.com"
+  end
+end
diff --git a/spec/features/projects/remote_mirror_spec.rb b/spec/features/projects/remote_mirror_spec.rb
new file mode 100644
index 00000000000..81a6b613cc8
--- /dev/null
+++ b/spec/features/projects/remote_mirror_spec.rb
@@ -0,0 +1,34 @@
+require 'spec_helper'
+
+feature 'Project remote mirror', :feature do
+  let(:project) { create(:project, :repository, :remote_mirror) }
+  let(:remote_mirror) { project.remote_mirrors.first }
+  let(:user) { create(:user) }
+
+  describe 'On a project', :js do
+    before do
+      project.add_master(user)
+      sign_in user
+    end
+
+    context 'when last_error is present but last_update_at is not' do
+      it 'renders error message without timstamp' do
+        remote_mirror.update_attributes(last_error: 'Some new error', last_update_at: nil)
+
+        visit project_mirror_path(project)
+
+        expect(page).to have_content('The remote repository failed to update.')
+      end
+    end
+
+    context 'when last_error and last_update_at are present' do
+      it 'renders error message with timestamp' do
+        remote_mirror.update_attributes(last_error: 'Some new error', last_update_at: Time.now - 5.minutes)
+
+        visit project_mirror_path(project)
+
+        expect(page).to have_content('The remote repository failed to update 5 minutes ago.')
+      end
+    end
+  end
+end
diff --git a/spec/features/projects/settings/repository_settings_spec.rb b/spec/features/projects/settings/repository_settings_spec.rb
index e1dfe617691..162aee63942 100644
--- a/spec/features/projects/settings/repository_settings_spec.rb
+++ b/spec/features/projects/settings/repository_settings_spec.rb
@@ -115,5 +115,20 @@ describe 'Projects > Settings > Repository settings' do
         expect(page).to have_content('Your new project deploy token has been created')
       end
     end
+
+    context 'remote mirror settings' do
+      let(:user2) { create(:user) }
+
+      before do
+        project.add_master(user2)
+
+        visit project_settings_repository_path(project)
+      end
+
+      it 'shows push mirror settings' do
+        expect(page).to have_selector('#project_remote_mirrors_attributes_0_enabled')
+        expect(page).to have_selector('#project_remote_mirrors_attributes_0_url')
+      end
+    end
   end
 end
diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml
index a08e0c93df9..ad76adcc2e5 100644
--- a/spec/lib/gitlab/import_export/all_models.yml
+++ b/spec/lib/gitlab/import_export/all_models.yml
@@ -268,6 +268,7 @@ project:
 - pages_domains
 - authorized_users
 - project_authorizations
+- remote_mirrors
 - route
 - redirect_routes
 - statistics
diff --git a/spec/lib/gitlab/usage_data_spec.rb b/spec/lib/gitlab/usage_data_spec.rb
index 9e6aa109a4b..a716e6f5434 100644
--- a/spec/lib/gitlab/usage_data_spec.rb
+++ b/spec/lib/gitlab/usage_data_spec.rb
@@ -96,6 +96,7 @@ describe Gitlab::UsageData do
         pages_domains
         protected_branches
         releases
+        remote_mirrors
         snippets
         todos
         uploads
diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb
index f3cf21cf279..41622fbbb6f 100644
--- a/spec/models/project_spec.rb
+++ b/spec/models/project_spec.rb
@@ -1852,6 +1852,85 @@ describe Project do
     it { expect(project.gitea_import?).to be true }
   end
 
+  describe '#has_remote_mirror?' do
+    let(:project) { create(:project, :remote_mirror, :import_started) }
+    subject { project.has_remote_mirror? }
+
+    before do
+      allow_any_instance_of(RemoteMirror).to receive(:refresh_remote)
+    end
+
+    it 'returns true when a remote mirror is enabled' do
+      is_expected.to be_truthy
+    end
+
+    it 'returns false when remote mirror is disabled' do
+      project.remote_mirrors.first.update_attributes(enabled: false)
+
+      is_expected.to be_falsy
+    end
+  end
+
+  describe '#update_remote_mirrors' do
+    let(:project) { create(:project, :remote_mirror, :import_started) }
+    delegate :update_remote_mirrors, to: :project
+
+    before do
+      allow_any_instance_of(RemoteMirror).to receive(:refresh_remote)
+    end
+
+    it 'syncs enabled remote mirror' do
+      expect_any_instance_of(RemoteMirror).to receive(:sync)
+
+      update_remote_mirrors
+    end
+
+    # TODO: study if remote_mirror_available_overridden is still a necessary attribute considering that
+    # it is no longer under any license
+    it 'does nothing when remote mirror is disabled globally and not overridden' do
+      stub_application_setting(mirror_available: false)
+      project.remote_mirror_available_overridden = false
+
+      expect_any_instance_of(RemoteMirror).not_to receive(:sync)
+
+      update_remote_mirrors
+    end
+
+    it 'does not sync disabled remote mirrors' do
+      project.remote_mirrors.first.update_attributes(enabled: false)
+
+      expect_any_instance_of(RemoteMirror).not_to receive(:sync)
+
+      update_remote_mirrors
+    end
+  end
+
+  describe '#remote_mirror_available?' do
+    let(:project) { create(:project) }
+
+    context 'when remote mirror global setting is enabled' do
+      it 'returns true' do
+        expect(project.remote_mirror_available?).to be(true)
+      end
+    end
+
+    context 'when remote mirror global setting is disabled' do
+      before do
+        stub_application_setting(mirror_available: false)
+      end
+
+      it 'returns true when overridden' do
+        project.remote_mirror_available_overridden = true
+
+        expect(project.remote_mirror_available?).to be(true)
+      end
+
+      it 'returns false when not overridden' do
+        expect(project.remote_mirror_available?).to be(false)
+      end
+    end
+  end
+
   describe '#ancestors_upto', :nested_groups do
     let(:parent) { create(:group) }
     let(:child) { create(:group, parent: parent) }
diff --git a/spec/models/remote_mirror_spec.rb b/spec/models/remote_mirror_spec.rb
new file mode 100644
index 00000000000..a80800c6c92
--- /dev/null
+++ b/spec/models/remote_mirror_spec.rb
@@ -0,0 +1,267 @@
+require 'rails_helper'
+
+describe RemoteMirror do
+  describe 'URL validation' do
+    context 'with a valid URL' do
+      it 'should be valid' do
+        remote_mirror = build(:remote_mirror)
+        expect(remote_mirror).to be_valid
+      end
+    end
+
+    context 'with an invalid URL' do
+      it 'should not be valid' do
+        remote_mirror = build(:remote_mirror, url: 'ftp://invalid.invalid')
+        expect(remote_mirror).not_to be_valid
+        expect(remote_mirror.errors[:url].size).to eq(2)
+      end
+    end
+  end
+
+  describe 'encrypting credentials' do
+    context 'when setting URL for a first time' do
+      it 'stores the URL without credentials' do
+        mirror = create_mirror(url: 'http://foo:bar@test.com')
+
+        expect(mirror.read_attribute(:url)).to eq('http://test.com')
+      end
+
+      it 'stores the credentials on a separate field' do
+        mirror = create_mirror(url: 'http://foo:bar@test.com')
+
+        expect(mirror.credentials).to eq({ user: 'foo', password: 'bar' })
+      end
+
+      it 'handles credentials with large content' do
+        mirror = create_mirror(url: 'http://bxnhm8dote33ct932r3xavslj81wxmr7o8yux8do10oozckkif:9ne7fuvjn40qjt35dgt8v86q9m9g9essryxj76sumg2ccl2fg26c0krtz2gzfpyq4hf22h328uhq6npuiq6h53tpagtsj7vsrz75@test.com')
+
+        expect(mirror.credentials).to eq({
+          user: 'bxnhm8dote33ct932r3xavslj81wxmr7o8yux8do10oozckkif',
+          password: '9ne7fuvjn40qjt35dgt8v86q9m9g9essryxj76sumg2ccl2fg26c0krtz2gzfpyq4hf22h328uhq6npuiq6h53tpagtsj7vsrz75'
+        })
+      end
+    end
+
+    context 'when updating the URL' do
+      it 'allows a new URL without credentials' do
+        mirror = create_mirror(url: 'http://foo:bar@test.com')
+
+        mirror.update_attribute(:url, 'http://test.com')
+
+        expect(mirror.url).to eq('http://test.com')
+        expect(mirror.credentials).to eq({ user: nil, password: nil })
+      end
+
+      it 'allows a new URL with credentials' do
+        mirror = create_mirror(url: 'http://test.com')
+
+        mirror.update_attribute(:url, 'http://foo:bar@test.com')
+
+        expect(mirror.url).to eq('http://foo:bar@test.com')
+        expect(mirror.credentials).to eq({ user: 'foo', password: 'bar' })
+      end
+
+      it 'updates the remote config if credentials changed' do
+        mirror = create_mirror(url: 'http://foo:bar@test.com')
+        repo = mirror.project.repository
+
+        mirror.update_attribute(:url, 'http://foo:baz@test.com')
+
+        config = repo.raw_repository.rugged.config
+        expect(config["remote.#{mirror.remote_name}.url"]).to eq('http://foo:baz@test.com')
+      end
+
+      it 'removes previous remote' do
+        mirror = create_mirror(url: 'http://foo:bar@test.com')
+
+        expect(RepositoryRemoveRemoteWorker).to receive(:perform_async).with(mirror.project.id, mirror.remote_name).and_call_original
+
+        mirror.update_attributes(url: 'http://test.com')
+      end
+    end
+  end
+
+  describe '#remote_name' do
+    context 'when remote name is persisted in the database' do
+      it 'returns remote name with random value' do
+        allow(SecureRandom).to receive(:hex).and_return('secret')
+
+        remote_mirror = create(:remote_mirror)
+
+        expect(remote_mirror.remote_name).to eq("remote_mirror_secret")
+      end
+    end
+
+    context 'when remote name is not persisted in the database' do
+      it 'returns remote name with remote mirror id' do
+        remote_mirror = create(:remote_mirror)
+        remote_mirror.remote_name = nil
+
+        expect(remote_mirror.remote_name).to eq("remote_mirror_#{remote_mirror.id}")
+      end
+    end
+
+    context 'when remote is not persisted in the database' do
+      it 'returns nil' do
+        remote_mirror = build(:remote_mirror, remote_name: nil)
+
+        expect(remote_mirror.remote_name).to be_nil
+      end
+    end
+  end
+
+  describe '#safe_url' do
+    context 'when URL contains credentials' do
+      it 'masks the credentials' do
+        mirror = create_mirror(url: 'http://foo:bar@test.com')
+
+        expect(mirror.safe_url).to eq('http://*****:*****@test.com')
+      end
+    end
+
+    context 'when URL does not contain credentials' do
+      it 'shows the full URL' do
+        mirror = create_mirror(url: 'http://test.com')
+
+        expect(mirror.safe_url).to eq('http://test.com')
+      end
+    end
+  end
+
+  context 'when remote mirror gets destroyed' do
+    it 'removes remote' do
+      mirror = create_mirror(url: 'http://foo:bar@test.com')
+
+      expect(RepositoryRemoveRemoteWorker).to receive(:perform_async).with(mirror.project.id, mirror.remote_name).and_call_original
+
+      mirror.destroy!
+    end
+  end
+
+  context 'stuck mirrors' do
+    it 'includes mirrors stuck in started with no last_update_at set' do
+      mirror = create_mirror(url: 'http://cantbeblank',
+                             update_status: 'started',
+                             last_update_at: nil,
+                             updated_at: 25.hours.ago)
+
+      expect(described_class.stuck.last).to eq(mirror)
+    end
+  end
+
+  context '#sync' do
+    let(:remote_mirror) { create(:project, :repository, :remote_mirror).remote_mirrors.first }
+
+    around do |example|
+      Timecop.freeze { example.run }
+    end
+
+    context 'with remote mirroring disabled' do
+      it 'returns nil' do
+        remote_mirror.update_attributes(enabled: false)
+
+        expect(remote_mirror.sync).to be_nil
+      end
+    end
+
+    context 'with remote mirroring enabled' do
+      context 'with only protected branches enabled' do
+        context 'when it did not update in the last minute' do
+          it 'schedules a RepositoryUpdateRemoteMirrorWorker to run now' do
+            expect(RepositoryUpdateRemoteMirrorWorker).to receive(:perform_async).with(remote_mirror.id, Time.now)
+
+            remote_mirror.sync
+          end
+        end
+
+        context 'when it did update in the last minute' do
+          it 'schedules a RepositoryUpdateRemoteMirrorWorker to run in the next minute' do
+            remote_mirror.last_update_started_at = Time.now - 30.seconds
+
+            expect(RepositoryUpdateRemoteMirrorWorker).to receive(:perform_in).with(RemoteMirror::PROTECTED_BACKOFF_DELAY, remote_mirror.id, Time.now)
+
+            remote_mirror.sync
+          end
+        end
+      end
+
+      context 'with only protected branches disabled' do
+        before do
+          remote_mirror.only_protected_branches = false
+        end
+
+        context 'when it did not update in the last 5 minutes' do
+          it 'schedules a RepositoryUpdateRemoteMirrorWorker to run now' do
+            expect(RepositoryUpdateRemoteMirrorWorker).to receive(:perform_async).with(remote_mirror.id, Time.now)
+
+            remote_mirror.sync
+          end
+        end
+
+        context 'when it did update within the last 5 minutes' do
+          it 'schedules a RepositoryUpdateRemoteMirrorWorker to run in the next 5 minutes' do
+            remote_mirror.last_update_started_at = Time.now - 30.seconds
+
+            expect(RepositoryUpdateRemoteMirrorWorker).to receive(:perform_in).with(RemoteMirror::UNPROTECTED_BACKOFF_DELAY, remote_mirror.id, Time.now)
+
+            remote_mirror.sync
+          end
+        end
+      end
+    end
+  end
+
+  context '#updated_since?' do
+    let(:remote_mirror) { create(:project, :repository, :remote_mirror).remote_mirrors.first }
+    let(:timestamp) { Time.now - 5.minutes }
+
+    around do |example|
+      Timecop.freeze { example.run }
+    end
+
+    before do
+      remote_mirror.update_attributes(last_update_started_at: Time.now)
+    end
+
+    context 'when remote mirror does not have status failed' do
+      it 'returns true when last update started after the timestamp' do
+        expect(remote_mirror.updated_since?(timestamp)).to be true
+      end
+
+      it 'returns false when last update started before the timestamp' do
+        expect(remote_mirror.updated_since?(Time.now + 5.minutes)).to be false
+      end
+    end
+
+    context 'when remote mirror has status failed' do
+      it 'returns false when last update started after the timestamp' do
+        remote_mirror.update_attributes(update_status: 'failed')
+
+        expect(remote_mirror.updated_since?(timestamp)).to be false
+      end
+    end
+  end
+
+  context 'no project' do
+    it 'includes mirror with a project in pending_delete' do
+      mirror = create_mirror(url: 'http://cantbeblank',
+                             update_status: 'finished',
+                             enabled: true,
+                             last_update_at: nil,
+                             updated_at: 25.hours.ago)
+      project = mirror.project
+      project.pending_delete = true
+      project.save
+      mirror.reload
+
+      expect(mirror.sync).to be_nil
+      expect(mirror.valid?).to be_truthy
+      expect(mirror.update_status).to eq('finished')
+    end
+  end
+
+  def create_mirror(params)
+    project = FactoryBot.create(:project, :repository)
+    project.remote_mirrors.create!(params)
+  end
+end
diff --git a/spec/models/repository_spec.rb b/spec/models/repository_spec.rb
index 630b9e0519f..4b736b02b7d 100644
--- a/spec/models/repository_spec.rb
+++ b/spec/models/repository_spec.rb
@@ -758,6 +758,38 @@ describe Repository do
     end
   end
 
+  describe '#async_remove_remote' do
+    before do
+      masterrev = repository.find_branch('master').dereferenced_target
+      create_remote_branch('joe', 'remote_branch', masterrev)
+    end
+
+    context 'when worker is scheduled successfully' do
+      before do
+        masterrev = repository.find_branch('master').dereferenced_target
+        create_remote_branch('remote_name', 'remote_branch', masterrev)
+
+        allow(RepositoryRemoveRemoteWorker).to receive(:perform_async).and_return('1234')
+      end
+
+      it 'returns job_id' do
+        expect(repository.async_remove_remote('joe')).to eq('1234')
+      end
+    end
+
+    context 'when worker does not schedule successfully' do
+      before do
+        allow(RepositoryRemoveRemoteWorker).to receive(:perform_async).and_return(nil)
+      end
+
+      it 'returns nil' do
+        expect(Rails.logger).to receive(:info).with("Remove remote job failed to create for #{project.id} with remote name joe.")
+
+        expect(repository.async_remove_remote('joe')).to be_nil
+      end
+    end
+  end
+
   describe '#fetch_ref' do
     let(:broken_repository) { create(:project, :broken_storage).repository }
 
@@ -2338,6 +2370,11 @@ describe Repository do
     end
   end
 
+  def create_remote_branch(remote_name, branch_name, target)
+    rugged = repository.rugged
+    rugged.references.create("refs/remotes/#{remote_name}/#{branch_name}", target.id)
+  end
+
   describe '#ancestor?' do
     let(:commit) { repository.commit }
     let(:ancestor) { commit.parents.first }
diff --git a/spec/services/git_push_service_spec.rb b/spec/services/git_push_service_spec.rb
index 26fdf8d4b24..35826de5814 100644
--- a/spec/services/git_push_service_spec.rb
+++ b/spec/services/git_push_service_spec.rb
@@ -14,6 +14,72 @@ describe GitPushService, services: true do
     project.add_master(user)
   end
 
+  describe 'with remote mirrors' do
+    let(:project)  { create(:project, :repository, :remote_mirror) }
+
+    subject do
+      described_class.new(project, user, oldrev: oldrev, newrev: newrev, ref: ref)
+    end
+
+    context 'when remote mirror feature is enabled' do
+      it 'fails stuck remote mirrors' do
+        allow(project).to receive(:update_remote_mirrors).and_return(project.remote_mirrors)
+        expect(project).to receive(:mark_stuck_remote_mirrors_as_failed!)
+
+        subject.execute
+      end
+
+      it 'updates remote mirrors' do
+        expect(project).to receive(:update_remote_mirrors)
+
+        subject.execute
+      end
+    end
+
+    context 'when remote mirror feature is disabled' do
+      before do
+        stub_application_setting(mirror_available: false)
+      end
+
+      context 'with remote mirrors global setting overridden' do
+        before do
+          project.remote_mirror_available_overridden = true
+        end
+
+        it 'fails stuck remote mirrors' do
+          allow(project).to receive(:update_remote_mirrors).and_return(project.remote_mirrors)
+          expect(project).to receive(:mark_stuck_remote_mirrors_as_failed!)
+
+          subject.execute
+        end
+
+        it 'updates remote mirrors' do
+          expect(project).to receive(:update_remote_mirrors)
+
+          subject.execute
+        end
+      end
+
+      context 'without remote mirrors global setting overridden' do
+        before do
+          project.remote_mirror_available_overridden = false
+        end
+
+        it 'does not fails stuck remote mirrors' do
+          expect(project).not_to receive(:mark_stuck_remote_mirrors_as_failed!)
+
+          subject.execute
+        end
+
+        it 'does not updates remote mirrors' do
+          expect(project).not_to receive(:update_remote_mirrors)
+
+          subject.execute
+        end
+      end
+    end
+  end
+
   describe 'Push branches' do
     subject do
       execute_service(project, user, oldrev, newrev, ref)
diff --git a/spec/services/projects/destroy_service_spec.rb b/spec/services/projects/destroy_service_spec.rb
index b2c52214f48..b63f409579e 100644
--- a/spec/services/projects/destroy_service_spec.rb
+++ b/spec/services/projects/destroy_service_spec.rb
@@ -65,6 +65,19 @@ describe Projects::DestroyService do
       Sidekiq::Testing.inline! { destroy_project(project, user, {}) }
     end
 
+    context 'when has remote mirrors' do
+      let!(:project) do
+        create(:project, :repository, namespace: user.namespace).tap do |project|
+          project.remote_mirrors.create(url: 'http://test.com')
+        end
+      end
+      let!(:async) { true }
+
+      it 'destroys them' do
+        expect(RemoteMirror.count).to eq(0)
+      end
+    end
+
     it_behaves_like 'deleting the project'
 
     it 'invalidates personal_project_count cache' do
diff --git a/spec/services/projects/update_remote_mirror_service_spec.rb b/spec/services/projects/update_remote_mirror_service_spec.rb
new file mode 100644
index 00000000000..be09afd9f36
--- /dev/null
+++ b/spec/services/projects/update_remote_mirror_service_spec.rb
@@ -0,0 +1,355 @@
+require 'spec_helper'
+
+describe Projects::UpdateRemoteMirrorService do
+  let(:project) { create(:project, :repository) }
+  let(:remote_project) { create(:forked_project_with_submodules) }
+  let(:repository) { project.repository }
+  let(:raw_repository) { repository.raw }
+  let(:remote_mirror) { project.remote_mirrors.create!(url: remote_project.http_url_to_repo, enabled: true, only_protected_branches: false) }
+
+  subject { described_class.new(project, project.creator) }
+
+  describe "#execute", :skip_gitaly_mock do
+    before do
+      create_branch(repository, 'existing-branch')
+      allow(raw_repository).to receive(:remote_tags) do
+        generate_tags(repository, 'v1.0.0', 'v1.1.0')
+      end
+      allow(raw_repository).to receive(:push_remote_branches).and_return(true)
+    end
+
+    it "fetches the remote repository" do
+      expect(repository).to receive(:fetch_remote).with(remote_mirror.remote_name, no_tags: true) do
+        sync_remote(repository, remote_mirror.remote_name, local_branch_names)
+      end
+
+      subject.execute(remote_mirror)
+    end
+
+    it "succeeds" do
+      allow(repository).to receive(:fetch_remote) { sync_remote(repository, remote_mirror.remote_name, local_branch_names) }
+
+      result = subject.execute(remote_mirror)
+
+      expect(result[:status]).to eq(:success)
+    end
+
+    describe 'Syncing branches' do
+      it "push all the branches the first time" do
+        allow(repository).to receive(:fetch_remote)
+
+        expect(raw_repository).to receive(:push_remote_branches).with(remote_mirror.remote_name, local_branch_names)
+
+        subject.execute(remote_mirror)
+      end
+
+      it "does not push anything is remote is up to date" do
+        allow(repository).to receive(:fetch_remote) { sync_remote(repository, remote_mirror.remote_name, local_branch_names) }
+
+        expect(raw_repository).not_to receive(:push_remote_branches)
+
+        subject.execute(remote_mirror)
+      end
+
+      it "sync new branches" do
+        # call local_branch_names early so it is not called after the new branch has been created
+        current_branches = local_branch_names
+        allow(repository).to receive(:fetch_remote) { sync_remote(repository, remote_mirror.remote_name, current_branches) }
+        create_branch(repository, 'my-new-branch')
+
+        expect(raw_repository).to receive(:push_remote_branches).with(remote_mirror.remote_name, ['my-new-branch'])
+
+        subject.execute(remote_mirror)
+      end
+
+      it "sync updated branches" do
+        allow(repository).to receive(:fetch_remote) do
+          sync_remote(repository, remote_mirror.remote_name, local_branch_names)
+          update_branch(repository, 'existing-branch')
+        end
+
+        expect(raw_repository).to receive(:push_remote_branches).with(remote_mirror.remote_name, ['existing-branch'])
+
+        subject.execute(remote_mirror)
+      end
+
+      context 'when push only protected branches option is set' do
+        let(:unprotected_branch_name) { 'existing-branch' }
+        let(:protected_branch_name) do
+          project.repository.branch_names.find { |n| n != unprotected_branch_name }
+        end
+        let!(:protected_branch) do
+          create(:protected_branch, project: project, name: protected_branch_name)
+        end
+
+        before do
+          project.reload
+          remote_mirror.only_protected_branches = true
+        end
+
+        it "sync updated protected branches" do
+          allow(repository).to receive(:fetch_remote) do
+            sync_remote(repository, remote_mirror.remote_name, local_branch_names)
+            update_branch(repository, protected_branch_name)
+          end
+
+          expect(raw_repository).to receive(:push_remote_branches).with(remote_mirror.remote_name, [protected_branch_name])
+
+          subject.execute(remote_mirror)
+        end
+
+        it 'does not sync unprotected branches' do
+          allow(repository).to receive(:fetch_remote) do
+            sync_remote(repository, remote_mirror.remote_name, local_branch_names)
+            update_branch(repository, unprotected_branch_name)
+          end
+
+          expect(raw_repository).not_to receive(:push_remote_branches).with(remote_mirror.remote_name, [unprotected_branch_name])
+
+          subject.execute(remote_mirror)
+        end
+      end
+
+      context 'when branch exists in local and remote repo' do
+        context 'when it has diverged' do
+          it 'syncs branches' do
+            allow(repository).to receive(:fetch_remote) do
+              sync_remote(repository, remote_mirror.remote_name, local_branch_names)
+              update_remote_branch(repository, remote_mirror.remote_name, 'markdown')
+            end
+
+            expect(raw_repository).to receive(:push_remote_branches).with(remote_mirror.remote_name, ['markdown'])
+
+            subject.execute(remote_mirror)
+          end
+        end
+      end
+
+      describe 'for delete' do
+        context 'when branch exists in local and remote repo' do
+          it 'deletes the branch from remote repo' do
+            allow(repository).to receive(:fetch_remote) do
+              sync_remote(repository, remote_mirror.remote_name, local_branch_names)
+              delete_branch(repository, 'existing-branch')
+            end
+
+            expect(raw_repository).to receive(:delete_remote_branches).with(remote_mirror.remote_name, ['existing-branch'])
+
+            subject.execute(remote_mirror)
+          end
+        end
+
+        context 'when push only protected branches option is set' do
+          before do
+            remote_mirror.only_protected_branches = true
+          end
+
+          context 'when branch exists in local and remote repo' do
+            let!(:protected_branch_name) { local_branch_names.first }
+
+            before do
+              create(:protected_branch, project: project, name: protected_branch_name)
+              project.reload
+            end
+
+            it 'deletes the protected branch from remote repo' do
+              allow(repository).to receive(:fetch_remote) do
+                sync_remote(repository, remote_mirror.remote_name, local_branch_names)
+                delete_branch(repository, protected_branch_name)
+              end
+
+              expect(raw_repository).not_to receive(:delete_remote_branches).with(remote_mirror.remote_name, [protected_branch_name])
+
+              subject.execute(remote_mirror)
+            end
+
+            it 'does not delete the unprotected branch from remote repo' do
+              allow(repository).to receive(:fetch_remote) do
+                sync_remote(repository, remote_mirror.remote_name, local_branch_names)
+                delete_branch(repository, 'existing-branch')
+              end
+
+              expect(raw_repository).not_to receive(:delete_remote_branches).with(remote_mirror.remote_name, ['existing-branch'])
+
+              subject.execute(remote_mirror)
+            end
+          end
+
+          context 'when branch only exists on remote repo' do
+            let!(:protected_branch_name) { 'remote-branch' }
+
+            before do
+              create(:protected_branch, project: project, name: protected_branch_name)
+            end
+
+            context 'when it has diverged' do
+              it 'does not delete the remote branch' do
+                allow(repository).to receive(:fetch_remote) do
+                  sync_remote(repository, remote_mirror.remote_name, local_branch_names)
+
+                  rev = repository.find_branch('markdown').dereferenced_target
+                  create_remote_branch(repository, remote_mirror.remote_name, 'remote-branch', rev.id)
+                end
+
+                expect(raw_repository).not_to receive(:delete_remote_branches)
+
+                subject.execute(remote_mirror)
+              end
+            end
+
+            context 'when it has not diverged' do
+              it 'deletes the remote branch' do
+                allow(repository).to receive(:fetch_remote) do
+                  sync_remote(repository, remote_mirror.remote_name, local_branch_names)
+
+                  masterrev = repository.find_branch('master').dereferenced_target
+                  create_remote_branch(repository, remote_mirror.remote_name, protected_branch_name, masterrev.id)
+                end
+
+                expect(raw_repository).to receive(:delete_remote_branches).with(remote_mirror.remote_name, [protected_branch_name])
+
+                subject.execute(remote_mirror)
+              end
+            end
+          end
+        end
+
+        context 'when branch only exists on remote repo' do
+          context 'when it has diverged' do
+            it 'does not delete the remote branch' do
+              allow(repository).to receive(:fetch_remote) do
+                sync_remote(repository, remote_mirror.remote_name, local_branch_names)
+
+                rev = repository.find_branch('markdown').dereferenced_target
+                create_remote_branch(repository, remote_mirror.remote_name, 'remote-branch', rev.id)
+              end
+
+              expect(raw_repository).not_to receive(:delete_remote_branches)
+
+              subject.execute(remote_mirror)
+            end
+          end
+
+          context 'when it has not diverged' do
+            it 'deletes the remote branch' do
+              allow(repository).to receive(:fetch_remote) do
+                sync_remote(repository, remote_mirror.remote_name, local_branch_names)
+
+                masterrev = repository.find_branch('master').dereferenced_target
+                create_remote_branch(repository, remote_mirror.remote_name, 'remote-branch', masterrev.id)
+              end
+
+              expect(raw_repository).to receive(:delete_remote_branches).with(remote_mirror.remote_name, ['remote-branch'])
+
+              subject.execute(remote_mirror)
+            end
+          end
+        end
+      end
+    end
+
+    describe 'Syncing tags' do
+      before do
+        allow(repository).to receive(:fetch_remote) { sync_remote(repository, remote_mirror.remote_name, local_branch_names) }
+      end
+
+      context 'when there are not tags to push' do
+        it 'does not try to push tags' do
+          allow(repository).to receive(:remote_tags) { {} }
+          allow(repository).to receive(:tags) { [] }
+
+          expect(repository).not_to receive(:push_tags)
+
+          subject.execute(remote_mirror)
+        end
+      end
+
+      context 'when there are some tags to push' do
+        it 'pushes tags to remote' do
+          allow(raw_repository).to receive(:remote_tags) { {} }
+
+          expect(raw_repository).to receive(:push_remote_branches).with(remote_mirror.remote_name, ['v1.0.0', 'v1.1.0'])
+
+          subject.execute(remote_mirror)
+        end
+      end
+
+      context 'when there are some tags to delete' do
+        it 'deletes tags from remote' do
+          remote_tags = generate_tags(repository, 'v1.0.0', 'v1.1.0')
+          allow(raw_repository).to receive(:remote_tags) { remote_tags }
+
+          repository.rm_tag(create(:user), 'v1.0.0')
+
+          expect(raw_repository).to receive(:delete_remote_branches).with(remote_mirror.remote_name, ['v1.0.0'])
+
+          subject.execute(remote_mirror)
+        end
+      end
+    end
+  end
+
+  def create_branch(repository, branch_name)
+    rugged = repository.rugged
+    masterrev = repository.find_branch('master').dereferenced_target
+    parentrev = repository.commit(masterrev).parent_id
+
+    rugged.references.create("refs/heads/#{branch_name}", parentrev)
+
+    repository.expire_branches_cache
+  end
+
+  def create_remote_branch(repository, remote_name, branch_name, source_id)
+    rugged = repository.rugged
+
+    rugged.references.create("refs/remotes/#{remote_name}/#{branch_name}", source_id)
+  end
+
+  def sync_remote(repository, remote_name, local_branch_names)
+    rugged = repository.rugged
+
+    local_branch_names.each do |branch|
+      target = repository.find_branch(branch).try(:dereferenced_target)
+      rugged.references.create("refs/remotes/#{remote_name}/#{branch}", target.id) if target
+    end
+  end
+
+  def update_remote_branch(repository, remote_name, branch)
+    rugged = repository.rugged
+    masterrev = repository.find_branch('master').dereferenced_target.id
+
+    rugged.references.create("refs/remotes/#{remote_name}/#{branch}", masterrev, force: true)
+    repository.expire_branches_cache
+  end
+
+  def update_branch(repository, branch)
+    rugged = repository.rugged
+    masterrev = repository.find_branch('master').dereferenced_target.id
+
+    # Updated existing branch
+    rugged.references.create("refs/heads/#{branch}", masterrev, force: true)
+    repository.expire_branches_cache
+  end
+
+  def delete_branch(repository, branch)
+    rugged = repository.rugged
+
+    rugged.references.delete("refs/heads/#{branch}")
+    repository.expire_branches_cache
+  end
+
+  def generate_tags(repository, *tag_names)
+    tag_names.each_with_object([]) do |name, tags|
+      tag = repository.find_tag(name)
+      target = tag.try(:target)
+      target_commit = tag.try(:dereferenced_target)
+      tags << Gitlab::Git::Tag.new(repository.raw_repository, name, target, target_commit)
+    end
+  end
+
+  def local_branch_names
+    branch_names = repository.branches.map(&:name)
+    # we want the protected branch to be pushed first
+    branch_names.unshift(branch_names.delete('master'))
+  end
+end
diff --git a/spec/workers/repository_remove_remote_worker_spec.rb b/spec/workers/repository_remove_remote_worker_spec.rb
new file mode 100644
index 00000000000..f22d7c1d073
--- /dev/null
+++ b/spec/workers/repository_remove_remote_worker_spec.rb
@@ -0,0 +1,50 @@
+require 'rails_helper'
+
+describe RepositoryRemoveRemoteWorker do
+  subject(:worker) { described_class.new }
+
+  describe '#perform' do
+    let(:remote_name) { 'joe'}
+    let!(:project) { create(:project, :repository) }
+
+    context 'when it cannot obtain lease' do
+      it 'logs error' do
+        allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain) { nil }
+
+        expect_any_instance_of(Repository).not_to receive(:remove_remote)
+        expect(worker).to receive(:log_error).with('Cannot obtain an exclusive lease. There must be another instance already in execution.')
+
+        worker.perform(project.id, remote_name)
+      end
+    end
+
+    context 'when it gets the lease' do
+      before do
+        allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain).and_return(true)
+      end
+
+      context 'when project does not exist' do
+        it 'returns nil' do
+          expect(worker.perform(-1, 'remote_name')).to be_nil
+        end
+      end
+
+      context 'when project exists' do
+        it 'removes remote from repository' do
+          masterrev = project.repository.find_branch('master').dereferenced_target
+
+          create_remote_branch(remote_name, 'remote_branch', masterrev)
+
+          expect_any_instance_of(Repository).to receive(:remove_remote).with(remote_name).and_call_original
+
+          worker.perform(project.id, remote_name)
+        end
+      end
+    end
+  end
+
+  def create_remote_branch(remote_name, branch_name, target)
+    rugged = project.repository.rugged
+    rugged.references.create("refs/remotes/#{remote_name}/#{branch_name}", target.id)
+  end
+end
diff --git a/spec/workers/repository_update_remote_mirror_worker_spec.rb b/spec/workers/repository_update_remote_mirror_worker_spec.rb
new file mode 100644
index 00000000000..152ba2509b9
--- /dev/null
+++ b/spec/workers/repository_update_remote_mirror_worker_spec.rb
@@ -0,0 +1,84 @@
+require 'rails_helper'
+
+describe RepositoryUpdateRemoteMirrorWorker do
+  subject { described_class.new }
+
+  let(:remote_mirror) { create(:project, :repository, :remote_mirror).remote_mirrors.first }
+  let(:scheduled_time) { Time.now - 5.minutes }
+
+  around do |example|
+    Timecop.freeze(Time.now) { example.run }
+  end
+
+  describe '#perform' do
+    context 'with status none' do
+      before do
+        remote_mirror.update_attributes(update_status: 'none')
+      end
+
+      it 'sets status as finished when update remote mirror service executes successfully' do
+        expect_any_instance_of(Projects::UpdateRemoteMirrorService).to receive(:execute).with(remote_mirror).and_return(status: :success)
+
+        expect { subject.perform(remote_mirror.id, Time.now) }.to change { remote_mirror.reload.update_status }.to('finished')
+      end
+
+      it 'sets status as failed when update remote mirror service executes with errors' do
+        error_message = 'fail!'
+
+        expect_any_instance_of(Projects::UpdateRemoteMirrorService).to receive(:execute).with(remote_mirror).and_return(status: :error, message: error_message)
+        expect do
+          subject.perform(remote_mirror.id, Time.now)
+        end.to raise_error(RepositoryUpdateRemoteMirrorWorker::UpdateError, error_message)
+
+        expect(remote_mirror.reload.update_status).to eq('failed')
+      end
+
+      it 'does nothing if last_update_started_at is higher than the time the job was scheduled in' do
+        remote_mirror.update_attributes(last_update_started_at: Time.now)
+
+        expect_any_instance_of(RemoteMirror).to receive(:updated_since?).with(scheduled_time).and_return(true)
+        expect_any_instance_of(Projects::UpdateRemoteMirrorService).not_to receive(:execute).with(remote_mirror)
+
+        expect(subject.perform(remote_mirror.id, scheduled_time)).to be_nil
+      end
+    end
+
+    context 'with unexpected error' do
+      it 'marks mirror as failed' do
+        allow_any_instance_of(Projects::UpdateRemoteMirrorService).to receive(:execute).with(remote_mirror).and_raise(RuntimeError)
+
+        expect do
+          subject.perform(remote_mirror.id, Time.now)
+        end.to raise_error(RepositoryUpdateRemoteMirrorWorker::UpdateError)
+        expect(remote_mirror.reload.update_status).to eq('failed')
+      end
+    end
+
+    context 'with another worker already running' do
+      before do
+        remote_mirror.update_attributes(update_status: 'started')
+      end
+
+      it 'raises RemoteMirrorUpdateAlreadyInProgressError' do
+        expect do
+          subject.perform(remote_mirror.id, Time.now)
+        end.to raise_error(RepositoryUpdateRemoteMirrorWorker::UpdateAlreadyInProgressError)
+      end
+    end
+
+    context 'with status failed' do
+      before do
+        remote_mirror.update_attributes(update_status: 'failed')
+      end
+
+      it 'sets status as finished if last_update_started_at is higher than the time the job was scheduled in' do
+        remote_mirror.update_attributes(last_update_started_at: Time.now)
+
+        expect_any_instance_of(RemoteMirror).to receive(:updated_since?).with(scheduled_time).and_return(false)
+        expect_any_instance_of(Projects::UpdateRemoteMirrorService).to receive(:execute).with(remote_mirror).and_return(status: :success)
+
+        expect { subject.perform(remote_mirror.id, scheduled_time) }.to change { remote_mirror.reload.update_status }.to('finished')
+      end
+    end
+  end
+end