Merge branch '19376-post-bfg-cleanup' into 'master'

Allow internal references to be removed

Closes #19376

See merge request gitlab-org/gitlab-ce!23189

10 files changed, 299 insertions, 0 deletions

diff --git a/spec/controllers/projects/settings/repository_controller_spec.rb b/spec/controllers/projects/settings/repository_controller_spec.rb
index 9cee40b7553..69ec971bb75 100644
--- a/spec/controllers/projects/settings/repository_controller_spec.rb
+++ b/spec/controllers/projects/settings/repository_controller_spec.rb
@@ -17,4 +17,35 @@ describe Projects::Settings::RepositoryController do
       expect(response).to render_template(:show)
     end
   end
+
+  describe 'PUT cleanup' do
+    def do_put!
+      object_map = fixture_file_upload('spec/fixtures/bfg_object_map.txt')
+
+      Sidekiq::Testing.fake! do
+        put :cleanup, namespace_id: project.namespace, project_id: project, project: { object_map: object_map }
+      end
+    end
+
+    context 'feature enabled' do
+      it 'enqueues a RepositoryCleanupWorker' do
+        stub_feature_flags(project_cleanup: true)
+
+        do_put!
+
+        expect(response).to redirect_to project_settings_repository_path(project)
+        expect(RepositoryCleanupWorker.jobs.count).to eq(1)
+      end
+    end
+
+    context 'feature disabled' do
+      it 'shows a 404 error' do
+        stub_feature_flags(project_cleanup: false)
+
+        do_put!
+
+        expect(response).to have_gitlab_http_status(404)
+      end
+    end
+  end
 end
diff --git a/spec/features/projects/settings/repository_settings_spec.rb b/spec/features/projects/settings/repository_settings_spec.rb
index b7a22316d26..418e22f8c35 100644
--- a/spec/features/projects/settings/repository_settings_spec.rb
+++ b/spec/features/projects/settings/repository_settings_spec.rb
@@ -196,5 +196,40 @@ describe 'Projects > Settings > Repository settings' do
         end
       end
     end
+
+    context 'repository cleanup settings' do
+      let(:object_map_file) { Rails.root.join('spec', 'fixtures', 'bfg_object_map.txt') }
+
+      context 'feature enabled' do
+        it 'uploads an object map file', :js do
+          stub_feature_flags(project_cleanup: true)
+
+          visit project_settings_repository_path(project)
+
+          expect(page).to have_content('Repository cleanup')
+
+          page.within('#cleanup') do
+            attach_file('project[bfg_object_map]', object_map_file, visible: false)
+
+            Sidekiq::Testing.fake! do
+              click_button 'Start cleanup'
+            end
+          end
+
+          expect(page).to have_content('Repository cleanup has started')
+          expect(RepositoryCleanupWorker.jobs.count).to eq(1)
+        end
+      end
+
+      context 'feature disabled' do
+        it 'does not show the settings' do
+          stub_feature_flags(project_cleanup: false)
+
+          visit project_settings_repository_path(project)
+
+          expect(page).not_to have_content('Repository cleanup')
+        end
+      end
+    end
   end
 end
diff --git a/spec/fixtures/bfg_object_map.txt b/spec/fixtures/bfg_object_map.txt
new file mode 100644
index 00000000000..c60171d8770
--- /dev/null
+++ b/spec/fixtures/bfg_object_map.txt
@@ -0,0 +1 @@
+f1d2d2f924e986ac86fdf7b36c94bcdf32beec15 e242ed3bffccdf271b7fbaf34ed72d089537b42f
diff --git a/spec/helpers/projects_helper_spec.rb b/spec/helpers/projects_helper_spec.rb
index 976b6c312b4..a857b7646b2 100644
--- a/spec/helpers/projects_helper_spec.rb
+++ b/spec/helpers/projects_helper_spec.rb
@@ -471,6 +471,31 @@ describe ProjectsHelper do
     end
   end
 
+  describe 'link_to_bfg' do
+    subject { helper.link_to_bfg }
+
+    it 'generates a hardcoded link to the BFG Repo-Cleaner' do
+      result = helper.link_to_bfg
+      doc = Nokogiri::HTML.fragment(result)
+
+      expect(doc.children.size).to eq(1)
+
+      link = doc.children.first
+
+      aggregate_failures do
+        expect(result).to be_html_safe
+
+        expect(link.name).to eq('a')
+        expect(link[:target]).to eq('_blank')
+        expect(link[:rel]).to eq('noopener noreferrer')
+        expect(link[:href]).to eq('https://rtyley.github.io/bfg-repo-cleaner/')
+        expect(link.inner_html).to eq('BFG')
+
+        expect(result).to be_html_safe
+      end
+    end
+  end
+
   describe '#legacy_render_context' do
     it 'returns the redcarpet engine' do
       params = { legacy_render: '1' }
diff --git a/spec/javascripts/lib/utils/file_upload_spec.js b/spec/javascripts/lib/utils/file_upload_spec.js
new file mode 100644
index 00000000000..92c9cc70aaf
--- /dev/null
+++ b/spec/javascripts/lib/utils/file_upload_spec.js
@@ -0,0 +1,36 @@
+import fileUpload from '~/lib/utils/file_upload';
+
+describe('File upload', () => {
+  beforeEach(() => {
+    setFixtures(`
+      <form>
+        <button class="js-button" type="button">Click me!</button>
+        <input type="text" class="js-input" />
+        <span class="js-filename"></span>
+      </form>
+    `);
+
+    fileUpload('.js-button', '.js-input');
+  });
+
+  it('clicks file input after clicking button', () => {
+    const btn = document.querySelector('.js-button');
+    const input = document.querySelector('.js-input');
+
+    spyOn(input, 'click');
+
+    btn.click();
+
+    expect(input.click).toHaveBeenCalled();
+  });
+
+  it('updates file name text', () => {
+    const input = document.querySelector('.js-input');
+
+    input.value = 'path/to/file/index.js';
+
+    input.dispatchEvent(new CustomEvent('change'));
+
+    expect(document.querySelector('.js-filename').textContent).toEqual('index.js');
+  });
+});
diff --git a/spec/lib/gitlab/git/repository_cleaner_spec.rb b/spec/lib/gitlab/git/repository_cleaner_spec.rb
new file mode 100644
index 00000000000..a9d9e67ef94
--- /dev/null
+++ b/spec/lib/gitlab/git/repository_cleaner_spec.rb
@@ -0,0 +1,32 @@
+require 'spec_helper'
+
+describe Gitlab::Git::RepositoryCleaner do
+  let(:project) { create(:project, :repository) }
+  let(:repository) { project.repository }
+  let(:head_sha) { repository.head_commit.id }
+
+  let(:object_map) { StringIO.new("#{head_sha} #{'0' * 40}") }
+
+  subject(:cleaner) { described_class.new(repository.raw) }
+
+  describe '#apply_bfg_object_map' do
+    it 'removes internal references pointing at SHAs in the object map' do
+      # Create some refs we expect to be removed
+      repository.keep_around(head_sha)
+      repository.create_ref(head_sha, 'refs/environments/1')
+      repository.create_ref(head_sha, 'refs/merge-requests/1')
+      repository.create_ref(head_sha, 'refs/heads/_keep')
+      repository.create_ref(head_sha, 'refs/tags/_keep')
+
+      cleaner.apply_bfg_object_map(object_map)
+
+      aggregate_failures do
+        expect(repository.kept_around?(head_sha)).to be_falsy
+        expect(repository.ref_exists?('refs/environments/1')).to be_falsy
+        expect(repository.ref_exists?('refs/merge-requests/1')).to be_falsy
+        expect(repository.ref_exists?('refs/heads/_keep')).to be_truthy
+        expect(repository.ref_exists?('refs/tags/_keep')).to be_truthy
+      end
+    end
+  end
+end
diff --git a/spec/lib/gitlab/gitaly_client/cleanup_service_spec.rb b/spec/lib/gitlab/gitaly_client/cleanup_service_spec.rb
new file mode 100644
index 00000000000..369deff732a
--- /dev/null
+++ b/spec/lib/gitlab/gitaly_client/cleanup_service_spec.rb
@@ -0,0 +1,19 @@
+require 'spec_helper'
+
+describe Gitlab::GitalyClient::CleanupService do
+  let(:project) { create(:project) }
+  let(:storage_name) { project.repository_storage }
+  let(:relative_path) { project.disk_path + '.git' }
+  let(:client) { described_class.new(project.repository) }
+
+  describe '#apply_bfg_object_map' do
+    it 'sends an apply_bfg_object_map message' do
+      expect_any_instance_of(Gitaly::CleanupService::Stub)
+        .to receive(:apply_bfg_object_map)
+        .with(kind_of(Enumerator), kind_of(Hash))
+        .and_return(double)
+
+      client.apply_bfg_object_map(StringIO.new)
+    end
+  end
+end
diff --git a/spec/services/notification_service_spec.rb b/spec/services/notification_service_spec.rb
index 2d8da7673dc..0f6c2604984 100644
--- a/spec/services/notification_service_spec.rb
+++ b/spec/services/notification_service_spec.rb
@@ -2146,6 +2146,27 @@ describe NotificationService, :mailer do
     end
   end
 
+  describe 'Repository cleanup' do
+    let(:user) { create(:user) }
+    let(:project) { create(:project) }
+
+    describe '#repository_cleanup_success' do
+      it 'emails the specified user only' do
+        notification.repository_cleanup_success(project, user)
+
+        should_email(user)
+      end
+    end
+
+    describe '#repository_cleanup_failure' do
+      it 'emails the specified user only' do
+        notification.repository_cleanup_failure(project, user, 'Some error')
+
+        should_email(user)
+      end
+    end
+  end
+
   def build_team(project)
     @u_watcher               = create_global_setting_for(create(:user), :watch)
     @u_participating         = create_global_setting_for(create(:user), :participating)
diff --git a/spec/services/projects/cleanup_service_spec.rb b/spec/services/projects/cleanup_service_spec.rb
new file mode 100644
index 00000000000..3d4587ce2a1
--- /dev/null
+++ b/spec/services/projects/cleanup_service_spec.rb
@@ -0,0 +1,44 @@
+require 'spec_helper'
+
+describe Projects::CleanupService do
+  let(:project) { create(:project, :repository, bfg_object_map: fixture_file_upload('spec/fixtures/bfg_object_map.txt')) }
+  let(:object_map) { project.bfg_object_map }
+
+  subject(:service) { described_class.new(project) }
+
+  describe '#execute' do
+    it 'runs the apply_bfg_object_map gitaly RPC' do
+      expect_next_instance_of(Gitlab::Git::RepositoryCleaner) do |cleaner|
+        expect(cleaner).to receive(:apply_bfg_object_map).with(kind_of(IO))
+      end
+
+      service.execute
+    end
+
+    it 'runs garbage collection on the repository' do
+      expect_next_instance_of(GitGarbageCollectWorker) do |worker|
+        expect(worker).to receive(:perform)
+      end
+
+      service.execute
+    end
+
+    it 'clears the repository cache' do
+      expect(project.repository).to receive(:expire_all_method_caches)
+
+      service.execute
+    end
+
+    it 'removes the object map file' do
+      service.execute
+
+      expect(object_map.exists?).to be_falsy
+    end
+
+    it 'raises an error if no object map can be found' do
+      object_map.remove!
+
+      expect { service.execute }.to raise_error(described_class::NoUploadError)
+    end
+  end
+end
diff --git a/spec/workers/repository_cleanup_worker_spec.rb b/spec/workers/repository_cleanup_worker_spec.rb
new file mode 100644
index 00000000000..3adae0b6cfa
--- /dev/null
+++ b/spec/workers/repository_cleanup_worker_spec.rb
@@ -0,0 +1,55 @@
+require 'spec_helper'
+
+describe RepositoryCleanupWorker do
+  let(:project) { create(:project) }
+  let(:user) { create(:user) }
+
+  subject(:worker) { described_class.new }
+
+  describe '#perform' do
+    it 'executes the cleanup service and sends a success notification' do
+      expect_next_instance_of(Projects::CleanupService) do |service|
+        expect(service.project).to eq(project)
+        expect(service.current_user).to eq(user)
+
+        expect(service).to receive(:execute)
+      end
+
+      expect_next_instance_of(NotificationService) do |service|
+        expect(service).to receive(:repository_cleanup_success).with(project, user)
+      end
+
+      worker.perform(project.id, user.id)
+    end
+
+    it 'raises an error if the project cannot be found' do
+      project.destroy
+
+      expect { worker.perform(project.id, user.id) }.to raise_error(ActiveRecord::RecordNotFound)
+    end
+
+    it 'raises an error if the user cannot be found' do
+      user.destroy
+
+      expect { worker.perform(project.id, user.id) }.to raise_error(ActiveRecord::RecordNotFound)
+    end
+  end
+
+  describe '#sidekiq_retries_exhausted' do
+    let(:job) { { 'args' => [project.id, user.id], 'error_message' => 'Error' } }
+
+    it 'does not send a failure notification for a RecordNotFound error' do
+      expect(NotificationService).not_to receive(:new)
+
+      described_class.sidekiq_retries_exhausted_block.call(job, ActiveRecord::RecordNotFound.new)
+    end
+
+    it 'sends a failure notification' do
+      expect_next_instance_of(NotificationService) do |service|
+        expect(service).to receive(:repository_cleanup_failure).with(project, user, 'Error')
+      end
+
+      described_class.sidekiq_retries_exhausted_block.call(job, StandardError.new)
+    end
+  end
+end