Add latest changes from gitlab-org/gitlab@13-11-stable-eev13.11.0-rc43

3 files changed, 107 insertions, 36 deletions

diff --git a/spec/tasks/gitlab/gitaly_rake_spec.rb b/spec/tasks/gitlab/gitaly_rake_spec.rb
index efc983d526f..1b38580f484 100644
--- a/spec/tasks/gitlab/gitaly_rake_spec.rb
+++ b/spec/tasks/gitlab/gitaly_rake_spec.rb
@@ -41,6 +41,7 @@ RSpec.describe 'gitlab:gitaly namespace rake task' do
 
     describe 'checkout or clone' do
       before do
+        stub_env('CI', false)
         expect(Dir).to receive(:chdir).with(clone_path)
       end
 
@@ -86,18 +87,14 @@ RSpec.describe 'gitlab:gitaly namespace rake task' do
         end
 
         context 'when Rails.env is test' do
-          let(:command) do
-            %W[make
-               BUNDLE_FLAGS=--no-deployment
-               GEM_HOME=#{Bundler.bundle_path}]
-          end
+          let(:command) { %w[make] }
 
           before do
             stub_rails_env('test')
           end
 
-          it 'calls make in the gitaly directory with --no-deployment flag for bundle' do
-            expect(Gitlab::Popen).to receive(:popen).with(command, nil, { "BUNDLE_GEMFILE" => nil, "RUBYOPT" => nil }).and_return(true)
+          it 'calls make in the gitaly directory with BUNDLE_DEPLOYMENT and GEM_HOME variables' do
+            expect(Gitlab::Popen).to receive(:popen).with(command, nil, { "BUNDLE_GEMFILE" => nil, "RUBYOPT" => nil, "BUNDLE_DEPLOYMENT" => 'false', "GEM_HOME" => Bundler.bundle_path.to_s }).and_return(true)
 
             subject
           end
diff --git a/spec/tasks/gitlab/pages_rake_spec.rb b/spec/tasks/gitlab/pages_rake_spec.rb
index 08194f4d1c9..664899c361b 100644
--- a/spec/tasks/gitlab/pages_rake_spec.rb
+++ b/spec/tasks/gitlab/pages_rake_spec.rb
@@ -12,10 +12,9 @@ RSpec.describe 'gitlab:pages' do
 
     it 'calls migration service' do
       expect_next_instance_of(::Pages::MigrateFromLegacyStorageService, anything,
-                              migration_threads: 3,
-                              batch_size: 10,
-                              ignore_invalid_entries: false) do |service|
-        expect(service).to receive(:execute).and_call_original
+                              ignore_invalid_entries: false,
+                              mark_projects_as_not_deployed: false) do |service|
+        expect(service).to receive(:execute_with_threads).with(threads: 3, batch_size: 10).and_call_original
       end
 
       subject
@@ -25,10 +24,9 @@ RSpec.describe 'gitlab:pages' do
       stub_env('PAGES_MIGRATION_THREADS', '5')
 
       expect_next_instance_of(::Pages::MigrateFromLegacyStorageService, anything,
-                              migration_threads: 5,
-                              batch_size: 10,
-                              ignore_invalid_entries: false) do |service|
-        expect(service).to receive(:execute).and_call_original
+                              ignore_invalid_entries: false,
+                              mark_projects_as_not_deployed: false) do |service|
+        expect(service).to receive(:execute_with_threads).with(threads: 5, batch_size: 10).and_call_original
       end
 
       subject
@@ -38,10 +36,9 @@ RSpec.describe 'gitlab:pages' do
       stub_env('PAGES_MIGRATION_BATCH_SIZE', '100')
 
       expect_next_instance_of(::Pages::MigrateFromLegacyStorageService, anything,
-                              migration_threads: 3,
-                              batch_size: 100,
-                              ignore_invalid_entries: false) do |service|
-        expect(service).to receive(:execute).and_call_original
+                              ignore_invalid_entries: false,
+                              mark_projects_as_not_deployed: false) do |service|
+        expect(service).to receive(:execute_with_threads).with(threads: 3, batch_size: 100).and_call_original
       end
 
       subject
@@ -51,10 +48,21 @@ RSpec.describe 'gitlab:pages' do
       stub_env('PAGES_MIGRATION_IGNORE_INVALID_ENTRIES', 'true')
 
       expect_next_instance_of(::Pages::MigrateFromLegacyStorageService, anything,
-                              migration_threads: 3,
-                              batch_size: 10,
-                              ignore_invalid_entries: true) do |service|
-        expect(service).to receive(:execute).and_call_original
+                              ignore_invalid_entries: true,
+                              mark_projects_as_not_deployed: false) do |service|
+        expect(service).to receive(:execute_with_threads).with(threads: 3, batch_size: 10).and_call_original
+      end
+
+      subject
+    end
+
+    it 'uses PAGES_MIGRATION_MARK_PROJECTS_AS_NOT_DEPLOYED environment variable' do
+      stub_env('PAGES_MIGRATION_MARK_PROJECTS_AS_NOT_DEPLOYED', 'true')
+
+      expect_next_instance_of(::Pages::MigrateFromLegacyStorageService, anything,
+                              ignore_invalid_entries: false,
+                              mark_projects_as_not_deployed: true) do |service|
+        expect(service).to receive(:execute_with_threads).with(threads: 3, batch_size: 10).and_call_original
       end
 
       subject
@@ -78,4 +86,80 @@ RSpec.describe 'gitlab:pages' do
       expect(PagesDeployment.find_by_id(migrated_deployment.id)).to be_nil
     end
   end
+
+  describe 'gitlab:pages:deployments:migrate_to_object_storage' do
+    subject { run_rake_task('gitlab:pages:deployments:migrate_to_object_storage') }
+
+    before do
+      stub_pages_object_storage(::Pages::DeploymentUploader, enabled: object_storage_enabled)
+    end
+
+    let!(:deployment) { create(:pages_deployment, file_store: store) }
+    let(:object_storage_enabled) { true }
+
+    context 'when local storage is used' do
+      let(:store) { ObjectStorage::Store::LOCAL }
+
+      context 'and remote storage is defined' do
+        it 'migrates file to remote storage' do
+          subject
+
+          expect(deployment.reload.file_store).to eq(ObjectStorage::Store::REMOTE)
+        end
+      end
+
+      context 'and remote storage is not defined' do
+        let(:object_storage_enabled) { false }
+
+        it 'fails to migrate to remote storage' do
+          subject
+
+          expect(deployment.reload.file_store).to eq(ObjectStorage::Store::LOCAL)
+        end
+      end
+    end
+
+    context 'when remote storage is used' do
+      let(:store) { ObjectStorage::Store::REMOTE }
+
+      it 'file stays on remote storage' do
+        subject
+
+        expect(deployment.reload.file_store).to eq(ObjectStorage::Store::REMOTE)
+      end
+    end
+  end
+
+  describe 'gitlab:pages:deployments:migrate_to_local' do
+    subject { run_rake_task('gitlab:pages:deployments:migrate_to_local') }
+
+    before do
+      stub_pages_object_storage(::Pages::DeploymentUploader, enabled: object_storage_enabled)
+    end
+
+    let!(:deployment) { create(:pages_deployment, file_store: store) }
+    let(:object_storage_enabled) { true }
+
+    context 'when remote storage is used' do
+      let(:store) { ObjectStorage::Store::REMOTE }
+
+      context 'and job has remote file store defined' do
+        it 'migrates file to local storage' do
+          subject
+
+          expect(deployment.reload.file_store).to eq(ObjectStorage::Store::LOCAL)
+        end
+      end
+    end
+
+    context 'when local storage is used' do
+      let(:store) { ObjectStorage::Store::LOCAL }
+
+      it 'file stays on local storage' do
+        subject
+
+        expect(deployment.reload.file_store).to eq(ObjectStorage::Store::LOCAL)
+      end
+    end
+  end
 end
diff --git a/spec/tasks/gitlab/usage_data_rake_spec.rb b/spec/tasks/gitlab/usage_data_rake_spec.rb
index 0ee6fbef53f..84269568b8b 100644
--- a/spec/tasks/gitlab/usage_data_rake_spec.rb
+++ b/spec/tasks/gitlab/usage_data_rake_spec.rb
@@ -3,22 +3,12 @@
 require 'rake_helper'
 
 RSpec.describe 'gitlab:usage data take tasks' do
+  include UsageDataHelpers
+
   before do
     Rake.application.rake_require 'tasks/gitlab/usage_data'
     # stub prometheus external http calls https://gitlab.com/gitlab-org/gitlab/-/issues/245277
-    stub_request(:get, %r{^http[s]?://::1:9090/-/ready})
-      .to_return(
-        status: 200,
-        body: [{}].to_json,
-        headers: { 'Content-Type' => 'application/json' }
-      )
-
-    stub_request(:get, %r{^http[s]?://::1:9090/api/v1/query\?query=.*})
-      .to_return(
-        status: 200,
-        body: [{}].to_json,
-        headers: { 'Content-Type' => 'application/json' }
-      )
+    stub_prometheus_queries
   end
 
   describe 'dump_sql_in_yaml' do