Add latest changes from gitlab-org/gitlab@13-9-stable-eev13.9.0-rc42

6 files changed, 227 insertions, 63 deletions

diff --git a/spec/tasks/gitlab/cleanup_rake_spec.rb b/spec/tasks/gitlab/cleanup_rake_spec.rb
index dc460611169..08d8651dcef 100644
--- a/spec/tasks/gitlab/cleanup_rake_spec.rb
+++ b/spec/tasks/gitlab/cleanup_rake_spec.rb
@@ -109,8 +109,7 @@ RSpec.describe 'gitlab:cleanup rake tasks' do
       it 'passes dry_run correctly' do
         expect(Gitlab::Cleanup::OrphanJobArtifactFiles)
           .to receive(:new)
-          .with(limit: anything,
-                dry_run: false,
+          .with(dry_run: false,
                 niceness: anything,
                 logger: anything)
           .and_call_original
@@ -145,7 +144,6 @@ RSpec.describe 'gitlab:cleanup rake tasks' do
         expect(Gitlab::Cleanup::OrphanLfsFileReferences)
           .to receive(:new)
           .with(project,
-                limit: anything,
                 dry_run: false,
                 logger: anything)
           .and_call_original
@@ -153,24 +151,6 @@ RSpec.describe 'gitlab:cleanup rake tasks' do
         rake_task
       end
     end
-
-    context 'with LIMIT set to 100' do
-      before do
-        stub_env('LIMIT', '100')
-      end
-
-      it 'passes limit as integer' do
-        expect(Gitlab::Cleanup::OrphanLfsFileReferences)
-          .to receive(:new)
-          .with(project,
-                limit: 100,
-                dry_run: true,
-                logger: anything)
-          .and_call_original
-
-        rake_task
-      end
-    end
   end
 
   describe 'gitlab:cleanup:orphan_lfs_files' do
diff --git a/spec/tasks/gitlab/db_rake_spec.rb b/spec/tasks/gitlab/db_rake_spec.rb
index edfdb44022b..d1f4a12d8fc 100644
--- a/spec/tasks/gitlab/db_rake_spec.rb
+++ b/spec/tasks/gitlab/db_rake_spec.rb
@@ -297,6 +297,57 @@ RSpec.describe 'gitlab:db namespace rake task' do
     end
   end
 
+  describe '#migrate_with_instrumentation' do
+    subject { run_rake_task('gitlab:db:migration_testing', "[#{filename}]") }
+
+    let(:ctx) { double('ctx', migrations: all_migrations, schema_migration: double, get_all_versions: existing_versions) }
+    let(:instrumentation) { instance_double(Gitlab::Database::Migrations::Instrumentation, observations: observations) }
+    let(:existing_versions) { [1] }
+    let(:all_migrations) { [double('migration1', version: 1), pending_migration] }
+    let(:pending_migration) { double('migration2', version: 2) }
+    let(:filename) { 'results-file.json'}
+    let(:buffer) { StringIO.new }
+    let(:observations) { %w[some data] }
+
+    before do
+      allow(ActiveRecord::Base.connection).to receive(:migration_context).and_return(ctx)
+      allow(Gitlab::Database::Migrations::Instrumentation).to receive(:new).and_return(instrumentation)
+      allow(ActiveRecord::Migrator).to receive_message_chain('new.run').with(any_args).with(no_args)
+
+      allow(instrumentation).to receive(:observe).and_yield
+
+      allow(File).to receive(:open).with(filename, 'wb+').and_yield(buffer)
+    end
+
+    it 'fails when given no filename argument' do
+      expect { run_rake_task('gitlab:db:migration_testing') }.to raise_error(/specify result_file/)
+    end
+
+    it 'fails when the given file already exists' do
+      expect(File).to receive(:exist?).with(filename).and_return(true)
+
+      expect { subject }.to raise_error(/File exists/)
+    end
+
+    it 'instruments the pending migration' do
+      expect(instrumentation).to receive(:observe).with(2).and_yield
+
+      subject
+    end
+
+    it 'executes the pending migration' do
+      expect(ActiveRecord::Migrator).to receive_message_chain('new.run').with(:up, ctx.migrations, ctx.schema_migration, pending_migration.version).with(no_args)
+
+      subject
+    end
+
+    it 'writes observations out to JSON file' do
+      subject
+
+      expect(buffer.string).to eq(observations.to_json)
+    end
+  end
+
   def run_rake_task(task_name, arguments = '')
     Rake::Task[task_name].reenable
     Rake.application.invoke_task("#{task_name}#{arguments}")
diff --git a/spec/tasks/gitlab/git_rake_spec.rb b/spec/tasks/gitlab/git_rake_spec.rb
index 4d89b126c9e..50ec2632d83 100644
--- a/spec/tasks/gitlab/git_rake_spec.rb
+++ b/spec/tasks/gitlab/git_rake_spec.rb
@@ -9,8 +9,6 @@ RSpec.describe 'gitlab:git rake tasks' do
   before do
     Rake.application.rake_require 'tasks/gitlab/git'
 
-    allow_any_instance_of(String).to receive(:color) { |string, _color| string }
-
     stub_warn_user_is_not_gitlab
   end
 
diff --git a/spec/tasks/gitlab/pages_rake_spec.rb b/spec/tasks/gitlab/pages_rake_spec.rb
index 76808f52890..08194f4d1c9 100644
--- a/spec/tasks/gitlab/pages_rake_spec.rb
+++ b/spec/tasks/gitlab/pages_rake_spec.rb
@@ -2,66 +2,80 @@
 
 require 'rake_helper'
 
-RSpec.describe 'gitlab:pages:migrate_legacy_storagerake task' do
+RSpec.describe 'gitlab:pages' do
   before(:context) do
     Rake.application.rake_require 'tasks/gitlab/pages'
   end
 
-  subject { run_rake_task('gitlab:pages:migrate_legacy_storage') }
+  describe 'migrate_legacy_storage task' do
+    subject { run_rake_task('gitlab:pages:migrate_legacy_storage') }
 
-  let(:project) { create(:project) }
+    it 'calls migration service' do
+      expect_next_instance_of(::Pages::MigrateFromLegacyStorageService, anything,
+                              migration_threads: 3,
+                              batch_size: 10,
+                              ignore_invalid_entries: false) do |service|
+        expect(service).to receive(:execute).and_call_original
+      end
 
-  it 'does not try to migrate pages if pages are not deployed' do
-    expect(::Pages::MigrateLegacyStorageToDeploymentService).not_to receive(:new)
+      subject
+    end
 
-    subject
-  end
+    it 'uses PAGES_MIGRATION_THREADS environment variable' do
+      stub_env('PAGES_MIGRATION_THREADS', '5')
 
-  context 'when pages are marked as deployed' do
-    before do
-      project.mark_pages_as_deployed
+      expect_next_instance_of(::Pages::MigrateFromLegacyStorageService, anything,
+                              migration_threads: 5,
+                              batch_size: 10,
+                              ignore_invalid_entries: false) do |service|
+        expect(service).to receive(:execute).and_call_original
+      end
+
+      subject
     end
 
-    context 'when pages directory does not exist' do
-      it 'tries to migrate the project, but does not crash' do
-        expect_next_instance_of(::Pages::MigrateLegacyStorageToDeploymentService, project) do |service|
-          expect(service).to receive(:execute).and_call_original
-        end
+    it 'uses PAGES_MIGRATION_BATCH_SIZE environment variable' do
+      stub_env('PAGES_MIGRATION_BATCH_SIZE', '100')
 
-        subject
+      expect_next_instance_of(::Pages::MigrateFromLegacyStorageService, anything,
+                              migration_threads: 3,
+                              batch_size: 100,
+                              ignore_invalid_entries: false) do |service|
+        expect(service).to receive(:execute).and_call_original
       end
+
+      subject
     end
 
-    context 'when pages directory exists on disk' do
-      before do
-        FileUtils.mkdir_p File.join(project.pages_path, "public")
-        File.open(File.join(project.pages_path, "public/index.html"), "w") do |f|
-          f.write("Hello!")
-        end
+    it 'uses PAGES_MIGRATION_IGNORE_INVALID_ENTRIES environment variable' do
+      stub_env('PAGES_MIGRATION_IGNORE_INVALID_ENTRIES', 'true')
+
+      expect_next_instance_of(::Pages::MigrateFromLegacyStorageService, anything,
+                              migration_threads: 3,
+                              batch_size: 10,
+                              ignore_invalid_entries: true) do |service|
+        expect(service).to receive(:execute).and_call_original
       end
 
-      it 'migrates pages projects without deployments' do
-        expect_next_instance_of(::Pages::MigrateLegacyStorageToDeploymentService, project) do |service|
-          expect(service).to receive(:execute).and_call_original
-        end
+      subject
+    end
+  end
+
+  describe 'clean_migrated_zip_storage task' do
+    it 'removes only migrated deployments' do
+      regular_deployment = create(:pages_deployment)
+      migrated_deployment = create(:pages_deployment, :migrated)
 
-        expect do
-          subject
-        end.to change { project.pages_metadatum.reload.pages_deployment }.from(nil)
-      end
+      regular_deployment.project.update_pages_deployment!(regular_deployment)
+      migrated_deployment.project.update_pages_deployment!(migrated_deployment)
 
-      context 'when deployed already exists for the project' do
-        before do
-          deployment = create(:pages_deployment, project: project)
-          project.set_first_pages_deployment!(deployment)
-        end
+      expect(PagesDeployment.all).to contain_exactly(regular_deployment, migrated_deployment)
 
-        it 'does not try to migrate project' do
-          expect(::Pages::MigrateLegacyStorageToDeploymentService).not_to receive(:new)
+      run_rake_task('gitlab:pages:clean_migrated_zip_storage')
 
-          subject
-        end
-      end
+      expect(PagesDeployment.all).to contain_exactly(regular_deployment)
+      expect(PagesDeployment.find_by_id(regular_deployment.id)).not_to be_nil
+      expect(PagesDeployment.find_by_id(migrated_deployment.id)).to be_nil
     end
   end
 end
diff --git a/spec/tasks/gitlab/password_rake_spec.rb b/spec/tasks/gitlab/password_rake_spec.rb
new file mode 100644
index 00000000000..d5320f3b4af
--- /dev/null
+++ b/spec/tasks/gitlab/password_rake_spec.rb
@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require 'rake_helper'
+
+RSpec.describe 'gitlab:password rake tasks' do
+  let_it_be(:user_1) { create(:user, username: 'foobar', password: 'initial_password') }
+
+  def stub_username(username)
+    allow(Gitlab::TaskHelpers).to receive(:prompt).with('Enter username: ').and_return(username)
+  end
+
+  def stub_password(password, confirmation = nil)
+    confirmation ||= password
+    allow(Gitlab::TaskHelpers).to receive(:prompt_for_password).and_return(password)
+    allow(Gitlab::TaskHelpers).to receive(:prompt_for_password).with('Confirm password: ').and_return(confirmation)
+  end
+
+  before do
+    Rake.application.rake_require 'tasks/gitlab/password'
+
+    stub_username('foobar')
+    stub_password('secretpassword')
+  end
+
+  describe ':reset' do
+    context 'when all inputs are correct' do
+      it 'updates the password properly' do
+        run_rake_task('gitlab:password:reset', user_1.username)
+        expect(user_1.reload.valid_password?('secretpassword')).to eq(true)
+      end
+    end
+
+    context 'when username is not provided' do
+      it 'asks for username' do
+        expect(Gitlab::TaskHelpers).to receive(:prompt).with('Enter username: ')
+
+        run_rake_task('gitlab:password:reset')
+      end
+
+      context 'when username is empty' do
+        it 'aborts with an error' do
+          stub_username('')
+          expect { run_rake_task('gitlab:password:reset') }.to raise_error(/Username can not be empty./)
+        end
+      end
+    end
+
+    context 'when username is passed as argument' do
+      it 'does not ask for username' do
+        expect(Gitlab::TaskHelpers).not_to receive(:prompt)
+
+        run_rake_task('gitlab:password:reset', 'foobar')
+      end
+    end
+
+    context 'when passwords do not match' do
+      before do
+        stub_password('randompassword', 'differentpassword')
+      end
+
+      it 'aborts with an error' do
+        expect { run_rake_task('gitlab:password:reset') }.to raise_error(%r{Unable to change password of the user with username foobar.\nPassword confirmation doesn't match Password})
+      end
+    end
+
+    context 'when user cannot be found' do
+      before do
+        stub_username('nonexistentuser')
+      end
+
+      it 'aborts with an error' do
+        expect { run_rake_task('gitlab:password:reset') }.to raise_error(/Unable to find user with username nonexistentuser./)
+      end
+    end
+  end
+end
diff --git a/spec/tasks/gitlab/terraform/migrate_rake_spec.rb b/spec/tasks/gitlab/terraform/migrate_rake_spec.rb
new file mode 100644
index 00000000000..4188521df8e
--- /dev/null
+++ b/spec/tasks/gitlab/terraform/migrate_rake_spec.rb
@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require 'rake_helper'
+
+RSpec.describe 'gitlab:terraform_states' do
+  let_it_be(:version) { create(:terraform_state_version) }
+
+  let(:logger) { instance_double(Logger) }
+  let(:helper) { double }
+
+  before(:all) do
+    Rake.application.rake_require 'tasks/gitlab/terraform/migrate'
+  end
+
+  before do
+    allow(Logger).to receive(:new).with(STDOUT).and_return(logger)
+  end
+
+  describe 'gitlab:terraform_states:migrate' do
+    subject { run_rake_task('gitlab:terraform_states:migrate') }
+
+    it 'invokes the migration helper to move files to object storage' do
+      expect(Gitlab::Terraform::StateMigrationHelper).to receive(:migrate_to_remote_storage).and_yield(version)
+      expect(logger).to receive(:info).with('Starting transfer of Terraform states to object storage')
+      expect(logger).to receive(:info).with(/Transferred Terraform state version ID #{version.id}/)
+
+      subject
+    end
+
+    context 'an error is raised while migrating' do
+      let(:error_message) { 'Something went wrong' }
+
+      before do
+        allow(Gitlab::Terraform::StateMigrationHelper).to receive(:migrate_to_remote_storage).and_raise(StandardError, error_message)
+      end
+
+      it 'logs the error' do
+        expect(logger).to receive(:info).with('Starting transfer of Terraform states to object storage')
+        expect(logger).to receive(:error).with("Failed to migrate: #{error_message}")
+
+        subject
+      end
+    end
+  end
+end