Add latest changes from gitlab-org/gitlab@13-0-stable-ee

3 files changed, 265 insertions, 41 deletions

diff --git a/spec/lib/gitlab/background_migration/backfill_environment_id_deployment_merge_requests_spec.rb b/spec/lib/gitlab/background_migration/backfill_environment_id_deployment_merge_requests_spec.rb
new file mode 100644
index 00000000000..34ac70071bb
--- /dev/null
+++ b/spec/lib/gitlab/background_migration/backfill_environment_id_deployment_merge_requests_spec.rb
@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::BackgroundMigration::BackfillEnvironmentIdDeploymentMergeRequests, schema: 20200312134637 do
+  let(:environments)              { table(:environments) }
+  let(:merge_requests)            { table(:merge_requests) }
+  let(:deployments)               { table(:deployments) }
+  let(:deployment_merge_requests) { table(:deployment_merge_requests) }
+  let(:namespaces)                { table(:namespaces) }
+  let(:projects)                  { table(:projects) }
+
+  subject(:migration) { described_class.new }
+
+  it 'correctly backfills environment_id column' do
+    namespace = namespaces.create!(name: 'foo', path: 'foo')
+    project = projects.create!(namespace_id: namespace.id)
+
+    production = environments.create!(project_id: project.id, name: 'production', slug: 'production')
+    staging = environments.create!(project_id: project.id, name: 'staging', slug: 'staging')
+
+    mr = merge_requests.create!(source_branch: 'x', target_branch: 'master', target_project_id: project.id)
+
+    deployment1 = deployments.create!(environment_id: staging.id, iid: 1, project_id: project.id, ref: 'master', tag: false, sha: '123abcdef', status: 1)
+    deployment2 = deployments.create!(environment_id: production.id, iid: 2, project_id: project.id, ref: 'master', tag: false, sha: '123abcdef', status: 1)
+    deployment3 = deployments.create!(environment_id: production.id, iid: 3, project_id: project.id, ref: 'master', tag: false, sha: '123abcdef', status: 1)
+
+    # mr is tracked twice in production through deployment2 and deployment3
+    deployment_merge_requests.create!(deployment_id: deployment1.id, merge_request_id: mr.id)
+    deployment_merge_requests.create!(deployment_id: deployment2.id, merge_request_id: mr.id)
+    deployment_merge_requests.create!(deployment_id: deployment3.id, merge_request_id: mr.id)
+
+    expect(deployment_merge_requests.where(environment_id: nil).count).to eq(3)
+
+    migration.backfill_range(1, mr.id)
+
+    expect(deployment_merge_requests.where(environment_id: nil).count).to be_zero
+    expect(deployment_merge_requests.count).to eq(2)
+
+    production_deployments = deployment_merge_requests.where(environment_id: production.id)
+    expect(production_deployments.count).to eq(1)
+    expect(production_deployments.first.deployment_id).to eq(deployment2.id)
+
+    expect(deployment_merge_requests.where(environment_id: staging.id).count).to eq(1)
+  end
+end
diff --git a/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb b/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb
index 08d3b7bec6a..27ae60eb278 100644
--- a/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb
+++ b/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb
@@ -2,13 +2,31 @@
 
 require 'spec_helper'
 
-describe Gitlab::BackgroundMigration::BackfillSnippetRepositories, :migration, schema: 2020_02_26_162723 do
+describe Gitlab::BackgroundMigration::BackfillSnippetRepositories, :migration, schema: 2020_04_20_094444 do
   let(:gitlab_shell) { Gitlab::Shell.new }
   let(:users) { table(:users) }
   let(:snippets) { table(:snippets) }
   let(:snippet_repositories) { table(:snippet_repositories) }
 
-  let(:user) { users.create(id: 1, email: 'user@example.com', projects_limit: 10, username: 'test', name: 'Test') }
+  let(:user_state) { 'active' }
+  let(:ghost) { false }
+  let(:user_type) { nil }
+  let(:user_name) { 'Test' }
+
+  let!(:user) do
+    users.create(id: 1,
+                 email: 'user@example.com',
+                 projects_limit: 10,
+                 username: 'test',
+                 name: user_name,
+                 state: user_state,
+                 ghost: ghost,
+                 last_activity_on: 1.minute.ago,
+                 user_type: user_type,
+                 confirmed_at: 1.day.ago)
+  end
+
+  let(:migration_bot) { User.migration_bot }
   let!(:snippet_with_repo) { snippets.create(id: 1, type: 'PersonalSnippet', author_id: user.id, file_name: file_name, content: content) }
   let!(:snippet_with_empty_repo) { snippets.create(id: 2, type: 'PersonalSnippet', author_id: user.id, file_name: file_name, content: content) }
   let!(:snippet_without_repo) { snippets.create(id: 3, type: 'PersonalSnippet', author_id: user.id, file_name: file_name, content: content) }
@@ -53,15 +71,52 @@ describe Gitlab::BackgroundMigration::BackfillSnippetRepositories, :migration, s
       end
     end
 
-    shared_examples 'commits the file to the repository' do
+    shared_examples 'migration_bot user commits files' do
       it do
         subject
 
-        blob = blob_at(snippet, file_name)
+        last_commit = raw_repository(snippet).commit
 
-        aggregate_failures do
-          expect(blob).to be
-          expect(blob.data).to eq content
+        expect(last_commit.author_name).to eq migration_bot.name
+        expect(last_commit.author_email).to eq migration_bot.email
+      end
+    end
+
+    shared_examples 'commits the file to the repository' do
+      context 'when author can update snippet and use git' do
+        it 'creates the repository and commit the file' do
+          subject
+
+          blob = blob_at(snippet, file_name)
+          last_commit = raw_repository(snippet).commit
+
+          aggregate_failures do
+            expect(blob).to be
+            expect(blob.data).to eq content
+            expect(last_commit.author_name).to eq user.name
+            expect(last_commit.author_email).to eq user.email
+          end
+        end
+      end
+
+      context 'when author cannot update snippet or use git' do
+        context 'when user is blocked' do
+          let(:user_state) { 'blocked' }
+
+          it_behaves_like 'migration_bot user commits files'
+        end
+
+        context 'when user is deactivated' do
+          let(:user_state) { 'deactivated' }
+
+          it_behaves_like 'migration_bot user commits files'
+        end
+
+        context 'when user is a ghost' do
+          let(:ghost) { true }
+          let(:user_type) { 'ghost' }
+
+          it_behaves_like 'migration_bot user commits files'
         end
       end
     end
@@ -123,6 +178,124 @@ describe Gitlab::BackgroundMigration::BackfillSnippetRepositories, :migration, s
         end
       end
     end
+
+    context 'with invalid file names' do
+      using RSpec::Parameterized::TableSyntax
+
+      where(:invalid_file_name, :converted_file_name) do
+        'filename.js // with comment'       | 'filename-js-with-comment'
+        '.git/hooks/pre-commit'             | 'git-hooks-pre-commit'
+        'https://gitlab.com'                | 'https-gitlab-com'
+        'html://web.title%mp4/mpg/mpeg.net' | 'html-web-title-mp4-mpg-mpeg-net'
+        '../../etc/passwd'                  | 'etc-passwd'
+        '.'                                 | 'snippetfile1.txt'
+      end
+
+      with_them do
+        let!(:snippet_with_invalid_path) { snippets.create(id: 4, type: 'PersonalSnippet', author_id: user.id, file_name: invalid_file_name, content: content) }
+        let!(:snippet_with_valid_path) { snippets.create(id: 5, type: 'PersonalSnippet', author_id: user.id, file_name: file_name, content: content) }
+        let(:ids) { [4, 5] }
+
+        after do
+          raw_repository(snippet_with_invalid_path).remove
+          raw_repository(snippet_with_valid_path).remove
+        end
+
+        it 'checks for file path errors when errors are raised' do
+          expect(service).to receive(:set_file_path_error).once.and_call_original
+
+          subject
+        end
+
+        it 'converts invalid filenames' do
+          subject
+
+          expect(blob_at(snippet_with_invalid_path, converted_file_name)).to be
+        end
+
+        it 'does not convert valid filenames on subsequent migrations' do
+          subject
+
+          expect(blob_at(snippet_with_valid_path, file_name)).to be
+        end
+      end
+    end
+
+    context 'when snippet content size is higher than the existing limit' do
+      let(:limit) { 15 }
+      let(:content) { 'a' * (limit + 1) }
+      let(:snippet) { snippet_without_repo }
+      let(:ids) { [snippet.id, snippet.id] }
+
+      before do
+        allow(Gitlab::CurrentSettings).to receive(:snippet_size_limit).and_return(limit)
+      end
+
+      it_behaves_like 'migration_bot user commits files'
+    end
+
+    context 'when user name is invalid' do
+      let(:user_name) { '.' }
+      let!(:snippet) { snippets.create(id: 4, type: 'PersonalSnippet', author_id: user.id, file_name: file_name, content: content) }
+      let(:ids) { [4, 4] }
+
+      after do
+        raw_repository(snippet).remove
+      end
+
+      it_behaves_like 'migration_bot user commits files'
+    end
+
+    context 'when both user name and snippet file_name are invalid' do
+      let(:user_name) { '.' }
+      let!(:other_user) do
+        users.create(id: 2,
+                     email: 'user2@example.com',
+                     projects_limit: 10,
+                     username: 'test2',
+                     name: 'Test2',
+                     state: user_state,
+                     ghost: ghost,
+                     last_activity_on: 1.minute.ago,
+                     user_type: user_type,
+                     confirmed_at: 1.day.ago)
+      end
+      let!(:invalid_snippet) { snippets.create(id: 4, type: 'PersonalSnippet', author_id: user.id, file_name: '.', content: content) }
+      let!(:snippet) { snippets.create(id: 5, type: 'PersonalSnippet', author_id: other_user.id, file_name: file_name, content: content) }
+      let(:ids) { [4, 5] }
+
+      after do
+        raw_repository(snippet).remove
+        raw_repository(invalid_snippet).remove
+      end
+
+      it 'updates the file_name only when it is invalid' do
+        subject
+
+        expect(blob_at(invalid_snippet, 'snippetfile1.txt')).to be
+        expect(blob_at(snippet, file_name)).to be
+      end
+
+      it_behaves_like 'migration_bot user commits files' do
+        let(:snippet) { invalid_snippet }
+      end
+
+      it 'does not alter the commit author in subsequent migrations' do
+        subject
+
+        last_commit = raw_repository(snippet).commit
+
+        expect(last_commit.author_name).to eq other_user.name
+        expect(last_commit.author_email).to eq other_user.email
+      end
+
+      it "increases the number of retries temporarily from #{described_class::MAX_RETRIES} to #{described_class::MAX_RETRIES + 1}" do
+        expect(service).to receive(:create_commit).with(Snippet.find(invalid_snippet.id)).exactly(described_class::MAX_RETRIES + 1).times.and_call_original
+        expect(service).to receive(:create_commit).with(Snippet.find(snippet.id)).once.and_call_original
+
+        subject
+      end
+    end
   end
 
   def blob_at(snippet, path)
diff --git a/spec/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data_spec.rb b/spec/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data_spec.rb
index 7dae28f72a5..4411dca3fd9 100644
--- a/spec/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data_spec.rb
+++ b/spec/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data_spec.rb
@@ -4,40 +4,45 @@ require 'spec_helper'
 describe Gitlab::BackgroundMigration::MigrateIssueTrackersSensitiveData, schema: 20200130145430 do
   let(:services) { table(:services) }
 
-  # we need to define the classes due to encryption
-  class IssueTrackerData < ApplicationRecord
-    self.table_name = 'issue_tracker_data'
-
-    def self.encryption_options
-      {
-        key: Settings.attr_encrypted_db_key_base_32,
-        encode: true,
-        mode: :per_attribute_iv,
-        algorithm: 'aes-256-gcm'
-      }
+  before do
+    # we need to define the classes due to encryption
+    issue_tracker_data = Class.new(ApplicationRecord) do
+      self.table_name = 'issue_tracker_data'
+
+      def self.encryption_options
+        {
+          key: Settings.attr_encrypted_db_key_base_32,
+          encode: true,
+          mode: :per_attribute_iv,
+          algorithm: 'aes-256-gcm'
+        }
+      end
+
+      attr_encrypted :project_url, encryption_options
+      attr_encrypted :issues_url, encryption_options
+      attr_encrypted :new_issue_url, encryption_options
     end
 
-    attr_encrypted :project_url, encryption_options
-    attr_encrypted :issues_url, encryption_options
-    attr_encrypted :new_issue_url, encryption_options
-  end
+    jira_tracker_data = Class.new(ApplicationRecord) do
+      self.table_name = 'jira_tracker_data'
 
-  class JiraTrackerData < ApplicationRecord
-    self.table_name = 'jira_tracker_data'
+      def self.encryption_options
+        {
+          key: Settings.attr_encrypted_db_key_base_32,
+          encode: true,
+          mode: :per_attribute_iv,
+          algorithm: 'aes-256-gcm'
+        }
+      end
 
-    def self.encryption_options
-      {
-        key: Settings.attr_encrypted_db_key_base_32,
-        encode: true,
-        mode: :per_attribute_iv,
-        algorithm: 'aes-256-gcm'
-      }
+      attr_encrypted :url, encryption_options
+      attr_encrypted :api_url, encryption_options
+      attr_encrypted :username, encryption_options
+      attr_encrypted :password, encryption_options
     end
 
-    attr_encrypted :url, encryption_options
-    attr_encrypted :api_url, encryption_options
-    attr_encrypted :username, encryption_options
-    attr_encrypted :password, encryption_options
+    stub_const('IssueTrackerData', issue_tracker_data)
+    stub_const('JiraTrackerData', jira_tracker_data)
   end
 
   let(:url) { 'http://base-url.tracker.com' }
@@ -90,7 +95,7 @@ describe Gitlab::BackgroundMigration::MigrateIssueTrackersSensitiveData, schema:
     end
   end
 
-  context 'with jira service' do
+  context 'with Jira service' do
     let!(:service) do
       services.create(id: 10, type: 'JiraService', title: nil, properties: jira_properties.to_json, category: 'issue_tracker')
     end
@@ -202,7 +207,7 @@ describe Gitlab::BackgroundMigration::MigrateIssueTrackersSensitiveData, schema:
     end
   end
 
-  context 'with jira service which has data fields record inconsistent with properties field' do
+  context 'with Jira service which has data fields record inconsistent with properties field' do
     let!(:service) do
       services.create(id: 16, type: 'CustomIssueTrackerService', description: 'Existing description', properties: jira_properties.to_json, category: 'issue_tracker').tap do |service|
         JiraTrackerData.create!(service_id: service.id, url: 'http://other_jira_url')
@@ -241,7 +246,7 @@ describe Gitlab::BackgroundMigration::MigrateIssueTrackersSensitiveData, schema:
     end
   end
 
-  context 'jira service with empty properties' do
+  context 'Jira service with empty properties' do
     let!(:service) do
       services.create(id: 18, type: 'JiraService', properties: '', category: 'issue_tracker')
     end
@@ -253,7 +258,7 @@ describe Gitlab::BackgroundMigration::MigrateIssueTrackersSensitiveData, schema:
     end
   end
 
-  context 'jira service with nil properties' do
+  context 'Jira service with nil properties' do
     let!(:service) do
       services.create(id: 18, type: 'JiraService', properties: nil, category: 'issue_tracker')
     end
@@ -265,7 +270,7 @@ describe Gitlab::BackgroundMigration::MigrateIssueTrackersSensitiveData, schema:
     end
   end
 
-  context 'jira service with invalid properties' do
+  context 'Jira service with invalid properties' do
     let!(:service) do
       services.create(id: 18, type: 'JiraService', properties: 'invalid data', category: 'issue_tracker')
     end
@@ -277,7 +282,7 @@ describe Gitlab::BackgroundMigration::MigrateIssueTrackersSensitiveData, schema:
     end
   end
 
-  context 'with jira service with invalid properties, valid jira service and valid bugzilla service' do
+  context 'with Jira service with invalid properties, valid Jira service and valid bugzilla service' do
     let!(:jira_service_invalid) do
       services.create(id: 19, title: 'invalid - title', description: 'invalid - description', type: 'JiraService', properties: 'invalid data', category: 'issue_tracker')
     end