Add latest changes from gitlab-org/gitlab@13-10-stable-eev13.10.0-rc40

10 files changed, 265 insertions, 74 deletions

diff --git a/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb b/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb
index 61eb3b332de..7484027a0fa 100644
--- a/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb
+++ b/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb
@@ -5,7 +5,7 @@ module Gitlab
     # Update existent project update_at column after their repository storage was moved
     class BackfillProjectUpdatedAtAfterRepositoryStorageMove
       def perform(*project_ids)
-        updated_repository_storages = ProjectRepositoryStorageMove.select("project_id, MAX(updated_at) as updated_at").where(project_id: project_ids).group(:project_id)
+        updated_repository_storages = Projects::RepositoryStorageMove.select("project_id, MAX(updated_at) as updated_at").where(project_id: project_ids).group(:project_id)
 
         Project.connection.execute <<-SQL
           WITH repository_storage_cte as (
diff --git a/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb
new file mode 100644
index 00000000000..80693728e86
--- /dev/null
+++ b/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    module BatchingStrategies
+      # Generic batching class for use with a BatchedBackgroundMigration.
+      # Batches over the given table and column combination, returning the MIN() and MAX()
+      # values for the next batch as an array.
+      #
+      # If no more batches exist in the table, returns nil.
+      class PrimaryKeyBatchingStrategy
+        include Gitlab::Database::DynamicModelHelpers
+
+        # Finds and returns the next batch in the table.
+        #
+        # table_name - The table to batch over
+        # column_name - The column to batch over
+        # batch_min_value - The minimum value which the next batch will start at
+        # batch_size - The size of the next batch
+        def next_batch(table_name, column_name, batch_min_value:, batch_size:)
+          model_class = define_batchable_model(table_name)
+
+          quoted_column_name = model_class.connection.quote_column_name(column_name)
+          relation = model_class.where("#{quoted_column_name} >= ?", batch_min_value)
+          next_batch_bounds = nil
+
+          relation.each_batch(of: batch_size, column: column_name) do |batch| # rubocop:disable Lint/UnreachableLoop
+            next_batch_bounds = batch.pluck(Arel.sql("MIN(#{quoted_column_name}), MAX(#{quoted_column_name})")).first
+
+            break
+          end
+
+          next_batch_bounds
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb b/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb
index 16c0de39a3b..60682bd2ec1 100644
--- a/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb
+++ b/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb
@@ -2,13 +2,11 @@
 
 module Gitlab
   module BackgroundMigration
-    # Background migration that extends CopyColumn to update the value of a
+    # Background migration that updates the value of a
     # column using the value of another column in the same table.
     #
     # - The {start_id, end_id} arguments are at the start so that it can be used
-    #   with `queue_background_migration_jobs_by_range_at_intervals`
-    # - Provides support for background job tracking through the use of
-    #   Gitlab::Database::BackgroundMigrationJob
+    #   with `queue_batched_background_migration`
     # - Uses sub-batching so that we can keep each update's execution time at
     #   low 100s ms, while being able to update more records per 2 minutes
     #   that we allow background migration jobs to be scheduled one after the other
@@ -22,28 +20,24 @@ module Gitlab
 
       # start_id - The start ID of the range of rows to update.
       # end_id - The end ID of the range of rows to update.
-      # table - The name of the table that contains the columns.
-      # primary_key - The primary key column of the table.
-      # copy_from - The column containing the data to copy.
-      # copy_to - The column to copy the data to.
+      # batch_table - The name of the table that contains the columns.
+      # batch_column - The name of the column we use to batch over the table.
       # sub_batch_size - We don't want updates to take more than ~100ms
       #                  This allows us to run multiple smaller batches during
       #                  the minimum 2.minute interval that we can schedule jobs
-      def perform(start_id, end_id, table, primary_key, copy_from, copy_to, sub_batch_size)
+      # copy_from - The column containing the data to copy.
+      # copy_to - The column to copy the data to.
+      def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, copy_from, copy_to)
         quoted_copy_from = connection.quote_column_name(copy_from)
         quoted_copy_to = connection.quote_column_name(copy_to)
 
-        parent_batch_relation = relation_scoped_to_range(table, primary_key, start_id, end_id)
+        parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id)
 
-        parent_batch_relation.each_batch(column: primary_key, of: sub_batch_size) do |sub_batch|
+        parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch|
           sub_batch.update_all("#{quoted_copy_to}=#{quoted_copy_from}")
 
           sleep(PAUSE_SECONDS)
         end
-
-        # We have to add all arguments when marking a job as succeeded as they
-        #  are all used to track the job by `queue_background_migration_jobs_by_range_at_intervals`
-        mark_job_as_succeeded(start_id, end_id, table, primary_key, copy_from, copy_to, sub_batch_size)
       end
 
       private
@@ -52,10 +46,6 @@ module Gitlab
         ActiveRecord::Base.connection
       end
 
-      def mark_job_as_succeeded(*arguments)
-        Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(self.class.name, arguments)
-      end
-
       def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id)
         define_batchable_model(source_table).where(source_key_column => start_id..stop_id)
       end
diff --git a/lib/gitlab/background_migration/merge_request_assignees_migration_progress_check.rb b/lib/gitlab/background_migration/merge_request_assignees_migration_progress_check.rb
deleted file mode 100644
index de0c357ab1c..00000000000
--- a/lib/gitlab/background_migration/merge_request_assignees_migration_progress_check.rb
+++ /dev/null
@@ -1,43 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module BackgroundMigration
-    # rubocop: disable Style/Documentation
-    class MergeRequestAssigneesMigrationProgressCheck
-      include Gitlab::Utils::StrongMemoize
-
-      RESCHEDULE_DELAY = 3.hours
-      WORKER = 'PopulateMergeRequestAssigneesTable'
-      DeadJobsError = Class.new(StandardError)
-
-      def perform
-        raise DeadJobsError, "Only dead background jobs in the queue for #{WORKER}" if !ongoing? && dead_jobs?
-
-        if ongoing?
-          BackgroundMigrationWorker.perform_in(RESCHEDULE_DELAY, self.class.name)
-        else
-          Feature.enable(:multiple_merge_request_assignees)
-        end
-      end
-
-      private
-
-      def dead_jobs?
-        strong_memoize(:dead_jobs) do
-          migration_klass.dead_jobs?(WORKER)
-        end
-      end
-
-      def ongoing?
-        strong_memoize(:ongoing) do
-          migration_klass.exists?(WORKER) || migration_klass.retrying_jobs?(WORKER)
-        end
-      end
-
-      def migration_klass
-        Gitlab::BackgroundMigration
-      end
-    end
-    # rubocop: enable Style/Documentation
-  end
-end
diff --git a/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb b/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb
new file mode 100644
index 00000000000..4eaef26c9c6
--- /dev/null
+++ b/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb
@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # This migration moves projects.container_registry_enabled values to
+    # project_features.container_registry_access_level for the projects within
+    # the given range of ids.
+    class MoveContainerRegistryEnabledToProjectFeature
+      MAX_BATCH_SIZE = 1_000
+
+      module Migratable
+        # Migration model namespace isolated from application code.
+        class ProjectFeature < ActiveRecord::Base
+          ENABLED = 20
+          DISABLED = 0
+        end
+      end
+
+      def perform(from_id, to_id)
+        (from_id..to_id).each_slice(MAX_BATCH_SIZE) do |batch|
+          process_batch(batch.first, batch.last)
+        end
+      end
+
+      private
+
+      def process_batch(from_id, to_id)
+        ActiveRecord::Base.connection.execute(update_sql(from_id, to_id))
+
+        logger.info(message: "#{self.class}: Copied container_registry_enabled values for projects with IDs between #{from_id}..#{to_id}")
+      end
+
+      # For projects that have a project_feature:
+      # Set project_features.container_registry_access_level to ENABLED (20) or DISABLED (0)
+      #   depending if container_registry_enabled is true or false.
+      def update_sql(from_id, to_id)
+        <<~SQL
+        UPDATE project_features
+        SET container_registry_access_level = (CASE p.container_registry_enabled
+                                              WHEN true THEN #{ProjectFeature::ENABLED}
+                                              WHEN false THEN #{ProjectFeature::DISABLED}
+                                              ELSE #{ProjectFeature::DISABLED}
+                                              END)
+        FROM projects p
+        WHERE project_id = p.id AND
+        project_id BETWEEN #{from_id} AND #{to_id}
+        SQL
+      end
+
+      def logger
+        @logger ||= Gitlab::BackgroundMigration::Logger.build
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/populate_finding_uuid_for_vulnerability_feedback.rb b/lib/gitlab/background_migration/populate_finding_uuid_for_vulnerability_feedback.rb
index 52b09e07fd5..dc31f995ae0 100644
--- a/lib/gitlab/background_migration/populate_finding_uuid_for_vulnerability_feedback.rb
+++ b/lib/gitlab/background_migration/populate_finding_uuid_for_vulnerability_feedback.rb
@@ -61,16 +61,12 @@ module Gitlab
         private
 
         def calculated_uuid
-          Gitlab::UUID.v5(uuid_components)
-        end
-
-        def uuid_components
-          [
-            category,
-            vulnerability_finding.primary_identifier.fingerprint,
-            vulnerability_finding.location_fingerprint,
-            project_id
-          ].join('-')
+          ::Security::VulnerabilityUUID.generate(
+            report_type: category,
+            primary_identifier_fingerprint: vulnerability_finding.primary_identifier.fingerprint,
+            location_fingerprint: vulnerability_finding.location_fingerprint,
+            project_id: project_id
+          )
         end
 
         def finding_key
diff --git a/lib/gitlab/background_migration/populate_namespace_statistics.rb b/lib/gitlab/background_migration/populate_namespace_statistics.rb
new file mode 100644
index 00000000000..e352ae71de6
--- /dev/null
+++ b/lib/gitlab/background_migration/populate_namespace_statistics.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # This class creates/updates those namespace statistics
+    # that haven't been created nor initialized.
+    # It also updates the related namespace statistics
+    # This is only required in EE
+    class PopulateNamespaceStatistics
+      def perform(group_ids, statistics)
+      end
+    end
+  end
+end
+
+Gitlab::BackgroundMigration::PopulateNamespaceStatistics.prepend_if_ee('EE::Gitlab::BackgroundMigration::PopulateNamespaceStatistics')
diff --git a/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb b/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb
index 3d3970f50e1..4aff9d1e2c1 100644
--- a/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb
+++ b/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb
@@ -10,7 +10,7 @@ module Gitlab
         NOP_RELATION.new
       end
 
-      def perform(_scan_ids); end
+      def perform(*_scan_ids); end
     end
   end
 end
diff --git a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb
new file mode 100644
index 00000000000..7b18e617c81
--- /dev/null
+++ b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb
@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+# rubocop: disable Style/Documentation
+class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid
+  # rubocop: disable Gitlab/NamespacedClass
+  class VulnerabilitiesIdentifier < ActiveRecord::Base
+    self.table_name = "vulnerability_identifiers"
+    has_many :primary_findings, class_name: 'VulnerabilitiesFinding', inverse_of: :primary_identifier, foreign_key: 'primary_identifier_id'
+  end
+
+  class VulnerabilitiesFinding < ActiveRecord::Base
+    self.table_name = "vulnerability_occurrences"
+    belongs_to :primary_identifier, class_name: 'VulnerabilitiesIdentifier', inverse_of: :primary_findings, foreign_key: 'primary_identifier_id'
+    REPORT_TYPES = {
+      sast: 0,
+      dependency_scanning: 1,
+      container_scanning: 2,
+      dast: 3,
+      secret_detection: 4,
+      coverage_fuzzing: 5,
+      api_fuzzing: 6
+    }.with_indifferent_access.freeze
+    enum report_type: REPORT_TYPES
+  end
+
+  class CalculateFindingUUID
+    FINDING_NAMESPACES_IDS = {
+      development: "a143e9e2-41b3-47bc-9a19-081d089229f4",
+      test: "a143e9e2-41b3-47bc-9a19-081d089229f4",
+      staging: "a6930898-a1b2-4365-ab18-12aa474d9b26",
+      production:  "58dc0f06-936c-43b3-93bb-71693f1b6570"
+    }.freeze
+
+    NAMESPACE_REGEX = /(\h{8})-(\h{4})-(\h{4})-(\h{4})-(\h{4})(\h{8})/.freeze
+    PACK_PATTERN = "NnnnnN".freeze
+
+    def self.call(value)
+      Digest::UUID.uuid_v5(namespace_id, value)
+    end
+
+    def self.namespace_id
+      namespace_uuid = FINDING_NAMESPACES_IDS.fetch(Rails.env.to_sym)
+      # Digest::UUID is broken when using an UUID in namespace_id
+      # https://github.com/rails/rails/issues/37681#issue-520718028
+      namespace_uuid.scan(NAMESPACE_REGEX).flatten.map { |s| s.to_i(16) }.pack(PACK_PATTERN)
+    end
+  end
+  # rubocop: enable Gitlab/NamespacedClass
+
+  def perform(start_id, end_id)
+    findings = VulnerabilitiesFinding
+      .joins(:primary_identifier)
+      .select(:id, :report_type, :fingerprint, :location_fingerprint, :project_id)
+      .where(id: start_id..end_id)
+
+    mappings = findings.each_with_object({}) do |finding, hash|
+      hash[finding] = { uuid: calculate_uuid_v5_for_finding(finding) }
+    end
+
+    ::Gitlab::Database::BulkUpdate.execute(%i[uuid], mappings)
+  end
+
+  private
+
+  def calculate_uuid_v5_for_finding(vulnerability_finding)
+    return unless vulnerability_finding
+
+    uuid_v5_name_components = {
+      report_type: vulnerability_finding.report_type,
+      primary_identifier_fingerprint: vulnerability_finding.fingerprint,
+      location_fingerprint: vulnerability_finding.location_fingerprint,
+      project_id: vulnerability_finding.project_id
+    }
+
+    name = uuid_v5_name_components.values.join('-')
+
+    CalculateFindingUUID.call(name)
+  end
+end
diff --git a/lib/gitlab/background_migration/set_default_iteration_cadences.rb b/lib/gitlab/background_migration/set_default_iteration_cadences.rb
new file mode 100644
index 00000000000..42f9d33ab71
--- /dev/null
+++ b/lib/gitlab/background_migration/set_default_iteration_cadences.rb
@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # rubocop:disable Style/Documentation
+    class SetDefaultIterationCadences
+      class Iteration < ApplicationRecord
+        self.table_name = 'sprints'
+      end
+
+      class IterationCadence < ApplicationRecord
+        self.table_name = 'iterations_cadences'
+
+        include BulkInsertSafe
+      end
+
+      class Group < ApplicationRecord
+        self.table_name = 'namespaces'
+
+        self.inheritance_column = :_type_disabled
+      end
+
+      def perform(*group_ids)
+        create_iterations_cadences(group_ids)
+        assign_iterations_cadences(group_ids)
+      end
+
+      private
+
+      def create_iterations_cadences(group_ids)
+        groups_with_cadence = IterationCadence.select(:group_id)
+
+        new_cadences = Group.where(id: group_ids).where.not(id: groups_with_cadence).map do |group|
+          last_iteration = Iteration.where(group_id: group.id).order(:start_date)&.last
+
+          next unless last_iteration
+
+          time = Time.now
+          IterationCadence.new(
+            group_id: group.id,
+            title: "#{group.name} Iterations",
+            start_date: last_iteration.start_date,
+            last_run_date: last_iteration.start_date,
+            automatic: false,
+            created_at: time,
+            updated_at: time
+          )
+        end
+
+        IterationCadence.bulk_insert!(new_cadences.compact, skip_duplicates: true)
+      end
+
+      def assign_iterations_cadences(group_ids)
+        IterationCadence.where(group_id: group_ids).each do |cadence|
+          Iteration.where(iterations_cadence_id: nil).where(group_id: cadence.group_id).update_all(iterations_cadence_id: cadence.id)
+        end
+      end
+    end
+  end
+end