Add latest changes from gitlab-org/gitlab@master

2 files changed, 181 insertions, 0 deletions

diff --git a/lib/gitlab/background_migration/backfill_compliance_violations.rb b/lib/gitlab/background_migration/backfill_compliance_violations.rb
new file mode 100644
index 00000000000..131b4a05e41
--- /dev/null
+++ b/lib/gitlab/background_migration/backfill_compliance_violations.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # rubocop: disable Style/Documentation
+    class BackfillComplianceViolations < Gitlab::BackgroundMigration::BatchedMigrationJob
+      feature_category :compliance_management
+
+      def perform
+        # no-op. The logic is defined in EE module.
+      end
+    end
+    # rubocop: enable Style/Documentation
+  end
+end
+
+::Gitlab::BackgroundMigration::BackfillComplianceViolations.prepend_mod
diff --git a/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings.rb b/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings.rb
new file mode 100644
index 00000000000..7a2f4dab742
--- /dev/null
+++ b/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings.rb
@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+module Vulnerabilities
+  # The class is mimicking Vulnerabilites::Remediation
+  class Remediation < ApplicationRecord
+    include FileStoreMounter
+    include ShaAttribute
+
+    self.table_name = 'vulnerability_remediations'
+
+    sha_attribute :checksum
+
+    mount_file_store_uploader AttachmentUploader
+
+    def retrieve_upload(_identifier, paths)
+      Upload.find_by(model: self, path: paths)
+    end
+  end
+end
+
+module Gitlab
+  module BackgroundMigration
+    # The class to migrate the remediation data into their own records from the json attribute
+    class MigrateRemediationsForVulnerabilityFindings < BatchedMigrationJob
+      feature_category :vulnerability_management
+      operation_name :migrate_remediations_for_vulnerability_findings
+
+      # The class to encapsulate checksum and file for uploading
+      class DiffFile < StringIO
+        # This method is used by the `carrierwave` gem
+        def original_filename
+          @original_filename ||= self.class.original_filename(checksum)
+        end
+
+        def checksum
+          @checksum ||= self.class.checksum(string)
+        end
+
+        def self.checksum(value)
+          Digest::SHA256.hexdigest(value)
+        end
+
+        def self.original_filename(checksum)
+          "#{checksum}.diff"
+        end
+      end
+
+      # The class is mimicking Vulnerabilites::Finding
+      class Finding < ApplicationRecord
+        self.table_name = 'vulnerability_occurrences'
+
+        validates :details, json_schema: { filename: 'vulnerability_finding_details', draft: 7 }, if: false
+      end
+
+      # The class is mimicking Vulnerabilites::FindingRemediation
+      class FindingRemediation < ApplicationRecord
+        self.table_name = 'vulnerability_findings_remediations'
+      end
+
+      def perform
+        each_sub_batch do |sub_batch|
+          migrate_remediations(sub_batch)
+        end
+      end
+
+      private
+
+      def migrate_remediations(sub_batch)
+        sub_batch.each do |finding|
+          FindingRemediation.transaction do
+            remediations = append_remediations_diff_checksum(finding.raw_metadata)
+
+            result_ids = create_remediations(finding, remediations)
+
+            create_finding_remediations(finding.id, result_ids)
+          end
+        rescue StandardError => e
+          logger.error(
+            message: e.message,
+            class: self.class.name,
+            model_id: finding.id
+          )
+        end
+      end
+
+      def create_finding_remediations(finding_id, result_ids)
+        attrs = result_ids.map do |result_id|
+          build_finding_remediation_attrs(finding_id, result_id)
+        end
+
+        return unless attrs.present?
+
+        FindingRemediation.upsert_all(
+          attrs,
+          returning: false,
+          unique_by: [:vulnerability_occurrence_id, :vulnerability_remediation_id]
+        )
+      end
+
+      def create_remediations(finding, remediations)
+        attrs = remediations.map do |remediation|
+          build_remediation_attrs(finding, remediation)
+        end
+
+        return [] unless attrs.present?
+
+        ids_checksums = ::Vulnerabilities::Remediation.upsert_all(
+          attrs,
+          returning: %w[id checksum],
+          unique_by: [:project_id, :checksum]
+        )
+
+        ids_checksums.each do |id_checksum|
+          upload_file(id_checksum['id'], id_checksum['checksum'], remediations)
+        end
+
+        ids_checksums.pluck('id')
+      end
+
+      def upload_file(id, checksum, remediations)
+        deserialized_checksum = Gitlab::Database::ShaAttribute.new.deserialize(checksum)
+        diff = remediations.find { |rem| rem['checksum'] == deserialized_checksum }["diff"]
+        file = DiffFile.new(diff)
+        ::Vulnerabilities::Remediation.find_by(id: id).update!(file: file)
+      end
+
+      def build_remediation_attrs(finding, remediation)
+        {
+          project_id: finding.project_id,
+          summary: remediation['summary'],
+          file: DiffFile.original_filename(remediation['checksum']),
+          checksum: remediation['checksum'],
+          created_at: Time.current,
+          updated_at: Time.current
+        }
+      end
+
+      def build_finding_remediation_attrs(finding_id, remediation_id)
+        {
+          vulnerability_occurrence_id: finding_id,
+          vulnerability_remediation_id: remediation_id,
+          created_at: Time.current,
+          updated_at: Time.current
+        }
+      end
+
+      def append_remediations_diff_checksum(metadata)
+        parsed_metadata = Gitlab::Json.parse(metadata)
+
+        return [] unless parsed_metadata['remediations']
+
+        parsed_metadata['remediations'].filter_map do |remediation|
+          next unless remediation
+
+          remediation.merge('checksum' => DiffFile.checksum(remediation['diff']))
+        end.compact.uniq
+      end
+
+      def logger
+        @logger ||= ::Gitlab::AppLogger
+      end
+    end
+  end
+end