Extend reports to support security features

3 files changed, 75 insertions, 28 deletions

diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index 63aaa0f7bcc..3dadb95443a 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -68,8 +68,12 @@ module Ci
         '', Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id').archive)
     end
 
+    scope :with_existing_job_artifacts, ->(query) do
+      where('EXISTS (?)', ::Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id').merge(query))
+    end
+
     scope :with_archived_trace, ->() do
-      where('EXISTS (?)', Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id').trace)
+      with_existing_job_artifacts(Ci::JobArtifact.trace)
     end
 
     scope :without_archived_trace, ->() do
@@ -77,10 +81,12 @@ module Ci
     end
 
     scope :with_test_reports, ->() do
-      includes(:job_artifacts_junit) # Prevent N+1 problem when iterating each ci_job_artifact row
-        .where('EXISTS (?)', Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id').test_reports)
+      with_existing_job_artifacts(Ci::JobArtifact.test_reports)
+        .eager_load_job_artifacts
     end
 
+    scope :eager_load_job_artifacts, -> { includes(:job_artifacts) }
+
     scope :with_artifacts_stored_locally, -> { with_artifacts_archive.where(artifacts_file_store: [nil, LegacyArtifactUploader::Store::LOCAL]) }
     scope :with_archived_trace_stored_locally, -> { with_archived_trace.where(artifacts_file_store: [nil, LegacyArtifactUploader::Store::LOCAL]) }
     scope :with_artifacts_not_expired, ->() { with_artifacts_archive.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) }
@@ -404,8 +410,8 @@ module Ci
       trace.exist?
     end
 
-    def has_test_reports?
-      job_artifacts.test_reports.any?
+    def has_job_artifacts?
+      job_artifacts.any?
     end
 
     def has_old_trace?
@@ -469,28 +475,23 @@ module Ci
       end
     end
 
-    def erase_artifacts!
-      remove_artifacts_file!
-      remove_artifacts_metadata!
-      save
-    end
-
-    def erase_test_reports!
-      # TODO: Use fast_destroy_all in the context of https://gitlab.com/gitlab-org/gitlab-ce/issues/35240
-      job_artifacts_junit&.destroy
+    # and use that for `ExpireBuildInstanceArtifactsWorker`?
+    def erase_erasable_artifacts!
+      job_artifacts.erasable.destroy_all # rubocop: disable DestroyAll
+      erase_old_artifacts!
     end
 
     def erase(opts = {})
       return false unless erasable?
 
-      erase_artifacts!
-      erase_test_reports!
+      job_artifacts.destroy_all # rubocop: disable DestroyAll
+      erase_old_artifacts!
       erase_trace!
       update_erased!(opts[:erased_by])
     end
 
     def erasable?
-      complete? && (artifacts? || has_test_reports? || has_trace?)
+      complete? && (artifacts? || has_job_artifacts? || has_trace?)
     end
 
     def erased?
@@ -648,8 +649,8 @@ module Ci
 
     def collect_test_reports!(test_reports)
       test_reports.get_suite(group_name).tap do |test_suite|
-        each_test_report do |file_type, blob|
-          Gitlab::Ci::Parsers.fabricate!(file_type).parse!(blob, test_suite)
+        each_report(Ci::JobArtifact::TEST_REPORT_FILE_TYPES) do |file_type, blob|
+          Gitlab::Ci::Parsers::Test.fabricate!(file_type).parse!(blob, test_suite)
         end
       end
     end
@@ -669,6 +670,13 @@ module Ci
 
     private
 
+    def erase_old_artifacts!
+      # TODO: To be removed once we get rid of
+      remove_artifacts_file!
+      remove_artifacts_metadata!
+      save
+    end
+
     def successful_deployment_status
       if success? && last_deployment&.last?
         return :last
@@ -679,14 +687,19 @@ module Ci
       :creating
     end
 
-    def each_test_report
-      Ci::JobArtifact::TEST_REPORT_FILE_TYPES.each do |file_type|
-        public_send("job_artifacts_#{file_type}").each_blob do |blob| # rubocop:disable GitlabSecurity/PublicSend
-          yield file_type, blob
+    def each_report(report_types)
+      job_artifacts_for_types(report_types).each do |report_artifact|
+        report_artifact.each_blob do |blob|
+          yield report_artifact.file_type, blob
         end
       end
     end
 
+    def job_artifacts_for_types(report_types)
+      # Use select to leverage cached associations and avoid N+1 queries
+      job_artifacts.select { |artifact| artifact.file_type.in?(report_types) }
+    end
+
     def update_artifacts_size
       self.artifacts_size = legacy_artifacts_file&.size
     end
diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb
index 93fc1b145b2..259d85e2fe5 100644
--- a/app/models/ci/job_artifact.rb
+++ b/app/models/ci/job_artifact.rb
@@ -9,8 +9,28 @@ module Ci
     NotSupportedAdapterError = Class.new(StandardError)
 
     TEST_REPORT_FILE_TYPES = %w[junit].freeze
-    DEFAULT_FILE_NAMES = { junit: 'junit.xml' }.freeze
-    TYPE_AND_FORMAT_PAIRS = { archive: :zip, metadata: :gzip, trace: :raw, junit: :gzip }.freeze
+    NON_ERASABLE_FILE_TYPES = %w[trace].freeze
+    DEFAULT_FILE_NAMES = {
+      archive: nil,
+      metadata: nil,
+      trace: nil,
+      junit: 'junit.xml',
+      sast: 'gl-sast-report.json',
+      dependency_scanning: 'gl-dependency-scanning-report.json',
+      container_scanning: 'gl-container-scanning-report.json',
+      dast: 'gl-dast-report.json'
+    }.freeze
+
+    TYPE_AND_FORMAT_PAIRS = {
+      archive: :zip,
+      metadata: :gzip,
+      trace: :raw,
+      junit: :gzip,
+      sast: :gzip,
+      dependency_scanning: :gzip,
+      container_scanning: :gzip,
+      dast: :gzip
+    }.freeze
 
     belongs_to :project
     belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id
@@ -27,8 +47,18 @@ module Ci
 
     scope :with_files_stored_locally, -> { where(file_store: [nil, ::JobArtifactUploader::Store::LOCAL]) }
 
+    scope :with_file_types, -> (file_types) do
+      types = self.file_types.select { |file_type| file_types.include?(file_type) }.values
+
+      where(file_type: types)
+    end
+
     scope :test_reports, -> do
-      types = self.file_types.select { |file_type| TEST_REPORT_FILE_TYPES.include?(file_type) }.values
+      with_file_types(TEST_REPORT_FILE_TYPES)
+    end
+
+    scope :erasable, -> do
+      types = self.file_types.reject { |file_type| NON_ERASABLE_FILE_TYPES.include?(file_type) }.values
 
       where(file_type: types)
     end
@@ -39,7 +69,11 @@ module Ci
       archive: 1,
       metadata: 2,
       trace: 3,
-      junit: 4
+      junit: 4,
+      sast: 5, ## EE-specific
+      dependency_scanning: 6, ## EE-specific
+      container_scanning: 7, ## EE-specific
+      dast: 8 ## EE-specific
     }
 
     enum file_format: {
diff --git a/app/workers/expire_build_instance_artifacts_worker.rb b/app/workers/expire_build_instance_artifacts_worker.rb
index 4fcd1e5bd24..94426dcf921 100644
--- a/app/workers/expire_build_instance_artifacts_worker.rb
+++ b/app/workers/expire_build_instance_artifacts_worker.rb
@@ -13,7 +13,7 @@ class ExpireBuildInstanceArtifactsWorker
     return unless build&.project && !build.project.pending_delete
 
     Rails.logger.info "Removing artifacts for build #{build.id}..."
-    build.erase_artifacts!
+    build.erase_erasable_artifacts!
   end
   # rubocop: enable CodeReuse/ActiveRecord
 end