Merge branch 'add-background-migration-for-legacy-traces' into 'master'

Migrate job artifacts data from `ci_builds` to `ci_job_artifacts` table (with Background migrations)

Closes #46652

See merge request gitlab-org/gitlab-ce!18615

2 files changed, 34 insertions, 3 deletions

diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb
index 17b7ee4f07e..32d7cb3424e 100644
--- a/app/models/ci/job_artifact.rb
+++ b/app/models/ci/job_artifact.rb
@@ -48,6 +48,20 @@ module Ci
       gzip: 3
     }
 
+    # `file_location` indicates where actual files are stored.
+    # Ideally, actual files should be stored in the same directory, and use the same
+    # convention to generate its path. However, sometimes we can't do so due to backward-compatibility.
+    #
+    # legacy_path ... The actual file is stored at a path consists of a timestamp
+    #                 and raw project/model IDs. Those rows were migrated from
+    #                 `ci_builds.artifacts_file` and `ci_builds.artifacts_metadata`
+    # hashed_path ... The actual file is stored at a path consists of a SHA2 based on the project ID.
+    #                 This is the default value.
+    enum file_location: {
+      legacy_path: 1,
+      hashed_path: 2
+    }
+
     FILE_FORMAT_ADAPTERS = {
       gzip: Gitlab::Ci::Build::Artifacts::GzipFileAdapter
     }.freeze
@@ -72,6 +86,10 @@ module Ci
       [nil, ::JobArtifactUploader::Store::LOCAL].include?(self.file_store)
     end
 
+    def hashed_path?
+      super || self.file_location.nil?
+    end
+
     def expire_in
       expire_at - Time.now if expire_at
     end
@@ -108,7 +126,7 @@ module Ci
     end
 
     def update_project_statistics_after_destroy
-      update_project_statistics(-self.size)
+      update_project_statistics(-self.size.to_i)
     end
 
     def update_project_statistics(difference)
diff --git a/app/uploaders/job_artifact_uploader.rb b/app/uploaders/job_artifact_uploader.rb
index f6af023e0f9..557b13a8bd6 100644
--- a/app/uploaders/job_artifact_uploader.rb
+++ b/app/uploaders/job_artifact_uploader.rb
@@ -5,6 +5,7 @@ class JobArtifactUploader < GitlabUploader
   include ObjectStorage::Concern
 
   ObjectNotReadyError = Class.new(StandardError)
+  UnknownFileLocationError = Class.new(StandardError)
 
   storage_options Gitlab.config.artifacts
 
@@ -23,10 +24,22 @@ class JobArtifactUploader < GitlabUploader
   def dynamic_segment
     raise ObjectNotReadyError, 'JobArtifact is not ready' unless model.id
 
-    creation_date = model.created_at.utc.strftime('%Y_%m_%d')
+    if model.hashed_path?
+      hashed_path
+    elsif model.legacy_path?
+      legacy_path
+    else
+      raise UnknownFileLocationError
+    end
+  end
 
+  def hashed_path
     File.join(disk_hash[0..1], disk_hash[2..3], disk_hash,
-              creation_date, model.job_id.to_s, model.id.to_s)
+      model.created_at.utc.strftime('%Y_%m_%d'), model.job_id.to_s, model.id.to_s)
+  end
+
+  def legacy_path
+    File.join(model.created_at.utc.strftime('%Y_%m'), model.project_id.to_s, model.job_id.to_s)
   end
 
   def disk_hash