Merge branch 'proper-fix-for-artifacts-service' into 'master'

Add archive feature to trace

Closes #43022 and gitlab-ee#4170

See merge request gitlab-org/gitlab-ce!17314

2 files changed, 70 insertions, 0 deletions

diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb
index f2e5124c8a8..c5356af2f02 100644
--- a/lib/gitlab/ci/trace.rb
+++ b/lib/gitlab/ci/trace.rb
@@ -1,6 +1,8 @@
 module Gitlab
   module Ci
     class Trace
+      ArchiveError = Class.new(StandardError)
+
       attr_reader :job
 
       delegate :old_trace, to: :job
@@ -93,8 +95,52 @@ module Gitlab
         job.erase_old_trace!
       end
 
+      def archive!
+        raise ArchiveError, 'Already archived' if trace_artifact
+        raise ArchiveError, 'Job is not finished yet' unless job.complete?
+
+        if current_path
+          File.open(current_path) do |stream|
+            archive_stream!(stream)
+            FileUtils.rm(current_path)
+          end
+        elsif old_trace
+          StringIO.new(old_trace, 'rb').tap do |stream|
+            archive_stream!(stream)
+            job.erase_old_trace!
+          end
+        end
+      end
+
       private
 
+      def archive_stream!(stream)
+        clone_file!(stream, JobArtifactUploader.workhorse_upload_path) do |clone_path|
+          create_job_trace!(job, clone_path)
+        end
+      end
+
+      def clone_file!(src_stream, temp_dir)
+        FileUtils.mkdir_p(temp_dir)
+        Dir.mktmpdir('tmp-trace', temp_dir) do |dir_path|
+          temp_path = File.join(dir_path, "job.log")
+          FileUtils.touch(temp_path)
+          size = IO.copy_stream(src_stream, temp_path)
+          raise ArchiveError, 'Failed to copy stream' unless size == src_stream.size
+
+          yield(temp_path)
+        end
+      end
+
+      def create_job_trace!(job, path)
+        File.open(path) do |stream|
+          job.create_job_artifacts_trace!(
+            project: job.project,
+            file_type: :trace,
+            file: stream)
+        end
+      end
+
       def ensure_path
         return current_path if current_path
 
diff --git a/lib/tasks/gitlab/traces.rake b/lib/tasks/gitlab/traces.rake
new file mode 100644
index 00000000000..fd2a4f2d11a
--- /dev/null
+++ b/lib/tasks/gitlab/traces.rake
@@ -0,0 +1,24 @@
+require 'logger'
+require 'resolv-replace'
+
+desc "GitLab | Archive legacy traces to trace artifacts"
+namespace :gitlab do
+  namespace :traces do
+    task archive: :environment do
+      logger = Logger.new(STDOUT)
+      logger.info('Archiving legacy traces')
+
+      Ci::Build.finished
+        .where('NOT EXISTS (?)',
+          Ci::JobArtifact.select(1).trace.where('ci_builds.id = ci_job_artifacts.job_id'))
+        .order(id: :asc)
+        .find_in_batches(batch_size: 1000) do |jobs|
+        job_ids = jobs.map { |job| [job.id] }
+
+        ArchiveTraceWorker.bulk_perform_async(job_ids)
+
+        logger.info("Scheduled #{job_ids.count} jobs. From #{job_ids.min} to #{job_ids.max}")
+      end
+    end
+  end
+end