Add latest changes from gitlab-org/gitlab@14-4-stable-eev14.4.0-rc42

3 files changed, 175 insertions, 0 deletions

diff --git a/lib/gitlab/ci/trace/archive.rb b/lib/gitlab/ci/trace/archive.rb
new file mode 100644
index 00000000000..5047cf04562
--- /dev/null
+++ b/lib/gitlab/ci/trace/archive.rb
@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Trace
+      class Archive
+        include ::Gitlab::Utils::StrongMemoize
+        include Checksummable
+
+        def initialize(job, trace_metadata, metrics = ::Gitlab::Ci::Trace::Metrics.new)
+          @job = job
+          @trace_metadata = trace_metadata
+          @metrics = metrics
+        end
+
+        def execute!(stream)
+          clone_file!(stream, JobArtifactUploader.workhorse_upload_path) do |clone_path|
+            md5_checksum    = self.class.md5_hexdigest(clone_path)
+            sha256_checksum = self.class.sha256_hexdigest(clone_path)
+
+            job.transaction do
+              self.trace_artifact = create_build_trace!(clone_path, sha256_checksum)
+              trace_metadata.track_archival!(trace_artifact.id, md5_checksum)
+            end
+          end
+
+          validate_archived_trace
+        end
+
+        private
+
+        attr_reader :job, :trace_metadata, :metrics
+        attr_accessor :trace_artifact
+
+        def clone_file!(src_stream, temp_dir)
+          FileUtils.mkdir_p(temp_dir)
+          Dir.mktmpdir("tmp-trace-#{job.id}", temp_dir) do |dir_path|
+            temp_path = File.join(dir_path, "job.log")
+            FileUtils.touch(temp_path)
+            size = IO.copy_stream(src_stream, temp_path)
+            raise ::Gitlab::Ci::Trace::ArchiveError, 'Failed to copy stream' unless size == src_stream.size
+
+            yield(temp_path)
+          end
+        end
+
+        def create_build_trace!(path, file_sha256)
+          File.open(path) do |stream|
+            # TODO: Set `file_format: :raw` after we've cleaned up legacy traces migration
+            # https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/20307
+            job.create_job_artifacts_trace!(
+              project: job.project,
+              file_type: :trace,
+              file: stream,
+              file_sha256: file_sha256)
+          end
+        end
+
+        def validate_archived_trace
+          return unless remote_checksum
+
+          trace_metadata.update!(remote_checksum: remote_checksum)
+
+          unless trace_metadata.remote_checksum_valid?
+            metrics.increment_error_counter(type: :archive_invalid_checksum)
+          end
+        end
+
+        def remote_checksum
+          strong_memoize(:remote_checksum) do
+            ::Gitlab::Ci::Trace::RemoteChecksum.new(trace_artifact).md5_checksum
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/trace/metrics.rb b/lib/gitlab/ci/trace/metrics.rb
index fcd70634630..174a5f184ff 100644
--- a/lib/gitlab/ci/trace/metrics.rb
+++ b/lib/gitlab/ci/trace/metrics.rb
@@ -21,6 +21,12 @@ module Gitlab
           :corrupted  # malformed trace found after comparing CRC32 and size
         ].freeze
 
+        TRACE_ERROR_TYPES = [
+          :chunks_invalid_size,       # used to be :corrupted
+          :chunks_invalid_checksum,   # used to be :invalid
+          :archive_invalid_checksum   # malformed trace found into object store after comparing MD5
+        ].freeze
+
         def increment_trace_operation(operation: :unknown)
           unless OPERATIONS.include?(operation)
             raise ArgumentError, "unknown trace operation: #{operation}"
@@ -33,6 +39,14 @@ module Gitlab
           self.class.trace_bytes.increment({}, size.to_i)
         end
 
+        def increment_error_counter(type: :unknown)
+          unless TRACE_ERROR_TYPES.include?(type)
+            raise ArgumentError, "unknown error type: #{type}"
+          end
+
+          self.class.trace_errors_counter.increment(type: type)
+        end
+
         def observe_migration_duration(seconds)
           self.class.finalize_histogram.observe({}, seconds.to_f)
         end
@@ -65,6 +79,15 @@ module Gitlab
             ::Gitlab::Metrics.histogram(name, comment, labels, buckets)
           end
         end
+
+        def self.trace_errors_counter
+          strong_memoize(:trace_errors_counter) do
+            name = :gitlab_ci_build_trace_errors_total
+            comment = 'Total amount of different error types on a build trace'
+
+            Gitlab::Metrics.counter(name, comment)
+          end
+        end
       end
     end
   end
diff --git a/lib/gitlab/ci/trace/remote_checksum.rb b/lib/gitlab/ci/trace/remote_checksum.rb
new file mode 100644
index 00000000000..d57f3888ec0
--- /dev/null
+++ b/lib/gitlab/ci/trace/remote_checksum.rb
@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Trace
+      ##
+      # RemoteChecksum class is responsible for fetching the MD5 checksum of
+      # an uploaded build trace.
+      #
+      class RemoteChecksum
+        include Gitlab::Utils::StrongMemoize
+
+        def initialize(trace_artifact)
+          @trace_artifact = trace_artifact
+        end
+
+        def md5_checksum
+          strong_memoize(:md5_checksum) do
+            fetch_md5_checksum
+          end
+        end
+
+        private
+
+        attr_reader :trace_artifact
+        delegate :aws?, :google?, to: :object_store_config, prefix: :provider
+
+        def fetch_md5_checksum
+          return unless Feature.enabled?(:ci_archived_build_trace_checksum, trace_artifact.project, default_enabled: :yaml)
+          return unless object_store_config.enabled?
+          return if trace_artifact.local_store?
+
+          remote_checksum_value
+        end
+
+        def remote_checksum_value
+          strong_memoize(:remote_checksum_value) do
+            if provider_google?
+              checksum_from_google
+            elsif provider_aws?
+              checksum_from_aws
+            end
+          end
+        end
+
+        def object_store_config
+          strong_memoize(:object_store_config) do
+            trace_artifact.file.class.object_store_config
+          end
+        end
+
+        def checksum_from_google
+          content_md5 = upload_attributes.fetch(:content_md5)
+
+          Base64
+            .decode64(content_md5)
+            .unpack1('H*')
+        end
+
+        def checksum_from_aws
+          upload_attributes.fetch(:etag)
+        end
+
+        # Carrierwave caches attributes for the local file and does not replace
+        # them with the ones from object store after the upload completes.
+        # We need to force it to fetch them directly from the object store.
+        def upload_attributes
+          strong_memoize(:upload_attributes) do
+            ::Ci::JobArtifact.find(trace_artifact.id).file.file.attributes
+          end
+        end
+      end
+    end
+  end
+end