Add latest changes from gitlab-org/gitlab@13-5-stable-eev13.5.0-rc42

2 files changed, 113 insertions, 3 deletions

diff --git a/lib/gitlab/ci/trace/checksum.rb b/lib/gitlab/ci/trace/checksum.rb
new file mode 100644
index 00000000000..62532ef1cd2
--- /dev/null
+++ b/lib/gitlab/ci/trace/checksum.rb
@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Trace
+      ##
+      # Trace::Checksum class is responsible for calculating a CRC32 checksum
+      # of an entire build trace using partial build trace chunks stored in a
+      # database.
+      #
+      # CRC32 checksum can be easily calculated by combining partial checksums
+      # in a right order.
+      #
+      # Then we compare CRC32 checksum provided by a GitLab Runner and expect
+      # it to be the same as the CRC32 checksum derived from partial chunks.
+      #
+      class Checksum
+        include Gitlab::Utils::StrongMemoize
+
+        attr_reader :build
+
+        def initialize(build)
+          @build = build
+        end
+
+        def valid?
+          return false unless state_crc32.present?
+
+          state_crc32 == chunks_crc32
+        end
+
+        def state_crc32
+          strong_memoize(:state_crc32) { build.pending_state&.crc32 }
+        end
+
+        def chunks_crc32
+          strong_memoize(:chunks_crc32) do
+            trace_chunks.reduce(0) do |crc32, chunk|
+              Zlib.crc32_combine(crc32, chunk.crc32, chunk_size(chunk))
+            end
+          end
+        end
+
+        def last_chunk
+          strong_memoize(:last_chunk) { trace_chunks.max }
+        end
+
+        ##
+        # Trace chunks will be persisted in a database if an object store is
+        # not configured - in that case we do not want to load entire raw data
+        # of all the chunks into memory.
+        #
+        # We ignore `raw_data` attribute instead, and rely on internal build
+        # trace chunk database adapter to handle
+        # `ActiveModel::MissingAttributeError` exception.
+        #
+        # Alternative solution would be separating chunk data from chunk
+        # metadata on the database level too.
+        #
+        def trace_chunks
+          strong_memoize(:trace_chunks) do
+            build.trace_chunks.persisted
+              .select(::Ci::BuildTraceChunk.metadata_attributes)
+          end
+        end
+
+        def chunks_count
+          trace_chunks.to_a.size
+        end
+
+        private
+
+        def chunk_size(chunk)
+          if chunk == last_chunk
+            chunk.size
+          else
+            ::Ci::BuildTraceChunk::CHUNK_SIZE
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/trace/metrics.rb b/lib/gitlab/ci/trace/metrics.rb
index 82a7d5fb83c..097436d84ea 100644
--- a/lib/gitlab/ci/trace/metrics.rb
+++ b/lib/gitlab/ci/trace/metrics.rb
@@ -6,8 +6,20 @@ module Gitlab
       class Metrics
         extend Gitlab::Utils::StrongMemoize
 
-        OPERATIONS = [:appended, :streamed, :chunked, :mutated, :overwrite,
-                      :accepted, :finalized, :discarded, :conflict].freeze
+        OPERATIONS = [
+          :appended,  # new trace data has been written to a chunk
+          :streamed,  # new trace data has been sent by a runner
+          :chunked,   # new trace chunk has been created
+          :mutated,   # trace has been mutated when removing secrets
+          :overwrite, # runner requested overwritting a build trace
+          :accepted,  # scheduled chunks for migration and responded with 202
+          :finalized, # all live build trace chunks have been persisted
+          :discarded, # failed to persist live chunks before timeout
+          :conflict,  # runner has sent unrecognized build state details
+          :locked,    # build trace has been locked by a different mechanism
+          :stalled,   # failed to migrate chunk due to a worker duplication
+          :invalid    # malformed build trace has been detected using CRC32
+        ].freeze
 
         def increment_trace_operation(operation: :unknown)
           unless OPERATIONS.include?(operation)
@@ -18,7 +30,11 @@ module Gitlab
         end
 
         def increment_trace_bytes(size)
-          self.class.trace_bytes.increment(by: size.to_i)
+          self.class.trace_bytes.increment({}, size.to_i)
+        end
+
+        def observe_migration_duration(seconds)
+          self.class.finalize_histogram.observe({}, seconds.to_f)
         end
 
         def self.trace_operations
@@ -38,6 +54,17 @@ module Gitlab
             Gitlab::Metrics.counter(name, comment)
           end
         end
+
+        def self.finalize_histogram
+          strong_memoize(:finalize_histogram) do
+            name = :gitlab_ci_trace_finalize_duration_seconds
+            comment = 'Duration of build trace chunks migration to object storage'
+            buckets = [0.1, 0.5, 1.0, 2.0, 3.0, 5.0, 7.0, 10.0, 20.0, 30.0, 60.0, 300.0]
+            labels = {}
+
+            ::Gitlab::Metrics.histogram(name, comment, labels, buckets)
+          end
+        end
       end
     end
   end