Add latest changes from gitlab-org/gitlab@master

3 files changed, 76 insertions, 11 deletions

diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb
index f9798023838..d4dbf2ac297 100644
--- a/lib/gitlab/ci/trace.rb
+++ b/lib/gitlab/ci/trace.rb
@@ -4,6 +4,7 @@ module Gitlab
   module Ci
     class Trace
       include ::Gitlab::ExclusiveLeaseHelpers
+      include ::Gitlab::Utils::StrongMemoize
       include Checksummable
 
       LOCK_TTL = 10.minutes
@@ -23,6 +24,8 @@ module Gitlab
       attr_reader :job
 
       delegate :old_trace, to: :job
+      delegate :can_attempt_archival_now?, :increment_archival_attempts!,
+        :archival_attempts_message, to: :trace_metadata
 
       def initialize(job)
         @job = job
@@ -251,11 +254,19 @@ module Gitlab
         File.open(path) do |stream|
           # TODO: Set `file_format: :raw` after we've cleaned up legacy traces migration
           # https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/20307
-          job.create_job_artifacts_trace!(
+          trace_artifact = job.create_job_artifacts_trace!(
             project: job.project,
             file_type: :trace,
             file: stream,
             file_sha256: self.class.hexdigest(path))
+
+          trace_metadata.track_archival!(trace_artifact.id)
+        end
+      end
+
+      def trace_metadata
+        strong_memoize(:trace_metadata) do
+          job.ensure_trace_metadata!
         end
       end
 
diff --git a/lib/gitlab/ci/trace/backoff.rb b/lib/gitlab/ci/trace/backoff.rb
new file mode 100644
index 00000000000..c13d88cced1
--- /dev/null
+++ b/lib/gitlab/ci/trace/backoff.rb
@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Trace
+      ##
+      # Trace::Backoff class is responsible for calculating a backoff value
+      # for when to be able to retry archiving a build's trace
+      #
+      # Because we're updating `last_archival_attempt_at` timestamp with every
+      # failed archival attempt, we need to be sure that sum of the backoff values
+      # for 1..MAX_ATTEMPTS is under 7 days(CHUNK_REDIS_TTL).
+      #
+      class Backoff
+        include Gitlab::Utils::StrongMemoize
+
+        MAX_JITTER_VALUE = 4
+
+        attr_reader :archival_attempts
+
+        def initialize(archival_attempts)
+          @archival_attempts = archival_attempts
+        end
+
+        def value
+          (((chunks_ttl / (3.5 * max_attempts)) * archival_attempts) / 1.hour).hours
+        end
+
+        # This formula generates an increasing delay between executions
+        # 9.6, 19.2, 28.8, 38.4, 48.0 + a random amount of time to
+        # change the order of execution for the jobs.
+        # With maximum value for each call to rand(4), this sums up to 6.8 days
+        # and with minimum values is 6 days.
+        #
+        def value_with_jitter
+          value + jitter
+        end
+
+        private
+
+        def jitter
+          rand(MAX_JITTER_VALUE).hours
+        end
+
+        def chunks_ttl
+          ::Ci::BuildTraceChunks::RedisBase::CHUNK_REDIS_TTL
+        end
+
+        def max_attempts
+          ::Ci::BuildTraceMetadata::MAX_ATTEMPTS
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 76a6408da09..0e2cca507ba 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -360,18 +360,17 @@ module Gitlab
         end
       end
 
-      def new_blobs(newrev, dynamic_timeout: nil)
-        return [] if newrev.blank? || newrev == ::Gitlab::Git::BLANK_SHA
+      def new_blobs(newrevs, dynamic_timeout: nil)
+        newrevs = Array.wrap(newrevs).reject { |rev| rev.blank? || rev == ::Gitlab::Git::BLANK_SHA }
+        return [] if newrevs.empty?
 
-        strong_memoize("new_blobs_#{newrev}") do
-          if Feature.enabled?(:new_blobs_via_list_blobs)
-            blobs(['--not', '--all', '--not', newrev], with_paths: true, dynamic_timeout: dynamic_timeout)
-          else
-            wrapped_gitaly_errors do
-              gitaly_ref_client.list_new_blobs(newrev, REV_LIST_COMMIT_LIMIT, dynamic_timeout: dynamic_timeout)
-            end
-          end
+        newrevs = newrevs.uniq.sort
+
+        @new_blobs ||= Hash.new do |h, revs|
+          h[revs] = blobs(['--not', '--all', '--not'] + newrevs, with_paths: true, dynamic_timeout: dynamic_timeout)
         end
+
+        @new_blobs[newrevs]
       end
 
       # List blobs reachable via a set of revisions. Supports the