Add latest changes from gitlab-org/gitlab@master

9 files changed, 22 insertions, 335 deletions

diff --git a/lib/api/debian_project_packages.rb b/lib/api/debian_project_packages.rb
index 9dedc4390f7..03f0f97b805 100644
--- a/lib/api/debian_project_packages.rb
+++ b/lib/api/debian_project_packages.rb
@@ -81,11 +81,7 @@ module API
 
             package = ::Packages::Debian::FindOrCreateIncomingService.new(authorized_user_project, current_user).execute
 
-            package_file = ::Packages::Debian::CreatePackageFileService.new(package, file_params).execute
-
-            if params['file_name'].end_with? '.changes'
-              ::Packages::Debian::ProcessChangesWorker.perform_async(package_file.id, current_user.id) # rubocop:disable CodeReuse/Worker
-            end
+            ::Packages::Debian::CreatePackageFileService.new(package: package, current_user: current_user, params: file_params).execute
 
             created!
           rescue ObjectStorage::RemoteStoreError => e
diff --git a/lib/bulk_imports/network_error.rb b/lib/bulk_imports/network_error.rb
index 335d992d5a7..fda4bb74a30 100644
--- a/lib/bulk_imports/network_error.rb
+++ b/lib/bulk_imports/network_error.rb
@@ -2,7 +2,8 @@
 
 module BulkImports
   class NetworkError < Error
-    COUNTER_KEY = 'bulk_imports/%{entity_id}/%{stage}/%{tracker_id}/network_error/%{error}'
+    TRACKER_COUNTER_KEY = 'bulk_imports/%{entity_id}/%{stage}/%{tracker_id}/network_error/%{error}'
+    ENTITY_COUNTER_KEY = 'bulk_imports/%{entity_id}/network_error/%{error}'
 
     RETRIABLE_EXCEPTIONS = Gitlab::HTTP::HTTP_TIMEOUT_ERRORS + [
       EOFError, SocketError, OpenSSL::SSL::SSLError, OpenSSL::OpenSSLError,
@@ -24,9 +25,9 @@ module BulkImports
       @response = response
     end
 
-    def retriable?(tracker)
+    def retriable?(object)
       if retriable_exception? || retriable_http_code?
-        increment(tracker) <= MAX_RETRIABLE_COUNT
+        increment(object) <= MAX_RETRIABLE_COUNT
       else
         false
       end
@@ -50,15 +51,27 @@ module BulkImports
       RETRIABLE_HTTP_CODES.include?(response&.code)
     end
 
-    def increment(tracker)
-      key = COUNTER_KEY % {
+    def increment(object)
+      key = object.is_a?(BulkImports::Entity) ? entity_cache_key(object) : tracker_cache_key(object)
+
+      Gitlab::Cache::Import::Caching.increment(key)
+    end
+
+    def tracker_cache_key(tracker)
+      TRACKER_COUNTER_KEY % {
         stage: tracker.stage,
         tracker_id: tracker.id,
         entity_id: tracker.entity.id,
         error: cause.class.name
       }
+    end
 
-      Gitlab::Cache::Import::Caching.increment(key)
+    def entity_cache_key(entity)
+      ENTITY_COUNTER_KEY % {
+        import_id: entity.bulk_import_id,
+        entity_id: entity.id,
+        error: cause.class.name
+      }
     end
   end
 end
diff --git a/lib/feature/shared.rb b/lib/feature/shared.rb
index edfc39aea0c..6af24451322 100644
--- a/lib/feature/shared.rb
+++ b/lib/feature/shared.rb
@@ -53,8 +53,6 @@ module Feature
         default_enabled: false,
         example: <<-EOS
           experiment(:my_experiment, project: project, actor: current_user) { ...variant code... }
-          # or
-          Gitlab::Experimentation.in_experiment_group?(:my_experiment, subject: current_user)
         EOS
       }
     }.freeze
diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb
index ec628399785..138e79db331 100644
--- a/lib/gitlab/ci/config/external/context.rb
+++ b/lib/gitlab/ci/config/external/context.rb
@@ -10,7 +10,6 @@ module Gitlab
           TimeoutError = Class.new(StandardError)
 
           MAX_INCLUDES = 100
-          TRIAL_MAX_INCLUDES = 250
 
           include ::Gitlab::Utils::StrongMemoize
 
@@ -31,7 +30,7 @@ module Gitlab
             @expandset = Set.new
             @execution_deadline = 0
             @logger = logger || Gitlab::Ci::Pipeline::Logger.new(project: project)
-            @max_includes = Feature.enabled?(:ci_increase_includes_to_250, project) ? TRIAL_MAX_INCLUDES : MAX_INCLUDES
+            @max_includes = MAX_INCLUDES
             yield self if block_given?
           end
 
diff --git a/lib/gitlab/ci/pipeline/logger.rb b/lib/gitlab/ci/pipeline/logger.rb
index 44d905faced..4b7cbae5004 100644
--- a/lib/gitlab/ci/pipeline/logger.rb
+++ b/lib/gitlab/ci/pipeline/logger.rb
@@ -86,6 +86,7 @@ module Gitlab
               'count' => values.size,
               'min' => values.min,
               'max' => values.max,
+              'sum' => values.sum,
               'avg' => values.sum / values.size
             }
           end.compact
diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb
deleted file mode 100644
index 142d0e55593..00000000000
--- a/lib/gitlab/experimentation.rb
+++ /dev/null
@@ -1,110 +0,0 @@
-# frozen_string_literal: true
-
-# == Experimentation
-#
-# Utility module for A/B testing experimental features. Define your experiments in the `EXPERIMENTS` constant.
-# Experiment options:
-# - tracking_category (optional, used to set the category when tracking an experiment event)
-# - rollout_strategy: default is `:cookie` based rollout. We may also set it to `:user` based rollout
-#
-# The experiment is controlled by a Feature Flag (https://docs.gitlab.com/ee/development/feature_flags/controls.html),
-# which is named "#{experiment_key}_experiment_percentage" and *must* be set with a percentage and not be used for other purposes.
-#
-# To enable the experiment for 10% of the time:
-#
-# chatops: `/chatops run feature set experiment_key_experiment_percentage 10 --random`
-# console: `Feature.enable_percentage_of_time(:experiment_key_experiment_percentage, 10)`
-#
-# To disable the experiment:
-#
-# chatops: `/chatops run feature delete experiment_key_experiment_percentage`
-# console: `Feature.remove(:experiment_key_experiment_percentage)`
-#
-# To check the current rollout percentage:
-#
-# chatops: `/chatops run feature get experiment_key_experiment_percentage`
-# console: `Feature.get(:experiment_key_experiment_percentage).percentage_of_time_value`
-#
-
-# TODO: see https://gitlab.com/gitlab-org/gitlab/-/issues/217490
-module Gitlab
-  module Experimentation
-    EXPERIMENTS = {
-    }.freeze
-
-    class << self
-      def get_experiment(experiment_key)
-        return unless EXPERIMENTS.key?(experiment_key)
-
-        ::Gitlab::Experimentation::Experiment.new(experiment_key, **EXPERIMENTS[experiment_key])
-      end
-
-      def active?(experiment_key)
-        experiment = get_experiment(experiment_key)
-        return false unless experiment
-
-        experiment.active?
-      end
-
-      def in_experiment_group?(experiment_key, subject:)
-        return false if subject.blank?
-        return false unless active?(experiment_key)
-
-        log_invalid_rollout(experiment_key, subject)
-
-        experiment = get_experiment(experiment_key)
-        return false unless experiment
-
-        experiment.enabled_for_index?(index_for_subject(experiment, subject))
-      end
-
-      def rollout_strategy(experiment_key)
-        experiment = get_experiment(experiment_key)
-        return unless experiment
-
-        experiment.rollout_strategy
-      end
-
-      def log_invalid_rollout(experiment_key, subject)
-        return if valid_subject_for_rollout_strategy?(experiment_key, subject)
-
-        logger = Gitlab::ExperimentationLogger.build
-        logger.warn message: 'Subject must conform to the rollout strategy',
-                    experiment_key: experiment_key,
-                    subject: subject.class.to_s,
-                    rollout_strategy: rollout_strategy(experiment_key)
-      end
-
-      def valid_subject_for_rollout_strategy?(experiment_key, subject)
-        case rollout_strategy(experiment_key)
-        when :user
-          subject.is_a?(User)
-        when :group
-          subject.is_a?(Group)
-        when :cookie
-          subject.nil? || subject.is_a?(String)
-        else
-          false
-        end
-      end
-
-      private
-
-      def index_for_subject(experiment, subject)
-        index = Zlib.crc32("#{experiment.key}#{subject_id(subject)}")
-
-        index % 100
-      end
-
-      def subject_id(subject)
-        if subject.respond_to?(:to_global_id)
-          subject.to_global_id.to_s
-        elsif subject.respond_to?(:to_s)
-          subject.to_s
-        else
-          raise ArgumentError, 'Subject must respond to `to_global_id` or `to_s`'
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/experimentation/controller_concern.rb b/lib/gitlab/experimentation/controller_concern.rb
deleted file mode 100644
index b09d67b8d5f..00000000000
--- a/lib/gitlab/experimentation/controller_concern.rb
+++ /dev/null
@@ -1,156 +0,0 @@
-# frozen_string_literal: true
-
-require 'zlib'
-
-# Controller concern that checks if an `experimentation_subject_id cookie` is present and sets it if absent.
-# Used for A/B testing of experimental features. Exposes the `experiment_enabled?(experiment_name, subject: nil)` method
-# to controllers and views. It returns true when the experiment is enabled and the user is selected as part
-# of the experimental group.
-#
-module Gitlab
-  module Experimentation
-    module ControllerConcern
-      include ::Gitlab::Experimentation::GroupTypes
-      include Gitlab::Tracking::Helpers
-      extend ActiveSupport::Concern
-
-      included do
-        before_action :set_experimentation_subject_id_cookie, unless: :dnt_enabled?
-        helper_method :experiment_enabled?, :experiment_tracking_category_and_group, :record_experiment_group
-      end
-
-      def set_experimentation_subject_id_cookie
-        if Gitlab.com?
-          return if cookies[:experimentation_subject_id].present?
-
-          cookies.permanent.signed[:experimentation_subject_id] = {
-            value: SecureRandom.uuid,
-            secure: ::Gitlab.config.gitlab.https,
-            httponly: true
-          }
-        else
-          # We set the cookie before, although experiments are not conducted on self managed instances.
-          cookies.delete(:experimentation_subject_id)
-        end
-      end
-
-      def push_frontend_experiment(experiment_key, subject: nil)
-        var_name = experiment_key.to_s.camelize(:lower)
-
-        enabled = experiment_enabled?(experiment_key, subject: subject)
-
-        gon.push({ experiments: { var_name => enabled } }, true)
-      end
-
-      def experiment_enabled?(experiment_key, subject: nil)
-        return true if forced_enabled?(experiment_key)
-        return false if dnt_enabled?
-
-        Experimentation.log_invalid_rollout(experiment_key, subject)
-
-        subject ||= experimentation_subject_id
-
-        Experimentation.in_experiment_group?(experiment_key, subject: subject)
-      end
-
-      def track_experiment_event(experiment_key, action, value = nil, subject: nil)
-        return if dnt_enabled?
-
-        track_experiment_event_for(experiment_key, action, value, subject: subject) do |tracking_data|
-          ::Gitlab::Tracking.event(tracking_data.delete(:category), tracking_data.delete(:action), **tracking_data.merge!(user: current_user))
-        end
-      end
-
-      def frontend_experimentation_tracking_data(experiment_key, action, value = nil, subject: nil)
-        return if dnt_enabled?
-
-        track_experiment_event_for(experiment_key, action, value, subject: subject) do |tracking_data|
-          gon.push(tracking_data: tracking_data)
-        end
-      end
-
-      def record_experiment_user(experiment_key, context = {})
-        return if dnt_enabled?
-        return unless Experimentation.active?(experiment_key) && current_user
-
-        subject = Experimentation.rollout_strategy(experiment_key) == :cookie ? nil : current_user
-
-        ::Experiment.add_user(experiment_key, tracking_group(experiment_key, nil, subject: subject), current_user, context)
-      end
-
-      def record_experiment_group(experiment_key, group)
-        return if dnt_enabled?
-        return unless Experimentation.active?(experiment_key) && group
-
-        variant_subject = Experimentation.rollout_strategy(experiment_key) == :cookie ? nil : group
-        variant = tracking_group(experiment_key, nil, subject: variant_subject)
-
-        ::Experiment.add_group(experiment_key, group: group, variant: variant)
-      end
-
-      def record_experiment_conversion_event(experiment_key, context = {})
-        return if dnt_enabled?
-        return unless current_user
-        return unless Experimentation.active?(experiment_key)
-
-        ::Experiment.record_conversion_event(experiment_key, current_user, context)
-      end
-
-      def experiment_tracking_category_and_group(experiment_key, subject: nil)
-        "#{tracking_category(experiment_key)}:#{tracking_group(experiment_key, '_group', subject: subject)}"
-      end
-
-      private
-
-      def experimentation_subject_id
-        cookies.signed[:experimentation_subject_id]
-      end
-
-      def track_experiment_event_for(experiment_key, action, value, subject: nil)
-        return unless Experimentation.active?(experiment_key)
-
-        yield experimentation_tracking_data(experiment_key, action, value, subject: subject)
-      end
-
-      def experimentation_tracking_data(experiment_key, action, value, subject: nil)
-        {
-          category: tracking_category(experiment_key),
-          action: action,
-          property: tracking_group(experiment_key, "_group", subject: subject),
-          label: tracking_label(subject),
-          value: value
-        }.compact
-      end
-
-      def tracking_category(experiment_key)
-        Experimentation.get_experiment(experiment_key).tracking_category
-      end
-
-      def tracking_group(experiment_key, suffix = nil, subject: nil)
-        return unless Experimentation.active?(experiment_key)
-
-        subject ||= experimentation_subject_id
-        group = experiment_enabled?(experiment_key, subject: subject) ? GROUP_EXPERIMENTAL : GROUP_CONTROL
-
-        suffix ? "#{group}#{suffix}" : group
-      end
-
-      def forced_enabled?(experiment_key)
-        return true if params.has_key?(:force_experiment) && params[:force_experiment] == experiment_key.to_s
-        return false if cookies[:force_experiment].blank?
-
-        cookies[:force_experiment].to_s.split(',').any? { |experiment| experiment.strip == experiment_key.to_s }
-      end
-
-      def tracking_label(subject = nil)
-        return experimentation_subject_id if subject.blank?
-
-        if subject.respond_to?(:to_global_id)
-          Digest::SHA256.hexdigest(subject.to_global_id.to_s)
-        else
-          Digest::SHA256.hexdigest(subject.to_s)
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/experimentation/experiment.rb b/lib/gitlab/experimentation/experiment.rb
deleted file mode 100644
index 0c7091d19e3..00000000000
--- a/lib/gitlab/experimentation/experiment.rb
+++ /dev/null
@@ -1,45 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module Experimentation
-    class Experiment
-      FEATURE_FLAG_SUFFIX = "_experiment_percentage"
-
-      attr_reader :key, :tracking_category, :rollout_strategy
-
-      def initialize(key, **params)
-        @key = key
-        @tracking_category = params[:tracking_category]
-        @rollout_strategy = params[:rollout_strategy] || :cookie
-      end
-
-      def active?
-        # TODO: just touch a feature flag
-        # Temporary change, we will change `experiment_percentage` in future to `Feature.enabled?
-        Feature.enabled?(feature_flag_name, type: :experiment)
-
-        ::Gitlab.com? && experiment_percentage > 0
-      end
-
-      def enabled_for_index?(index)
-        return false if index.blank?
-
-        index <= experiment_percentage
-      end
-
-      private
-
-      def experiment_percentage
-        feature_flag.percentage_of_time_value
-      end
-
-      def feature_flag
-        Feature.get(feature_flag_name) # rubocop:disable Gitlab/AvoidFeatureGet
-      end
-
-      def feature_flag_name
-        :"#{key}#{FEATURE_FLAG_SUFFIX}"
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/experimentation_logger.rb b/lib/gitlab/experimentation_logger.rb
deleted file mode 100644
index ba1b60d6b4c..00000000000
--- a/lib/gitlab/experimentation_logger.rb
+++ /dev/null
@@ -1,9 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  class ExperimentationLogger < ::Gitlab::JsonLogger
-    def self.file_name_noext
-      'experimentation_json'
-    end
-  end
-end