diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-10-17 18:10:37 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-10-17 18:10:37 +0300 |
commit | 3884d9d7160e80a70ad327813ada6cab03cded65 (patch) | |
tree | 5a827e4b94a90e9fa62417d07db7e434002bcdad /lib | |
parent | 8060e5c60901ab0f6b890414dccbdf5d1b95c3ad (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'lib')
-rw-r--r-- | lib/api/debian_project_packages.rb | 6 | ||||
-rw-r--r-- | lib/bulk_imports/network_error.rb | 25 | ||||
-rw-r--r-- | lib/feature/shared.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/ci/config/external/context.rb | 3 | ||||
-rw-r--r-- | lib/gitlab/ci/pipeline/logger.rb | 1 | ||||
-rw-r--r-- | lib/gitlab/experimentation.rb | 110 | ||||
-rw-r--r-- | lib/gitlab/experimentation/controller_concern.rb | 156 | ||||
-rw-r--r-- | lib/gitlab/experimentation/experiment.rb | 45 | ||||
-rw-r--r-- | lib/gitlab/experimentation_logger.rb | 9 |
9 files changed, 22 insertions, 335 deletions
diff --git a/lib/api/debian_project_packages.rb b/lib/api/debian_project_packages.rb index 9dedc4390f7..03f0f97b805 100644 --- a/lib/api/debian_project_packages.rb +++ b/lib/api/debian_project_packages.rb @@ -81,11 +81,7 @@ module API package = ::Packages::Debian::FindOrCreateIncomingService.new(authorized_user_project, current_user).execute - package_file = ::Packages::Debian::CreatePackageFileService.new(package, file_params).execute - - if params['file_name'].end_with? '.changes' - ::Packages::Debian::ProcessChangesWorker.perform_async(package_file.id, current_user.id) # rubocop:disable CodeReuse/Worker - end + ::Packages::Debian::CreatePackageFileService.new(package: package, current_user: current_user, params: file_params).execute created! rescue ObjectStorage::RemoteStoreError => e diff --git a/lib/bulk_imports/network_error.rb b/lib/bulk_imports/network_error.rb index 335d992d5a7..fda4bb74a30 100644 --- a/lib/bulk_imports/network_error.rb +++ b/lib/bulk_imports/network_error.rb @@ -2,7 +2,8 @@ module BulkImports class NetworkError < Error - COUNTER_KEY = 'bulk_imports/%{entity_id}/%{stage}/%{tracker_id}/network_error/%{error}' + TRACKER_COUNTER_KEY = 'bulk_imports/%{entity_id}/%{stage}/%{tracker_id}/network_error/%{error}' + ENTITY_COUNTER_KEY = 'bulk_imports/%{entity_id}/network_error/%{error}' RETRIABLE_EXCEPTIONS = Gitlab::HTTP::HTTP_TIMEOUT_ERRORS + [ EOFError, SocketError, OpenSSL::SSL::SSLError, OpenSSL::OpenSSLError, @@ -24,9 +25,9 @@ module BulkImports @response = response end - def retriable?(tracker) + def retriable?(object) if retriable_exception? || retriable_http_code? - increment(tracker) <= MAX_RETRIABLE_COUNT + increment(object) <= MAX_RETRIABLE_COUNT else false end @@ -50,15 +51,27 @@ module BulkImports RETRIABLE_HTTP_CODES.include?(response&.code) end - def increment(tracker) - key = COUNTER_KEY % { + def increment(object) + key = object.is_a?(BulkImports::Entity) ? entity_cache_key(object) : tracker_cache_key(object) + + Gitlab::Cache::Import::Caching.increment(key) + end + + def tracker_cache_key(tracker) + TRACKER_COUNTER_KEY % { stage: tracker.stage, tracker_id: tracker.id, entity_id: tracker.entity.id, error: cause.class.name } + end - Gitlab::Cache::Import::Caching.increment(key) + def entity_cache_key(entity) + ENTITY_COUNTER_KEY % { + import_id: entity.bulk_import_id, + entity_id: entity.id, + error: cause.class.name + } end end end diff --git a/lib/feature/shared.rb b/lib/feature/shared.rb index edfc39aea0c..6af24451322 100644 --- a/lib/feature/shared.rb +++ b/lib/feature/shared.rb @@ -53,8 +53,6 @@ module Feature default_enabled: false, example: <<-EOS experiment(:my_experiment, project: project, actor: current_user) { ...variant code... } - # or - Gitlab::Experimentation.in_experiment_group?(:my_experiment, subject: current_user) EOS } }.freeze diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb index ec628399785..138e79db331 100644 --- a/lib/gitlab/ci/config/external/context.rb +++ b/lib/gitlab/ci/config/external/context.rb @@ -10,7 +10,6 @@ module Gitlab TimeoutError = Class.new(StandardError) MAX_INCLUDES = 100 - TRIAL_MAX_INCLUDES = 250 include ::Gitlab::Utils::StrongMemoize @@ -31,7 +30,7 @@ module Gitlab @expandset = Set.new @execution_deadline = 0 @logger = logger || Gitlab::Ci::Pipeline::Logger.new(project: project) - @max_includes = Feature.enabled?(:ci_increase_includes_to_250, project) ? TRIAL_MAX_INCLUDES : MAX_INCLUDES + @max_includes = MAX_INCLUDES yield self if block_given? end diff --git a/lib/gitlab/ci/pipeline/logger.rb b/lib/gitlab/ci/pipeline/logger.rb index 44d905faced..4b7cbae5004 100644 --- a/lib/gitlab/ci/pipeline/logger.rb +++ b/lib/gitlab/ci/pipeline/logger.rb @@ -86,6 +86,7 @@ module Gitlab 'count' => values.size, 'min' => values.min, 'max' => values.max, + 'sum' => values.sum, 'avg' => values.sum / values.size } end.compact diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb deleted file mode 100644 index 142d0e55593..00000000000 --- a/lib/gitlab/experimentation.rb +++ /dev/null @@ -1,110 +0,0 @@ -# frozen_string_literal: true - -# == Experimentation -# -# Utility module for A/B testing experimental features. Define your experiments in the `EXPERIMENTS` constant. -# Experiment options: -# - tracking_category (optional, used to set the category when tracking an experiment event) -# - rollout_strategy: default is `:cookie` based rollout. We may also set it to `:user` based rollout -# -# The experiment is controlled by a Feature Flag (https://docs.gitlab.com/ee/development/feature_flags/controls.html), -# which is named "#{experiment_key}_experiment_percentage" and *must* be set with a percentage and not be used for other purposes. -# -# To enable the experiment for 10% of the time: -# -# chatops: `/chatops run feature set experiment_key_experiment_percentage 10 --random` -# console: `Feature.enable_percentage_of_time(:experiment_key_experiment_percentage, 10)` -# -# To disable the experiment: -# -# chatops: `/chatops run feature delete experiment_key_experiment_percentage` -# console: `Feature.remove(:experiment_key_experiment_percentage)` -# -# To check the current rollout percentage: -# -# chatops: `/chatops run feature get experiment_key_experiment_percentage` -# console: `Feature.get(:experiment_key_experiment_percentage).percentage_of_time_value` -# - -# TODO: see https://gitlab.com/gitlab-org/gitlab/-/issues/217490 -module Gitlab - module Experimentation - EXPERIMENTS = { - }.freeze - - class << self - def get_experiment(experiment_key) - return unless EXPERIMENTS.key?(experiment_key) - - ::Gitlab::Experimentation::Experiment.new(experiment_key, **EXPERIMENTS[experiment_key]) - end - - def active?(experiment_key) - experiment = get_experiment(experiment_key) - return false unless experiment - - experiment.active? - end - - def in_experiment_group?(experiment_key, subject:) - return false if subject.blank? - return false unless active?(experiment_key) - - log_invalid_rollout(experiment_key, subject) - - experiment = get_experiment(experiment_key) - return false unless experiment - - experiment.enabled_for_index?(index_for_subject(experiment, subject)) - end - - def rollout_strategy(experiment_key) - experiment = get_experiment(experiment_key) - return unless experiment - - experiment.rollout_strategy - end - - def log_invalid_rollout(experiment_key, subject) - return if valid_subject_for_rollout_strategy?(experiment_key, subject) - - logger = Gitlab::ExperimentationLogger.build - logger.warn message: 'Subject must conform to the rollout strategy', - experiment_key: experiment_key, - subject: subject.class.to_s, - rollout_strategy: rollout_strategy(experiment_key) - end - - def valid_subject_for_rollout_strategy?(experiment_key, subject) - case rollout_strategy(experiment_key) - when :user - subject.is_a?(User) - when :group - subject.is_a?(Group) - when :cookie - subject.nil? || subject.is_a?(String) - else - false - end - end - - private - - def index_for_subject(experiment, subject) - index = Zlib.crc32("#{experiment.key}#{subject_id(subject)}") - - index % 100 - end - - def subject_id(subject) - if subject.respond_to?(:to_global_id) - subject.to_global_id.to_s - elsif subject.respond_to?(:to_s) - subject.to_s - else - raise ArgumentError, 'Subject must respond to `to_global_id` or `to_s`' - end - end - end - end -end diff --git a/lib/gitlab/experimentation/controller_concern.rb b/lib/gitlab/experimentation/controller_concern.rb deleted file mode 100644 index b09d67b8d5f..00000000000 --- a/lib/gitlab/experimentation/controller_concern.rb +++ /dev/null @@ -1,156 +0,0 @@ -# frozen_string_literal: true - -require 'zlib' - -# Controller concern that checks if an `experimentation_subject_id cookie` is present and sets it if absent. -# Used for A/B testing of experimental features. Exposes the `experiment_enabled?(experiment_name, subject: nil)` method -# to controllers and views. It returns true when the experiment is enabled and the user is selected as part -# of the experimental group. -# -module Gitlab - module Experimentation - module ControllerConcern - include ::Gitlab::Experimentation::GroupTypes - include Gitlab::Tracking::Helpers - extend ActiveSupport::Concern - - included do - before_action :set_experimentation_subject_id_cookie, unless: :dnt_enabled? - helper_method :experiment_enabled?, :experiment_tracking_category_and_group, :record_experiment_group - end - - def set_experimentation_subject_id_cookie - if Gitlab.com? - return if cookies[:experimentation_subject_id].present? - - cookies.permanent.signed[:experimentation_subject_id] = { - value: SecureRandom.uuid, - secure: ::Gitlab.config.gitlab.https, - httponly: true - } - else - # We set the cookie before, although experiments are not conducted on self managed instances. - cookies.delete(:experimentation_subject_id) - end - end - - def push_frontend_experiment(experiment_key, subject: nil) - var_name = experiment_key.to_s.camelize(:lower) - - enabled = experiment_enabled?(experiment_key, subject: subject) - - gon.push({ experiments: { var_name => enabled } }, true) - end - - def experiment_enabled?(experiment_key, subject: nil) - return true if forced_enabled?(experiment_key) - return false if dnt_enabled? - - Experimentation.log_invalid_rollout(experiment_key, subject) - - subject ||= experimentation_subject_id - - Experimentation.in_experiment_group?(experiment_key, subject: subject) - end - - def track_experiment_event(experiment_key, action, value = nil, subject: nil) - return if dnt_enabled? - - track_experiment_event_for(experiment_key, action, value, subject: subject) do |tracking_data| - ::Gitlab::Tracking.event(tracking_data.delete(:category), tracking_data.delete(:action), **tracking_data.merge!(user: current_user)) - end - end - - def frontend_experimentation_tracking_data(experiment_key, action, value = nil, subject: nil) - return if dnt_enabled? - - track_experiment_event_for(experiment_key, action, value, subject: subject) do |tracking_data| - gon.push(tracking_data: tracking_data) - end - end - - def record_experiment_user(experiment_key, context = {}) - return if dnt_enabled? - return unless Experimentation.active?(experiment_key) && current_user - - subject = Experimentation.rollout_strategy(experiment_key) == :cookie ? nil : current_user - - ::Experiment.add_user(experiment_key, tracking_group(experiment_key, nil, subject: subject), current_user, context) - end - - def record_experiment_group(experiment_key, group) - return if dnt_enabled? - return unless Experimentation.active?(experiment_key) && group - - variant_subject = Experimentation.rollout_strategy(experiment_key) == :cookie ? nil : group - variant = tracking_group(experiment_key, nil, subject: variant_subject) - - ::Experiment.add_group(experiment_key, group: group, variant: variant) - end - - def record_experiment_conversion_event(experiment_key, context = {}) - return if dnt_enabled? - return unless current_user - return unless Experimentation.active?(experiment_key) - - ::Experiment.record_conversion_event(experiment_key, current_user, context) - end - - def experiment_tracking_category_and_group(experiment_key, subject: nil) - "#{tracking_category(experiment_key)}:#{tracking_group(experiment_key, '_group', subject: subject)}" - end - - private - - def experimentation_subject_id - cookies.signed[:experimentation_subject_id] - end - - def track_experiment_event_for(experiment_key, action, value, subject: nil) - return unless Experimentation.active?(experiment_key) - - yield experimentation_tracking_data(experiment_key, action, value, subject: subject) - end - - def experimentation_tracking_data(experiment_key, action, value, subject: nil) - { - category: tracking_category(experiment_key), - action: action, - property: tracking_group(experiment_key, "_group", subject: subject), - label: tracking_label(subject), - value: value - }.compact - end - - def tracking_category(experiment_key) - Experimentation.get_experiment(experiment_key).tracking_category - end - - def tracking_group(experiment_key, suffix = nil, subject: nil) - return unless Experimentation.active?(experiment_key) - - subject ||= experimentation_subject_id - group = experiment_enabled?(experiment_key, subject: subject) ? GROUP_EXPERIMENTAL : GROUP_CONTROL - - suffix ? "#{group}#{suffix}" : group - end - - def forced_enabled?(experiment_key) - return true if params.has_key?(:force_experiment) && params[:force_experiment] == experiment_key.to_s - return false if cookies[:force_experiment].blank? - - cookies[:force_experiment].to_s.split(',').any? { |experiment| experiment.strip == experiment_key.to_s } - end - - def tracking_label(subject = nil) - return experimentation_subject_id if subject.blank? - - if subject.respond_to?(:to_global_id) - Digest::SHA256.hexdigest(subject.to_global_id.to_s) - else - Digest::SHA256.hexdigest(subject.to_s) - end - end - end - end -end diff --git a/lib/gitlab/experimentation/experiment.rb b/lib/gitlab/experimentation/experiment.rb deleted file mode 100644 index 0c7091d19e3..00000000000 --- a/lib/gitlab/experimentation/experiment.rb +++ /dev/null @@ -1,45 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Experimentation - class Experiment - FEATURE_FLAG_SUFFIX = "_experiment_percentage" - - attr_reader :key, :tracking_category, :rollout_strategy - - def initialize(key, **params) - @key = key - @tracking_category = params[:tracking_category] - @rollout_strategy = params[:rollout_strategy] || :cookie - end - - def active? - # TODO: just touch a feature flag - # Temporary change, we will change `experiment_percentage` in future to `Feature.enabled? - Feature.enabled?(feature_flag_name, type: :experiment) - - ::Gitlab.com? && experiment_percentage > 0 - end - - def enabled_for_index?(index) - return false if index.blank? - - index <= experiment_percentage - end - - private - - def experiment_percentage - feature_flag.percentage_of_time_value - end - - def feature_flag - Feature.get(feature_flag_name) # rubocop:disable Gitlab/AvoidFeatureGet - end - - def feature_flag_name - :"#{key}#{FEATURE_FLAG_SUFFIX}" - end - end - end -end diff --git a/lib/gitlab/experimentation_logger.rb b/lib/gitlab/experimentation_logger.rb deleted file mode 100644 index ba1b60d6b4c..00000000000 --- a/lib/gitlab/experimentation_logger.rb +++ /dev/null @@ -1,9 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - class ExperimentationLogger < ::Gitlab::JsonLogger - def self.file_name_noext - 'experimentation_json' - end - end -end |