diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-11-30 14:02:35 +0300 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-11-30 14:02:35 +0300 |
| commit | 434a0ce52d75e13d48eac9ce83774954c7c5d48d (patch) | |
| tree | de3b7a7cf1ce8b07555f28df592297c76894c90f /lib/gitlab | |
| parent | 0a0d9493ca481c56b739a3df27c31262283150fe (diff) | |
Add latest changes from gitlab-org/gitlab@13-7-stable-eev13.7.0-rc2
Diffstat (limited to 'lib/gitlab')
50 files changed, 850 insertions, 388 deletions
diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index e92bbe4f529..fbba86d1253 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -34,7 +34,8 @@ module Gitlab group_testing_hook: { threshold: 5, interval: 1.minute }, profile_add_new_email: { threshold: 5, interval: 1.minute }, profile_resend_email_confirmation: { threshold: 5, interval: 1.minute }, - update_environment_canary_ingress: { threshold: 1, interval: 1.minute } + update_environment_canary_ingress: { threshold: 1, interval: 1.minute }, + auto_rollback_deployment: { threshold: 1, interval: 3.minutes } }.freeze end diff --git a/lib/gitlab/background_migration/populate_vulnerability_historical_statistics.rb b/lib/gitlab/background_migration/populate_vulnerability_historical_statistics.rb index a0c89cc4664..2e81b1615d8 100644 --- a/lib/gitlab/background_migration/populate_vulnerability_historical_statistics.rb +++ b/lib/gitlab/background_migration/populate_vulnerability_historical_statistics.rb @@ -5,7 +5,7 @@ module Gitlab # This class creates/updates those project historical vulnerability statistics # that haven't been created nor initialized. It should only be executed in EE. class PopulateVulnerabilityHistoricalStatistics - def perform(project_ids) + def perform(project_ids, retention_period = 90) end end end diff --git a/lib/gitlab/background_migration/update_existing_users_that_require_two_factor_auth.rb b/lib/gitlab/background_migration/update_existing_users_that_require_two_factor_auth.rb new file mode 100644 index 00000000000..d97765cd398 --- /dev/null +++ b/lib/gitlab/background_migration/update_existing_users_that_require_two_factor_auth.rb @@ -0,0 +1,110 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class UpdateExistingUsersThatRequireTwoFactorAuth # rubocop:disable Metrics/ClassLength + def perform(start_id, stop_id) + ActiveRecord::Base.connection.execute <<~SQL + UPDATE + users + SET + require_two_factor_authentication_from_group = FALSE + WHERE + users.id BETWEEN #{start_id} + AND #{stop_id} + AND users.require_two_factor_authentication_from_group = TRUE + AND users.id NOT IN ( SELECT DISTINCT + users_groups_query.user_id + FROM ( + SELECT + users.id AS user_id, + members.source_id AS group_ids + FROM + users + LEFT JOIN members ON members.source_type = 'Namespace' + AND members.requested_at IS NULL + AND members.user_id = users.id + AND members.type = 'GroupMember' + WHERE + users.require_two_factor_authentication_from_group = TRUE + AND users.id BETWEEN #{start_id} + AND #{stop_id}) AS users_groups_query + INNER JOIN LATERAL ( WITH RECURSIVE "base_and_ancestors" AS ( + ( + SELECT + "namespaces"."type", + "namespaces"."id", + "namespaces"."parent_id", + "namespaces"."require_two_factor_authentication" + FROM + "namespaces" + WHERE + "namespaces"."type" = 'Group' + AND "namespaces"."id" = users_groups_query.group_ids) + UNION ( + SELECT + "namespaces"."type", + "namespaces"."id", + "namespaces"."parent_id", + "namespaces"."require_two_factor_authentication" + FROM + "namespaces", + "base_and_ancestors" + WHERE + "namespaces"."type" = 'Group' + AND "namespaces"."id" = "base_and_ancestors"."parent_id")), + "base_and_descendants" AS ( + ( + SELECT + "namespaces"."type", + "namespaces"."id", + "namespaces"."parent_id", + "namespaces"."require_two_factor_authentication" + FROM + "namespaces" + WHERE + "namespaces"."type" = 'Group' + AND "namespaces"."id" = users_groups_query.group_ids) + UNION ( + SELECT + "namespaces"."type", + "namespaces"."id", + "namespaces"."parent_id", + "namespaces"."require_two_factor_authentication" + FROM + "namespaces", + "base_and_descendants" + WHERE + "namespaces"."type" = 'Group' + AND "namespaces"."parent_id" = "base_and_descendants"."id")) + SELECT + "namespaces".* + FROM (( + SELECT + "namespaces"."type", + "namespaces"."id", + "namespaces"."parent_id", + "namespaces"."require_two_factor_authentication" + FROM + "base_and_ancestors" AS "namespaces" + WHERE + "namespaces"."type" = 'Group') + UNION ( + SELECT + "namespaces"."type", + "namespaces"."id", + "namespaces"."parent_id", + "namespaces"."require_two_factor_authentication" + FROM + "base_and_descendants" AS "namespaces" + WHERE + "namespaces"."type" = 'Group')) namespaces + WHERE + "namespaces"."type" = 'Group' + AND "namespaces"."require_two_factor_authentication" = TRUE) AS hierarchy_tree ON TRUE); + SQL + end + end + end +end diff --git a/lib/gitlab/ci/build/rules/rule/clause/changes.rb b/lib/gitlab/ci/build/rules/rule/clause/changes.rb index cbecce57163..9c2f6eea1dd 100644 --- a/lib/gitlab/ci/build/rules/rule/clause/changes.rb +++ b/lib/gitlab/ci/build/rules/rule/clause/changes.rb @@ -11,7 +11,7 @@ module Gitlab def satisfied_by?(pipeline, context) return true if pipeline.modified_paths.nil? - expanded_globs = expand_globs(pipeline, context) + expanded_globs = expand_globs(context) pipeline.modified_paths.any? do |path| expanded_globs.any? do |glob| File.fnmatch?(glob, path, File::FNM_PATHNAME | File::FNM_DOTMATCH | File::FNM_EXTGLOB) @@ -19,8 +19,7 @@ module Gitlab end end - def expand_globs(pipeline, context) - return @globs unless ::Feature.enabled?(:ci_variable_expansion_in_rules_changes, pipeline.project, default_enabled: true) + def expand_globs(context) return @globs unless context @globs.map do |glob| diff --git a/lib/gitlab/ci/parsers.rb b/lib/gitlab/ci/parsers.rb index 0e44475607b..57f73c265b2 100644 --- a/lib/gitlab/ci/parsers.rb +++ b/lib/gitlab/ci/parsers.rb @@ -10,7 +10,8 @@ module Gitlab junit: ::Gitlab::Ci::Parsers::Test::Junit, cobertura: ::Gitlab::Ci::Parsers::Coverage::Cobertura, terraform: ::Gitlab::Ci::Parsers::Terraform::Tfplan, - accessibility: ::Gitlab::Ci::Parsers::Accessibility::Pa11y + accessibility: ::Gitlab::Ci::Parsers::Accessibility::Pa11y, + codequality: ::Gitlab::Ci::Parsers::Codequality::CodeClimate } end diff --git a/lib/gitlab/ci/parsers/codequality/code_climate.rb b/lib/gitlab/ci/parsers/codequality/code_climate.rb new file mode 100644 index 00000000000..628d50b84cb --- /dev/null +++ b/lib/gitlab/ci/parsers/codequality/code_climate.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Codequality + class CodeClimate + def parse!(json_data, codequality_report) + root = Gitlab::Json.parse(json_data) + + parse_all(root, codequality_report) + rescue JSON::ParserError => e + codequality_report.set_error_message("JSON parsing failed: #{e}") + end + + private + + def parse_all(root, codequality_report) + return unless root.present? + + root.each do |degradation| + break unless codequality_report.add_degradation(degradation) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb index a864c843dd8..2ca51930c19 100644 --- a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb +++ b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb @@ -35,7 +35,7 @@ module Gitlab # rubocop: enable CodeReuse/ActiveRecord def pipelines - if ::Feature.enabled?(:ci_auto_cancel_all_pipelines, project, default_enabled: false) + if ::Feature.enabled?(:ci_auto_cancel_all_pipelines, project, default_enabled: true) project.all_pipelines.ci_and_parent_sources else project.ci_pipelines diff --git a/lib/gitlab/ci/reports/codequality_reports.rb b/lib/gitlab/ci/reports/codequality_reports.rb new file mode 100644 index 00000000000..060a1e2399b --- /dev/null +++ b/lib/gitlab/ci/reports/codequality_reports.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + class CodequalityReports + attr_reader :degradations, :error_message + + CODECLIMATE_SCHEMA_PATH = Rails.root.join('app', 'validators', 'json_schemas', 'codeclimate.json').to_s + + def initialize + @degradations = {}.with_indifferent_access + @error_message = nil + end + + def add_degradation(degradation) + valid_degradation?(degradation) && @degradations[degradation.dig('fingerprint')] = degradation + end + + def set_error_message(error) + @error_message = error + end + + def degradations_count + @degradations.size + end + + def all_degradations + @degradations.values + end + + private + + def valid_degradation?(degradation) + JSON::Validator.validate!(CODECLIMATE_SCHEMA_PATH, degradation) + rescue JSON::Schema::ValidationError => e + set_error_message("Invalid degradation format: #{e.message}") + false + end + end + end + end +end diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index 385959389de..e5b40e5f49a 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ .auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.0.0-beta.2" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.0.0" dependencies: [] review: diff --git a/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml b/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml index 65abee1f5eb..3faf07546de 100644 --- a/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: ayufan/openshift-cli +image: openshift/origin-cli stages: - build # dummy stage to follow the template guidelines diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index 3cbde9d30c8..5ea2363a0c5 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -8,7 +8,7 @@ variables: container_scanning: stage: test - image: $SECURE_ANALYZERS_PREFIX/klar:$CS_MAJOR_VERSION + image: "$CS_ANALYZER_IMAGE" variables: # By default, use the latest clair vulnerabilities database, however, allow it to be overridden here with a specific image # to enable container scanning to run offline, or to provide a consistent list of vulnerabilities for integration testing purposes @@ -18,6 +18,7 @@ container_scanning: # file. See https://docs.gitlab.com/ee/user/application_security/container_scanning/index.html#overriding-the-container-scanning-template # for details GIT_STRATEGY: none + CS_ANALYZER_IMAGE: $SECURE_ANALYZERS_PREFIX/klar:$CS_MAJOR_VERSION allow_failure: true services: - name: $CLAIR_DB_IMAGE diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index 3789f0edc1c..b534dad9593 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -28,11 +28,8 @@ dependency_scanning: .ds-analyzer: extends: dependency_scanning allow_failure: true - rules: - - if: $DEPENDENCY_SCANNING_DISABLED - when: never - - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ + # `rules` must be overridden explicitly by each child job + # see https://gitlab.com/gitlab-org/gitlab/-/issues/218444 script: - /analyzer run diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index a51cb61da6d..671e2346fcb 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -30,10 +30,8 @@ sast: .sast-analyzer: extends: sast allow_failure: true - rules: - - if: $SAST_DISABLED - when: never - - if: $CI_COMMIT_BRANCH + # `rules` must be overridden explicitly by each child job + # see https://gitlab.com/gitlab-org/gitlab/-/issues/218444 script: - /analyzer run diff --git a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml index 6ebff102ccb..8ca1d2e08ba 100644 --- a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml @@ -14,6 +14,9 @@ variables: stage: test image: "$SECURE_ANALYZERS_PREFIX/secrets:$SECRETS_ANALYZER_VERSION" services: [] + allow_failure: true + # `rules` must be overridden explicitly by each child job + # see https://gitlab.com/gitlab-org/gitlab/-/issues/218444 artifacts: reports: secret_detection: gl-secret-detection-report.json diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml index e455bfac9de..910e711f046 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml @@ -56,5 +56,6 @@ cache: .destroy: &destroy stage: cleanup script: + - cd ${TF_ROOT} - gitlab-terraform destroy when: manual diff --git a/lib/gitlab/danger/changelog.rb b/lib/gitlab/danger/changelog.rb index 607ca1200a0..92af6849b2f 100644 --- a/lib/gitlab/danger/changelog.rb +++ b/lib/gitlab/danger/changelog.rb @@ -39,6 +39,7 @@ module Gitlab def required? git.added_files.any? { |path| path =~ %r{\Adb/(migrate|post_migrate)/} } end + alias_method :db_changes?, :required? def optional? categories_need_changelog? && without_no_changelog_label? diff --git a/lib/gitlab/database/batch_count.rb b/lib/gitlab/database/batch_count.rb index 6f79e965cd5..5a506da0d05 100644 --- a/lib/gitlab/database/batch_count.rb +++ b/lib/gitlab/database/batch_count.rb @@ -49,6 +49,8 @@ module Gitlab MAX_ALLOWED_LOOPS = 10_000 SLEEP_TIME_IN_SECONDS = 0.01 # 10 msec sleep ALLOWED_MODES = [:itself, :distinct].freeze + FALLBACK_FINISH = 0 + OFFSET_BY_ONE = 1 # Each query should take < 500ms https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22705 DEFAULT_DISTINCT_BATCH_SIZE = 10_000 @@ -65,7 +67,7 @@ module Gitlab (@operation == :count && batch_size <= MIN_REQUIRED_BATCH_SIZE) || (@operation == :sum && batch_size < DEFAULT_SUM_BATCH_SIZE) || (finish - start) / batch_size >= MAX_ALLOWED_LOOPS || - start > finish + start >= finish end def count(batch_size: nil, mode: :itself, start: nil, finish: nil) @@ -85,11 +87,13 @@ module Gitlab results = nil batch_start = start - while batch_start <= finish - batch_relation = build_relation_batch(batch_start, batch_start + batch_size, mode) + while batch_start < finish + batch_end = [batch_start + batch_size, finish].min + batch_relation = build_relation_batch(batch_start, batch_end, mode) + begin results = merge_results(results, batch_relation.send(@operation, *@operation_args)) # rubocop:disable GitlabSecurity/PublicSend - batch_start += batch_size + batch_start = batch_end rescue ActiveRecord::QueryCanceled => error # retry with a safe batch size & warmer cache if batch_size >= 2 * MIN_REQUIRED_BATCH_SIZE @@ -99,6 +103,7 @@ module Gitlab return FALLBACK end end + sleep(SLEEP_TIME_IN_SECONDS) end @@ -138,7 +143,7 @@ module Gitlab end def actual_finish(finish) - finish || @relation.unscope(:group, :having).maximum(@column) || 0 + (finish || @relation.unscope(:group, :having).maximum(@column) || FALLBACK_FINISH) + OFFSET_BY_ONE end def check_mode!(mode) diff --git a/lib/gitlab/database/migrations/background_migration_helpers.rb b/lib/gitlab/database/migrations/background_migration_helpers.rb index a6cc03aa9eb..36073844765 100644 --- a/lib/gitlab/database/migrations/background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/background_migration_helpers.rb @@ -55,7 +55,8 @@ module Gitlab bulk_migrate_async(jobs) unless jobs.empty? end - # Queues background migration jobs for an entire table, batched by ID range. + # Queues background migration jobs for an entire table in batches. + # The default batching column used is the standard primary key `id`. # Each job is scheduled with a `delay_interval` in between. # If you use a small interval, then some jobs may run at the same time. # @@ -68,6 +69,7 @@ module Gitlab # is scheduled to be run. These records can be used to trace execution of the background job, but there is no # builtin support to manage that automatically at this time. You should only set this flag if you are aware of # how it works, and intend to manually cleanup the database records in your background job. + # primary_column_name - The name of the primary key column if the primary key is not `id` # # *Returns the final migration delay* # @@ -87,8 +89,9 @@ module Gitlab # # do something # end # end - def queue_background_migration_jobs_by_range_at_intervals(model_class, job_class_name, delay_interval, batch_size: BACKGROUND_MIGRATION_BATCH_SIZE, other_job_arguments: [], initial_delay: 0, track_jobs: false) - raise "#{model_class} does not have an ID to use for batch ranges" unless model_class.column_names.include?('id') + def queue_background_migration_jobs_by_range_at_intervals(model_class, job_class_name, delay_interval, batch_size: BACKGROUND_MIGRATION_BATCH_SIZE, other_job_arguments: [], initial_delay: 0, track_jobs: false, primary_column_name: :id) + raise "#{model_class} does not have an ID column of #{primary_column_name} to use for batch ranges" unless model_class.column_names.include?(primary_column_name.to_s) + raise "#{primary_column_name} is not an integer column" unless model_class.columns_hash[primary_column_name.to_s].type == :integer # To not overload the worker too much we enforce a minimum interval both # when scheduling and performing jobs. @@ -99,7 +102,7 @@ module Gitlab final_delay = 0 model_class.each_batch(of: batch_size) do |relation, index| - start_id, end_id = relation.pluck(Arel.sql('MIN(id), MAX(id)')).first + start_id, end_id = relation.pluck(Arel.sql("MIN(#{primary_column_name}), MAX(#{primary_column_name})")).first # `BackgroundMigrationWorker.bulk_perform_in` schedules all jobs for # the same time, which is not helpful in most cases where we wish to diff --git a/lib/gitlab/database/postgres_hll_batch_distinct_counter.rb b/lib/gitlab/database/postgres_hll_batch_distinct_counter.rb new file mode 100644 index 00000000000..4b2afcb128f --- /dev/null +++ b/lib/gitlab/database/postgres_hll_batch_distinct_counter.rb @@ -0,0 +1,156 @@ +# frozen_string_literal: true + +module Gitlab + module Database + # For large tables, PostgreSQL can take a long time to count rows due to MVCC. + # Implements a distinct batch counter based on HyperLogLog algorithm + # Needs indexes on the column below to calculate max, min and range queries + # For larger tables just set higher batch_size with index optimization + # + # In order to not use a possible complex time consuming query when calculating min and max values, + # the start and finish can be sent specifically, start and finish should contain max and min values for PRIMARY KEY of + # relation (most cases `id` column) rather than counted attribute eg: + # estimate_distinct_count(start: ::Project.with_active_services.minimum(:id), finish: ::Project.with_active_services.maximum(:id)) + # + # Grouped relations are NOT supported yet. + # + # @example Usage + # ::Gitlab::Database::PostgresHllBatchDistinctCount.new(::Project, :creator_id).estimate_distinct_count + # ::Gitlab::Database::PostgresHllBatchDistinctCount.new(::Project.with_active_services.service_desk_enabled.where(time_period)) + # .estimate_distinct_count( + # batch_size: 1_000, + # start: ::Project.with_active_services.service_desk_enabled.where(time_period).minimum(:id), + # finish: ::Project.with_active_services.service_desk_enabled.where(time_period).maximum(:id) + # ) + # + # @note HyperLogLog is an PROBABILISTIC algorithm that ESTIMATES distinct count of given attribute value for supplied relation + # Like all probabilistic algorithm is has ERROR RATE margin, that can affect values, + # for given implementation no higher value was reported (https://gitlab.com/gitlab-org/gitlab/-/merge_requests/45673#accuracy-estimation) than 5.3% + # for the most of a cases this value is lower. However, if the exact value is necessary other tools has to be used. + class PostgresHllBatchDistinctCounter + FALLBACK = -1 + MIN_REQUIRED_BATCH_SIZE = 1_250 + MAX_ALLOWED_LOOPS = 10_000 + SLEEP_TIME_IN_SECONDS = 0.01 # 10 msec sleep + + # Each query should take < 500ms https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22705 + DEFAULT_BATCH_SIZE = 100_000 + + BIT_31_MASK = "B'0#{'1' * 31}'" + BIT_9_MASK = "B'#{'0' * 23}#{'1' * 9}'" + # @example source_query + # SELECT CAST(('X' || md5(CAST(%{column} as text))) as bit(32)) attr_hash_32_bits + # FROM %{relation} + # WHERE %{pkey} >= %{batch_start} + # AND %{pkey} < %{batch_end} + # AND %{column} IS NOT NULL + BUCKETED_DATA_SQL = <<~SQL + WITH hashed_attributes AS (%{source_query}) + SELECT (attr_hash_32_bits & #{BIT_9_MASK})::int AS bucket_num, + (31 - floor(log(2, min((attr_hash_32_bits & #{BIT_31_MASK})::int))))::int as bucket_hash + FROM hashed_attributes + GROUP BY 1 ORDER BY 1 + SQL + + TOTAL_BUCKETS_NUMBER = 512 + + def initialize(relation, column = nil) + @relation = relation + @column = column || relation.primary_key + end + + def unwanted_configuration?(finish, batch_size, start) + batch_size <= MIN_REQUIRED_BATCH_SIZE || + (finish - start) / batch_size >= MAX_ALLOWED_LOOPS || + start > finish + end + + def estimate_distinct_count(batch_size: nil, start: nil, finish: nil) + raise 'BatchCount can not be run inside a transaction' if ActiveRecord::Base.connection.transaction_open? + + batch_size ||= DEFAULT_BATCH_SIZE + + start = actual_start(start) + finish = actual_finish(finish) + + raise "Batch counting expects positive values only for #{@column}" if start < 0 || finish < 0 + return FALLBACK if unwanted_configuration?(finish, batch_size, start) + + batch_start = start + hll_blob = {} + + while batch_start <= finish + begin + hll_blob.merge!(hll_blob_for_batch(batch_start, batch_start + batch_size)) {|_key, old, new| new > old ? new : old } + batch_start += batch_size + end + sleep(SLEEP_TIME_IN_SECONDS) + end + + estimate_cardinality(hll_blob) + end + + private + + # arbitrary values that are present in #estimate_cardinality + # are sourced from https://www.sisense.com/blog/hyperloglog-in-pure-sql/ + # article, they are not representing any entity and serves as tune value + # for the whole equation + def estimate_cardinality(hll_blob) + num_zero_buckets = TOTAL_BUCKETS_NUMBER - hll_blob.size + + num_uniques = ( + ((TOTAL_BUCKETS_NUMBER**2) * (0.7213 / (1 + 1.079 / TOTAL_BUCKETS_NUMBER))) / + (num_zero_buckets + hll_blob.values.sum { |bucket_hash, _| 2**(-1 * bucket_hash)} ) + ).to_i + + if num_zero_buckets > 0 && num_uniques < 2.5 * TOTAL_BUCKETS_NUMBER + ((0.7213 / (1 + 1.079 / TOTAL_BUCKETS_NUMBER)) * (TOTAL_BUCKETS_NUMBER * + Math.log2(TOTAL_BUCKETS_NUMBER.to_f / num_zero_buckets))) + else + num_uniques + end + end + + def hll_blob_for_batch(start, finish) + @relation + .connection + .execute(BUCKETED_DATA_SQL % { source_query: source_query(start, finish) }) + .map(&:values) + .to_h + end + + # Generate the source query SQL snippet for the provided id range + # + # @example SQL query template + # SELECT CAST(('X' || md5(CAST(%{column} as text))) as bit(32)) attr_hash_32_bits + # FROM %{relation} + # WHERE %{pkey} >= %{batch_start} AND %{pkey} < %{batch_end} + # AND %{column} IS NOT NULL + # + # @param start initial id range + # @param finish final id range + # @return [String] SQL query fragment + def source_query(start, finish) + col_as_arel = @column.is_a?(Arel::Attributes::Attribute) ? @column : Arel.sql(@column.to_s) + col_as_text = Arel::Nodes::NamedFunction.new('CAST', [col_as_arel.as('text')]) + md5_of_col = Arel::Nodes::NamedFunction.new('md5', [col_as_text]) + md5_as_hex = Arel::Nodes::Concat.new(Arel.sql("'X'"), md5_of_col) + bits = Arel::Nodes::NamedFunction.new('CAST', [md5_as_hex.as('bit(32)')]) + + @relation + .where(@relation.primary_key => (start...finish)) + .where(col_as_arel.not_eq(nil)) + .select(bits.as('attr_hash_32_bits')).to_sql + end + + def actual_start(start) + start || @relation.unscope(:group, :having).minimum(@relation.primary_key) || 0 + end + + def actual_finish(finish) + finish || @relation.unscope(:group, :having).maximum(@relation.primary_key) || 0 + end + end + end +end diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index 6e39776bbd4..d511972757f 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -4,7 +4,6 @@ # # Utility module for A/B testing experimental features. Define your experiments in the `EXPERIMENTS` constant. # Experiment options: -# - environment (optional, defaults to enabled for development and GitLab.com) # - tracking_category (optional, used to set the category when tracking an experiment event) # - use_backwards_compatible_subject_index (optional, set this to true if you need backwards compatibility) # @@ -55,10 +54,6 @@ module Gitlab tracking_category: 'Growth::Expansion::Experiment::InviteMembersEmptyGroupVersionA', use_backwards_compatible_subject_index: true }, - new_create_project_ui: { - tracking_category: 'Manage::Import::Experiment::NewCreateProjectUi', - use_backwards_compatible_subject_index: true - }, contact_sales_btn_in_app: { tracking_category: 'Growth::Conversion::Experiment::ContactSalesInApp', use_backwards_compatible_subject_index: true @@ -87,14 +82,13 @@ module Gitlab class << self def experiment(key) - Experiment.new(EXPERIMENTS[key].merge(key: key)) + Gitlab::Experimentation::Experiment.new(key, **EXPERIMENTS[key]) end def enabled?(experiment_key) return false unless EXPERIMENTS.key?(experiment_key) - experiment = experiment(experiment_key) - experiment.enabled_for_environment? && experiment.enabled? + experiment(experiment_key).enabled? end def enabled_for_attribute?(experiment_key, attribute) @@ -106,36 +100,5 @@ module Gitlab enabled?(experiment_key) && experiment(experiment_key).enabled_for_index?(value) end end - - Experiment = Struct.new( - :key, - :environment, - :tracking_category, - :use_backwards_compatible_subject_index, - keyword_init: true - ) do - def enabled? - experiment_percentage > 0 - end - - def enabled_for_environment? - return ::Gitlab.dev_env_or_com? if environment.nil? - - environment - end - - def enabled_for_index?(index) - return false if index.blank? - - index <= experiment_percentage - end - - private - - # When a feature does not exist, the `percentage_of_time_value` method will return 0 - def experiment_percentage - @experiment_percentage ||= Feature.get(:"#{key}_experiment_percentage").percentage_of_time_value # rubocop:disable Gitlab/AvoidFeatureGet - end - end end end diff --git a/lib/gitlab/experimentation/experiment.rb b/lib/gitlab/experimentation/experiment.rb new file mode 100644 index 00000000000..bacbaa6949c --- /dev/null +++ b/lib/gitlab/experimentation/experiment.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module Experimentation + class Experiment + attr_reader :tracking_category, :use_backwards_compatible_subject_index + + def initialize(key, **params) + @tracking_category = params[:tracking_category] + @use_backwards_compatible_subject_index = params[:use_backwards_compatible_subject_index] + + @experiment_percentage = Feature.get(:"#{key}_experiment_percentage").percentage_of_time_value # rubocop:disable Gitlab/AvoidFeatureGet + end + + def enabled? + ::Gitlab.dev_env_or_com? && experiment_percentage > 0 + end + + def enabled_for_index?(index) + return false if index.blank? + + index <= experiment_percentage + end + + private + + attr_reader :experiment_percentage + end + end +end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 96f3487fd6f..a2215366bdc 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -17,6 +17,7 @@ module Gitlab CommitError = Class.new(BaseError) OSError = Class.new(BaseError) UnknownRef = Class.new(BaseError) + CommandTimedOut = Class.new(CommandError) class << self include Gitlab::EncodingHelper diff --git a/lib/gitlab/git/wraps_gitaly_errors.rb b/lib/gitlab/git/wraps_gitaly_errors.rb index 9963bcfbf1c..2009683d32c 100644 --- a/lib/gitlab/git/wraps_gitaly_errors.rb +++ b/lib/gitlab/git/wraps_gitaly_errors.rb @@ -9,6 +9,8 @@ module Gitlab raise Gitlab::Git::Repository::NoRepository.new(e) rescue GRPC::InvalidArgument => e raise ArgumentError.new(e) + rescue GRPC::DeadlineExceeded => e + raise Gitlab::Git::CommandTimedOut.new(e) rescue GRPC::BadStatus => e raise Gitlab::Git::CommandError.new(e) end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 2d41ad76618..7261589473a 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -83,3 +83,5 @@ module Gitlab end end end + +Gitlab::GonHelper.prepend_if_ee('EE::Gitlab::GonHelper') diff --git a/lib/gitlab/graphql/authorize/authorize_resource.rb b/lib/gitlab/graphql/authorize/authorize_resource.rb index c70127553fd..6ee446011d4 100644 --- a/lib/gitlab/graphql/authorize/authorize_resource.rb +++ b/lib/gitlab/graphql/authorize/authorize_resource.rb @@ -62,8 +62,8 @@ module Gitlab end end - def raise_resource_not_available_error! - raise Gitlab::Graphql::Errors::ResourceNotAvailable, RESOURCE_ACCESS_ERROR + def raise_resource_not_available_error!(msg = RESOURCE_ACCESS_ERROR) + raise Gitlab::Graphql::Errors::ResourceNotAvailable, msg end end end diff --git a/lib/gitlab/graphql/connection_collection_methods.rb b/lib/gitlab/graphql/connection_collection_methods.rb new file mode 100644 index 00000000000..0e2c4a98bb6 --- /dev/null +++ b/lib/gitlab/graphql/connection_collection_methods.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module ConnectionCollectionMethods + extend ActiveSupport::Concern + + included do + delegate :to_a, :size, :include?, :empty?, to: :nodes + end + end + end +end diff --git a/lib/gitlab/graphql/connection_redaction.rb b/lib/gitlab/graphql/connection_redaction.rb new file mode 100644 index 00000000000..5e037bb9f63 --- /dev/null +++ b/lib/gitlab/graphql/connection_redaction.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module ConnectionRedaction + class RedactionState + attr_reader :redactor + attr_reader :redacted_nodes + + def redactor=(redactor) + @redactor = redactor + @redacted_nodes = nil + end + + def redacted(&block) + @redacted_nodes ||= redactor.present? ? redactor.redact(yield) : yield + end + end + + delegate :redactor=, to: :redaction_state + + def nodes + redaction_state.redacted { super.to_a } + end + + private + + def redaction_state + @redaction_state ||= RedactionState.new + end + end + end +end diff --git a/lib/gitlab/graphql/pagination/array_connection.rb b/lib/gitlab/graphql/pagination/array_connection.rb new file mode 100644 index 00000000000..efc912eaeca --- /dev/null +++ b/lib/gitlab/graphql/pagination/array_connection.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +# We use the Keyset / Stable cursor connection by default for ActiveRecord::Relation. +# However, there are times when that may not be powerful enough (yet), and we +# want to use standard offset pagination. +module Gitlab + module Graphql + module Pagination + class ArrayConnection < ::GraphQL::Pagination::ArrayConnection + prepend ::Gitlab::Graphql::ConnectionRedaction + include ::Gitlab::Graphql::ConnectionCollectionMethods + end + end + end +end diff --git a/lib/gitlab/graphql/pagination/connections.rb b/lib/gitlab/graphql/pagination/connections.rb index 8f37fa3f474..54a84be4274 100644 --- a/lib/gitlab/graphql/pagination/connections.rb +++ b/lib/gitlab/graphql/pagination/connections.rb @@ -12,6 +12,10 @@ module Gitlab schema.connections.add( Gitlab::Graphql::ExternallyPaginatedArray, Gitlab::Graphql::Pagination::ExternallyPaginatedArrayConnection) + + schema.connections.add( + Array, + Gitlab::Graphql::Pagination::ArrayConnection) end end end diff --git a/lib/gitlab/graphql/pagination/externally_paginated_array_connection.rb b/lib/gitlab/graphql/pagination/externally_paginated_array_connection.rb index 12e047420bf..90a20861b0d 100644 --- a/lib/gitlab/graphql/pagination/externally_paginated_array_connection.rb +++ b/lib/gitlab/graphql/pagination/externally_paginated_array_connection.rb @@ -5,6 +5,9 @@ module Gitlab module Graphql module Pagination class ExternallyPaginatedArrayConnection < GraphQL::Pagination::ArrayConnection + include ::Gitlab::Graphql::ConnectionCollectionMethods + prepend ::Gitlab::Graphql::ConnectionRedaction + def start_cursor items.previous_cursor end diff --git a/lib/gitlab/graphql/pagination/keyset/connection.rb b/lib/gitlab/graphql/pagination/keyset/connection.rb index 252f6371765..2ad8d2f7ab7 100644 --- a/lib/gitlab/graphql/pagination/keyset/connection.rb +++ b/lib/gitlab/graphql/pagination/keyset/connection.rb @@ -31,6 +31,8 @@ module Gitlab module Keyset class Connection < GraphQL::Pagination::ActiveRecordRelationConnection include Gitlab::Utils::StrongMemoize + include ::Gitlab::Graphql::ConnectionCollectionMethods + prepend ::Gitlab::Graphql::ConnectionRedaction # rubocop: disable Naming/PredicateName # https://relay.dev/graphql/connections.htm#sec-undefined.PageInfo.Fields diff --git a/lib/gitlab/graphql/pagination/keyset/order_info.rb b/lib/gitlab/graphql/pagination/keyset/order_info.rb index f3ce3a10703..d37264c1343 100644 --- a/lib/gitlab/graphql/pagination/keyset/order_info.rb +++ b/lib/gitlab/graphql/pagination/keyset/order_info.rb @@ -127,3 +127,5 @@ module Gitlab end end end + +Gitlab::Graphql::Pagination::Keyset::OrderInfo.prepend_if_ee('EE::Gitlab::Graphql::Pagination::Keyset::OrderInfo') diff --git a/lib/gitlab/graphql/pagination/offset_active_record_relation_connection.rb b/lib/gitlab/graphql/pagination/offset_active_record_relation_connection.rb index 33f84701562..4a57b7aceca 100644 --- a/lib/gitlab/graphql/pagination/offset_active_record_relation_connection.rb +++ b/lib/gitlab/graphql/pagination/offset_active_record_relation_connection.rb @@ -7,6 +7,8 @@ module Gitlab module Graphql module Pagination class OffsetActiveRecordRelationConnection < GraphQL::Pagination::ActiveRecordRelationConnection + prepend ::Gitlab::Graphql::ConnectionRedaction + include ::Gitlab::Graphql::ConnectionCollectionMethods end end end diff --git a/lib/gitlab/i18n/html_todo.yml b/lib/gitlab/i18n/html_todo.yml index 91e01f8a0b8..610381b9b48 100644 --- a/lib/gitlab/i18n/html_todo.yml +++ b/lib/gitlab/i18n/html_todo.yml @@ -55,260 +55,3 @@ - "La branĉo <strong>%{branch_name}</strong> estis kreita. Por agordi aŭtomatan disponigadon, bonvolu elekti Yaml-ŝablonon por GitLab CI kaj enmeti viajn ŝanĝojn. %{link_to_autodeploy_doc}" - "La branche <strong>%{branch_name}</strong> a été créée. Pour mettre en place le déploiement automatisé, sélectionnez un modèle de fichier YAML pour l’intégration continue (CI) de GitLab, et validez les modifications. %{link_to_autodeploy_doc}" - "La rama <strong>%{branch_name}</strong> fue creada. Para configurar el auto despliegue, escoge una plantilla Yaml para GitLab CI y envía tus cambios. %{link_to_autodeploy_doc}" -"GitLabPages|GitLab Pages are disabled for this project. You can enable them on your project's %{strong_start}Settings > General > Visibility%{strong_end} page.": - plural_id: - translations: - - "GitLab Pagesはこのプロジェクトでは無効になっています。 プロジェクトの%{strong_start} 設定> 全般> 可視性%{strong_end}ページで有効にできます。" - - "GitLab Pages отключены для этого проекта. Вы можете включить в поле %{strong_start}Настройки > Общие > Видимость%{strong_end} вашего проекта." - - "此项目禁用GitLab Pages。您可以在您的项目的%{strong_start}设置 > 常规 > 可见性%{strong_end} 页面启用。" - - "GitLab Pages вимкнено для цього проєкту. Ви можете їх увімкнути перейшовши на сторінку проєкту %{strong_start}Налаштування > Загальні > Видимість%{strong_end}." - - "Las páginas de GitLab están deshabilitadas para este proyecto. Puede habilitarlas en los ajustes %{strong_start} de su proyecto > General > Visibilidad%{strong_end}." -"You can invite a new member to <strong>%{project_name}</strong> or invite another group.": - plural_id: - translations: - - "新しいメンバーを<strong>%{project_name} </strong>に招待するか、別のグループを招待することができます。" - - "Podes convidar um novo para <strong>%{project_name}</strong> ou convidar outro grupo." - - "邀请新成员或另一个群组加入<strong>%{project_name}</strong>。" - - "Puede invitar a un nuevo miembro a <strong>%{project_name}</strong> o invitar a otro grupo." - - "<strong>%{project_name}</strong> projesine yeni bir üye davet edebilir veya başka bir grubu davet edebilirsiniz." - - "Вы можете пригласить нового участника в <strong>%{project_name}</strong> или пригласить другую группу." - - "Ви можете запросити нового учасника до <strong>%{project_name}</strong> або запросити іншу групу." -"You can invite a new member to <strong>%{project_name}</strong>.": - plural_id: - translations: - - "新しいメンバーを<strong>%{project_name} </strong>に招待できます。" - - "Podes convidar um novo membro para <strong>%{project_name}</strong>." - - "邀请新成员加入<strong>%{project_name}</strong>。" - - "Puedes invitar a un nuevo miembro a <strong>%{project_name}</strong>." - - "<strong>%{project_name}</strong> projesine yeni bir üye davet edebilirsiniz." - - "Вы можете пригласить нового участника в <strong>%{project_name}</strong>." - - "Ви можете запросити нового учасника до <strong>%{project_name}</strong>." -"You can invite another group to <strong>%{project_name}</strong>.": - plural_id: - translations: - - "他のグループを<strong>%{project_name} </strong>に招待できます。" - - "Podes convidar outro grupo para <strong>%{project_name}</strong>." - - "您可以邀请另一个群组加入<strong>%{project_name}</strong>。" - - "Ви можете запросити нову групу до <strong>%{project_name}</strong>." - - "Puedes invitar a otro grupo a <strong>%{project_name}</strong>." -"Example: <code>192.168.0.0/24</code>. %{read_more_link}.": - plural_id: - translations: -"Note that PostgreSQL %{pg_version_upcoming} will become the minimum required version in GitLab %{gl_version_upcoming} (%{gl_version_upcoming_date}). Please consider upgrading your environment to a supported PostgreSQL version soon, see <a href=\\\"%{pg_version_upcoming_url}\\\">the related epic</a> for details.": - plural_id: - translations: -"Authorize <strong>%{user}</strong> to use your account?": - plural_id: - translations: -"DeployFreeze|Specify times when deployments are not allowed for an environment. The <code>gitlab-ci.yml</code> file must be updated to make deployment jobs aware of the %{freeze_period_link_start}freeze period%{freeze_period_link_end}.": - plural_id: - translations: -"<project name>": - translations: - - "<название проекта>" - - "<project name>" - - "<proje adı>" - - "<naziv projekta>" - - "<ім’я проєкту>" - - "<프로젝트 이름>" -"<strong>Deletes</strong> source branch": - plural_id: - translations: - - "<strong>刪除</strong>來源分支" - - "<strong>Apagar</strong> branch de origem" - - "ソースブランチを<strong>削除</strong>" - - "<strong>刪除</strong>來源分支" - - "<strong>Apagar</strong> o ramo de origem" - - "<strong>Удаляет</strong> исходную ветку" - - "<strong>删除</strong>源分支" - - "<strong>Видаляє</strong> гілку-джерело" - - "<strong>Löscht</strong> den Quellbranch" - - "소스 브랜치 <strong>삭제</strong>" - - "<strong>Supprime</strong> la branche source" - - "<strong>elimina</strong> la rama origen" - - "Kaynak dalı <strong>siler</strong>" -"Badges|You are going to delete this badge. Deleted badges <strong>cannot</strong> be restored.": - plural_id: - translations: - - "Você está prestes a excluir este selo. Selos excluídos <strong>não podem</strong> ser restaurados." - - "このバッジを削除しようとしています。削除されたバッジは<strong>復元できません</strong>。" - - "Estás prestes a apagar este emblema. Emblemas apagados <strong>não podem</strong> ser restaurados." - - "Вы собираетесь удалить этот значок. Удаленные значки <strong>не могут</strong> быть восстановлены." - - "您即将删除此徽章。徽章被删除后 <strong>不能</strong> 恢复。" - - "Ви збираєтеся видалити цей значок. Вилучені значки <strong>не можуть</strong> бути відновлені." - - "Du bist gerade dabei dieses Badge zu entfernen. Entfernte Badges können <strong>nicht</strong> rückgängig gemacht werden." - - "이 배지를 삭제하려고합니다. 삭제 된 배지는 <strong>복원 할 수 없습니다</strong>." - - "Vous êtes sur le point de supprimer ce badge. Les badges supprimés <strong>ne peuvent pas</strong> être restaurés." - - "Va a eliminar esta insignia. Las insignias eliminadas <strong>no se pueden</strong> restaurar." - - "Bu rozeti sileceksiniz. Silinen rozetler geri <strong>yüklenemez</strong>." -"ClusterIntegration| This will permanently delete the following resources: <ul> <li>All installed applications and related resources</li> <li>The <code>gitlab-managed-apps</code> namespace</li> <li>Any project namespaces</li> <li><code>clusterroles</code></li> <li><code>clusterrolebindings</code></li> </ul>": - plural_id: - translations: - - "これにより、次のリソースは完全に削除されます <ul> <li>インストールされているすべてのアプリケーションと関連したリソース</li> <li> <code>gitlab-managed-apps</code> 名前空間</li> <li>任意のプロジェクト名前空間</li> <li><code>clusterroles</code></li> <li><code>clusterrolebindings</code></li> </ul>" - - "此操作将永久删除下列资源： <ul> <li>所有已安装的应用程序和相关资源</li> <li> <code>GitLab管理的应用</code> 命名空间</li> <li>任何项目命名空间</li> <li><code>clusterroles</code></li> <li><code>clusterrolebindings</code></li> </ul>" - - "Esto eliminará permanentemente los siguientes recursos: <ul> <li>Todas las aplicaciones instaladas y sus recursos relacionados</li> <li>El espacio de nombres <code>gitlab-managed-apps</code></li> <li>Cualquier espacio de nombres de proyecto</li> <li><code> clusterroles </code></li> <li><code>clusterrolebindings</code></li> </ul>" -"Configure a <code>.gitlab-webide.yml</code> file in the <code>.gitlab</code> directory to start using the Web Terminal. %{helpStart}Learn more.%{helpEnd}": - plural_id: - translations: - - "Configure um arquivo <code>.gitlab-webide.yml</code> no diretório <code>.gitlab</code> para começar a usar o Terminal Web. %{helpStart}Saiba mais.%{helpEnd}" - - "Webターミナルの使用を開始するには、 <code>.gitlab</code> ディレクトリの <code>.gitlab-webide.yml</code> ファイルを設定します。 詳細は%{helpStart}こちら%{helpEnd}です。" - - "Сконфигурируйте файл <code>.gitlab-webide.yml</code> в каталоге <code>.gitlab</code> чтобы начать использовать веб-терминал. %{helpStart}Узнайте больше.%{helpEnd}" - - "在 <code>.gitlab</code> 目录中配置 <code>.gitlab-webide.yml</code> 文件以开始使用Web终端。 %{helpStart}了解更多。%{helpEnd}" - - "Налаштуйте файл <code>.gitlab-webide.yml</code> у директорії <code>.gitlab</code>, щоб почати використовувати Веб-термінал. %{helpStart}Докладніше.%{helpEnd}" - - "웹 터미널 사용을 시작하도록 <code>.gitlab</code> 디렉토리에서 <code>.gitlab-webide.yml</code> 파일을 구성하십시오. %{helpStart}자세히 알아보십시오.%{helpEnd}" - - "Configure un archivo <code>.gitlab-webide.yml</code> en el directorio <code>.gitlab</code> para comenzar a utilizar el Terminal Web. %{helpStart}Aprende más.%{helpEnd}" -"Depends on <strong>%d closed</strong> merge request.": - plural_id: "Depends on <strong>%d closed</strong> merge requests." - translations: - - "В зависимости от <strong>%d закрытого</strong> запроса на слияние." - - "В зависимости от <strong>%d закрытых</strong> запросов на слияние." - - "В зависимости от <strong>%d закрытых</strong> запросов на слияние." - - "В зависимости от <strong>%d закрытых</strong> запросов на слияние." - - "依赖于<strong>%d个已关闭的</strong>合并请求" - - "Залежить від %d <strong>закритого</strong> запиту на злиття." - - "Залежить від %d <strong>закритих</strong> запитів на злиття." - - "Залежить від %d <strong>закритих</strong> запитів на злиття." - - "Залежить від %d <strong>закритих</strong> запитів на злиття." - - "<strong>%d kapanan</strong> birleştirme isteğine bağlıdır." - - "<strong>%d kapanan</strong> birleştirme isteğine bağlıdır." -"Go to <strong>Issues</strong> > <strong>Boards</strong> to access your personalized learning issue board.": - plural_id: - translations: - - "转至<strong>议题</strong> > <strong>看板</strong>访问您的个性化学习议题看板。" -"Labels|<span>Promote label</span> %{labelTitle} <span>to Group Label?</span>": - plural_id: - translations: - - "<span>要讓標籤</span> %{labelTitle} <span>提升到群組標籤嗎？</span>" - - "<span>Promover a etiqueta</span> %{labelTitle} <span>para etiqueta do Grupo?</span>" - - "%{labelTitle} <span>ラベルをグループラベルに昇格しますか？</span>" - - "<span>Повысить метку</span> %{labelTitle} <span>до групповой метки?</span>" - - "<span>将标记</span> %{labelTitle} <span>升级为群组标记？</span>" - - "<span>Перенести мітку</span> %{labelTitle} <span>на рівень групи?</span>" - - "<span>Label</span> %{labelTitle} <span>zu Gruppenlabel hochstufen?</span>" - - "<span>라벨</span> %{labelTitle} <span>(을)를 그룹 라벨로 승격하시겠습니까?</span>" - - "<span>Promouvoir l’étiquette</span> %{labelTitle} <span>en étiquette de groupe ?</span>" - - "<span>¿Promocionar la etiqueta</span> %{labelTitle} <span>a etiqueta de grupo?</span>" -"Lock this %{issuableDisplayName}? Only <strong>project members</strong> will be able to comment.": - plural_id: - translations: - - "Travar este %{issuableDisplayName}? Apenas <strong>membros do projeto</strong> poderão comentar." - - "%{issuableDisplayName} をロックしますか？<strong>プロジェクトメンバー</strong> のみコメントできます。" - - "锁定此%{issuableDisplayName}吗？锁定后将只有<strong>项目成员</strong>可以发表评论。" - - "Заблокувати цю %{issuableDisplayName}? Лише <strong>учасники проекту</strong> зможуть коментувати." - - "%{issuableDisplayName} sperren? Es werden nur noch <strong>Projektmitglieder</strong> kommentieren können." - - "Verrouiller ce·t·te %{issuableDisplayName} ? Seuls les <strong>membres du projet</strong> seront en mesure de commenter." - - "¿Bloquear este %{issuableDisplayName}? Sólo los <strong>miembros del proyecto</strong> podrán comentar." -"PrometheusService|<p class=\\\"text-tertiary\\\">No <a href=\\\"%{docsUrl}\\\">common metrics</a> were found</p>": - plural_id: - translations: - - "<p class=\\\"text-tertiary\\\">Nenhuma <a href=\\\"%{docsUrl}\\\">métrica comum</a> foi encontrada</p>" - - "<p class=\\\"text-tertiary\\\"><a href=\\\"%{docsUrl}\\\">共通メトリクス</a>は見つかりませんでした</p>" - - "<p class=\\\"text-tertiary\\\">Ни одной <a href=\\\"%{docsUrl}\\\">общей метрики</a> не найдено</p>" - - "<p class=\\\"text-tertiary\\\">无<a href=\\\"%{docsUrl}\\\">常用指标</a> </p>" - - "<p class=\\\"text-tertiary\\\">Ніяких <a href=\\\"%{docsUrl}\\\">загальних метрик</a> не знайдено</p>" - - "<p class=\\\"text-tertiary\\\">Es wurden keine <a href=\\\"%{docsUrl}\\\">allgemeinen Metriken</a> gefunden</p>" - - "<p class=\\\"text-tertiary\\\"><a href=\\\"%{docsUrl}\\\">공통 메트릭스</a>가 발견되지 않았습니다.</p>" - - "<p class=\\\"text-tertiary\\\">Aucune <a href=\\\"%{docsUrl}\\\">métrique commune</a> trouvée</p>" - - "<p class=\\\"text-tertiary\\\">No se han encontrado<a href=\\\"%{docsUrl}\\\">métricas comunes</a> </p>" -"This project does not have billing enabled. To create a cluster, <a href=%{linkToBilling} target=\\\"_blank\\\" rel=\\\"noopener noreferrer\\\">enable billing <i class=\\\"fa fa-external-link\\\" aria-hidden=\\\"true\\\"></i></a> and try again.": - plural_id: - translations: - - "Este projeto não possui faturamento ativado. Para criar um cluster, <a href=%{linkToBilling} target=\\\"_blank\\\" rel=\\\"noopener noreferrer\\\">ative o faturamento <i class=\\\"fa fa-external-link\\\" aria-hidden=\\\"true\\\"></i></a> e tente novamente." - - "このプロジェクトでは課金が有効になっていません。クラスターを作成するには、<a href=%{linkToBilling} target=\\\"_blank\\\" rel=\\\"noopener noreferrer\\\"> 課金を有効<i class=\\\"fa fa-external-link\\\" aria-hidden=\\\"true\\\"></i></a> にして再度お試しください。" - - "此项目未启用账单。要创建群集，请 <a href=%{linkToBilling} target=\\\"_blank\\\" rel=\\\"noopener noreferrer\\\">启用账单 <i class=\\\"fa fa-external-link\\\" aria-hidden=\\\"true\\\"></i></a> 并重试。" - - "Для цього проекту вимкнено білінг. Щоб створити кластер, <a href=%{linkToBilling} target=\\\"_blank\\\" rel=\\\"noopener noreferrer\\\">увімкніть білінг <i class=\\\"fa fa-external-link\\\" aria-hidden=\\\"true\\\"></i></a> і спробуйте знову." - - "Für dieses Projekt ist keine Abrechnung aktiviert. Um ein Cluster zu erstellen, <a href=%{linkToBilling} target=\\\"_blank\\\" rel=\\\"noopener noreferrer\\\">aktiviere die Abrechnung<i class=\\\"fa fa-external-link\\\" aria-hidden=\\\"true\\\"></i></a> und versuche es erneut." - - "Ce projet n’a pas de facturation activée. Afin de créer une grappe de serveurs, veuillez <a href=%{linkToBilling} target=\\\"_blank\\\" rel=\\\"noopener noreferrer\\\">activer la facturation<i class=\\\"fa fa-external-link\\\" aria-hidden=\\\"true\\\"></i></a> et réessayer." - - "Este proyecto no tiene la facturación habilitada. Para crear un clúster, <a href=%{linkToBilling} target=\\\"_blank\\\" rel=\\\"noopener noreferrer\\\">habilite la facturación <i class=\\\"fa fa-external-link\\\" aria-hidden=\\\"true\\\"></i></a> e inténtelo de nuevo." -"Unlock this %{issuableDisplayName}? <strong>Everyone</strong> will be able to comment.": - plural_id: - translations: - - "Desbloquear este %{issuableDisplayName}? <strong>Todos</strong> poderão comentar." - - "%{issuableDisplayName} のロックを解除しますか？ <strong>全員</strong>がコメントできるようになります。" - - "解锁此%{issuableDisplayName}吗？解锁后<strong>所有人</strong>都将可以发表评论。" - - "Розблокувати %{issuableDisplayName}? <strong>Будь-хто</strong> зможе залишати коментарі." - - "Dieses %{issuableDisplayName} entsperren? <strong>Jeder</strong> wird in der Lage sein zu kommentieren." - - "%{issuableDisplayName}(을)를 잠금해제 하시겠습니까? <strong>모두가</strong> 코멘트 할 수 있게 됩니다." - - "Déverrouiller %{issuableDisplayName} ? <strong>Tout le monde</strong> sera en mesure de commenter." - - "Desbloquear este %{issuableDisplayName}? <strong>Todos</strong> podrán comentar." -"confidentiality|You are going to turn off the confidentiality. This means <strong>everyone</strong> will be able to see and leave a comment on this issue.": - plural_id: - translations: - - "Você está prestes a desligar a confidencialidade. Isso significa que <strong>todos</strong> serão capazes de ver e deixar comentários nesse issue." - - "あなたは公開設定に変更しようとしています。これは<strong>すべての人</strong> が閲覧可能になり、課題に対してコメントを残すことができるようになることを意味します。" - - "即将关闭私密性。这将使得 <strong>所有用户</strong>都可以查看并且评论当前议题。" - - "Ви вимикаєте конфіденційність. Це означає, що <strong>будь-хто</strong> зможе бачити і залишати коментарі для цієї задачі." - - "Du willst die Vertraulichkeit deaktivieren. Das bedeutet, dass <strong>alle</strong> das Ticket betrachten und kommentieren können." - - "Vous êtes sur le point de désactiver la confidentialité. Cela signifie que <strong>tout le monde</strong> sera en mesure de voir et de laisser un commentaire sur ce ticket." - - "Va a desactivar la confidencialidad. Esto significa que <strong>todos</strong> podrán ver y dejar un comentario sobre este tema." -"confidentiality|You are going to turn on the confidentiality. This means that only team members with <strong>at least Reporter access</strong> are able to see and leave comments on the issue.": - plural_id: - translations: - - "Você está prestes a ligar a confidencialidade. Isso significa que apenas membros da equipe com <strong>ao menos acesso de Relator</strong> serão capazes de ver e deixar comentários nesse issue." - - "あなたは公開設定に変更しようとしています。これはチームに限定していた<strong>最小限の報告権限</strong>をなくし、課題に対してコメントを残すことができるようになることを意味します。" - - "即将设置私密性。这将使得 <strong>至少有Reporter以上权限</strong>的团队成员才能查看并且评论当前议题。" - - "Ви вмикаєте конфіденційність. Це означає що лише учасники команди <strong>рівня репортер або вище</strong> матимуть змогу бачити та залишати коментарі для цієї задачі." - - "Du willst die Vertraulichkeit aktivieren. Das bedeutet, dass nur Teammitglieder mit <strong>mindestens Reporter-Zugriff</strong> das Ticket betrachten und kommentieren können." - - "Vous êtes sur le point de d’activer la confidentialité. Cela signifie que seuls les membres de l’équipe avec <strong>au moins un accès en tant que rapporteur</strong> seront en mesure de voir et de laisser des commentaires sur le ticket." - - "Va a activar la confidencialidad. Esto significa que solo los miembros del equipo con como mínimo,<strong>acceso como Reporter</strong> podrán ver y dejar comentarios sobre la incidencia." - - "あなたは非公開設定をオンにしようとしています。これは、最低でも<strong>報告権限</strong>を持ったチームメンバーのみが課題を表示したりコメントを残したりすることができるようになるということです。" -" or <!merge request id>": - translations: - - " ወይም <!merge request id>" - - " ou <!merge request id>" - - " または <!merge request id>" - - "或 <!合併請求 id>" - - " или <!merge request id>" - - "或<!merge request id>" - - " або <!merge request id>" - - " oder <!merge request id>" - - " o <!merge request id>" - - " 또는 <!merge request id>" - - " o <!merge request id>" - - " veya <!merge request id>" - - " neu <!merge request id>" - - " neu <#issue id>" -" or <#issue id>": - translations: - - "或 <#issue id>" - - " ወይም ‹#issue id›" - - " ou <identificación #issue>" - - " ou <#issue id>" - - " または <#課題 ID>" - - " o <#issue id>" - - "或 <#議題 id>" - - " ou <#issue id>" - - " или <#issue id>" - - "或 <#issue id>" - - " або <#issue id>" - - " oder <#issue id>" - - " o <#issue id>" - - " 또는 <#issue id>" - - " ou <#issue id>" - - " o <#issue id>" - - " veya <#issue id>" - - " neu <#issue id>" -" or <&epic id>": - translations: - - " ወይም <&epic id>" - - " または <&エピックID>" - - " 或 <#史詩 id>" - - " или <&epic id>" - - " 或<#epic id>" - - " або <&epic id>" - - " oder <&epic id>" - - " o <&epic id>" - - " veya <&epic id>" - - " neu <#epic id>" - - " 또는 <&epic id>" -"< 1 hour": - translations: - - "1 時間未満" - - "< 1 小時" - - "< 1 часа" - - "< 1小时" - - "< 1 години" - - "< 1 hora" - - "< 1 saat" - - "< 1 Stunde" - - "< 1시간" diff --git a/lib/gitlab/import_export/import_failure_service.rb b/lib/gitlab/import_export/import_failure_service.rb index d4eca551b49..bf7200726a1 100644 --- a/lib/gitlab/import_export/import_failure_service.rb +++ b/lib/gitlab/import_export/import_failure_service.rb @@ -28,23 +28,26 @@ module Gitlab end def log_import_failure(source:, relation_key: nil, relation_index: nil, exception:, retry_count: 0) - extra = { - source: source, - relation_key: relation_key, + attributes = { relation_index: relation_index, - retry_count: retry_count + source: source, + retry_count: retry_count, + importable_column_name => importable.id } - extra[importable_column_name] = importable.id - - Gitlab::ErrorTracking.track_exception(exception, extra) - - attributes = { - exception_class: exception.class.to_s, - exception_message: exception.message.truncate(255), - correlation_id_value: Labkit::Correlation::CorrelationId.current_or_new_id - }.merge(extra) - ImportFailure.create(attributes) + Gitlab::ErrorTracking.track_exception( + exception, + attributes.merge(relation_name: relation_key) + ) + + ImportFailure.create( + attributes.merge( + exception_class: exception.class.to_s, + exception_message: exception.message.truncate(255), + correlation_id_value: Labkit::Correlation::CorrelationId.current_or_new_id, + relation_key: relation_key + ) + ) end private diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index ae7ddbc5eba..8c094603d53 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -169,6 +169,7 @@ excluded_attributes: - :compliance_framework_setting - :show_default_award_emojis - :services + - :exported_protected_branches namespaces: - :runners_token - :runners_token_encrypted diff --git a/lib/gitlab/kubernetes/helm/v2/client_command.rb b/lib/gitlab/kubernetes/helm/v2/client_command.rb index 88693a28d6c..8b15af9aeea 100644 --- a/lib/gitlab/kubernetes/helm/v2/client_command.rb +++ b/lib/gitlab/kubernetes/helm/v2/client_command.rb @@ -22,17 +22,6 @@ module Gitlab def repository_update_command 'helm repo update' end - - def optional_tls_flags - return [] unless files.key?(:'ca.pem') - - [ - '--tls', - '--tls-ca-cert', "#{files_dir}/ca.pem", - '--tls-cert', "#{files_dir}/cert.pem", - '--tls-key', "#{files_dir}/key.pem" - ] - end end end end diff --git a/lib/gitlab/kubernetes/helm/v2/reset_command.rb b/lib/gitlab/kubernetes/helm/v2/reset_command.rb index 172a0884c49..00626501a9a 100644 --- a/lib/gitlab/kubernetes/helm/v2/reset_command.rb +++ b/lib/gitlab/kubernetes/helm/v2/reset_command.rb @@ -9,9 +9,8 @@ module Gitlab def generate_script super + [ - reset_helm_command, - delete_tiller_replicaset, - delete_tiller_clusterrolebinding + init_command, + reset_helm_command ].join("\n") end @@ -21,27 +20,8 @@ module Gitlab private - # This method can be delete once we upgrade Helm to > 12.13.0 - # https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/27096#note_159695900 - # - # Tracking this method to be removed here: - # https://gitlab.com/gitlab-org/gitlab-foss/issues/52791#note_199374155 - def delete_tiller_replicaset - delete_args = %w[replicaset -n gitlab-managed-apps -l name=tiller] - - Gitlab::Kubernetes::KubectlCmd.delete(*delete_args) - end - - def delete_tiller_clusterrolebinding - delete_args = %w[clusterrolebinding tiller-admin] - - Gitlab::Kubernetes::KubectlCmd.delete(*delete_args) - end - def reset_helm_command - command = %w[helm reset] + optional_tls_flags - - command.shelljoin + 'helm reset --force' end end end diff --git a/lib/gitlab/rack_attack.rb b/lib/gitlab/rack_attack.rb new file mode 100644 index 00000000000..222f5d57adf --- /dev/null +++ b/lib/gitlab/rack_attack.rb @@ -0,0 +1,98 @@ +# frozen_string_literal: true + +# Integration specs for throttling can be found in: +# spec/requests/rack_attack_global_spec.rb +module Gitlab + module RackAttack + def self.configure(rack_attack) + # This adds some methods used by our throttles to the `Rack::Request` + rack_attack::Request.include(Gitlab::RackAttack::Request) + # Configure the throttles + configure_throttles(rack_attack) + end + + def self.configure_throttles(rack_attack) + throttle_or_track(rack_attack, 'throttle_unauthenticated', Gitlab::Throttle.unauthenticated_options) do |req| + if !req.should_be_skipped? && + Gitlab::Throttle.settings.throttle_unauthenticated_enabled && + req.unauthenticated? + req.ip + end + end + + throttle_or_track(rack_attack, 'throttle_authenticated_api', Gitlab::Throttle.authenticated_api_options) do |req| + if req.api_request? && + Gitlab::Throttle.settings.throttle_authenticated_api_enabled + req.authenticated_user_id([:api]) + end + end + + # Product analytics feature is in experimental stage. + # At this point we want to limit amount of events registered + # per application (aid stands for application id). + throttle_or_track(rack_attack, 'throttle_product_analytics_collector', limit: 100, period: 60) do |req| + if req.product_analytics_collector_request? + req.params['aid'] + end + end + + throttle_or_track(rack_attack, 'throttle_authenticated_web', Gitlab::Throttle.authenticated_web_options) do |req| + if req.web_request? && + Gitlab::Throttle.settings.throttle_authenticated_web_enabled + req.authenticated_user_id([:api, :rss, :ics]) + end + end + + throttle_or_track(rack_attack, 'throttle_unauthenticated_protected_paths', Gitlab::Throttle.protected_paths_options) do |req| + if req.post? && + !req.should_be_skipped? && + req.protected_path? && + Gitlab::Throttle.protected_paths_enabled? && + req.unauthenticated? + req.ip + end + end + + throttle_or_track(rack_attack, 'throttle_authenticated_protected_paths_api', Gitlab::Throttle.protected_paths_options) do |req| + if req.post? && + req.api_request? && + req.protected_path? && + Gitlab::Throttle.protected_paths_enabled? + req.authenticated_user_id([:api]) + end + end + + throttle_or_track(rack_attack, 'throttle_authenticated_protected_paths_web', Gitlab::Throttle.protected_paths_options) do |req| + if req.post? && + req.web_request? && + req.protected_path? && + Gitlab::Throttle.protected_paths_enabled? + req.authenticated_user_id([:api, :rss, :ics]) + end + end + + rack_attack.safelist('throttle_bypass_header') do |req| + Gitlab::Throttle.bypass_header.present? && + req.get_header(Gitlab::Throttle.bypass_header) == '1' + end + end + + def self.throttle_or_track(rack_attack, throttle_name, *args, &block) + if track?(throttle_name) + rack_attack.track(throttle_name, *args, &block) + else + rack_attack.throttle(throttle_name, *args, &block) + end + end + + def self.track?(name) + dry_run_config = ENV['GITLAB_THROTTLE_DRY_RUN'].to_s.strip + + return false if dry_run_config.empty? + return true if dry_run_config == '*' + + dry_run_config.split(',').map(&:strip).include?(name) + end + end +end +::Gitlab::RackAttack.prepend_if_ee('::EE::Gitlab::RackAttack') diff --git a/lib/gitlab/rack_attack/request.rb b/lib/gitlab/rack_attack/request.rb new file mode 100644 index 00000000000..a75f66af263 --- /dev/null +++ b/lib/gitlab/rack_attack/request.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module Gitlab + module RackAttack + module Request + def unauthenticated? + !(authenticated_user_id([:api, :rss, :ics]) || authenticated_runner_id) + end + + def authenticated_user_id(request_formats) + request_authenticator.user(request_formats)&.id + end + + def authenticated_runner_id + request_authenticator.runner&.id + end + + def api_request? + path.start_with?('/api') + end + + def api_internal_request? + path =~ %r{^/api/v\d+/internal/} + end + + def health_check_request? + path =~ %r{^/-/(health|liveness|readiness|metrics)} + end + + def product_analytics_collector_request? + path.start_with?('/-/collector/i') + end + + def should_be_skipped? + api_internal_request? || health_check_request? + end + + def web_request? + !api_request? && !health_check_request? + end + + def protected_path? + !protected_path_regex.nil? + end + + def protected_path_regex + path =~ protected_paths_regex + end + + private + + def request_authenticator + @request_authenticator ||= Gitlab::Auth::RequestAuthenticator.new(self) + end + + def protected_paths + Gitlab::CurrentSettings.current_application_settings.protected_paths + end + + def protected_paths_regex + Regexp.union(protected_paths.map { |path| /\A#{Regexp.escape(path)}/ }) + end + end + end +end +::Gitlab::RackAttack::Request.prepend_if_ee('::EE::Gitlab::RackAttack::Request') diff --git a/lib/gitlab/throttle.rb b/lib/gitlab/throttle.rb new file mode 100644 index 00000000000..aebf8d92cb3 --- /dev/null +++ b/lib/gitlab/throttle.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + class Throttle + def self.settings + Gitlab::CurrentSettings.current_application_settings + end + + # Returns true if we should use the Admin Area protected paths throttle + def self.protected_paths_enabled? + self.settings.throttle_protected_paths_enabled? + end + + def self.omnibus_protected_paths_present? + Rack::Attack.throttles.key?('protected paths') + end + + def self.bypass_header + env_value = ENV['GITLAB_THROTTLE_BYPASS_HEADER'] + return unless env_value.present? + + "HTTP_#{env_value.upcase.tr('-', '_')}" + end + + def self.unauthenticated_options + limit_proc = proc { |req| settings.throttle_unauthenticated_requests_per_period } + period_proc = proc { |req| settings.throttle_unauthenticated_period_in_seconds.seconds } + { limit: limit_proc, period: period_proc } + end + + def self.authenticated_api_options + limit_proc = proc { |req| settings.throttle_authenticated_api_requests_per_period } + period_proc = proc { |req| settings.throttle_authenticated_api_period_in_seconds.seconds } + { limit: limit_proc, period: period_proc } + end + + def self.authenticated_web_options + limit_proc = proc { |req| settings.throttle_authenticated_web_requests_per_period } + period_proc = proc { |req| settings.throttle_authenticated_web_period_in_seconds.seconds } + { limit: limit_proc, period: period_proc } + end + + def self.protected_paths_options + limit_proc = proc { |req| settings.throttle_protected_paths_requests_per_period } + period_proc = proc { |req| settings.throttle_protected_paths_period_in_seconds.seconds } + + { limit: limit_proc, period: period_proc } + end + end +end diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb index 19be468e3d5..461b5dc3afc 100644 --- a/lib/gitlab/tracking.rb +++ b/lib/gitlab/tracking.rb @@ -26,6 +26,7 @@ module Gitlab def event(category, action, label: nil, property: nil, value: nil, context: nil) snowplow.event(category, action, label: label, property: property, value: value, context: context) + product_analytics.event(category, action, label: label, property: property, value: value, context: context) end def self_describing_event(schema_url, event_data_json, context: nil) @@ -49,6 +50,10 @@ module Gitlab def snowplow @snowplow ||= Gitlab::Tracking::Destinations::Snowplow.new end + + def product_analytics + @product_analytics ||= Gitlab::Tracking::Destinations::ProductAnalytics.new + end end end end diff --git a/lib/gitlab/tracking/destinations/product_analytics.rb b/lib/gitlab/tracking/destinations/product_analytics.rb new file mode 100644 index 00000000000..cacedbc5b83 --- /dev/null +++ b/lib/gitlab/tracking/destinations/product_analytics.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Tracking + module Destinations + class ProductAnalytics < Base + extend ::Gitlab::Utils::Override + include ::Gitlab::Utils::StrongMemoize + + override :event + def event(category, action, label: nil, property: nil, value: nil, context: nil) + return unless event_allowed?(category, action) + return unless enabled? + + tracker.track_struct_event(category, action, label, property, value, context, (Time.now.to_f * 1000).to_i) + end + + private + + def event_allowed?(category, action) + category == 'epics' && action == 'promote' + end + + def enabled? + Feature.enabled?(:product_analytics_tracking, type: :ops) && + Gitlab::CurrentSettings.usage_ping_enabled? && + Gitlab::CurrentSettings.self_monitoring_project_id.present? + end + + def tracker + @tracker ||= SnowplowTracker::Tracker.new( + SnowplowTracker::AsyncEmitter.new(::ProductAnalytics::Tracker::COLLECTOR_URL, protocol: Gitlab.config.gitlab.protocol), + SnowplowTracker::Subject.new, + Gitlab::Tracking::SNOWPLOW_NAMESPACE, + Gitlab::CurrentSettings.self_monitoring_project_id.to_s + ) + end + end + end + end +end diff --git a/lib/gitlab/uploads/migration_helper.rb b/lib/gitlab/uploads/migration_helper.rb index 9377ccfec1e..b610d2a10c6 100644 --- a/lib/gitlab/uploads/migration_helper.rb +++ b/lib/gitlab/uploads/migration_helper.rb @@ -75,3 +75,5 @@ module Gitlab end end end + +Gitlab::Uploads::MigrationHelper.prepend_if_ee('EE::Gitlab::Uploads::MigrationHelper') diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 4b0dd54683b..77a74c86c63 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -296,20 +296,7 @@ module Gitlab # @return [Array<#totals>] An array of objects that respond to `#totals` def usage_data_counters - [ - Gitlab::UsageDataCounters::WikiPageCounter, - Gitlab::UsageDataCounters::WebIdeCounter, - Gitlab::UsageDataCounters::NoteCounter, - Gitlab::UsageDataCounters::SnippetCounter, - Gitlab::UsageDataCounters::SearchCounter, - Gitlab::UsageDataCounters::CycleAnalyticsCounter, - Gitlab::UsageDataCounters::ProductivityAnalyticsCounter, - Gitlab::UsageDataCounters::SourceCodeCounter, - Gitlab::UsageDataCounters::MergeRequestCounter, - Gitlab::UsageDataCounters::DesignsCounter, - Gitlab::UsageDataCounters::KubernetesAgentCounter, - Gitlab::UsageDataCounters::StaticSiteEditorCounter - ] + Gitlab::UsageDataCounters.counters end def components_usage_data diff --git a/lib/gitlab/usage_data_counters.rb b/lib/gitlab/usage_data_counters.rb new file mode 100644 index 00000000000..79f0cd117bc --- /dev/null +++ b/lib/gitlab/usage_data_counters.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + COUNTERS = [ + WikiPageCounter, + WebIdeCounter, + NoteCounter, + SnippetCounter, + SearchCounter, + CycleAnalyticsCounter, + ProductivityAnalyticsCounter, + SourceCodeCounter, + MergeRequestCounter, + DesignsCounter, + KubernetesAgentCounter, + StaticSiteEditorCounter + ].freeze + + UsageDataCounterError = Class.new(StandardError) + UnknownEvent = Class.new(UsageDataCounterError) + + class << self + def counters + self::COUNTERS + end + + def count(event_name) + counters.each do |counter| + event = counter.fetch_supported_event(event_name) + + return counter.count(event) if event + end + + raise UnknownEvent, "Cannot find counter for event #{event_name}" + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml b/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml index 97ec8423b95..dd43b60bd45 100644 --- a/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml +++ b/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml @@ -15,3 +15,28 @@ - name: product_analytics_test_metrics_intersection operator: AND events: ['i_search_total', 'i_search_advanced', 'i_search_paid'] +- name: incident_management_alerts_total_unique_counts + operator: OR + events: [ + 'incident_management_alert_status_changed', + 'incident_management_alert_assigned', + 'incident_management_alert_todo', + 'incident_management_alert_create_incident' + ] + feature_flag: usage_data_incident_management_alerts_total_unique_counts +- name: incident_management_incidents_total_unique_counts + operator: OR + events: [ + 'incident_management_incident_created', + 'incident_management_incident_reopened', + 'incident_management_incident_closed', + 'incident_management_incident_assigned', + 'incident_management_incident_todo', + 'incident_management_incident_comment', + 'incident_management_incident_zoom_meeting', + 'incident_management_incident_published', + 'incident_management_incident_relate', + 'incident_management_incident_unrelate', + 'incident_management_incident_change_confidential' + ] + feature_flag: usage_data_incident_management_incidents_total_unique_counts diff --git a/lib/gitlab/usage_data_counters/base_counter.rb b/lib/gitlab/usage_data_counters/base_counter.rb index 44893645cc2..d28fd17a989 100644 --- a/lib/gitlab/usage_data_counters/base_counter.rb +++ b/lib/gitlab/usage_data_counters/base_counter.rb @@ -29,6 +29,12 @@ module Gitlab::UsageDataCounters known_events.map { |event| [counter_key(event), -1] }.to_h end + def fetch_supported_event(event_name) + return if prefix.present? && !event_name.start_with?(prefix) + + known_events.find { |event| counter_key(event) == event_name.to_sym } + end + private def require_known_event(event) diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index 85f16ea807b..58b023d374c 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -229,6 +229,12 @@ category: incident_management aggregation: weekly feature_flag: usage_data_incident_management_incident_change_confidential +# Incident management alerts +- name: incident_management_alert_create_incident + redis_slot: incident_management + category: incident_management_alerts + aggregation: weekly + feature_flag: usage_data_incident_management_alert_create_incident # Testing category - name: i_testing_test_case_parsed category: testing diff --git a/lib/gitlab/usage_data_counters/search_counter.rb b/lib/gitlab/usage_data_counters/search_counter.rb index 61f98887adc..46aec52b95a 100644 --- a/lib/gitlab/usage_data_counters/search_counter.rb +++ b/lib/gitlab/usage_data_counters/search_counter.rb @@ -4,6 +4,7 @@ module Gitlab module UsageDataCounters class SearchCounter < BaseCounter KNOWN_EVENTS = %w[all_searches navbar_searches].freeze + PREFIX = nil class << self def redis_key(event) |