diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-03-12 19:26:10 +0300 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-03-12 19:26:10 +0300 |
| commit | 6653ccc011dec86e5140a5d09ea3b2357eab6714 (patch) | |
| tree | 897193f37bcd98152a0ac214f80a3c4cfe1047c5 /lib/gitlab | |
| parent | bff35a05aed6a31380a73c39113808fd262c2c37 (diff) | |
Add latest changes from gitlab-org/gitlab@13-10-stable-eev13.10.0-rc41
Diffstat (limited to 'lib/gitlab')
210 files changed, 3979 insertions, 1137 deletions
diff --git a/lib/gitlab/alert_management/payload/generic.rb b/lib/gitlab/alert_management/payload/generic.rb index 0eb1bee8181..e2db9b62dd5 100644 --- a/lib/gitlab/alert_management/payload/generic.rb +++ b/lib/gitlab/alert_management/payload/generic.rb @@ -5,7 +5,7 @@ module Gitlab module AlertManagement module Payload class Generic < Base - DEFAULT_TITLE = 'New: Incident' + DEFAULT_TITLE = 'New: Alert' attribute :description, paths: 'description' attribute :ends_at, paths: 'end_time', type: :time diff --git a/lib/gitlab/analytics/cycle_analytics/average.rb b/lib/gitlab/analytics/cycle_analytics/average.rb new file mode 100644 index 00000000000..a449b71b165 --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/average.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + class Average + include Gitlab::Utils::StrongMemoize + include StageQueryHelpers + + def initialize(stage:, query:) + @stage = stage + @query = query + end + + def seconds + select_average ? select_average['average'] : nil + end + + def days + seconds ? seconds.fdiv(1.day) : nil + end + + private + + attr_reader :stage + + # rubocop: disable CodeReuse/ActiveRecord + def select_average + strong_memoize(:select_average) do + execute_query(@query.select(average_in_seconds.as('average')).reorder(nil)).first + end + end + # rubocop: enable CodeReuse/ActiveRecord + + def average + Arel::Nodes::NamedFunction.new( + 'AVG', + [duration] + ) + end + + def average_in_seconds + Arel::Nodes::Extract.new(average, :epoch) + end + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/data_collector.rb b/lib/gitlab/analytics/cycle_analytics/data_collector.rb index 5eca364a697..10a008a76d5 100644 --- a/lib/gitlab/analytics/cycle_analytics/data_collector.rb +++ b/lib/gitlab/analytics/cycle_analytics/data_collector.rb @@ -31,6 +31,12 @@ module Gitlab end end + def average + strong_memoize(:average) do + Average.new(stage: stage, query: query) + end + end + private attr_reader :stage, :params diff --git a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb index be5d9be3d64..178ebe0d4d4 100644 --- a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb +++ b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb @@ -29,6 +29,8 @@ module Gitlab @stage = stage @query = query @params = params + @sort = params[:sort] || :end_event + @direction = params[:direction] || :desc end def serialized_records @@ -52,7 +54,7 @@ module Gitlab private - attr_reader :stage, :query, :params + attr_reader :stage, :query, :params, :sort, :direction def columns MAPPINGS.fetch(subject_class).fetch(:columns_for_select).map do |column_name| @@ -90,7 +92,7 @@ module Gitlab end def ordered_and_limited_query - order_by_end_event(query, columns).limit(MAX_RECORDS) + order_by(query, sort, direction, columns).limit(MAX_RECORDS) end def records diff --git a/lib/gitlab/analytics/cycle_analytics/sorting.rb b/lib/gitlab/analytics/cycle_analytics/sorting.rb new file mode 100644 index 00000000000..828879d466d --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/sorting.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + class Sorting + # rubocop: disable CodeReuse/ActiveRecord + SORTING_OPTIONS = { + end_event: { + asc: -> (query, stage) { query.reorder(stage.end_event.timestamp_projection.asc) }, + desc: -> (query, stage) { query.reorder(stage.end_event.timestamp_projection.desc) } + }.freeze, + duration: { + asc: -> (query, stage) { query.reorder(Arel::Nodes::Subtraction.new(stage.end_event.timestamp_projection, stage.start_event.timestamp_projection).asc) }, + desc: -> (query, stage) { query.reorder(Arel::Nodes::Subtraction.new(stage.end_event.timestamp_projection, stage.start_event.timestamp_projection).desc) } + }.freeze + }.freeze + # rubocop: enable CodeReuse/ActiveRecord, + + def self.apply(query, stage, sort, direction) + sort_lambda = SORTING_OPTIONS.dig(sort, direction) || SORTING_OPTIONS.dig(:end_event, :desc) + sort_lambda.call(query, stage) + end + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events.rb b/lib/gitlab/analytics/cycle_analytics/stage_events.rb index 27fc8bd9a1a..02b1024b8b3 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events.rb @@ -8,10 +8,12 @@ module Gitlab # Issue: < 100 # MergeRequest: >= 100 && < 1000 # Custom events for default stages: >= 1000 (legacy) + # + # To avoid duplications, verify that the value does not exist in ee/lib/ee/gitlab/analytics/cycle_analytics/stage_events.rb ENUM_MAPPING = { StageEvents::IssueCreated => 1, StageEvents::IssueFirstMentionedInCommit => 2, - StageEvents::IssueDeployedToProduction => 3, + StageEvents::IssueDeployedToProduction => 10, StageEvents::MergeRequestCreated => 100, StageEvents::MergeRequestFirstDeployedToProduction => 101, StageEvents::MergeRequestLastBuildFinished => 102, diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb index 79738747e71..cfc9300a710 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb @@ -19,6 +19,10 @@ module Gitlab raise NotImplementedError end + def markdown_description + self.class.name + end + def self.identifier raise NotImplementedError end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb b/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb index 80e426e6e17..777a8278e6e 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb @@ -24,8 +24,8 @@ module Gitlab end # rubocop: disable CodeReuse/ActiveRecord - def order_by_end_event(query, extra_columns_to_select = [:id]) - ordered_query = query.reorder(stage.end_event.timestamp_projection.desc) + def order_by(query, sort, direction, extra_columns_to_select = [:id]) + ordered_query = Gitlab::Analytics::CycleAnalytics::Sorting.apply(query, stage, sort, direction) # When filtering for more than one label, postgres requires the columns in ORDER BY to be present in the GROUP BY clause if requires_grouping? diff --git a/lib/gitlab/analytics/instance_statistics/workers_argument_builder.rb b/lib/gitlab/analytics/usage_trends/workers_argument_builder.rb index 54b3bbb3ce6..a502f46287d 100644 --- a/lib/gitlab/analytics/instance_statistics/workers_argument_builder.rb +++ b/lib/gitlab/analytics/usage_trends/workers_argument_builder.rb @@ -2,7 +2,7 @@ module Gitlab module Analytics - module InstanceStatistics + module UsageTrends class WorkersArgumentBuilder def initialize(measurement_identifiers: [], recorded_at: Time.zone.now) @measurement_identifiers = measurement_identifiers @@ -35,11 +35,11 @@ module Gitlab end def custom_min_max_queries - ::Analytics::InstanceStatistics::Measurement.identifier_min_max_queries + ::Analytics::UsageTrends::Measurement.identifier_min_max_queries end def query_mappings - ::Analytics::InstanceStatistics::Measurement.identifier_query_mapping + ::Analytics::UsageTrends::Measurement.identifier_query_mapping end end end diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb index cefe983848c..a75da3a682b 100644 --- a/lib/gitlab/application_context.rb +++ b/lib/gitlab/application_context.rb @@ -4,6 +4,7 @@ module Gitlab # A GitLab-rails specific accessor for `Labkit::Logging::ApplicationContext` class ApplicationContext include Gitlab::Utils::LazyAttributes + include Gitlab::Utils::StrongMemoize Attribute = Struct.new(:name, :type) @@ -11,6 +12,7 @@ module Gitlab Attribute.new(:project, Project), Attribute.new(:namespace, Namespace), Attribute.new(:user, User), + Attribute.new(:runner, ::Ci::Runner), Attribute.new(:caller_id, String), Attribute.new(:remote_ip, String), Attribute.new(:related_class, String), @@ -27,8 +29,12 @@ module Gitlab Labkit::Context.push(application_context.to_lazy_hash) end + def self.current + Labkit::Context.current.to_h + end + def self.current_context_include?(attribute_name) - Labkit::Context.current.to_h.include?(Labkit::Context.log_key(attribute_name)) + current.include?(Labkit::Context.log_key(attribute_name)) end def initialize(**args) @@ -43,8 +49,9 @@ module Gitlab def to_lazy_hash {}.tap do |hash| hash[:user] = -> { username } if set_values.include?(:user) - hash[:project] = -> { project_path } if set_values.include?(:project) + hash[:project] = -> { project_path } if set_values.include?(:project) || set_values.include?(:runner) hash[:root_namespace] = -> { root_namespace_path } if include_namespace? + hash[:client_id] = -> { client } if include_client? hash[:caller_id] = caller_id if set_values.include?(:caller_id) hash[:remote_ip] = remote_ip if set_values.include?(:remote_ip) hash[:related_class] = related_class if set_values.include?(:related_class) @@ -71,7 +78,8 @@ module Gitlab end def project_path - project&.full_path + associated_routable = project || runner_project + associated_routable&.full_path end def username @@ -79,15 +87,43 @@ module Gitlab end def root_namespace_path - if namespace - namespace.full_path_components.first + associated_routable = namespace || project || runner_project || runner_group + associated_routable&.full_path_components&.first + end + + def include_namespace? + set_values.include?(:namespace) || set_values.include?(:project) || set_values.include?(:runner) + end + + def include_client? + set_values.include?(:user) || set_values.include?(:runner) || set_values.include?(:remote_ip) + end + + def client + if user + "user/#{user.id}" + elsif runner + "runner/#{runner.id}" else - project&.full_path_components&.first + "ip/#{remote_ip}" end end - def include_namespace? - set_values.include?(:namespace) || set_values.include?(:project) + def runner_project + strong_memoize(:runner_project) do + next unless runner&.project_type? + + projects = runner.projects.take(2) # rubocop: disable CodeReuse/ActiveRecord + projects.first if projects.one? + end + end + + def runner_group + strong_memoize(:runner_group) do + next unless runner&.group_type? + + runner.groups.first + end end end end diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index 0a69a9c503d..f74edf2b767 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -47,7 +47,7 @@ module Gitlab # @option scope [Array<ActiveRecord>] Array of ActiveRecord models to scope throttling to a specific request (e.g. per user per project) # @option threshold [Integer] Optional threshold value to override default one registered in `.rate_limits` # @option interval [Integer] Optional interval value to override default one registered in `.rate_limits` - # @option users_allowlist [Array<String>] Optional list of usernames to excepted from the limit. This param will only be functional if Scope includes a current user. + # @option users_allowlist [Array<String>] Optional list of usernames to exclude from the limit. This param will only be functional if Scope includes a current user. # # @return [Boolean] Whether or not a request should be throttled def throttled?(key, **options) diff --git a/lib/gitlab/auth/current_user_mode.rb b/lib/gitlab/auth/current_user_mode.rb index 0f327a39f61..a6d706c2a49 100644 --- a/lib/gitlab/auth/current_user_mode.rb +++ b/lib/gitlab/auth/current_user_mode.rb @@ -77,7 +77,7 @@ module Gitlab return false unless user Gitlab::SafeRequestStore.fetch(admin_mode_rs_key) do - user.admin? && session_with_admin_mode? + user.admin? && (privileged_runtime? || session_with_admin_mode?) end end @@ -154,6 +154,11 @@ module Gitlab Gitlab::SafeRequestStore.delete(admin_mode_rs_key) Gitlab::SafeRequestStore.delete(admin_mode_requested_rs_key) end + + # Runtimes which imply shell access get admin mode automatically, see Gitlab::Runtime + def privileged_runtime? + Gitlab::Runtime.rake? || Gitlab::Runtime.rails_runner? || Gitlab::Runtime.console? + end end end end diff --git a/lib/gitlab/auth/o_auth/provider.rb b/lib/gitlab/auth/o_auth/provider.rb index 57ff3fcd1f0..ab6ac815601 100644 --- a/lib/gitlab/auth/o_auth/provider.rb +++ b/lib/gitlab/auth/o_auth/provider.rb @@ -5,11 +5,12 @@ module Gitlab module OAuth class Provider LABELS = { - "github" => "GitHub", - "gitlab" => "GitLab.com", - "google_oauth2" => "Google", - "azure_oauth2" => "Azure AD", - 'atlassian_oauth2' => 'Atlassian' + "github" => "GitHub", + "gitlab" => "GitLab.com", + "google_oauth2" => "Google", + "azure_oauth2" => "Azure AD", + "azure_activedirectory_v2" => "Azure AD v2", + 'atlassian_oauth2' => 'Atlassian' }.freeze def self.authentication(user, provider) diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index f556a7f40e9..fe1bf730e76 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -239,8 +239,9 @@ module Gitlab end def update_profile - clear_user_synced_attributes_metadata + return unless gl_user + clear_user_synced_attributes_metadata return unless sync_profile_from_provider? || creating_linked_ldap_user? metadata = gl_user.build_user_synced_attributes_metadata diff --git a/lib/gitlab/avatar_cache.rb b/lib/gitlab/avatar_cache.rb new file mode 100644 index 00000000000..30c8e089061 --- /dev/null +++ b/lib/gitlab/avatar_cache.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +module Gitlab + class AvatarCache + class << self + # Increment this if a breaking change requires + # immediate cache expiry of all avatar caches. + # + # @return [Integer] + VERSION = 1 + + # @return [Symbol] + BASE_KEY = :avatar_cache + + # @return [ActiveSupport::Duration] + DEFAULT_EXPIRY = 7.days + + # Look up cached avatar data by email address. + # This accepts a block to provide the value to be + # cached in the event nothing is found. + # + # Multiple calls in the same request will be served from the + # request store. + # + # @param email [String] + # @param additional_keys [*Object] all must respond to `#to_s` + # @param expires_in [ActiveSupport::Duration, Integer] + # @yield [email, *additional_keys] yields the supplied params back to the block + # @return [String] + def by_email(email, *additional_keys, expires_in: DEFAULT_EXPIRY) + key = email_key(email) + subkey = additional_keys.join(":") + + Gitlab::SafeRequestStore.fetch([key, subkey]) do + with do |redis| + # Look for existing cache value + cached = redis.hget(key, subkey) + + # Return the cached entry if set + break cached unless cached.nil? + + # Otherwise, call the block to get the value + to_cache = yield(email, *additional_keys).to_s + + # Set it in the cache + redis.hset(key, subkey, to_cache) + + # Update the expiry time + redis.expire(key, expires_in) + + # Return this new value + break to_cache + end + end + end + + # Remove one or more emails from the cache + # + # @param emails [String] one or more emails to delete + # @return [Integer] the number of keys deleted + def delete_by_email(*emails) + return 0 if emails.empty? + + with do |redis| + keys = emails.map { |email| email_key(email) } + + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + redis.unlink(*keys) + end + end + end + + private + + # @param email [String] + # @return [String] + def email_key(email) + "#{BASE_KEY}:v#{VERSION}:#{email}" + end + + def with(&blk) + Gitlab::Redis::Cache.with(&blk) # rubocop:disable CodeReuse/ActiveRecord + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb b/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb index 61eb3b332de..7484027a0fa 100644 --- a/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb +++ b/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb @@ -5,7 +5,7 @@ module Gitlab # Update existent project update_at column after their repository storage was moved class BackfillProjectUpdatedAtAfterRepositoryStorageMove def perform(*project_ids) - updated_repository_storages = ProjectRepositoryStorageMove.select("project_id, MAX(updated_at) as updated_at").where(project_id: project_ids).group(:project_id) + updated_repository_storages = Projects::RepositoryStorageMove.select("project_id, MAX(updated_at) as updated_at").where(project_id: project_ids).group(:project_id) Project.connection.execute <<-SQL WITH repository_storage_cte as ( diff --git a/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb new file mode 100644 index 00000000000..80693728e86 --- /dev/null +++ b/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module BatchingStrategies + # Generic batching class for use with a BatchedBackgroundMigration. + # Batches over the given table and column combination, returning the MIN() and MAX() + # values for the next batch as an array. + # + # If no more batches exist in the table, returns nil. + class PrimaryKeyBatchingStrategy + include Gitlab::Database::DynamicModelHelpers + + # Finds and returns the next batch in the table. + # + # table_name - The table to batch over + # column_name - The column to batch over + # batch_min_value - The minimum value which the next batch will start at + # batch_size - The size of the next batch + def next_batch(table_name, column_name, batch_min_value:, batch_size:) + model_class = define_batchable_model(table_name) + + quoted_column_name = model_class.connection.quote_column_name(column_name) + relation = model_class.where("#{quoted_column_name} >= ?", batch_min_value) + next_batch_bounds = nil + + relation.each_batch(of: batch_size, column: column_name) do |batch| # rubocop:disable Lint/UnreachableLoop + next_batch_bounds = batch.pluck(Arel.sql("MIN(#{quoted_column_name}), MAX(#{quoted_column_name})")).first + + break + end + + next_batch_bounds + end + end + end + end +end diff --git a/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb b/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb index 16c0de39a3b..60682bd2ec1 100644 --- a/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb +++ b/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb @@ -2,13 +2,11 @@ module Gitlab module BackgroundMigration - # Background migration that extends CopyColumn to update the value of a + # Background migration that updates the value of a # column using the value of another column in the same table. # # - The {start_id, end_id} arguments are at the start so that it can be used - # with `queue_background_migration_jobs_by_range_at_intervals` - # - Provides support for background job tracking through the use of - # Gitlab::Database::BackgroundMigrationJob + # with `queue_batched_background_migration` # - Uses sub-batching so that we can keep each update's execution time at # low 100s ms, while being able to update more records per 2 minutes # that we allow background migration jobs to be scheduled one after the other @@ -22,28 +20,24 @@ module Gitlab # start_id - The start ID of the range of rows to update. # end_id - The end ID of the range of rows to update. - # table - The name of the table that contains the columns. - # primary_key - The primary key column of the table. - # copy_from - The column containing the data to copy. - # copy_to - The column to copy the data to. + # batch_table - The name of the table that contains the columns. + # batch_column - The name of the column we use to batch over the table. # sub_batch_size - We don't want updates to take more than ~100ms # This allows us to run multiple smaller batches during # the minimum 2.minute interval that we can schedule jobs - def perform(start_id, end_id, table, primary_key, copy_from, copy_to, sub_batch_size) + # copy_from - The column containing the data to copy. + # copy_to - The column to copy the data to. + def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, copy_from, copy_to) quoted_copy_from = connection.quote_column_name(copy_from) quoted_copy_to = connection.quote_column_name(copy_to) - parent_batch_relation = relation_scoped_to_range(table, primary_key, start_id, end_id) + parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id) - parent_batch_relation.each_batch(column: primary_key, of: sub_batch_size) do |sub_batch| + parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| sub_batch.update_all("#{quoted_copy_to}=#{quoted_copy_from}") sleep(PAUSE_SECONDS) end - - # We have to add all arguments when marking a job as succeeded as they - # are all used to track the job by `queue_background_migration_jobs_by_range_at_intervals` - mark_job_as_succeeded(start_id, end_id, table, primary_key, copy_from, copy_to, sub_batch_size) end private @@ -52,10 +46,6 @@ module Gitlab ActiveRecord::Base.connection end - def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(self.class.name, arguments) - end - def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) define_batchable_model(source_table).where(source_key_column => start_id..stop_id) end diff --git a/lib/gitlab/background_migration/merge_request_assignees_migration_progress_check.rb b/lib/gitlab/background_migration/merge_request_assignees_migration_progress_check.rb deleted file mode 100644 index de0c357ab1c..00000000000 --- a/lib/gitlab/background_migration/merge_request_assignees_migration_progress_check.rb +++ /dev/null @@ -1,43 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # rubocop: disable Style/Documentation - class MergeRequestAssigneesMigrationProgressCheck - include Gitlab::Utils::StrongMemoize - - RESCHEDULE_DELAY = 3.hours - WORKER = 'PopulateMergeRequestAssigneesTable' - DeadJobsError = Class.new(StandardError) - - def perform - raise DeadJobsError, "Only dead background jobs in the queue for #{WORKER}" if !ongoing? && dead_jobs? - - if ongoing? - BackgroundMigrationWorker.perform_in(RESCHEDULE_DELAY, self.class.name) - else - Feature.enable(:multiple_merge_request_assignees) - end - end - - private - - def dead_jobs? - strong_memoize(:dead_jobs) do - migration_klass.dead_jobs?(WORKER) - end - end - - def ongoing? - strong_memoize(:ongoing) do - migration_klass.exists?(WORKER) || migration_klass.retrying_jobs?(WORKER) - end - end - - def migration_klass - Gitlab::BackgroundMigration - end - end - # rubocop: enable Style/Documentation - end -end diff --git a/lib/gitlab/background_migration/populate_finding_uuid_for_vulnerability_feedback.rb b/lib/gitlab/background_migration/populate_finding_uuid_for_vulnerability_feedback.rb index 52b09e07fd5..dc31f995ae0 100644 --- a/lib/gitlab/background_migration/populate_finding_uuid_for_vulnerability_feedback.rb +++ b/lib/gitlab/background_migration/populate_finding_uuid_for_vulnerability_feedback.rb @@ -61,16 +61,12 @@ module Gitlab private def calculated_uuid - Gitlab::UUID.v5(uuid_components) - end - - def uuid_components - [ - category, - vulnerability_finding.primary_identifier.fingerprint, - vulnerability_finding.location_fingerprint, - project_id - ].join('-') + ::Security::VulnerabilityUUID.generate( + report_type: category, + primary_identifier_fingerprint: vulnerability_finding.primary_identifier.fingerprint, + location_fingerprint: vulnerability_finding.location_fingerprint, + project_id: project_id + ) end def finding_key diff --git a/lib/gitlab/background_migration/populate_namespace_statistics.rb b/lib/gitlab/background_migration/populate_namespace_statistics.rb new file mode 100644 index 00000000000..e352ae71de6 --- /dev/null +++ b/lib/gitlab/background_migration/populate_namespace_statistics.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class creates/updates those namespace statistics + # that haven't been created nor initialized. + # It also updates the related namespace statistics + # This is only required in EE + class PopulateNamespaceStatistics + def perform(group_ids, statistics) + end + end + end +end + +Gitlab::BackgroundMigration::PopulateNamespaceStatistics.prepend_if_ee('EE::Gitlab::BackgroundMigration::PopulateNamespaceStatistics') diff --git a/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb b/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb index 3d3970f50e1..4aff9d1e2c1 100644 --- a/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb +++ b/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb @@ -10,7 +10,7 @@ module Gitlab NOP_RELATION.new end - def perform(_scan_ids); end + def perform(*_scan_ids); end end end end diff --git a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb new file mode 100644 index 00000000000..7b18e617c81 --- /dev/null +++ b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +# rubocop: disable Style/Documentation +class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid + # rubocop: disable Gitlab/NamespacedClass + class VulnerabilitiesIdentifier < ActiveRecord::Base + self.table_name = "vulnerability_identifiers" + has_many :primary_findings, class_name: 'VulnerabilitiesFinding', inverse_of: :primary_identifier, foreign_key: 'primary_identifier_id' + end + + class VulnerabilitiesFinding < ActiveRecord::Base + self.table_name = "vulnerability_occurrences" + belongs_to :primary_identifier, class_name: 'VulnerabilitiesIdentifier', inverse_of: :primary_findings, foreign_key: 'primary_identifier_id' + REPORT_TYPES = { + sast: 0, + dependency_scanning: 1, + container_scanning: 2, + dast: 3, + secret_detection: 4, + coverage_fuzzing: 5, + api_fuzzing: 6 + }.with_indifferent_access.freeze + enum report_type: REPORT_TYPES + end + + class CalculateFindingUUID + FINDING_NAMESPACES_IDS = { + development: "a143e9e2-41b3-47bc-9a19-081d089229f4", + test: "a143e9e2-41b3-47bc-9a19-081d089229f4", + staging: "a6930898-a1b2-4365-ab18-12aa474d9b26", + production: "58dc0f06-936c-43b3-93bb-71693f1b6570" + }.freeze + + NAMESPACE_REGEX = /(\h{8})-(\h{4})-(\h{4})-(\h{4})-(\h{4})(\h{8})/.freeze + PACK_PATTERN = "NnnnnN".freeze + + def self.call(value) + Digest::UUID.uuid_v5(namespace_id, value) + end + + def self.namespace_id + namespace_uuid = FINDING_NAMESPACES_IDS.fetch(Rails.env.to_sym) + # Digest::UUID is broken when using an UUID in namespace_id + # https://github.com/rails/rails/issues/37681#issue-520718028 + namespace_uuid.scan(NAMESPACE_REGEX).flatten.map { |s| s.to_i(16) }.pack(PACK_PATTERN) + end + end + # rubocop: enable Gitlab/NamespacedClass + + def perform(start_id, end_id) + findings = VulnerabilitiesFinding + .joins(:primary_identifier) + .select(:id, :report_type, :fingerprint, :location_fingerprint, :project_id) + .where(id: start_id..end_id) + + mappings = findings.each_with_object({}) do |finding, hash| + hash[finding] = { uuid: calculate_uuid_v5_for_finding(finding) } + end + + ::Gitlab::Database::BulkUpdate.execute(%i[uuid], mappings) + end + + private + + def calculate_uuid_v5_for_finding(vulnerability_finding) + return unless vulnerability_finding + + uuid_v5_name_components = { + report_type: vulnerability_finding.report_type, + primary_identifier_fingerprint: vulnerability_finding.fingerprint, + location_fingerprint: vulnerability_finding.location_fingerprint, + project_id: vulnerability_finding.project_id + } + + name = uuid_v5_name_components.values.join('-') + + CalculateFindingUUID.call(name) + end +end diff --git a/lib/gitlab/background_migration/set_default_iteration_cadences.rb b/lib/gitlab/background_migration/set_default_iteration_cadences.rb new file mode 100644 index 00000000000..42f9d33ab71 --- /dev/null +++ b/lib/gitlab/background_migration/set_default_iteration_cadences.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop:disable Style/Documentation + class SetDefaultIterationCadences + class Iteration < ApplicationRecord + self.table_name = 'sprints' + end + + class IterationCadence < ApplicationRecord + self.table_name = 'iterations_cadences' + + include BulkInsertSafe + end + + class Group < ApplicationRecord + self.table_name = 'namespaces' + + self.inheritance_column = :_type_disabled + end + + def perform(*group_ids) + create_iterations_cadences(group_ids) + assign_iterations_cadences(group_ids) + end + + private + + def create_iterations_cadences(group_ids) + groups_with_cadence = IterationCadence.select(:group_id) + + new_cadences = Group.where(id: group_ids).where.not(id: groups_with_cadence).map do |group| + last_iteration = Iteration.where(group_id: group.id).order(:start_date)&.last + + next unless last_iteration + + time = Time.now + IterationCadence.new( + group_id: group.id, + title: "#{group.name} Iterations", + start_date: last_iteration.start_date, + last_run_date: last_iteration.start_date, + automatic: false, + created_at: time, + updated_at: time + ) + end + + IterationCadence.bulk_insert!(new_cadences.compact, skip_duplicates: true) + end + + def assign_iterations_cadences(group_ids) + IterationCadence.where(group_id: group_ids).each do |cadence| + Iteration.where(iterations_cadence_id: nil).where(group_id: cadence.group_id).update_all(iterations_cadence_id: cadence.id) + end + end + end + end +end diff --git a/lib/gitlab/blame.rb b/lib/gitlab/blame.rb index 5382bdab7eb..78a8f39e143 100644 --- a/lib/gitlab/blame.rb +++ b/lib/gitlab/blame.rb @@ -19,16 +19,14 @@ module Gitlab commit = Commit.new(commit, project) commit.lazy_author # preload author - sha = commit.sha - if prev_sha != sha + if prev_sha != commit.sha groups << current_group if current_group current_group = { commit: commit, lines: [] } end - line = highlighted_lines[i].html_safe if highlight - current_group[:lines] << line + current_group[:lines] << (highlight ? highlighted_lines[i].html_safe : line) - prev_sha = sha + prev_sha = commit.sha i += 1 end groups << current_group if current_group diff --git a/lib/gitlab/checks/branch_check.rb b/lib/gitlab/checks/branch_check.rb index ad2a718ef67..a8287a97cc3 100644 --- a/lib/gitlab/checks/branch_check.rb +++ b/lib/gitlab/checks/branch_check.rb @@ -51,7 +51,7 @@ module Gitlab logger.log_timed(LOG_MESSAGES[:protected_branch_checks]) do return unless ProtectedBranch.protected?(project, branch_name) # rubocop:disable Cop/AvoidReturnFromBlocks - if forced_push? + if forced_push? && !ProtectedBranch.allow_force_push?(project, branch_name) raise GitAccess::ForbiddenError, ERROR_MESSAGES[:force_push_protected_branch] end end diff --git a/lib/gitlab/checks/lfs_check.rb b/lib/gitlab/checks/lfs_check.rb index b70a6a69b93..38f0b82c8b4 100644 --- a/lib/gitlab/checks/lfs_check.rb +++ b/lib/gitlab/checks/lfs_check.rb @@ -13,6 +13,7 @@ module Gitlab return unless project.lfs_enabled? return if skip_lfs_integrity_check + return if deletion? logger.log_timed(LOG_MESSAGE) do lfs_check = Checks::LfsIntegrity.new(project, newrev, logger.time_left) diff --git a/lib/gitlab/ci/build/context/build.rb b/lib/gitlab/ci/build/context/build.rb index dfd86d3ad72..641aa71fb4e 100644 --- a/lib/gitlab/ci/build/context/build.rb +++ b/lib/gitlab/ci/build/context/build.rb @@ -21,7 +21,7 @@ module Gitlab # to the CI variables to evaluate rules before we persist a Build # with the result. We should refactor away the extra Build.new, # but be able to get CI Variables directly from the Seed::Build. - stub_build.scoped_variables_hash + stub_build.scoped_variables end end diff --git a/lib/gitlab/ci/build/context/global.rb b/lib/gitlab/ci/build/context/global.rb index fdd3ac358d5..dd0bc54d8b2 100644 --- a/lib/gitlab/ci/build/context/global.rb +++ b/lib/gitlab/ci/build/context/global.rb @@ -19,8 +19,7 @@ module Gitlab # to the CI variables to evaluate workflow:rules # with the result. We should refactor away the extra Build.new, # but be able to get CI Variables directly from the Seed::Build. - stub_build.scoped_variables_hash - .reject { |key, _value| key =~ /\ACI_(JOB|BUILD)/ } + stub_build.scoped_variables.reject { |var| var[:key] =~ /\ACI_(JOB|BUILD)/ } end end diff --git a/lib/gitlab/ci/build/credentials/registry/dependency_proxy.rb b/lib/gitlab/ci/build/credentials/registry/dependency_proxy.rb index b6ac06cfb53..76eec2172b1 100644 --- a/lib/gitlab/ci/build/credentials/registry/dependency_proxy.rb +++ b/lib/gitlab/ci/build/credentials/registry/dependency_proxy.rb @@ -7,7 +7,7 @@ module Gitlab module Registry class DependencyProxy < GitlabRegistry def url - "#{Gitlab.config.gitlab.host}:#{Gitlab.config.gitlab.port}" + Gitlab.host_with_port end def valid? diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index dbb48a81030..d3f030c3b36 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -99,10 +99,18 @@ module Gitlab initial_config end + def find_sha(project) + branches = project&.repository&.branches || [] + + unless branches.empty? + project.repository.root_ref_sha + end + end + def build_context(project:, sha:, user:, parent_pipeline:) Config::External::Context.new( project: project, - sha: sha || project&.repository&.root_ref_sha, + sha: sha || find_sha(project), user: user, parent_pipeline: parent_pipeline, variables: project&.predefined_variables&.to_runner_variables) diff --git a/lib/gitlab/ci/config/entry/cache.rb b/lib/gitlab/ci/config/entry/cache.rb index 6b036182706..cf599ce5294 100644 --- a/lib/gitlab/ci/config/entry/cache.rb +++ b/lib/gitlab/ci/config/entry/cache.rb @@ -7,52 +7,90 @@ module Gitlab ## # Entry that represents a cache configuration # - class Cache < ::Gitlab::Config::Entry::Node - include ::Gitlab::Config::Entry::Configurable - include ::Gitlab::Config::Entry::Validatable - include ::Gitlab::Config::Entry::Attributable - - ALLOWED_KEYS = %i[key untracked paths when policy].freeze - ALLOWED_POLICY = %w[pull-push push pull].freeze - DEFAULT_POLICY = 'pull-push' - ALLOWED_WHEN = %w[on_success on_failure always].freeze - DEFAULT_WHEN = 'on_success' - - validations do - validates :config, type: Hash, allowed_keys: ALLOWED_KEYS - validates :policy, - inclusion: { in: ALLOWED_POLICY, message: 'should be pull-push, push, or pull' }, - allow_blank: true - - with_options allow_nil: true do - validates :when, - inclusion: { - in: ALLOWED_WHEN, - message: 'should be on_success, on_failure or always' - } + class Cache < ::Gitlab::Config::Entry::Simplifiable + strategy :Caches, if: -> (config) { Feature.enabled?(:multiple_cache_per_job) } + strategy :Cache, if: -> (config) { Feature.disabled?(:multiple_cache_per_job) } + + class Caches < ::Gitlab::Config::Entry::ComposableArray + include ::Gitlab::Config::Entry::Validatable + + MULTIPLE_CACHE_LIMIT = 4 + + validations do + validates :config, presence: true + + validate do + unless config.is_a?(Hash) || config.is_a?(Array) + errors.add(:config, 'can only be a Hash or an Array') + end + + if config.is_a?(Array) && config.count > MULTIPLE_CACHE_LIMIT + errors.add(:config, "no more than #{MULTIPLE_CACHE_LIMIT} caches can be created") + end + end + end + + def initialize(*args) + super + + @key = nil + end + + def composable_class + Entry::Cache::Cache end end - entry :key, Entry::Key, - description: 'Cache key used to define a cache affinity.' + class Cache < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + ALLOWED_KEYS = %i[key untracked paths when policy].freeze + ALLOWED_POLICY = %w[pull-push push pull].freeze + DEFAULT_POLICY = 'pull-push' + ALLOWED_WHEN = %w[on_success on_failure always].freeze + DEFAULT_WHEN = 'on_success' + + validations do + validates :config, type: Hash, allowed_keys: ALLOWED_KEYS + validates :policy, + inclusion: { in: ALLOWED_POLICY, message: 'should be pull-push, push, or pull' }, + allow_blank: true + + with_options allow_nil: true do + validates :when, + inclusion: { + in: ALLOWED_WHEN, + message: 'should be on_success, on_failure or always' + } + end + end - entry :untracked, ::Gitlab::Config::Entry::Boolean, - description: 'Cache all untracked files.' + entry :key, Entry::Key, + description: 'Cache key used to define a cache affinity.' - entry :paths, Entry::Paths, - description: 'Specify which paths should be cached across builds.' + entry :untracked, ::Gitlab::Config::Entry::Boolean, + description: 'Cache all untracked files.' - attributes :policy, :when + entry :paths, Entry::Paths, + description: 'Specify which paths should be cached across builds.' - def value - result = super + attributes :policy, :when - result[:key] = key_value - result[:policy] = policy || DEFAULT_POLICY - # Use self.when to avoid conflict with reserved word - result[:when] = self.when || DEFAULT_WHEN + def value + result = super + + result[:key] = key_value + result[:policy] = policy || DEFAULT_POLICY + # Use self.when to avoid conflict with reserved word + result[:when] = self.when || DEFAULT_WHEN + + result + end + end - result + class UnknownStrategy < ::Gitlab::Config::Entry::Node end end end diff --git a/lib/gitlab/ci/config/entry/environment.rb b/lib/gitlab/ci/config/entry/environment.rb index 64e6d48133f..2066e9be3b1 100644 --- a/lib/gitlab/ci/config/entry/environment.rb +++ b/lib/gitlab/ci/config/entry/environment.rb @@ -10,7 +10,7 @@ module Gitlab class Environment < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Configurable - ALLOWED_KEYS = %i[name url action on_stop auto_stop_in kubernetes].freeze + ALLOWED_KEYS = %i[name url action on_stop auto_stop_in kubernetes deployment_tier].freeze entry :kubernetes, Entry::Kubernetes, description: 'Kubernetes deployment configuration.' @@ -47,6 +47,11 @@ module Gitlab inclusion: { in: %w[start stop prepare], message: 'should be start, stop or prepare' }, allow_nil: true + validates :deployment_tier, + type: String, + inclusion: { in: ::Environment.tiers.keys, message: "must be one of #{::Environment.tiers.keys.join(', ')}" }, + allow_nil: true + validates :on_stop, type: String, allow_nil: true validates :kubernetes, type: Hash, allow_nil: true validates :auto_stop_in, duration: true, allow_nil: true @@ -85,6 +90,10 @@ module Gitlab value[:auto_stop_in] end + def deployment_tier + value[:deployment_tier] + end + def value case @config when String then { name: @config, action: 'start' } diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb index d1a366125ef..27ea9c2558a 100644 --- a/lib/gitlab/ci/features.rb +++ b/lib/gitlab/ci/features.rb @@ -38,10 +38,6 @@ module Gitlab ::Feature.enabled?(:ci_disallow_to_create_merge_request_pipelines_in_target_project, target_project) end - def self.project_transactionless_destroy?(project) - Feature.enabled?(:project_transactionless_destroy, project, default_enabled: false) - end - def self.trace_overwrite? ::Feature.enabled?(:ci_trace_overwrite, type: :ops, default_enabled: false) end @@ -55,14 +51,6 @@ module Gitlab ::Feature.enabled?(:ci_trace_log_invalid_chunks, project, type: :ops, default_enabled: false) end - def self.ci_pipeline_editor_page_enabled?(project) - ::Feature.enabled?(:ci_pipeline_editor_page, project, default_enabled: :yaml) - end - - def self.rules_variables_enabled?(project) - ::Feature.enabled?(:ci_rules_variables, project, default_enabled: true) - end - def self.validate_build_dependencies?(project) ::Feature.enabled?(:ci_validate_build_dependencies, project, default_enabled: :yaml) && ::Feature.disabled?(:ci_validate_build_dependencies_override, project) @@ -76,8 +64,8 @@ module Gitlab ::Feature.enabled?(:codequality_backend_comparison, project, default_enabled: :yaml) end - def self.use_coverage_data_new_finder?(record) - ::Feature.enabled?(:coverage_data_new_finder, record, default_enabled: :yaml) + def self.multiple_cache_per_job? + ::Feature.enabled?(:multiple_cache_per_job, default_enabled: :yaml) end end end diff --git a/lib/gitlab/ci/jwt.rb b/lib/gitlab/ci/jwt.rb index 0870c74053a..af06e124736 100644 --- a/lib/gitlab/ci/jwt.rb +++ b/lib/gitlab/ci/jwt.rb @@ -60,7 +60,7 @@ module Gitlab ref_protected: build.protected.to_s } - if include_environment_claims? + if environment.present? fields.merge!( environment: environment.name, environment_protected: environment_protected?.to_s @@ -119,10 +119,6 @@ module Gitlab def environment_protected? false # Overridden in EE end - - def include_environment_claims? - Feature.enabled?(:ci_jwt_include_environment) && environment.present? - end end end end diff --git a/lib/gitlab/ci/lint.rb b/lib/gitlab/ci/lint.rb index 364e67db02b..4a7c11ee26e 100644 --- a/lib/gitlab/ci/lint.rb +++ b/lib/gitlab/ci/lint.rb @@ -21,7 +21,7 @@ module Gitlab def initialize(project:, current_user:, sha: nil) @project = project @current_user = current_user - @sha = sha || project.repository.commit.sha + @sha = sha || project.repository.commit&.sha end def validate(content, dry_run: false) diff --git a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb index f0214bb4e38..1c0dfbdbee3 100644 --- a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb +++ b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb @@ -10,7 +10,7 @@ module Gitlab def perform! return unless project.auto_cancel_pending_pipelines? - Gitlab::OptimisticLocking.retry_lock(auto_cancelable_pipelines) do |cancelables| + Gitlab::OptimisticLocking.retry_lock(auto_cancelable_pipelines, name: 'cancel_pending_pipelines') do |cancelables| cancelables.find_each do |cancelable| cancelable.auto_cancel_running(pipeline) end diff --git a/lib/gitlab/ci/pipeline/expression/statement.rb b/lib/gitlab/ci/pipeline/expression/statement.rb index 0e81e1bd34c..5f3310dd668 100644 --- a/lib/gitlab/ci/pipeline/expression/statement.rb +++ b/lib/gitlab/ci/pipeline/expression/statement.rb @@ -7,9 +7,9 @@ module Gitlab class Statement StatementError = Class.new(Expression::ExpressionError) - def initialize(statement, variables = {}) + def initialize(statement, variables = nil) @lexer = Expression::Lexer.new(statement) - @variables = variables.with_indifferent_access + @variables = variables&.to_hash end def parse_tree diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 3770bb4b328..896dba9f117 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -28,8 +28,16 @@ module Gitlab .fabricate(attributes.delete(:except)) @rules = Gitlab::Ci::Build::Rules .new(attributes.delete(:rules), default_when: 'on_success') - @cache = Seed::Build::Cache - .new(pipeline, attributes.delete(:cache)) + + if multiple_cache_per_job? + cache = Array.wrap(attributes.delete(:cache)) + @cache = cache.map do |cache| + Seed::Build::Cache.new(pipeline, cache) + end + else + @cache = Seed::Build::Cache + .new(pipeline, attributes.delete(:cache)) + end end def name @@ -52,7 +60,7 @@ module Gitlab return unless included? strong_memoize(:errors) do - needs_errors + [needs_errors, variable_expansion_errors].compact.flatten end end @@ -153,6 +161,12 @@ module Gitlab @pipeline.project.actual_limits.ci_needs_size_limit end + def variable_expansion_errors + sorted_collection = evaluate_context.variables.sorted_collection(@pipeline.project) + errors = sorted_collection.errors + ["#{name}: #{errors}"] if errors + end + def pipeline_attributes { pipeline: @pipeline, @@ -169,15 +183,11 @@ module Gitlab strong_memoize(:rules_attributes) do next {} unless @using_rules - if ::Gitlab::Ci::Features.rules_variables_enabled?(@pipeline.project) - rules_variables_result = ::Gitlab::Ci::Variables::Helpers.merge_variables( - @seed_attributes[:yaml_variables], rules_result.variables - ) + rules_variables_result = ::Gitlab::Ci::Variables::Helpers.merge_variables( + @seed_attributes[:yaml_variables], rules_result.variables + ) - rules_result.build_attributes.merge(yaml_variables: rules_variables_result) - else - rules_result.build_attributes - end + rules_result.build_attributes.merge(yaml_variables: rules_variables_result) end end @@ -195,7 +205,21 @@ module Gitlab def cache_attributes strong_memoize(:cache_attributes) do - @cache.build_attributes + if multiple_cache_per_job? + if @cache.empty? + {} + else + { options: { cache: @cache.map(&:attributes) } } + end + else + @cache.build_attributes + end + end + end + + def multiple_cache_per_job? + strong_memoize(:multiple_cache_per_job) do + ::Gitlab::Ci::Features.multiple_cache_per_job? end end diff --git a/lib/gitlab/ci/pipeline/seed/build/cache.rb b/lib/gitlab/ci/pipeline/seed/build/cache.rb index 8d6fe13c3b9..78ffaaa7e81 100644 --- a/lib/gitlab/ci/pipeline/seed/build/cache.rb +++ b/lib/gitlab/ci/pipeline/seed/build/cache.rb @@ -18,18 +18,18 @@ module Gitlab raise ArgumentError, "unknown cache keys: #{local_cache.keys}" if local_cache.any? end - def build_attributes + def attributes { - options: { - cache: { - key: key_string, - paths: @paths, - policy: @policy, - untracked: @untracked, - when: @when - }.compact.presence - }.compact - } + key: key_string, + paths: @paths, + policy: @policy, + untracked: @untracked, + when: @when + }.compact + end + + def build_attributes + { options: { cache: attributes.presence }.compact } end private diff --git a/lib/gitlab/ci/pipeline/seed/environment.rb b/lib/gitlab/ci/pipeline/seed/environment.rb index 5dff0788ec9..2abedd3b664 100644 --- a/lib/gitlab/ci/pipeline/seed/environment.rb +++ b/lib/gitlab/ci/pipeline/seed/environment.rb @@ -13,7 +13,9 @@ module Gitlab def to_resource environments.safe_find_or_create_by(name: expanded_environment_name) do |environment| + # Initialize the attributes at creation environment.auto_stop_in = auto_stop_in + environment.tier = deployment_tier if ::Feature.enabled?(:environment_tier, job.project, default_enabled: :yaml) end end @@ -27,6 +29,10 @@ module Gitlab job.environment_auto_stop_in end + def deployment_tier + job.environment_deployment_tier + end + def expanded_environment_name job.expanded_environment_name end diff --git a/lib/gitlab/ci/queue/metrics.rb b/lib/gitlab/ci/queue/metrics.rb new file mode 100644 index 00000000000..5398c19e536 --- /dev/null +++ b/lib/gitlab/ci/queue/metrics.rb @@ -0,0 +1,210 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Queue + class Metrics + extend Gitlab::Utils::StrongMemoize + + QUEUE_DURATION_SECONDS_BUCKETS = [1, 3, 10, 30, 60, 300, 900, 1800, 3600].freeze + QUEUE_ACTIVE_RUNNERS_BUCKETS = [1, 3, 10, 30, 60, 300, 900, 1800, 3600].freeze + QUEUE_DEPTH_TOTAL_BUCKETS = [1, 2, 3, 5, 8, 16, 32, 50, 100, 250, 500, 1000, 2000, 5000].freeze + QUEUE_SIZE_TOTAL_BUCKETS = [1, 5, 10, 50, 100, 500, 1000, 2000, 5000].freeze + QUEUE_ITERATION_DURATION_SECONDS_BUCKETS = [0.1, 0.3, 0.5, 1, 5, 10, 30, 60, 180, 300].freeze + + METRICS_SHARD_TAG_PREFIX = 'metrics_shard::' + DEFAULT_METRICS_SHARD = 'default' + JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET = 5.freeze + + OPERATION_COUNTERS = [ + :build_can_pick, + :build_not_pick, + :build_not_pending, + :build_temporary_locked, + :build_conflict_lock, + :build_conflict_exception, + :build_conflict_transition, + :queue_attempt, + :queue_conflict, + :queue_iteration, + :queue_depth_limit, + :queue_replication_lag, + :runner_pre_assign_checks_failed, + :runner_pre_assign_checks_success, + :runner_queue_tick + ].to_set.freeze + + QUEUE_DEPTH_HISTOGRAMS = [ + :found, + :not_found, + :conflict + ].to_set.freeze + + attr_reader :runner + + def initialize(runner) + @runner = runner + end + + def register_failure + self.class.failed_attempt_counter.increment + self.class.attempt_counter.increment + end + + def register_success(job) + labels = { shared_runner: runner.instance_type?, + jobs_running_for_project: jobs_running_for_project(job), + shard: DEFAULT_METRICS_SHARD } + + if runner.instance_type? + shard = runner.tag_list.sort.find { |name| name.starts_with?(METRICS_SHARD_TAG_PREFIX) } + labels[:shard] = shard.gsub(METRICS_SHARD_TAG_PREFIX, '') if shard + end + + self.class.job_queue_duration_seconds.observe(labels, Time.current - job.queued_at) unless job.queued_at.nil? + self.class.attempt_counter.increment + end + + # rubocop: disable CodeReuse/ActiveRecord + def jobs_running_for_project(job) + return '+Inf' unless runner.instance_type? + + # excluding currently started job + running_jobs_count = job.project.builds.running.where(runner: ::Ci::Runner.instance_type) + .limit(JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET + 1).count - 1 + running_jobs_count < JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET ? running_jobs_count : "#{JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET}+" + end + # rubocop: enable CodeReuse/ActiveRecord + + def increment_queue_operation(operation) + if !Rails.env.production? && !OPERATION_COUNTERS.include?(operation) + raise ArgumentError, "unknown queue operation: #{operation}" + end + + self.class.queue_operations_total.increment(operation: operation) + end + + def observe_queue_depth(queue, size) + return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false) + + if !Rails.env.production? && !QUEUE_DEPTH_HISTOGRAMS.include?(queue) + raise ArgumentError, "unknown queue depth label: #{queue}" + end + + self.class.queue_depth_total.observe({ queue: queue }, size.to_f) + end + + def observe_queue_size(size_proc) + return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false) + + self.class.queue_size_total.observe({}, size_proc.call.to_f) + end + + def observe_queue_time + start_time = ::Gitlab::Metrics::System.monotonic_time + + result = yield + + return result unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false) + + seconds = ::Gitlab::Metrics::System.monotonic_time - start_time + self.class.queue_iteration_duration_seconds.observe({}, seconds.to_f) + + result + end + + def self.observe_active_runners(runners_proc) + return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false) + + queue_active_runners_total.observe({}, runners_proc.call.to_f) + end + + def self.increment_runner_tick(runner) + self.new(runner).increment_queue_operation(:runner_queue_tick) + end + + def self.failed_attempt_counter + strong_memoize(:failed_attempt_counter) do + name = :job_register_attempts_failed_total + comment = 'Counts the times a runner tries to register a job' + + Gitlab::Metrics.counter(name, comment) + end + end + + def self.attempt_counter + strong_memoize(:attempt_counter) do + name = :job_register_attempts_total + comment = 'Counts the times a runner tries to register a job' + + Gitlab::Metrics.counter(name, comment) + end + end + + def self.job_queue_duration_seconds + strong_memoize(:job_queue_duration_seconds) do + name = :job_queue_duration_seconds + comment = 'Request handling execution time' + buckets = QUEUE_DURATION_SECONDS_BUCKETS + labels = {} + + Gitlab::Metrics.histogram(name, comment, labels, buckets) + end + end + + def self.queue_operations_total + strong_memoize(:queue_operations_total) do + name = :gitlab_ci_queue_operations_total + comment = 'Counts all the operations that are happening inside a queue' + + Gitlab::Metrics.counter(name, comment) + end + end + + def self.queue_depth_total + strong_memoize(:queue_depth_total) do + name = :gitlab_ci_queue_depth_total + comment = 'Size of a CI/CD builds queue in relation to the operation result' + buckets = QUEUE_DEPTH_TOTAL_BUCKETS + labels = {} + + Gitlab::Metrics.histogram(name, comment, labels, buckets) + end + end + + def self.queue_size_total + strong_memoize(:queue_size_total) do + name = :gitlab_ci_queue_size_total + comment = 'Size of initialized CI/CD builds queue' + buckets = QUEUE_SIZE_TOTAL_BUCKETS + labels = {} + + Gitlab::Metrics.histogram(name, comment, labels, buckets) + end + end + + def self.queue_iteration_duration_seconds + strong_memoize(:queue_iteration_duration_seconds) do + name = :gitlab_ci_queue_iteration_duration_seconds + comment = 'Time it takes to find a build in CI/CD queue' + buckets = QUEUE_ITERATION_DURATION_SECONDS_BUCKETS + labels = {} + + Gitlab::Metrics.histogram(name, comment, labels, buckets) + end + end + + def self.queue_active_runners_total + strong_memoize(:queue_active_runners_total) do + name = :gitlab_ci_queue_active_runners_total + comment = 'The amount of active runners that can process queue in a project' + buckets = QUEUE_ACTIVE_RUNNERS_BUCKETS + labels = {} + + Gitlab::Metrics.histogram(name, comment, labels, buckets) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/codequality_reports_comparer.rb b/lib/gitlab/ci/reports/codequality_reports_comparer.rb index 88e02cd9004..10748b8ca02 100644 --- a/lib/gitlab/ci/reports/codequality_reports_comparer.rb +++ b/lib/gitlab/ci/reports/codequality_reports_comparer.rb @@ -5,7 +5,7 @@ module Gitlab module Reports class CodequalityReportsComparer < ReportsComparer def initialize(base_report, head_report) - @base_report = base_report || CodequalityReports.new + @base_report = base_report @head_report = head_report end @@ -15,12 +15,16 @@ module Gitlab def existing_errors strong_memoize(:existing_errors) do + next [] if not_found? + base_report.all_degradations & head_report.all_degradations end end def new_errors strong_memoize(:new_errors) do + next [] if not_found? + fingerprints = head_report.degradations.keys - base_report.degradations.keys head_report.degradations.fetch_values(*fingerprints) end @@ -28,6 +32,8 @@ module Gitlab def resolved_errors strong_memoize(:resolved_errors) do + next [] if not_found? + fingerprints = base_report.degradations.keys - head_report.degradations.keys base_report.degradations.fetch_values(*fingerprints) end diff --git a/lib/gitlab/ci/reports/reports_comparer.rb b/lib/gitlab/ci/reports/reports_comparer.rb index d413d3a74f6..16a7f6478b7 100644 --- a/lib/gitlab/ci/reports/reports_comparer.rb +++ b/lib/gitlab/ci/reports/reports_comparer.rb @@ -8,6 +8,7 @@ module Gitlab STATUS_SUCCESS = 'success' STATUS_FAILED = 'failed' + STATUS_NOT_FOUND = 'not_found' attr_reader :base_report, :head_report @@ -17,7 +18,13 @@ module Gitlab end def status - success? ? STATUS_SUCCESS : STATUS_FAILED + if base_report.nil? || head_report.nil? + STATUS_NOT_FOUND + elsif success? + STATUS_SUCCESS + else + STATUS_FAILED + end end def success? @@ -47,6 +54,10 @@ module Gitlab def total_count existing_errors.size + new_errors.size end + + def not_found? + status == STATUS_NOT_FOUND + end end end end diff --git a/lib/gitlab/ci/status/composite.rb b/lib/gitlab/ci/status/composite.rb index 9a4f5644f7d..5368e020a50 100644 --- a/lib/gitlab/ci/status/composite.rb +++ b/lib/gitlab/ci/status/composite.rb @@ -7,7 +7,10 @@ module Gitlab include Gitlab::Utils::StrongMemoize # This class accepts an array of arrays/hashes/or objects - def initialize(all_statuses, with_allow_failure: true, dag: false) + # + # The parameter `project` is only used for the feature flag check, and will be removed with + # https://gitlab.com/gitlab-org/gitlab/-/issues/321972 + def initialize(all_statuses, with_allow_failure: true, dag: false, project: nil) unless all_statuses.respond_to?(:pluck) raise ArgumentError, "all_statuses needs to respond to `.pluck`" end @@ -16,6 +19,7 @@ module Gitlab @status_key = 0 @allow_failure_key = 1 if with_allow_failure @dag = dag + @project = project consume_all_statuses(all_statuses) end @@ -32,7 +36,7 @@ module Gitlab return if none? strong_memoize(:status) do - if @dag && any_of?(:skipped) + if @dag && any_skipped_or_ignored? # The DAG job is skipped if one of the needs does not run at all. 'skipped' elsif @dag && !only_of?(:success, :failed, :canceled, :skipped, :success_with_warnings) @@ -90,6 +94,14 @@ module Gitlab matching == @status_set.size end + def any_skipped_or_ignored? + if ::Feature.enabled?(:ci_fix_pipeline_status_for_dag_needs_manual, @project, default_enabled: :yaml) + any_of?(:skipped) || any_of?(:ignored) + else + any_of?(:skipped) + end + end + def consume_all_statuses(all_statuses) columns = [] columns[@status_key] = :status diff --git a/lib/gitlab/ci/templates/Chef.gitlab-ci.yml b/lib/gitlab/ci/templates/Chef.gitlab-ci.yml index 5f17c93b853..d879e27dfcb 100644 --- a/lib/gitlab/ci/templates/Chef.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Chef.gitlab-ci.yml @@ -20,11 +20,6 @@ stages: - functional - deploy -foodcritic: - stage: lint - script: - - chef exec foodcritic . - cookstyle: stage: lint script: diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index daed75a42ee..fd6c51ea350 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -7,7 +7,7 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.22" + CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.23" needs: [] script: - export SOURCE_CODE=$PWD diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index c4e194bd658..29edada4041 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -23,7 +23,7 @@ review: rules: - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' when: never - - if: '$CI_COMMIT_BRANCH == "master"' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' when: never - if: '$REVIEW_DISABLED' when: never @@ -44,7 +44,7 @@ stop_review: rules: - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' when: never - - if: '$CI_COMMIT_BRANCH == "master"' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' when: never - if: '$REVIEW_DISABLED' when: never @@ -73,7 +73,7 @@ staging: rules: - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' when: never - - if: '$CI_COMMIT_BRANCH != "master"' + - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never - if: '$STAGING_ENABLED' @@ -98,7 +98,7 @@ canary: rules: - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' when: never - - if: '$CI_COMMIT_BRANCH != "master"' + - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never - if: '$CANARY_ENABLED' when: manual @@ -136,7 +136,7 @@ production: when: never - if: '$INCREMENTAL_ROLLOUT_MODE' when: never - - if: '$CI_COMMIT_BRANCH == "master"' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' production_manual: <<: *production_template @@ -148,12 +148,12 @@ production_manual: when: never - if: '$INCREMENTAL_ROLLOUT_MODE' when: never - - if: '$CI_COMMIT_BRANCH == "master" && $STAGING_ENABLED' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $STAGING_ENABLED' when: manual - - if: '$CI_COMMIT_BRANCH == "master" && $CANARY_ENABLED' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CANARY_ENABLED' when: manual -# This job implements incremental rollout on for every push to `master`. +# This job implements incremental rollout on for every push to the default branch. .rollout: &rollout_template extends: .auto-deploy @@ -184,7 +184,7 @@ production_manual: when: never - if: '$INCREMENTAL_ROLLOUT_MODE == "timed"' when: never - - if: '$CI_COMMIT_BRANCH != "master"' + - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never # $INCREMENTAL_ROLLOUT_ENABLED is for compatibility with pre-GitLab 11.4 syntax - if: '$INCREMENTAL_ROLLOUT_MODE == "manual" || $INCREMENTAL_ROLLOUT_ENABLED' @@ -197,7 +197,7 @@ production_manual: when: never - if: '$INCREMENTAL_ROLLOUT_MODE == "manual"' when: never - - if: '$CI_COMMIT_BRANCH != "master"' + - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never - if: '$INCREMENTAL_ROLLOUT_MODE == "timed"' when: delayed diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index e5b40e5f49a..530ab1d0f99 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ .auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.0.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.6.0" dependencies: [] review: @@ -23,7 +23,7 @@ review: rules: - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' when: never - - if: '$CI_COMMIT_BRANCH == "master"' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' when: never - if: '$REVIEW_DISABLED' when: never @@ -44,7 +44,7 @@ stop_review: rules: - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' when: never - - if: '$CI_COMMIT_BRANCH == "master"' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' when: never - if: '$REVIEW_DISABLED' when: never @@ -73,7 +73,7 @@ staging: rules: - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' when: never - - if: '$CI_COMMIT_BRANCH != "master"' + - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never - if: '$STAGING_ENABLED' @@ -98,7 +98,7 @@ canary: rules: - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' when: never - - if: '$CI_COMMIT_BRANCH != "master"' + - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never - if: '$CANARY_ENABLED' when: manual @@ -135,7 +135,7 @@ production: when: never - if: '$INCREMENTAL_ROLLOUT_MODE' when: never - - if: '$CI_COMMIT_BRANCH == "master"' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' production_manual: <<: *production_template @@ -147,12 +147,12 @@ production_manual: when: never - if: '$INCREMENTAL_ROLLOUT_MODE' when: never - - if: '$CI_COMMIT_BRANCH == "master" && $STAGING_ENABLED' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $STAGING_ENABLED' when: manual - - if: '$CI_COMMIT_BRANCH == "master" && $CANARY_ENABLED' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CANARY_ENABLED' when: manual -# This job implements incremental rollout on for every push to `master`. +# This job implements incremental rollout on for every push to the default branch. .rollout: &rollout_template extends: .auto-deploy @@ -181,7 +181,7 @@ production_manual: when: never - if: '$INCREMENTAL_ROLLOUT_MODE == "timed"' when: never - - if: '$CI_COMMIT_BRANCH != "master"' + - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never # $INCREMENTAL_ROLLOUT_ENABLED is for compatibility with pre-GitLab 11.4 syntax - if: '$INCREMENTAL_ROLLOUT_MODE == "manual" || $INCREMENTAL_ROLLOUT_ENABLED' @@ -194,7 +194,7 @@ production_manual: when: never - if: '$INCREMENTAL_ROLLOUT_MODE == "manual"' when: never - - if: '$CI_COMMIT_BRANCH != "master"' + - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never - if: '$INCREMENTAL_ROLLOUT_MODE == "timed"' when: delayed diff --git a/lib/gitlab/ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml index ed2172ef7f5..7efbcab221b 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml @@ -20,7 +20,7 @@ review_ec2: when: never - if: '$REVIEW_DISABLED' when: never - - if: '$CI_COMMIT_BRANCH == "master"' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' @@ -34,6 +34,6 @@ production_ec2: when: never - if: '$CI_KUBERNETES_ACTIVE' when: never - - if: '$CI_COMMIT_BRANCH != "master"' + - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' diff --git a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml index 0289ba1c473..332c58c8695 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml @@ -46,7 +46,7 @@ review_ecs: when: never - if: '$REVIEW_DISABLED' when: never - - if: '$CI_COMMIT_BRANCH == "master"' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' @@ -62,7 +62,7 @@ stop_review_ecs: when: never - if: '$REVIEW_DISABLED' when: never - - if: '$CI_COMMIT_BRANCH == "master"' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' when: manual @@ -81,7 +81,7 @@ review_fargate: when: never - if: '$REVIEW_DISABLED' when: never - - if: '$CI_COMMIT_BRANCH == "master"' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' @@ -97,7 +97,7 @@ stop_review_fargate: when: never - if: '$REVIEW_DISABLED' when: never - - if: '$CI_COMMIT_BRANCH == "master"' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' when: manual @@ -109,7 +109,7 @@ production_ecs: when: never - if: '$CI_KUBERNETES_ACTIVE' when: never - - if: '$CI_COMMIT_BRANCH != "master"' + - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' @@ -120,6 +120,6 @@ production_fargate: when: never - if: '$CI_KUBERNETES_ACTIVE' when: never - - if: '$CI_COMMIT_BRANCH != "master"' + - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' diff --git a/lib/gitlab/ci/templates/LaTeX.gitlab-ci.yml b/lib/gitlab/ci/templates/LaTeX.gitlab-ci.yml index a4aed36889e..e4ed7fadfaa 100644 --- a/lib/gitlab/ci/templates/LaTeX.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/LaTeX.gitlab-ci.yml @@ -1,11 +1,20 @@ -# use docker image with latex preinstalled -# since there is no official latex image, use https://github.com/blang/latex-docker -# possible alternative: https://github.com/natlownes/docker-latex -image: blang/latex +--- +variables: + # Feel free to choose the image that suits you best. + # blang/latex:latest ... Former image used in this template. No longer maintained by author. + # listx/texlive:2020 ... The default, referring to TexLive 2020. Current at least to 2021-02-02. + + # Additional alternatives with high Docker pull counts: + # thomasweise/docker-texlive-full + # thomasweise/texlive + # adnrv/texlive + LATEX_IMAGE: listx/texlive:2020 build: + image: $LATEX_IMAGE script: - latexmk -pdf + artifacts: paths: - "*.pdf" diff --git a/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml index d2dd3fbfb75..90cd8472916 100644 --- a/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml @@ -11,5 +11,5 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml index ba422c08614..7435afef572 100644 --- a/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml @@ -9,5 +9,5 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml index 3a6eac63892..708c5063cc6 100644 --- a/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml @@ -13,5 +13,5 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml index 92f25280c6e..694446dd6c9 100644 --- a/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml @@ -8,5 +8,5 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml index 0e206423fa5..a2fd6620909 100644 --- a/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml @@ -11,5 +11,5 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml index d91a8d7421f..fd75e47e899 100644 --- a/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml @@ -13,5 +13,5 @@ pages: paths: - node_modules key: project - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml index 975cb3b7698..a6a605e35f0 100644 --- a/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml @@ -10,7 +10,8 @@ test: script: - hugo except: - - master + variables: + - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH pages: script: @@ -19,4 +20,5 @@ pages: paths: - public only: - - master + variables: + - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml index 7a441a2f70f..1be2f4bad76 100644 --- a/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml @@ -11,7 +11,8 @@ test: - pip install hyde - hyde gen except: - - master + variables: + - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH pages: stage: deploy @@ -22,4 +23,5 @@ pages: paths: - public only: - - master + variables: + - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml index f2f92fe0704..01e063c50ad 100644 --- a/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml @@ -18,7 +18,8 @@ test: paths: - test except: - - master + variables: + - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH pages: stage: deploy @@ -28,4 +29,5 @@ pages: paths: - public only: - - master + variables: + - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml index 2d26b86a328..e39aa8a2063 100644 --- a/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml @@ -33,5 +33,5 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml index 93ab8e0be0d..13d3089f4fa 100644 --- a/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml @@ -8,5 +8,5 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml index 6524405133a..e65cf3928f2 100644 --- a/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml @@ -12,5 +12,5 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml index 462b4737c4e..377fd8c396e 100644 --- a/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml @@ -12,7 +12,8 @@ test: - bundle install --path vendor - bundle exec middleman build except: - - master + variables: + - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH pages: script: @@ -23,5 +24,5 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml index b512f8d77e9..89281b41b66 100644 --- a/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml @@ -8,5 +8,5 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml index 4318aadcaa6..8fd4702b90d 100644 --- a/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml @@ -11,5 +11,5 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/SwaggerUI.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/SwaggerUI.gitlab-ci.yml index 8fd08ea7995..9fa8b07f7cb 100644 --- a/lib/gitlab/ci/templates/Pages/SwaggerUI.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/SwaggerUI.gitlab-ci.yml @@ -25,5 +25,5 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml index 135f0df99fe..654a03ced5f 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml @@ -45,13 +45,10 @@ apifuzzer_fuzz: entrypoint: ["/bin/bash", "-l", "-c"] variables: FUZZAPI_PROJECT: $CI_PROJECT_PATH - FUZZAPI_API: http://apifuzzer:80 + FUZZAPI_API: http://localhost:80 FUZZAPI_NEW_REPORT: 1 + FUZZAPI_LOG_SCANNER: gl-apifuzzing-api-scanner.log TZ: America/Los_Angeles - services: - - name: $FUZZAPI_IMAGE - alias: apifuzzer - entrypoint: ["dotnet", "/peach/Peach.Web.dll"] allow_failure: true rules: - if: $FUZZAPI_D_TARGET_IMAGE @@ -80,17 +77,26 @@ apifuzzer_fuzz: # Make sure asset path exists - mkdir -p $FUZZAPI_REPORT_ASSET_PATH # + # Start API Security background process + - dotnet /peach/Peach.Web.dll &> $FUZZAPI_LOG_SCANNER & + - APISEC_PID=$! + # # Start scanning - worker-entry # # Run user provided post-script - sh -c "$FUZZAPI_POST_SCRIPT" # + # Shutdown API Security + - kill $APISEC_PID + - wait $APISEC_PID + # artifacts: when: always paths: - $FUZZAPI_REPORT_ASSET_PATH - $FUZZAPI_REPORT + - $FUZZAPI_LOG_SCANNER reports: api_fuzzing: $FUZZAPI_REPORT @@ -172,6 +178,7 @@ apifuzzer_fuzz_dnd: -e FUZZAPI_HAR \ -e FUZZAPI_OPENAPI \ -e FUZZAPI_POSTMAN_COLLECTION \ + -e FUZZAPI_POSTMAN_COLLECTION_VARIABLES \ -e FUZZAPI_TARGET_URL \ -e FUZZAPI_OVERRIDES_FILE \ -e FUZZAPI_OVERRIDES_ENV \ @@ -214,6 +221,7 @@ apifuzzer_fuzz_dnd: -e FUZZAPI_HAR \ -e FUZZAPI_OPENAPI \ -e FUZZAPI_POSTMAN_COLLECTION \ + -e FUZZAPI_POSTMAN_COLLECTION_VARIABLES \ -e FUZZAPI_TARGET_URL \ -e FUZZAPI_OVERRIDES_FILE \ -e FUZZAPI_OVERRIDES_ENV \ diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index 5ea2363a0c5..64001c2828a 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -18,6 +18,9 @@ container_scanning: # file. See https://docs.gitlab.com/ee/user/application_security/container_scanning/index.html#overriding-the-container-scanning-template # for details GIT_STRATEGY: none + # CS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. CS_ANALYZER_IMAGE: $SECURE_ANALYZERS_PREFIX/klar:$CS_MAJOR_VERSION allow_failure: true services: diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index b534dad9593..3039d64514b 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -38,6 +38,9 @@ gemnasium-dependency_scanning: image: name: "$DS_ANALYZER_IMAGE" variables: + # DS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gemnasium:$DS_MAJOR_VERSION" rules: - if: $DEPENDENCY_SCANNING_DISABLED @@ -61,6 +64,9 @@ gemnasium-maven-dependency_scanning: image: name: "$DS_ANALYZER_IMAGE" variables: + # DS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gemnasium-maven:$DS_MAJOR_VERSION" rules: - if: $DEPENDENCY_SCANNING_DISABLED @@ -79,6 +85,9 @@ gemnasium-python-dependency_scanning: image: name: "$DS_ANALYZER_IMAGE" variables: + # DS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gemnasium-python:$DS_MAJOR_VERSION" rules: - if: $DEPENDENCY_SCANNING_DISABLED @@ -104,6 +113,9 @@ bundler-audit-dependency_scanning: image: name: "$DS_ANALYZER_IMAGE" variables: + # DS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/bundler-audit:$DS_MAJOR_VERSION" rules: - if: $DEPENDENCY_SCANNING_DISABLED @@ -119,6 +131,9 @@ retire-js-dependency_scanning: image: name: "$DS_ANALYZER_IMAGE" variables: + # DS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/retire.js:$DS_MAJOR_VERSION" rules: - if: $DEPENDENCY_SCANNING_DISABLED diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index 828352743b4..0391fb429a7 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -41,6 +41,9 @@ bandit-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/bandit:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -57,6 +60,9 @@ brakeman-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/brakeman:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -74,6 +80,9 @@ eslint-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/eslint:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -94,6 +103,9 @@ flawfinder-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/flawfinder:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -111,6 +123,9 @@ kubesec-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kubesec:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -126,6 +141,9 @@ gosec-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gosec:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -140,11 +158,16 @@ gosec-sast: mobsf-android-sast: extends: .sast-analyzer services: - - name: opensecurity/mobile-security-framework-mobsf:latest + # this version must match with analyzer version mentioned in: https://gitlab.com/gitlab-org/security-products/analyzers/mobsf/-/blob/master/Dockerfile + # Unfortunately, we need to keep track of mobsf version in 2 different places for now. + - name: opensecurity/mobile-security-framework-mobsf:v3.2.9 alias: mobsf image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/mobsf:$SAST_ANALYZER_IMAGE_TAG" MOBSF_API_KEY: key rules: @@ -161,11 +184,16 @@ mobsf-android-sast: mobsf-ios-sast: extends: .sast-analyzer services: - - name: opensecurity/mobile-security-framework-mobsf:latest + # this version must match with analyzer version mentioned in: https://gitlab.com/gitlab-org/security-products/analyzers/mobsf/-/blob/master/Dockerfile + # Unfortunately, we need to keep track of mobsf version in 2 different places for now. + - name: opensecurity/mobile-security-framework-mobsf:v3.2.9 alias: mobsf image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/mobsf:$SAST_ANALYZER_IMAGE_TAG" MOBSF_API_KEY: key rules: @@ -184,6 +212,9 @@ nodejs-scan-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/nodejs-scan:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -200,6 +231,9 @@ phpcs-security-audit-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/phpcs-security-audit:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -216,6 +250,9 @@ pmd-apex-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/pmd-apex:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -232,6 +269,9 @@ security-code-scan-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/security-code-scan:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -249,6 +289,9 @@ semgrep-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/semgrep:latest" rules: - if: $SAST_DISABLED @@ -266,6 +309,9 @@ sobelow-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/sobelow:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -282,6 +328,9 @@ spotbugs-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/spotbugs:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_EXCLUDED_ANALYZERS =~ /spotbugs/ diff --git a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml index d2a6fa06dd8..c255fb4707a 100644 --- a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml @@ -1,7 +1,7 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/secret_detection # # Configure the scanning tool through the environment variables. -# List of the variables: https://gitlab.com/gitlab-org/security-products/secret_detection#available-variables +# List of the variables: https://docs.gitlab.com/ee/user/application_security/secret_detection/#available-variables # How to set: https://docs.gitlab.com/ee/ci/yaml/#variables variables: diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml index c2db0fc44f1..200388a274c 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml @@ -52,7 +52,8 @@ cache: - gitlab-terraform apply when: manual only: - - master + variables: + - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH .destroy: &destroy stage: cleanup diff --git a/lib/gitlab/ci/variables/collection.rb b/lib/gitlab/ci/variables/collection.rb index f7bbb58df7e..45e2c535d3a 100644 --- a/lib/gitlab/ci/variables/collection.rb +++ b/lib/gitlab/ci/variables/collection.rb @@ -6,14 +6,22 @@ module Gitlab class Collection include Enumerable - def initialize(variables = []) + attr_reader :errors + + def initialize(variables = [], errors = nil) @variables = [] + @variables_by_key = {} + @errors = errors variables.each { |variable| self.append(variable) } end def append(resource) - tap { @variables.append(Collection::Item.fabricate(resource)) } + item = Collection::Item.fabricate(resource) + @variables.append(item) + @variables_by_key[item[:key]] = item + + self end def concat(resources) @@ -33,14 +41,31 @@ module Gitlab end end + def [](key) + @variables_by_key[key] + end + + def size + @variables.size + end + def to_runner_variables self.map(&:to_runner_variable) end def to_hash self.to_runner_variables - .map { |env| [env.fetch(:key), env.fetch(:value)] } - .to_h.with_indifferent_access + .to_h { |env| [env.fetch(:key), env.fetch(:value)] } + .with_indifferent_access + end + + def reject(&block) + Collection.new(@variables.reject(&block)) + end + + # Returns a sorted Collection object, and sets errors property in case of an error + def sorted_collection(project) + Sort.new(self, project).collection end end end diff --git a/lib/gitlab/ci/variables/collection/item.rb b/lib/gitlab/ci/variables/collection/item.rb index 84a9280e507..b52dfdc4bc1 100644 --- a/lib/gitlab/ci/variables/collection/item.rb +++ b/lib/gitlab/ci/variables/collection/item.rb @@ -5,13 +5,20 @@ module Gitlab module Variables class Collection class Item - def initialize(key:, value:, public: true, file: false, masked: false) + include Gitlab::Utils::StrongMemoize + + attr_reader :raw + + def initialize(key:, value:, public: true, file: false, masked: false, raw: false) raise ArgumentError, "`#{key}` must be of type String or nil value, while it was: #{value.class}" unless value.is_a?(String) || value.nil? - @variable = { - key: key, value: value, public: public, file: file, masked: masked - } + @variable = { key: key, value: value, public: public, file: file, masked: masked } + @raw = raw + end + + def value + @variable.fetch(:value) end def [](key) @@ -22,6 +29,16 @@ module Gitlab to_runner_variable == self.class.fabricate(other).to_runner_variable end + def depends_on + strong_memoize(:depends_on) do + next if raw + + next unless ExpandVariables.possible_var_reference?(value) + + value.scan(ExpandVariables::VARIABLES_REGEXP).map(&:first) + end + end + ## # If `file: true` has been provided we expose it, otherwise we # don't expose `file` attribute at all (stems from what the runner diff --git a/lib/gitlab/ci/variables/collection/sort.rb b/lib/gitlab/ci/variables/collection/sort.rb new file mode 100644 index 00000000000..94273ab3d67 --- /dev/null +++ b/lib/gitlab/ci/variables/collection/sort.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Variables + class Collection + class Sort + include TSort + include Gitlab::Utils::StrongMemoize + + def initialize(collection, project) + raise(ArgumentError, "A Gitlab::Ci::Variables::Collection object was expected") unless + collection.is_a?(Collection) + + @collection = collection + @project = project + end + + def valid? + errors.nil? + end + + # errors sorts an array of variables, ignoring unknown variable references, + # and returning an error string if a circular variable reference is found + def errors + return if Feature.disabled?(:variable_inside_variable, @project) + + strong_memoize(:errors) do + # Check for cyclic dependencies and build error message in that case + cyclic_vars = each_strongly_connected_component.filter_map do |component| + component.map { |v| v[:key] }.inspect if component.size > 1 + end + + "circular variable reference detected: #{cyclic_vars.join(', ')}" if cyclic_vars.any? + end + end + + # collection sorts a collection of variables, ignoring unknown variable references. + # If a circular variable reference is found, a new collection with the original array and an error is returned + def collection + return @collection if Feature.disabled?(:variable_inside_variable, @project) + + return Gitlab::Ci::Variables::Collection.new(@collection, errors) if errors + + Gitlab::Ci::Variables::Collection.new(tsort) + end + + private + + def tsort_each_node(&block) + @collection.each(&block) + end + + def tsort_each_child(var_item, &block) + depends_on = var_item.depends_on + return unless depends_on + + depends_on.filter_map { |var_ref_name| @collection[var_ref_name] }.each(&block) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/variables/collection/sorted.rb b/lib/gitlab/ci/variables/collection/sorted.rb deleted file mode 100644 index e641df10462..00000000000 --- a/lib/gitlab/ci/variables/collection/sorted.rb +++ /dev/null @@ -1,78 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Variables - class Collection - class Sorted - include TSort - include Gitlab::Utils::StrongMemoize - - def initialize(variables, project) - @variables = variables - @project = project - end - - def valid? - errors.nil? - end - - # errors sorts an array of variables, ignoring unknown variable references, - # and returning an error string if a circular variable reference is found - def errors - return if Feature.disabled?(:variable_inside_variable, @project) - - strong_memoize(:errors) do - # Check for cyclic dependencies and build error message in that case - errors = each_strongly_connected_component.filter_map do |component| - component.map { |v| v[:key] }.inspect if component.size > 1 - end - - "circular variable reference detected: #{errors.join(', ')}" if errors.any? - end - end - - # sort sorts an array of variables, ignoring unknown variable references. - # If a circular variable reference is found, the original array is returned - def sort - return @variables if Feature.disabled?(:variable_inside_variable, @project) - return @variables if errors - - tsort - end - - private - - def tsort_each_node(&block) - @variables.each(&block) - end - - def tsort_each_child(variable, &block) - each_variable_reference(variable[:value], &block) - end - - def input_vars - strong_memoize(:input_vars) do - @variables.index_by { |env| env.fetch(:key) } - end - end - - def walk_references(value) - return unless ExpandVariables.possible_var_reference?(value) - - value.scan(ExpandVariables::VARIABLES_REGEXP) do |var_ref| - yield(input_vars, var_ref.first) - end - end - - def each_variable_reference(value) - walk_references(value) do |vars_hash, ref_var_name| - variable = vars_hash.dig(ref_var_name) - yield variable if variable - end - end - end - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/summary/deploy.rb b/lib/gitlab/cycle_analytics/summary/deploy.rb index aaa2554dbfa..c247ef0d2a7 100644 --- a/lib/gitlab/cycle_analytics/summary/deploy.rb +++ b/lib/gitlab/cycle_analytics/summary/deploy.rb @@ -15,16 +15,10 @@ module Gitlab private def deployments_count - if Feature.enabled?(:query_deploymenys_via_finished_at_in_vsa, default_enabled: :yaml) - DeploymentsFinder - .new(project: @project, finished_after: @from, finished_before: @to, status: :success) - .execute - .count - else - query = @project.deployments.success.where("created_at >= ?", @from) - query = query.where("created_at <= ?", @to) if @to - query.count - end + DeploymentsFinder + .new(project: @project, finished_after: @from, finished_before: @to, status: :success) + .execute + .count end end end diff --git a/lib/gitlab/data_builder/build.rb b/lib/gitlab/data_builder/build.rb index e17bd25e57e..c4af5e6608e 100644 --- a/lib/gitlab/data_builder/build.rb +++ b/lib/gitlab/data_builder/build.rb @@ -62,7 +62,9 @@ module Gitlab git_http_url: project.http_url_to_repo, git_ssh_url: project.ssh_url_to_repo, visibility_level: project.visibility_level - } + }, + + environment: build_environment(build) } data @@ -86,6 +88,15 @@ module Gitlab tags: runner.tags&.map(&:name) } end + + def build_environment(build) + return unless build.has_environment? + + { + name: build.expanded_environment_name, + action: build.environment_action + } + end end end end diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index 3036bc57ca5..7fd1b9cd228 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -67,7 +67,8 @@ module Gitlab artifacts_file: { filename: build.artifacts_file&.filename, size: build.artifacts_size - } + }, + environment: environment_hook_attrs(build) } end @@ -80,6 +81,15 @@ module Gitlab tags: runner.tags&.map(&:name) } end + + def environment_hook_attrs(build) + return unless build.has_environment? + + { + name: build.expanded_environment_name, + action: build.environment_action + } + end end end end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 45d271a2fd4..2b3c98ffa14 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -313,28 +313,18 @@ module Gitlab ActiveRecord::Base.prepend(ActiveRecordBaseTransactionMetrics) end - # observe_transaction_duration is called from ActiveRecordBaseTransactionMetrics.transaction and used to - # record transaction durations. - def self.observe_transaction_duration(duration_seconds) - if current_transaction = ::Gitlab::Metrics::Transaction.current - current_transaction.observe(:gitlab_database_transaction_seconds, duration_seconds) do - docstring "Time spent in database transactions, in seconds" - end - end - rescue Prometheus::Client::LabelSetValidator::LabelSetError => err - # Ensure that errors in recording these metrics don't affect the operation of the application - Gitlab::AppLogger.error("Unable to observe database transaction duration: #{err}") - end - # MonkeyPatch for ActiveRecord::Base for adding observability module ActiveRecordBaseTransactionMetrics - # A monkeypatch over ActiveRecord::Base.transaction. - # It provides observability into transactional methods. - def transaction(options = {}, &block) - start_time = Gitlab::Metrics::System.monotonic_time - super(options, &block) - ensure - Gitlab::Database.observe_transaction_duration(Gitlab::Metrics::System.monotonic_time - start_time) + extend ActiveSupport::Concern + + class_methods do + # A monkeypatch over ActiveRecord::Base.transaction. + # It provides observability into transactional methods. + def transaction(**options, &block) + ActiveSupport::Notifications.instrument('transaction.active_record', { connection: connection }) do + super(**options, &block) + end + end end end end diff --git a/lib/gitlab/database/background_migration/batched_job.rb b/lib/gitlab/database/background_migration/batched_job.rb new file mode 100644 index 00000000000..3b624df2bfd --- /dev/null +++ b/lib/gitlab/database/background_migration/batched_job.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module BackgroundMigration + class BatchedJob < ActiveRecord::Base # rubocop:disable Rails/ApplicationRecord + self.table_name = :batched_background_migration_jobs + + belongs_to :batched_migration, foreign_key: :batched_background_migration_id + + enum status: { + pending: 0, + running: 1, + failed: 2, + succeeded: 3 + } + + delegate :aborted?, :job_class, :table_name, :column_name, :job_arguments, + to: :batched_migration, prefix: :migration + end + end + end +end diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb new file mode 100644 index 00000000000..0c9add9b355 --- /dev/null +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module BackgroundMigration + class BatchedMigration < ActiveRecord::Base # rubocop:disable Rails/ApplicationRecord + JOB_CLASS_MODULE = 'Gitlab::BackgroundMigration' + BATCH_CLASS_MODULE = "#{JOB_CLASS_MODULE}::BatchingStrategies".freeze + + self.table_name = :batched_background_migrations + + has_many :batched_jobs, foreign_key: :batched_background_migration_id + has_one :last_job, -> { order(id: :desc) }, + class_name: 'Gitlab::Database::BackgroundMigration::BatchedJob', + foreign_key: :batched_background_migration_id + + scope :queue_order, -> { order(id: :asc) } + + enum status: { + paused: 0, + active: 1, + aborted: 2, + finished: 3 + } + + def interval_elapsed? + last_job.nil? || last_job.created_at <= Time.current - interval + end + + def create_batched_job!(min, max) + batched_jobs.create!(min_value: min, max_value: max, batch_size: batch_size, sub_batch_size: sub_batch_size) + end + + def next_min_value + last_job&.max_value&.next || min_value + end + + def job_class + "#{JOB_CLASS_MODULE}::#{job_class_name}".constantize + end + + def batch_class + "#{BATCH_CLASS_MODULE}::#{batch_class_name}".constantize + end + + def job_class_name=(class_name) + write_attribute(:job_class_name, class_name.demodulize) + end + + def batch_class_name=(class_name) + write_attribute(:batch_class_name, class_name.demodulize) + end + end + end + end +end diff --git a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb new file mode 100644 index 00000000000..299bd992197 --- /dev/null +++ b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module BackgroundMigration + class BatchedMigrationWrapper + def perform(batch_tracking_record) + start_tracking_execution(batch_tracking_record) + + execute_batch(batch_tracking_record) + + batch_tracking_record.status = :succeeded + rescue => e + batch_tracking_record.status = :failed + + raise e + ensure + finish_tracking_execution(batch_tracking_record) + end + + private + + def start_tracking_execution(tracking_record) + tracking_record.update!(attempts: tracking_record.attempts + 1, status: :running, started_at: Time.current) + end + + def execute_batch(tracking_record) + job_instance = tracking_record.migration_job_class.new + + job_instance.perform( + tracking_record.min_value, + tracking_record.max_value, + tracking_record.migration_table_name, + tracking_record.migration_column_name, + tracking_record.sub_batch_size, + *tracking_record.migration_job_arguments) + end + + def finish_tracking_execution(tracking_record) + tracking_record.finished_at = Time.current + tracking_record.save! + end + end + end + end +end diff --git a/lib/gitlab/database/background_migration/scheduler.rb b/lib/gitlab/database/background_migration/scheduler.rb new file mode 100644 index 00000000000..5f8a5ec06a5 --- /dev/null +++ b/lib/gitlab/database/background_migration/scheduler.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module BackgroundMigration + class Scheduler + def perform(migration_wrapper: BatchedMigrationWrapper.new) + active_migration = BatchedMigration.active.queue_order.first + + return unless active_migration&.interval_elapsed? + + if next_batched_job = create_next_batched_job!(active_migration) + migration_wrapper.perform(next_batched_job) + else + finish_active_migration(active_migration) + end + end + + private + + def create_next_batched_job!(active_migration) + next_batch_range = find_next_batch_range(active_migration) + + return if next_batch_range.nil? + + active_migration.create_batched_job!(next_batch_range.min, next_batch_range.max) + end + + def find_next_batch_range(active_migration) + batching_strategy = active_migration.batch_class.new + batch_min_value = active_migration.next_min_value + + next_batch_bounds = batching_strategy.next_batch( + active_migration.table_name, + active_migration.column_name, + batch_min_value: batch_min_value, + batch_size: active_migration.batch_size) + + return if next_batch_bounds.nil? + + clamped_batch_range(active_migration, next_batch_bounds) + end + + def clamped_batch_range(active_migration, next_bounds) + min_value, max_value = next_bounds + + return if min_value > active_migration.max_value + + max_value = max_value.clamp(min_value, active_migration.max_value) + + (min_value..max_value) + end + + def finish_active_migration(active_migration) + active_migration.finished! + end + end + end + end +end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 6b169a504f3..e8ed3bb1258 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -1015,7 +1015,7 @@ module Gitlab 'CopyColumnUsingBackgroundMigrationJob', interval, batch_size: batch_size, - other_job_arguments: [table, primary_key, column, tmp_column, sub_batch_size], + other_job_arguments: [table, primary_key, sub_batch_size, column, tmp_column], track_jobs: true, primary_column_name: primary_key ) diff --git a/lib/gitlab/database/migrations/background_migration_helpers.rb b/lib/gitlab/database/migrations/background_migration_helpers.rb index 12dcf68da2f..e8cbea72887 100644 --- a/lib/gitlab/database/migrations/background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/background_migration_helpers.rb @@ -4,8 +4,12 @@ module Gitlab module Database module Migrations module BackgroundMigrationHelpers - BACKGROUND_MIGRATION_BATCH_SIZE = 1_000 # Number of rows to process per job - BACKGROUND_MIGRATION_JOB_BUFFER_SIZE = 1_000 # Number of jobs to bulk queue at a time + BATCH_SIZE = 1_000 # Number of rows to process per job + SUB_BATCH_SIZE = 100 # Number of rows to process per sub-batch + JOB_BUFFER_SIZE = 1_000 # Number of jobs to bulk queue at a time + BATCH_CLASS_NAME = 'PrimaryKeyBatchingStrategy' # Default batch class for batched migrations + BATCH_MIN_VALUE = 1 # Default minimum value for batched migrations + BATCH_MIN_DELAY = 2.minutes.freeze # Minimum delay between batched migrations # Bulk queues background migration jobs for an entire table, batched by ID range. # "Bulk" meaning many jobs will be pushed at a time for efficiency. @@ -31,7 +35,7 @@ module Gitlab # # do something # end # end - def bulk_queue_background_migration_jobs_by_range(model_class, job_class_name, batch_size: BACKGROUND_MIGRATION_BATCH_SIZE) + def bulk_queue_background_migration_jobs_by_range(model_class, job_class_name, batch_size: BATCH_SIZE) raise "#{model_class} does not have an ID to use for batch ranges" unless model_class.column_names.include?('id') jobs = [] @@ -40,7 +44,7 @@ module Gitlab model_class.each_batch(of: batch_size) do |relation| start_id, end_id = relation.pluck("MIN(#{table_name}.id)", "MAX(#{table_name}.id)").first - if jobs.length >= BACKGROUND_MIGRATION_JOB_BUFFER_SIZE + if jobs.length >= JOB_BUFFER_SIZE # Note: This code path generally only helps with many millions of rows # We push multiple jobs at a time to reduce the time spent in # Sidekiq/Redis operations. We're using this buffer based approach so we @@ -89,7 +93,7 @@ module Gitlab # # do something # end # end - def queue_background_migration_jobs_by_range_at_intervals(model_class, job_class_name, delay_interval, batch_size: BACKGROUND_MIGRATION_BATCH_SIZE, other_job_arguments: [], initial_delay: 0, track_jobs: false, primary_column_name: :id) + def queue_background_migration_jobs_by_range_at_intervals(model_class, job_class_name, delay_interval, batch_size: BATCH_SIZE, other_job_arguments: [], initial_delay: 0, track_jobs: false, primary_column_name: :id) raise "#{model_class} does not have an ID column of #{primary_column_name} to use for batch ranges" unless model_class.column_names.include?(primary_column_name.to_s) raise "#{primary_column_name} is not an integer column" unless model_class.columns_hash[primary_column_name.to_s].type == :integer @@ -127,6 +131,79 @@ module Gitlab final_delay end + # Creates a batched background migration for the given table. A batched migration runs one job + # at a time, computing the bounds of the next batch based on the current migration settings and the previous + # batch bounds. Each job's execution status is tracked in the database as the migration runs. The given job + # class must be present in the Gitlab::BackgroundMigration module, and the batch class (if specified) must be + # present in the Gitlab::BackgroundMigration::BatchingStrategies module. + # + # job_class_name - The background migration job class as a string + # batch_table_name - The name of the table the migration will batch over + # batch_column_name - The name of the column the migration will batch over + # job_arguments - Extra arguments to pass to the job instance when the migration runs + # job_interval - The pause interval between each job's execution, minimum of 2 minutes + # batch_min_value - The value in the column the batching will begin at + # batch_max_value - The value in the column the batching will end at, defaults to `SELECT MAX(batch_column)` + # batch_class_name - The name of the class that will be called to find the range of each next batch + # batch_size - The maximum number of rows per job + # sub_batch_size - The maximum number of rows processed per "iteration" within the job + # + # + # *Returns the created BatchedMigration record* + # + # Example: + # + # queue_batched_background_migration( + # 'CopyColumnUsingBackgroundMigrationJob', + # :events, + # :id, + # job_interval: 2.minutes, + # other_job_arguments: ['column1', 'column2']) + # + # Where the the background migration exists: + # + # class Gitlab::BackgroundMigration::CopyColumnUsingBackgroundMigrationJob + # def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, *other_args) + # # do something + # end + # end + def queue_batched_background_migration( # rubocop:disable Metrics/ParameterLists + job_class_name, + batch_table_name, + batch_column_name, + *job_arguments, + job_interval:, + batch_min_value: BATCH_MIN_VALUE, + batch_max_value: nil, + batch_class_name: BATCH_CLASS_NAME, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + + job_interval = BATCH_MIN_DELAY if job_interval < BATCH_MIN_DELAY + + batch_max_value ||= connection.select_value(<<~SQL) + SELECT MAX(#{connection.quote_column_name(batch_column_name)}) + FROM #{connection.quote_table_name(batch_table_name)} + SQL + + migration_status = batch_max_value.nil? ? :finished : :active + batch_max_value ||= batch_min_value + + Gitlab::Database::BackgroundMigration::BatchedMigration.create!( + job_class_name: job_class_name, + table_name: batch_table_name, + column_name: batch_column_name, + interval: job_interval, + min_value: batch_min_value, + max_value: batch_max_value, + batch_class_name: batch_class_name, + batch_size: batch_size, + sub_batch_size: sub_batch_size, + job_arguments: job_arguments, + status: migration_status) + end + def perform_background_migration_inline? Rails.env.test? || Rails.env.development? end diff --git a/lib/gitlab/database/migrations/observation.rb b/lib/gitlab/database/migrations/observation.rb index 518c2c560d2..046843824a4 100644 --- a/lib/gitlab/database/migrations/observation.rb +++ b/lib/gitlab/database/migrations/observation.rb @@ -7,7 +7,8 @@ module Gitlab :migration, :walltime, :success, - :total_database_size_change + :total_database_size_change, + :query_statistics ) end end diff --git a/lib/gitlab/database/migrations/observers.rb b/lib/gitlab/database/migrations/observers.rb index 4b931d3c19c..592993aeac5 100644 --- a/lib/gitlab/database/migrations/observers.rb +++ b/lib/gitlab/database/migrations/observers.rb @@ -6,7 +6,8 @@ module Gitlab module Observers def self.all_observers [ - TotalDatabaseSizeChange.new + TotalDatabaseSizeChange.new, + QueryStatistics.new ] end end diff --git a/lib/gitlab/database/migrations/observers/query_statistics.rb b/lib/gitlab/database/migrations/observers/query_statistics.rb new file mode 100644 index 00000000000..466f4724256 --- /dev/null +++ b/lib/gitlab/database/migrations/observers/query_statistics.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + module Observers + # This observer gathers statistics from the pg_stat_statements extension. + # Notice that this extension is not installed by default. In case it cannot + # be found, the observer does nothing and doesn't throw an error. + class QueryStatistics < MigrationObserver + include Gitlab::Database::SchemaHelpers + + def before + return unless enabled? + + connection.execute('select pg_stat_statements_reset()') + end + + def record(observation) + return unless enabled? + + observation.query_statistics = connection.execute(<<~SQL) + SELECT query, calls, total_time, max_time, mean_time, rows + FROM pg_stat_statements + ORDER BY total_time DESC + SQL + end + + private + + def enabled? + function_exists?(:pg_stat_statements_reset) && connection.view_exists?(:pg_stat_statements) + end + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb index f4cf576dda7..1c289391e21 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb @@ -9,7 +9,7 @@ module Gitlab include ::Gitlab::Database::MigrationHelpers include ::Gitlab::Database::Migrations::BackgroundMigrationHelpers - ALLOWED_TABLES = %w[audit_events].freeze + ALLOWED_TABLES = %w[audit_events web_hook_logs].freeze ERROR_SCOPE = 'table partitioning' MIGRATION_CLASS_NAME = "::#{module_parent_name}::BackfillPartitionedTable" diff --git a/lib/gitlab/database/similarity_score.rb b/lib/gitlab/database/similarity_score.rb index ff78fd0218c..40845c0d5e0 100644 --- a/lib/gitlab/database/similarity_score.rb +++ b/lib/gitlab/database/similarity_score.rb @@ -74,9 +74,14 @@ module Gitlab end # (SIMILARITY ...) + (SIMILARITY ...) - expressions.inject(first_expression) do |expression1, expression2| + additions = expressions.inject(first_expression) do |expression1, expression2| Arel::Nodes::Addition.new(expression1, expression2) end + + score_as_numeric = Arel::Nodes::NamedFunction.new('CAST', [Arel::Nodes::Grouping.new(additions).as('numeric')]) + + # Rounding the score to two decimals + Arel::Nodes::NamedFunction.new('ROUND', [score_as_numeric, 2]) end def self.order_by_similarity?(arel_query) diff --git a/lib/gitlab/dependency_linker/base_linker.rb b/lib/gitlab/dependency_linker/base_linker.rb index d735fb55652..36a840372c5 100644 --- a/lib/gitlab/dependency_linker/base_linker.rb +++ b/lib/gitlab/dependency_linker/base_linker.rb @@ -80,7 +80,7 @@ module Gitlab highlighted_lines.map!.with_index do |rich_line, i| marker = StringRegexMarker.new((plain_lines[i].chomp! || plain_lines[i]), rich_line.html_safe) - marker.mark(regex, group: :name) do |text, left:, right:| + marker.mark(regex, group: :name) do |text, left:, right:, mode:| url = yield(text) url ? link_tag(text, url) : text end diff --git a/lib/gitlab/dependency_linker/go_mod_linker.rb b/lib/gitlab/dependency_linker/go_mod_linker.rb index 4d6fe366333..fae4ee23383 100644 --- a/lib/gitlab/dependency_linker/go_mod_linker.rb +++ b/lib/gitlab/dependency_linker/go_mod_linker.rb @@ -22,7 +22,7 @@ module Gitlab i, j = match.offset(:name) marker = StringRangeMarker.new(plain_line, rich_line.html_safe) - marker.mark([i..(j - 1)]) do |text, left:, right:| + marker.mark([i..(j - 1)]) do |text, left:, right:, mode:| url = package_url(text, match[:version]) url ? link_tag(text, url) : text end diff --git a/lib/gitlab/dependency_linker/go_sum_linker.rb b/lib/gitlab/dependency_linker/go_sum_linker.rb index 20dc82ede9f..44826332f66 100644 --- a/lib/gitlab/dependency_linker/go_sum_linker.rb +++ b/lib/gitlab/dependency_linker/go_sum_linker.rb @@ -21,7 +21,7 @@ module Gitlab i2, j2 = match.offset(:checksum) marker = StringRangeMarker.new(plain_line, rich_line.html_safe) - marker.mark([i0..(j0 - 1), i2..(j2 - 1)]) do |text, left:, right:| + marker.mark([i0..(j0 - 1), i2..(j2 - 1)]) do |text, left:, right:, mode:| if left url = package_url(text, match[:version]) url ? link_tag(text, url) : text diff --git a/lib/gitlab/diff/char_diff.rb b/lib/gitlab/diff/char_diff.rb index c8bb39e9f5d..1b3af8f75ca 100644 --- a/lib/gitlab/diff/char_diff.rb +++ b/lib/gitlab/diff/char_diff.rb @@ -32,12 +32,12 @@ module Gitlab end if action == :delete - old_diffs << (old_pointer..(old_pointer + content_size - 1)) + old_diffs << MarkerRange.new(old_pointer, old_pointer + content_size - 1, mode: MarkerRange::DELETION) old_pointer += content_size end if action == :insert - new_diffs << (new_pointer..(new_pointer + content_size - 1)) + new_diffs << MarkerRange.new(new_pointer, new_pointer + content_size - 1, mode: MarkerRange::ADDITION) new_pointer += content_size end end diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 035084d4861..baa46e7e306 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -31,6 +31,12 @@ module Gitlab if line_inline_diffs = inline_diffs[i] begin + # MarkerRange objects are converted to Ranges to keep the previous behavior + # Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/324068 + if Feature.disabled?(:introduce_marker_ranges, project, default_enabled: :yaml) + line_inline_diffs = line_inline_diffs.map { |marker_range| marker_range.to_range } + end + rich_line = InlineDiffMarker.new(diff_line.text, rich_line).mark(line_inline_diffs) # This should only happen when the encoding of the diff doesn't # match the blob, which is a bug. But we shouldn't fail to render @@ -67,7 +73,7 @@ module Gitlab end def inline_diffs - @inline_diffs ||= InlineDiff.for_lines(@raw_lines, project: project) + @inline_diffs ||= InlineDiff.for_lines(@raw_lines) end def old_lines diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb index 7932cd2a837..c5e9bfdc321 100644 --- a/lib/gitlab/diff/highlight_cache.rb +++ b/lib/gitlab/diff/highlight_cache.rb @@ -7,8 +7,7 @@ module Gitlab include Gitlab::Utils::StrongMemoize EXPIRATION = 1.week - VERSION = 1 - NEXT_VERSION = 2 + VERSION = 2 delegate :diffable, to: :@diff_collection delegate :diff_options, to: :@diff_collection @@ -70,20 +69,17 @@ module Gitlab def key strong_memoize(:redis_key) do - ['highlighted-diff-files', diffable.cache_key, version, diff_options].join(":") + [ + 'highlighted-diff-files', + diffable.cache_key, VERSION, + diff_options, + Feature.enabled?(:introduce_marker_ranges, diffable.project, default_enabled: :yaml) + ].join(":") end end private - def version - if Feature.enabled?(:improved_merge_diff_highlighting, diffable.project, default_enabled: :yaml) - NEXT_VERSION - else - VERSION - end - end - def set_highlighted_diff_lines(diff_file, content) diff_file.highlighted_diff_lines = content.map do |line| Gitlab::Diff::Line.safe_init_from_hash(line) diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb index cf769262958..dd73e4d6c15 100644 --- a/lib/gitlab/diff/inline_diff.rb +++ b/lib/gitlab/diff/inline_diff.rb @@ -3,22 +3,6 @@ module Gitlab module Diff class InlineDiff - # Regex to find a run of deleted lines followed by the same number of added lines - LINE_PAIRS_PATTERN = %r{ - # Runs start at the beginning of the string (the first line) or after a space (for an unchanged line) - (?:\A|\s) - - # This matches a number of `-`s followed by the same number of `+`s through recursion - (?<del_ins> - - - \g<del_ins>? - \+ - ) - - # Runs end at the end of the string (the last line) or before a space (for an unchanged line) - (?=\s|\z) - }x.freeze - attr_accessor :old_line, :new_line, :offset def initialize(old_line, new_line, offset: 0) @@ -27,28 +11,24 @@ module Gitlab @offset = offset end - def inline_diffs(project: nil) + def inline_diffs # Skip inline diff if empty line was replaced with content return if old_line == "" - if Feature.enabled?(:improved_merge_diff_highlighting, project, default_enabled: :yaml) - CharDiff.new(old_line, new_line).changed_ranges(offset: offset) - else - deprecated_diff - end + CharDiff.new(old_line, new_line).changed_ranges(offset: offset) end class << self - def for_lines(lines, project: nil) - changed_line_pairs = find_changed_line_pairs(lines) + def for_lines(lines) + pair_selector = Gitlab::Diff::PairSelector.new(lines) inline_diffs = [] - changed_line_pairs.each do |old_index, new_index| + pair_selector.each do |old_index, new_index| old_line = lines[old_index] new_line = lines[new_index] - old_diffs, new_diffs = new(old_line, new_line, offset: 1).inline_diffs(project: project) + old_diffs, new_diffs = new(old_line, new_line, offset: 1).inline_diffs inline_diffs[old_index] = old_diffs inline_diffs[new_index] = new_diffs @@ -56,74 +36,6 @@ module Gitlab inline_diffs end - - private - - # Finds pairs of old/new line pairs that represent the same line that changed - # rubocop: disable CodeReuse/ActiveRecord - def find_changed_line_pairs(lines) - # Prefixes of all diff lines, indicating their types - # For example: `" - + -+ ---+++ --+ -++"` - line_prefixes = lines.each_with_object(+"") { |line, s| s << (line[0] || ' ') }.gsub(/[^ +-]/, ' ') - - changed_line_pairs = [] - line_prefixes.scan(LINE_PAIRS_PATTERN) do - # For `"---+++"`, `begin_index == 0`, `end_index == 6` - begin_index, end_index = Regexp.last_match.offset(:del_ins) - - # For `"---+++"`, `changed_line_count == 3` - changed_line_count = (end_index - begin_index) / 2 - - halfway_index = begin_index + changed_line_count - (begin_index...halfway_index).each do |i| - # For `"---+++"`, index 1 maps to 1 + 3 = 4 - changed_line_pairs << [i, i + changed_line_count] - end - end - - changed_line_pairs - end - # rubocop: enable CodeReuse/ActiveRecord - end - - private - - # See: https://gitlab.com/gitlab-org/gitlab/-/issues/299884 - def deprecated_diff - lcp = longest_common_prefix(old_line, new_line) - lcs = longest_common_suffix(old_line[lcp..-1], new_line[lcp..-1]) - - lcp += offset - old_length = old_line.length + offset - new_length = new_line.length + offset - - old_diff_range = lcp..(old_length - lcs - 1) - new_diff_range = lcp..(new_length - lcs - 1) - - old_diffs = [old_diff_range] if old_diff_range.begin <= old_diff_range.end - new_diffs = [new_diff_range] if new_diff_range.begin <= new_diff_range.end - - [old_diffs, new_diffs] - end - - def longest_common_prefix(a, b) # rubocop:disable Naming/UncommunicativeMethodParamName - max_length = [a.length, b.length].max - - length = 0 - (0..max_length - 1).each do |pos| - old_char = a[pos] - new_char = b[pos] - - break if old_char != new_char - - length += 1 - end - - length - end - - def longest_common_suffix(a, b) # rubocop:disable Naming/UncommunicativeMethodParamName - longest_common_prefix(a.reverse, b.reverse) end end end diff --git a/lib/gitlab/diff/inline_diff_markdown_marker.rb b/lib/gitlab/diff/inline_diff_markdown_marker.rb index 3c536c43a9e..d8d596ebce7 100644 --- a/lib/gitlab/diff/inline_diff_markdown_marker.rb +++ b/lib/gitlab/diff/inline_diff_markdown_marker.rb @@ -8,8 +8,8 @@ module Gitlab deletion: "-" }.freeze - def mark(line_inline_diffs, mode: nil) - super(line_inline_diffs) do |text, left:, right:| + def mark(line_inline_diffs) + super(line_inline_diffs) do |text, left:, right:, mode:| symbol = MARKDOWN_SYMBOLS[mode] "{#{symbol}#{text}#{symbol}}" end diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb index 29dff699ba5..c8cc1c0e649 100644 --- a/lib/gitlab/diff/inline_diff_marker.rb +++ b/lib/gitlab/diff/inline_diff_marker.rb @@ -7,8 +7,8 @@ module Gitlab super(line, rich_line || line) end - def mark(line_inline_diffs, mode: nil) - super(line_inline_diffs) do |text, left:, right:| + def mark(line_inline_diffs) + super(line_inline_diffs) do |text, left:, right:, mode:| %{<span class="#{html_class_names(left, right, mode)}">#{text}</span>}.html_safe end end diff --git a/lib/gitlab/diff/pair_selector.rb b/lib/gitlab/diff/pair_selector.rb new file mode 100644 index 00000000000..2e5ee3a7363 --- /dev/null +++ b/lib/gitlab/diff/pair_selector.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Gitlab + module Diff + class PairSelector + include Enumerable + + # Regex to find a run of deleted lines followed by the same number of added lines + # rubocop: disable Lint/MixedRegexpCaptureTypes + LINE_PAIRS_PATTERN = %r{ + # Runs start at the beginning of the string (the first line) or after a space (for an unchanged line) + (?:\A|\s) + + # This matches a number of `-`s followed by the same number of `+`s through recursion + (?<del_ins> + - + \g<del_ins>? + \+ + ) + + # Runs end at the end of the string (the last line) or before a space (for an unchanged line) + (?=\s|\z) + }x.freeze + # rubocop: enable Lint/MixedRegexpCaptureTypes + + def initialize(lines) + @lines = lines + end + + # Finds pairs of old/new line pairs that represent the same line that changed + # rubocop: disable CodeReuse/ActiveRecord + def each + # Prefixes of all diff lines, indicating their types + # For example: `" - + -+ ---+++ --+ -++"` + line_prefixes = lines.each_with_object(+"") { |line, s| s << (line[0] || ' ') }.gsub(/[^ +-]/, ' ') + + line_prefixes.scan(LINE_PAIRS_PATTERN) do + # For `"---+++"`, `begin_index == 0`, `end_index == 6` + begin_index, end_index = Regexp.last_match.offset(:del_ins) + + # For `"---+++"`, `changed_line_count == 3` + changed_line_count = (end_index - begin_index) / 2 + + halfway_index = begin_index + changed_line_count + (begin_index...halfway_index).each do |i| + # For `"---+++"`, index 1 maps to 1 + 3 = 4 + yield [i, i + changed_line_count] + end + end + end + # rubocop: enable CodeReuse/ActiveRecord + + private + + attr_reader :lines + end + end +end diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb index d1dd616385d..80e8b726099 100644 --- a/lib/gitlab/email/handler/service_desk_handler.rb +++ b/lib/gitlab/email/handler/service_desk_handler.rb @@ -79,7 +79,7 @@ module Gitlab @issue = Issues::CreateService.new( project, User.support_bot, - title: issue_title, + title: mail.subject, description: message_including_template, confidential: true, external_author: from_address @@ -137,12 +137,6 @@ module Gitlab (mail.reply_to || []).first || mail.from.first || mail.sender end - def issue_title - from = "(from #{from_address})" if from_address - - "Service Desk #{from}: #{mail.subject}" - end - def can_handle_legacy_format? project_path && project_path.include?('/') && !mail_key.include?('+') end diff --git a/lib/gitlab/error_tracking.rb b/lib/gitlab/error_tracking.rb index 1a8e5aaf07a..dfed8db8df0 100644 --- a/lib/gitlab/error_tracking.rb +++ b/lib/gitlab/error_tracking.rb @@ -27,33 +27,16 @@ module Gitlab config.sanitize_fields = Rails.application.config.filter_parameters.map(&:to_s) config.processors << ::Gitlab::ErrorTracking::Processor::SidekiqProcessor config.processors << ::Gitlab::ErrorTracking::Processor::GrpcErrorProcessor + config.processors << ::Gitlab::ErrorTracking::Processor::ContextPayloadProcessor # Sanitize authentication headers config.sanitize_http_headers = %w[Authorization Private-Token] - config.tags = extra_tags_from_env.merge(program: Gitlab.process_name) config.before_send = method(:before_send) yield config if block_given? end end - def with_context(current_user = nil) - last_user_context = Raven.context.user - - user_context = { - id: current_user&.id, - email: current_user&.email, - username: current_user&.username - }.compact - - Raven.tags_context(default_tags) - Raven.user_context(user_context) - - yield - ensure - Raven.user_context(last_user_context) - end - # This should be used when you want to passthrough exception handling: # rescue and raise to be catched in upper layers of the application. # @@ -118,37 +101,20 @@ module Gitlab end def process_exception(exception, sentry: false, logging: true, extra:) - exception.try(:sentry_extra_data)&.tap do |data| - extra = extra.merge(data) if data.is_a?(Hash) - end - - extra = sanitize_request_parameters(extra) + context_payload = Gitlab::ErrorTracking::ContextPayloadGenerator.generate(exception, extra) if sentry && Raven.configuration.server - Raven.capture_exception(exception, tags: default_tags, extra: extra) + Raven.capture_exception(exception, **context_payload) end if logging - # TODO: this logic could migrate into `Gitlab::ExceptionLogFormatter` - # and we could also flatten deep nested hashes if required for search - # (e.g. if `extra` includes hash of hashes). - # In the current implementation, we don't flatten multi-level folded hashes. - log_hash = {} - Raven.context.tags.each { |name, value| log_hash["tags.#{name}"] = value } - Raven.context.user.each { |name, value| log_hash["user.#{name}"] = value } - Raven.context.extra.merge(extra).each { |name, value| log_hash["extra.#{name}"] = value } - - Gitlab::ExceptionLogFormatter.format!(exception, log_hash) + formatter = Gitlab::ErrorTracking::LogFormatter.new + log_hash = formatter.generate_log(exception, context_payload) Gitlab::ErrorTracking::Logger.error(log_hash) end end - def sanitize_request_parameters(parameters) - filter = ActiveSupport::ParameterFilter.new(::Rails.application.config.filter_parameters) - filter.filter(parameters) - end - def sentry_dsn return unless Rails.env.production? || Rails.env.development? return unless Gitlab.config.sentry.enabled @@ -160,22 +126,6 @@ module Gitlab Rails.env.development? || Rails.env.test? end - def default_tags - { - Labkit::Correlation::CorrelationId::LOG_KEY.to_sym => Labkit::Correlation::CorrelationId.current_id, - locale: I18n.locale - } - end - - # Static tags that are set on application start - def extra_tags_from_env - Gitlab::Json.parse(ENV.fetch('GITLAB_SENTRY_EXTRA_TAGS', '{}')).to_hash - rescue => e - Gitlab::AppLogger.debug("GITLAB_SENTRY_EXTRA_TAGS could not be parsed as JSON: #{e.class.name}: #{e.message}") - - {} - end - # Group common, mostly non-actionable exceptions by type and message, # rather than cause def custom_fingerprinting(event, ex) diff --git a/lib/gitlab/error_tracking/context_payload_generator.rb b/lib/gitlab/error_tracking/context_payload_generator.rb new file mode 100644 index 00000000000..c99283b3d20 --- /dev/null +++ b/lib/gitlab/error_tracking/context_payload_generator.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module Gitlab + module ErrorTracking + class ContextPayloadGenerator + def self.generate(exception, extra = {}) + new.generate(exception, extra) + end + + def generate(exception, extra = {}) + { + extra: extra_payload(exception, extra), + tags: tags_payload, + user: user_payload + } + end + + private + + def extra_payload(exception, extra) + inline_extra = exception.try(:sentry_extra_data) + if inline_extra.present? && inline_extra.is_a?(Hash) + extra = extra.merge(inline_extra) + end + + sanitize_request_parameters(extra) + end + + def sanitize_request_parameters(parameters) + filter = ActiveSupport::ParameterFilter.new(::Rails.application.config.filter_parameters) + filter.filter(parameters) + end + + def tags_payload + extra_tags_from_env.merge!( + program: Gitlab.process_name, + locale: I18n.locale, + feature_category: current_context['meta.feature_category'], + Labkit::Correlation::CorrelationId::LOG_KEY.to_sym => Labkit::Correlation::CorrelationId.current_id + ) + end + + def user_payload + { + username: current_context['meta.user'] + } + end + + # Static tags that are set on application start + def extra_tags_from_env + Gitlab::Json.parse(ENV.fetch('GITLAB_SENTRY_EXTRA_TAGS', '{}')).to_hash + rescue => e + Gitlab::AppLogger.debug("GITLAB_SENTRY_EXTRA_TAGS could not be parsed as JSON: #{e.class.name}: #{e.message}") + + {} + end + + def current_context + # In case Gitlab::ErrorTracking is used when the app starts + return {} unless defined?(::Gitlab::ApplicationContext) + + ::Gitlab::ApplicationContext.current.to_h + end + end + end +end diff --git a/lib/gitlab/error_tracking/detailed_error.rb b/lib/gitlab/error_tracking/detailed_error.rb index 5d272efa64a..d0b3fc176aa 100644 --- a/lib/gitlab/error_tracking/detailed_error.rb +++ b/lib/gitlab/error_tracking/detailed_error.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +# This should be in the ErrorTracking namespace. For more details, see: +# https://gitlab.com/gitlab-org/gitlab/-/issues/323342 module Gitlab module ErrorTracking class DetailedError diff --git a/lib/gitlab/error_tracking/error.rb b/lib/gitlab/error_tracking/error.rb index 6bfb9dae610..a256f87ec3d 100644 --- a/lib/gitlab/error_tracking/error.rb +++ b/lib/gitlab/error_tracking/error.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +# This should be in the ErrorTracking namespace. For more details, see: +# https://gitlab.com/gitlab-org/gitlab/-/issues/323342 module Gitlab module ErrorTracking class Error diff --git a/lib/gitlab/error_tracking/error_collection.rb b/lib/gitlab/error_tracking/error_collection.rb index 56bcb671363..d01064bb677 100644 --- a/lib/gitlab/error_tracking/error_collection.rb +++ b/lib/gitlab/error_tracking/error_collection.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +# This should be in the ErrorTracking namespace. For more details, see: +# https://gitlab.com/gitlab-org/gitlab/-/issues/323342 module Gitlab module ErrorTracking class ErrorCollection diff --git a/lib/gitlab/error_tracking/error_event.rb b/lib/gitlab/error_tracking/error_event.rb index 015d2c0ead0..d80289f6bc9 100644 --- a/lib/gitlab/error_tracking/error_event.rb +++ b/lib/gitlab/error_tracking/error_event.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +# This should be in the ErrorTracking namespace. For more details, see: +# https://gitlab.com/gitlab-org/gitlab/-/issues/323342 module Gitlab module ErrorTracking class ErrorEvent diff --git a/lib/gitlab/error_tracking/log_formatter.rb b/lib/gitlab/error_tracking/log_formatter.rb new file mode 100644 index 00000000000..d004c4e20bb --- /dev/null +++ b/lib/gitlab/error_tracking/log_formatter.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module ErrorTracking + class LogFormatter + # Note: all the accesses to Raven's contexts here are to keep the + # backward-compatibility to Sentry's built-in integrations. In future, + # they can be removed. + def generate_log(exception, context_payload) + payload = {} + + Gitlab::ExceptionLogFormatter.format!(exception, payload) + append_user_to_log!(payload, context_payload) + append_tags_to_log!(payload, context_payload) + append_extra_to_log!(payload, context_payload) + + payload + end + + private + + def append_user_to_log!(payload, context_payload) + user_context = Raven.context.user.merge(context_payload[:user]) + user_context.each do |key, value| + payload["user.#{key}"] = value + end + end + + def append_tags_to_log!(payload, context_payload) + tags_context = Raven.context.tags.merge(context_payload[:tags]) + tags_context.each do |key, value| + payload["tags.#{key}"] = value + end + end + + def append_extra_to_log!(payload, context_payload) + extra = Raven.context.extra.merge(context_payload[:extra]) + extra = extra.except(:server) + + # The extra value for sidekiq is a hash whose keys are strings. + if extra[:sidekiq].is_a?(Hash) && extra[:sidekiq].key?('args') + sidekiq_extra = extra.delete(:sidekiq) + sidekiq_extra['args'] = Gitlab::ErrorTracking::Processor::SidekiqProcessor.loggable_arguments( + sidekiq_extra['args'], sidekiq_extra['class'] + ) + payload["extra.sidekiq"] = sidekiq_extra + end + + extra.each do |key, value| + payload["extra.#{key}"] = value + end + end + end + end +end diff --git a/lib/gitlab/error_tracking/processor/context_payload_processor.rb b/lib/gitlab/error_tracking/processor/context_payload_processor.rb new file mode 100644 index 00000000000..5185205e94e --- /dev/null +++ b/lib/gitlab/error_tracking/processor/context_payload_processor.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module ErrorTracking + module Processor + class ContextPayloadProcessor < ::Raven::Processor + # This processor is added to inject application context into Sentry + # events generated by Sentry built-in integrations. When the + # integrations are re-implemented and use Gitlab::ErrorTracking, this + # processor should be removed. + def process(payload) + context_payload = Gitlab::ErrorTracking::ContextPayloadGenerator.generate(nil, {}) + payload.deep_merge!(context_payload) + end + end + end + end +end diff --git a/lib/gitlab/error_tracking/project.rb b/lib/gitlab/error_tracking/project.rb index 93e81da5034..a4ed8831e38 100644 --- a/lib/gitlab/error_tracking/project.rb +++ b/lib/gitlab/error_tracking/project.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +# This should be in the ErrorTracking namespace. For more details, see: +# https://gitlab.com/gitlab-org/gitlab/-/issues/323342 module Gitlab module ErrorTracking class Project diff --git a/lib/gitlab/error_tracking/repo.rb b/lib/gitlab/error_tracking/repo.rb index 50611943bac..e88ac58ff0f 100644 --- a/lib/gitlab/error_tracking/repo.rb +++ b/lib/gitlab/error_tracking/repo.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +# This should be in the ErrorTracking namespace. For more details, see: +# https://gitlab.com/gitlab-org/gitlab/-/issues/323342 module Gitlab module ErrorTracking class Repo diff --git a/lib/gitlab/error_tracking/stack_trace_highlight_decorator.rb b/lib/gitlab/error_tracking/stack_trace_highlight_decorator.rb index 1e490e52c43..24f4c2a2dcf 100644 --- a/lib/gitlab/error_tracking/stack_trace_highlight_decorator.rb +++ b/lib/gitlab/error_tracking/stack_trace_highlight_decorator.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +# This should be in the ErrorTracking namespace. For more details, see: +# https://gitlab.com/gitlab-org/gitlab/-/issues/323342 module Gitlab module ErrorTracking module StackTraceHighlightDecorator diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb index fc3c05c57b2..8c916375a98 100644 --- a/lib/gitlab/etag_caching/middleware.rb +++ b/lib/gitlab/etag_caching/middleware.rb @@ -17,12 +17,12 @@ module Gitlab def call(env) request = ActionDispatch::Request.new(env) - route = Gitlab::EtagCaching::Router.match(request.path_info) + route = Gitlab::EtagCaching::Router.match(request) return @app.call(env) unless route track_event(:etag_caching_middleware_used, route) - etag, cached_value_present = get_etag(request) + etag, cached_value_present = get_etag(request, route) if_none_match = env['HTTP_IF_NONE_MATCH'] if if_none_match == etag @@ -36,8 +36,8 @@ module Gitlab private - def get_etag(request) - cache_key = request.path + def get_etag(request, route) + cache_key = route.cache_key(request) store = Gitlab::EtagCaching::Store.new current_value = store.get(cache_key) cached_value_present = current_value.present? diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb index 769ac2784d1..742b72ecde9 100644 --- a/lib/gitlab/etag_caching/router.rb +++ b/lib/gitlab/etag_caching/router.rb @@ -2,99 +2,24 @@ module Gitlab module EtagCaching - class Router - Route = Struct.new(:regexp, :name, :feature_category) - # We enable an ETag for every request matching the regex. - # To match a regex the path needs to match the following: - # - Don't contain a reserved word (expect for the words used in the - # regex itself) - # - Ending in `noteable/issue/<id>/notes` for the `issue_notes` route - # - Ending in `issues/id`/realtime_changes` for the `issue_title` route - USED_IN_ROUTES = %w[noteable issue notes issues realtime_changes - commit pipelines merge_requests builds - new environments].freeze - RESERVED_WORDS = Gitlab::PathRegex::ILLEGAL_PROJECT_PATH_WORDS - USED_IN_ROUTES - RESERVED_WORDS_REGEX = Regexp.union(*RESERVED_WORDS.map(&Regexp.method(:escape))) - RESERVED_WORDS_PREFIX = %Q(^(?!.*\/(#{RESERVED_WORDS_REGEX})\/).*) + module Router + Route = Struct.new(:regexp, :name, :feature_category, :router) do + delegate :match, to: :regexp + delegate :cache_key, to: :router + end - ROUTES = [ - Gitlab::EtagCaching::Router::Route.new( - %r(#{RESERVED_WORDS_PREFIX}/noteable/issue/\d+/notes\z), - 'issue_notes', - 'issue_tracking' - ), - Gitlab::EtagCaching::Router::Route.new( - %r(#{RESERVED_WORDS_PREFIX}/noteable/merge_request/\d+/notes\z), - 'merge_request_notes', - 'code_review' - ), - Gitlab::EtagCaching::Router::Route.new( - %r(#{RESERVED_WORDS_PREFIX}/issues/\d+/realtime_changes\z), - 'issue_title', - 'issue_tracking' - ), - Gitlab::EtagCaching::Router::Route.new( - %r(#{RESERVED_WORDS_PREFIX}/commit/\S+/pipelines\.json\z), - 'commit_pipelines', - 'continuous_integration' - ), - Gitlab::EtagCaching::Router::Route.new( - %r(#{RESERVED_WORDS_PREFIX}/merge_requests/new\.json\z), - 'new_merge_request_pipelines', - 'continuous_integration' - ), - Gitlab::EtagCaching::Router::Route.new( - %r(#{RESERVED_WORDS_PREFIX}/merge_requests/\d+/pipelines\.json\z), - 'merge_request_pipelines', - 'continuous_integration' - ), - Gitlab::EtagCaching::Router::Route.new( - %r(#{RESERVED_WORDS_PREFIX}/pipelines\.json\z), - 'project_pipelines', - 'continuous_integration' - ), - Gitlab::EtagCaching::Router::Route.new( - %r(#{RESERVED_WORDS_PREFIX}/pipelines/\d+\.json\z), - 'project_pipeline', - 'continuous_integration' - ), - Gitlab::EtagCaching::Router::Route.new( - %r(#{RESERVED_WORDS_PREFIX}/builds/\d+\.json\z), - 'project_build', - 'continuous_integration' - ), - Gitlab::EtagCaching::Router::Route.new( - %r(#{RESERVED_WORDS_PREFIX}/clusters/\d+/environments\z), - 'cluster_environments', - 'continuous_delivery' - ), - Gitlab::EtagCaching::Router::Route.new( - %r(#{RESERVED_WORDS_PREFIX}/environments\.json\z), - 'environments', - 'continuous_delivery' - ), - Gitlab::EtagCaching::Router::Route.new( - %r(#{RESERVED_WORDS_PREFIX}/import/github/realtime_changes\.json\z), - 'realtime_changes_import_github', - 'importers' - ), - Gitlab::EtagCaching::Router::Route.new( - %r(#{RESERVED_WORDS_PREFIX}/import/gitea/realtime_changes\.json\z), - 'realtime_changes_import_gitea', - 'importers' - ), - Gitlab::EtagCaching::Router::Route.new( - %r(#{RESERVED_WORDS_PREFIX}/merge_requests/\d+/cached_widget\.json\z), - 'merge_request_widget', - 'code_review' - ) - ].freeze + module Helpers + def build_route(attrs) + EtagCaching::Router::Route.new(*attrs, self) + end + end - def self.match(path) - ROUTES.find { |route| route.regexp.match(path) } + # Performing RESTful routing match before GraphQL would be more expensive + # for the GraphQL requests because we need to traverse all of the RESTful + # route definitions before falling back to GraphQL. + def self.match(request) + Router::Graphql.match(request) || Router::Restful.match(request) end end end end - -Gitlab::EtagCaching::Router.prepend_if_ee('EE::Gitlab::EtagCaching::Router') diff --git a/lib/gitlab/etag_caching/router/graphql.rb b/lib/gitlab/etag_caching/router/graphql.rb new file mode 100644 index 00000000000..f1737f0ce5a --- /dev/null +++ b/lib/gitlab/etag_caching/router/graphql.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module EtagCaching + module Router + class Graphql + extend EtagCaching::Router::Helpers + GRAPHQL_ETAG_RESOURCE_HEADER = 'X-GITLAB-GRAPHQL-RESOURCE-ETAG' + + ROUTES = [ + [ + %r(\Apipelines/id/\d+\z), + 'pipelines_graph', + 'continuous_integration' + ] + ].map(&method(:build_route)).freeze + + def self.match(request) + return unless request.path_info == graphql_api_path + + graphql_resource = request.headers[GRAPHQL_ETAG_RESOURCE_HEADER] + return unless graphql_resource + + ROUTES.find { |route| route.match(graphql_resource) } + end + + def self.cache_key(request) + [ + request.path, + request.headers[GRAPHQL_ETAG_RESOURCE_HEADER] + ].compact.join(':') + end + + def self.graphql_api_path + @graphql_api_path ||= Gitlab::Routing.url_helpers.api_graphql_path + end + private_class_method :graphql_api_path + end + end + end +end diff --git a/lib/gitlab/etag_caching/router/restful.rb b/lib/gitlab/etag_caching/router/restful.rb new file mode 100644 index 00000000000..08c20e30a48 --- /dev/null +++ b/lib/gitlab/etag_caching/router/restful.rb @@ -0,0 +1,112 @@ +# frozen_string_literal: true + +module Gitlab + module EtagCaching + module Router + class Restful + extend EtagCaching::Router::Helpers + + # We enable an ETag for every request matching the regex. + # To match a regex the path needs to match the following: + # - Don't contain a reserved word (expect for the words used in the + # regex itself) + # - Ending in `noteable/issue/<id>/notes` for the `issue_notes` route + # - Ending in `issues/id`/realtime_changes` for the `issue_title` route + USED_IN_ROUTES = %w[noteable issue notes issues realtime_changes + commit pipelines merge_requests builds + new environments].freeze + RESERVED_WORDS = Gitlab::PathRegex::ILLEGAL_PROJECT_PATH_WORDS - USED_IN_ROUTES + RESERVED_WORDS_REGEX = Regexp.union(*RESERVED_WORDS.map(&Regexp.method(:escape))) + RESERVED_WORDS_PREFIX = %Q(^(?!.*\/(#{RESERVED_WORDS_REGEX})\/).*) + + ROUTES = [ + [ + %r(#{RESERVED_WORDS_PREFIX}/noteable/issue/\d+/notes\z), + 'issue_notes', + 'issue_tracking' + ], + [ + %r(#{RESERVED_WORDS_PREFIX}/noteable/merge_request/\d+/notes\z), + 'merge_request_notes', + 'code_review' + ], + [ + %r(#{RESERVED_WORDS_PREFIX}/issues/\d+/realtime_changes\z), + 'issue_title', + 'issue_tracking' + ], + [ + %r(#{RESERVED_WORDS_PREFIX}/commit/\S+/pipelines\.json\z), + 'commit_pipelines', + 'continuous_integration' + ], + [ + %r(#{RESERVED_WORDS_PREFIX}/merge_requests/new\.json\z), + 'new_merge_request_pipelines', + 'continuous_integration' + ], + [ + %r(#{RESERVED_WORDS_PREFIX}/merge_requests/\d+/pipelines\.json\z), + 'merge_request_pipelines', + 'continuous_integration' + ], + [ + %r(#{RESERVED_WORDS_PREFIX}/pipelines\.json\z), + 'project_pipelines', + 'continuous_integration' + ], + [ + %r(#{RESERVED_WORDS_PREFIX}/pipelines/\d+\.json\z), + 'project_pipeline', + 'continuous_integration' + ], + [ + %r(#{RESERVED_WORDS_PREFIX}/builds/\d+\.json\z), + 'project_build', + 'continuous_integration' + ], + [ + %r(#{RESERVED_WORDS_PREFIX}/clusters/\d+/environments\z), + 'cluster_environments', + 'continuous_delivery' + ], + [ + %r(#{RESERVED_WORDS_PREFIX}/environments\.json\z), + 'environments', + 'continuous_delivery' + ], + [ + %r(#{RESERVED_WORDS_PREFIX}/import/github/realtime_changes\.json\z), + 'realtime_changes_import_github', + 'importers' + ], + [ + %r(#{RESERVED_WORDS_PREFIX}/import/gitea/realtime_changes\.json\z), + 'realtime_changes_import_gitea', + 'importers' + ], + [ + %r(#{RESERVED_WORDS_PREFIX}/merge_requests/\d+/cached_widget\.json\z), + 'merge_request_widget', + 'code_review' + ] + ].map(&method(:build_route)).freeze + + # Overridden in EE to add more routes + def self.all_routes + ROUTES + end + + def self.match(request) + all_routes.find { |route| route.match(request.path_info) } + end + + def self.cache_key(request) + request.path + end + end + end + end +end + +Gitlab::EtagCaching::Router::Restful.prepend_if_ee('EE::Gitlab::EtagCaching::Router::Restful') diff --git a/lib/gitlab/etag_caching/store.rb b/lib/gitlab/etag_caching/store.rb index 1d2f0d7bbf4..d0d790a7c72 100644 --- a/lib/gitlab/etag_caching/store.rb +++ b/lib/gitlab/etag_caching/store.rb @@ -3,6 +3,8 @@ module Gitlab module EtagCaching class Store + InvalidKeyError = Class.new(StandardError) + EXPIRY_TIME = 20.minutes SHARED_STATE_NAMESPACE = 'etag:' @@ -27,9 +29,28 @@ module Gitlab end def redis_shared_state_key(key) - raise 'Invalid key' if !Rails.env.production? && !Gitlab::EtagCaching::Router.match(key) + raise InvalidKeyError, "#{key} is invalid" unless valid_key?(key) "#{SHARED_STATE_NAMESPACE}#{key}" + rescue InvalidKeyError => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) + end + + def valid_key?(key) + return true if skip_validation? + + path, header = key.split(':', 2) + env = { + 'PATH_INFO' => path, + 'HTTP_X_GITLAB_GRAPHQL_RESOURCE_ETAG' => header + } + + fake_request = ActionDispatch::Request.new(env) + !!Gitlab::EtagCaching::Router.match(fake_request) + end + + def skip_validation? + Rails.env.production? end end end diff --git a/lib/gitlab/exception_log_formatter.rb b/lib/gitlab/exception_log_formatter.rb index 6aff8f909f3..9898651c9e3 100644 --- a/lib/gitlab/exception_log_formatter.rb +++ b/lib/gitlab/exception_log_formatter.rb @@ -12,16 +12,6 @@ module Gitlab 'exception.message' => exception.message ) - payload.delete('extra.server') - - payload['extra.sidekiq'].tap do |value| - if value.is_a?(Hash) && value.key?('args') - value = value.dup - payload['extra.sidekiq']['args'] = Gitlab::ErrorTracking::Processor::SidekiqProcessor - .loggable_arguments(value['args'], value['class']) - end - end - if exception.backtrace payload['exception.backtrace'] = Rails.backtrace_cleaner.clean(exception.backtrace) end diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index 423f238a0a2..1bb29ba3eac 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -34,18 +34,10 @@ module Gitlab module Experimentation EXPERIMENTS = { - ci_notification_dot: { - tracking_category: 'Growth::Expansion::Experiment::CiNotificationDot', - use_backwards_compatible_subject_index: true - }, upgrade_link_in_user_menu_a: { tracking_category: 'Growth::Expansion::Experiment::UpgradeLinkInUserMenuA', use_backwards_compatible_subject_index: true }, - invite_members_version_a: { - tracking_category: 'Growth::Expansion::Experiment::InviteMembersVersionA', - use_backwards_compatible_subject_index: true - }, invite_members_version_b: { tracking_category: 'Growth::Expansion::Experiment::InviteMembersVersionB', use_backwards_compatible_subject_index: true @@ -58,30 +50,16 @@ module Gitlab tracking_category: 'Growth::Conversion::Experiment::ContactSalesInApp', use_backwards_compatible_subject_index: true }, - customize_homepage: { - tracking_category: 'Growth::Expansion::Experiment::CustomizeHomepage', - use_backwards_compatible_subject_index: true - }, - group_only_trials: { - tracking_category: 'Growth::Conversion::Experiment::GroupOnlyTrials', - use_backwards_compatible_subject_index: true - }, remove_known_trial_form_fields: { tracking_category: 'Growth::Conversion::Experiment::RemoveKnownTrialFormFields' }, - trimmed_skip_trial_copy: { - tracking_category: 'Growth::Conversion::Experiment::TrimmedSkipTrialCopy' - }, - trial_registration_with_social_signin: { - tracking_category: 'Growth::Conversion::Experiment::TrialRegistrationWithSocialSigning' - }, invite_members_empty_project_version_a: { tracking_category: 'Growth::Expansion::Experiment::InviteMembersEmptyProjectVersionA' }, trial_during_signup: { tracking_category: 'Growth::Conversion::Experiment::TrialDuringSignup' }, - ci_syntax_templates: { + ci_syntax_templates_b: { tracking_category: 'Growth::Activation::Experiment::CiSyntaxTemplates', rollout_strategy: :user }, diff --git a/lib/gitlab/experimentation/controller_concern.rb b/lib/gitlab/experimentation/controller_concern.rb index 2b38b12c914..248abfeada5 100644 --- a/lib/gitlab/experimentation/controller_concern.rb +++ b/lib/gitlab/experimentation/controller_concern.rb @@ -15,7 +15,7 @@ module Gitlab included do before_action :set_experimentation_subject_id_cookie, unless: :dnt_enabled? - helper_method :experiment_enabled?, :experiment_tracking_category_and_group, :tracking_label + helper_method :experiment_enabled?, :experiment_tracking_category_and_group, :record_experiment_group, :tracking_label end def set_experimentation_subject_id_cookie @@ -72,12 +72,22 @@ module Gitlab ::Experiment.add_user(experiment_key, tracking_group(experiment_key, nil, subject: subject), current_user, context) end - def record_experiment_conversion_event(experiment_key) + def record_experiment_group(experiment_key, group) + return if dnt_enabled? + return unless Experimentation.active?(experiment_key) && group + + variant_subject = Experimentation.rollout_strategy(experiment_key) == :cookie ? nil : group + variant = tracking_group(experiment_key, nil, subject: variant_subject) + + ::Experiment.add_group(experiment_key, group: group, variant: variant) + end + + def record_experiment_conversion_event(experiment_key, context = {}) return if dnt_enabled? return unless current_user return unless Experimentation.active?(experiment_key) - ::Experiment.record_conversion_event(experiment_key, current_user) + ::Experiment.record_conversion_event(experiment_key, current_user, context) end def experiment_tracking_category_and_group(experiment_key, subject: nil) diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb index b118eda37f8..9e24306c05e 100644 --- a/lib/gitlab/git/blame.rb +++ b/lib/gitlab/git/blame.rb @@ -38,9 +38,11 @@ module Gitlab if line[0, 1] == "\t" lines << line[1, line.size] elsif m = /^(\w{40}) (\d+) (\d+)/.match(line) - commit_id, old_lineno, lineno = m[1], m[2].to_i, m[3].to_i + # Removed these instantiations for performance but keeping them for reference: + # commit_id, old_lineno, lineno = m[1], m[2].to_i, m[3].to_i + commit_id = m[1] commits[commit_id] = nil unless commits.key?(commit_id) - info[lineno] = [commit_id, old_lineno] + info[m[3].to_i] = [commit_id, m[2].to_i] end end @@ -50,8 +52,7 @@ module Gitlab # get it together info.sort.each do |lineno, (commit_id, old_lineno)| - commit = commits[commit_id] - final << BlameLine.new(lineno, old_lineno, commit, lines[lineno - 1]) + final << BlameLine.new(lineno, old_lineno, commits[commit_id], lines[lineno - 1]) end @lines = final diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 35c3dc5b0b3..ff99803d8de 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -20,6 +20,7 @@ module Gitlab ].freeze attr_accessor(*SERIALIZE_KEYS) + attr_reader :repository def ==(other) return false unless other.is_a?(Gitlab::Git::Commit) diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 3c7fa88977e..e3788814dd5 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -246,9 +246,7 @@ module Gitlab def self.route_to_primary return {} unless Gitlab::SafeRequestStore.active? - return {} unless Gitlab::SafeRequestStore[:gitlab_git_env] - - return {} if Gitlab::SafeRequestStore[:gitlab_git_env].empty? + return {} if Gitlab::SafeRequestStore[:gitlab_git_env].blank? { 'gitaly-route-repository-accessor-policy' => 'primary-only' } end diff --git a/lib/gitlab/gitaly_client/storage_settings.rb b/lib/gitlab/gitaly_client/storage_settings.rb index 5e50ac72965..7edd42f9ef7 100644 --- a/lib/gitlab/gitaly_client/storage_settings.rb +++ b/lib/gitlab/gitaly_client/storage_settings.rb @@ -60,7 +60,8 @@ module Gitlab end def legacy_disk_path - if self.class.disk_access_denied? + # Do not use self.class due to Spring reloading issues + if Gitlab::GitalyClient::StorageSettings.disk_access_denied? raise DirectPathAccessError, "git disk access denied" end diff --git a/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb b/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb index 11181edf0e9..8173fdd5e3e 100644 --- a/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb @@ -12,25 +12,27 @@ module Gitlab def execute merge_request = project.merge_requests.find_by_iid(pull_request.iid) + timestamp = Time.new.utc + merged_at = pull_request.merged_at user_finder = GithubImport::UserFinder.new(project, client) gitlab_user_id = user_finder.user_id_for(pull_request.merged_by) - if gitlab_user_id - timestamp = Time.new.utc - MergeRequest::Metrics.upsert({ - target_project_id: project.id, - merge_request_id: merge_request.id, - merged_by_id: gitlab_user_id, - created_at: timestamp, - updated_at: timestamp - }, unique_by: :merge_request_id) - else + MergeRequest::Metrics.upsert({ + target_project_id: project.id, + merge_request_id: merge_request.id, + merged_by_id: gitlab_user_id, + merged_at: merged_at, + created_at: timestamp, + updated_at: timestamp + }, unique_by: :merge_request_id) + + unless gitlab_user_id merge_request.notes.create!( importing: true, - note: "*Merged by: #{pull_request.merged_by.login}*", + note: missing_author_note, author_id: project.creator_id, project: project, - created_at: pull_request.created_at + created_at: merged_at ) end end @@ -38,6 +40,13 @@ module Gitlab private attr_reader :project, :pull_request, :client + + def missing_author_note + s_("GitHubImporter|*Merged by: %{author} at %{timestamp}*") % { + author: pull_request.merged_by.login, + timestamp: pull_request.merged_at + } + end end end end diff --git a/lib/gitlab/github_import/importer/pull_request_review_importer.rb b/lib/gitlab/github_import/importer/pull_request_review_importer.rb index 14ee69ba089..9f495913897 100644 --- a/lib/gitlab/github_import/importer/pull_request_review_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_review_importer.rb @@ -77,12 +77,22 @@ module Gitlab def add_approval!(user_id) return unless review.review_type == 'APPROVED' - add_approval_system_note!(user_id) - - merge_request.approvals.create!( + approval_attribues = { + merge_request_id: merge_request.id, user_id: user_id, - created_at: review.submitted_at + created_at: review.submitted_at, + updated_at: review.submitted_at + } + + result = ::Approval.insert( + approval_attribues, + returning: [:id], + unique_by: [:user_id, :merge_request_id] ) + + if result.rows.present? + add_approval_system_note!(user_id) + end end def add_approval_system_note!(user_id) diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 3dd317c5a64..c7e215c143f 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -7,13 +7,14 @@ module Gitlab include WebpackHelper def add_gon_variables - gon.api_version = 'v4' - gon.default_avatar_url = default_avatar_url - gon.max_file_size = Gitlab::CurrentSettings.max_attachment_size - gon.asset_host = ActionController::Base.asset_host - gon.webpack_public_path = webpack_public_path - gon.relative_url_root = Gitlab.config.gitlab.relative_url_root - gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class + gon.api_version = 'v4' + gon.default_avatar_url = default_avatar_url + gon.max_file_size = Gitlab::CurrentSettings.max_attachment_size + gon.asset_host = ActionController::Base.asset_host + gon.webpack_public_path = webpack_public_path + gon.relative_url_root = Gitlab.config.gitlab.relative_url_root + gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class + gon.markdown_surround_selection = current_user&.markdown_surround_selection if Gitlab.config.sentry.enabled gon.sentry_dsn = Gitlab.config.sentry.clientside_dsn diff --git a/lib/gitlab/graphql/calls_gitaly.rb b/lib/gitlab/graphql/calls_gitaly.rb deleted file mode 100644 index 40cd74a34f2..00000000000 --- a/lib/gitlab/graphql/calls_gitaly.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Graphql - # Wraps the field resolution to count Gitaly calls before and after. - # Raises an error if the field calls Gitaly but hadn't declared such. - module CallsGitaly - extend ActiveSupport::Concern - - def self.use(schema_definition) - schema_definition.instrument(:field, Gitlab::Graphql::CallsGitaly::Instrumentation.new, after_built_ins: true) - end - end - end -end diff --git a/lib/gitlab/graphql/calls_gitaly/field_extension.rb b/lib/gitlab/graphql/calls_gitaly/field_extension.rb new file mode 100644 index 00000000000..32530b47ce3 --- /dev/null +++ b/lib/gitlab/graphql/calls_gitaly/field_extension.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module CallsGitaly + # Check if any `calls_gitaly: true` declarations need to be added + # + # See BaseField: this extension is not applied if the field does not + # need it (i.e. it has a constant complexity or knows that it calls + # gitaly) + class FieldExtension < ::GraphQL::Schema::FieldExtension + include Laziness + + def resolve(object:, arguments:, **rest) + yield(object, arguments, [current_gitaly_call_count, accounted_for]) + end + + def after_resolve(value:, memo:, **rest) + (value, count) = value_with_count(value, memo) + calls_gitaly_check(count) + accounted_for(count) + + value + end + + private + + # Resolutions are not nested nicely (due to laziness), so we have to + # know not just how many calls were made before resolution started, but + # also how many were accounted for by fields with the correct settings + # in between. + # + # e.g. the following is not just plausible, but common: + # + # enter A.user (lazy) + # enter A.x + # leave A.x + # enter A.calls_gitaly + # leave A.calls_gitaly (accounts for 1 call) + # leave A.user + # + # In this circumstance we need to mark the calls made by A.calls_gitaly + # as accounted for, even though they were made after we yielded + # in A.user + def value_with_count(value, (previous_count, previous_accounted_for)) + newly_accounted_for = accounted_for - previous_accounted_for + value = force(value) + count = [current_gitaly_call_count - (previous_count + newly_accounted_for), 0].max + + [value, count] + end + + def current_gitaly_call_count + Gitlab::GitalyClient.get_request_count || 0 + end + + def calls_gitaly_check(calls) + return if calls < 1 || field.may_call_gitaly? + + error = RuntimeError.new(<<~ERROR) + #{field_name} unexpectedly calls Gitaly! + + Please either specify a constant complexity or add `calls_gitaly: true` + to the field declaration + ERROR + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) + end + + def accounted_for(count = nil) + return 0 unless Gitlab::SafeRequestStore.active? + + Gitlab::SafeRequestStore["graphql_gitaly_accounted_for"] ||= 0 + + if count.nil? + Gitlab::SafeRequestStore["graphql_gitaly_accounted_for"] + else + Gitlab::SafeRequestStore["graphql_gitaly_accounted_for"] += count + end + end + + def field_name + "#{field.owner.graphql_name}.#{field.graphql_name}" + end + end + end + end +end diff --git a/lib/gitlab/graphql/calls_gitaly/instrumentation.rb b/lib/gitlab/graphql/calls_gitaly/instrumentation.rb deleted file mode 100644 index 11d3c50e093..00000000000 --- a/lib/gitlab/graphql/calls_gitaly/instrumentation.rb +++ /dev/null @@ -1,40 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Graphql - module CallsGitaly - class Instrumentation - # Check if any `calls_gitaly: true` declarations need to be added - # Do nothing if a constant complexity was provided - def instrument(_type, field) - type_object = field.metadata[:type_class] - return field unless type_object.respond_to?(:calls_gitaly?) - return field if type_object.constant_complexity? || type_object.calls_gitaly? - - old_resolver_proc = field.resolve_proc - - gitaly_wrapped_resolve = -> (typed_object, args, ctx) do - previous_gitaly_call_count = Gitlab::GitalyClient.get_request_count - result = old_resolver_proc.call(typed_object, args, ctx) - current_gitaly_call_count = Gitlab::GitalyClient.get_request_count - calls_gitaly_check(type_object, current_gitaly_call_count - previous_gitaly_call_count) - result - end - - field.redefine do - resolve(gitaly_wrapped_resolve) - end - end - - def calls_gitaly_check(type_object, calls) - return if calls < 1 - - # Will inform you if there needs to be `calls_gitaly: true` as a kwarg in the field declaration - # if there is at least 1 Gitaly call involved with the field resolution. - error = RuntimeError.new("Gitaly is called for field '#{type_object.name}' on #{type_object.owner.try(:name)} - please either specify a constant complexity or add `calls_gitaly: true` to the field declaration") - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) - end - end - end - end -end diff --git a/lib/gitlab/graphql/docs/helper.rb b/lib/gitlab/graphql/docs/helper.rb index ad9e08e189c..68a2a78d0d4 100644 --- a/lib/gitlab/graphql/docs/helper.rb +++ b/lib/gitlab/graphql/docs/helper.rb @@ -28,16 +28,20 @@ module Gitlab end def render_name_and_description(object) - content = "### #{object[:name]}\n" + content = "### `#{object[:name]}`\n" if object[:description].present? - content += "\n#{object[:description]}.\n" + content += "\n#{object[:description]}" + content += '.' unless object[:description].ends_with?('.') + content += "\n" end content end def sorted_by_name(objects) + return [] unless objects.present? + objects.sort_by { |o| o[:name] } end @@ -98,6 +102,10 @@ module Gitlab end end + def queries + graphql_operation_types.find { |type| type[:name] == 'Query' }.to_h.values_at(:fields, :connections).flatten + end + # We ignore the built-in enum types. def enums graphql_enum_types.select do |enum_type| diff --git a/lib/gitlab/graphql/docs/templates/default.md.haml b/lib/gitlab/graphql/docs/templates/default.md.haml index 9dfb9b090a8..5ae83fd51c5 100644 --- a/lib/gitlab/graphql/docs/templates/default.md.haml +++ b/lib/gitlab/graphql/docs/templates/default.md.haml @@ -14,13 +14,29 @@ WARNING: Fields that are deprecated are marked with **{warning-solid}**. - Items (fields, enums, etc) that have been removed according to our [deprecation process](../index.md#deprecation-process) can be found + Items (fields, enums, etc) that have been removed according to our [deprecation process](../index.md#deprecation-and-removal-process) can be found in [Removed Items](../removed_items.md). <!-- vale gitlab.Spelling = NO --> \ :plain + ## `Query` type + + The `Query` type contains the API's top-level entry points for all executable queries. +\ + +- sorted_by_name(queries).each do |query| + = render_name_and_description(query) + - unless query[:arguments].empty? + ~ "#### Arguments\n" + ~ "| Name | Type | Description |" + ~ "| ---- | ---- | ----------- |" + - sorted_by_name(query[:arguments]).each do |argument| + = render_field(argument) + \ + +:plain ## Object types Object types represent the resources that the GitLab GraphQL API can return. diff --git a/lib/gitlab/graphql/extensions/externally_paginated_array_extension.rb b/lib/gitlab/graphql/extensions/externally_paginated_array_extension.rb index 1adedb500e6..f787e7be94a 100644 --- a/lib/gitlab/graphql/extensions/externally_paginated_array_extension.rb +++ b/lib/gitlab/graphql/extensions/externally_paginated_array_extension.rb @@ -4,7 +4,7 @@ module Gitlab module Extensions class ExternallyPaginatedArrayExtension < GraphQL::Schema::Field::ConnectionExtension def resolve(object:, arguments:, context:) - yield(object, arguments) + yield(object, arguments, arguments) end end end diff --git a/lib/gitlab/graphql/pagination/keyset/connection.rb b/lib/gitlab/graphql/pagination/keyset/connection.rb index f95c91c5706..e525996ec10 100644 --- a/lib/gitlab/graphql/pagination/keyset/connection.rb +++ b/lib/gitlab/graphql/pagination/keyset/connection.rb @@ -33,6 +33,7 @@ module Gitlab include Gitlab::Utils::StrongMemoize include ::Gitlab::Graphql::ConnectionCollectionMethods prepend ::Gitlab::Graphql::ConnectionRedaction + prepend GenericKeysetPagination # rubocop: disable Naming/PredicateName # https://relay.dev/graphql/connections.htm#sec-undefined.PageInfo.Fields diff --git a/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb b/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb new file mode 100644 index 00000000000..318c6e1734f --- /dev/null +++ b/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Pagination + module Keyset + # Use the generic keyset implementation if the given ActiveRecord scope supports it. + # Note: this module is temporary, at some point it will be merged with Keyset::Connection + module GenericKeysetPagination + extend ActiveSupport::Concern + + def ordered_items + return super unless Gitlab::Pagination::Keyset::Order.keyset_aware?(items) + + items + end + + def cursor_for(node) + return super unless Gitlab::Pagination::Keyset::Order.keyset_aware?(items) + + order = Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(items) + encode(order.cursor_attributes_for_node(node).to_json) + end + + def slice_nodes(sliced, encoded_cursor, before_or_after) + return super unless Gitlab::Pagination::Keyset::Order.keyset_aware?(sliced) + + order = Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(sliced) + order = order.reversed_order if before_or_after == :before + + decoded_cursor = ordering_from_encoded_json(encoded_cursor) + order.apply_cursor_conditions(sliced, decoded_cursor) + end + + def sliced_nodes + return super unless Gitlab::Pagination::Keyset::Order.keyset_aware?(items) + + sliced = ordered_items + sliced = slice_nodes(sliced, before, :before) if before.present? + sliced = slice_nodes(sliced, after, :after) if after.present? + sliced + end + end + end + end + end +end diff --git a/lib/gitlab/graphql/pagination/keyset/last_items.rb b/lib/gitlab/graphql/pagination/keyset/last_items.rb index 45bf15236c1..960567a6fbc 100644 --- a/lib/gitlab/graphql/pagination/keyset/last_items.rb +++ b/lib/gitlab/graphql/pagination/keyset/last_items.rb @@ -10,46 +10,14 @@ module Gitlab class LastItems # rubocop: disable CodeReuse/ActiveRecord def self.take_items(scope, count) - if custom_order = lookup_custom_reverse_order(scope.order_values) - items = scope.reorder(*custom_order).first(count) # returns a single record when count is nil + if Gitlab::Pagination::Keyset::Order.keyset_aware?(scope) + order = Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(scope) + items = scope.reorder(order.reversed_order).first(count) items.is_a?(Array) ? items.reverse : items else scope.last(count) end end - # rubocop: enable CodeReuse/ActiveRecord - - # Detect special ordering and provide the reversed order - def self.lookup_custom_reverse_order(order_values) - if ordering_by_merged_at_and_mr_id_desc?(order_values) - [ - Gitlab::Database.nulls_first_order('merge_request_metrics.merged_at', 'ASC'), # reversing the order - MergeRequest.arel_table[:id].asc - ] - elsif ordering_by_merged_at_and_mr_id_asc?(order_values) - [ - Gitlab::Database.nulls_first_order('merge_request_metrics.merged_at', 'DESC'), - MergeRequest.arel_table[:id].asc - ] - end - end - - def self.ordering_by_merged_at_and_mr_id_desc?(order_values) - order_values.size == 2 && - order_values.first.to_s == Gitlab::Database.nulls_last_order('merge_request_metrics.merged_at', 'DESC') && - order_values.last.is_a?(Arel::Nodes::Descending) && - order_values.last.to_sql == MergeRequest.arel_table[:id].desc.to_sql - end - - def self.ordering_by_merged_at_and_mr_id_asc?(order_values) - order_values.size == 2 && - order_values.first.to_s == Gitlab::Database.nulls_last_order('merge_request_metrics.merged_at', 'ASC') && - order_values.last.is_a?(Arel::Nodes::Descending) && - order_values.last.to_sql == MergeRequest.arel_table[:id].desc.to_sql - end - - private_class_method :ordering_by_merged_at_and_mr_id_desc? - private_class_method :ordering_by_merged_at_and_mr_id_asc? end end end diff --git a/lib/gitlab/graphql/pagination/keyset/order_info.rb b/lib/gitlab/graphql/pagination/keyset/order_info.rb index d37264c1343..0494329bfd9 100644 --- a/lib/gitlab/graphql/pagination/keyset/order_info.rb +++ b/lib/gitlab/graphql/pagination/keyset/order_info.rb @@ -92,8 +92,6 @@ module Gitlab def extract_attribute_values(order_value) if ordering_by_lower?(order_value) [order_value.expr.expressions[0].name.to_s, order_value.direction, order_value.expr] - elsif ordering_by_similarity?(order_value) - ['similarity', order_value.direction, order_value.expr] elsif ordering_by_case?(order_value) ['case_order_value', order_value.direction, order_value.expr] elsif ordering_by_array_position?(order_value) @@ -113,11 +111,6 @@ module Gitlab order_value.expr.is_a?(Arel::Nodes::NamedFunction) && order_value.expr&.name&.downcase == 'array_position' end - # determine if ordering using SIMILARITY scoring based on Gitlab::Database::SimilarityScore - def ordering_by_similarity?(order_value) - Gitlab::Database::SimilarityScore.order_by_similarity?(order_value) - end - # determine if ordering using CASE def ordering_by_case?(order_value) order_value.expr.is_a?(Arel::Nodes::Case) diff --git a/lib/gitlab/graphql/present.rb b/lib/gitlab/graphql/present.rb index 6d86d632ab4..fdaf075eb25 100644 --- a/lib/gitlab/graphql/present.rb +++ b/lib/gitlab/graphql/present.rb @@ -12,11 +12,30 @@ module Gitlab def self.presenter_class @presenter_class end + + def self.present(object, attrs) + klass = @presenter_class + return object if !klass || object.is_a?(klass) + + @presenter_class.new(object, **attrs) + end + end + + def unpresented + unwrapped || object end - def self.use(schema_definition) - schema_definition.instrument(:field, ::Gitlab::Graphql::Present::Instrumentation.new) + def present(object_type, attrs) + return unless object_type.respond_to?(:present) + + self.unwrapped ||= object + # @object belongs to Schema::Object, which does not expose a writer. + @object = object_type.present(unwrapped, attrs) # rubocop: disable Gitlab/ModuleWithInstanceVariables end + + private + + attr_accessor :unwrapped end end end diff --git a/lib/gitlab/graphql/present/field_extension.rb b/lib/gitlab/graphql/present/field_extension.rb new file mode 100644 index 00000000000..2e211b70d35 --- /dev/null +++ b/lib/gitlab/graphql/present/field_extension.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Present + class FieldExtension < ::GraphQL::Schema::FieldExtension + SAFE_CONTEXT_KEYS = %i[current_user].freeze + + def resolve(object:, arguments:, context:) + attrs = safe_context_values(context) + + # We need to handle the object being either a Schema::Object or an + # inner Schema::Object#object. This depends on whether the field + # has a @resolver_proc or not. + if object.is_a?(::Types::BaseObject) + object.present(field.owner, attrs) + yield(object, arguments) + else + # This is the legacy code-path, hit if the field has a @resolver_proc + # TODO: remove this when resolve procs are removed from the + # graphql-ruby library, and all field instrumentation is removed. + # See: https://github.com/rmosolgo/graphql-ruby/issues/3385 + presented = field.owner.try(:present, object, attrs) || object + yield(presented, arguments) + end + end + + private + + def safe_context_values(context) + context.to_h.slice(*SAFE_CONTEXT_KEYS) + end + end + end + end +end diff --git a/lib/gitlab/graphql/present/instrumentation.rb b/lib/gitlab/graphql/present/instrumentation.rb deleted file mode 100644 index b8535575da5..00000000000 --- a/lib/gitlab/graphql/present/instrumentation.rb +++ /dev/null @@ -1,49 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Graphql - module Present - class Instrumentation - SAFE_CONTEXT_KEYS = %i[current_user].freeze - - def instrument(type, field) - return field unless field.metadata[:type_class] - - presented_in = field.metadata[:type_class].owner - return field unless presented_in.respond_to?(:presenter_class) - return field unless presented_in.presenter_class - - old_resolver = field.resolve_proc - - resolve_with_presenter = -> (presented_type, args, context) do - # We need to wrap the original presentation type into a type that - # uses the presenter as an object. - object = presented_type.object - - if object.is_a?(presented_in.presenter_class) - next old_resolver.call(presented_type, args, context) - end - - attrs = safe_context_values(context) - presenter = presented_in.presenter_class.new(object, **attrs) - - # we have to use the new `authorized_new` method, as `new` is protected - wrapped = presented_type.class.authorized_new(presenter, context) - - old_resolver.call(wrapped, args, context) - end - - field.redefine do - resolve(resolve_with_presenter) - end - end - - private - - def safe_context_values(context) - context.to_h.slice(*SAFE_CONTEXT_KEYS) - end - end - end - end -end diff --git a/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb b/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb index 0665ea8b6c9..8acd27869a9 100644 --- a/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb +++ b/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb @@ -9,10 +9,6 @@ module Gitlab FIELD_USAGE_ANALYZER = GraphQL::Analysis::FieldUsage.new { |query, used_fields, used_deprecated_fields| [used_fields, used_deprecated_fields] } ALL_ANALYZERS = [COMPLEXITY_ANALYZER, DEPTH_ANALYZER, FIELD_USAGE_ANALYZER].freeze - def analyze?(query) - Feature.enabled?(:graphql_logging, default_enabled: true) - end - def initial_value(query) variables = process_variables(query.provided_variables) default_initial_values(query).merge({ diff --git a/lib/gitlab/hook_data/project_member_builder.rb b/lib/gitlab/hook_data/project_member_builder.rb new file mode 100644 index 00000000000..90fc83fdf21 --- /dev/null +++ b/lib/gitlab/hook_data/project_member_builder.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module Gitlab + module HookData + class ProjectMemberBuilder < BaseBuilder + alias_method :project_member, :object + + # Sample data + + # { + # :created_at=>"2021-03-02T10:43:17Z", + # :updated_at=>"2021-03-02T10:43:17Z", + # :project_name=>"gitlab", + # :project_path=>"gitlab", + # :project_path_with_namespace=>"namespace1/gitlab", + # :project_id=>1, + # :user_username=>"johndoe", + # :user_name=>"John Doe", + # :user_email=>"john@example.com", + # :user_id=>2, + # :access_level=>"Developer", + # :project_visibility=>"internal", + # :event_name=>"user_update_for_team" + # } + + def build(event) + [ + timestamps_data, + project_member_data, + event_data(event) + ].reduce(:merge) + end + + private + + def project_member_data + project = project_member.project || Project.unscoped.find(project_member.source_id) + + { + project_name: project.name, + project_path: project.path, + project_path_with_namespace: project.full_path, + project_id: project.id, + user_username: project_member.user.username, + user_name: project_member.user.name, + user_email: project_member.user.email, + user_id: project_member.user.id, + access_level: project_member.human_access, + project_visibility: project.visibility + } + end + + def event_data(event) + event_name = case event + when :create + 'user_add_to_team' + when :destroy + 'user_remove_from_team' + when :update + 'user_update_for_team' + end + { event_name: event_name } + end + end + end +end diff --git a/lib/gitlab/http_connection_adapter.rb b/lib/gitlab/http_connection_adapter.rb index 84eb60f3a5d..37f618ae879 100644 --- a/lib/gitlab/http_connection_adapter.rb +++ b/lib/gitlab/http_connection_adapter.rb @@ -11,13 +11,18 @@ # This option will take precedence over the global setting. module Gitlab class HTTPConnectionAdapter < HTTParty::ConnectionAdapter + extend ::Gitlab::Utils::Override + + override :connection def connection - begin - @uri, hostname = Gitlab::UrlBlocker.validate!(uri, allow_local_network: allow_local_requests?, - allow_localhost: allow_local_requests?, - dns_rebind_protection: dns_rebind_protection?) - rescue Gitlab::UrlBlocker::BlockedUrlError => e - raise Gitlab::HTTP::BlockedUrlError, "URL '#{uri}' is blocked: #{e.message}" + @uri, hostname = validate_url!(uri) + + if options.key?(:http_proxyaddr) + proxy_uri_with_port = uri_with_port(options[:http_proxyaddr], options[:http_proxyport]) + proxy_uri_validated = validate_url!(proxy_uri_with_port).first + + @options[:http_proxyaddr] = proxy_uri_validated.omit(:port).to_s + @options[:http_proxyport] = proxy_uri_validated.port end super.tap do |http| @@ -27,6 +32,14 @@ module Gitlab private + def validate_url!(url) + Gitlab::UrlBlocker.validate!(url, allow_local_network: allow_local_requests?, + allow_localhost: allow_local_requests?, + dns_rebind_protection: dns_rebind_protection?) + rescue Gitlab::UrlBlocker::BlockedUrlError => e + raise Gitlab::HTTP::BlockedUrlError, "URL '#{url}' is blocked: #{e.message}" + end + def allow_local_requests? options.fetch(:allow_local_requests, allow_settings_local_requests?) end @@ -40,5 +53,11 @@ module Gitlab def allow_settings_local_requests? Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services? end + + def uri_with_port(address, port) + uri = Addressable::URI.parse(address) + uri.port = port if port.present? + uri + end end end diff --git a/lib/gitlab/marginalia.rb b/lib/gitlab/marginalia.rb index 325a8c5c325..c99cf113638 100644 --- a/lib/gitlab/marginalia.rb +++ b/lib/gitlab/marginalia.rb @@ -2,8 +2,6 @@ module Gitlab module Marginalia - cattr_accessor :enabled, default: false - def self.set_application_name ::Marginalia.application_name = Gitlab.process_name end @@ -13,12 +11,5 @@ module Gitlab ::Marginalia::SidekiqInstrumentation.enable! end end - - def self.set_enabled_from_feature_flag - # During db:create and db:bootstrap skip feature query as DB is not available yet. - return false unless Gitlab::Database.cached_table_exists?('features') - - self.enabled = Feature.enabled?(:marginalia, type: :ops) - end end end diff --git a/lib/gitlab/marginalia/active_record_instrumentation.rb b/lib/gitlab/marginalia/active_record_instrumentation.rb deleted file mode 100644 index 452f472bf6a..00000000000 --- a/lib/gitlab/marginalia/active_record_instrumentation.rb +++ /dev/null @@ -1,12 +0,0 @@ -# frozen_string_literal: true - -# Patch to annotate sql only when the feature is enabled. -module Gitlab - module Marginalia - module ActiveRecordInstrumentation - def annotate_sql(sql) - Gitlab::Marginalia.enabled ? super(sql) : sql - end - end - end -end diff --git a/lib/gitlab/marginalia/comment.rb b/lib/gitlab/marginalia/comment.rb index 7b4e4b06f00..ee15d3b1812 100644 --- a/lib/gitlab/marginalia/comment.rb +++ b/lib/gitlab/marginalia/comment.rb @@ -37,6 +37,10 @@ module Gitlab job end end + + def endpoint_id + Labkit::Context.current&.get_attribute(:caller_id) + end end end end diff --git a/lib/gitlab/marker_range.rb b/lib/gitlab/marker_range.rb new file mode 100644 index 00000000000..50a59adebdf --- /dev/null +++ b/lib/gitlab/marker_range.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +# It is a Range object extended with `mode` attribute +# MarkerRange not only keeps information about changed characters, but also +# the type of changes +module Gitlab + class MarkerRange < Range + DELETION = :deletion + ADDITION = :addition + + # Converts Range object to MarkerRange class + def self.from_range(range) + return range if range.is_a?(self) + + new(range.begin, range.end, exclude_end: range.exclude_end?) + end + + def initialize(first, last, exclude_end: false, mode: nil) + super(first, last, exclude_end) + @mode = mode + end + + def to_range + Range.new(self.begin, self.end, self.exclude_end?) + end + + attr_reader :mode + end +end diff --git a/lib/gitlab/memory/instrumentation.rb b/lib/gitlab/memory/instrumentation.rb index 76e84e54d3a..8f9f6d19ce8 100644 --- a/lib/gitlab/memory/instrumentation.rb +++ b/lib/gitlab/memory/instrumentation.rb @@ -25,7 +25,7 @@ module Gitlab def self.ensure_feature_flag! return unless available? - enabled = Feature.enabled?(:trace_memory_allocations) + enabled = Feature.enabled?(:trace_memory_allocations, default_enabled: true) return if enabled == Thread.trace_memory_allocations MUTEX.synchronize do diff --git a/lib/gitlab/metrics/background_transaction.rb b/lib/gitlab/metrics/background_transaction.rb new file mode 100644 index 00000000000..3dda68bf93f --- /dev/null +++ b/lib/gitlab/metrics/background_transaction.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + class BackgroundTransaction < Transaction + # Separate web transaction instance and background transaction instance + BACKGROUND_THREAD_KEY = :_gitlab_metrics_background_transaction + BACKGROUND_BASE_LABEL_KEYS = %i(endpoint_id feature_category).freeze + + class << self + def current + Thread.current[BACKGROUND_THREAD_KEY] + end + + def prometheus_metric(name, type, &block) + fetch_metric(type, name) do + # set default metric options + docstring "#{name.to_s.humanize} #{type}" + + evaluate(&block) + # always filter sensitive labels and merge with base ones + label_keys BACKGROUND_BASE_LABEL_KEYS | (label_keys - ::Gitlab::Metrics::Transaction::FILTERED_LABEL_KEYS) + end + end + end + + def run + Thread.current[BACKGROUND_THREAD_KEY] = self + + yield + ensure + Thread.current[BACKGROUND_THREAD_KEY] = nil + end + + def labels + @labels ||= { + endpoint_id: current_context&.get_attribute(:caller_id), + feature_category: current_context&.get_attribute(:feature_category) + } + end + + private + + def current_context + Labkit::Context.current + end + end + end +end diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index 76175b465e4..3d29d38fa1f 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -7,7 +7,7 @@ module Gitlab module Samplers class RubySampler < BaseSampler DEFAULT_SAMPLING_INTERVAL_SECONDS = 60 - GC_REPORT_BUCKETS = [0.005, 0.01, 0.02, 0.04, 0.07, 0.1, 0.5].freeze + GC_REPORT_BUCKETS = [0.01, 0.05, 0.1, 0.2, 0.3, 0.5, 1].freeze def initialize(*) GC::Profiler.clear diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index d725d8d7b29..5eefef02507 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -9,6 +9,16 @@ module Gitlab IGNORABLE_SQL = %w{BEGIN COMMIT}.freeze DB_COUNTERS = %i{db_count db_write_count db_cached_count}.freeze + SQL_COMMANDS_WITH_COMMENTS_REGEX = /\A(\/\*.*\*\/\s)?((?!(.*[^\w'"](DELETE|UPDATE|INSERT INTO)[^\w'"])))(WITH.*)?(SELECT)((?!(FOR UPDATE|FOR SHARE)).)*$/i.freeze + + DURATION_BUCKET = [0.05, 0.1, 0.25].freeze + + # This event is published from ActiveRecordBaseTransactionMetrics and + # used to record a database transaction duration when calling + # ActiveRecord::Base.transaction {} block. + def transaction(event) + observe(:gitlab_database_transaction_seconds, event) + end def sql(event) # Mark this thread as requiring a database connection. This is used @@ -17,51 +27,57 @@ module Gitlab Thread.current[:uses_db_connection] = true payload = event.payload - return if payload[:name] == 'SCHEMA' || IGNORABLE_SQL.include?(payload[:sql]) + return if ignored_query?(payload) - increment_db_counters(payload) + increment(:db_count) + increment(:db_cached_count) if cached_query?(payload) + increment(:db_write_count) unless select_sql_command?(payload) - current_transaction&.observe(:gitlab_sql_duration_seconds, event.duration / 1000.0) do - buckets [0.05, 0.1, 0.25] - end + observe(:gitlab_sql_duration_seconds, event) end def self.db_counter_payload return {} unless Gitlab::SafeRequestStore.active? - DB_COUNTERS.map do |counter| - [counter, Gitlab::SafeRequestStore[counter].to_i] - end.to_h + payload = {} + DB_COUNTERS.each do |counter| + payload[counter] = Gitlab::SafeRequestStore[counter].to_i + end + payload end private - def select_sql_command?(payload) - payload[:sql].match(/\A((?!(.*[^\w'"](DELETE|UPDATE|INSERT INTO)[^\w'"])))(WITH.*)?(SELECT)((?!(FOR UPDATE|FOR SHARE)).)*$/i) + def ignored_query?(payload) + payload[:name] == 'SCHEMA' || IGNORABLE_SQL.include?(payload[:sql]) end - def increment_db_counters(payload) - increment(:db_count) - - if payload.fetch(:cached, payload[:name] == 'CACHE') - increment(:db_cached_count) - end + def cached_query?(payload) + payload.fetch(:cached, payload[:name] == 'CACHE') + end - increment(:db_write_count) unless select_sql_command?(payload) + def select_sql_command?(payload) + payload[:sql].match(SQL_COMMANDS_WITH_COMMENTS_REGEX) end def increment(counter) current_transaction&.increment("gitlab_transaction_#{counter}_total".to_sym, 1) - if Gitlab::SafeRequestStore.active? - Gitlab::SafeRequestStore[counter] = Gitlab::SafeRequestStore[counter].to_i + 1 + Gitlab::SafeRequestStore[counter] = Gitlab::SafeRequestStore[counter].to_i + 1 + end + + def observe(histogram, event) + current_transaction&.observe(histogram, event.duration / 1000.0) do + buckets DURATION_BUCKET end end def current_transaction - ::Gitlab::Metrics::Transaction.current + ::Gitlab::Metrics::WebTransaction.current || ::Gitlab::Metrics::BackgroundTransaction.current end end end end end + +Gitlab::Metrics::Subscribers::ActiveRecord.prepend_if_ee('EE::Gitlab::Metrics::Subscribers::ActiveRecord') diff --git a/lib/gitlab/optimistic_locking.rb b/lib/gitlab/optimistic_locking.rb index d51d718c826..b29240985f1 100644 --- a/lib/gitlab/optimistic_locking.rb +++ b/lib/gitlab/optimistic_locking.rb @@ -2,22 +2,61 @@ module Gitlab module OptimisticLocking + MAX_RETRIES = 100 + module_function - def retry_lock(subject, retries = nil, &block) - retries ||= 100 - # TODO(Observability): We should be recording details of the number of retries and the duration of the total execution here - ActiveRecord::Base.transaction do - yield(subject) - end - rescue ActiveRecord::StaleObjectError - retries -= 1 - raise unless retries >= 0 + def retry_lock(subject, max_retries = MAX_RETRIES, name:, &block) + start_time = Gitlab::Metrics::System.monotonic_time + retry_attempts = 0 + + begin + ActiveRecord::Base.transaction do + yield(subject) + end + rescue ActiveRecord::StaleObjectError + raise unless retry_attempts < max_retries + + subject.reset - subject.reset - retry + retry_attempts += 1 + retry + ensure + retry_lock_histogram.observe({}, retry_attempts) + + log_optimistic_lock_retries( + name: name, + retry_attempts: retry_attempts, + start_time: start_time) + end end alias_method :retry_optimistic_lock, :retry_lock + + def log_optimistic_lock_retries(name:, retry_attempts:, start_time:) + return unless retry_attempts > 0 + + elapsed_time = Gitlab::Metrics::System.monotonic_time - start_time + + retry_lock_logger.info( + message: "Optimistic Lock released with retries", + name: name, + retries: retry_attempts, + time_s: elapsed_time) + end + + def retry_lock_logger + @retry_lock_logger ||= Gitlab::Services::Logger.build + end + + def retry_lock_histogram + @retry_lock_histogram ||= + Gitlab::Metrics.histogram( + :gitlab_optimistic_locking_retries, + 'Number of retry attempts to execute optimistic retry lock', + {}, + [0, 1, 2, 3, 5, 10, 50] + ) + end end end diff --git a/lib/gitlab/pagination/keyset/column_order_definition.rb b/lib/gitlab/pagination/keyset/column_order_definition.rb new file mode 100644 index 00000000000..0c8ec02a56b --- /dev/null +++ b/lib/gitlab/pagination/keyset/column_order_definition.rb @@ -0,0 +1,224 @@ +# frozen_string_literal: true + +module Gitlab + module Pagination + module Keyset + # This class stores information for one column (or SQL expression) which can be used in an + # ORDER BY SQL clasue. + # The goal of this class is to encapsulate all the metadata in one place which are needed to + # make keyset pagination work in a generalized way. + # + # == Arguments + # + # **order expression** (Arel::Nodes::Node | String) + # + # The actual SQL expression for the ORDER BY clause. + # + # Examples: + # # Arel column order definition + # Project.arel_table[:id].asc # ORDER BY projects.id ASC + # + # # Arel expression, calculated order definition + # Arel::Nodes::NamedFunction.new("COALESCE", [Project.arel_table[:issue_count].asc, 0]).asc # ORDER BY COALESCE(projects.issue_count, 0) + # + # # Another Arel expression + # Arel::Nodes::Multiplication(Issue.arel_table[:weight], Issue.arel_table[:time_spent]).desc + # + # # Raw string order definition + # 'issues.type DESC NULLS LAST' + # + # **column_expression** (Arel::Nodes::Node | String) + # + # Expression for the database column or an expression. This value will be used with logical operations (>, <, =, !=) + # when building the database query for the next page. + # + # Examples: + # # Arel column reference + # Issue.arel_table[:title] + # + # # Calculated value + # Arel::Nodes::Multiplication(Issue.arel_table[:weight], Issue.arel_table[:time_spent]) + # + # **attribute_name** (String | Symbol) + # + # An attribute on the loaded ActiveRecord model where the value can be obtained. + # + # Examples: + # # Simple attribute definition + # attribute_name = :title + # + # # Later on this attribute will be used like this: + # my_record = Issue.find(x) + # value = my_record[attribute_name] # reads data from the title column + # + # # Calculated value based on an Arel or raw SQL expression + # + # attribute_name = :lowercase_title + # + # # `lowercase_title` is not is not a table column therefore we need to make sure it's available in the `SELECT` clause + # + # my_record = Issue.select(:id, 'LOWER(title) as lowercase_title').last + # value = my_record[:lowercase_title] + # + # **distinct** + # + # Boolean value. + # + # Tells us whether the database column contains only distinct values. If the column is covered by + # a unique index then set to true. + # + # **nullable** (:not_nullable | :nulls_last | :nulls_first) + # + # Tells us whether the database column is nullable or not. This information can be + # obtained from the DB schema. + # + # If the column is not nullable, set this attribute to :not_nullable. + # + # If the column is nullable, then additional information is needed. Based on the ordering, the null values + # will show up at the top or at the bottom of the resultset. + # + # Examples: + # # Nulls are showing up at the top (for example: ORDER BY column ASC): + # nullable = :nulls_first + # + # # Nulls are showing up at the bottom (for example: ORDER BY column DESC): + # nullable = :nulls_last + # + # **order_direction** + # + # :asc or :desc + # + # Note: this is an optional attribute, the value will be inferred from the order_expression. + # Sometimes it's not possible to infer the order automatically. In this case an exception will be + # raised (when the query is executed). If the reverse order cannot be computed, it must be provided explicitly. + # + # **reversed_order_expression** + # + # The reversed version of the order_expression. + # + # A ColumnOrderDefinition object is able to reverse itself which is used when paginating backwards. + # When a complex order_expression is provided (raw string), then reversing the order automatically + # is not possible. In this case an exception will be raised. + # + # Example: + # + # order_expression = Project.arel_table[:id].asc + # reversed_order_expression = Project.arel_table[:id].desc + # + # **add_to_projections** + # + # Set to true if the column is not part of the queried table. (Not part of SELECT *) + # + # Example: + # + # - When the order is a calculated expression or the column is in another table (JOIN-ed) + # + # If the add_to_projections is true, the query builder will automatically add the column to the SELECT values + class ColumnOrderDefinition + REVERSED_ORDER_DIRECTIONS = { asc: :desc, desc: :asc }.freeze + REVERSED_NULL_POSITIONS = { nulls_first: :nulls_last, nulls_last: :nulls_first }.freeze + AREL_ORDER_CLASSES = { Arel::Nodes::Ascending => :asc, Arel::Nodes::Descending => :desc }.freeze + ALLOWED_NULLABLE_VALUES = [:not_nullable, :nulls_first, :nulls_last].freeze + + attr_reader :attribute_name, :column_expression, :order_expression, :add_to_projections + + def initialize(attribute_name:, order_expression:, column_expression: nil, reversed_order_expression: nil, nullable: :not_nullable, distinct: true, order_direction: nil, add_to_projections: false) + @attribute_name = attribute_name + @order_expression = order_expression + @column_expression = column_expression || calculate_column_expression(order_expression) + @distinct = distinct + @reversed_order_expression = reversed_order_expression || calculate_reversed_order(order_expression) + @nullable = parse_nullable(nullable, distinct) + @order_direction = parse_order_direction(order_expression, order_direction) + @add_to_projections = add_to_projections + end + + def reverse + self.class.new( + attribute_name: attribute_name, + column_expression: column_expression, + order_expression: reversed_order_expression, + reversed_order_expression: order_expression, + nullable: not_nullable? ? :not_nullable : REVERSED_NULL_POSITIONS[nullable], + distinct: distinct, + order_direction: REVERSED_ORDER_DIRECTIONS[order_direction] + ) + end + + def ascending_order? + order_direction == :asc + end + + def descending_order? + order_direction == :desc + end + + def nulls_first? + nullable == :nulls_first + end + + def nulls_last? + nullable == :nulls_last + end + + def not_nullable? + nullable == :not_nullable + end + + def nullable? + !not_nullable? + end + + def distinct? + distinct + end + + private + + attr_reader :reversed_order_expression, :nullable, :distinct, :order_direction + + def calculate_reversed_order(order_expression) + unless AREL_ORDER_CLASSES.has_key?(order_expression.class) # Arel can reverse simple orders + raise "Couldn't determine reversed order for `#{order_expression}`, please provide the `reversed_order_expression` parameter." + end + + order_expression.reverse + end + + def calculate_column_expression(order_expression) + if order_expression.respond_to?(:expr) + order_expression.expr + else + raise("Couldn't calculate the column expression. Please pass an ARel node as the order_expression, not a string.") + end + end + + def parse_order_direction(order_expression, order_direction) + transformed_order_direction = if order_direction.nil? && AREL_ORDER_CLASSES[order_expression.class] + AREL_ORDER_CLASSES[order_expression.class] + elsif order_direction.present? + order_direction.to_s.downcase.to_sym + end + + unless REVERSED_ORDER_DIRECTIONS.has_key?(transformed_order_direction) + raise "Invalid or missing `order_direction` (value: #{order_direction}) was given, the allowed values are: :asc or :desc" + end + + transformed_order_direction + end + + def parse_nullable(nullable, distinct) + if ALLOWED_NULLABLE_VALUES.exclude?(nullable) + raise "Invalid `nullable` is given (value: #{nullable}), the allowed values are: #{ALLOWED_NULLABLE_VALUES.join(', ')}" + end + + if nullable != :not_nullable && distinct + raise 'Invalid column definition, `distinct` and `nullable` columns are not allowed at the same time' + end + + nullable + end + end + end + end +end diff --git a/lib/gitlab/pagination/keyset/order.rb b/lib/gitlab/pagination/keyset/order.rb new file mode 100644 index 00000000000..e8e68a5c4a5 --- /dev/null +++ b/lib/gitlab/pagination/keyset/order.rb @@ -0,0 +1,248 @@ +# frozen_string_literal: true + +module Gitlab + module Pagination + module Keyset + # This class is a special ORDER BY clause which is compatible with ActiveRecord. It helps + # building keyset paginated queries. + # + # In ActiveRecord we use the `order()` method which will generate the `ORDER BY X` SQL clause + # + # Project.where(active: true).order(id: :asc) + # + # # Or + # + # Project.where(active: true).order(created_at: :asc, id: desc) + # + # Gitlab::Pagination::Keyset::Order class encapsulates more information about the order columns + # in order to implement keyset pagination in a generic way + # + # - Extract values from a record (usually the last item of the previous query) + # - Build query conditions based on the column configuration + # + # Example 1: Order by primary key + # + # # Simple order definition for the primary key as an ActiveRecord scope + # scope :id_asc_ordered, -> { + # keyset_order = Gitlab::Pagination::Keyset::Order.build([ + # Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + # attribute: :id, + # order_expression: Project.arel_table[:id].asc + # ) + # ]) + # + # reorder(keyset_order) + # } + # + # # ... Later in the application code: + # + # # Compatible with ActiveRecord's `order()` method + # page1 = Project.where(active: true).id_asc_ordered.limit(5) + # keyset_order = Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(page1) + # + # last_record = page1.last + # cursor_values = keyset_order.cursor_attributes_for_node(last_record) # { id: x } + # + # page2 = keyset_order.apply_cursor_conditions(Project.where(active: true).id_asc_ordered, cursor_values).limit(5) + # + # last_record = page2.last + # cursor_values = keyset_order.cursor_attributes_for_node(last_record) + # + # page3 = keyset_order.apply_cursor_conditions(Project.where(active: true).id_asc_ordered, cursor_values).limit(5) + # + # Example 2: Order by creation time and primary key (primary key is the tie breaker) + # + # scope :created_at_ordered, -> { + # keyset_order = Gitlab::Pagination::Keyset::Order.build([ + # Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + # attribute: :created_at, + # column_expression: Project.arel_table[:created_at], + # order_expression: Project.arel_table[:created_at].asc, + # distinct: false, # values in the column are not unique + # nullable: :nulls_last # we might see NULL values (bottom) + # ), + # Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + # attribute: :id, + # order_expression: Project.arel_table[:id].asc + # ) + # ]) + # + # reorder(keyset_order) + # } + # + class Order < Arel::Nodes::SqlLiteral + attr_reader :column_definitions + + def initialize(column_definitions:) + @column_definitions = column_definitions + + super(to_sql_literal(@column_definitions)) + end + + # Tells whether the given ActiveRecord::Relation has keyset ordering + def self.keyset_aware?(scope) + scope.order_values.first.is_a?(self) && scope.order_values.one? + end + + def self.extract_keyset_order_object(scope) + scope.order_values.first + end + + def self.build(column_definitions) + new(column_definitions: column_definitions) + end + + def cursor_attributes_for_node(node) + column_definitions.each_with_object({}) do |column_definition, hash| + field_value = node[column_definition.attribute_name] + hash[column_definition.attribute_name] = if field_value.is_a?(Time) + field_value.strftime('%Y-%m-%d %H:%M:%S.%N %Z') + elsif field_value.nil? + nil + else + field_value.to_s + end + end + end + + # This methods builds the conditions for the keyset pagination + # + # Example: + # + # |created_at|id| + # |----------|--| + # |2020-01-01| 1| + # | null| 2| + # | null| 3| + # |2020-02-01| 4| + # + # Note: created_at is not distinct and nullable + # Order `ORDER BY created_at DESC, id DESC` + # + # We get the following cursor values from the previous page: + # { id: 4, created_at: '2020-02-01' } + # + # To get the next rows, we need to build the following conditions: + # + # (created_at = '2020-02-01' AND id < 4) OR (created_at < '2020-01-01') + # + # DESC ordering ensures that NULL values are on top so we don't need conditions for NULL values + # + # Another cursor example: + # { id: 3, created_at: nil } + # + # To get the next rows, we need to build the following conditions: + # + # (id < 3 AND created_at IS NULL) OR (created_at IS NOT NULL) + def build_where_values(values) + return if values.blank? + + verify_incoming_values!(values) + + where_values = [] + + reversed_column_definitions = column_definitions.reverse + reversed_column_definitions.each_with_index do |column_definition, i| + value = values[column_definition.attribute_name] + + conditions_for_column(column_definition, value).each do |condition| + column_definitions_after_index = reversed_column_definitions.last(column_definitions.reverse.size - i - 1) + + equal_conditon_for_rest = column_definitions_after_index.map do |definition| + definition.column_expression.eq(values[definition.attribute_name]) + end + + where_values << Arel::Nodes::Grouping.new(Arel::Nodes::And.new([condition, *equal_conditon_for_rest].compact)) + end + end + + build_or_query(where_values) + end + + # rubocop: disable CodeReuse/ActiveRecord + def apply_cursor_conditions(scope, values = {}) + scope = apply_custom_projections(scope) + scope.where(build_where_values(values)) + end + # rubocop: enable CodeReuse/ActiveRecord + + def reversed_order + self.class.build(column_definitions.map(&:reverse)) + end + + private + + # Adds extra columns to the SELECT clause + def apply_custom_projections(scope) + additional_projections = column_definitions.select(&:add_to_projections).map do |column_definition| + # avoid mutating the original column_expression + column_definition.column_expression.dup.as(column_definition.attribute_name).to_sql + end + + scope = scope.select(*scope.arel.projections, *additional_projections) if additional_projections + scope + end + + def conditions_for_column(column_definition, value) + conditions = [] + # Depending on the order, build a query condition fragment for taking the next rows + if column_definition.distinct? || (!column_definition.distinct? && value.present?) + conditions << compare_column_with_value(column_definition, value) + end + + # When the column is nullable, additional conditions for NULL a NOT NULL values are necessary. + # This depends on the position of the nulls (top or bottom of the resultset). + if column_definition.nulls_first? && value.blank? + conditions << column_definition.column_expression.not_eq(nil) + elsif column_definition.nulls_last? && value.present? + conditions << column_definition.column_expression.eq(nil) + end + + conditions + end + + def compare_column_with_value(column_definition, value) + if column_definition.descending_order? + column_definition.column_expression.lt(value) + else + column_definition.column_expression.gt(value) + end + end + + def build_or_query(expressions) + or_expression = expressions.reduce { |or_expression, expression| Arel::Nodes::Or.new(or_expression, expression) } + + Arel::Nodes::Grouping.new(or_expression) + end + + def to_sql_literal(column_definitions) + column_definitions.map do |column_definition| + if column_definition.order_expression.respond_to?(:to_sql) + column_definition.order_expression.to_sql + else + column_definition.order_expression.to_s + end + end.join(', ') + end + + def verify_incoming_values!(values) + value_keys = values.keys.map(&:to_s) + order_attrbute_names = column_definitions.map(&:attribute_name).map(&:to_s) + missing_items = order_attrbute_names - value_keys + extra_items = value_keys - order_attrbute_names + + if missing_items.any? || extra_items.any? + error_text = ['Incorrect cursor values were given'] + + error_text << "Extra items: #{extra_items.join(', ')}" if extra_items.any? + error_text << "Missing items: #{missing_items.join(', ')}" if missing_items.any? + + error_text.compact + + raise error_text.join('. ') + end + end + end + end + end +end diff --git a/lib/gitlab/performance_bar/redis_adapter_when_peek_enabled.rb b/lib/gitlab/performance_bar/redis_adapter_when_peek_enabled.rb index 133d777fc32..ac5c907465e 100644 --- a/lib/gitlab/performance_bar/redis_adapter_when_peek_enabled.rb +++ b/lib/gitlab/performance_bar/redis_adapter_when_peek_enabled.rb @@ -17,7 +17,7 @@ module Gitlab # to a structured log # rubocop:disable Gitlab/ModuleWithInstanceVariables def enqueue_stats_job(request_id) - return unless gather_stats? + return unless Feature.enabled?(:performance_bar_stats) @client.sadd(GitlabPerformanceBarStatsWorker::STATS_KEY, request_id) @@ -43,12 +43,6 @@ module Gitlab ) end # rubocop:enable Gitlab/ModuleWithInstanceVariables - - def gather_stats? - return unless Feature.enabled?(:performance_bar_stats) - - Gitlab.com? || Gitlab.staging? || !Rails.env.production? - end end end end diff --git a/lib/gitlab/query_limiting.rb b/lib/gitlab/query_limiting.rb index 31e6b120e45..5e46e26e14e 100644 --- a/lib/gitlab/query_limiting.rb +++ b/lib/gitlab/query_limiting.rb @@ -4,9 +4,8 @@ module Gitlab module QueryLimiting # Returns true if we should enable tracking of query counts. # - # This is only enabled in production/staging if we're running on GitLab.com. - # This ensures we don't produce any errors that users can't do anything - # about themselves. + # This is only enabled in development and test to ensure we don't produce + # any errors that users of other environments can't do anything about themselves. def self.enable? Rails.env.development? || Rails.env.test? end @@ -19,7 +18,7 @@ module Gitlab # The issue URL is only meant to push developers into creating an issue # instead of blindly whitelisting offending blocks of code. def self.whitelist(issue_url) - return unless enable_whitelist? + return unless enable? unless issue_url.start_with?('https://') raise( @@ -30,9 +29,5 @@ module Gitlab Transaction&.current&.whitelisted = true end - - def self.enable_whitelist? - Rails.env.development? || Rails.env.test? - end end end diff --git a/lib/gitlab/query_limiting/active_support_subscriber.rb b/lib/gitlab/query_limiting/active_support_subscriber.rb index 065862174bb..138fae7b641 100644 --- a/lib/gitlab/query_limiting/active_support_subscriber.rb +++ b/lib/gitlab/query_limiting/active_support_subscriber.rb @@ -6,9 +6,10 @@ module Gitlab attach_to :active_record def sql(event) - unless event.payload.fetch(:cached, event.payload[:name] == 'CACHE') - Transaction.current&.increment - end + return if !Transaction.current || event.payload.fetch(:cached, event.payload[:name] == 'CACHE') + + Transaction.current.increment + Transaction.current.executed_sql(event.payload[:sql]) end end end diff --git a/lib/gitlab/query_limiting/transaction.rb b/lib/gitlab/query_limiting/transaction.rb index e8fad067fa6..196072dddda 100644 --- a/lib/gitlab/query_limiting/transaction.rb +++ b/lib/gitlab/query_limiting/transaction.rb @@ -15,6 +15,7 @@ module Gitlab # the sake of keeping things simple we hardcode this value here, it's not # supposed to be changed very often anyway. THRESHOLD = 100 + LOG_THRESHOLD = THRESHOLD * 1.5 # Error that is raised whenever exceeding the maximum number of queries. ThresholdExceededError = Class.new(StandardError) @@ -45,6 +46,7 @@ module Gitlab @action = nil @count = 0 @whitelisted = false + @sql_executed = [] end # Sends a notification based on the number of executed SQL queries. @@ -60,6 +62,10 @@ module Gitlab @count += 1 unless whitelisted end + def executed_sql(sql) + @sql_executed << sql if @count <= LOG_THRESHOLD + end + def raise_error? Rails.env.test? end @@ -71,8 +77,11 @@ module Gitlab def error_message header = 'Too many SQL queries were executed' header = "#{header} in #{action}" if action + msg = "a maximum of #{THRESHOLD} is allowed but #{count} SQL queries were executed" + log = @sql_executed.each_with_index.map { |sql, i| "#{i}: #{sql}" }.join("\n").presence + ellipsis = '...' if @count > LOG_THRESHOLD - "#{header}: a maximum of #{THRESHOLD} is allowed but #{count} SQL queries were executed" + ["#{header}: #{msg}", log, ellipsis].compact.join("\n") end end end diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb index c162ee545c6..7bf348346ea 100644 --- a/lib/gitlab/quick_actions/issue_actions.rb +++ b/lib/gitlab/quick_actions/issue_actions.rb @@ -235,6 +235,35 @@ module Gitlab @execution_message[:remove_zoom] = result.message end + desc _('Add email participant(s)') + explanation _('Adds email participant(s)') + params 'email1@example.com email2@example.com (up to 6 emails)' + types Issue + condition do + Feature.enabled?(:issue_email_participants, parent) && + current_user.can?(:"admin_#{quick_action_target.to_ability_name}", quick_action_target) + end + command :invite_email do |emails = ""| + MAX_NUMBER_OF_EMAILS = 6 + + existing_emails = quick_action_target.email_participants_downcase + emails_to_add = emails.split(' ').index_by { |email| [email.downcase, email] }.except(*existing_emails).each_value.first(MAX_NUMBER_OF_EMAILS) + added_emails = [] + + emails_to_add.each do |email| + new_participant = quick_action_target.issue_email_participants.create(email: email) + added_emails << email if new_participant.persisted? + end + + if added_emails.any? + message = _("added %{emails}") % { emails: added_emails.to_sentence } + SystemNoteService.add_email_participants(quick_action_target, quick_action_target.project, current_user, message) + @execution_message[:invite_email] = message.upcase_first << "." + else + @execution_message[:invite_email] = _("No email participants were added. Either none were provided, or they already exist.") + end + end + private def zoom_link_service diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 96f2b7570b3..00739c05386 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -61,6 +61,10 @@ module Gitlab maven_app_name_regex end + def npm_package_name_regex + @npm_package_name_regex ||= %r{\A(?:@(#{Gitlab::PathRegex::NAMESPACE_FORMAT_REGEX})/)?[-+\.\_a-zA-Z0-9]+\z} + end + def nuget_package_name_regex @nuget_package_name_regex ||= %r{\A[-+\.\_a-zA-Z0-9]+\z}.freeze end diff --git a/lib/gitlab/relative_positioning/closed_range.rb b/lib/gitlab/relative_positioning/closed_range.rb new file mode 100644 index 00000000000..8916d1face5 --- /dev/null +++ b/lib/gitlab/relative_positioning/closed_range.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module RelativePositioning + class ClosedRange < RelativePositioning::Range + def initialize(lhs, rhs) + @lhs, @rhs = lhs, rhs + raise IllegalRange, 'Either lhs or rhs is missing' unless lhs && rhs + raise IllegalRange, 'lhs and rhs cannot be the same object' if lhs == rhs + end + end + end +end diff --git a/lib/gitlab/relative_positioning/ending_at.rb b/lib/gitlab/relative_positioning/ending_at.rb new file mode 100644 index 00000000000..61060638ee6 --- /dev/null +++ b/lib/gitlab/relative_positioning/ending_at.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module RelativePositioning + class EndingAt < RelativePositioning::Range + include Gitlab::Utils::StrongMemoize + + def initialize(rhs) + @rhs = rhs + raise IllegalRange, 'rhs is required' unless rhs + end + + def lhs + strong_memoize(:lhs) { rhs.lhs_neighbour } + end + end + end +end diff --git a/lib/gitlab/relative_positioning/range.rb b/lib/gitlab/relative_positioning/range.rb index 0b0ccdf5be4..3214c72eb8b 100644 --- a/lib/gitlab/relative_positioning/range.rb +++ b/lib/gitlab/relative_positioning/range.rb @@ -31,39 +31,5 @@ module Gitlab other.is_a?(RelativePositioning::Range) && lhs == other.lhs && rhs == other.rhs end end - - class ClosedRange < RelativePositioning::Range - def initialize(lhs, rhs) - @lhs, @rhs = lhs, rhs - raise IllegalRange, 'Either lhs or rhs is missing' unless lhs && rhs - raise IllegalRange, 'lhs and rhs cannot be the same object' if lhs == rhs - end - end - - class StartingFrom < RelativePositioning::Range - include Gitlab::Utils::StrongMemoize - - def initialize(lhs) - @lhs = lhs - raise IllegalRange, 'lhs is required' unless lhs - end - - def rhs - strong_memoize(:rhs) { lhs.rhs_neighbour } - end - end - - class EndingAt < RelativePositioning::Range - include Gitlab::Utils::StrongMemoize - - def initialize(rhs) - @rhs = rhs - raise IllegalRange, 'rhs is required' unless rhs - end - - def lhs - strong_memoize(:lhs) { rhs.lhs_neighbour } - end - end end end diff --git a/lib/gitlab/relative_positioning/starting_from.rb b/lib/gitlab/relative_positioning/starting_from.rb new file mode 100644 index 00000000000..6ddd35a39ad --- /dev/null +++ b/lib/gitlab/relative_positioning/starting_from.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module RelativePositioning + class StartingFrom < RelativePositioning::Range + include Gitlab::Utils::StrongMemoize + + def initialize(lhs) + @lhs = lhs + raise IllegalRange, 'lhs is required' unless lhs + end + + def rhs + strong_memoize(:rhs) { lhs.rhs_neighbour } + end + end + end +end diff --git a/lib/gitlab/runtime.rb b/lib/gitlab/runtime.rb index 647ac169f05..968ef06b085 100644 --- a/lib/gitlab/runtime.rb +++ b/lib/gitlab/runtime.rb @@ -82,7 +82,10 @@ module Gitlab end def puma_in_clustered_mode? - puma? && Puma.cli_config.options[:workers].to_i > 0 + return unless puma? + return unless Puma.respond_to?(:cli_config) + + Puma.cli_config.options[:workers].to_i > 0 end def max_threads diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb index 48f204e0b86..7561e36cc33 100644 --- a/lib/gitlab/setup_helper.rb +++ b/lib/gitlab/setup_helper.rb @@ -115,7 +115,7 @@ module Gitlab config[:storage] = storages - internal_socket_dir = File.join(gitaly_dir, 'internal_sockets') + internal_socket_dir = options[:internal_socket_dir] || File.join(gitaly_dir, 'internal_sockets') FileUtils.mkdir(internal_socket_dir) unless File.exist?(internal_socket_dir) config[:internal_socket_dir] = internal_socket_dir diff --git a/lib/gitlab/sidekiq_middleware.rb b/lib/gitlab/sidekiq_middleware.rb index 5f912818605..a2696e17078 100644 --- a/lib/gitlab/sidekiq_middleware.rb +++ b/lib/gitlab/sidekiq_middleware.rb @@ -36,6 +36,8 @@ module Gitlab chain.add ::Gitlab::SidekiqMiddleware::DuplicateJobs::Client chain.add ::Gitlab::SidekiqStatus::ClientMiddleware chain.add ::Gitlab::SidekiqMiddleware::AdminMode::Client + # Size limiter should be placed at the bottom, but before the metrics midleware + chain.add ::Gitlab::SidekiqMiddleware::SizeLimiter::Client chain.add ::Gitlab::SidekiqMiddleware::ClientMetrics end end diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb index 4ab8d313ad8..cf768811ffd 100644 --- a/lib/gitlab/sidekiq_middleware/server_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb @@ -34,7 +34,8 @@ module Gitlab monotonic_time_start = Gitlab::Metrics::System.monotonic_time job_thread_cputime_start = get_thread_cputime begin - yield + transaction = Gitlab::Metrics::BackgroundTransaction.new + transaction.run { yield } job_succeeded = true ensure monotonic_time_end = Gitlab::Metrics::System.monotonic_time diff --git a/lib/gitlab/sidekiq_middleware/size_limiter/client.rb b/lib/gitlab/sidekiq_middleware/size_limiter/client.rb new file mode 100644 index 00000000000..bc8b1989e78 --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/size_limiter/client.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + module SizeLimiter + # This midleware is inserted into Sidekiq **client** middleware chain. It + # prevents the caller from dispatching a too-large job payload. The job + # payload should be small and simple. Read more at: + # https://github.com/mperham/sidekiq/wiki/Best-Practices#1-make-your-job-parameters-small-and-simple + class Client + def call(worker_class, job, queue, _redis_pool) + ::Gitlab::SidekiqMiddleware::SizeLimiter::Validator.validate!(worker_class, job) + + yield + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/size_limiter/exceed_limit_error.rb b/lib/gitlab/sidekiq_middleware/size_limiter/exceed_limit_error.rb new file mode 100644 index 00000000000..da6c903ccae --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/size_limiter/exceed_limit_error.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + module SizeLimiter + # A custom exception for size limiter. It contains worker class and its + # size to easier track later + class ExceedLimitError < StandardError + attr_reader :worker_class, :size, :size_limit + + def initialize(worker_class, size, size_limit) + @worker_class = worker_class + @size = size + @size_limit = size_limit + + super "#{@worker_class} job exceeds payload size limit (#{size}/#{size_limit})" + end + + def sentry_extra_data + { + worker_class: @worker_class.to_s, + size: @size.to_i, + size_limit: @size_limit.to_i + } + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb b/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb new file mode 100644 index 00000000000..2c50c4a2157 --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb @@ -0,0 +1,97 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + module SizeLimiter + # Validate a Sidekiq job payload limit based on current configuration. + # This validator pulls the configuration from the environment variables: + # + # - GITLAB_SIDEKIQ_SIZE_LIMITER_MODE: the current mode of the size + # limiter. This must be either `track` or `raise`. + # + # - GITLAB_SIDEKIQ_SIZE_LIMITER_LIMIT_BYTES: the size limit in bytes. + # + # If the size of job payload after serialization exceeds the limit, an + # error is tracked raised adhering to the mode. + class Validator + def self.validate!(worker_class, job) + new(worker_class, job).validate! + end + + DEFAULT_SIZE_LIMIT = 0 + + MODES = [ + TRACK_MODE = 'track', + RAISE_MODE = 'raise' + ].freeze + + attr_reader :mode, :size_limit + + def initialize( + worker_class, job, + mode: ENV['GITLAB_SIDEKIQ_SIZE_LIMITER_MODE'], + size_limit: ENV['GITLAB_SIDEKIQ_SIZE_LIMITER_LIMIT_BYTES'] + ) + @worker_class = worker_class + @job = job + + @mode = (mode || TRACK_MODE).to_s.strip + unless MODES.include?(@mode) + ::Sidekiq.logger.warn "Invalid Sidekiq size limiter mode: #{@mode}. Fallback to #{TRACK_MODE} mode." + @mode = TRACK_MODE + end + + @size_limit = (size_limit || DEFAULT_SIZE_LIMIT).to_i + if @size_limit < 0 + ::Sidekiq.logger.warn "Invalid Sidekiq size limiter limit: #{@size_limit}" + end + end + + def validate! + return unless @size_limit > 0 + + return if allow_big_payload? + return if job_size <= @size_limit + + exception = ExceedLimitError.new(@worker_class, job_size, @size_limit) + # This should belong to Gitlab::ErrorTracking. We'll remove this + # after this epic is done: + # https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/396 + exception.set_backtrace(backtrace) + + if raise_mode? + raise exception + else + track(exception) + end + end + + private + + def job_size + # This maynot be the optimal solution, but can be acceptable solution + # for now. Internally, Sidekiq calls Sidekiq.dump_json everywhere. + # There is no clean way to intefere to prevent double serialization. + @job_size ||= ::Sidekiq.dump_json(@job).bytesize + end + + def allow_big_payload? + worker_class = @worker_class.to_s.safe_constantize + worker_class.respond_to?(:big_payload?) && worker_class.big_payload? + end + + def raise_mode? + @mode == RAISE_MODE + end + + def track(exception) + Gitlab::ErrorTracking.track_exception(exception) + end + + def backtrace + Gitlab::BacktraceCleaner.clean_backtrace(caller) + end + end + end + end +end diff --git a/lib/gitlab/string_range_marker.rb b/lib/gitlab/string_range_marker.rb index 780fe4c7725..5ddc88edf50 100644 --- a/lib/gitlab/string_range_marker.rb +++ b/lib/gitlab/string_range_marker.rb @@ -15,8 +15,10 @@ module Gitlab end end - def mark(marker_ranges) - return rich_line unless marker_ranges&.any? + def mark(ranges) + return rich_line unless ranges&.any? + + marker_ranges = ranges.map { |range| Gitlab::MarkerRange.from_range(range) } if html_escaped rich_marker_ranges = [] @@ -24,7 +26,7 @@ module Gitlab # Map the inline-diff range based on the raw line to character positions in the rich line rich_positions = position_mapping[range].flatten # Turn the array of character positions into ranges - rich_marker_ranges.concat(collapse_ranges(rich_positions)) + rich_marker_ranges.concat(collapse_ranges(rich_positions, range.mode)) end else rich_marker_ranges = marker_ranges @@ -36,7 +38,7 @@ module Gitlab offset_range = (range.begin + offset)..(range.end + offset) original_text = rich_line[offset_range] - text = yield(original_text, left: i == 0, right: i == rich_marker_ranges.length - 1) + text = yield(original_text, left: i == 0, right: i == rich_marker_ranges.length - 1, mode: range.mode) rich_line[offset_range] = text @@ -90,21 +92,21 @@ module Gitlab end # Takes an array of integers, and returns an array of ranges covering the same integers - def collapse_ranges(positions) + def collapse_ranges(positions, mode) return [] if positions.empty? ranges = [] start = prev = positions[0] - range = start..prev + range = MarkerRange.new(start, prev, mode: mode) positions[1..-1].each do |pos| if pos == prev + 1 - range = start..pos + range = MarkerRange.new(start, pos, mode: mode) prev = pos else ranges << range start = prev = pos - range = start..prev + range = MarkerRange.new(start, prev, mode: mode) end end ranges << range diff --git a/lib/gitlab/template/base_template.rb b/lib/gitlab/template/base_template.rb index 3be77aff07e..dc006877129 100644 --- a/lib/gitlab/template/base_template.rb +++ b/lib/gitlab/template/base_template.rb @@ -87,11 +87,11 @@ module Gitlab raise NotImplementedError end - def by_category(category, project = nil) + def by_category(category, project = nil, empty_category_title: nil) directory = category_directory(category) files = finder(project).list_files_for(directory) - files.map { |f| new(f, project, category: category) }.sort + files.map { |f| new(f, project, category: category.presence || empty_category_title) }.sort end def category_directory(category) diff --git a/lib/gitlab/template/issue_template.rb b/lib/gitlab/template/issue_template.rb index 30fff9bdae9..6e579018e45 100644 --- a/lib/gitlab/template/issue_template.rb +++ b/lib/gitlab/template/issue_template.rb @@ -25,6 +25,10 @@ module Gitlab # follow-up issue: https://gitlab.com/gitlab-org/gitlab/-/issues/300279 project.repository.issue_template_names_hash end + + def by_category(category, project = nil, empty_category_title: nil) + super(category, project, empty_category_title: _('Project Templates')) + end end end end diff --git a/lib/gitlab/template/merge_request_template.rb b/lib/gitlab/template/merge_request_template.rb index 7491c8f26c6..241a823d870 100644 --- a/lib/gitlab/template/merge_request_template.rb +++ b/lib/gitlab/template/merge_request_template.rb @@ -25,6 +25,10 @@ module Gitlab # follow-up issue: https://gitlab.com/gitlab-org/gitlab/-/issues/300279 project.repository.merge_request_template_names_hash end + + def by_category(category, project = nil, empty_category_title: nil) + super(category, project, empty_category_title: _('Project Templates')) + end end end end diff --git a/lib/gitlab/tracking/standard_context.rb b/lib/gitlab/tracking/standard_context.rb index 92fdd008249..8ce16c11267 100644 --- a/lib/gitlab/tracking/standard_context.rb +++ b/lib/gitlab/tracking/standard_context.rb @@ -15,10 +15,14 @@ module Gitlab end def environment - return 'production' if Gitlab.com_and_canary? - return 'staging' if Gitlab.staging? + return 'production' if Gitlab.com? + + return 'org' if Gitlab.org? + + return 'self-managed' if Rails.env.production? + 'development' end diff --git a/lib/gitlab/tree_summary.rb b/lib/gitlab/tree_summary.rb index bc7b8bd2b94..86cd91f0a32 100644 --- a/lib/gitlab/tree_summary.rb +++ b/lib/gitlab/tree_summary.rb @@ -104,12 +104,12 @@ module Gitlab end def fetch_last_cached_commits_list - cache_key = ['projects', project.id, 'last_commits_list', commit.id, ensured_path, offset, limit] + cache_key = ['projects', project.id, 'last_commits', commit.id, ensured_path, offset, limit] commits = Rails.cache.fetch(cache_key, expires_in: CACHE_EXPIRE_IN) do repository .list_last_commits_for_tree(commit.id, ensured_path, offset: offset, limit: limit, literal_pathspec: true) - .transform_values!(&:to_hash) + .transform_values! { |commit| commit_to_hash(commit) } end commits.transform_values! { |value| Commit.from_hash(value, project) } @@ -121,6 +121,12 @@ module Gitlab resolved_commits[commit.id] ||= commit end + def commit_to_hash(commit) + commit.to_hash.tap do |hash| + hash[:message] = hash[:message].to_s.truncate_bytes(1.kilobyte, omission: '...') + end + end + def commit_path(commit) Gitlab::Routing.url_helpers.project_commit_path(project, commit) end diff --git a/lib/gitlab/usage/docs/helper.rb b/lib/gitlab/usage/docs/helper.rb index 8483334800b..1dc660e574b 100644 --- a/lib/gitlab/usage/docs/helper.rb +++ b/lib/gitlab/usage/docs/helper.rb @@ -5,9 +5,6 @@ module Gitlab module Docs # Helper with functions to be used by HAML templates module Helper - HEADER = %w(field value).freeze - SKIP_KEYS = %i(description).freeze - def auto_generated_comment <<-MARKDOWN.strip_heredoc --- @@ -27,35 +24,33 @@ module Gitlab end def render_name(name) - "## `#{name}`\n" + "### `#{name}`" end def render_description(object) - object.description + return 'Missing description' unless object[:description].present? + + object[:description] end - def render_attribute_row(key, value) - value = Gitlab::Usage::Docs::ValueFormatter.format(key, value) - table_row(["`#{key}`", value]) + def render_yaml_link(yaml_path) + "[YAML definition](#{yaml_path})" end - def render_attributes_table(object) - <<~MARKDOWN + def render_status(object) + "Status: #{format(:status, object[:status])}" + end - #{table_row(HEADER)} - #{table_row(HEADER.map { '---' })} - #{table_value_rows(object.attributes)} - MARKDOWN + def render_owner(object) + "Group: `#{object[:product_group]}`" end - def table_value_rows(attributes) - attributes.reject { |k, _| k.in?(SKIP_KEYS) }.map do |key, value| - render_attribute_row(key, value) - end.join("\n") + def render_tiers(object) + "Tiers:#{format(:tier, object[:tier])}" end - def table_row(array) - "| #{array.join(' | ')} |" + def format(key, value) + Gitlab::Usage::Docs::ValueFormatter.format(key, value) end end end diff --git a/lib/gitlab/usage/docs/templates/default.md.haml b/lib/gitlab/usage/docs/templates/default.md.haml index 86e93be66c7..19ad668019e 100644 --- a/lib/gitlab/usage/docs/templates/default.md.haml +++ b/lib/gitlab/usage/docs/templates/default.md.haml @@ -13,16 +13,26 @@ The Metrics Dictionary is based on the following metrics definition YAML files: - - [`config/metrics`]('https://gitlab.com/gitlab-org/gitlab/-/tree/master/config/metrics') + - [`config/metrics`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/config/metrics) - [`ee/config/metrics`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/ee/config/metrics) -Each table includes a `milestone`, which corresponds to the GitLab version when the metric -was released. + Each table includes a `milestone`, which corresponds to the GitLab version when the metric + was released. + + ## Metrics Definitions + \ - metrics_definitions.each do |name, object| = render_name(name) - - = render_description(object) - - = render_attributes_table(object) + \ + = render_description(object.attributes) + \ + = render_yaml_link(object.yaml_path) + \ + = render_owner(object.attributes) + \ + = render_status(object.attributes) + \ + = render_tiers(object.attributes) + \ diff --git a/lib/gitlab/usage/docs/value_formatter.rb b/lib/gitlab/usage/docs/value_formatter.rb index a2dc9b081f8..379e5df4d52 100644 --- a/lib/gitlab/usage/docs/value_formatter.rb +++ b/lib/gitlab/usage/docs/value_formatter.rb @@ -5,17 +5,19 @@ module Gitlab module Docs class ValueFormatter def self.format(key, value) + return '' unless value.present? + case key when :key_path "**`#{value}`**" when :data_source value.to_s.capitalize - when :product_group + when :product_group, :product_category, :status "`#{value}`" when :introduced_by_url "[Introduced by](#{value})" when :distribution, :tier - Array(value).join(', ') + Array(value).map { |tier| " `#{tier}`" }.join(',') else value end diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb index 01d202e4d45..3964eb39ad9 100644 --- a/lib/gitlab/usage/metric_definition.rb +++ b/lib/gitlab/usage/metric_definition.rb @@ -4,6 +4,7 @@ module Gitlab module Usage class MetricDefinition METRIC_SCHEMA_PATH = Rails.root.join('config', 'metrics', 'schema.json') + BASE_REPO_PATH = 'https://gitlab.com/gitlab-org/gitlab/-/blob/master' attr_reader :path attr_reader :attributes @@ -21,6 +22,10 @@ module Gitlab attributes end + def yaml_path + "#{BASE_REPO_PATH}#{path.delete_prefix(Rails.root.to_s)}" + end + def validate! unless skip_validation? self.class.schemer.validate(attributes.stringify_keys).each do |error| diff --git a/lib/gitlab/usage/metrics/aggregates/aggregate.rb b/lib/gitlab/usage/metrics/aggregates/aggregate.rb index 1fc40798320..1aeca87d849 100644 --- a/lib/gitlab/usage/metrics/aggregates/aggregate.rb +++ b/lib/gitlab/usage/metrics/aggregates/aggregate.rb @@ -11,6 +11,7 @@ module Gitlab AggregatedMetricError = Class.new(StandardError) UnknownAggregationOperator = Class.new(AggregatedMetricError) UnknownAggregationSource = Class.new(AggregatedMetricError) + DisallowedAggregationTimeFrame = Class.new(AggregatedMetricError) DATABASE_SOURCE = 'database' REDIS_SOURCE = 'redis' @@ -30,25 +31,38 @@ module Gitlab @recorded_at = recorded_at end + def all_time_data + aggregated_metrics_data(start_date: nil, end_date: nil, time_frame: Gitlab::Utils::UsageData::ALL_TIME_TIME_FRAME_NAME) + end + def monthly_data - aggregated_metrics_data(**monthly_time_range) + aggregated_metrics_data(**monthly_time_range.merge(time_frame: Gitlab::Utils::UsageData::TWENTY_EIGHT_DAYS_TIME_FRAME_NAME)) end def weekly_data - aggregated_metrics_data(**weekly_time_range) + aggregated_metrics_data(**weekly_time_range.merge(time_frame: Gitlab::Utils::UsageData::SEVEN_DAYS_TIME_FRAME_NAME)) end private attr_accessor :aggregated_metrics, :recorded_at - def aggregated_metrics_data(start_date:, end_date:) + def aggregated_metrics_data(start_date:, end_date:, time_frame:) aggregated_metrics.each_with_object({}) do |aggregation, data| next if aggregation[:feature_flag] && Feature.disabled?(aggregation[:feature_flag], default_enabled: :yaml, type: :development) + next unless aggregation[:time_frame].include?(time_frame) case aggregation[:source] when REDIS_SOURCE - data[aggregation[:name]] = calculate_count_for_aggregation(aggregation: aggregation, start_date: start_date, end_date: end_date) + if time_frame == Gitlab::Utils::UsageData::ALL_TIME_TIME_FRAME_NAME + data[aggregation[:name]] = Gitlab::Utils::UsageData::FALLBACK + Gitlab::ErrorTracking + .track_and_raise_for_dev_exception( + DisallowedAggregationTimeFrame.new("Aggregation time frame: 'all' is not allowed for aggregation with source: '#{REDIS_SOURCE}'") + ) + else + data[aggregation[:name]] = calculate_count_for_aggregation(aggregation: aggregation, start_date: start_date, end_date: end_date) + end when DATABASE_SOURCE next unless Feature.enabled?('database_sourced_aggregated_metrics', default_enabled: false, type: :development) @@ -155,3 +169,5 @@ module Gitlab end end end + +Gitlab::Usage::Metrics::Aggregates::Aggregate.prepend_if_ee('EE::Gitlab::Usage::Metrics::Aggregates::Aggregate') diff --git a/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb b/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb index 33678d2b813..a01efbdb1a6 100644 --- a/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb +++ b/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb @@ -55,15 +55,15 @@ module Gitlab end def time_period_to_human_name(time_period) - return Gitlab::Utils::UsageData::ALL_TIME_PERIOD_HUMAN_NAME if time_period.blank? + return Gitlab::Utils::UsageData::ALL_TIME_TIME_FRAME_NAME if time_period.blank? start_date = time_period.first.to_date end_date = time_period.last.to_date if (end_date - start_date).to_i > 7 - Gitlab::Utils::UsageData::MONTHLY_PERIOD_HUMAN_NAME + Gitlab::Utils::UsageData::TWENTY_EIGHT_DAYS_TIME_FRAME_NAME else - Gitlab::Utils::UsageData::WEEKLY_PERIOD_HUMAN_NAME + Gitlab::Utils::UsageData::SEVEN_DAYS_TIME_FRAME_NAME end end end diff --git a/lib/gitlab/usage/metrics/names_suggestions/generator.rb b/lib/gitlab/usage/metrics/names_suggestions/generator.rb new file mode 100644 index 00000000000..e560f6979b4 --- /dev/null +++ b/lib/gitlab/usage/metrics/names_suggestions/generator.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module NamesSuggestions + class Generator < ::Gitlab::UsageData + FREE_TEXT_METRIC_NAME = "<please fill metric name>" + + class << self + def generate(key_path) + uncached_data.deep_stringify_keys.dig(*key_path.split('.')) + end + + private + + def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) + "count_#{parse_target_and_source(column, relation)}" + end + + def distinct_count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) + "count_distinct_#{parse_target_and_source(column, relation)}" + end + + def redis_usage_counter + FREE_TEXT_METRIC_NAME + end + + def alt_usage_data(*) + FREE_TEXT_METRIC_NAME + end + + def redis_usage_data_totals(counter) + counter.fallback_totals.transform_values { |_| FREE_TEXT_METRIC_NAME} + end + + def sum(relation, column, *rest) + "sum_#{parse_target_and_source(column, relation)}" + end + + def estimate_batch_distinct_count(relation, column = nil, *rest) + "estimate_distinct_#{parse_target_and_source(column, relation)}" + end + + def add(*args) + "add_#{args.join('_and_')}" + end + + def parse_target_and_source(column, relation) + if column + "#{column}_from_#{relation.table_name}" + else + relation.table_name + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 8e096a9f351..0d9b76f3d34 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -60,6 +60,7 @@ module Gitlab .merge(compliance_unique_visits_data) .merge(search_unique_visits_data) .merge(redis_hll_counters) + .deep_merge(aggregated_metrics_data) end end @@ -192,7 +193,7 @@ module Gitlab container_expiration_policies_usage, service_desk_counts ).tap do |data| - data[:snippets] = data[:personal_snippets] + data[:project_snippets] + data[:snippets] = add(data[:personal_snippets], data[:project_snippets]) end } end @@ -224,10 +225,9 @@ module Gitlab project_snippets: count(ProjectSnippet.where(last_28_days_time_period)), projects_with_alerts_created: distinct_count(::AlertManagement::Alert.where(last_28_days_time_period), :project_id) }.merge( - snowplow_event_counts(last_28_days_time_period(column: :collector_tstamp)), - aggregated_metrics_monthly + snowplow_event_counts(last_28_days_time_period(column: :collector_tstamp)) ).tap do |data| - data[:snippets] = data[:personal_snippets] + data[:project_snippets] + data[:snippets] = add(data[:personal_snippets], data[:project_snippets]) end } end @@ -242,17 +242,15 @@ module Gitlab def system_usage_data_settings { settings: { - ldap_encrypted_secrets_enabled: alt_usage_data(fallback: nil) { Gitlab::Auth::Ldap::Config.encrypted_secrets.active? } + ldap_encrypted_secrets_enabled: alt_usage_data(fallback: nil) { Gitlab::Auth::Ldap::Config.encrypted_secrets.active? }, + operating_system: alt_usage_data(fallback: nil) { operating_system } } } end def system_usage_data_weekly { - counts_weekly: { - }.merge( - aggregated_metrics_weekly - ) + counts_weekly: {} } end @@ -505,6 +503,17 @@ module Gitlab end end + def operating_system + ohai_data = Ohai::System.new.tap do |oh| + oh.all_plugins(['platform']) + end.data + + platform = ohai_data['platform'] + platform = 'raspbian' if ohai_data['platform'] == 'debian' && /armv/.match?(ohai_data['kernel']['machine']) + + "#{platform}-#{ohai_data['platform_version']}" + end + def last_28_days_time_period(column: :created_at) { column => 30.days.ago..2.days.ago } end @@ -701,15 +710,13 @@ module Gitlab { redis_hll_counters: ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events_data } end - def aggregated_metrics_monthly - { - aggregated_metrics: aggregated_metrics.monthly_data - } - end - - def aggregated_metrics_weekly + def aggregated_metrics_data { - aggregated_metrics: aggregated_metrics.weekly_data + counts_weekly: { aggregated_metrics: aggregated_metrics.weekly_data }, + counts_monthly: { aggregated_metrics: aggregated_metrics.monthly_data }, + counts: aggregated_metrics + .all_time_data + .to_h { |key, value| ["aggregate_#{key}".to_sym, value.round] } } end @@ -821,11 +828,9 @@ module Gitlab def total_alert_issues # Remove prometheus table queries once they are deprecated # To be removed with https://gitlab.com/gitlab-org/gitlab/-/issues/217407. - [ - count(Issue.with_alert_management_alerts, start: issue_minimum_id, finish: issue_maximum_id), + add count(Issue.with_alert_management_alerts, start: issue_minimum_id, finish: issue_maximum_id), count(::Issue.with_self_managed_prometheus_alert_events, start: issue_minimum_id, finish: issue_maximum_id), count(::Issue.with_prometheus_alert_events, start: issue_minimum_id, finish: issue_maximum_id) - ].reduce(:+) end def user_minimum_id @@ -952,7 +957,7 @@ module Gitlab csv_issue_imports = distinct_count(Issues::CsvImport.where(time_period), :user_id) group_imports = distinct_count(::GroupImportState.where(time_period), :user_id) - project_imports + bulk_imports + jira_issue_imports + csv_issue_imports + group_imports + add(project_imports, bulk_imports, jira_issue_imports, csv_issue_imports, group_imports) end # rubocop:enable CodeReuse/ActiveRecord diff --git a/lib/gitlab/usage_data_counters/aggregated_metrics/code_review.yml b/lib/gitlab/usage_data_counters/aggregated_metrics/code_review.yml new file mode 100644 index 00000000000..4c2355d526a --- /dev/null +++ b/lib/gitlab/usage_data_counters/aggregated_metrics/code_review.yml @@ -0,0 +1,108 @@ +# code_review_extension_category_monthly_active_users +# This is only metrics related to the VS Code Extension for now. +# +# code_review_category_monthly_active_users +# This is the user based metrics. These should only be user based metrics and only be related to the Code Review things inside of GitLab. +# +# code_review_group_monthly_active_users +# This is an aggregation of both of the above aggregations. It's intended to represent all users who interact with our group across all of our categories. +--- +- name: code_review_group_monthly_active_users + operator: OR + feature_flag: usage_data_code_review_aggregation + source: redis + time_frame: [7d, 28d] + events: [ + 'i_code_review_user_single_file_diffs', + 'i_code_review_user_create_mr', + 'i_code_review_user_close_mr', + 'i_code_review_user_reopen_mr', + 'i_code_review_user_resolve_thread', + 'i_code_review_user_unresolve_thread', + 'i_code_review_edit_mr_title', + 'i_code_review_edit_mr_desc', + 'i_code_review_user_merge_mr', + 'i_code_review_user_create_mr_comment', + 'i_code_review_user_edit_mr_comment', + 'i_code_review_user_remove_mr_comment', + 'i_code_review_user_create_review_note', + 'i_code_review_user_publish_review', + 'i_code_review_user_create_multiline_mr_comment', + 'i_code_review_user_edit_multiline_mr_comment', + 'i_code_review_user_remove_multiline_mr_comment', + 'i_code_review_user_add_suggestion', + 'i_code_review_user_apply_suggestion', + 'i_code_review_user_assigned', + 'i_code_review_user_review_requested', + 'i_code_review_user_approve_mr', + 'i_code_review_user_unapprove_mr', + 'i_code_review_user_marked_as_draft', + 'i_code_review_user_unmarked_as_draft', + 'i_code_review_user_approval_rule_added', + 'i_code_review_user_approval_rule_deleted', + 'i_code_review_user_approval_rule_edited', + 'i_code_review_user_vs_code_api_request', + 'i_code_review_user_toggled_task_item_status', + 'i_code_review_user_create_mr_from_issue', + 'i_code_review_user_mr_discussion_locked', + 'i_code_review_user_mr_discussion_unlocked', + 'i_code_review_user_time_estimate_changed', + 'i_code_review_user_time_spent_changed', + 'i_code_review_user_assignees_changed', + 'i_code_review_user_reviewers_changed', + 'i_code_review_user_milestone_changed', + 'i_code_review_user_labels_changed' + ] +- name: code_review_category_monthly_active_users + operator: OR + feature_flag: usage_data_code_review_aggregation + source: redis + time_frame: [7d, 28d] + events: [ + 'i_code_review_user_single_file_diffs', + 'i_code_review_user_create_mr', + 'i_code_review_user_close_mr', + 'i_code_review_user_reopen_mr', + 'i_code_review_user_resolve_thread', + 'i_code_review_user_unresolve_thread', + 'i_code_review_edit_mr_title', + 'i_code_review_edit_mr_desc', + 'i_code_review_user_merge_mr', + 'i_code_review_user_create_mr_comment', + 'i_code_review_user_edit_mr_comment', + 'i_code_review_user_remove_mr_comment', + 'i_code_review_user_create_review_note', + 'i_code_review_user_publish_review', + 'i_code_review_user_create_multiline_mr_comment', + 'i_code_review_user_edit_multiline_mr_comment', + 'i_code_review_user_remove_multiline_mr_comment', + 'i_code_review_user_add_suggestion', + 'i_code_review_user_apply_suggestion', + 'i_code_review_user_assigned', + 'i_code_review_user_review_requested', + 'i_code_review_user_approve_mr', + 'i_code_review_user_unapprove_mr', + 'i_code_review_user_marked_as_draft', + 'i_code_review_user_unmarked_as_draft', + 'i_code_review_user_approval_rule_added', + 'i_code_review_user_approval_rule_deleted', + 'i_code_review_user_approval_rule_edited', + 'i_code_review_user_toggled_task_item_status', + 'i_code_review_user_create_mr_from_issue', + 'i_code_review_user_mr_discussion_locked', + 'i_code_review_user_mr_discussion_unlocked', + 'i_code_review_user_time_estimate_changed', + 'i_code_review_user_time_spent_changed', + 'i_code_review_user_assignees_changed', + 'i_code_review_user_reviewers_changed', + 'i_code_review_user_milestone_changed', + 'i_code_review_user_labels_changed' + ] +- name: code_review_extension_category_monthly_active_users + operator: OR + feature_flag: usage_data_code_review_aggregation + source: redis + time_frame: [7d, 28d] + events: [ + 'i_code_review_user_vs_code_api_request' + ] diff --git a/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml b/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml index 4d92202e7fd..73a55b5d5fa 100644 --- a/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml +++ b/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml @@ -1,3 +1,5 @@ +# Aggregated metrics that include EE only event names within `events:` attribute have to be defined at ee/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml +# instead of this file. #- name: unique name of aggregated metric # operator: aggregation operator. Valid values are: # - "OR": counts unique elements that were observed triggering any of following events @@ -7,6 +9,10 @@ # source: defines which datasource will be used to locate events that should be included in aggregated metric. Valid values are: # - database # - redis +# time_frame: defines time frames for aggregated metrics: +# - 7d - last 7 days +# - 28d - last 28 days +# - all - all historical available data, this time frame is not available for redis source # feature_flag: name of development feature flag that will be checked before metrics aggregation is performed. # Corresponding feature flag should have `default_enabled` attribute set to `false`. # This attribute is OPTIONAL and can be omitted, when `feature_flag` is missing no feature flag will be checked. @@ -14,18 +20,22 @@ - name: compliance_features_track_unique_visits_union operator: OR source: redis + time_frame: [7d, 28d] events: ['g_compliance_audit_events', 'g_compliance_dashboard', 'i_compliance_audit_events', 'a_compliance_audit_events_api', 'i_compliance_credential_inventory'] - name: product_analytics_test_metrics_union operator: OR source: redis + time_frame: [7d, 28d] events: ['i_search_total', 'i_search_advanced', 'i_search_paid'] - name: product_analytics_test_metrics_intersection operator: AND source: redis + time_frame: [7d, 28d] events: ['i_search_total', 'i_search_advanced', 'i_search_paid'] - name: incident_management_alerts_total_unique_counts operator: OR source: redis + time_frame: [7d, 28d] events: [ 'incident_management_alert_status_changed', 'incident_management_alert_assigned', @@ -35,6 +45,7 @@ - name: incident_management_incidents_total_unique_counts operator: OR source: redis + time_frame: [7d, 28d] events: [ 'incident_management_incident_created', 'incident_management_incident_reopened', @@ -51,10 +62,11 @@ - name: i_testing_paid_monthly_active_user_total operator: OR source: redis + time_frame: [7d, 28d] events: [ - 'i_testing_web_performance_widget_total', - 'i_testing_full_code_quality_report_total', - 'i_testing_group_code_coverage_visit_total', - 'i_testing_load_performance_widget_total', - 'i_testing_metrics_report_widget_total' -] + 'i_testing_web_performance_widget_total', + 'i_testing_full_code_quality_report_total', + 'i_testing_group_code_coverage_visit_total', + 'i_testing_load_performance_widget_total', + 'i_testing_metrics_report_widget_total' + ] diff --git a/lib/gitlab/usage_data_counters/counter_events/package_events.yml b/lib/gitlab/usage_data_counters/counter_events/package_events.yml index f6bddabdd44..e1648245f3f 100644 --- a/lib/gitlab/usage_data_counters/counter_events/package_events.yml +++ b/lib/gitlab/usage_data_counters/counter_events/package_events.yml @@ -41,6 +41,9 @@ - i_package_pypi_delete_package - i_package_pypi_pull_package - i_package_pypi_push_package +- i_package_rubygems_delete_package +- i_package_rubygems_pull_package +- i_package_rubygems_push_package - i_package_tag_delete_package - i_package_tag_pull_package - i_package_tag_push_package diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index 68ae239debb..336bef081a6 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -127,11 +127,15 @@ module Gitlab return unless Gitlab::CurrentSettings.usage_ping_enabled? event = event_for(event_name) - raise UnknownEvent, "Unknown event #{event_name}" unless event.present? + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(UnknownEvent.new("Unknown event #{event_name}")) unless event.present? return unless feature_enabled?(event) Gitlab::Redis::HLL.add(key: redis_key(event, time, context), value: values, expiry: expiry(event)) + rescue => e + # Ignore any exceptions unless is dev or test env + # The application flow should not be blocked by erros in tracking + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) end # The array of valid context on which we allow tracking diff --git a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml index d657c5487d7..18c5dc73de2 100644 --- a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml @@ -164,3 +164,43 @@ category: code_review aggregation: weekly feature_flag: usage_data_i_code_review_user_create_mr_from_issue +- name: i_code_review_user_mr_discussion_locked + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_mr_discussion_locked +- name: i_code_review_user_mr_discussion_unlocked + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_mr_discussion_unlocked +- name: i_code_review_user_time_estimate_changed + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_time_estimate_changed +- name: i_code_review_user_time_spent_changed + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_time_spent_changed +- name: i_code_review_user_assignees_changed + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_assignees_changed +- name: i_code_review_user_reviewers_changed + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_reviewers_changed +- name: i_code_review_user_milestone_changed + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_milestone_changed +- name: i_code_review_user_labels_changed + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_labels_changed diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index 79f319b2d58..80a79682338 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -439,3 +439,18 @@ redis_slot: pipeline_authoring aggregation: weekly feature_flag: usage_data_unique_users_committing_ciconfigfile +- name: o_pipeline_authoring_unique_users_pushing_mr_ciconfigfile + category: pipeline_authoring + redis_slot: pipeline_authoring + aggregation: weekly + feature_flag: usage_data_o_pipeline_authoring_unique_users_pushing_mr_ciconfigfile +# Epic events +# +# We are using the same slot of issue events 'project_management' for +# epic events to allow data aggregation. +# More information in: https://gitlab.com/gitlab-org/gitlab/-/issues/322405 +- name: g_project_management_epic_created + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity diff --git a/lib/gitlab/usage_data_counters/known_events/ecosystem.yml b/lib/gitlab/usage_data_counters/known_events/ecosystem.yml index 3fd02164f74..1c765bb1830 100644 --- a/lib/gitlab/usage_data_counters/known_events/ecosystem.yml +++ b/lib/gitlab/usage_data_counters/known_events/ecosystem.yml @@ -20,3 +20,49 @@ redis_slot: ecosystem aggregation: weekly feature_flag: usage_data_track_ecosystem_jira_service +- name: i_ecosystem_slack_service_issue_notification + category: ecosystem + redis_slot: ecosystem + aggregation: weekly + feature_flag: usage_data_track_ecosystem_slack_service +- name: i_ecosystem_slack_service_push_notification + category: ecosystem + redis_slot: ecosystem + aggregation: weekly + feature_flag: usage_data_track_ecosystem_slack_service +- name: i_ecosystem_slack_service_deployment_notification + category: ecosystem + redis_slot: ecosystem + aggregation: weekly + feature_flag: usage_data_track_ecosystem_slack_service +- name: i_ecosystem_slack_service_wiki_page_notification + category: ecosystem + redis_slot: ecosystem + aggregation: weekly + feature_flag: usage_data_track_ecosystem_slack_service +- name: i_ecosystem_slack_service_merge_request_notification + category: ecosystem + redis_slot: ecosystem + aggregation: weekly + feature_flag: usage_data_track_ecosystem_slack_service +- name: i_ecosystem_slack_service_note_notification + category: ecosystem + redis_slot: ecosystem + aggregation: weekly + feature_flag: usage_data_track_ecosystem_slack_service +- name: i_ecosystem_slack_service_tag_push_notification + category: ecosystem + redis_slot: ecosystem + aggregation: weekly + feature_flag: usage_data_track_ecosystem_slack_service +- name: i_ecosystem_slack_service_confidential_note_notification + category: ecosystem + redis_slot: ecosystem + aggregation: weekly + feature_flag: usage_data_track_ecosystem_slack_service +- name: i_ecosystem_slack_service_confidential_issue_notification + category: ecosystem + redis_slot: ecosystem + aggregation: weekly + feature_flag: usage_data_track_ecosystem_slack_service + diff --git a/lib/gitlab/usage_data_counters/known_events/package_events.yml b/lib/gitlab/usage_data_counters/known_events/package_events.yml index 78a2a587b34..b7e583003c8 100644 --- a/lib/gitlab/usage_data_counters/known_events/package_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/package_events.yml @@ -3,109 +3,95 @@ category: deploy_token_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis - name: i_package_composer_user category: user_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis - name: i_package_conan_deploy_token category: deploy_token_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis - name: i_package_conan_user category: user_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis - name: i_package_container_deploy_token category: deploy_token_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis - name: i_package_container_user category: user_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis - name: i_package_debian_deploy_token category: deploy_token_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis - name: i_package_debian_user category: user_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis - name: i_package_generic_deploy_token category: deploy_token_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis - name: i_package_generic_user category: user_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis - name: i_package_golang_deploy_token category: deploy_token_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis - name: i_package_golang_user category: user_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis - name: i_package_maven_deploy_token category: deploy_token_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis - name: i_package_maven_user category: user_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis - name: i_package_npm_deploy_token category: deploy_token_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis - name: i_package_npm_user category: user_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis - name: i_package_nuget_deploy_token category: deploy_token_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis - name: i_package_nuget_user category: user_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis - name: i_package_pypi_deploy_token category: deploy_token_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis - name: i_package_pypi_user category: user_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis +- name: i_package_rubygems_deploy_token + category: deploy_token_packages + aggregation: weekly + redis_slot: package +- name: i_package_rubygems_user + category: user_packages + aggregation: weekly + redis_slot: package - name: i_package_tag_deploy_token category: deploy_token_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis - name: i_package_tag_user category: user_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis diff --git a/lib/gitlab/usage_data_counters/known_events/quickactions.yml b/lib/gitlab/usage_data_counters/known_events/quickactions.yml index bf292047da0..0fe65afb237 100644 --- a/lib/gitlab/usage_data_counters/known_events/quickactions.yml +++ b/lib/gitlab/usage_data_counters/known_events/quickactions.yml @@ -324,3 +324,13 @@ redis_slot: quickactions aggregation: weekly feature_flag: usage_data_track_quickactions +- name: i_quickactions_invite_email_single + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_invite_email_multiple + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions diff --git a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb index b9856e1f74a..eb28a387a97 100644 --- a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb @@ -35,6 +35,15 @@ module Gitlab MR_EDIT_MR_TITLE_ACTION = 'i_code_review_edit_mr_title' MR_EDIT_MR_DESC_ACTION = 'i_code_review_edit_mr_desc' MR_CREATE_FROM_ISSUE_ACTION = 'i_code_review_user_create_mr_from_issue' + MR_DISCUSSION_LOCKED_ACTION = 'i_code_review_user_mr_discussion_locked' + MR_DISCUSSION_UNLOCKED_ACTION = 'i_code_review_user_mr_discussion_unlocked' + MR_TIME_ESTIMATE_CHANGED_ACTION = 'i_code_review_user_time_estimate_changed' + MR_TIME_SPENT_CHANGED_ACTION = 'i_code_review_user_time_spent_changed' + MR_ASSIGNEES_CHANGED_ACTION = 'i_code_review_user_assignees_changed' + MR_REVIEWERS_CHANGED_ACTION = 'i_code_review_user_reviewers_changed' + MR_INCLUDING_CI_CONFIG_ACTION = 'o_pipeline_authoring_unique_users_pushing_mr_ciconfigfile' + MR_MILESTONE_CHANGED_ACTION = 'i_code_review_user_milestone_changed' + MR_LABELS_CHANGED_ACTION = 'i_code_review_user_labels_changed' class << self def track_mr_diffs_action(merge_request:) @@ -153,6 +162,45 @@ module Gitlab track_unique_action_by_user(MR_CREATE_FROM_ISSUE_ACTION, user) end + def track_discussion_locked_action(user:) + track_unique_action_by_user(MR_DISCUSSION_LOCKED_ACTION, user) + end + + def track_discussion_unlocked_action(user:) + track_unique_action_by_user(MR_DISCUSSION_UNLOCKED_ACTION, user) + end + + def track_time_estimate_changed_action(user:) + track_unique_action_by_user(MR_TIME_ESTIMATE_CHANGED_ACTION, user) + end + + def track_time_spent_changed_action(user:) + track_unique_action_by_user(MR_TIME_SPENT_CHANGED_ACTION, user) + end + + def track_assignees_changed_action(user:) + track_unique_action_by_user(MR_ASSIGNEES_CHANGED_ACTION, user) + end + + def track_reviewers_changed_action(user:) + track_unique_action_by_user(MR_REVIEWERS_CHANGED_ACTION, user) + end + + def track_mr_including_ci_config(user:, merge_request:) + return unless Feature.enabled?(:usage_data_o_pipeline_authoring_unique_users_pushing_mr_ciconfigfile, user, default_enabled: :yaml) + return unless merge_request.includes_ci_config? + + track_unique_action_by_user(MR_INCLUDING_CI_CONFIG_ACTION, user) + end + + def track_milestone_changed_action(user:) + track_unique_action_by_user(MR_MILESTONE_CHANGED_ACTION, user) + end + + def track_labels_changed_action(user:) + track_unique_action_by_user(MR_LABELS_CHANGED_ACTION, user) + end + private def track_unique_action_by_merge_request(action, merge_request) diff --git a/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb index f757b51f73c..15c68fb3945 100644 --- a/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb @@ -34,6 +34,8 @@ module Gitlab event_name_for_unassign(args) when 'unlabel', 'remove_label' event_name_for_unlabel(args) + when 'invite_email' + 'invite_email' + event_name_quantifier(args.split) else name end @@ -44,10 +46,8 @@ module Gitlab if args.count == 1 && args.first == 'me' 'assign_self' - elsif args.count == 1 - 'assign_single' else - 'assign_multiple' + 'assign' + event_name_quantifier(args) end end @@ -82,6 +82,14 @@ module Gitlab 'unlabel_all' end end + + def event_name_quantifier(args) + if args.count == 1 + '_single' + else + '_multiple' + end + end end end end diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb index b275bdbacde..c00e7a2aa13 100644 --- a/lib/gitlab/usage_data_queries.rb +++ b/lib/gitlab/usage_data_queries.rb @@ -32,6 +32,10 @@ module Gitlab raw_sql(relation, column, :distinct) end + def add(*args) + 'SELECT ' + args.map {|arg| "(#{arg})" }.join(' + ') + end + private def raw_sql(relation, column, distinct = nil) diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb index 28dc66e19f8..cb652868d58 100644 --- a/lib/gitlab/utils/usage_data.rb +++ b/lib/gitlab/utils/usage_data.rb @@ -24,7 +24,8 @@ # alt_usage_data(fallback: nil) { Gitlab.config.registry.enabled } # # * redis_usage_data method -# handles ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent +# handles ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent, +# Gitlab::UsageDataCounters::HLLRedisCounter::EventError # returns -1 when a block is sent or hash with all values -1 when a counter is sent # different behaviour due to 2 different implementations of redis counter # @@ -39,9 +40,9 @@ module Gitlab FALLBACK = -1 DISTRIBUTED_HLL_FALLBACK = -2 - ALL_TIME_PERIOD_HUMAN_NAME = "all_time" - WEEKLY_PERIOD_HUMAN_NAME = "weekly" - MONTHLY_PERIOD_HUMAN_NAME = "monthly" + ALL_TIME_TIME_FRAME_NAME = "all" + SEVEN_DAYS_TIME_FRAME_NAME = "7d" + TWENTY_EIGHT_DAYS_TIME_FRAME_NAME = "28d" def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) if batch @@ -86,6 +87,14 @@ module Gitlab FALLBACK end + def add(*args) + return -1 if args.any?(&:negative?) + + args.sum + rescue StandardError + FALLBACK + end + def alt_usage_data(value = nil, fallback: FALLBACK, &block) if block_given? yield @@ -126,8 +135,6 @@ module Gitlab # @param event_name [String] the event name # @param values [Array|String] the values counted def track_usage_event(event_name, values) - return unless Feature.enabled?(:"usage_data_#{event_name}", default_enabled: true) - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name.to_s, values: values) end @@ -160,7 +167,7 @@ module Gitlab def redis_usage_counter yield - rescue ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent + rescue ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent, Gitlab::UsageDataCounters::HLLRedisCounter::EventError FALLBACK end diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index 76cf769d041..abfb7e2310e 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -44,9 +44,9 @@ module Gitlab def options { - N_('VisibilityLevel|Private') => PRIVATE, - N_('VisibilityLevel|Internal') => INTERNAL, - N_('VisibilityLevel|Public') => PUBLIC + s_('VisibilityLevel|Private') => PRIVATE, + s_('VisibilityLevel|Internal') => INTERNAL, + s_('VisibilityLevel|Public') => PUBLIC } end @@ -104,12 +104,7 @@ module Gitlab end def level_name(level) - level_name = N_('VisibilityLevel|Unknown') - options.each do |name, lvl| - level_name = name if lvl == level.to_i - end - - s_(level_name) + options.key(level.to_i) || s_('VisibilityLevel|Unknown') end def level_value(level) diff --git a/lib/gitlab/word_diff/chunk_collection.rb b/lib/gitlab/word_diff/chunk_collection.rb new file mode 100644 index 00000000000..dd388f75302 --- /dev/null +++ b/lib/gitlab/word_diff/chunk_collection.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module WordDiff + class ChunkCollection + def initialize + @chunks = [] + end + + def add(chunk) + @chunks << chunk + end + + def content + @chunks.join('') + end + + def reset + @chunks = [] + end + end + end +end diff --git a/lib/gitlab/word_diff/line_processor.rb b/lib/gitlab/word_diff/line_processor.rb new file mode 100644 index 00000000000..49263962dd6 --- /dev/null +++ b/lib/gitlab/word_diff/line_processor.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +# Converts a line from `git diff --word-diff=porcelain` output into a segment +# +# Possible options: +# 1. Diff hunk +# 2. Chunk +# 3. Newline +module Gitlab + module WordDiff + class LineProcessor + def initialize(line) + @line = line + end + + def extract + return if empty_line? + return Segments::DiffHunk.new(full_line) if diff_hunk? + return Segments::Newline.new if newline_delimiter? + + Segments::Chunk.new(full_line) + end + + private + + attr_reader :line + + def diff_hunk? + line =~ /^@@ -/ + end + + def empty_line? + full_line == ' ' + end + + def newline_delimiter? + full_line == '~' + end + + def full_line + @full_line ||= line.delete("\n") + end + end + end +end diff --git a/lib/gitlab/word_diff/parser.rb b/lib/gitlab/word_diff/parser.rb new file mode 100644 index 00000000000..3b6d4d4d384 --- /dev/null +++ b/lib/gitlab/word_diff/parser.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +# Converts git diff --word-diff=porcelain output to Gitlab::Diff::Line objects +# see: https://git-scm.com/docs/git-diff#Documentation/git-diff.txt-porcelain +module Gitlab + module WordDiff + class Parser + include Enumerable + + def parse(lines, diff_file: nil) + return [] if lines.blank? + + # By returning an Enumerator we make it possible to search for a single line (with #find) + # without having to instantiate all the others that come after it. + Enumerator.new do |yielder| + @chunks = ChunkCollection.new + @counter = PositionsCounter.new + + lines.each do |line| + segment = LineProcessor.new(line).extract + + case segment + when Segments::DiffHunk + next if segment.first_line? + + counter.set_pos_num(old: segment.pos_old, new: segment.pos_new) + + yielder << build_line(segment.to_s, 'match', parent_file: diff_file) + + when Segments::Chunk + @chunks.add(segment) + + when Segments::Newline + yielder << build_line(@chunks.content, nil, parent_file: diff_file) + + @chunks.reset + counter.increase_pos_num + end + end + end + end + + private + + attr_reader :counter + + def build_line(content, type, options = {}) + Gitlab::Diff::Line.new( + content, type, + counter.line_obj_index, counter.pos_old, counter.pos_new, + **options).tap do + counter.increase_obj_index + end + end + end + end +end diff --git a/lib/gitlab/word_diff/positions_counter.rb b/lib/gitlab/word_diff/positions_counter.rb new file mode 100644 index 00000000000..ca66b43755f --- /dev/null +++ b/lib/gitlab/word_diff/positions_counter.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +# Responsible for keeping track of line numbers and created Gitlab::Diff::Line objects +module Gitlab + module WordDiff + class PositionsCounter + def initialize + @pos_old = 1 + @pos_new = 1 + @line_obj_index = 0 + end + + attr_reader :pos_old, :pos_new, :line_obj_index + + def increase_pos_num + @pos_old += 1 + @pos_new += 1 + end + + def increase_obj_index + @line_obj_index += 1 + end + + def set_pos_num(old:, new:) + @pos_old = old + @pos_new = new + end + end + end +end diff --git a/lib/gitlab/word_diff/segments/chunk.rb b/lib/gitlab/word_diff/segments/chunk.rb new file mode 100644 index 00000000000..7c5850666f9 --- /dev/null +++ b/lib/gitlab/word_diff/segments/chunk.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +# Chunk is a part of the line that starts with ` `, `-`, `+` +# Consecutive chunks build a line. Line that starts with `~` is an identifier of +# end of the line. +module Gitlab + module WordDiff + module Segments + class Chunk + def initialize(line) + @line = line + end + + def removed? + line[0] == '-' + end + + def added? + line[0] == '+' + end + + def to_s + line[1..] || '' + end + + def length + to_s.length + end + + private + + attr_reader :line + end + end + end +end diff --git a/lib/gitlab/word_diff/segments/diff_hunk.rb b/lib/gitlab/word_diff/segments/diff_hunk.rb new file mode 100644 index 00000000000..88b6817676f --- /dev/null +++ b/lib/gitlab/word_diff/segments/diff_hunk.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +# Diff hunk is line that starts with @@ +# It contains information about start line numbers +# +# Example: +# @@ -1,4 +1,5 @@ +# +# See more: https://www.gnu.org/software/diffutils/manual/html_node/Detailed-Unified.html +module Gitlab + module WordDiff + module Segments + class DiffHunk + def initialize(line) + @line = line + end + + def pos_old + line.match(/\-[0-9]*/)[0].to_i.abs rescue 0 + end + + def pos_new + line.match(/\+[0-9]*/)[0].to_i.abs rescue 0 + end + + def first_line? + pos_old <= 1 && pos_new <= 1 + end + + def to_s + line + end + + private + + attr_reader :line + end + end + end +end diff --git a/lib/gitlab/word_diff/segments/newline.rb b/lib/gitlab/word_diff/segments/newline.rb new file mode 100644 index 00000000000..de8bbf252ff --- /dev/null +++ b/lib/gitlab/word_diff/segments/newline.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module WordDiff + module Segments + class Newline + def to_s + '' + end + end + end + end +end diff --git a/lib/gitlab/x509/signature.rb b/lib/gitlab/x509/signature.rb index 7d4d4d9d13a..edff1540cb3 100644 --- a/lib/gitlab/x509/signature.rb +++ b/lib/gitlab/x509/signature.rb @@ -52,6 +52,12 @@ module Gitlab strong_memoize(:cert_store) do store = OpenSSL::X509::Store.new store.set_default_paths + + if Feature.enabled?(:x509_forced_cert_loading, type: :ops) + # Forcibly load the default cert file because the OpenSSL library seemingly ignores it + store.add_file(OpenSSL::X509::DEFAULT_CERT_FILE) if File.exist?(OpenSSL::X509::DEFAULT_CERT_FILE) + end + # valid_signing_time? checks the time attributes already # this flag is required, otherwise expired certificates would become # unverified when notAfter within certificate attribute is reached |