diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-12-20 17:22:11 +0300 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-12-20 17:22:11 +0300 |
| commit | 0c872e02b2c822e3397515ec324051ff540f0cd5 (patch) | |
| tree | ce2fb6ce7030e4dad0f4118d21ab6453e5938cdd /lib/gitlab | |
| parent | f7e05a6853b12f02911494c4b3fe53d9540d74fc (diff) | |
Add latest changes from gitlab-org/gitlab@15-7-stable-eev15.7.0-rc42
Diffstat (limited to 'lib/gitlab')
304 files changed, 4278 insertions, 2308 deletions
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events.rb b/lib/gitlab/analytics/cycle_analytics/stage_events.rb index b7a11bc0418..9ac433944a8 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events.rb @@ -80,6 +80,10 @@ module Gitlab def self.internal_events INTERNAL_EVENTS end + + def self.selectable_events + (events - internal_events).sort_by(&:name) + end end end end diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb index b6ad25e700b..06ce1dbdc77 100644 --- a/lib/gitlab/application_context.rb +++ b/lib/gitlab/application_context.rb @@ -164,7 +164,11 @@ module Gitlab end def include_client? - set_values.include?(:user) || set_values.include?(:runner) || set_values.include?(:remote_ip) + # Don't overwrite an existing more specific client id with an `ip/` one. + original_client_id = self.class.current_context_attribute(:client_id).to_s + return false if original_client_id.starts_with?('user/') || original_client_id.starts_with?('runner/') + + include_user? || set_values.include?(:runner) || set_values.include?(:remote_ip) end def include_user? @@ -178,8 +182,8 @@ module Gitlab def client if runner "runner/#{runner.id}" - elsif user - "user/#{user.id}" + elsif user_id + "user/#{user_id}" else "ip/#{remote_ip}" end diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index 507f94d87a5..5b1bf99e297 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -51,8 +51,11 @@ module Gitlab project_testing_integration: { threshold: 5, interval: 1.minute }, email_verification: { threshold: 10, interval: 10.minutes }, email_verification_code_send: { threshold: 10, interval: 1.hour }, + phone_verification_send_code: { threshold: 10, interval: 1.hour }, + phone_verification_verify_code: { threshold: 10, interval: 10.minutes }, namespace_exists: { threshold: 20, interval: 1.minute }, - fetch_google_ip_list: { threshold: 10, interval: 1.minute } + fetch_google_ip_list: { threshold: 10, interval: 1.minute }, + jobs_index: { threshold: 600, interval: 1.minute } }.freeze end diff --git a/lib/gitlab/audit/auditor.rb b/lib/gitlab/audit/auditor.rb index 4a6e4e2e06e..fddc1f830aa 100644 --- a/lib/gitlab/audit/auditor.rb +++ b/lib/gitlab/audit/auditor.rb @@ -59,7 +59,8 @@ module Gitlab @context = context @name = @context.fetch(:name, 'audit_operation') - @stream_only = @context.fetch(:stream_only, false) + @is_audit_event_yaml_defined = Gitlab::Audit::Type::Definition.defined?(@name) + @stream_only = stream_only? @author = @context.fetch(:author) @scope = @context.fetch(:scope) @target = @context.fetch(:target) @@ -70,6 +71,14 @@ module Gitlab @target_details = @context[:target_details] @authentication_event = @context.fetch(:authentication_event, false) @authentication_provider = @context[:authentication_provider] + + # TODO: Remove this code once we close https://gitlab.com/gitlab-org/gitlab/-/issues/367870 + return unless @is_audit_event_yaml_defined + + # rubocop:disable Gitlab/RailsLogger + Rails.logger.warn('WARNING: Logging audit events without an event type definition will be deprecated soon.') + Rails.logger.warn('See https://docs.gitlab.com/ee/development/audit_event_guide/#event-type-definitions') + # rubocop:enable Gitlab/RailsLogger end def single_audit @@ -84,14 +93,23 @@ module Gitlab end def record(events) - log_events(events) unless @stream_only - send_to_stream(events) + @stream_only ? send_to_stream(events) : log_events_and_stream(events) end - def log_events(events) + def log_events_and_stream(events) log_authentication_event - log_to_database(events) + saved_events = log_to_database(events) + + # we only want to override events with saved_events when it successfully saves into database. + # we are doing so to ensure events in memory reflects events saved in database and have id column. + events = saved_events if saved_events.present? + + log_to_file_and_stream(events) + end + + def log_to_file_and_stream(events) log_to_file(events) + send_to_stream(events) end def audit_enabled? @@ -102,6 +120,14 @@ module Gitlab @authentication_event end + def stream_only? + if @is_audit_event_yaml_defined + Gitlab::Audit::Type::Definition.stream_only?(@name) + else + @context.fetch(:stream_only, false) + end + end + def log_authentication_event return unless Gitlab::Database.read_write? && authentication_event? @@ -145,7 +171,13 @@ module Gitlab end def log_to_database(events) - AuditEvent.bulk_insert!(events) + if events.one? + events.first.save! + events + else + event_ids = AuditEvent.bulk_insert!(events, returns: :ids) + AuditEvent.id_in(event_ids) + end rescue ActiveRecord::RecordInvalid => e ::Gitlab::ErrorTracking.track_exception(e, audit_operation: @name) end diff --git a/lib/gitlab/audit/type/definition.rb b/lib/gitlab/audit/type/definition.rb index f64f66f4ca4..81c88a3a0ae 100644 --- a/lib/gitlab/audit/type/definition.rb +++ b/lib/gitlab/audit/type/definition.rb @@ -59,19 +59,36 @@ module Gitlab end class << self + include ::Gitlab::Utils::StrongMemoize + def paths @paths ||= [Rails.root.join('config', 'audit_events', 'types', '*.yml')] end def definitions - # We lazily load all definitions - @definitions ||= load_all! + load_all! end + strong_memoize_attr :definitions def get(key) definitions[key.to_sym] end + def event_names + definitions.keys.map(&:to_s) + end + + def defined?(key) + get(key).present? + end + + def stream_only?(key) + event_definition = get(key) + return false unless event_definition + + event_definition.streamed && !event_definition.saved_to_database + end + private def load_all! diff --git a/lib/gitlab/audit/type/shared.rb b/lib/gitlab/audit/type/shared.rb index 999b7de13e2..1e3e26d3735 100644 --- a/lib/gitlab/audit/type/shared.rb +++ b/lib/gitlab/audit/type/shared.rb @@ -14,7 +14,7 @@ module Gitlab description introduced_by_issue introduced_by_mr - group + feature_category milestone saved_to_database streamed diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index c567df8e133..7e8f9c76dea 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -135,15 +135,13 @@ module Gitlab # it is important to reset the ban counter once the client has proven # they are not a 'bad guy'. rate_limiter.reset! - else + elsif rate_limiter.register_fail! # Register a login failure so that Rack::Attack can block the next # request from this IP if needed. # This returns true when the failures are over the threshold and the IP # is banned. - if rate_limiter.register_fail! - Gitlab::AppLogger.info "IP #{rate_limiter.ip} failed to login " \ + Gitlab::AppLogger.info "IP #{rate_limiter.ip} failed to login " \ "as #{login} but has been temporarily banned from Git auth" - end end end diff --git a/lib/gitlab/auth/current_user_mode.rb b/lib/gitlab/auth/current_user_mode.rb index fc391543f4d..9bd4711c4bb 100644 --- a/lib/gitlab/auth/current_user_mode.rb +++ b/lib/gitlab/auth/current_user_mode.rb @@ -106,8 +106,8 @@ module Gitlab end def enable_admin_mode!(password: nil, skip_password_validation: false) - return unless user&.admin? - return unless skip_password_validation || user&.valid_password?(password) + return false unless user&.admin? + return false unless skip_password_validation || user&.valid_password?(password) raise NotRequestedError unless admin_mode_requested? @@ -115,6 +115,10 @@ module Gitlab current_session_data[ADMIN_MODE_REQUESTED_TIME_KEY] = nil current_session_data[ADMIN_MODE_START_TIME_KEY] = Time.now + + audit_user_enable_admin_mode + + true end def disable_admin_mode! @@ -175,6 +179,10 @@ module Gitlab def privileged_runtime? Gitlab::Runtime.rake? || Gitlab::Runtime.rails_runner? || Gitlab::Runtime.console? end + + def audit_user_enable_admin_mode; end end end end + +Gitlab::Auth::CurrentUserMode.prepend_mod_with('Gitlab::Auth::CurrentUserMode') diff --git a/lib/gitlab/auth/ldap/access.rb b/lib/gitlab/auth/ldap/access.rb index 62a817d7c4d..ea098ff8057 100644 --- a/lib/gitlab/auth/ldap/access.rb +++ b/lib/gitlab/auth/ldap/access.rb @@ -12,7 +12,7 @@ module Gitlab def self.open(user, &block) Gitlab::Auth::Ldap::Adapter.open(user.ldap_identity.provider) do |adapter| - block.call(self.new(user, adapter)) + yield(self.new(user, adapter)) end end diff --git a/lib/gitlab/auth/ldap/adapter.rb b/lib/gitlab/auth/ldap/adapter.rb index 47eca74aa5b..9aedc131e92 100644 --- a/lib/gitlab/auth/ldap/adapter.rb +++ b/lib/gitlab/auth/ldap/adapter.rb @@ -11,7 +11,7 @@ module Gitlab def self.open(provider, &block) Net::LDAP.open(config(provider).adapter_options) do |ldap| - block.call(self.new(provider, ldap)) + yield(self.new(provider, ldap)) end end diff --git a/lib/gitlab/auth/ldap/config.rb b/lib/gitlab/auth/ldap/config.rb index 9dafd59561a..6c99b505797 100644 --- a/lib/gitlab/auth/ldap/config.rb +++ b/lib/gitlab/auth/ldap/config.rb @@ -82,7 +82,8 @@ module Gitlab def adapter_options opts = base_options.merge( - encryption: encryption_options + encryption: encryption_options, + instrumentation_service: ActiveSupport::Notifications ) opts.merge!(auth_options) if has_auth? diff --git a/lib/gitlab/auth/ldap/dn.rb b/lib/gitlab/auth/ldap/dn.rb index a188aa168c1..84bf455c98a 100644 --- a/lib/gitlab/auth/ldap/dn.rb +++ b/lib/gitlab/auth/ldap/dn.rb @@ -51,7 +51,7 @@ module Gitlab ## # Parse a DN into key value pairs using ASN from - # http://tools.ietf.org/html/rfc2253 section 3. + # https://www.rfc-editor.org/rfc/rfc2253 section 3. # rubocop:disable Metrics/AbcSize # rubocop:disable Metrics/CyclomaticComplexity # rubocop:disable Metrics/PerceivedComplexity @@ -231,7 +231,7 @@ module Gitlab self.class.new(*to_a).to_s.downcase end - # https://tools.ietf.org/html/rfc4514 section 2.4 lists these exceptions + # https://www.rfc-editor.org/rfc/rfc4514 section 2.4 lists these exceptions # for DN values. All of the following must be escaped in any normal string # using a single backslash ('\') as escape. The space character is left # out here because in a "normalized" string, spaces should only be escaped diff --git a/lib/gitlab/background_migration/backfill_environment_tiers.rb b/lib/gitlab/background_migration/backfill_environment_tiers.rb new file mode 100644 index 00000000000..6f381577274 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_environment_tiers.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class backfills the `environments.tier` column by using `guess_tier` logic. + # Environments created after 13.10 already have a value, however, environments created before 13.10 don't. + # See https://gitlab.com/gitlab-org/gitlab/-/issues/300741 for more information. + class BackfillEnvironmentTiers < BatchedMigrationJob + operation_name :backfill_environment_tiers + + # Equivalent to `Environment#guess_tier` pattern matching. + PRODUCTION_TIER = 0 + STAGING_TIER = 1 + TESTING_TIER = 2 + DEVELOPMENT_TIER = 3 + OTHER_TIER = 4 + + TIER_REGEXP_PAIR = [ + { tier: DEVELOPMENT_TIER, regexp: '(dev|review|trunk)' }, + { tier: TESTING_TIER, regexp: '(test|tst|int|ac(ce|)pt|qa|qc|control|quality)' }, + { tier: STAGING_TIER, regexp: '(st(a|)g|mod(e|)l|pre|demo|non)' }, + { tier: PRODUCTION_TIER, regexp: '(pr(o|)d|live)' } + ].freeze + + def perform + TIER_REGEXP_PAIR.each do |pair| + each_sub_batch( + batching_scope: ->(relation) { relation.where(tier: nil).where("name ~* '#{pair[:regexp]}'") } # rubocop:disable GitlabSecurity/SqlInjection + ) do |sub_batch| + sub_batch.update_all(tier: pair[:tier]) + end + end + + each_sub_batch(batching_scope: ->(relation) { relation.where(tier: nil) }) do |sub_batch| + sub_batch.update_all(tier: OTHER_TIER) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_note_discussion_id.rb b/lib/gitlab/background_migration/backfill_note_discussion_id.rb index da2c31ebd11..ce2698b3cb8 100644 --- a/lib/gitlab/background_migration/backfill_note_discussion_id.rb +++ b/lib/gitlab/background_migration/backfill_note_discussion_id.rb @@ -33,8 +33,8 @@ module Gitlab private def update_discussion_ids(notes) - mapping = notes.each_with_object({}) do |note, hash| - hash[note] = { discussion_id: note.generate_discussion_id } + mapping = notes.index_with do |note| + { discussion_id: note.generate_discussion_id } end Gitlab::Database::BulkUpdate.execute(%i(discussion_id), mapping) diff --git a/lib/gitlab/background_migration/backfill_project_statistics_storage_size_without_uploads_size.rb b/lib/gitlab/background_migration/backfill_project_statistics_storage_size_without_uploads_size.rb new file mode 100644 index 00000000000..1a3dd88ea31 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_project_statistics_storage_size_without_uploads_size.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Back-fill storage_size for project_statistics + class BackfillProjectStatisticsStorageSizeWithoutUploadsSize < Gitlab::BackgroundMigration::BatchedMigrationJob + def perform + # no-op + end + end + end +end + +Gitlab::BackgroundMigration::BackfillProjectStatisticsStorageSizeWithoutUploadsSize.prepend_mod_with('Gitlab::BackgroundMigration::BackfillProjectStatisticsStorageSizeWithoutUploadsSize') # rubocop:disable Layout/LineLength diff --git a/lib/gitlab/background_migration/batched_migration_job.rb b/lib/gitlab/background_migration/batched_migration_job.rb index 64401bc0674..973ab20f547 100644 --- a/lib/gitlab/background_migration/batched_migration_job.rb +++ b/lib/gitlab/background_migration/batched_migration_job.rb @@ -9,55 +9,68 @@ module Gitlab # see https://docs.gitlab.com/ee/development/database/batched_background_migrations.html#job-arguments. class BatchedMigrationJob include Gitlab::Database::DynamicModelHelpers + include Gitlab::ClassAttributes - def initialize( - start_id:, end_id:, batch_table:, batch_column:, sub_batch_size:, pause_ms:, job_arguments: [], connection: - ) + DEFAULT_FEATURE_CATEGORY = :database - @start_id = start_id - @end_id = end_id - @batch_table = batch_table - @batch_column = batch_column - @sub_batch_size = sub_batch_size - @pause_ms = pause_ms - @job_arguments = job_arguments - @connection = connection - end + class << self + def generic_instance(batch_table:, batch_column:, job_arguments: [], connection:) + new( + batch_table: batch_table, batch_column: batch_column, + job_arguments: job_arguments, connection: connection, + start_id: 0, end_id: 0, sub_batch_size: 0, pause_ms: 0 + ) + end - def self.generic_instance(batch_table:, batch_column:, job_arguments: [], connection:) - new( - batch_table: batch_table, batch_column: batch_column, - job_arguments: job_arguments, connection: connection, - start_id: 0, end_id: 0, sub_batch_size: 0, pause_ms: 0 - ) - end + def job_arguments_count + 0 + end - def self.job_arguments_count - 0 - end + def operation_name(operation) + define_method('operation_name') do + operation + end + end - def self.operation_name(operation) - define_method('operation_name') do - operation + def job_arguments(*args) + args.each.with_index do |arg, index| + define_method(arg) do + @job_arguments[index] + end + end + + define_singleton_method(:job_arguments_count) do + args.count + end end - end - def self.job_arguments(*args) - args.each.with_index do |arg, index| - define_method(arg) do - @job_arguments[index] + def scope_to(scope) + define_method(:filter_batch) do |relation| + instance_exec(relation, &scope) end end - define_singleton_method(:job_arguments_count) do - args.count + def feature_category(feature_category_name = nil) + if feature_category_name.present? + set_class_attribute(:feature_category, feature_category_name) + else + get_class_attribute(:feature_category) || DEFAULT_FEATURE_CATEGORY + end end end - def self.scope_to(scope) - define_method(:filter_batch) do |relation| - instance_exec(relation, &scope) - end + def initialize( + start_id:, end_id:, batch_table:, batch_column:, sub_batch_size:, pause_ms:, job_arguments: [], connection: + ) + + @start_id = start_id + @end_id = end_id + @batch_table = batch_table + @batch_column = batch_column + @sub_batch_size = sub_batch_size + @pause_ms = pause_ms + @job_arguments = job_arguments + @connection = connection end def filter_batch(relation) diff --git a/lib/gitlab/background_migration/delete_orphans_approval_merge_request_rules.rb b/lib/gitlab/background_migration/delete_orphans_approval_merge_request_rules.rb new file mode 100644 index 00000000000..4b7b7d42c77 --- /dev/null +++ b/lib/gitlab/background_migration/delete_orphans_approval_merge_request_rules.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Deletes orphans records whenever report_type equals to scan_finding (i.e., 4) + class DeleteOrphansApprovalMergeRequestRules < BatchedMigrationJob + scope_to ->(relation) { relation.where(report_type: 4) } + + operation_name :delete_all + + def perform + each_sub_batch do |sub_batch| + sub_batch.where(security_orchestration_policy_configuration_id: nil).delete_all + end + end + end + end +end diff --git a/lib/gitlab/background_migration/delete_orphans_approval_project_rules.rb b/lib/gitlab/background_migration/delete_orphans_approval_project_rules.rb new file mode 100644 index 00000000000..33aa1a8d29d --- /dev/null +++ b/lib/gitlab/background_migration/delete_orphans_approval_project_rules.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Deletes orphans records whenever report_type equals to scan_finding (i.e., 4) + class DeleteOrphansApprovalProjectRules < BatchedMigrationJob + operation_name :delete_all + + def perform + each_sub_batch do |sub_batch| + sub_batch.where(report_type: 4, security_orchestration_policy_configuration_id: nil).delete_all + end + end + end + end +end diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_five_mb.rb b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_five_mb.rb new file mode 100644 index 00000000000..dcef4f086e2 --- /dev/null +++ b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_five_mb.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Set `project_settings.legacy_open_source_license_available` to false for projects less than 5 MB + class DisableLegacyOpenSourceLicenseForProjectsLessThanFiveMb < ::Gitlab::BackgroundMigration::BatchedMigrationJob + scope_to ->(relation) do + relation + .where(legacy_open_source_license_available: true) + end + + operation_name :disable_legacy_open_source_license_for_projects_less_than_five_mb + + def perform + each_sub_batch do |sub_batch| + updates = { legacy_open_source_license_available: false, updated_at: Time.current } + + sub_batch + .joins('INNER JOIN project_statistics ON project_statistics.project_id = project_settings.project_id') + .where('project_statistics.repository_size < ?', 5.megabyte) + .update_all(updates) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/fix_approval_project_rules_without_protected_branches.rb b/lib/gitlab/background_migration/fix_approval_project_rules_without_protected_branches.rb new file mode 100644 index 00000000000..4b283bae79d --- /dev/null +++ b/lib/gitlab/background_migration/fix_approval_project_rules_without_protected_branches.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class doesn't update approval project rules + # as this feature exists only in EE + class FixApprovalProjectRulesWithoutProtectedBranches < BatchedMigrationJob + def perform; end + end + end +end + +# rubocop:disable Layout/LineLength +Gitlab::BackgroundMigration::FixApprovalProjectRulesWithoutProtectedBranches.prepend_mod_with('Gitlab::BackgroundMigration::FixApprovalProjectRulesWithoutProtectedBranches') +# rubocop:enable Layout/LineLength diff --git a/lib/gitlab/background_migration/fix_security_scan_statuses.rb b/lib/gitlab/background_migration/fix_security_scan_statuses.rb new file mode 100644 index 00000000000..b60e739f870 --- /dev/null +++ b/lib/gitlab/background_migration/fix_security_scan_statuses.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Fixes the `status` attribute of `security_scans` records + class FixSecurityScanStatuses < BatchedMigrationJob + def perform + # no-op. The logic is defined in EE module. + end + end + end +end + +::Gitlab::BackgroundMigration::FixSecurityScanStatuses.prepend_mod diff --git a/lib/gitlab/background_migration/migrate_vulnerabilities_feedback_to_vulnerabilities_state_transition.rb b/lib/gitlab/background_migration/migrate_vulnerabilities_feedback_to_vulnerabilities_state_transition.rb new file mode 100644 index 00000000000..81b29b5a6cd --- /dev/null +++ b/lib/gitlab/background_migration/migrate_vulnerabilities_feedback_to_vulnerabilities_state_transition.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +# rubocop:disable Style/Documentation +module Gitlab + module BackgroundMigration + class MigrateVulnerabilitiesFeedbackToVulnerabilitiesStateTransition < BatchedMigrationJob + def perform; end + end + end +end + +Gitlab::BackgroundMigration::MigrateVulnerabilitiesFeedbackToVulnerabilitiesStateTransition.prepend_mod +# rubocop:enable Style/Documentation diff --git a/lib/gitlab/background_migration/prune_stale_project_export_jobs.rb b/lib/gitlab/background_migration/prune_stale_project_export_jobs.rb new file mode 100644 index 00000000000..a91cda2c427 --- /dev/null +++ b/lib/gitlab/background_migration/prune_stale_project_export_jobs.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Background migration for deleting stale project import jobs + class PruneStaleProjectExportJobs < BatchedMigrationJob + EXPIRES_IN = 7.days + + scope_to ->(relation) { relation.where("updated_at < ?", EXPIRES_IN.ago) } + operation_name :delete_all + + def perform + each_sub_batch(&:delete_all) + end + end + end +end diff --git a/lib/gitlab/background_migration/reset_status_on_container_repositories.rb b/lib/gitlab/background_migration/reset_status_on_container_repositories.rb new file mode 100644 index 00000000000..09cd3b1895f --- /dev/null +++ b/lib/gitlab/background_migration/reset_status_on_container_repositories.rb @@ -0,0 +1,139 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job that: + # * pickup container repositories with delete_scheduled status. + # * check if there are tags linked to it. + # * if there are tags, reset the status to nil. + class ResetStatusOnContainerRepositories < BatchedMigrationJob + DELETE_SCHEDULED_STATUS = 0 + DUMMY_TAGS = %w[tag].freeze + MIGRATOR = 'ResetStatusOnContainerRepositories' + + scope_to ->(relation) { relation.where(status: DELETE_SCHEDULED_STATUS) } + operation_name :reset_status_on_container_repositories + + def perform + each_sub_batch do |sub_batch| + reset_status_if_tags(sub_batch) + end + end + + private + + def reset_status_if_tags(container_repositories) + container_repositories_with_tags = container_repositories.select { |cr| cr.becomes(ContainerRepository).tags? } # rubocop:disable Cop/AvoidBecomes + + ContainerRepository.where(id: container_repositories_with_tags.map(&:id)) + .update_all(status: nil) + end + + # rubocop:disable Style/Documentation + module Routable + extend ActiveSupport::Concern + + included do + has_one :route, + as: :source, + class_name: '::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Route' + end + + def full_path + route&.path || build_full_path + end + + def build_full_path + if parent && path + "#{parent.full_path}/#{path}" + else + path + end + end + end + + class Route < ::ApplicationRecord + self.table_name = 'routes' + end + + class Namespace < ::ApplicationRecord + include ::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Routable + include ::Namespaces::Traversal::Recursive + include ::Namespaces::Traversal::Linear + include ::Gitlab::Utils::StrongMemoize + + self.table_name = 'namespaces' + self.inheritance_column = :_type_disabled + + belongs_to :parent, + class_name: '::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Namespace' + + def self.polymorphic_name + 'Namespace' + end + end + + class Project < ::ApplicationRecord + include ::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Routable + + self.table_name = 'projects' + + belongs_to :namespace, + class_name: '::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Namespace' + + alias_method :parent, :namespace + alias_attribute :parent_id, :namespace_id + + delegate :root_ancestor, to: :namespace, allow_nil: true + end + + class ContainerRepository < ::ApplicationRecord + self.table_name = 'container_repositories' + + belongs_to :project, + class_name: '::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Project' + + def tags? + result = ContainerRegistry.tags_for(path).any? + ::Gitlab::BackgroundMigration::Logger.info( + migrator: MIGRATOR, + has_tags: result, + container_repository_id: id, + container_repository_path: path + ) + result + end + + def path + @path ||= [project.full_path, name].select(&:present?).join('/').downcase + end + end + + class ContainerRegistry + class << self + def tags_for(path) + response = ContainerRegistryClient.repository_tags(path, page_size: 1) + return DUMMY_TAGS unless response + + response['tags'] || [] + rescue StandardError + DUMMY_TAGS + end + end + end + + class ContainerRegistryClient + def self.repository_tags(path, page_size:) + registry_config = ::Gitlab.config.registry + + return { 'tags' => DUMMY_TAGS } unless registry_config.enabled && registry_config.api_url.present? + + pull_token = ::Auth::ContainerRegistryAuthenticationService.pull_access_token(path) + client = ::ContainerRegistry::Client.new(registry_config.api_url, token: pull_token) + client.repository_tags(path, page_size: page_size) + end + end + # rubocop:enable Style/Documentation + end + end +end diff --git a/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb b/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb index e9a38916999..8aab7d13b45 100644 --- a/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb +++ b/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb @@ -37,16 +37,16 @@ module Gitlab end end - cloud_mappings = cloud.each_with_object({}) do |tracker_data, hash| - hash[tracker_data] = { deployment_type: 2 } + cloud_mappings = cloud.index_with do + { deployment_type: 2 } end - server_mappings = server.each_with_object({}) do |tracker_data, hash| - hash[tracker_data] = { deployment_type: 1 } + server_mappings = server.index_with do + { deployment_type: 1 } end - unknown_mappings = unknown.each_with_object({}) do |tracker_data, hash| - hash[tracker_data] = { deployment_type: 0 } + unknown_mappings = unknown.index_with do + { deployment_type: 0 } end mappings = cloud_mappings.merge(server_mappings, unknown_mappings) diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 7de6be45349..49b8ab760f3 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -98,7 +98,7 @@ module Gitlab create_labels - issue_type_id = WorkItems::Type.default_issue_type.id + issue_type_id = ::WorkItems::Type.default_issue_type.id client.issues(repo).each do |issue| import_issue(issue, issue_type_id) diff --git a/lib/gitlab/bullet.rb b/lib/gitlab/bullet.rb index f5f8a316855..9759a82be0c 100644 --- a/lib/gitlab/bullet.rb +++ b/lib/gitlab/bullet.rb @@ -10,7 +10,7 @@ module Gitlab alias_method :extra_logging_enabled?, :enabled? def configure_bullet? - defined?(::Bullet) && (enabled? || Rails.env.development?) + defined?(::Bullet) && (enabled? || Gitlab.config.bullet.enabled) end end end diff --git a/lib/gitlab/changes_list.rb b/lib/gitlab/changes_list.rb index fb75a78a978..999d2ee4356 100644 --- a/lib/gitlab/changes_list.rb +++ b/lib/gitlab/changes_list.rb @@ -15,14 +15,12 @@ module Gitlab end def changes - @changes ||= begin - @raw_changes.map do |change| - next if change.blank? + @changes ||= @raw_changes.map do |change| + next if change.blank? - oldrev, newrev, ref = change.strip.split(' ') - { oldrev: oldrev, newrev: newrev, ref: ref } - end.compact - end + oldrev, newrev, ref = change.strip.split(' ') + { oldrev: oldrev, newrev: newrev, ref: ref } + end.compact end end end diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb index 19819ff7275..42a8b561d34 100644 --- a/lib/gitlab/ci/ansi2html.rb +++ b/lib/gitlab/ci/ansi2html.rb @@ -448,8 +448,8 @@ module Gitlab end def state - state = STATE_PARAMS.each_with_object({}) do |param, h| - h[param] = send(param) # rubocop:disable GitlabSecurity/PublicSend + state = STATE_PARAMS.index_with do |param| + send(param) # rubocop:disable GitlabSecurity/PublicSend end Base64.urlsafe_encode64(state.to_json) end diff --git a/lib/gitlab/ci/build/cache.rb b/lib/gitlab/ci/build/cache.rb index 375e6b4a96f..1cddc9fcc98 100644 --- a/lib/gitlab/ci/build/cache.rb +++ b/lib/gitlab/ci/build/cache.rb @@ -8,9 +8,9 @@ module Gitlab def initialize(cache, pipeline) cache = Array.wrap(cache) - @cache = cache.map do |cache| + @cache = cache.map.with_index do |cache, index| Gitlab::Ci::Pipeline::Seed::Build::Cache - .new(pipeline, cache) + .new(pipeline, cache, index) end end diff --git a/lib/gitlab/ci/build/context/build.rb b/lib/gitlab/ci/build/context/build.rb index a1a8e9288c7..1025e1cc2d7 100644 --- a/lib/gitlab/ci/build/context/build.rb +++ b/lib/gitlab/ci/build/context/build.rb @@ -9,25 +9,29 @@ module Gitlab attr_reader :attributes - def initialize(pipeline, attributes = {}) + def initialize(pipeline, attributes = {}, build = nil) super(pipeline) + @build = build @attributes = attributes end def variables - strong_memoize(:variables) do - # This is a temporary piece of technical debt to allow us access - # to the CI variables to evaluate rules before we persist a Build - # with the result. We should refactor away the extra Build.new, - # but be able to get CI Variables directly from the Seed::Build. - stub_build.scoped_variables - end + build.scoped_variables end + strong_memoize_attr :variables private + def build + @build || stub_build + end + def stub_build + # This is a temporary piece of technical debt to allow us access + # to the CI variables to evaluate rules before we persist a Build + # with the result. We should refactor away the extra Build.new, + # but be able to get CI Variables directly from the Seed::Build. ::Ci::Build.new(build_attributes) end diff --git a/lib/gitlab/ci/build/hook.rb b/lib/gitlab/ci/build/hook.rb new file mode 100644 index 00000000000..b731228678c --- /dev/null +++ b/lib/gitlab/ci/build/hook.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + class Hook + attr_reader :name, :script + + class << self + def from_hooks(job) + job.options[:hooks].to_a.map do |name, script| + new(name.to_s, script) + end + end + end + + def initialize(name, script) + @name = name + @script = script + end + end + end + end +end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index ee537f4efe5..142f0b8dfd8 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -10,7 +10,7 @@ module Gitlab ConfigError = Class.new(StandardError) TIMEOUT_SECONDS = 30.seconds - TIMEOUT_MESSAGE = 'Resolving config took longer than expected' + TIMEOUT_MESSAGE = 'Request timed out when fetching configuration files.' RESCUE_ERRORS = [ Gitlab::Config::Loader::FormatError, @@ -26,7 +26,7 @@ module Gitlab @source_ref_path = pipeline&.source_ref_path @project = project - @context = self.logger.instrument(:config_build_context) do + @context = self.logger.instrument(:config_build_context, once: true) do pipeline ||= ::Ci::Pipeline.new(project: project, sha: sha, user: user, source: source) build_context(project: project, pipeline: pipeline, sha: sha, user: user, parent_pipeline: parent_pipeline) end @@ -35,12 +35,16 @@ module Gitlab @source = source - @config = self.logger.instrument(:config_expand) do + @config = self.logger.instrument(:config_expand, once: true) do expand_config(config) end - @root = self.logger.instrument(:config_compose) do - Entry::Root.new(@config, project: project, user: user).tap(&:compose!) + @root = self.logger.instrument(:config_root, once: true) do + Entry::Root.new(@config, project: project, user: user, logger: self.logger) + end + + self.logger.instrument(:config_root_compose, once: true) do + @root.compose! end rescue *rescue_errors => e raise Config::ConfigError, e.message @@ -123,23 +127,23 @@ module Gitlab end def build_config(config) - initial_config = logger.instrument(:config_yaml_load) do + initial_config = logger.instrument(:config_yaml_load, once: true) do Config::Yaml.load!(config) end - initial_config = logger.instrument(:config_external_process) do + initial_config = logger.instrument(:config_external_process, once: true) do Config::External::Processor.new(initial_config, @context).perform end - initial_config = logger.instrument(:config_yaml_extend) do + initial_config = logger.instrument(:config_yaml_extend, once: true) do Config::Extendable.new(initial_config).to_hash end - initial_config = logger.instrument(:config_tags_resolve) do + initial_config = logger.instrument(:config_tags_resolve, once: true) do Config::Yaml::Tags::Resolver.new(initial_config).to_hash end - logger.instrument(:config_stages_inject) do + logger.instrument(:config_stages_inject, once: true) do Config::EdgeStagesInjector.new(initial_config).to_hash end end @@ -163,7 +167,7 @@ module Gitlab end def build_variables(pipeline:) - logger.instrument(:config_build_variables) do + logger.instrument(:config_build_variables, once: true) do pipeline .variables_builder .config_variables diff --git a/lib/gitlab/ci/config/entry/artifacts.rb b/lib/gitlab/ci/config/entry/artifacts.rb index 3b0cbc6b69e..27206d7e3a8 100644 --- a/lib/gitlab/ci/config/entry/artifacts.rb +++ b/lib/gitlab/ci/config/entry/artifacts.rb @@ -12,6 +12,7 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable + ALLOWED_WHEN = %w[on_success on_failure always].freeze ALLOWED_KEYS = %i[name untracked paths reports when expire_in expose_as exclude public].freeze EXPOSE_AS_REGEX = /\A\w[-\w ]*\z/.freeze EXPOSE_AS_ERROR_MESSAGE = "can contain only letters, digits, '-', '_' and spaces" @@ -38,10 +39,10 @@ module Gitlab validates :expose_as, format: { with: EXPOSE_AS_REGEX, message: EXPOSE_AS_ERROR_MESSAGE }, if: :expose_as_present? validates :exclude, array_of_strings: true validates :reports, type: Hash - validates :when, - inclusion: { in: %w[on_success on_failure always], - message: 'should be on_success, on_failure ' \ - 'or always' } + validates :when, type: String, inclusion: { + in: ALLOWED_WHEN, + message: "should be one of: #{ALLOWED_WHEN.join(', ')}" + } validates :expire_in, duration: { parser: ::Gitlab::Ci::Build::DurationParser } end end diff --git a/lib/gitlab/ci/config/entry/cache.rb b/lib/gitlab/ci/config/entry/cache.rb index ab79add688b..a5481071fc5 100644 --- a/lib/gitlab/ci/config/entry/cache.rb +++ b/lib/gitlab/ci/config/entry/cache.rb @@ -17,16 +17,16 @@ module Gitlab validations do validates :config, type: Hash, allowed_keys: ALLOWED_KEYS - validates :policy, - inclusion: { in: ALLOWED_POLICY, message: 'should be pull-push, push, or pull' }, - allow_blank: true + validates :policy, type: String, allow_blank: true, inclusion: { + in: ALLOWED_POLICY, + message: "should be one of: #{ALLOWED_POLICY.join(', ')}" + } with_options allow_nil: true do - validates :when, - inclusion: { - in: ALLOWED_WHEN, - message: 'should be on_success, on_failure or always' - } + validates :when, type: String, inclusion: { + in: ALLOWED_WHEN, + message: "should be one of: #{ALLOWED_WHEN.join(', ')}" + } end end diff --git a/lib/gitlab/ci/config/entry/default.rb b/lib/gitlab/ci/config/entry/default.rb index 12d68b755b3..e996b6b1312 100644 --- a/lib/gitlab/ci/config/entry/default.rb +++ b/lib/gitlab/ci/config/entry/default.rb @@ -13,9 +13,8 @@ module Gitlab include ::Gitlab::Config::Entry::Configurable include ::Gitlab::Config::Entry::Inheritable - ALLOWED_KEYS = %i[before_script image services - after_script cache interruptible - timeout retry tags artifacts].freeze + ALLOWED_KEYS = %i[before_script after_script hooks cache image services + interruptible timeout retry tags artifacts].freeze validations do validates :config, allowed_keys: ALLOWED_KEYS @@ -25,22 +24,27 @@ module Gitlab description: 'Script that will be executed before each job.', inherit: true - entry :image, Entry::Image, - description: 'Docker image that will be used to execute jobs.', - inherit: true - - entry :services, Entry::Services, - description: 'Docker images that will be linked to the container.', - inherit: true - entry :after_script, Entry::Commands, description: 'Script that will be executed after each job.', inherit: true + entry :hooks, Entry::Hooks, + description: 'Commands that will be executed on Runner before/after some events ' \ + 'such as `clone` and `build-script`.', + inherit: false + entry :cache, Entry::Caches, description: 'Configure caching between build jobs.', inherit: true + entry :image, Entry::Image, + description: 'Docker image that will be used to execute jobs.', + inherit: true + + entry :services, Entry::Services, + description: 'Docker images that will be linked to the container.', + inherit: true + entry :interruptible, ::Gitlab::Config::Entry::Boolean, description: 'Set jobs interruptible default value.', inherit: false diff --git a/lib/gitlab/ci/config/entry/hooks.rb b/lib/gitlab/ci/config/entry/hooks.rb new file mode 100644 index 00000000000..28bc2e4e7ce --- /dev/null +++ b/lib/gitlab/ci/config/entry/hooks.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + class Hooks < ::Gitlab::Config::Entry::Node + # `Configurable` alreadys adds `Validatable` + include ::Gitlab::Config::Entry::Configurable + + # NOTE: If a new hook is added, inheriting should be changed because a `job:hooks` overrides all + # `default:hooks` now. We should implement merging; each hook must be overridden individually. + ALLOWED_HOOKS = %i[pre_get_sources_script].freeze + + validations do + validates :config, type: Hash, allowed_keys: ALLOWED_HOOKS + end + + entry :pre_get_sources_script, Entry::Commands, + description: 'Commands that will be executed on Runner before cloning/fetching the Git repository.' + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/id_token.rb b/lib/gitlab/ci/config/entry/id_token.rb new file mode 100644 index 00000000000..12e0975d1b1 --- /dev/null +++ b/lib/gitlab/ci/config/entry/id_token.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents a JWT definition. + # + class IdToken < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Attributable + include ::Gitlab::Config::Entry::Validatable + + attributes %i[aud] + + validations do + validates :config, required_keys: %i[aud], allowed_keys: %i[aud] + validates :aud, array_of_strings_or_string: true + end + + def value + { aud: aud } + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 8e7f6ba4326..7c49b59a7f0 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -12,9 +12,9 @@ module Gitlab ALLOWED_WHEN = %w[on_success on_failure always manual delayed].freeze ALLOWED_KEYS = %i[tags script image services start_in artifacts - cache dependencies before_script after_script + cache dependencies before_script after_script hooks environment coverage retry parallel interruptible timeout - release].freeze + release id_tokens].freeze validations do validates :config, allowed_keys: Gitlab::Ci::Config::Entry::Job.allowed_keys + PROCESSABLE_ALLOWED_KEYS @@ -59,6 +59,10 @@ module Gitlab description: 'Commands that will be executed when finishing job.', inherit: true + entry :hooks, Entry::Hooks, + description: 'Commands that will be executed on Runner before/after some events; clone, build-script.', + inherit: true + entry :cache, Entry::Caches, description: 'Cache definition for this job.', inherit: true @@ -116,6 +120,11 @@ module Gitlab description: 'Indicates whether this job is allowed to fail or not.', inherit: false + entry :id_tokens, ::Gitlab::Config::Entry::ComposableHash, + description: 'Configured JWTs for this job', + inherit: false, + metadata: { composable_class: ::Gitlab::Ci::Config::Entry::IdToken } + attributes :script, :tags, :when, :dependencies, :needs, :retry, :parallel, :start_in, :interruptible, :timeout, @@ -155,10 +164,12 @@ module Gitlab artifacts: artifacts_value, release: release_value, after_script: after_script_value, + hooks: hooks_pre_get_sources_script_enabled? ? hooks_value : nil, ignore: ignored?, allow_failure_criteria: allow_failure_criteria, needs: needs_defined? ? needs_value : nil, - scheduling_type: needs_defined? ? :dag : :stage + scheduling_type: needs_defined? ? :dag : :stage, + id_tokens: id_tokens_value ).compact end @@ -183,6 +194,10 @@ module Gitlab allow_failure_value end + + def hooks_pre_get_sources_script_enabled? + YamlProcessor::FeatureFlags.enabled?(:ci_hooks_pre_get_sources_script) + end end end end diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb index f77876cc926..16844fa88db 100644 --- a/lib/gitlab/ci/config/entry/reports.rb +++ b/lib/gitlab/ci/config/entry/reports.rb @@ -16,8 +16,8 @@ module Gitlab %i[junit codequality sast secret_detection dependency_scanning container_scanning dast performance browser_performance load_performance license_scanning metrics lsif dotenv terraform accessibility - requirements coverage_fuzzing api_fuzzing cluster_image_scanning - coverage_report cyclonedx].freeze + coverage_fuzzing api_fuzzing cluster_image_scanning + requirements requirements_v2 coverage_report cyclonedx].freeze attributes ALLOWED_KEYS @@ -48,6 +48,7 @@ module Gitlab validates :terraform, array_of_strings_or_string: true validates :accessibility, array_of_strings_or_string: true validates :requirements, array_of_strings_or_string: true + validates :requirements_v2, array_of_strings_or_string: true validates :cyclonedx, array_of_strings_or_string: true end end diff --git a/lib/gitlab/ci/config/entry/root.rb b/lib/gitlab/ci/config/entry/root.rb index a30e6a0d9c3..a3d57ab6ac6 100644 --- a/lib/gitlab/ci/config/entry/root.rb +++ b/lib/gitlab/ci/config/entry/root.rb @@ -50,7 +50,7 @@ module Gitlab entry :variables, Entry::Variables, description: 'Environment variables that will be used.', - metadata: { allowed_value_data: %i[value description expand], allow_array_value: true }, + metadata: { allowed_value_data: %i[value description expand options] }, reserved: true entry :stages, Entry::Stages, @@ -103,12 +103,16 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def compose_jobs! - factory = ::Gitlab::Config::Entry::Factory.new(Entry::Jobs) - .value(jobs_config) - .with(key: :jobs, parent: self, - description: 'Jobs definition for this pipeline') + factory = logger.instrument(:config_root_compose_jobs_factory, once: true) do + ::Gitlab::Config::Entry::Factory.new(Entry::Jobs) + .value(jobs_config) + .with(key: :jobs, parent: self, + description: 'Jobs definition for this pipeline') + end - @entries[:jobs] = factory.create! + @entries[:jobs] = logger.instrument(:config_root_compose_jobs_create, once: true) do + factory.create! + end end # rubocop: enable CodeReuse/ActiveRecord @@ -123,6 +127,10 @@ module Gitlab @config = @config.except(*@jobs_config.keys) end + + def logger + metadata[:logger] + end end end end diff --git a/lib/gitlab/ci/config/entry/trigger.rb b/lib/gitlab/ci/config/entry/trigger.rb index 0f94b3f94fe..4c254a4fa07 100644 --- a/lib/gitlab/ci/config/entry/trigger.rb +++ b/lib/gitlab/ci/config/entry/trigger.rb @@ -41,7 +41,7 @@ module Gitlab validations do validates :config, presence: true validates :config, allowed_keys: ALLOWED_KEYS - validates :project, presence: true + validates :project, type: String, presence: true validates :branch, type: String, allow_nil: true validates :strategy, type: String, inclusion: { in: %w[depend], message: 'should be depend' }, allow_nil: true end diff --git a/lib/gitlab/ci/config/entry/variable.rb b/lib/gitlab/ci/config/entry/variable.rb index 16091758916..decb568ffc9 100644 --- a/lib/gitlab/ci/config/entry/variable.rb +++ b/lib/gitlab/ci/config/entry/variable.rb @@ -10,7 +10,6 @@ module Gitlab class Variable < ::Gitlab::Config::Entry::Simplifiable strategy :SimpleVariable, if: -> (config) { SimpleVariable.applies_to?(config) } strategy :ComplexVariable, if: -> (config) { ComplexVariable.applies_to?(config) } - strategy :ComplexArrayVariable, if: -> (config) { ComplexArrayVariable.applies_to?(config) } class SimpleVariable < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable @@ -41,20 +40,24 @@ module Gitlab class ComplexVariable < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable class << self def applies_to?(config) - config.is_a?(Hash) && !config[:value].is_a?(Array) + config.is_a?(Hash) end end + attributes :value, :description, :expand, :options, prefix: :config + validations do validates :key, alphanumeric: true - validates :config_value, alphanumeric: true, allow_nil: false, if: :config_value_defined? - validates :config_description, alphanumeric: true, allow_nil: false, if: :config_description_defined? - validates :config_expand, boolean: true, - allow_nil: false, - if: -> { ci_raw_variables_in_yaml_config_enabled? && config_expand_defined? } + validates :config_value, alphanumeric: true, allow_nil: true + validates :config_description, alphanumeric: true, allow_nil: true + validates :config_expand, boolean: true, allow_nil: true, if: -> { + ci_raw_variables_in_yaml_config_enabled? + } + validates :config_options, array_of_strings: true, allow_nil: true validate do allowed_value_data = Array(opt(:allowed_value_data)) @@ -66,91 +69,43 @@ module Gitlab else errors.add(:config, "must be a string") end + + if config_options.present? && config_options.exclude?(config_value) + errors.add(:config, 'value must be present in options') + end end end def value + # Needed since the `Entry::Node` provides `value` (which is current hash) config_value.to_s end def value_with_data if ci_raw_variables_in_yaml_config_enabled? { - value: value, - raw: (!config_expand if config_expand_defined?) + value: config_value.to_s, + raw: (!config_expand if has_config_expand?) }.compact else { - value: value + value: config_value.to_s }.compact end end def value_with_prefill_data value_with_data.merge( - description: config_description + description: config_description, + options: config_options ).compact end - def config_value - @config[:value] - end - - def config_description - @config[:description] - end - - def config_expand - @config[:expand] - end - - def config_value_defined? - config.key?(:value) - end - - def config_description_defined? - config.key?(:description) - end - - def config_expand_defined? - config.key?(:expand) - end - def ci_raw_variables_in_yaml_config_enabled? YamlProcessor::FeatureFlags.enabled?(:ci_raw_variables_in_yaml_config) end end - class ComplexArrayVariable < ComplexVariable - include ::Gitlab::Config::Entry::Validatable - - class << self - def applies_to?(config) - config.is_a?(Hash) && config[:value].is_a?(Array) - end - end - - validations do - validates :config_value, array_of_strings: true, allow_nil: false, if: :config_value_defined? - - validate do - next if opt(:allow_array_value) - - errors.add(:config, 'value must be an alphanumeric string') - end - end - - def value - config_value.first - end - - def value_with_prefill_data - super.merge( - value_options: config_value - ).compact - end - end - class UnknownStrategy < ::Gitlab::Config::Entry::Node def errors ["variable definition must be either a string or a hash"] diff --git a/lib/gitlab/ci/config/entry/variables.rb b/lib/gitlab/ci/config/entry/variables.rb index ef4f74b9f56..e338bce3109 100644 --- a/lib/gitlab/ci/config/entry/variables.rb +++ b/lib/gitlab/ci/config/entry/variables.rb @@ -42,7 +42,7 @@ module Gitlab end def composable_metadata - { allowed_value_data: opt(:allowed_value_data), allow_array_value: opt(:allow_array_value) } + { allowed_value_data: opt(:allowed_value_data) } end end end diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb index 57ff606c9ee..65caf4ac47d 100644 --- a/lib/gitlab/ci/config/external/file/base.rb +++ b/lib/gitlab/ci/config/external/file/base.rb @@ -47,7 +47,7 @@ module Gitlab end def validate! - validate_execution_time! + context.check_execution_time! if ::Feature.disabled?(:ci_refactoring_external_mapper, context.project) validate_location! validate_context! if valid? fetch_and_validate_content! if valid? @@ -87,10 +87,6 @@ module Gitlab nil end - def validate_execution_time! - context.check_execution_time! - end - def validate_location! if invalid_location_type? errors.push("Included file `#{masked_location}` needs to be a string") diff --git a/lib/gitlab/ci/config/external/file/remote.rb b/lib/gitlab/ci/config/external/file/remote.rb index b0c540685d4..ed37357dc53 100644 --- a/lib/gitlab/ci/config/external/file/remote.rb +++ b/lib/gitlab/ci/config/external/file/remote.rb @@ -53,7 +53,7 @@ module Gitlab errors.push("Remote file `#{masked_location}` could not be fetched because of a timeout error!") rescue Gitlab::HTTP::Error errors.push("Remote file `#{masked_location}` could not be fetched because of HTTP error!") - rescue Gitlab::HTTP::BlockedUrlError => e + rescue Errno::ECONNREFUSED, Gitlab::HTTP::BlockedUrlError => e errors.push("Remote file could not be fetched because #{e}!") end diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index fc03ac125fd..a41bc2b39f2 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -7,6 +7,7 @@ module Gitlab class Mapper include Gitlab::Utils::StrongMemoize + # Will be removed with FF ci_refactoring_external_mapper FILE_CLASSES = [ External::File::Local, External::File::Project, @@ -15,6 +16,7 @@ module Gitlab External::File::Artifact ].freeze + # Will be removed with FF ci_refactoring_external_mapper FILE_SUBKEYS = FILE_CLASSES.map { |f| f.name.demodulize.downcase }.freeze Error = Class.new(StandardError) @@ -22,27 +24,43 @@ module Gitlab TooManyIncludesError = Class.new(Error) def initialize(values, context) - @locations = Array.wrap(values.fetch(:include, [])) + @locations = Array.wrap(values.fetch(:include, [])).compact @context = context end def process - return [] if locations.empty? + return [] if @locations.empty? - logger.instrument(:config_mapper_process) do - process_without_instrumentation + context.logger.instrument(:config_mapper_process) do + if ::Feature.enabled?(:ci_refactoring_external_mapper, context.project) + process_without_instrumentation + else + legacy_process_without_instrumentation + end end end private - attr_reader :locations, :context + attr_reader :context delegate :expandset, :logger, to: :context def process_without_instrumentation - locations - .compact + locations = Normalizer.new(context).process(@locations) + locations = Filter.new(context).process(locations) + locations = LocationExpander.new(context).process(locations) + locations = VariablesExpander.new(context).process(locations) + + files = Matcher.new(context).process(locations) + Verifier.new(context).process(files) + + files + end + + # This and the following methods will be removed with FF ci_refactoring_external_mapper + def legacy_process_without_instrumentation + @locations .map(&method(:normalize_location)) .filter_map(&method(:verify_rules)) .flat_map(&method(:expand_project_files)) @@ -52,14 +70,8 @@ module Gitlab .each(&method(:verify!)) end - def normalize_location(location) - logger.instrument(:config_mapper_normalize) do - normalize_location_without_instrumentation(location) - end - end - # convert location if String to canonical form - def normalize_location_without_instrumentation(location) + def normalize_location(location) if location.is_a?(String) expanded_location = expand_variables(location) normalize_location_string(expanded_location) diff --git a/lib/gitlab/ci/config/external/mapper/base.rb b/lib/gitlab/ci/config/external/mapper/base.rb new file mode 100644 index 00000000000..d2f56d0b8f6 --- /dev/null +++ b/lib/gitlab/ci/config/external/mapper/base.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + class Mapper + # Base class for mapper classes + class Base + def initialize(context) + @context = context + end + + def process(*args) + context.logger.instrument(mapper_instrumentation_key) do + process_without_instrumentation(*args) + end + end + + private + + attr_reader :context + + def process_without_instrumentation + raise NotImplementedError + end + + def mapper_instrumentation_key + "config_mapper_#{self.class.name.demodulize.downcase}".to_sym + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/mapper/filter.rb b/lib/gitlab/ci/config/external/mapper/filter.rb new file mode 100644 index 00000000000..4d2b26c7d98 --- /dev/null +++ b/lib/gitlab/ci/config/external/mapper/filter.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + class Mapper + # Filters locations according to rules + class Filter < Base + private + + def process_without_instrumentation(locations) + locations.select do |location| + Rules.new(location[:rules]).evaluate(context).pass? + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/mapper/location_expander.rb b/lib/gitlab/ci/config/external/mapper/location_expander.rb new file mode 100644 index 00000000000..a4ca058f0d9 --- /dev/null +++ b/lib/gitlab/ci/config/external/mapper/location_expander.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + class Mapper + # Expands locations to include all files matching the pattern + class LocationExpander < Base + private + + def process_without_instrumentation(locations) + locations.flat_map do |location| + if location[:project] + expand_project_files(location) + elsif location[:local] + expand_wildcard_paths(location) + else + location + end + end + end + + def expand_project_files(location) + Array.wrap(location[:file]).map do |file| + location.merge(file: file) + end + end + + def expand_wildcard_paths(location) + return location unless location[:local].include?('*') + + context.project.repository.search_files_by_wildcard_path(location[:local], context.sha).map do |path| + { local: path } + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/mapper/matcher.rb b/lib/gitlab/ci/config/external/mapper/matcher.rb new file mode 100644 index 00000000000..85e19ff1ced --- /dev/null +++ b/lib/gitlab/ci/config/external/mapper/matcher.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + class Mapper + # Matches the first file type that matches the given location + class Matcher < Base + FILE_CLASSES = [ + External::File::Local, + External::File::Project, + External::File::Remote, + External::File::Template, + External::File::Artifact + ].freeze + + FILE_SUBKEYS = FILE_CLASSES.map { |f| f.name.demodulize.downcase }.freeze + + private + + def process_without_instrumentation(locations) + locations.map do |location| + matching = FILE_CLASSES.map do |file_class| + file_class.new(location, context) + end.select(&:matching?) + + if matching.one? + matching.first + elsif matching.empty? + raise Mapper::AmbigiousSpecificationError, + "`#{masked_location(location.to_json)}` does not have a valid subkey for include. " \ + "Valid subkeys are: `#{FILE_SUBKEYS.join('`, `')}`" + else + raise Mapper::AmbigiousSpecificationError, + "Each include must use only one of: `#{FILE_SUBKEYS.join('`, `')}`" + end + end + end + + def masked_location(location) + context.mask_variables_from(location) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/mapper/normalizer.rb b/lib/gitlab/ci/config/external/mapper/normalizer.rb new file mode 100644 index 00000000000..8fc798e78a0 --- /dev/null +++ b/lib/gitlab/ci/config/external/mapper/normalizer.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + class Mapper + # Converts locations to canonical form (local:/remote:) if String + class Normalizer < Base + def initialize(context) + super + + @variables_expander = VariablesExpander.new(context) + end + + private + + attr_reader :variables_expander + + def process_without_instrumentation(locations) + locations.map do |location| + if location.is_a?(String) + # We need to expand before normalizing because the information of + # whether if it's a remote or local path may be hidden inside the variable. + location = variables_expander.expand(location) + + normalize_location_string(location) + else + location.deep_symbolize_keys + end + end + end + + def normalize_location_string(location) + if ::Gitlab::UrlSanitizer.valid?(location) + { remote: location } + else + { local: location } + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/mapper/variables_expander.rb b/lib/gitlab/ci/config/external/mapper/variables_expander.rb new file mode 100644 index 00000000000..fddf04984d8 --- /dev/null +++ b/lib/gitlab/ci/config/external/mapper/variables_expander.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + class Mapper + # Handles variable expansion + class VariablesExpander < Base + def expand(data) + if data.is_a?(String) + expand_variable(data) + else + transform_and_expand_variable(data) + end + end + + private + + def process_without_instrumentation(locations) + locations.map { |location| expand(location) } + end + + def transform_and_expand_variable(data) + data.transform_values do |values| + case values + when Array + values.map { |value| expand_variable(value.to_s) } + when String + expand_variable(values) + else + values + end + end + end + + def expand_variable(data) + ExpandVariables.expand(data, -> { variables }) + end + + def variables + @variables ||= context.variables_hash + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/mapper/verifier.rb b/lib/gitlab/ci/config/external/mapper/verifier.rb new file mode 100644 index 00000000000..6d6f227b940 --- /dev/null +++ b/lib/gitlab/ci/config/external/mapper/verifier.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + class Mapper + # Fetches file contents and verifies them + class Verifier < Base + private + + def process_without_instrumentation(files) + files.select do |file| + verify_max_includes! + verify_execution_time! + + file.validate! + + context.expandset.add(file) + end + end + + def verify_max_includes! + return if context.expandset.count < context.max_includes + + raise Mapper::TooManyIncludesError, "Maximum of #{context.max_includes} nested includes are allowed!" + end + + def verify_execution_time! + context.check_execution_time! + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/processor.rb b/lib/gitlab/ci/config/external/processor.rb index 6a4aee26d80..e15b51fbff4 100644 --- a/lib/gitlab/ci/config/external/processor.rb +++ b/lib/gitlab/ci/config/external/processor.rb @@ -32,9 +32,7 @@ module Gitlab def validate_external_files! @external_files.each do |file| - logger.instrument(:config_external_verify) do - raise IncludeError, file.error_message unless file.valid? - end + raise IncludeError, file.error_message unless file.valid? end end diff --git a/lib/gitlab/ci/environment_matcher.rb b/lib/gitlab/ci/environment_matcher.rb new file mode 100644 index 00000000000..7d7a7742f68 --- /dev/null +++ b/lib/gitlab/ci/environment_matcher.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class EnvironmentMatcher + def initialize(pattern) + @pattern = pattern + end + + def match?(environment) + return false if pattern.blank? + + exact_match?(environment) || wildcard_match?(environment) + end + + private + + attr_reader :pattern, :match_type + + def exact_match?(environment) + pattern == environment + end + + def wildcard_match?(environment) + return false unless wildcard? + + wildcard_regex.match?(environment) + end + + def wildcard? + pattern.include?('*') + end + + def wildcard_regex + @wildcard_regex ||= Regexp.new(pattern.gsub('*', '.*')) + end + end + end +end diff --git a/lib/gitlab/ci/lint.rb b/lib/gitlab/ci/lint.rb index 51743a1f273..e0112a1b1c2 100644 --- a/lib/gitlab/ci/lint.rb +++ b/lib/gitlab/ci/lint.rb @@ -73,7 +73,7 @@ module Gitlab end def yaml_processor_result(content, logger) - logger.instrument(:yaml_process) do + logger.instrument(:yaml_process, once: true) do Gitlab::Ci::YamlProcessor.new(content, project: @project, user: @current_user, sha: @sha, @@ -119,7 +119,7 @@ module Gitlab environment: job[:environment], when: job[:when], allow_failure: job[:allow_failure], - needs: job.dig(:needs_attributes) + needs: job[:needs_attributes] } end end @@ -130,10 +130,10 @@ module Gitlab def build_logger Gitlab::Ci::Pipeline::Logger.new(project: @project) do |l| l.log_when do |observations| - values = observations['yaml_process_duration_s'] - next false if values.empty? + duration = observations['yaml_process_duration_s'] + next false unless duration - values.max >= LOG_MAX_DURATION_THRESHOLD + duration >= LOG_MAX_DURATION_THRESHOLD end end end diff --git a/lib/gitlab/ci/parsers/security/common.rb b/lib/gitlab/ci/parsers/security/common.rb index 0ac012b9fd1..67817c9f832 100644 --- a/lib/gitlab/ci/parsers/security/common.rb +++ b/lib/gitlab/ci/parsers/security/common.rb @@ -65,8 +65,14 @@ module Gitlab ) end + # New Oj parsers are not thread safe, therefore, + # we need to initialize them for each thread. + def introspect_parser + Thread.current[:introspect_parser] ||= Oj::Introspect.new(filter: "remediations") + end + def report_data - @report_data ||= Gitlab::Json.parse!(json_data) + @report_data ||= introspect_parser.parse(json_data) end def report_version diff --git a/lib/gitlab/ci/pipeline/chain/build/associations.rb b/lib/gitlab/ci/pipeline/chain/build/associations.rb index b5d63691849..b484a88a381 100644 --- a/lib/gitlab/ci/pipeline/chain/build/associations.rb +++ b/lib/gitlab/ci/pipeline/chain/build/associations.rb @@ -31,7 +31,8 @@ module Gitlab source_pipeline: @command.bridge.pipeline, source_project: @command.bridge.project, source_bridge: @command.bridge, - project: @command.project + project: @command.project, + source_partition_id: @command.bridge.partition_id ) end diff --git a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb index 07a3aff1862..53c8a7ac122 100644 --- a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb +++ b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb @@ -11,11 +11,10 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def perform! - ff_enabled = Feature.enabled?(:ci_skip_auto_cancelation_on_child_pipelines, project) - return if ff_enabled && pipeline.parent_pipeline? # skip if child pipeline + return if pipeline.parent_pipeline? # skip if child pipeline return unless project.auto_cancel_pending_pipelines? - Gitlab::OptimisticLocking.retry_lock(auto_cancelable_pipelines(ff_enabled), name: 'cancel_pending_pipelines') do |cancelables| + Gitlab::OptimisticLocking.retry_lock(auto_cancelable_pipelines, name: 'cancel_pending_pipelines') do |cancelables| cancelables.select(:id).each_batch(of: BATCH_SIZE) do |cancelables_batch| auto_cancel_interruptible_pipelines(cancelables_batch.ids) end @@ -29,19 +28,14 @@ module Gitlab private - def auto_cancelable_pipelines(ff_enabled) - relation = project.all_pipelines + def auto_cancelable_pipelines + project.all_pipelines .created_after(1.week.ago) .ci_and_parent_sources .for_ref(pipeline.ref) .where_not_sha(project.commit(pipeline.ref).try(:id)) .alive_or_scheduled - - if ff_enabled - relation.id_not_in(pipeline.id) - else - relation.id_not_in(pipeline.same_family_pipeline_ids) - end + .id_not_in(pipeline.id) end def auto_cancel_interruptible_pipelines(pipeline_ids) diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 5ec04b4889e..31b130b5ab7 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -98,7 +98,7 @@ module Gitlab def observe_step_duration(step_class, duration) step = step_class.name.underscore.parameterize(separator: '_') - logger.observe("pipeline_step_#{step}_duration_s", duration) + logger.observe("pipeline_step_#{step}_duration_s", duration, once: true) if Feature.enabled?(:ci_pipeline_creation_step_duration_tracking, type: :ops) metrics.pipeline_creation_step_duration_histogram @@ -107,14 +107,14 @@ module Gitlab end def observe_creation_duration(duration) - logger.observe(:pipeline_creation_duration_s, duration) + logger.observe(:pipeline_creation_duration_s, duration, once: true) metrics.pipeline_creation_duration_histogram - .observe({}, duration.seconds) + .observe({ gitlab: gitlab_org_project?.to_s }, duration.seconds) end def observe_pipeline_size(pipeline) - logger.observe(:pipeline_size_count, pipeline.total_size) + logger.observe(:pipeline_size_count, pipeline.total_size, once: true) metrics.pipeline_size_histogram .observe({ source: pipeline.source.to_s, plan: project.actual_plan_name }, pipeline.total_size) @@ -157,6 +157,10 @@ module Gitlab def full_git_ref_name_unavailable? ref == origin_ref end + + def gitlab_org_project? + project.full_path == 'gitlab-org/gitlab' + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb index 5548fca320f..ad6b2fd3411 100644 --- a/lib/gitlab/ci/pipeline/chain/config/process.rb +++ b/lib/gitlab/ci/pipeline/chain/config/process.rb @@ -11,7 +11,7 @@ module Gitlab def perform! raise ArgumentError, 'missing config content' unless @command.config_content - result = logger.instrument(:pipeline_config_process) do + result = logger.instrument(:pipeline_config_process, once: true) do processor = ::Gitlab::Ci::YamlProcessor.new( @command.config_content, { project: project, diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index 207b4b5ff8b..d4c4f94c7d3 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -9,7 +9,7 @@ module Gitlab include Gitlab::Utils::StrongMemoize def perform! - logger.instrument_with_sql(:pipeline_save) do + logger.instrument_once_with_sql(:pipeline_save) do BulkInsertableAssociations.with_bulk_insert do ::Ci::BulkInsertableTags.with_bulk_insert_tags do pipeline.transaction do diff --git a/lib/gitlab/ci/pipeline/chain/ensure_environments.rb b/lib/gitlab/ci/pipeline/chain/ensure_environments.rb index 1b9dd158733..ebea6a538ef 100644 --- a/lib/gitlab/ci/pipeline/chain/ensure_environments.rb +++ b/lib/gitlab/ci/pipeline/chain/ensure_environments.rb @@ -16,7 +16,7 @@ module Gitlab private def ensure_environment(build) - ::Environments::CreateForBuildService.new.execute(build, merge_request: @command.merge_request) + ::Environments::CreateForBuildService.new.execute(build) end end end diff --git a/lib/gitlab/ci/pipeline/chain/seed.rb b/lib/gitlab/ci/pipeline/chain/seed.rb index feae123f216..ae98c55e425 100644 --- a/lib/gitlab/ci/pipeline/chain/seed.rb +++ b/lib/gitlab/ci/pipeline/chain/seed.rb @@ -13,7 +13,7 @@ module Gitlab raise ArgumentError, 'missing workflow rules result' unless @command.workflow_rules_result # Allocate next IID. This operation must be outside of transactions of pipeline creations. - logger.instrument(:pipeline_allocate_seed_attributes) do + logger.instrument(:pipeline_allocate_seed_attributes, once: true) do pipeline.ensure_project_iid! pipeline.ensure_ci_ref! end @@ -25,7 +25,7 @@ module Gitlab ## # Gather all runtime build/stage errors # - seed_errors = logger.instrument(:pipeline_seed_evaluation) do + seed_errors = logger.instrument(:pipeline_seed_evaluation, once: true) do pipeline_seed.errors end @@ -44,7 +44,7 @@ module Gitlab def pipeline_seed strong_memoize(:pipeline_seed) do - logger.instrument(:pipeline_seed_initialization) do + logger.instrument(:pipeline_seed_initialization, once: true) do stages_attributes = @command.yaml_processor_result.stages_attributes Gitlab::Ci::Pipeline::Seed::Pipeline.new(context, stages_attributes) @@ -61,7 +61,7 @@ module Gitlab end def root_variables - logger.instrument(:pipeline_seed_merge_variables) do + strong_memoize(:root_variables) do ::Gitlab::Ci::Variables::Helpers.merge_variables( @command.yaml_processor_result.root_variables, @command.workflow_rules_result.variables diff --git a/lib/gitlab/ci/pipeline/logger.rb b/lib/gitlab/ci/pipeline/logger.rb index 4b7cbae5004..f393406b549 100644 --- a/lib/gitlab/ci/pipeline/logger.rb +++ b/lib/gitlab/ci/pipeline/logger.rb @@ -23,7 +23,7 @@ module Gitlab log_conditions.push(block) end - def instrument(operation) + def instrument(operation, once: false) return yield unless enabled? raise ArgumentError, 'block not given' unless block_given? @@ -32,63 +32,72 @@ module Gitlab result = yield - observe("#{operation}_duration_s", current_monotonic_time - op_started_at) + observe("#{operation}_duration_s", current_monotonic_time - op_started_at, once: once) result end - def instrument_with_sql(operation, &block) + def instrument_once_with_sql(operation, &block) op_start_db_counters = current_db_counter_payload - result = instrument(operation, &block) + result = instrument(operation, once: true, &block) - observe_sql_counters(operation, op_start_db_counters, current_db_counter_payload) + observe_sql_counters(operation, op_start_db_counters, current_db_counter_payload, once: true) result end - def observe(operation, value) + def observe(operation, value, once: false) return unless enabled? - observations[operation.to_s].push(value) + if once + observations[operation.to_s] = value + else + observations[operation.to_s] ||= [] + observations[operation.to_s].push(value) + end end def commit(pipeline:, caller:) return unless log? - attributes = { - class: self.class.name.to_s, - pipeline_creation_caller: caller, - project_id: project&.id, # project is not available when called from `/ci/lint` - pipeline_persisted: pipeline.persisted?, - pipeline_source: pipeline.source, - pipeline_creation_service_duration_s: age - } - - if pipeline.persisted? - attributes[:pipeline_builds_tags_count] = pipeline.tags_count - attributes[:pipeline_builds_distinct_tags_count] = pipeline.distinct_tags_count - attributes[:pipeline_id] = pipeline.id + Gitlab::ApplicationContext.with_context(project: project) do + attributes = Gitlab::ApplicationContext.current.merge( + class: self.class.name.to_s, + pipeline_creation_caller: caller, + project_id: project&.id, # project is not available when called from `/ci/lint` + pipeline_persisted: pipeline.persisted?, + pipeline_source: pipeline.source, + pipeline_creation_service_duration_s: age + ) + + if pipeline.persisted? + attributes[:pipeline_builds_tags_count] = pipeline.tags_count + attributes[:pipeline_builds_distinct_tags_count] = pipeline.distinct_tags_count + attributes[:pipeline_id] = pipeline.id + end + + attributes.compact! + attributes.stringify_keys! + attributes.merge!(observations_hash) + + destination.info(attributes) end - - attributes.compact! - attributes.stringify_keys! - attributes.merge!(observations_hash) - - destination.info(attributes) end def observations_hash - observations.transform_values do |values| - next if values.empty? - - { - 'count' => values.size, - 'min' => values.min, - 'max' => values.max, - 'sum' => values.sum, - 'avg' => values.sum / values.size - } + observations.transform_values do |observation| + next if observation.blank? + + if observation.is_a?(Array) + { + 'count' => observation.size, + 'max' => observation.max, + 'sum' => observation.sum + } + else + observation + end end.compact end @@ -110,21 +119,20 @@ module Gitlab end def enabled? - strong_memoize(:enabled) do - ::Feature.enabled?(:ci_pipeline_creation_logger, project, type: :ops) - end + ::Feature.enabled?(:ci_pipeline_creation_logger, project, type: :ops) end + strong_memoize_attr :enabled?, :enabled def observations - @observations ||= Hash.new { |hash, key| hash[key] = [] } + @observations ||= {} end - def observe_sql_counters(operation, start_db_counters, end_db_counters) + def observe_sql_counters(operation, start_db_counters, end_db_counters, once: false) end_db_counters.each do |key, value| result = value - start_db_counters.fetch(key, 0) next if result == 0 - observe("#{operation}_#{key}", result) + observe("#{operation}_#{key}", result, once: once) end end diff --git a/lib/gitlab/ci/pipeline/metrics.rb b/lib/gitlab/ci/pipeline/metrics.rb index c3e0f043b44..04565beeecc 100644 --- a/lib/gitlab/ci/pipeline/metrics.rb +++ b/lib/gitlab/ci/pipeline/metrics.rb @@ -9,7 +9,8 @@ module Gitlab def self.pipeline_creation_duration_histogram name = :gitlab_ci_pipeline_creation_duration_seconds comment = 'Pipeline creation duration' - labels = {} + # @gitlab: boolean value - if project is gitlab-org/gitlab + labels = { gitlab: false } buckets = [0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0, 20.0, 50.0, 240.0] ::Gitlab::Metrics.histogram(name, comment, labels, buckets) diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 2e4267e986b..b0b79b994c1 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -9,12 +9,13 @@ module Gitlab delegate :dig, to: :@seed_attributes - def initialize(context, attributes, stages_for_needs_lookup = []) + def initialize(context, attributes, stages_for_needs_lookup, stage) @context = context @pipeline = context.pipeline @seed_attributes = attributes @stages_for_needs_lookup = stages_for_needs_lookup.compact @needs_attributes = dig(:needs_attributes) + @stage = stage @resource_group_key = attributes.delete(:resource_group_key) @job_variables = @seed_attributes.delete(:job_variables) @root_variables_inheritance = @seed_attributes.delete(:root_variables_inheritance) { true } @@ -33,6 +34,8 @@ module Gitlab .new(attributes.delete(:cache), @pipeline) calculate_yaml_variables! + + @processable = initialize_processable end def name @@ -40,21 +43,20 @@ module Gitlab end def included? - strong_memoize(:inclusion) do - logger.instrument(:pipeline_seed_build_inclusion) do - if @using_rules - rules_result.pass? - elsif @using_only || @using_except - all_of_only? && none_of_except? - else - true - end + logger.instrument(:pipeline_seed_build_inclusion) do + if @using_rules + rules_result.pass? + elsif @using_only || @using_except + all_of_only? && none_of_except? + else + true end end end + strong_memoize_attr :included?, :inclusion def errors - strong_memoize(:errors) do + logger.instrument(:pipeline_seed_build_errors) do # We check rules errors before checking "included?" because rules affects its inclusion status. next rules_errors if rules_errors next unless included? @@ -62,14 +64,22 @@ module Gitlab [needs_errors, variable_expansion_errors].compact.flatten end end + strong_memoize_attr :errors + # TODO: Method used only in specs. Replace with `to_resource.attributes` when + # the feature flag ci_reuse_build_in_seed_context is removed. + # Then remove this method. def attributes - @seed_attributes - .deep_merge(pipeline_attributes) - .deep_merge(rules_attributes) - .deep_merge(allow_failure_criteria_attributes) - .deep_merge(@cache.cache_attributes) - .deep_merge(runner_tags) + if reuse_build_in_seed_context? + initial_attributes.deep_merge(evaluated_attributes) + else + @seed_attributes + .deep_merge(pipeline_attributes) + .deep_merge(rules_attributes) + .deep_merge(allow_failure_criteria_attributes) + .deep_merge(@cache.cache_attributes) + .deep_merge(runner_tags) + end end def bridge? @@ -80,12 +90,30 @@ module Gitlab end def to_resource - strong_memoize(:resource) do - initialize_processable + logger.instrument(:pipeline_seed_build_to_resource) do + if reuse_build_in_seed_context? + # The `options` attribute need to be entirely reassigned because they may + # be overridden by evaluated_attributes. + # We also don't want to reassign all the `initial_attributes` since those + # can affect performance. We only want to assign what's changed. + assignable_attributes = initial_attributes.slice(:options) + .deep_merge(evaluated_attributes) + processable.assign_attributes(assignable_attributes) + processable + else + legacy_initialize_processable + end end end + strong_memoize_attr :to_resource - def initialize_processable + private + + attr_reader :processable + + delegate :logger, to: :@context + + def legacy_initialize_processable if bridge? ::Ci::Bridge.new(attributes) else @@ -93,9 +121,28 @@ module Gitlab end end - private + def initialize_processable + return unless reuse_build_in_seed_context? - delegate :logger, to: :@context + if bridge? + ::Ci::Bridge.new(initial_attributes) + else + ::Ci::Build.new(initial_attributes) + end + end + + def initial_attributes + @seed_attributes + .deep_merge(pipeline_attributes) + .deep_merge(ci_stage: @stage) + .deep_merge(@cache.cache_attributes) + end + + def evaluated_attributes + rules_attributes + .deep_merge(allow_failure_criteria_attributes) + .deep_merge(runner_tags) + end def all_of_only? @only.all? { |spec| spec.satisfied_by?(@pipeline, evaluate_context) } @@ -155,40 +202,39 @@ module Gitlab end def rules_attributes - strong_memoize(:rules_attributes) do - next {} unless @using_rules + return {} unless @using_rules - rules_variables_result = ::Gitlab::Ci::Variables::Helpers.merge_variables( - @seed_attributes[:yaml_variables], rules_result.variables - ) + rules_variables_result = ::Gitlab::Ci::Variables::Helpers.merge_variables( + @seed_attributes[:yaml_variables], rules_result.variables + ) - rules_result.build_attributes.merge(yaml_variables: rules_variables_result) - end + rules_result.build_attributes.merge(yaml_variables: rules_variables_result) end + strong_memoize_attr :rules_attributes def rules_result - strong_memoize(:rules_result) do - @rules.evaluate(@pipeline, evaluate_context) - end + @rules.evaluate(@pipeline, evaluate_context) end + strong_memoize_attr :rules_result def rules_errors - strong_memoize(:rules_errors) do - ["Failed to parse rule for #{name}: #{rules_result.errors.join(', ')}"] if rules_result.errors.present? - end + ["Failed to parse rule for #{name}: #{rules_result.errors.join(', ')}"] if rules_result.errors.present? end + strong_memoize_attr :rules_errors def evaluate_context - strong_memoize(:evaluate_context) do + if reuse_build_in_seed_context? + Gitlab::Ci::Build::Context::Build.new(@pipeline, @seed_attributes, processable) + else Gitlab::Ci::Build::Context::Build.new(@pipeline, @seed_attributes) end end + strong_memoize_attr :evaluate_context def runner_tags - strong_memoize(:runner_tags) do - { tag_list: evaluate_runner_tags }.compact - end + { tag_list: evaluate_runner_tags }.compact end + strong_memoize_attr :runner_tags def evaluate_runner_tags @seed_attributes.delete(:tag_list)&.map do |tag| @@ -211,6 +257,11 @@ module Gitlab from: @context.root_variables, to: @job_variables, inheritance: @root_variables_inheritance ) end + + def reuse_build_in_seed_context? + Feature.enabled?(:ci_reuse_build_in_seed_context, @pipeline.project) + end + strong_memoize_attr :reuse_build_in_seed_context?, :reuse_build_in_seed_context end end end diff --git a/lib/gitlab/ci/pipeline/seed/build/cache.rb b/lib/gitlab/ci/pipeline/seed/build/cache.rb index 78ffaaa7e81..781065a63db 100644 --- a/lib/gitlab/ci/pipeline/seed/build/cache.rb +++ b/lib/gitlab/ci/pipeline/seed/build/cache.rb @@ -6,7 +6,7 @@ module Gitlab module Seed class Build class Cache - def initialize(pipeline, cache) + def initialize(pipeline, cache, custom_key_prefix) @pipeline = pipeline local_cache = cache.to_h.deep_dup @key = local_cache.delete(:key) @@ -14,6 +14,7 @@ module Gitlab @policy = local_cache.delete(:policy) @untracked = local_cache.delete(:untracked) @when = local_cache.delete(:when) + @custom_key_prefix = custom_key_prefix raise ArgumentError, "unknown cache keys: #{local_cache.keys}" if local_cache.any? end @@ -45,6 +46,7 @@ module Gitlab def key_from_files return unless @key.is_a?(Hash) + @key[:prefix] ||= @custom_key_prefix.to_s [@key[:prefix], files_digest].select(&:present?).join('-') end diff --git a/lib/gitlab/ci/pipeline/seed/pipeline.rb b/lib/gitlab/ci/pipeline/seed/pipeline.rb index 9e609debeed..57ad2546f1c 100644 --- a/lib/gitlab/ci/pipeline/seed/pipeline.rb +++ b/lib/gitlab/ci/pipeline/seed/pipeline.rb @@ -38,8 +38,10 @@ module Gitlab private + delegate :logger, to: :@context + def stage_seeds - strong_memoize(:stage_seeds) do + logger.instrument(:pipeline_seed_stage_seeds) do seeds = @stages_attributes.inject([]) do |previous_stages, attributes| seed = Gitlab::Ci::Pipeline::Seed::Stage.new(@context, attributes, previous_stages) previous_stages + [seed] @@ -48,6 +50,7 @@ module Gitlab seeds.select(&:included?) end end + strong_memoize_attr :stage_seeds end end end diff --git a/lib/gitlab/ci/pipeline/seed/stage.rb b/lib/gitlab/ci/pipeline/seed/stage.rb index 1c4247bd5ee..c3e94529634 100644 --- a/lib/gitlab/ci/pipeline/seed/stage.rb +++ b/lib/gitlab/ci/pipeline/seed/stage.rb @@ -10,54 +10,49 @@ module Gitlab delegate :size, to: :seeds delegate :dig, to: :seeds - def initialize(context, attributes, previous_stages) - @context = context - @pipeline = context.pipeline - @attributes = attributes - @previous_stages = previous_stages - - @builds = attributes.fetch(:builds).map do |attributes| - Seed::Build.new(context, attributes, previous_stages + [self]) + attr_reader :attributes + + def initialize(context, stage_attributes, previous_stages) + pipeline = context.pipeline + @attributes = { + name: stage_attributes.fetch(:name), + position: stage_attributes.fetch(:index), + pipeline: pipeline, + project: pipeline.project, + partition_id: pipeline.partition_id + } + + @stage = ::Ci::Stage.new(@attributes) + + @builds = stage_attributes.fetch(:builds).map do |build_attributes| + Seed::Build.new(context, build_attributes, previous_stages + [self], @stage) end end - def attributes - { name: @attributes.fetch(:name), - position: @attributes.fetch(:index), - pipeline: @pipeline, - project: @pipeline.project, - partition_id: @pipeline.partition_id } - end - def seeds - strong_memoize(:seeds) do - @builds.select(&:included?) - end + @builds.select(&:included?) end + strong_memoize_attr :seeds def errors - strong_memoize(:errors) do - @builds.flat_map(&:errors).compact - end + @builds.flat_map(&:errors).compact end + strong_memoize_attr :errors def seeds_names - strong_memoize(:seeds_names) do - seeds.map(&:name).to_set - end + seeds.map(&:name).to_set end + strong_memoize_attr :seeds_names def included? seeds.any? end def to_resource - strong_memoize(:stage) do - ::Ci::Stage.new(attributes).tap do |stage| - stage.statuses = seeds.map(&:to_resource) - end - end + @stage.statuses = seeds.map(&:to_resource) + @stage end + strong_memoize_attr :to_resource end end end diff --git a/lib/gitlab/ci/reports/security/finding.rb b/lib/gitlab/ci/reports/security/finding.rb index dd9b9cc6d55..92a91854358 100644 --- a/lib/gitlab/ci/reports/security/finding.rb +++ b/lib/gitlab/ci/reports/security/finding.rb @@ -83,8 +83,8 @@ module Gitlab message cve solution - ].each_with_object({}) do |key, hash| - hash[key] = public_send(key) # rubocop:disable GitlabSecurity/PublicSend + ].index_with do |key| + public_send(key) # rubocop:disable GitlabSecurity/PublicSend end end @@ -98,7 +98,7 @@ module Gitlab end def unsafe?(severity_levels, report_types) - severity.to_s.in?(severity_levels) && (report_types.blank? || report_type.to_s.in?(report_types) ) + severity.to_s.in?(severity_levels) && (report_types.blank? || report_type.to_s.in?(report_types)) end def eql?(other) diff --git a/lib/gitlab/ci/reports/security/finding_key.rb b/lib/gitlab/ci/reports/security/finding_key.rb index ad047fbf904..d42a0ea5b2e 100644 --- a/lib/gitlab/ci/reports/security/finding_key.rb +++ b/lib/gitlab/ci/reports/security/finding_key.rb @@ -15,7 +15,7 @@ module Gitlab has_fingerprints? && other.has_fingerprints? && location_fingerprint == other.location_fingerprint && - identifier_fingerprint == other.identifier_fingerprint + identifier_fingerprint == other.identifier_fingerprint end def hash diff --git a/lib/gitlab/ci/reports/security/identifier.rb b/lib/gitlab/ci/reports/security/identifier.rb index 4ba943cdcbc..0ff6be6acc4 100644 --- a/lib/gitlab/ci/reports/security/identifier.rb +++ b/lib/gitlab/ci/reports/security/identifier.rb @@ -31,8 +31,8 @@ module Gitlab fingerprint name url - ].each_with_object({}) do |key, hash| - hash[key] = public_send(key) # rubocop:disable GitlabSecurity/PublicSend + ].index_with do |key| + public_send(key) # rubocop:disable GitlabSecurity/PublicSend end end diff --git a/lib/gitlab/ci/reports/security/reports.rb b/lib/gitlab/ci/reports/security/reports.rb index 5c08381d5cc..8425881a4ab 100644 --- a/lib/gitlab/ci/reports/security/reports.rb +++ b/lib/gitlab/ci/reports/security/reports.rb @@ -21,29 +21,6 @@ module Gitlab def findings reports.values.flat_map(&:findings) end - - def violates_default_policy_against?(target_reports, vulnerabilities_allowed, severity_levels, vulnerability_states, report_types = []) - if Feature.enabled?(:require_approval_on_scan_removal, pipeline.project) && scan_removed?(target_reports) - return true - end - - unsafe_findings_count(target_reports, severity_levels, vulnerability_states, report_types) > vulnerabilities_allowed - end - - def unsafe_findings_uuids(severity_levels, report_types) - findings.select { |finding| finding.unsafe?(severity_levels, report_types) }.map(&:uuid) - end - - private - - def unsafe_findings_count(target_reports, severity_levels, vulnerability_states, report_types) - new_uuids = unsafe_findings_uuids(severity_levels, report_types) - target_reports&.unsafe_findings_uuids(severity_levels, report_types).to_a - new_uuids.count - end - - def scan_removed?(target_reports) - (target_reports&.reports&.keys.to_a - reports.keys).any? - end end end end diff --git a/lib/gitlab/ci/reports/test_suite.rb b/lib/gitlab/ci/reports/test_suite.rb index d0388c65f58..dcc593b4403 100644 --- a/lib/gitlab/ci/reports/test_suite.rb +++ b/lib/gitlab/ci/reports/test_suite.rb @@ -96,8 +96,8 @@ module Gitlab end def sort_by_execution_time_desc - @test_cases = @test_cases.keys.each_with_object({}) do |key, hash| - hash[key] = @test_cases[key].sort_by { |_key, test_case| -test_case.execution_time }.to_h + @test_cases = @test_cases.keys.index_with do |key| + @test_cases[key].sort_by { |_key, test_case| -test_case.execution_time }.to_h end end end diff --git a/lib/gitlab/ci/runner_instructions.rb b/lib/gitlab/ci/runner_instructions.rb index 68c911d3dbb..bcda2fec5ba 100644 --- a/lib/gitlab/ci/runner_instructions.rb +++ b/lib/gitlab/ci/runner_instructions.rb @@ -22,7 +22,8 @@ module Gitlab osx: { human_readable_name: "macOS", download_locations: { - amd64: "https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-darwin-amd64" + amd64: "https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-darwin-amd64", + arm64: "https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-darwin-arm64" }, install_script_template_path: "lib/gitlab/ci/runner_instructions/templates/osx/install.sh", runner_executable: "gitlab-runner" @@ -61,7 +62,7 @@ module Gitlab def install_script with_error_handling [Gitlab::Ci::RunnerInstructions::ArgumentError] do - raise Gitlab::Ci::RunnerInstructions::ArgumentError, s_('Architecture not found for OS') unless environment[:download_locations].key?(@arch.to_sym) + raise Gitlab::Ci::RunnerInstructions::ArgumentError, _('Architecture not found for OS') unless environment[:download_locations].key?(@arch.to_sym) replace_variables(get_file(environment[:install_script_template_path])) end @@ -69,7 +70,7 @@ module Gitlab def register_command with_error_handling [Gitlab::Ci::RunnerInstructions::ArgumentError, Gitlab::Access::AccessDeniedError] do - raise Gitlab::Ci::RunnerInstructions::ArgumentError, s_('No runner executable') unless environment[:runner_executable] + raise Gitlab::Ci::RunnerInstructions::ArgumentError, _('No runner executable') unless environment[:runner_executable] server_url = Gitlab::Routing.url_helpers.root_url(only_path: false) runner_executable = environment[:runner_executable] @@ -90,12 +91,12 @@ module Gitlab end def environment - @environment ||= OS[@os.to_sym] || ( raise Gitlab::Ci::RunnerInstructions::ArgumentError, s_('Invalid OS') ) + @environment ||= OS[@os.to_sym] || (raise Gitlab::Ci::RunnerInstructions::ArgumentError, _('Invalid OS')) end def validate_params - @errors << s_('Missing OS') unless @os.present? - @errors << s_('Missing arch') unless @arch.present? + @errors << _('Missing OS') unless @os.present? + @errors << _('Missing arch') unless @arch.present? end def replace_variables(expression) diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index fddcc1492a8..11420b05dfb 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -177,11 +177,11 @@ include: - template: Jobs/Browser-Performance-Testing.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml - template: Jobs/Helm-2to3.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Helm-2to3.gitlab-ci.yml - template: Security/DAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml - - template: Security/Container-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml - - template: Security/Dependency-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml - - template: Security/License-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml - - template: Security/SAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml - - template: Security/Secret-Detection.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml + - template: Jobs/Container-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml + - template: Jobs/Dependency-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml + - template: Jobs/License-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml + - template: Jobs/SAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml + - template: Jobs/Secret-Detection.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml # The latest build job generates a dotenv report artifact with a CI_APPLICATION_TAG # that also includes the image digest. This configures Auto Deploy to receive diff --git a/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml b/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml index 671925c5df6..16ce85548df 100644 --- a/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml @@ -11,13 +11,6 @@ image: gradle:alpine -# Disable the Gradle daemon for Continuous Integration servers as correctness -# is usually a priority over speed in CI environments. Using a fresh -# runtime for each build is more reliable since the runtime is completely -# isolated from any previous builds. -variables: - GRADLE_OPTS: "-Dorg.gradle.daemon=false" - before_script: - GRADLE_USER_HOME="$(pwd)/.gradle" - export GRADLE_USER_HOME diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml index fcf2ac7de7a..026ddf4a17a 100644 --- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml @@ -7,7 +7,7 @@ browser_performance: variables: DOCKER_TLS_CERTDIR: "" SITESPEED_IMAGE: sitespeedio/sitespeed.io - SITESPEED_VERSION: 14.1.0 + SITESPEED_VERSION: 26.1.0 SITESPEED_OPTIONS: '' services: - name: 'docker:20.10.12-dind' diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml index 04b7dacf2dd..218c2f79e6a 100644 --- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml @@ -7,7 +7,7 @@ browser_performance: variables: DOCKER_TLS_CERTDIR: "" SITESPEED_IMAGE: sitespeedio/sitespeed.io - SITESPEED_VERSION: 14.1.0 + SITESPEED_VERSION: latest SITESPEED_OPTIONS: '' services: - name: 'docker:20.10.12-dind' diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 23efed212f8..b4beeb60dfd 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -8,7 +8,8 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/gitlab-org/ci-cd/codequality:0.87.0" + CODE_QUALITY_IMAGE_TAG: "0.87.3" + CODE_QUALITY_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/gitlab-org/ci-cd/codequality:$CODE_QUALITY_IMAGE_TAG" needs: [] script: - export SOURCE_CODE=$PWD diff --git a/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml new file mode 100644 index 00000000000..fa609afc5a8 --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml @@ -0,0 +1,54 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml + +# Use this template to enable container scanning in your project. +# You should add this template to an existing `.gitlab-ci.yml` file by using the `include:` +# keyword. +# The template should work without modifications but you can customize the template settings if +# needed: https://docs.gitlab.com/ee/user/application_security/container_scanning/#customizing-the-container-scanning-settings +# +# Requirements: +# - A `test` stage to be present in the pipeline. +# - You must define the image to be scanned in the CS_IMAGE variable. If CS_IMAGE is the +# same as $CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG, you can skip this. +# - Container registry credentials defined by `CS_REGISTRY_USER` and `CS_REGISTRY_PASSWORD` variables if the +# image to be scanned is in a private registry. +# - For auto-remediation, a readable Dockerfile in the root of the project or as defined by the +# CS_DOCKERFILE_PATH variable. +# +# Configure container scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/container_scanning/#available-variables + +variables: + CS_ANALYZER_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/security-products/container-scanning:5" + +container_scanning: + image: "$CS_ANALYZER_IMAGE$CS_IMAGE_SUFFIX" + stage: test + variables: + # To provide a `vulnerability-allowlist.yml` file, override the GIT_STRATEGY variable in your + # `.gitlab-ci.yml` file and set it to `fetch`. + # For details, see the following links: + # https://docs.gitlab.com/ee/user/application_security/container_scanning/index.html#overriding-the-container-scanning-template + # https://docs.gitlab.com/ee/user/application_security/container_scanning/#vulnerability-allowlisting + GIT_STRATEGY: none + allow_failure: true + artifacts: + reports: + container_scanning: gl-container-scanning-report.json + dependency_scanning: gl-dependency-scanning-report.json + paths: [gl-container-scanning-report.json, gl-dependency-scanning-report.json] + dependencies: [] + script: + - gtcs scan + rules: + - if: $CONTAINER_SCANNING_DISABLED + when: never + - if: $CI_COMMIT_BRANCH && + $CI_GITLAB_FIPS_MODE == "true" && + $CS_ANALYZER_IMAGE !~ /-(fips|ubi)\z/ + variables: + CS_IMAGE_SUFFIX: -fips + - if: $CI_COMMIT_BRANCH diff --git a/lib/gitlab/ci/templates/Jobs/Container-Scanning.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Container-Scanning.latest.gitlab-ci.yml new file mode 100644 index 00000000000..f750bda2a3f --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/Container-Scanning.latest.gitlab-ci.yml @@ -0,0 +1,68 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Container-Scanning.latest.gitlab-ci.yml + +# Use this template to enable container scanning in your project. +# You should add this template to an existing `.gitlab-ci.yml` file by using the `include:` +# keyword. +# The template should work without modifications but you can customize the template settings if +# needed: https://docs.gitlab.com/ee/user/application_security/container_scanning/#customizing-the-container-scanning-settings +# +# Requirements: +# - A `test` stage to be present in the pipeline. +# - You must define the image to be scanned in the CS_IMAGE variable. If CS_IMAGE is the +# same as $CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG, you can skip this. +# - Container registry credentials defined by `CS_REGISTRY_USER` and `CS_REGISTRY_PASSWORD` variables if the +# image to be scanned is in a private registry. +# - For auto-remediation, a readable Dockerfile in the root of the project or as defined by the +# CS_DOCKERFILE_PATH variable. +# +# Configure container scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/container_scanning/#available-variables + +variables: + CS_ANALYZER_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/security-products/container-scanning:5" + +container_scanning: + image: "$CS_ANALYZER_IMAGE$CS_IMAGE_SUFFIX" + stage: test + variables: + # To provide a `vulnerability-allowlist.yml` file, override the GIT_STRATEGY variable in your + # `.gitlab-ci.yml` file and set it to `fetch`. + # For details, see the following links: + # https://docs.gitlab.com/ee/user/application_security/container_scanning/index.html#overriding-the-container-scanning-template + # https://docs.gitlab.com/ee/user/application_security/container_scanning/#vulnerability-allowlisting + GIT_STRATEGY: none + allow_failure: true + artifacts: + reports: + container_scanning: gl-container-scanning-report.json + dependency_scanning: gl-dependency-scanning-report.json + paths: [gl-container-scanning-report.json, gl-dependency-scanning-report.json] + dependencies: [] + script: + - gtcs scan + rules: + - if: $CONTAINER_SCANNING_DISABLED + when: never + + # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $CI_GITLAB_FIPS_MODE == "true" && + $CS_ANALYZER_IMAGE !~ /-(fips|ubi)\z/ + variables: + CS_IMAGE_SUFFIX: -fips + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + + # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + - if: $CI_OPEN_MERGE_REQUESTS + when: never + + # Add the job to branch pipelines. + - if: $CI_COMMIT_BRANCH && + $CI_GITLAB_FIPS_MODE == "true" && + $CS_ANALYZER_IMAGE !~ /-(fips|ubi)\z/ + variables: + CS_IMAGE_SUFFIX: -fips + - if: $CI_COMMIT_BRANCH diff --git a/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml index 936d8751fe1..12105e0e95d 100644 --- a/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml @@ -4,8 +4,8 @@ load_performance: allow_failure: true variables: DOCKER_TLS_CERTDIR: "" - K6_IMAGE: loadimpact/k6 - K6_VERSION: 0.27.0 + K6_IMAGE: grafana/k6 + K6_VERSION: 0.41.0 K6_TEST_FILE: raw.githubusercontent.com/grafana/k6/master/samples/http_get.js K6_OPTIONS: '' K6_DOCKER_OPTIONS: '' diff --git a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml index a6d47e31de2..2c5027cdb43 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml @@ -238,6 +238,8 @@ semgrep-sast: - '**/*.java' - '**/*.cs' - '**/*.html' + - '**/*.scala' + - '**/*.sc' sobelow-sast: extends: .sast-analyzer diff --git a/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml index 4600468ef30..58709d3ab62 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml @@ -299,6 +299,8 @@ semgrep-sast: - '**/*.java' - '**/*.html' - '**/*.cs' + - '**/*.scala' + - '**/*.sc' - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. when: never - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. @@ -313,6 +315,8 @@ semgrep-sast: - '**/*.java' - '**/*.html' - '**/*.cs' + - '**/*.scala' + - '**/*.sc' sobelow-sast: extends: .sast-analyzer diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index 79a08c33fdf..879d6a7a468 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -1,54 +1,5 @@ -# To contribute improvements to CI/CD templates, please follow the Development guide at: -# https://docs.gitlab.com/ee/development/cicd/templates.html -# This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +# This template moved to Jobs/Container-Scanning.gitlab-ci.yml in GitLab 15.6 +# Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/381665 -# Use this template to enable container scanning in your project. -# You should add this template to an existing `.gitlab-ci.yml` file by using the `include:` -# keyword. -# The template should work without modifications but you can customize the template settings if -# needed: https://docs.gitlab.com/ee/user/application_security/container_scanning/#customizing-the-container-scanning-settings -# -# Requirements: -# - A `test` stage to be present in the pipeline. -# - You must define the image to be scanned in the CS_IMAGE variable. If CS_IMAGE is the -# same as $CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG, you can skip this. -# - Container registry credentials defined by `CS_REGISTRY_USER` and `CS_REGISTRY_PASSWORD` variables if the -# image to be scanned is in a private registry. -# - For auto-remediation, a readable Dockerfile in the root of the project or as defined by the -# CS_DOCKERFILE_PATH variable. -# -# Configure container scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). -# List of available variables: https://docs.gitlab.com/ee/user/application_security/container_scanning/#available-variables - -variables: - CS_ANALYZER_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/security-products/container-scanning:5" - -container_scanning: - image: "$CS_ANALYZER_IMAGE$CS_IMAGE_SUFFIX" - stage: test - variables: - # To provide a `vulnerability-allowlist.yml` file, override the GIT_STRATEGY variable in your - # `.gitlab-ci.yml` file and set it to `fetch`. - # For details, see the following links: - # https://docs.gitlab.com/ee/user/application_security/container_scanning/index.html#overriding-the-container-scanning-template - # https://docs.gitlab.com/ee/user/application_security/container_scanning/#vulnerability-allowlisting - GIT_STRATEGY: none - allow_failure: true - artifacts: - reports: - container_scanning: gl-container-scanning-report.json - dependency_scanning: gl-dependency-scanning-report.json - paths: [gl-container-scanning-report.json, gl-dependency-scanning-report.json] - dependencies: [] - script: - - gtcs scan - rules: - - if: $CONTAINER_SCANNING_DISABLED - when: never - - if: $CI_COMMIT_BRANCH && - $CI_GITLAB_FIPS_MODE == "true" && - $CS_ANALYZER_IMAGE !~ /-(fips|ubi)\z/ - variables: - CS_IMAGE_SUFFIX: -fips - - if: $CI_COMMIT_BRANCH +include: + template: Jobs/Container-Scanning.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.latest.gitlab-ci.yml index f7b1d12b3b3..7a4f451314e 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.latest.gitlab-ci.yml @@ -1,68 +1,5 @@ -# To contribute improvements to CI/CD templates, please follow the Development guide at: -# https://docs.gitlab.com/ee/development/cicd/templates.html -# This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +# This template moved to Jobs/Container-Scanning.latest.gitlab-ci.yml in GitLab 15.6 +# Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/381665 -# Use this template to enable container scanning in your project. -# You should add this template to an existing `.gitlab-ci.yml` file by using the `include:` -# keyword. -# The template should work without modifications but you can customize the template settings if -# needed: https://docs.gitlab.com/ee/user/application_security/container_scanning/#customizing-the-container-scanning-settings -# -# Requirements: -# - A `test` stage to be present in the pipeline. -# - You must define the image to be scanned in the CS_IMAGE variable. If CS_IMAGE is the -# same as $CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG, you can skip this. -# - Container registry credentials defined by `CS_REGISTRY_USER` and `CS_REGISTRY_PASSWORD` variables if the -# image to be scanned is in a private registry. -# - For auto-remediation, a readable Dockerfile in the root of the project or as defined by the -# CS_DOCKERFILE_PATH variable. -# -# Configure container scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). -# List of available variables: https://docs.gitlab.com/ee/user/application_security/container_scanning/#available-variables - -variables: - CS_ANALYZER_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/security-products/container-scanning:5" - -container_scanning: - image: "$CS_ANALYZER_IMAGE$CS_IMAGE_SUFFIX" - stage: test - variables: - # To provide a `vulnerability-allowlist.yml` file, override the GIT_STRATEGY variable in your - # `.gitlab-ci.yml` file and set it to `fetch`. - # For details, see the following links: - # https://docs.gitlab.com/ee/user/application_security/container_scanning/index.html#overriding-the-container-scanning-template - # https://docs.gitlab.com/ee/user/application_security/container_scanning/#vulnerability-allowlisting - GIT_STRATEGY: none - allow_failure: true - artifacts: - reports: - container_scanning: gl-container-scanning-report.json - dependency_scanning: gl-dependency-scanning-report.json - paths: [gl-container-scanning-report.json, gl-dependency-scanning-report.json] - dependencies: [] - script: - - gtcs scan - rules: - - if: $CONTAINER_SCANNING_DISABLED - when: never - - # Add the job to merge request pipelines if there's an open merge request. - - if: $CI_PIPELINE_SOURCE == "merge_request_event" && - $CI_GITLAB_FIPS_MODE == "true" && - $CS_ANALYZER_IMAGE !~ /-(fips|ubi)\z/ - variables: - CS_IMAGE_SUFFIX: -fips - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - - # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. - - if: $CI_OPEN_MERGE_REQUESTS - when: never - - # Add the job to branch pipelines. - - if: $CI_COMMIT_BRANCH && - $CI_GITLAB_FIPS_MODE == "true" && - $CS_ANALYZER_IMAGE !~ /-(fips|ubi)\z/ - variables: - CS_IMAGE_SUFFIX: -fips - - if: $CI_COMMIT_BRANCH +include: + template: Jobs/Container-Scanning.latest.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml index d933007ec61..89944e347f6 100644 --- a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml @@ -16,7 +16,7 @@ variables: COVFUZZ_VERSION: v3 # This is for users who have an offline environment and will have to replicate gitlab-cov-fuzz release binaries # to their own servers - COVFUZZ_URL_PREFIX: "https://gitlab.com/gitlab-org/security-products/analyzers/gitlab-cov-fuzz/-/raw" + COVFUZZ_URL_PREFIX: "https://gitlab.com/security-products/gitlab-cov-fuzz/-/raw" coverage_fuzzing_unlicensed: diff --git a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.latest.gitlab-ci.yml index feed4c47157..4f6ba427058 100644 --- a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.latest.gitlab-ci.yml @@ -16,7 +16,7 @@ variables: COVFUZZ_VERSION: v3 # This is for users who have an offline environment and will have to replicate gitlab-cov-fuzz release binaries # to their own servers - COVFUZZ_URL_PREFIX: "https://gitlab.com/gitlab-org/security-products/analyzers/gitlab-cov-fuzz/-/raw" + COVFUZZ_URL_PREFIX: "https://gitlab.com/security-products/gitlab-cov-fuzz/-/raw" coverage_fuzzing_unlicensed: diff --git a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml index 40060e96dff..c43296b5865 100644 --- a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml @@ -51,7 +51,4 @@ dast: $REVIEW_DISABLED when: never - if: $CI_COMMIT_BRANCH && - ($CI_KUBERNETES_ACTIVE || $KUBECONFIG) && - $GITLAB_FEATURES =~ /\bdast\b/ - - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdast\b/ diff --git a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml index 50e9bb5431d..27bcc14bcf5 100644 --- a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml @@ -55,9 +55,6 @@ dast: # Add the job to merge request pipelines if there's an open merge request. - if: $CI_PIPELINE_SOURCE == "merge_request_event" && - ($CI_KUBERNETES_ACTIVE || $KUBECONFIG) && - $GITLAB_FEATURES =~ /\bdast\b/ - - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $GITLAB_FEATURES =~ /\bdast\b/ # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. @@ -66,9 +63,6 @@ dast: # Add the job to branch pipelines. - if: $CI_COMMIT_BRANCH && - ($CI_KUBERNETES_ACTIVE || $KUBECONFIG) && - $GITLAB_FEATURES =~ /\bdast\b/ - - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdast\b/ after_script: # Remove any debug.log files because they might contain secrets. diff --git a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml index fd04c86e6c7..631f6cecddf 100644 --- a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml @@ -9,9 +9,9 @@ # Usage: # # include: -# - template: Secure-Binaries.gitlab-ci.yml +# - template: Security/Secure-Binaries.gitlab-ci.yml # -# Docs: https://docs.gitlab.com/ee/topics/airgap/ +# Docs: https://docs.gitlab.com/ee/user/application_security/offline_deployments/ variables: # Setting this variable will affect all Security templates @@ -38,7 +38,7 @@ variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" services: - - docker:stable-dind + - docker:dind script: - docker info - env diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml index c3113ffebf3..c1a90955f7f 100644 --- a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml @@ -17,10 +17,10 @@ browser_performance: variables: URL: '' SITESPEED_IMAGE: sitespeedio/sitespeed.io - SITESPEED_VERSION: 14.1.0 + SITESPEED_VERSION: 26.1.0 SITESPEED_OPTIONS: '' services: - - docker:stable-dind + - docker:dind script: - mkdir gitlab-exporter # Busybox wget does not support proxied HTTPS, get the real thing. diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml index c9f0c173692..adc92fde5ae 100644 --- a/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml @@ -17,10 +17,10 @@ browser_performance: variables: URL: '' SITESPEED_IMAGE: sitespeedio/sitespeed.io - SITESPEED_VERSION: 14.1.0 + SITESPEED_VERSION: latest SITESPEED_OPTIONS: '' services: - - docker:stable-dind + - docker:dind script: - mkdir gitlab-exporter # Busybox wget does not support proxied HTTPS, get the real thing. diff --git a/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml index bf5cfbb519d..a907915587a 100644 --- a/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml @@ -15,13 +15,13 @@ load_performance: stage: performance image: docker:git variables: - K6_IMAGE: loadimpact/k6 - K6_VERSION: 0.27.0 + K6_IMAGE: grafana/k6 + K6_VERSION: 0.41.0 K6_TEST_FILE: raw.githubusercontent.com/grafana/k6/master/samples/http_get.js K6_OPTIONS: '' K6_DOCKER_OPTIONS: '' services: - - docker:stable-dind + - docker:dind script: - docker run --rm -v "$(pwd)":/k6 -w /k6 $K6_DOCKER_OPTIONS $K6_IMAGE:$K6_VERSION run $K6_TEST_FILE --summary-export=load-performance.json $K6_OPTIONS artifacts: diff --git a/lib/gitlab/ci/variables/builder.rb b/lib/gitlab/ci/variables/builder.rb index 8db8ea3a720..8e18d57b724 100644 --- a/lib/gitlab/ci/variables/builder.rb +++ b/lib/gitlab/ci/variables/builder.rb @@ -53,7 +53,7 @@ module Gitlab # https://gitlab.com/groups/gitlab-org/configure/-/epics/8 # Until then, we need to make both the old and the new KUBECONFIG contexts available collection.concat(deployment_variables(environment: environment, job: job)) - template = ::Ci::GenerateKubeconfigService.new(pipeline, token: job.try(:token)).execute + template = ::Ci::GenerateKubeconfigService.new(pipeline, token: job.try(:token), environment: environment).execute kubeconfig_yaml = collection['KUBECONFIG']&.value template.merge_yaml(kubeconfig_yaml) if kubeconfig_yaml.present? @@ -135,6 +135,9 @@ module Gitlab variables.append(key: 'CI_NODE_INDEX', value: job.options[:instance].to_s) if job.options&.include?(:instance) variables.append(key: 'CI_NODE_TOTAL', value: ci_node_total_value(job).to_s) + # Set environment name here so we can access it when evaluating the job's rules + variables.append(key: 'CI_ENVIRONMENT_NAME', value: job.environment) if job.environment + # legacy variables variables.append(key: 'CI_BUILD_NAME', value: job.name) variables.append(key: 'CI_BUILD_STAGE', value: job.stage_name) diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index ff255543d3b..f2c1ad0575d 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -107,6 +107,7 @@ module Gitlab cache: job[:cache], resource_group_key: job[:resource_group], scheduling_type: job[:scheduling_type], + id_tokens: job[:id_tokens], options: { image: job[:image], services: job[:services], @@ -118,6 +119,7 @@ module Gitlab before_script: job[:before_script], script: job[:script], after_script: job[:after_script], + hooks: job[:hooks], environment: job[:environment], resource_group_key: job[:resource_group], retry: job[:retry], diff --git a/lib/gitlab/cluster/rack_timeout_observer.rb b/lib/gitlab/cluster/rack_timeout_observer.rb index 5182b2be148..15dd6a59e19 100644 --- a/lib/gitlab/cluster/rack_timeout_observer.rb +++ b/lib/gitlab/cluster/rack_timeout_observer.rb @@ -3,6 +3,7 @@ module Gitlab module Cluster class RackTimeoutObserver + include ActionView::Helpers::SanitizeHelper TRANSITION_STATES = %i(ready active).freeze def initialize @@ -28,9 +29,9 @@ module Gitlab params = controller_params(env) || grape_params(env) || {} { - controller: params['controller'], - action: params['action'], - route: params['route'], + controller: sanitize(params['controller']), + action: sanitize(params['action']), + route: sanitize(params['route']), state: info.state } end diff --git a/lib/gitlab/color.rb b/lib/gitlab/color.rb index 01c534c15a0..7d9280ddba2 100644 --- a/lib/gitlab/color.rb +++ b/lib/gitlab/color.rb @@ -215,13 +215,11 @@ module Gitlab def rgb return [] unless valid? - @rgb ||= begin - if @value.length == 4 - @value[1, 4].scan(/./).map { |v| (v * 2).hex } - else - @value[1, 7].scan(/.{2}/).map(&:hex) - end - end + @rgb ||= if @value.length == 4 + @value[1, 4].scan(/./).map { |v| (v * 2).hex } + else + @value[1, 7].scan(/.{2}/).map(&:hex) + end end end end diff --git a/lib/gitlab/config/entry/attributable.rb b/lib/gitlab/config/entry/attributable.rb index d266d5218de..c8ad2521574 100644 --- a/lib/gitlab/config/entry/attributable.rb +++ b/lib/gitlab/config/entry/attributable.rb @@ -7,19 +7,21 @@ module Gitlab extend ActiveSupport::Concern class_methods do - def attributes(*attributes) + def attributes(*attributes, prefix: nil) attributes.flatten.each do |attribute| - if method_defined?(attribute) - raise ArgumentError, "Method '#{attribute}' already defined in '#{name}'" + attribute_method = prefix ? "#{prefix}_#{attribute}" : attribute + + if method_defined?(attribute_method) + raise ArgumentError, "Method '#{attribute_method}' already defined in '#{name}'" end - define_method(attribute) do + define_method(attribute_method) do return unless config.is_a?(Hash) config[attribute] end - define_method("has_#{attribute}?") do + define_method("has_#{attribute_method}?") do config.is_a?(Hash) && config.key?(attribute) end end diff --git a/lib/gitlab/conflict/file.rb b/lib/gitlab/conflict/file.rb index d40a6323d4f..7bcbcf84a4e 100644 --- a/lib/gitlab/conflict/file.rb +++ b/lib/gitlab/conflict/file.rb @@ -236,14 +236,12 @@ module Gitlab else :modified_target_removed_source end + elsif our_path.present? && their_path.present? + :both_added + elsif their_path.blank? + diff_file.renamed_file? ? :renamed_same_file : :removed_target_renamed_source else - if our_path.present? && their_path.present? - :both_added - elsif their_path.blank? - diff_file.renamed_file? ? :renamed_same_file : :removed_target_renamed_source - else - :removed_source_renamed_target - end + :removed_source_renamed_target end end diff --git a/lib/gitlab/content_security_policy/config_loader.rb b/lib/gitlab/content_security_policy/config_loader.rb index 29e8e631fb7..8b1298d0561 100644 --- a/lib/gitlab/content_security_policy/config_loader.rb +++ b/lib/gitlab/content_security_policy/config_loader.rb @@ -43,7 +43,10 @@ module Gitlab allow_websocket_connections(directives) allow_cdn(directives, Settings.gitlab.cdn_host) if Settings.gitlab.cdn_host.present? - allow_sentry(directives) if Gitlab.config.sentry&.enabled && Gitlab.config.sentry&.clientside_dsn + # Support for Sentry setup via configuration files will be removed in 16.0 + # in favor of Gitlab::CurrentSettings. + allow_legacy_sentry(directives) if Gitlab.config.sentry&.enabled && Gitlab.config.sentry&.clientside_dsn + allow_sentry(directives) if Gitlab::CurrentSettings.try(:sentry_enabled) && Gitlab::CurrentSettings.try(:sentry_clientside_dsn) allow_framed_gitlab_paths(directives) allow_customersdot(directives) if ENV['CUSTOMER_PORTAL_URL'].present? allow_review_apps(directives) if ENV['REVIEW_APPS_ENABLED'] @@ -135,13 +138,22 @@ module Gitlab append_to_directive(directives, 'frame_src', customersdot_host) end - def self.allow_sentry(directives) + def self.allow_legacy_sentry(directives) + # Support for Sentry setup via configuration files will be removed in 16.0 + # in favor of Gitlab::CurrentSettings. sentry_dsn = Gitlab.config.sentry.clientside_dsn sentry_uri = URI(sentry_dsn) append_to_directive(directives, 'connect_src', "#{sentry_uri.scheme}://#{sentry_uri.host}") end + def self.allow_sentry(directives) + sentry_dsn = Gitlab::CurrentSettings.sentry_clientside_dsn + sentry_uri = URI(sentry_dsn) + + append_to_directive(directives, 'connect_src', "#{sentry_uri.scheme}://#{sentry_uri.host}") + end + def self.allow_letter_opener(directives) append_to_directive(directives, 'frame_src', Gitlab::Utils.append_path(Gitlab.config.gitlab.url, '/rails/letter_opener/')) end diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index a45380aca6c..2068a9ae7d5 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -31,7 +31,7 @@ module Gitlab repo_events = events_created_between(start_time, end_time, :repository) .where(action: :pushed) issue_events = events_created_between(start_time, end_time, :issues) - .where(action: [:created, :closed], target_type: "Issue") + .where(action: [:created, :closed], target_type: %w[Issue WorkItem]) mr_events = events_created_between(start_time, end_time, :merge_requests) .where(action: [:merged, :created, :closed], target_type: "MergeRequest") note_events = events_created_between(start_time, end_time, :merge_requests) diff --git a/lib/gitlab/counters/buffered_counter.rb b/lib/gitlab/counters/buffered_counter.rb new file mode 100644 index 00000000000..56593b642a9 --- /dev/null +++ b/lib/gitlab/counters/buffered_counter.rb @@ -0,0 +1,113 @@ +# frozen_string_literal: true + +module Gitlab + module Counters + class BufferedCounter + include Gitlab::ExclusiveLeaseHelpers + + WORKER_DELAY = 10.minutes + WORKER_LOCK_TTL = 10.minutes + + LUA_FLUSH_INCREMENT_SCRIPT = <<~LUA + local increment_key, flushed_key = KEYS[1], KEYS[2] + local increment_value = redis.call("get", increment_key) or 0 + local flushed_value = redis.call("incrby", flushed_key, increment_value) + if flushed_value == 0 then + redis.call("del", increment_key, flushed_key) + else + redis.call("del", increment_key) + end + return flushed_value + LUA + + def initialize(counter_record, attribute) + @counter_record = counter_record + @attribute = attribute + end + + def get + redis_state do |redis| + redis.get(key).to_i + end + end + + def increment(amount) + result = redis_state do |redis| + redis.incrby(key, amount) + end + + FlushCounterIncrementsWorker.perform_in(WORKER_DELAY, counter_record.class.name, counter_record.id, attribute) + + result + end + + def reset! + counter_record.update!(attribute => 0) + + redis_state do |redis| + redis.del(key) + end + end + + def commit_increment! + with_exclusive_lease do + flush_amount = amount_to_be_flushed + next if flush_amount == 0 + + counter_record.transaction do + counter_record.update_counters_with_lease({ attribute => flush_amount }) + remove_flushed_key + end + + counter_record.execute_after_commit_callbacks + end + + counter_record.reset.read_attribute(attribute) + end + + # amount_to_be_flushed returns the total value to be flushed. + # The total value is the sum of the following: + # - current value in the increment_key + # - any existing value in the flushed_key that has not been flushed + def amount_to_be_flushed + redis_state do |redis| + redis.eval(LUA_FLUSH_INCREMENT_SCRIPT, keys: [key, flushed_key]) + end + end + + def key + project_id = counter_record.project.id + record_name = counter_record.class + record_id = counter_record.id + + "project:{#{project_id}}:counters:#{record_name}:#{record_id}:#{attribute}" + end + + def flushed_key + "#{key}:flushed" + end + + private + + attr_reader :counter_record, :attribute + + def remove_flushed_key + redis_state do |redis| + redis.del(flushed_key) + end + end + + def redis_state(&block) + Gitlab::Redis::SharedState.with(&block) + end + + def with_exclusive_lease(&block) + lock_key = "#{key}:locked" + + in_lock(lock_key, ttl: WORKER_LOCK_TTL, &block) + rescue Gitlab::ExclusiveLeaseHelpers::FailedToObtainLockError + # a worker is already updating the counters + end + end + end +end diff --git a/lib/gitlab/counters/legacy_counter.rb b/lib/gitlab/counters/legacy_counter.rb new file mode 100644 index 00000000000..06951514ec3 --- /dev/null +++ b/lib/gitlab/counters/legacy_counter.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module Counters + # This class is a wrapper over ActiveRecord counter + # for attributes that have not adopted Redis-backed BufferedCounter. + class LegacyCounter + def initialize(counter_record, attribute) + @counter_record = counter_record + @attribute = attribute + @current_value = counter_record.method(attribute).call + end + + def increment(amount) + updated = counter_record.class.update_counters(counter_record.id, { attribute => amount }) + + if updated == 1 + counter_record.execute_after_commit_callbacks + @current_value += amount + end + + @current_value + end + + def reset! + counter_record.update!(attribute => 0) + end + + private + + attr_reader :counter_record, :attribute + end + end +end diff --git a/lib/gitlab/data_builder/deployment.rb b/lib/gitlab/data_builder/deployment.rb index a9c69e3f997..7055f64937d 100644 --- a/lib/gitlab/data_builder/deployment.rb +++ b/lib/gitlab/data_builder/deployment.rb @@ -35,6 +35,8 @@ module Gitlab deployable_id: deployment.deployable_id, deployable_url: deployable_url, environment: deployment.environment.name, + environment_slug: deployment.environment.slug, + environment_external_url: deployment.environment.external_url, project: deployment.project.hook_attrs, short_sha: deployment.short_sha, user: deployment.deployed_by&.hook_attrs, diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 04cf056199c..51d5bfcee38 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -101,7 +101,8 @@ module Gitlab gitlab_main: [self.database_base_models.fetch(:main)], gitlab_ci: [self.database_base_models[:ci] || self.database_base_models.fetch(:main)], # use CI or fallback to main gitlab_shared: database_base_models_with_gitlab_shared.values, # all models - gitlab_internal: database_base_models.values # all models + gitlab_internal: database_base_models.values, # all models + gitlab_pm: [self.database_base_models.fetch(:main)] # package metadata models }.with_indifferent_access.freeze end diff --git a/lib/gitlab/database/bulk_update.rb b/lib/gitlab/database/bulk_update.rb index d68be19047e..4b4a9b38fd8 100644 --- a/lib/gitlab/database/bulk_update.rb +++ b/lib/gitlab/database/bulk_update.rb @@ -157,7 +157,7 @@ module Gitlab def self.execute(columns, mapping, &to_class) raise ArgumentError if mapping.blank? - entries_by_class = mapping.group_by { |k, v| to_class ? to_class.call(k) : k.class } + entries_by_class = mapping.group_by { |k, v| to_class ? yield(k) : k.class } entries_by_class.each do |model, entries| Setter.new(model, columns, entries).update! diff --git a/lib/gitlab/database/count/exact_count_strategy.rb b/lib/gitlab/database/count/exact_count_strategy.rb index 345c7e44b05..11c83786aa4 100644 --- a/lib/gitlab/database/count/exact_count_strategy.rb +++ b/lib/gitlab/database/count/exact_count_strategy.rb @@ -18,9 +18,7 @@ module Gitlab end def count - models.each_with_object({}) do |model, data| - data[model] = model.count - end + models.index_with(&:count) rescue *CONNECTION_ERRORS {} end diff --git a/lib/gitlab/database/gitlab_schema.rb b/lib/gitlab/database/gitlab_schema.rb index 365a4283d4c..0f848ed40fb 100644 --- a/lib/gitlab/database/gitlab_schema.rb +++ b/lib/gitlab/database/gitlab_schema.rb @@ -6,14 +6,16 @@ # Each table / view needs to have assigned gitlab_schema. Names supported today: # # - gitlab_shared - defines a set of tables that are found on all databases (data accessed is dependent on connection) -# - gitlab_main / gitlab_ci - defines a set of tables that can only exist on a given database +# - gitlab_main / gitlab_ci - defines a set of tables that can only exist on a given application database +# - gitlab_geo - defines a set of tables that can only exist on the geo database +# - gitlab_internal - defines all internal tables of Rails and PostgreSQL # # Tables for the purpose of tests should be prefixed with `_test_my_table_name` module Gitlab module Database module GitlabSchema - GITLAB_SCHEMAS_FILE = 'lib/gitlab/database/gitlab_schemas.yml' + DICTIONARY_PATH = 'db/docs/' # These tables are deleted/renamed, but still referenced by migrations. # This is needed for now, but should be removed in the future @@ -55,7 +57,7 @@ module Gitlab tables.map { |table| table_schema(table) }.to_set end - def self.table_schema(name) + def self.table_schema(name, undefined: true) schema_name, table_name = name.split('.', 2) # Strip schema name like: `public.` # Most of names do not have schemas, ensure that this is table @@ -68,7 +70,7 @@ module Gitlab table_name.gsub!(/_[0-9]+$/, '') # Tables that are properly mapped - if gitlab_schema = tables_to_schema[table_name] + if gitlab_schema = views_and_tables_to_schema[table_name] return gitlab_schema end @@ -84,6 +86,8 @@ module Gitlab return :gitlab_ci if table_name.start_with?('_test_gitlab_ci_') + return :gitlab_geo if table_name.start_with?('_test_gitlab_geo_') + # All tables that start with `_test_` without a following schema are shared and ignored return :gitlab_shared if table_name.start_with?('_test_') @@ -91,15 +95,39 @@ module Gitlab return :gitlab_internal if table_name.start_with?('pg_') # When undefined it's best to return a unique name so that we don't incorrectly assume that 2 undefined schemas belong on the same database - :"undefined_#{table_name}" + undefined ? :"undefined_#{table_name}" : nil + end + + def self.dictionary_path_globs + [Rails.root.join(DICTIONARY_PATH, '*.yml')] + end + + def self.view_path_globs + [Rails.root.join(DICTIONARY_PATH, 'views', '*.yml')] + end + + def self.views_and_tables_to_schema + @views_and_tables_to_schema ||= self.tables_to_schema.merge(self.views_to_schema) end def self.tables_to_schema - @tables_to_schema ||= YAML.load_file(Rails.root.join(GITLAB_SCHEMAS_FILE)) + @tables_to_schema ||= Dir.glob(self.dictionary_path_globs).each_with_object({}) do |file_path, dic| + data = YAML.load_file(file_path) + + dic[data['table_name']] = data['gitlab_schema'].to_sym + end + end + + def self.views_to_schema + @views_to_schema ||= Dir.glob(self.view_path_globs).each_with_object({}) do |file_path, dic| + data = YAML.load_file(file_path) + + dic[data['view_name']] = data['gitlab_schema'].to_sym + end end def self.schema_names - @schema_names ||= self.tables_to_schema.values.to_set + @schema_names ||= self.views_and_tables_to_schema.values.to_set end end end diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml deleted file mode 100644 index bf6ebb21f7d..00000000000 --- a/lib/gitlab/database/gitlab_schemas.yml +++ /dev/null @@ -1,606 +0,0 @@ -abuse_reports: :gitlab_main -agent_activity_events: :gitlab_main -agent_group_authorizations: :gitlab_main -agent_project_authorizations: :gitlab_main -alert_management_alert_assignees: :gitlab_main -alert_management_alerts: :gitlab_main -alert_management_alert_metric_images: :gitlab_main -alert_management_alert_user_mentions: :gitlab_main -alert_management_http_integrations: :gitlab_main -allowed_email_domains: :gitlab_main -analytics_cycle_analytics_aggregations: :gitlab_main -analytics_cycle_analytics_group_stages: :gitlab_main -analytics_cycle_analytics_group_value_streams: :gitlab_main -analytics_cycle_analytics_issue_stage_events: :gitlab_main -analytics_cycle_analytics_merge_request_stage_events: :gitlab_main -analytics_cycle_analytics_project_stages: :gitlab_main -analytics_cycle_analytics_project_value_streams: :gitlab_main -analytics_cycle_analytics_stage_event_hashes: :gitlab_main -analytics_devops_adoption_segments: :gitlab_main -analytics_devops_adoption_snapshots: :gitlab_main -analytics_language_trend_repository_languages: :gitlab_main -analytics_usage_trends_measurements: :gitlab_main -appearances: :gitlab_main -application_settings: :gitlab_main -application_setting_terms: :gitlab_main -approval_merge_request_rules_approved_approvers: :gitlab_main -approval_merge_request_rules: :gitlab_main -approval_merge_request_rules_groups: :gitlab_main -approval_merge_request_rule_sources: :gitlab_main -approval_merge_request_rules_users: :gitlab_main -approval_project_rules: :gitlab_main -approval_project_rules_groups: :gitlab_main -approval_project_rules_protected_branches: :gitlab_main -approval_project_rules_users: :gitlab_main -approvals: :gitlab_main -approver_groups: :gitlab_main -approvers: :gitlab_main -ar_internal_metadata: :gitlab_internal -atlassian_identities: :gitlab_main -audit_events_external_audit_event_destinations: :gitlab_main -audit_events: :gitlab_main -audit_events_streaming_headers: :gitlab_main -audit_events_streaming_event_type_filters: :gitlab_main -authentication_events: :gitlab_main -award_emoji: :gitlab_main -aws_roles: :gitlab_main -background_migration_jobs: :gitlab_shared -badges: :gitlab_main -banned_users: :gitlab_main -batched_background_migration_jobs: :gitlab_shared -batched_background_migrations: :gitlab_shared -board_assignees: :gitlab_main -board_group_recent_visits: :gitlab_main -board_labels: :gitlab_main -board_project_recent_visits: :gitlab_main -boards_epic_board_labels: :gitlab_main -boards_epic_board_positions: :gitlab_main -boards_epic_board_recent_visits: :gitlab_main -boards_epic_boards: :gitlab_main -boards_epic_lists: :gitlab_main -boards_epic_list_user_preferences: :gitlab_main -boards_epic_user_preferences: :gitlab_main -boards: :gitlab_main -board_user_preferences: :gitlab_main -broadcast_messages: :gitlab_main -bulk_import_configurations: :gitlab_main -bulk_import_entities: :gitlab_main -bulk_import_exports: :gitlab_main -bulk_import_export_uploads: :gitlab_main -bulk_import_failures: :gitlab_main -bulk_imports: :gitlab_main -bulk_import_trackers: :gitlab_main -chat_names: :gitlab_main -chat_teams: :gitlab_main -ci_build_needs: :gitlab_ci -ci_build_pending_states: :gitlab_ci -ci_build_report_results: :gitlab_ci -ci_builds: :gitlab_ci -ci_builds_metadata: :gitlab_ci -ci_builds_runner_session: :gitlab_ci -ci_build_trace_chunks: :gitlab_ci -ci_build_trace_metadata: :gitlab_ci -ci_daily_build_group_report_results: :gitlab_ci -ci_deleted_objects: :gitlab_ci -ci_freeze_periods: :gitlab_ci -ci_group_variables: :gitlab_ci -ci_instance_variables: :gitlab_ci -ci_job_artifacts: :gitlab_ci -ci_job_token_project_scope_links: :gitlab_ci -ci_job_variables: :gitlab_ci -ci_job_artifact_states: :gitlab_ci -ci_minutes_additional_packs: :gitlab_ci -ci_namespace_monthly_usages: :gitlab_ci -ci_namespace_mirrors: :gitlab_ci -ci_partitions: :gitlab_ci -ci_pending_builds: :gitlab_ci -ci_pipeline_artifacts: :gitlab_ci -ci_pipeline_chat_data: :gitlab_ci -ci_pipeline_messages: :gitlab_ci -ci_pipeline_schedules: :gitlab_ci -ci_pipeline_schedule_variables: :gitlab_ci -ci_pipelines_config: :gitlab_ci -ci_pipeline_metadata: :gitlab_ci -ci_pipelines: :gitlab_ci -ci_pipeline_variables: :gitlab_ci -ci_platform_metrics: :gitlab_ci -ci_project_monthly_usages: :gitlab_ci -ci_project_mirrors: :gitlab_ci -ci_refs: :gitlab_ci -ci_resource_groups: :gitlab_ci -ci_resources: :gitlab_ci -ci_runner_namespaces: :gitlab_ci -ci_runner_projects: :gitlab_ci -ci_runner_versions: :gitlab_ci -ci_runners: :gitlab_ci -ci_running_builds: :gitlab_ci -ci_sources_pipelines: :gitlab_ci -ci_secure_files: :gitlab_ci -ci_secure_file_states: :gitlab_ci -ci_sources_projects: :gitlab_ci -ci_stages: :gitlab_ci -ci_subscriptions_projects: :gitlab_ci -ci_trigger_requests: :gitlab_ci -ci_triggers: :gitlab_ci -ci_unit_test_failures: :gitlab_ci -ci_unit_tests: :gitlab_ci -ci_variables: :gitlab_ci -cluster_agents: :gitlab_main -cluster_agent_tokens: :gitlab_main -cluster_enabled_grants: :gitlab_main -cluster_groups: :gitlab_main -cluster_platforms_kubernetes: :gitlab_main -cluster_projects: :gitlab_main -cluster_providers_aws: :gitlab_main -cluster_providers_gcp: :gitlab_main -clusters_applications_cert_managers: :gitlab_main -clusters_applications_cilium: :gitlab_main -clusters_applications_crossplane: :gitlab_main -clusters_applications_helm: :gitlab_main -clusters_applications_ingress: :gitlab_main -clusters_applications_jupyter: :gitlab_main -clusters_applications_knative: :gitlab_main -clusters_applications_prometheus: :gitlab_main -clusters_applications_runners: :gitlab_main -clusters: :gitlab_main -clusters_integration_prometheus: :gitlab_main -clusters_kubernetes_namespaces: :gitlab_main -commit_user_mentions: :gitlab_main -compliance_management_frameworks: :gitlab_main -container_expiration_policies: :gitlab_main -container_repositories: :gitlab_main -content_blocked_states: :gitlab_main -conversational_development_index_metrics: :gitlab_main -coverage_fuzzing_corpuses: :gitlab_main -csv_issue_imports: :gitlab_main -custom_emoji: :gitlab_main -customer_relations_contacts: :gitlab_main -customer_relations_organizations: :gitlab_main -dast_profile_schedules: :gitlab_main -dast_profiles: :gitlab_main -dast_profiles_pipelines: :gitlab_main -dast_scanner_profiles_builds: :gitlab_main -dast_scanner_profiles: :gitlab_main -dast_site_profiles_builds: :gitlab_main -dast_site_profile_secret_variables: :gitlab_main -dast_site_profiles: :gitlab_main -dast_site_profiles_pipelines: :gitlab_main -dast_sites: :gitlab_main -dast_site_tokens: :gitlab_main -dast_site_validations: :gitlab_main -dependency_proxy_blob_states: :gitlab_main -dependency_proxy_blobs: :gitlab_main -dependency_proxy_group_settings: :gitlab_main -dependency_proxy_image_ttl_group_policies: :gitlab_main -dependency_proxy_manifests: :gitlab_main -deploy_keys_projects: :gitlab_main -deployment_approvals: :gitlab_main -deployment_clusters: :gitlab_main -deployment_merge_requests: :gitlab_main -deployments: :gitlab_main -deploy_tokens: :gitlab_main -description_versions: :gitlab_main -design_management_designs: :gitlab_main -design_management_designs_versions: :gitlab_main -design_management_versions: :gitlab_main -design_user_mentions: :gitlab_main -detached_partitions: :gitlab_shared -diff_note_positions: :gitlab_main -dora_configurations: :gitlab_main -dora_daily_metrics: :gitlab_main -draft_notes: :gitlab_main -elastic_index_settings: :gitlab_main -elastic_reindexing_slices: :gitlab_main -elastic_reindexing_subtasks: :gitlab_main -elastic_reindexing_tasks: :gitlab_main -elasticsearch_indexed_namespaces: :gitlab_main -elasticsearch_indexed_projects: :gitlab_main -emails: :gitlab_main -environments: :gitlab_main -epic_issues: :gitlab_main -epic_metrics: :gitlab_main -epics: :gitlab_main -epic_user_mentions: :gitlab_main -error_tracking_client_keys: :gitlab_main -error_tracking_error_events: :gitlab_main -error_tracking_errors: :gitlab_main -events: :gitlab_main -evidences: :gitlab_main -experiments: :gitlab_main -experiment_subjects: :gitlab_main -external_approval_rules: :gitlab_main -external_approval_rules_protected_branches: :gitlab_main -external_pull_requests: :gitlab_ci -external_status_checks: :gitlab_main -external_status_checks_protected_branches: :gitlab_main -feature_gates: :gitlab_main -features: :gitlab_main -fork_network_members: :gitlab_main -fork_networks: :gitlab_main -geo_cache_invalidation_events: :gitlab_main -geo_container_repository_updated_events: :gitlab_main -geo_event_log: :gitlab_main -geo_events: :gitlab_main -geo_hashed_storage_attachments_events: :gitlab_main -geo_hashed_storage_migrated_events: :gitlab_main -geo_node_namespace_links: :gitlab_main -geo_nodes: :gitlab_main -geo_node_statuses: :gitlab_main -geo_repositories_changed_events: :gitlab_main -geo_repository_created_events: :gitlab_main -geo_repository_deleted_events: :gitlab_main -geo_repository_renamed_events: :gitlab_main -geo_repository_updated_events: :gitlab_main -geo_reset_checksum_events: :gitlab_main -ghost_user_migrations: :gitlab_main -gitlab_subscription_histories: :gitlab_main -gitlab_subscriptions: :gitlab_main -gpg_keys: :gitlab_main -gpg_key_subkeys: :gitlab_main -gpg_signatures: :gitlab_main -grafana_integrations: :gitlab_main -group_custom_attributes: :gitlab_main -group_crm_settings: :gitlab_main -group_deletion_schedules: :gitlab_main -group_deploy_keys: :gitlab_main -group_deploy_keys_groups: :gitlab_main -group_deploy_tokens: :gitlab_main -group_features: :gitlab_main -group_group_links: :gitlab_main -group_import_states: :gitlab_main -group_merge_request_approval_settings: :gitlab_main -group_repository_storage_moves: :gitlab_main -group_wiki_repositories: :gitlab_main -historical_data: :gitlab_main -identities: :gitlab_main -import_export_uploads: :gitlab_main -import_failures: :gitlab_main -incident_management_escalation_policies: :gitlab_main -incident_management_escalation_rules: :gitlab_main -incident_management_issuable_escalation_statuses: :gitlab_main -incident_management_oncall_participants: :gitlab_main -incident_management_oncall_rotations: :gitlab_main -incident_management_oncall_schedules: :gitlab_main -incident_management_oncall_shifts: :gitlab_main -incident_management_pending_alert_escalations: :gitlab_main -incident_management_pending_issue_escalations: :gitlab_main -incident_management_timeline_events: :gitlab_main -incident_management_timeline_event_tags: :gitlab_main -incident_management_timeline_event_tag_links: :gitlab_main -index_statuses: :gitlab_main -in_product_marketing_emails: :gitlab_main -insights: :gitlab_main -integrations: :gitlab_main -internal_ids: :gitlab_main -ip_restrictions: :gitlab_main -issuable_metric_images: :gitlab_main -issuable_resource_links: :gitlab_main -issuable_severities: :gitlab_main -issuable_slas: :gitlab_main -issue_assignees: :gitlab_main -issue_customer_relations_contacts: :gitlab_main -issue_emails: :gitlab_main -issue_email_participants: :gitlab_main -issue_links: :gitlab_main -issue_metrics: :gitlab_main -issue_search_data: :gitlab_main -issues: :gitlab_main -issues_prometheus_alert_events: :gitlab_main -issues_self_managed_prometheus_alert_events: :gitlab_main -issue_tracker_data: :gitlab_main -issue_user_mentions: :gitlab_main -iterations_cadences: :gitlab_main -jira_connect_installations: :gitlab_main -jira_connect_subscriptions: :gitlab_main -jira_imports: :gitlab_main -jira_tracker_data: :gitlab_main -keys: :gitlab_main -label_links: :gitlab_main -label_priorities: :gitlab_main -labels: :gitlab_main -ldap_group_links: :gitlab_main -lfs_file_locks: :gitlab_main -lfs_objects: :gitlab_main -lfs_objects_projects: :gitlab_main -lfs_object_states: :gitlab_main -licenses: :gitlab_main -lists: :gitlab_main -list_user_preferences: :gitlab_main -loose_foreign_keys_deleted_records: :gitlab_shared -member_roles: :gitlab_main -member_tasks: :gitlab_main -members: :gitlab_main -merge_request_assignees: :gitlab_main -merge_request_blocks: :gitlab_main -merge_request_cleanup_schedules: :gitlab_main -merge_requests_compliance_violations: :gitlab_main -merge_request_context_commit_diff_files: :gitlab_main -merge_request_context_commits: :gitlab_main -merge_request_diff_commits: :gitlab_main -merge_request_diff_commit_users: :gitlab_main -merge_request_diff_details: :gitlab_main -merge_request_diff_files: :gitlab_main -merge_request_diffs: :gitlab_main -merge_request_metrics: :gitlab_main -merge_request_predictions: :gitlab_main -merge_request_reviewers: :gitlab_main -merge_requests_closing_issues: :gitlab_main -merge_requests: :gitlab_main -merge_request_user_mentions: :gitlab_main -merge_trains: :gitlab_main -metrics_dashboard_annotations: :gitlab_main -metrics_users_starred_dashboards: :gitlab_main -milestone_releases: :gitlab_main -milestones: :gitlab_main -ml_candidates: :gitlab_main -ml_experiments: :gitlab_main -ml_candidate_metrics: :gitlab_main -ml_candidate_params: :gitlab_main -namespace_admin_notes: :gitlab_main -namespace_aggregation_schedules: :gitlab_main -namespace_bans: :gitlab_main -namespace_limits: :gitlab_main -namespace_package_settings: :gitlab_main -namespace_root_storage_statistics: :gitlab_main -namespace_ci_cd_settings: :gitlab_main -namespace_commit_emails: :gitlab_main -namespace_settings: :gitlab_main -namespace_details: :gitlab_main -namespaces: :gitlab_main -namespaces_sync_events: :gitlab_main -namespace_statistics: :gitlab_main -note_diff_files: :gitlab_main -notes: :gitlab_main -notification_settings: :gitlab_main -oauth_access_grants: :gitlab_main -oauth_access_tokens: :gitlab_main -oauth_applications: :gitlab_main -oauth_openid_requests: :gitlab_main -onboarding_progresses: :gitlab_main -operations_feature_flags_clients: :gitlab_main -operations_feature_flag_scopes: :gitlab_main -operations_feature_flags: :gitlab_main -operations_feature_flags_issues: :gitlab_main -operations_scopes: :gitlab_main -operations_strategies: :gitlab_main -operations_strategies_user_lists: :gitlab_main -operations_user_lists: :gitlab_main -p_ci_builds_metadata: :gitlab_ci -packages_build_infos: :gitlab_main -packages_cleanup_policies: :gitlab_main -packages_composer_cache_files: :gitlab_main -packages_composer_metadata: :gitlab_main -packages_conan_file_metadata: :gitlab_main -packages_conan_metadata: :gitlab_main -packages_debian_file_metadata: :gitlab_main -packages_debian_group_architectures: :gitlab_main -packages_debian_group_component_files: :gitlab_main -packages_debian_group_components: :gitlab_main -packages_debian_group_distribution_keys: :gitlab_main -packages_debian_group_distributions: :gitlab_main -packages_debian_project_architectures: :gitlab_main -packages_debian_project_component_files: :gitlab_main -packages_debian_project_components: :gitlab_main -packages_debian_project_distribution_keys: :gitlab_main -packages_debian_project_distributions: :gitlab_main -packages_debian_publications: :gitlab_main -packages_dependencies: :gitlab_main -packages_dependency_links: :gitlab_main -packages_events: :gitlab_main -packages_helm_file_metadata: :gitlab_main -packages_maven_metadata: :gitlab_main -packages_npm_metadata: :gitlab_main -packages_rpm_metadata: :gitlab_main -packages_nuget_dependency_link_metadata: :gitlab_main -packages_nuget_metadata: :gitlab_main -packages_package_file_build_infos: :gitlab_main -packages_package_files: :gitlab_main -packages_rpm_repository_files: :gitlab_main -packages_packages: :gitlab_main -packages_pypi_metadata: :gitlab_main -packages_rubygems_metadata: :gitlab_main -packages_tags: :gitlab_main -pages_deployments: :gitlab_main -pages_deployment_states: :gitlab_main -pages_domain_acme_orders: :gitlab_main -pages_domains: :gitlab_main -path_locks: :gitlab_main -personal_access_tokens: :gitlab_main -plan_limits: :gitlab_main -plans: :gitlab_main -pool_repositories: :gitlab_main -postgres_async_indexes: :gitlab_shared -postgres_autovacuum_activity: :gitlab_shared -postgres_constraints: :gitlab_shared -postgres_foreign_keys: :gitlab_shared -postgres_index_bloat_estimates: :gitlab_shared -postgres_indexes: :gitlab_shared -postgres_partitioned_tables: :gitlab_shared -postgres_partitions: :gitlab_shared -postgres_reindex_actions: :gitlab_shared -postgres_reindex_queued_actions: :gitlab_shared -product_analytics_events_experimental: :gitlab_main -programming_languages: :gitlab_main -project_access_tokens: :gitlab_main -project_alerting_settings: :gitlab_main -project_aliases: :gitlab_main -project_authorizations: :gitlab_main -project_auto_devops: :gitlab_main -project_build_artifacts_size_refreshes: :gitlab_main -project_ci_cd_settings: :gitlab_main -project_ci_feature_usages: :gitlab_main -project_compliance_framework_settings: :gitlab_main -project_custom_attributes: :gitlab_main -project_daily_statistics: :gitlab_main -project_deploy_tokens: :gitlab_main -project_error_tracking_settings: :gitlab_main -project_export_jobs: :gitlab_main -project_features: :gitlab_main -project_feature_usages: :gitlab_main -project_group_links: :gitlab_main -project_import_data: :gitlab_main -project_incident_management_settings: :gitlab_main -project_metrics_settings: :gitlab_main -project_mirror_data: :gitlab_main -project_pages_metadata: :gitlab_main -project_relation_export_uploads: :gitlab_main -project_relation_exports: :gitlab_main -project_repositories: :gitlab_main -project_repository_states: :gitlab_main -project_repository_storage_moves: :gitlab_main -project_security_settings: :gitlab_main -project_settings: :gitlab_main -projects: :gitlab_main -projects_sync_events: :gitlab_main -project_statistics: :gitlab_main -project_topics: :gitlab_main -project_wiki_repositories: :gitlab_main -project_wiki_repository_states: :gitlab_main -prometheus_alert_events: :gitlab_main -prometheus_alerts: :gitlab_main -prometheus_metrics: :gitlab_main -protected_branches: :gitlab_main -protected_branch_merge_access_levels: :gitlab_main -protected_branch_push_access_levels: :gitlab_main -protected_branch_unprotect_access_levels: :gitlab_main -protected_environment_approval_rules: :gitlab_main -protected_environment_deploy_access_levels: :gitlab_main -protected_environments: :gitlab_main -protected_tag_create_access_levels: :gitlab_main -protected_tags: :gitlab_main -push_event_payloads: :gitlab_main -push_rules: :gitlab_main -raw_usage_data: :gitlab_main -redirect_routes: :gitlab_main -related_epic_links: :gitlab_main -release_links: :gitlab_main -releases: :gitlab_main -remote_mirrors: :gitlab_main -repository_languages: :gitlab_main -required_code_owners_sections: :gitlab_main -requirements: :gitlab_main -requirements_management_test_reports: :gitlab_main -resource_iteration_events: :gitlab_main -resource_label_events: :gitlab_main -resource_milestone_events: :gitlab_main -resource_state_events: :gitlab_main -resource_weight_events: :gitlab_main -reviews: :gitlab_main -routes: :gitlab_main -saml_group_links: :gitlab_main -saml_providers: :gitlab_main -saved_replies: :gitlab_main -sbom_components: :gitlab_main -sbom_occurrences: :gitlab_main -sbom_component_versions: :gitlab_main -sbom_sources: :gitlab_main -sbom_vulnerable_component_versions: :gitlab_main -schema_migrations: :gitlab_internal -scim_identities: :gitlab_main -scim_oauth_access_tokens: :gitlab_main -security_findings: :gitlab_main -security_orchestration_policy_configurations: :gitlab_main -security_orchestration_policy_rule_schedules: :gitlab_main -security_scans: :gitlab_main -security_training_providers: :gitlab_main -security_trainings: :gitlab_main -self_managed_prometheus_alert_events: :gitlab_main -sent_notifications: :gitlab_main -sentry_issues: :gitlab_main -serverless_domain_cluster: :gitlab_main -service_desk_settings: :gitlab_main -shards: :gitlab_main -slack_integrations: :gitlab_main -smartcard_identities: :gitlab_main -snippet_repositories: :gitlab_main -snippet_repository_storage_moves: :gitlab_main -snippets: :gitlab_main -snippet_statistics: :gitlab_main -snippet_user_mentions: :gitlab_main -software_license_policies: :gitlab_main -software_licenses: :gitlab_main -spam_logs: :gitlab_main -sprints: :gitlab_main -ssh_signatures: :gitlab_main -status_check_responses: :gitlab_main -status_page_published_incidents: :gitlab_main -status_page_settings: :gitlab_main -subscriptions: :gitlab_main -suggestions: :gitlab_main -system_note_metadata: :gitlab_main -taggings: :gitlab_ci -tags: :gitlab_ci -term_agreements: :gitlab_main -terraform_states: :gitlab_main -terraform_state_versions: :gitlab_main -timelogs: :gitlab_main -timelog_categories: :gitlab_main -todos: :gitlab_main -token_with_ivs: :gitlab_main -topics: :gitlab_main -trending_projects: :gitlab_main -u2f_registrations: :gitlab_main -upcoming_reconciliations: :gitlab_main -uploads: :gitlab_main -upload_states: :gitlab_main -user_agent_details: :gitlab_main -user_callouts: :gitlab_main -user_canonical_emails: :gitlab_main -user_credit_card_validations: :gitlab_main -user_custom_attributes: :gitlab_main -user_details: :gitlab_main -user_follow_users: :gitlab_main -user_group_callouts: :gitlab_main -user_project_callouts: :gitlab_main -user_highest_roles: :gitlab_main -user_interacted_projects: :gitlab_main -user_phone_number_validations: :gitlab_main -user_permission_export_uploads: :gitlab_main -user_preferences: :gitlab_main -users: :gitlab_main -users_ops_dashboard_projects: :gitlab_main -users_security_dashboard_projects: :gitlab_main -users_star_projects: :gitlab_main -users_statistics: :gitlab_main -user_statuses: :gitlab_main -user_synced_attributes_metadata: :gitlab_main -verification_codes: :gitlab_main -vulnerabilities: :gitlab_main -vulnerability_advisories: :gitlab_main -vulnerability_exports: :gitlab_main -vulnerability_external_issue_links: :gitlab_main -vulnerability_feedback: :gitlab_main -vulnerability_finding_evidences: :gitlab_main -vulnerability_finding_links: :gitlab_main -vulnerability_finding_signatures: :gitlab_main -vulnerability_findings_remediations: :gitlab_main -vulnerability_flags: :gitlab_main -vulnerability_historical_statistics: :gitlab_main -vulnerability_identifiers: :gitlab_main -vulnerability_issue_links: :gitlab_main -vulnerability_merge_request_links: :gitlab_main -vulnerability_occurrence_identifiers: :gitlab_main -vulnerability_occurrence_pipelines: :gitlab_main -vulnerability_occurrences: :gitlab_main -vulnerability_reads: :gitlab_main -vulnerability_remediations: :gitlab_main -vulnerability_scanners: :gitlab_main -vulnerability_state_transitions: :gitlab_main -vulnerability_statistics: :gitlab_main -vulnerability_user_mentions: :gitlab_main -webauthn_registrations: :gitlab_main -web_hook_logs: :gitlab_main -web_hooks: :gitlab_main -wiki_page_meta: :gitlab_main -wiki_page_slugs: :gitlab_main -work_item_parent_links: :gitlab_main -work_item_types: :gitlab_main -x509_certificates: :gitlab_main -x509_commit_signatures: :gitlab_main -x509_issuers: :gitlab_main -zentao_tracker_data: :gitlab_main -# dingtalk_tracker_data JiHu-specific, see https://jihulab.com/gitlab-cn/gitlab/-/merge_requests/417 -dingtalk_tracker_data: :gitlab_main -zoom_meetings: :gitlab_main -batched_background_migration_job_transition_logs: :gitlab_shared -user_namespace_callouts: :gitlab_main diff --git a/lib/gitlab/database/load_balancing/connection_proxy.rb b/lib/gitlab/database/load_balancing/connection_proxy.rb index 8799f8d8af8..f0343f9d8b5 100644 --- a/lib/gitlab/database/load_balancing/connection_proxy.rb +++ b/lib/gitlab/database/load_balancing/connection_proxy.rb @@ -95,7 +95,7 @@ module Gitlab # name - The name of the method to call on a connection object. def read_using_load_balancer(...) if current_session.use_primary? && - !current_session.use_replicas_for_read_queries? + !current_session.use_replicas_for_read_queries? @load_balancer.read_write do |connection| connection.send(...) end diff --git a/lib/gitlab/database/load_balancing/service_discovery.rb b/lib/gitlab/database/load_balancing/service_discovery.rb index 52a9e8798d4..3295301a2d7 100644 --- a/lib/gitlab/database/load_balancing/service_discovery.rb +++ b/lib/gitlab/database/load_balancing/service_discovery.rb @@ -125,13 +125,6 @@ module Gitlab old_host_list_length: current.length ) replace_hosts(from_dns) - else - ::Gitlab::Database::LoadBalancing::Logger.info( - event: :host_list_unchanged, - message: "Unchanged host list for service discovery", - host_list_length: from_dns.length, - old_host_list_length: current.length - ) end interval diff --git a/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb b/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb index 13afbd8fd37..619f11ae890 100644 --- a/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb +++ b/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb @@ -57,7 +57,7 @@ module Gitlab if uses_primary? load_balancer.primary_write_location else - load_balancer.host.database_replica_location + load_balancer.host&.database_replica_location || load_balancer.primary_write_location end end diff --git a/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb index 737852d5ccb..f7b8d2514ba 100644 --- a/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb +++ b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb @@ -97,14 +97,8 @@ module Gitlab end def databases_in_sync?(wal_locations) - locations = if Feature.enabled?(:indifferent_wal_location_keys) - wal_locations.with_indifferent_access - else - wal_locations - end - ::Gitlab::Database::LoadBalancing.each_load_balancer.all? do |lb| - if (location = locations[lb.name]) + if (location = wal_locations.with_indifferent_access[lb.name]) lb.select_up_to_date_host(location) else # If there's no entry for a load balancer it means the Sidekiq diff --git a/lib/gitlab/database/lock_writes_manager.rb b/lib/gitlab/database/lock_writes_manager.rb index 2594ee04b35..e3ae2892668 100644 --- a/lib/gitlab/database/lock_writes_manager.rb +++ b/lib/gitlab/database/lock_writes_manager.rb @@ -10,18 +10,34 @@ module Gitlab # See https://www.postgresql.org/message-id/16934.1568989957%40sss.pgh.pa.us EXPECTED_TRIGGER_RECORD_COUNT = 3 + def self.tables_to_lock(connection) + Gitlab::Database::GitlabSchema.tables_to_schema.each do |table_name, schema_name| + yield table_name, schema_name + end + + Gitlab::Database::SharedModel.using_connection(connection) do + Postgresql::DetachedPartition.find_each do |detached_partition| + yield detached_partition.fully_qualified_table_name, detached_partition.table_schema + end + end + end + def initialize(table_name:, connection:, database_name:, logger: nil, dry_run: false) @table_name = table_name @connection = connection @database_name = database_name @logger = logger @dry_run = dry_run + + @table_name_without_schema = ActiveRecord::ConnectionAdapters::PostgreSQL::Utils + .extract_schema_qualified_name(table_name) + .identifier end def table_locked_for_writes?(table_name) query = <<~SQL SELECT COUNT(*) from information_schema.triggers - WHERE event_object_table = '#{table_name}' + WHERE event_object_table = '#{table_name_without_schema}' AND trigger_name = '#{write_trigger_name(table_name)}' SQL @@ -56,7 +72,7 @@ module Gitlab private - attr_reader :table_name, :connection, :database_name, :logger, :dry_run + attr_reader :table_name, :connection, :database_name, :logger, :dry_run, :table_name_without_schema def execute_sql_statement(sql) if dry_run @@ -99,7 +115,7 @@ module Gitlab end def write_trigger_name(table_name) - "gitlab_schema_write_trigger_for_#{table_name}" + "gitlab_schema_write_trigger_for_#{table_name_without_schema}" end end end diff --git a/lib/gitlab/database/migration.rb b/lib/gitlab/database/migration.rb index ab8b6988c3d..4d38920f571 100644 --- a/lib/gitlab/database/migration.rb +++ b/lib/gitlab/database/migration.rb @@ -51,6 +51,10 @@ module Gitlab include Gitlab::Database::MigrationHelpers::RestrictGitlabSchema end + class V2_1 < V2_0 # rubocop:disable Naming/ClassAndModuleCamelCase + include Gitlab::Database::MigrationHelpers::AutomaticLockWritesOnTables + end + def self.[](version) version = version.to_s name = "V#{version.tr('.', '_')}" @@ -61,7 +65,7 @@ module Gitlab # The current version to be used in new migrations def self.current_version - 2.0 + 2.1 end end end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 16416dd2507..4858a96c173 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -10,6 +10,7 @@ module Gitlab include Migrations::TimeoutHelpers include Migrations::ConstraintsHelpers include Migrations::ExtensionHelpers + include Migrations::SidekiqHelpers include DynamicModelHelpers include RenameTableHelpers include AsyncIndexes::MigrationHelpers @@ -497,17 +498,6 @@ module Gitlab end end - # Adds a column with a default value without locking an entire table. - # - # @deprecated With PostgreSQL 11, adding columns with a default does not lead to a table rewrite anymore. - # As such, this method is not needed anymore and the default `add_column` helper should be used. - # This helper is subject to be removed in a >13.0 release. - def add_column_with_default(table, column, type, default:, limit: nil, allow_null: false) - raise 'Deprecated: add_column_with_default does not support being passed blocks anymore' if block_given? - - add_column(table, column, type, default: default, limit: limit, null: allow_null) - end - # Renames a column without requiring downtime. # # Concurrent renames work by using database triggers to ensure both the @@ -1027,38 +1017,6 @@ module Gitlab rescue ArgumentError end - # Remove any instances of deprecated job classes lingering in queues. - # - # rubocop:disable Cop/SidekiqApiUsage - def sidekiq_remove_jobs(job_klass:) - Sidekiq::Queue.new(job_klass.queue).each do |job| - job.delete if job.klass == job_klass.to_s - end - - Sidekiq::RetrySet.new.each do |retri| - retri.delete if retri.klass == job_klass.to_s - end - - Sidekiq::ScheduledSet.new.each do |scheduled| - scheduled.delete if scheduled.klass == job_klass.to_s - end - end - # rubocop:enable Cop/SidekiqApiUsage - - def sidekiq_queue_migrate(queue_from, to:) - while sidekiq_queue_length(queue_from) > 0 - Sidekiq.redis do |conn| - conn.rpoplpush "queue:#{queue_from}", "queue:#{to}" - end - end - end - - def sidekiq_queue_length(queue_name) - Sidekiq.redis do |conn| - conn.llen("queue:#{queue_name}") - end - end - def check_trigger_permissions!(table) unless Grant.create_and_execute_trigger?(table) dbname = ApplicationRecord.database.database_name diff --git a/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables.rb b/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables.rb new file mode 100644 index 00000000000..0aa4b0d01c4 --- /dev/null +++ b/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module MigrationHelpers + module AutomaticLockWritesOnTables + extend ActiveSupport::Concern + + included do + class_attribute :skip_automatic_lock_on_writes + end + + def exec_migration(connection, direction) + return super if %w[main ci].exclude?(Gitlab::Database.db_config_name(connection)) + return super if automatic_lock_on_writes_disabled? + + # This compares the tables only on the `public` schema. Partitions are not affected + tables = connection.tables + super + new_tables = connection.tables - tables + + new_tables.each do |table_name| + lock_writes_on_table(connection, table_name) if should_lock_writes_on_table?(table_name) + end + end + + private + + def automatic_lock_on_writes_disabled? + # Feature flags are set on the main database, see tables features/feature_gates. + # That is why we switch the ActiveRecord::Base.connection temporarily here back to the 'main' database + # for the cases when the migration is targeting another database, like the 'ci' database. + with_restored_connection_stack do |_| + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.with_suppressed do + skip_automatic_lock_on_writes || + Gitlab::Utils.to_boolean(ENV['SKIP_AUTOMATIC_LOCK_ON_WRITES']) || + Feature.disabled?(:automatic_lock_writes_on_table, type: :ops) + end + end + end + + def should_lock_writes_on_table?(table_name) + # currently gitlab_schema represents only present existing tables, this is workaround for deleted tables + # that should be skipped as they will be removed in a future migration. + return false if Gitlab::Database::GitlabSchema::DELETED_TABLES[table_name] + + table_schema = Gitlab::Database::GitlabSchema.table_schema(table_name.to_s, undefined: false) + + if table_schema.nil? + error_message = <<~ERROR + No gitlab_schema is defined for the table #{table_name}. Please consider + adding it to the database dictionary. + More info: https://docs.gitlab.com/ee/development/database/database_dictionary.html + ERROR + raise error_message + end + + return false unless %i[gitlab_main gitlab_ci].include?(table_schema) + + Gitlab::Database.gitlab_schemas_for_connection(connection).exclude?(table_schema) + end + + def lock_writes_on_table(connection, table_name) + database_name = Gitlab::Database.db_config_name(connection) + LockWritesManager.new( + table_name: table_name, + connection: connection, + database_name: database_name, + logger: Logger.new($stdout) + ).lock_writes + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/batched_migration_last_id.rb b/lib/gitlab/database/migrations/batched_migration_last_id.rb new file mode 100644 index 00000000000..c77a2e9a375 --- /dev/null +++ b/lib/gitlab/database/migrations/batched_migration_last_id.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + class BatchedMigrationLastId + FILE_NAME = 'last-batched-background-migration-id.txt' + + def initialize(connection, base_dir) + @connection = connection + @base_dir = base_dir + end + + def store + File.open(file_path, 'wb') { |file| file.write(last_background_migration_id) } + end + + # Reads the last id from the file + # + # @info casts the file content into an +Integer+. + # Casts any unexpected content to +nil+ + # + # @example + # Integer('4', exception: false) # => 4 + # Integer('', exception: false) # => nil + # + # @return [Integer, nil] + def read + return unless File.exist?(file_path) + + Integer(File.read(file_path).presence, exception: false) + end + + private + + attr_reader :connection, :base_dir + + def file_path + @file_path ||= base_dir.join(FILE_NAME) + end + + def last_background_migration_id + Gitlab::Database::SharedModel.using_connection(connection) do + Gitlab::Database::BackgroundMigration::BatchedMigration.maximum(:id) + end + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/runner.rb b/lib/gitlab/database/migrations/runner.rb index 27b161419b2..ed55081c9ab 100644 --- a/lib/gitlab/database/migrations/runner.rb +++ b/lib/gitlab/database/migrations/runner.rb @@ -29,16 +29,14 @@ module Gitlab def batched_background_migrations(for_database:, legacy_mode: false) runner = nil - result_dir = if legacy_mode - BASE_RESULT_DIR.join('background_migrations') - else - BASE_RESULT_DIR.join(for_database.to_s, 'background_migrations') - end + result_dir = background_migrations_dir(for_database, legacy_mode) # Only one loop iteration since we pass `only:` here Gitlab::Database::EachDatabase.each_database_connection(only: for_database) do |connection| + from_id = batched_migrations_last_id(for_database).read + runner = Gitlab::Database::Migrations::TestBatchedBackgroundRunner - .new(result_dir: result_dir, connection: connection) + .new(result_dir: result_dir, connection: connection, from_id: from_id) end runner @@ -66,6 +64,18 @@ module Gitlab end # rubocop:enable Database/MultipleDatabases + def batched_migrations_last_id(for_database) + runner = nil + base_dir = background_migrations_dir(for_database, false) + + Gitlab::Database::EachDatabase.each_database_connection(only: for_database) do |connection| + runner = Gitlab::Database::Migrations::BatchedMigrationLastId + .new(connection, base_dir) + end + + runner + end + private def migrations_for_up(database) @@ -90,6 +100,12 @@ module Gitlab existing_versions.include?(migration.version) && versions_this_branch.include?(migration.version) end end + + def background_migrations_dir(db, legacy_mode) + return BASE_RESULT_DIR.join('background_migrations') if legacy_mode + + BASE_RESULT_DIR.join(db.to_s, 'background_migrations') + end end attr_reader :direction, :result_dir, :migrations diff --git a/lib/gitlab/database/migrations/sidekiq_helpers.rb b/lib/gitlab/database/migrations/sidekiq_helpers.rb new file mode 100644 index 00000000000..c536b33bbdf --- /dev/null +++ b/lib/gitlab/database/migrations/sidekiq_helpers.rb @@ -0,0 +1,112 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + # rubocop:disable Cop/SidekiqApiUsage + # rubocop:disable Cop/SidekiqRedisCall + module SidekiqHelpers + # Constants for default sidekiq_remove_jobs values + DEFAULT_MAX_ATTEMPTS = 5 + DEFAULT_TIMES_IN_A_ROW = 2 + + # Probabilistically removes job_klasses from their specific queues, the + # retry set and the scheduled set. + # + # If jobs are still being processed at the same time, then there is a + # small chance it will not remove all instances of job_klass. To + # minimize this risk, it repeatedly removes matching jobs from each + # until nothing is removed twice in a row. + # + # Before calling this method, you should make sure that job_klass is no + # longer being scheduled within the running application. + def sidekiq_remove_jobs( + job_klasses:, + times_in_a_row: DEFAULT_TIMES_IN_A_ROW, + max_attempts: DEFAULT_MAX_ATTEMPTS + ) + + kwargs = { times_in_a_row: times_in_a_row, max_attempts: max_attempts } + + job_klasses_queues = job_klasses + .select { |job_klass| job_klass.to_s.safe_constantize.present? } + .map { |job_klass| job_klass.safe_constantize.queue } + .uniq + + job_klasses_queues.each do |queue| + delete_jobs_for( + set: Sidekiq::Queue.new(queue), + job_klasses: job_klasses, + kwargs: kwargs + ) + end + + delete_jobs_for( + set: Sidekiq::RetrySet.new, + kwargs: kwargs, + job_klasses: job_klasses + ) + + delete_jobs_for( + set: Sidekiq::ScheduledSet.new, + kwargs: kwargs, + job_klasses: job_klasses + ) + end + + def sidekiq_queue_migrate(queue_from, to:) + while sidekiq_queue_length(queue_from) > 0 + Sidekiq.redis do |conn| + conn.rpoplpush "queue:#{queue_from}", "queue:#{to}" + end + end + end + + def sidekiq_queue_length(queue_name) + Sidekiq.redis do |conn| + conn.llen("queue:#{queue_name}") + end + end + + private + + # Handle the "jobs deleted" tracking that is needed in order to track + # whether a job was deleted or not. + def delete_jobs_for(set:, kwargs:, job_klasses:) + until_equal_to(0, **kwargs) do + set.count do |job| + job_klasses.include?(job.klass) && job.delete + end + end + end + + # Control how many times in a row you want to see a job deleted 0 + # times. The idea is that if you see 0 jobs deleted x number of times + # in a row you've *likely* covered the case in which the queue was + # mutating while this was running. + def until_equal_to(target, times_in_a_row:, max_attempts:) + streak = 0 + + result = { attempts: 0, success: false } + + 1.upto(max_attempts) do |current_attempt| + # yield's return value is a count of "jobs_deleted" + if yield == target + streak += 1 + elsif streak > 0 + streak = 0 + end + + result[:attempts] = current_attempt + result[:success] = streak == times_in_a_row + + break if result[:success] + end + result + end + end + # rubocop:enable Cop/SidekiqApiUsage + # rubocop:enable Cop/SidekiqRedisCall + end + end +end diff --git a/lib/gitlab/database/migrations/test_batched_background_runner.rb b/lib/gitlab/database/migrations/test_batched_background_runner.rb index 46855ca1921..a16103f452c 100644 --- a/lib/gitlab/database/migrations/test_batched_background_runner.rb +++ b/lib/gitlab/database/migrations/test_batched_background_runner.rb @@ -6,16 +6,17 @@ module Gitlab class TestBatchedBackgroundRunner < BaseBackgroundRunner include Gitlab::Database::DynamicModelHelpers - def initialize(result_dir:, connection:) + def initialize(result_dir:, connection:, from_id:) super(result_dir: result_dir, connection: connection) @connection = connection + @from_id = from_id end def jobs_by_migration_name Gitlab::Database::SharedModel.using_connection(connection) do Gitlab::Database::BackgroundMigration::BatchedMigration .executable - .created_after(3.hours.ago) # Simple way to exclude migrations already running before migration testing + .where('id > ?', from_id) .to_h do |migration| batching_strategy = migration.batch_class.new(connection: connection) @@ -102,6 +103,10 @@ module Gitlab end end end + + private + + attr_reader :from_id end end end diff --git a/lib/gitlab/database/obsolete_ignored_columns.rb b/lib/gitlab/database/obsolete_ignored_columns.rb index ad5473f1b74..2b88ab12380 100644 --- a/lib/gitlab/database/obsolete_ignored_columns.rb +++ b/lib/gitlab/database/obsolete_ignored_columns.rb @@ -23,8 +23,8 @@ module Gitlab private def ignored_columns_safe_to_remove_for(klass) - ignores = ignored_and_not_present(klass).each_with_object({}) do |col, h| - h[col] = klass.ignored_columns_details[col.to_sym] + ignores = ignored_and_not_present(klass).index_with do |col| + klass.ignored_columns_details[col.to_sym] end ignores.select { |_, i| i&.safe_to_remove? } diff --git a/lib/gitlab/database/partitioning/single_numeric_list_partition.rb b/lib/gitlab/database/partitioning/single_numeric_list_partition.rb index 4e38eea963b..fd99062974c 100644 --- a/lib/gitlab/database/partitioning/single_numeric_list_partition.rb +++ b/lib/gitlab/database/partitioning/single_numeric_list_partition.rb @@ -19,7 +19,7 @@ module Gitlab attr_reader :table, :value - def initialize(table, value, partition_name: nil ) + def initialize(table, value, partition_name: nil) @table = table @value = value @partition_name = partition_name diff --git a/lib/gitlab/database/postgres_hll/buckets.rb b/lib/gitlab/database/postgres_hll/buckets.rb index cbc9544d905..3f64eee030e 100644 --- a/lib/gitlab/database/postgres_hll/buckets.rb +++ b/lib/gitlab/database/postgres_hll/buckets.rb @@ -61,7 +61,7 @@ module Gitlab num_uniques = ( ((TOTAL_BUCKETS**2) * (0.7213 / (1 + 1.079 / TOTAL_BUCKETS))) / - (num_zero_buckets + buckets.values.sum { |bucket_hash| 2**(-1 * bucket_hash) } ) + (num_zero_buckets + buckets.values.sum { |bucket_hash| 2**(-1 * bucket_hash) }) ).to_i if num_zero_buckets > 0 && num_uniques < 2.5 * TOTAL_BUCKETS diff --git a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb index 3b1751c863d..dd10e0d7992 100644 --- a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb +++ b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb @@ -165,8 +165,8 @@ module Gitlab def self.in_factory_bot_create? Rails.env.test? && caller_locations.any? do |l| l.path.end_with?('lib/factory_bot/evaluation.rb') && l.label == 'create' || - l.path.end_with?('lib/factory_bot/strategy/create.rb') || - l.path.end_with?('shoulda/matchers/active_record/validate_uniqueness_of_matcher.rb') && l.label == 'create_existing_record' + l.path.end_with?('lib/factory_bot/strategy/create.rb') || + l.path.end_with?('shoulda/matchers/active_record/validate_uniqueness_of_matcher.rb') && l.label == 'create_existing_record' end end end diff --git a/lib/gitlab/database/query_analyzers/query_recorder.rb b/lib/gitlab/database/query_analyzers/query_recorder.rb index 88fe829c3d2..b54f3442512 100644 --- a/lib/gitlab/database/query_analyzers/query_recorder.rb +++ b/lib/gitlab/database/query_analyzers/query_recorder.rb @@ -4,7 +4,7 @@ module Gitlab module Database module QueryAnalyzers class QueryRecorder < Base - LOG_FILE = 'rspec/query_recorder.ndjson' + LOG_PATH = 'query_recorder/' class << self def raw? @@ -12,8 +12,9 @@ module Gitlab end def enabled? - # Only enable QueryRecorder in CI - ENV['CI'].present? + # Only enable QueryRecorder in CI on database MRs or default branch + ENV['CI_MERGE_REQUEST_LABELS']&.include?('database') || + (ENV['CI_COMMIT_REF_NAME'].present? && ENV['CI_COMMIT_REF_NAME'] == ENV['CI_DEFAULT_BRANCH']) end def analyze(sql) @@ -24,11 +25,14 @@ module Gitlab log_query(payload) end + def log_file + Rails.root.join(LOG_PATH, "#{ENV.fetch('CI_JOB_NAME_SLUG', 'rspec')}.ndjson") + end + private def log_query(payload) - log_path = Rails.root.join(LOG_FILE) - log_dir = File.dirname(log_path) + log_dir = Rails.root.join(LOG_PATH) # Create log directory if it does not exist since it is only created # ahead of time by certain CI jobs @@ -36,7 +40,7 @@ module Gitlab log_line = "#{Gitlab::Json.dump(payload)}\n" - File.write(log_path, log_line, mode: 'a') + File.write(log_file, log_line, mode: 'a') end end end diff --git a/lib/gitlab/database/schema_cache_with_renamed_table.rb b/lib/gitlab/database/schema_cache_with_renamed_table.rb index 74900dc0d26..6da76803f7c 100644 --- a/lib/gitlab/database/schema_cache_with_renamed_table.rb +++ b/lib/gitlab/database/schema_cache_with_renamed_table.rb @@ -40,10 +40,8 @@ module Gitlab end def renamed_tables_cache - @renamed_tables ||= begin - Gitlab::Database::TABLES_TO_BE_RENAMED.select do |old_name, new_name| - connection.view_exists?(old_name) - end + @renamed_tables ||= Gitlab::Database::TABLES_TO_BE_RENAMED.select do |old_name, new_name| + connection.view_exists?(old_name) end end diff --git a/lib/gitlab/database/schema_cleaner.rb b/lib/gitlab/database/schema_cleaner.rb index c3cdcf1450d..2c8d0a4eb6d 100644 --- a/lib/gitlab/database/schema_cleaner.rb +++ b/lib/gitlab/database/schema_cleaner.rb @@ -25,7 +25,23 @@ module Gitlab # The intention here is to not introduce an assumption about the standard schema, # unless we have a good reason to do so. structure.gsub!(/public\.(\w+)/, '\1') - structure.gsub!(/CREATE EXTENSION IF NOT EXISTS (\w+) WITH SCHEMA public;/, 'CREATE EXTENSION IF NOT EXISTS \1;') + structure.gsub!( + /CREATE EXTENSION IF NOT EXISTS (\w+) WITH SCHEMA public;/, + 'CREATE EXTENSION IF NOT EXISTS \1;' + ) + + # Table lock-writes triggers should not be added to the schema + # These triggers are added by the rake task gitlab:db:lock_writes for a decomposed database. + structure.gsub!( + %r{ + ^CREATE.TRIGGER.gitlab_schema_write_trigger_\w+ + \s + BEFORE.INSERT.OR.DELETE.OR.UPDATE.OR.TRUNCATE.ON.\w+ + \s + FOR.EACH.STATEMENT.EXECUTE.FUNCTION.gitlab_schema_prevent_write\(\);$ + }x, + '' + ) structure.gsub!(/\n{3,}/, "\n\n") diff --git a/lib/gitlab/database/tables_sorted_by_foreign_keys.rb b/lib/gitlab/database/tables_sorted_by_foreign_keys.rb index 9f096904d31..b2a7f5442e9 100644 --- a/lib/gitlab/database/tables_sorted_by_foreign_keys.rb +++ b/lib/gitlab/database/tables_sorted_by_foreign_keys.rb @@ -26,15 +26,32 @@ module Gitlab # it maps the tables to the tables that depend on it def tables_dependencies - @tables.to_h do |table_name| - [table_name, all_foreign_keys[table_name]&.map(&:from_table).to_a] + @tables.index_with do |table_name| + all_foreign_keys[table_name] end end def all_foreign_keys - @all_foreign_keys ||= @tables.flat_map do |table_name| - @connection.foreign_keys(table_name) - end.group_by(&:to_table) + @all_foreign_keys ||= @tables.each_with_object(Hash.new { |h, k| h[k] = [] }) do |table, hash| + foreign_keys_for(table).each do |fk| + hash[fk.to_table] << table + end + end + end + + def foreign_keys_for(table) + # Detached partitions like gitlab_partitions_dynamic._test_gitlab_partition_20220101 + # store their foreign keys in the public schema. + # + # See spec/lib/gitlab/database/tables_sorted_by_foreign_keys_spec.rb + # for an example + name = ActiveRecord::ConnectionAdapters::PostgreSQL::Utils.extract_schema_qualified_name(table) + + if name.schema == ::Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA.to_s + @connection.foreign_keys(name.identifier) + else + @connection.foreign_keys(table) + end end end end diff --git a/lib/gitlab/database/tables_truncate.rb b/lib/gitlab/database/tables_truncate.rb index 8380bf23899..807ecdb862a 100644 --- a/lib/gitlab/database/tables_truncate.rb +++ b/lib/gitlab/database/tables_truncate.rb @@ -19,24 +19,32 @@ module Gitlab logger&.info "DRY RUN:" if dry_run - connection = Gitlab::Database.database_base_models[database_name].connection - schemas_for_connection = Gitlab::Database.gitlab_schemas_for_connection(connection) tables_to_truncate = Gitlab::Database::GitlabSchema.tables_to_schema.reject do |_, schema_name| - (GITLAB_SCHEMAS_TO_IGNORE.union(schemas_for_connection)).include?(schema_name) + GITLAB_SCHEMAS_TO_IGNORE.union(schemas_for_connection).include?(schema_name) end.keys + Gitlab::Database::SharedModel.using_connection(connection) do + Postgresql::DetachedPartition.find_each do |detached_partition| + next if GITLAB_SCHEMAS_TO_IGNORE.union(schemas_for_connection).include?(detached_partition.table_schema) + + tables_to_truncate << detached_partition.fully_qualified_table_name + end + end + tables_sorted = Gitlab::Database::TablesSortedByForeignKeys.new(connection, tables_to_truncate).execute # Checking if all the tables have the write-lock triggers # to make sure we are deleting the right tables on the right database. tables_sorted.flatten.each do |table_name| - query = <<~SQL - SELECT COUNT(*) from information_schema.triggers - WHERE event_object_table = '#{table_name}' - AND trigger_name = 'gitlab_schema_write_trigger_for_#{table_name}' - SQL - - if connection.select_value(query) == 0 + lock_writes_manager = Gitlab::Database::LockWritesManager.new( + table_name: table_name, + connection: connection, + database_name: database_name, + logger: logger, + dry_run: dry_run + ) + + unless lock_writes_manager.table_locked_for_writes?(table_name) raise "Table '#{table_name}' is not locked for writes. Run the rake task gitlab:db:lock_writes first" end end @@ -51,18 +59,26 @@ module Gitlab # min_batch_size is the minimum number of new tables to truncate at each stage. # But in each stage we have also have to truncate the already truncated tables in the previous stages logger&.info "Truncating legacy tables for the database #{database_name}" - truncate_tables_in_batches(connection, tables_sorted, min_batch_size) + truncate_tables_in_batches(tables_sorted) end private attr_accessor :database_name, :min_batch_size, :logger, :dry_run, :until_table - def truncate_tables_in_batches(connection, tables_sorted, min_batch_size) + def connection + @connection ||= Gitlab::Database.database_base_models[database_name].connection + end + + def truncate_tables_in_batches(tables_sorted) truncated_tables = [] tables_sorted.flatten.each do |table| - sql_statement = "SELECT set_config('lock_writes.#{table}', 'false', false)" + table_name_without_schema = ActiveRecord::ConnectionAdapters::PostgreSQL::Utils + .extract_schema_qualified_name(table) + .identifier + + sql_statement = "SELECT set_config('lock_writes.#{table_name_without_schema}', 'false', false)" logger&.info(sql_statement) connection.execute(sql_statement) unless dry_run end diff --git a/lib/gitlab/database/type/indifferent_jsonb.rb b/lib/gitlab/database/type/indifferent_jsonb.rb new file mode 100644 index 00000000000..69bbcb383ba --- /dev/null +++ b/lib/gitlab/database/type/indifferent_jsonb.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Type + # Extends Rails' Jsonb data type to deserialize it into indifferent access Hash. + # + # Example: + # + # class SomeModel < ApplicationRecord + # # some_model.a_field is of type `jsonb` + # attribute :a_field, :ind_jsonb + # end + class IndifferentJsonb < ::ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Jsonb + def type + :ind_jsonb + end + + def deserialize(value) + data = super + return unless data + + ::Gitlab::Utils.deep_indifferent_access(data) + end + end + end + end +end diff --git a/lib/gitlab/database_importers/work_items/hierarchy_restrictions_importer.rb b/lib/gitlab/database_importers/work_items/hierarchy_restrictions_importer.rb new file mode 100644 index 00000000000..1181c259a5c --- /dev/null +++ b/lib/gitlab/database_importers/work_items/hierarchy_restrictions_importer.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module DatabaseImporters + module WorkItems + module HierarchyRestrictionsImporter + def self.upsert_restrictions + objective = find_or_create_type(::WorkItems::Type::TYPE_NAMES[:objective]) + key_result = find_or_create_type(::WorkItems::Type::TYPE_NAMES[:key_result]) + issue = find_or_create_type(::WorkItems::Type::TYPE_NAMES[:issue]) + task = find_or_create_type(::WorkItems::Type::TYPE_NAMES[:task]) + incident = find_or_create_type(::WorkItems::Type::TYPE_NAMES[:incident]) + + restrictions = [ + { parent_type_id: objective.id, child_type_id: objective.id, maximum_depth: 9 }, + { parent_type_id: objective.id, child_type_id: key_result.id, maximum_depth: 1 }, + { parent_type_id: issue.id, child_type_id: task.id, maximum_depth: 1 }, + { parent_type_id: incident.id, child_type_id: task.id, maximum_depth: 1 } + ] + + ::WorkItems::HierarchyRestriction.upsert_all( + restrictions, + unique_by: :index_work_item_hierarchy_restrictions_on_parent_and_child + ) + end + + def self.find_or_create_type(name) + type = ::WorkItems::Type.find_by_name_and_namespace_id(name, nil) + return type if type + + Gitlab::DatabaseImporters::WorkItems::BaseTypeImporter.upsert_types + ::WorkItems::Type.find_by_name_and_namespace_id(name, nil) + end + end + end + end +end diff --git a/lib/gitlab/diff/file_collection/compare.rb b/lib/gitlab/diff/file_collection/compare.rb index 6d8395d048d..df0d71f3db2 100644 --- a/lib/gitlab/diff/file_collection/compare.rb +++ b/lib/gitlab/diff/file_collection/compare.rb @@ -4,7 +4,15 @@ module Gitlab module Diff module FileCollection class Compare < Base + delegate :limit_value, :current_page, :next_page, :prev_page, :total_count, :total_pages, to: :@pagination + def initialize(compare, project:, diff_options:, diff_refs: nil) + @pagination = Gitlab::PaginationDelegate.new( + page: diff_options&.delete(:page), + per_page: diff_options&.delete(:per_page), + count: diff_options&.delete(:count) + ) + super(compare, project: project, diff_options: diff_options, diff --git a/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb b/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb index 0a601bde612..56027d6a4de 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb @@ -10,6 +10,8 @@ module Gitlab # separate file keys (https://gitlab.com/gitlab-org/gitlab/issues/30550). # class MergeRequestDiffBatch < MergeRequestDiffBase + include PaginatedDiffs + DEFAULT_BATCH_PAGE = 1 DEFAULT_BATCH_SIZE = 30 @@ -25,41 +27,8 @@ module Gitlab } end - override :diffs - def diffs - strong_memoize(:diffs) do - @merge_request_diff.opening_external_diff do - # Avoiding any extra queries. - collection = @paginated_collection.to_a - - # The offset collection and calculation is required so that we - # know how much has been loaded in previous batches, collapsing - # the current paginated set accordingly (collection limit calculation). - # See: https://docs.gitlab.com/ee/development/diffs.html#diff-collection-limits - # - offset_index = collection.first&.index - options = diff_options.dup - - collection = - if offset_index && offset_index > 0 - offset_collection = relation.limit(offset_index) # rubocop:disable CodeReuse/ActiveRecord - options[:offset_index] = offset_index - offset_collection + collection - else - collection - end - - Gitlab::Git::DiffCollection.new(collection.map(&:to_hash), options) - end - end - end - private - def relation - @merge_request_diff.merge_request_diff_files - end - # rubocop: disable CodeReuse/ActiveRecord def load_paginated_collection(batch_page, batch_size, diff_options) batch_page ||= DEFAULT_BATCH_PAGE diff --git a/lib/gitlab/diff/file_collection/paginated_diffs.rb b/lib/gitlab/diff/file_collection/paginated_diffs.rb new file mode 100644 index 00000000000..63c186affe9 --- /dev/null +++ b/lib/gitlab/diff/file_collection/paginated_diffs.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module Diff + module FileCollection + module PaginatedDiffs + include Gitlab::Utils::StrongMemoize + extend ::Gitlab::Utils::Override + + override :diffs + def diffs + merge_request_diff.opening_external_diff do + # Avoiding any extra queries. + collection = paginated_collection.to_a + + # The offset collection and calculation is required so that we + # know how much has been loaded in previous batches, collapsing + # the current paginated set accordingly (collection limit calculation). + # See: https://docs.gitlab.com/ee/development/diffs.html#diff-collection-limits + # + offset_index = collection.first&.index + options = diff_options.dup + + collection = + if offset_index && offset_index > 0 + offset_collection = relation.limit(offset_index) # rubocop:disable CodeReuse/ActiveRecord + options[:offset_index] = offset_index + offset_collection + collection + else + collection + end + + Gitlab::Git::DiffCollection.new(collection.map(&:to_hash), options) + end + end + strong_memoize_attr :diffs + + private + + attr_reader :merge_request_diff, :paginated_collection + + def relation + merge_request_diff.merge_request_diff_files + end + end + end + end +end diff --git a/lib/gitlab/diff/file_collection/paginated_merge_request_diff.rb b/lib/gitlab/diff/file_collection/paginated_merge_request_diff.rb new file mode 100644 index 00000000000..37abad81305 --- /dev/null +++ b/lib/gitlab/diff/file_collection/paginated_merge_request_diff.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module Diff + module FileCollection + # Builds a traditional paginated diff file collection using Kaminari + # `per` and `per_page` which is different from how `MergeRequestDiffBatch` + # works (e.g. supports gradual loading). + class PaginatedMergeRequestDiff < MergeRequestDiffBase + include PaginatedDiffs + + DEFAULT_PAGE = 1 + DEFAULT_PER_PAGE = 30 + + delegate :limit_value, :current_page, :next_page, :prev_page, :total_count, + :total_pages, to: :paginated_collection + + def initialize(merge_request_diff, page, per_page) + super(merge_request_diff, diff_options: nil) + + @paginated_collection = load_paginated_collection(page, per_page) + end + + private + + def load_paginated_collection(page, per_page) + page ||= DEFAULT_PAGE + per_page ||= DEFAULT_PER_PAGE + + relation.page(page).per([per_page.to_i, DEFAULT_PER_PAGE].min) + end + end + end + end +end diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb index 924c28e3db5..b29c75ed467 100644 --- a/lib/gitlab/diff/parser.rb +++ b/lib/gitlab/diff/parser.rb @@ -73,7 +73,7 @@ module Gitlab private def filename?(line) - line.start_with?( '--- /dev/null', '+++ /dev/null', '--- a', '+++ b', + line.start_with?('--- /dev/null', '+++ /dev/null', '--- a', '+++ b', '+++ a', # The line will start with `+++ a` in the reverse diff of an orphan commit '--- /tmp/diffy', '+++ /tmp/diffy') end diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index 1e03f5d17ee..32794a6c99d 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -192,7 +192,7 @@ module Gitlab auto_submitted = mail.header['Auto-Submitted']&.value # Mail::Field#value would strip leading and trailing whitespace - # See also https://tools.ietf.org/html/rfc3834 + # See also https://www.rfc-editor.org/rfc/rfc3834 auto_submitted && auto_submitted != 'no' end diff --git a/lib/gitlab/error_tracking/error_repository/open_api_strategy.rb b/lib/gitlab/error_tracking/error_repository/open_api_strategy.rb index cc822e4c10b..e168fa10630 100644 --- a/lib/gitlab/error_tracking/error_repository/open_api_strategy.rb +++ b/lib/gitlab/error_tracking/error_repository/open_api_strategy.rb @@ -228,7 +228,7 @@ module Gitlab def configured_api_url url = Gitlab::CurrentSettings.current_application_settings.error_tracking_api_url || - 'http://localhost:8080' + 'http://localhost:8080' Gitlab::UrlBlocker.validate!(url, schemes: %w[http https], allow_localhost: true) diff --git a/lib/gitlab/favicon.rb b/lib/gitlab/favicon.rb index 721518c6fcc..8e48b482462 100644 --- a/lib/gitlab/favicon.rb +++ b/lib/gitlab/favicon.rb @@ -34,11 +34,9 @@ module Gitlab end def available_status_names - @available_status_names ||= begin - Dir.glob(Rails.root.join('app', 'assets', 'images', 'ci_favicons', '*.png')) + @available_status_names ||= Dir.glob(Rails.root.join('app', 'assets', 'images', 'ci_favicons', '*.png')) .map { |file| File.basename(file, '.png') } .sort - end end private diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb index 58b46a85aae..05d680c139c 100644 --- a/lib/gitlab/gfm/uploads_rewriter.rb +++ b/lib/gitlab/gfm/uploads_rewriter.rb @@ -54,8 +54,6 @@ module Gitlab file = find_file(match[:secret], match[:file]) # No file will be returned for a path traversal - return '' if file.nil? - return markdown unless file.try(:exists?) klass = @target_parent.is_a?(Namespace) ? NamespaceFileUploader : FileUploader diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 4b877bf44da..8e1b51fcec5 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -16,6 +16,7 @@ module Gitlab CommitError = Class.new(BaseError) OSError = Class.new(BaseError) UnknownRef = Class.new(BaseError) + AmbiguousRef = Class.new(BaseError) CommandTimedOut = Class.new(CommandError) InvalidPageToken = Class.new(BaseError) InvalidRefFormatError = Class.new(BaseError) diff --git a/lib/gitlab/git/base_error.rb b/lib/gitlab/git/base_error.rb index a7eaa82b347..0b0fdef54cc 100644 --- a/lib/gitlab/git/base_error.rb +++ b/lib/gitlab/git/base_error.rb @@ -1,20 +1,50 @@ # frozen_string_literal: true +require 'grpc' module Gitlab module Git class BaseError < StandardError DEBUG_ERROR_STRING_REGEX = /(.*?) debug_error_string:.*$/m.freeze + GRPC_CODES = { + '0' => 'ok', + '1' => 'cancelled', + '2' => 'unknown', + '3' => 'invalid_argument', + '4' => 'deadline_exceeded', + '5' => 'not_found', + '6' => 'already_exists', + '7' => 'permission_denied', + '8' => 'resource_exhausted', + '9' => 'failed_precondition', + '10' => 'aborted', + '11' => 'out_of_range', + '12' => 'unimplemented', + '13' => 'internal', + '14' => 'unavailable', + '15' => 'data_loss', + '16' => 'unauthenticated' + }.freeze + + attr_reader :status, :code, :service def initialize(msg = nil) - if msg - raw_message = msg.to_s - match = DEBUG_ERROR_STRING_REGEX.match(raw_message) - raw_message = match[1] if match + super && return if msg.nil? + + set_grpc_error_code(msg) if msg.is_a?(::GRPC::BadStatus) + + super(build_raw_message(msg)) + end + + def build_raw_message(message) + raw_message = message.to_s + match = DEBUG_ERROR_STRING_REGEX.match(raw_message) + match ? match[1] : raw_message + end - super(raw_message) - else - super - end + def set_grpc_error_code(grpc_error) + @status = grpc_error.code + @code = GRPC_CODES[@status.to_s] + @service = 'git' end end end diff --git a/lib/gitlab/git/cross_repo_comparer.rb b/lib/gitlab/git/cross_repo.rb index d42b2a3bd98..d44657e7db1 100644 --- a/lib/gitlab/git/cross_repo_comparer.rb +++ b/lib/gitlab/git/cross_repo.rb @@ -2,7 +2,7 @@ module Gitlab module Git - class CrossRepoComparer + class CrossRepo attr_reader :source_repo, :target_repo def initialize(source_repo, target_repo) @@ -10,15 +10,8 @@ module Gitlab @target_repo = target_repo end - def compare(source_ref, target_ref, straight:) - ensuring_ref_in_source(target_ref) do |target_commit_id| - Gitlab::Git::Compare.new( - source_repo, - target_commit_id, - source_ref, - straight: straight - ) - end + def execute(target_ref, &blk) + ensuring_ref_in_source(target_ref, &blk) end private diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 3b5151ef4f2..2f9cfe3e764 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -116,9 +116,9 @@ module Gitlab # Returns an Array of branch names # sorted by name ASC def branch_names - wrapped_gitaly_errors do - gitaly_ref_client.branch_names - end + refs = list_refs([Gitlab::Git::BRANCH_REF_PREFIX]) + + refs.map { |ref| Gitlab::Git.branch_name(ref.name) } end # Returns an Array of Branches @@ -134,6 +134,10 @@ module Gitlab wrapped_gitaly_errors do gitaly_ref_client.find_branch(name) end + rescue Gitlab::Git::AmbiguousRef + # Gitaly returns "reference is ambiguous" error in case when users request + # branch "my-branch", when another branch "my-branch/branch" exists. + # We handle this error here and return nil for this case. end def find_tag(name) @@ -158,9 +162,7 @@ module Gitlab # Returns the number of valid branches def branch_count - wrapped_gitaly_errors do - gitaly_ref_client.count_branch_names - end + branch_names.count end def rename(new_relative_path) @@ -202,16 +204,14 @@ module Gitlab # Returns the number of valid tags def tag_count - wrapped_gitaly_errors do - gitaly_ref_client.count_tag_names - end + tag_names.count end # Returns an Array of tag names def tag_names - wrapped_gitaly_errors do - gitaly_ref_client.tag_names - end + refs = list_refs([Gitlab::Git::TAG_REF_PREFIX]) + + refs.map { |ref| Gitlab::Git.tag_name(ref.name) } end # Returns an Array of Tags @@ -385,6 +385,12 @@ module Gitlab end end + def check_objects_exist(refs) + wrapped_gitaly_errors do + gitaly_commit_client.object_existence_map(Array.wrap(refs)) + end + end + def new_blobs(newrevs, dynamic_timeout: nil) newrevs = Array.wrap(newrevs).reject { |rev| rev.blank? || rev == ::Gitlab::Git::BLANK_SHA } return [] if newrevs.empty? @@ -823,9 +829,14 @@ module Gitlab end def compare_source_branch(target_branch_name, source_repository, source_branch_name, straight:) - CrossRepoComparer - .new(source_repository, self) - .compare(source_branch_name, target_branch_name, straight: straight) + CrossRepo.new(source_repository, self).execute(target_branch_name) do |target_commit_id| + Gitlab::Git::Compare.new( + source_repository, + target_commit_id, + source_branch_name, + straight: straight + ) + end end def write_ref(ref_path, ref, old_ref: nil) diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index da2a81983ec..344dd27589c 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -157,10 +157,10 @@ module Gitlab # for deploy tokens and builds def can_download? deploy_key_can_download_code? || - deploy_token_can_download? || - build_can_download? || - user_can_download? || - guest_can_download? + deploy_token_can_download? || + build_can_download? || + user_can_download? || + guest_can_download? end def check_container! @@ -339,7 +339,7 @@ module Gitlab def check_change_access! if changes == ANY can_push = deploy_key? || - user_can_push? || + user_can_push? || project&.any_branch_allows_collaboration?(user_access.user) unless can_push diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 6bcf4802fbe..de66ca7305f 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -571,7 +571,7 @@ module Gitlab end def encode_repeated(array) - Google::Protobuf::RepeatedField.new(:bytes, array.map { |s| encode_binary(s) } ) + Google::Protobuf::RepeatedField.new(:bytes, array.map { |s| encode_binary(s) }) end def call_find_commit(revision) diff --git a/lib/gitlab/gitaly_client/namespace_service.rb b/lib/gitlab/gitaly_client/namespace_service.rb index dbcebec3aa2..05aee2fa55d 100644 --- a/lib/gitlab/gitaly_client/namespace_service.rb +++ b/lib/gitlab/gitaly_client/namespace_service.rb @@ -40,6 +40,13 @@ module Gitlab gitaly_client_call(:rename_namespace, request, timeout: GitalyClient.fast_timeout) end + def exists?(name) + request = Gitaly::NamespaceExistsRequest.new(storage_name: @storage, name: name) + + response = gitaly_client_call(:namespace_exists, request, timeout: GitalyClient.fast_timeout) + response.exists + end + private def gitaly_client_call(type, request, timeout: nil) diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 2312def5efc..66f70ed9dc6 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -452,6 +452,14 @@ module Gitlab when :index_update raise Gitlab::Git::Index::IndexError, index_error_message(detailed_error.index_update) else + # Some invalid path errors are caught by Gitaly directly and returned + # as an :index_update error, while others are found by libgit2 and + # come as generic errors. We need to convert the latter as IndexErrors + # as well. + if e.to_status.details.start_with?('invalid path') + raise Gitlab::Git::Index::IndexError, e.to_status.details + end + raise e end end @@ -600,17 +608,17 @@ module Gitlab case index_error.error_type when :ERROR_TYPE_EMPTY_PATH - "Received empty path" + "You must provide a file path" when :ERROR_TYPE_INVALID_PATH - "Invalid path: #{encoded_path}" + "invalid path: '#{encoded_path}'" when :ERROR_TYPE_DIRECTORY_EXISTS - "Directory already exists: #{encoded_path}" + "A directory with this name already exists" when :ERROR_TYPE_DIRECTORY_TRAVERSAL - "Directory traversal in path escapes repository: #{encoded_path}" + "Path cannot include directory traversal" when :ERROR_TYPE_FILE_EXISTS - "File already exists: #{encoded_path}" + "A file with this name already exists" when :ERROR_TYPE_FILE_NOT_FOUND - "File not found: #{encoded_path}" + "A file with this name doesn't exist" else "Unknown error performing git operation" end diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index de76ade76cb..da579276101 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -17,6 +17,8 @@ module Gitlab 'desc' => Gitaly::SortDirection::DESCENDING }.freeze + AMBIGUOUS_REFERENCE = 'reference is ambiguous' + # 'repository' is a Gitlab::Git::Repository def initialize(repository) @repository = repository @@ -54,26 +56,6 @@ module Gitlab Gitlab::Git.branch_name(response.name) end - def branch_names - request = Gitaly::FindAllBranchNamesRequest.new(repository: @gitaly_repo) - response = gitaly_client_call(@storage, :ref_service, :find_all_branch_names, request, timeout: GitalyClient.fast_timeout) - consume_refs_response(response) { |name| Gitlab::Git.branch_name(name) } - end - - def tag_names - request = Gitaly::FindAllTagNamesRequest.new(repository: @gitaly_repo) - response = gitaly_client_call(@storage, :ref_service, :find_all_tag_names, request, timeout: GitalyClient.fast_timeout) - consume_refs_response(response) { |name| Gitlab::Git.tag_name(name) } - end - - def count_tag_names - tag_names.count - end - - def count_branch_names - branch_names.count - end - def local_branches(sort_by: nil, pagination_params: nil) request = Gitaly::FindLocalBranchesRequest.new(repository: @gitaly_repo, pagination_params: pagination_params) request.sort_by = sort_local_branches_by_param(sort_by) if sort_by @@ -109,6 +91,10 @@ module Gitlab target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target_commit) Gitlab::Git::Branch.new(@repository, branch.name.dup, branch.target_commit.id, target_commit) + rescue GRPC::BadStatus => e + raise e unless e.message.include?(AMBIGUOUS_REFERENCE) + + raise Gitlab::Git::AmbiguousRef, "branch is ambiguous: #{branch_name}" end def find_tag(tag_name) diff --git a/lib/gitlab/gitaly_client/with_feature_flag_actors.rb b/lib/gitlab/gitaly_client/with_feature_flag_actors.rb index 92fc524b724..3d81292da16 100644 --- a/lib/gitlab/gitaly_client/with_feature_flag_actors.rb +++ b/lib/gitlab/gitaly_client/with_feature_flag_actors.rb @@ -16,8 +16,6 @@ module Gitlab # gitaly_client_call performs Gitaly calls including collected feature flag actors. The actors are retrieved # from repository actor and memoized. The service must set `self.repository_actor = a_repository` beforehand. def gitaly_client_call(*args, **kargs) - return GitalyClient.call(*args, **kargs) unless actors_aware_gitaly_calls? - unless repository_actor Gitlab::ErrorTracking.track_and_raise_for_dev_exception( Feature::InvalidFeatureFlagError.new("gitaly_client_call called without setting repository_actor") @@ -34,11 +32,8 @@ module Gitlab end end - # gitaly_feature_flag_actors returns a hash of actors implied from input repository. If actors_aware_gitaly_calls - # flag is not on, this method returns an empty hash. + # gitaly_feature_flag_actors returns a hash of actors implied from input repository. def gitaly_feature_flag_actors(repository) - return {} unless actors_aware_gitaly_calls? - container = find_repository_container(repository) { repository: repository, @@ -92,10 +87,6 @@ module Gitlab repository.container end end - - def actors_aware_gitaly_calls? - Feature.enabled?(:actors_aware_gitaly_calls) - end end end end diff --git a/lib/gitlab/github_gists_import/importer/gist_importer.rb b/lib/gitlab/github_gists_import/importer/gist_importer.rb new file mode 100644 index 00000000000..a5e87d3cf7d --- /dev/null +++ b/lib/gitlab/github_gists_import/importer/gist_importer.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +module Gitlab + module GithubGistsImport + module Importer + class GistImporter + attr_reader :gist, :user + + FileCountLimitError = Class.new(StandardError) + + # gist - An instance of `Gitlab::GithubGistsImport::Representation::Gist`. + def initialize(gist, user_id) + @gist = gist + @user = User.find(user_id) + end + + def execute + snippet = build_snippet + import_repository(snippet) if snippet.save! + + return ServiceResponse.success unless max_snippet_files_count_exceeded?(snippet) + + fail_and_track(snippet) + end + + private + + def build_snippet + attrs = { + title: gist.truncated_title, + visibility_level: gist.visibility_level, + content: gist.first_file[:file_content], + file_name: gist.first_file[:file_name], + author: user, + created_at: gist.created_at, + updated_at: gist.updated_at + } + + PersonalSnippet.new(attrs) + end + + def import_repository(snippet) + resolved_address = get_resolved_address + + snippet.create_repository + snippet.repository.fetch_as_mirror(gist.git_pull_url, forced: true, resolved_address: resolved_address) + rescue StandardError + remove_snippet_and_repository(snippet) + + raise + end + + def get_resolved_address + validated_pull_url, host = Gitlab::UrlBlocker.validate!(gist.git_pull_url, + schemes: Project::VALID_IMPORT_PROTOCOLS, + ports: Project::VALID_IMPORT_PORTS, + allow_localhost: allow_local_requests?, + allow_local_network: allow_local_requests?) + + host.present? ? validated_pull_url.host.to_s : '' + end + + def max_snippet_files_count_exceeded?(snippet) + snippet.all_files.size > Snippet.max_file_limit + end + + def remove_snippet_and_repository(snippet) + snippet.repository.remove if snippet.repository_exists? + snippet.destroy + end + + def allow_local_requests? + Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services? + end + + def fail_and_track(snippet) + remove_snippet_and_repository(snippet) + + ServiceResponse.error(message: 'Snippet max file count exceeded').track_exception(as: FileCountLimitError) + end + end + end + end +end diff --git a/lib/gitlab/github_gists_import/importer/gists_importer.rb b/lib/gitlab/github_gists_import/importer/gists_importer.rb new file mode 100644 index 00000000000..08744dbaf5f --- /dev/null +++ b/lib/gitlab/github_gists_import/importer/gists_importer.rb @@ -0,0 +1,95 @@ +# frozen_string_literal: true + +module Gitlab + module GithubGistsImport + module Importer + class GistsImporter + attr_reader :user, :client, :already_imported_cache_key + + ALREADY_IMPORTED_CACHE_KEY = 'github-gists-importer/already-imported/%{user}' + RESULT_CONTEXT = Struct.new(:success?, :error, :waiter, :next_attempt_in, keyword_init: true) + + def initialize(user, token) + @user = user + @client = Gitlab::GithubImport::Client.new(token, parallel: true) + @already_imported_cache_key = format(ALREADY_IMPORTED_CACHE_KEY, user: user.id) + end + + def execute + waiter = spread_parallel_import + + expire_already_imported_cache! + + RESULT_CONTEXT.new(success?: true, waiter: waiter) + rescue Gitlab::GithubImport::RateLimitError => e + RESULT_CONTEXT.new(success?: false, error: e, next_attempt_in: client.rate_limit_resets_in) + rescue StandardError => e + RESULT_CONTEXT.new(success?: false, error: e) + end + + private + + def spread_parallel_import + waiter = JobWaiter.new + worker_arguments = fetch_gists_to_import.map { |gist_hash| [user.id, gist_hash, waiter.key] } + waiter.jobs_remaining = worker_arguments.size + + schedule_bulk_perform(worker_arguments) + waiter + end + + def fetch_gists_to_import + page_counter = Gitlab::GithubImport::PageCounter.new(user, :gists, 'github-gists-importer') + collection = [] + + client.each_page(:gists, nil, page: page_counter.current) do |page| + next unless page_counter.set(page.number) + + collection += gists_from(page) + end + + page_counter.expire! + + collection + end + + def gists_from(page) + page.objects.each.with_object([]) do |gist, page_collection| + gist = gist.to_h + next if already_imported?(gist) + + page_collection << ::Gitlab::GithubGistsImport::Representation::Gist.from_api_response(gist).to_hash + + mark_as_imported(gist) + end + end + + def schedule_bulk_perform(worker_arguments) + # rubocop:disable Scalability/BulkPerformWithContext + Gitlab::ApplicationContext.with_context(user: user) do + Gitlab::GithubGistsImport::ImportGistWorker.bulk_perform_in( + 1.second, + worker_arguments, + batch_size: 1000, + batch_delay: 1.minute + ) + end + # rubocop:enable Scalability/BulkPerformWithContext + end + + def already_imported?(gist) + Gitlab::Cache::Import::Caching.set_includes?(already_imported_cache_key, gist[:id]) + end + + def mark_as_imported(gist) + Gitlab::Cache::Import::Caching.set_add(already_imported_cache_key, gist[:id]) + end + + def expire_already_imported_cache! + Gitlab::Cache::Import::Caching + .expire(already_imported_cache_key, Gitlab::Cache::Import::Caching::SHORTER_TIMEOUT) + end + end + end + end +end diff --git a/lib/gitlab/github_gists_import/representation/gist.rb b/lib/gitlab/github_gists_import/representation/gist.rb new file mode 100644 index 00000000000..0d309a98f38 --- /dev/null +++ b/lib/gitlab/github_gists_import/representation/gist.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module Gitlab + module GithubGistsImport + module Representation + class Gist + include Gitlab::GithubImport::Representation::ToHash + include Gitlab::GithubImport::Representation::ExposeAttribute + + attr_reader :attributes + + expose_attribute :id, :description, :is_public, :created_at, :updated_at, :files, :git_pull_url + + # Builds a gist from a GitHub API response. + # + # gist - An instance of `Hash` containing the gist + # details. + def self.from_api_response(gist, additional_data = {}) + hash = { + id: gist[:id], + description: gist[:description], + is_public: gist[:public], + files: gist[:files], + git_pull_url: gist[:git_pull_url], + created_at: gist[:created_at], + updated_at: gist[:updated_at] + } + + new(hash) + end + + # Builds a new gist using a Hash that was built from a JSON payload. + def self.from_json_hash(raw_hash) + new(Gitlab::GithubImport::Representation.symbolize_hash(raw_hash)) + end + + # attributes - A hash containing the raw gist details. The keys of this + # Hash (and any nested hashes) must be symbols. + def initialize(attributes) + @attributes = attributes + end + + # Gist description can be an empty string, so we returning nil to use first file + # name as a title in such case on snippet creation + # Gist description has a limit of 256, while the snippet's title can be up to 255 + def truncated_title + title = description.presence || first_file[:file_name] + + title.truncate(255) + end + + def visibility_level + is_public ? Gitlab::VisibilityLevel::PUBLIC : Gitlab::VisibilityLevel::PRIVATE + end + + def first_file + _key, value = files.first + + { + file_name: value[:filename], + file_content: Gitlab::HTTP.try_get(value[:raw_url])&.body + } + end + + def github_identifiers + { id: id } + end + end + end + end +end diff --git a/lib/gitlab/github_gists_import/status.rb b/lib/gitlab/github_gists_import/status.rb new file mode 100644 index 00000000000..e997eb0bf88 --- /dev/null +++ b/lib/gitlab/github_gists_import/status.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module GithubGistsImport + class Status + IMPORT_STATUS_KEY = 'gitlab:github-gists-import:%{user_id}' + EXPIRATION_TIME = 24.hours + + def initialize(user_id) + @user_id = user_id + end + + def start! + change_status('started') + end + + def fail! + change_status('failed') + end + + def finish! + change_status('finished') + end + + def started? + Gitlab::Redis::SharedState.with { |redis| redis.get(import_status_key) == 'started' } + end + + private + + def change_status(status_name) + Gitlab::Redis::SharedState.with do |redis| + redis.set(import_status_key, status_name) + redis.expire(import_status_key, EXPIRATION_TIME) unless status_name == 'started' + end + end + + def import_status_key + format(IMPORT_STATUS_KEY, user_id: @user_id) + end + end + end +end diff --git a/lib/gitlab/github_import/bulk_importing.rb b/lib/gitlab/github_import/bulk_importing.rb index 28a39128ec9..0c91eff1d10 100644 --- a/lib/gitlab/github_import/bulk_importing.rb +++ b/lib/gitlab/github_import/bulk_importing.rb @@ -10,25 +10,38 @@ module Gitlab def initialize(project, client) @project = project @client = client + @validation_errors = [] end # Builds and returns an Array of objects to bulk insert into the - # database. + # database and array of validation errors if object is invalid. # # enum - An Enumerable that returns the objects to turn into database # rows. def build_database_rows(enum) + errors = [] rows = enum.each_with_object([]) do |(object, _), result| - result << build(object) unless already_imported?(object) + next if already_imported?(object) + + attrs = build_attributes(object) + build_record = model.new(attrs) + + if build_record.invalid? + log_error(object[:id], build_record.errors.full_messages) + errors << build_record.errors + next + end + + result << attrs end log_and_increment_counter(rows.size, :fetched) - rows + [rows, errors] end # Bulk inserts the given rows into the database. - def bulk_insert(model, rows, batch_size: 100) + def bulk_insert(rows, batch_size: 100) rows.each_slice(batch_size) do |slice| ApplicationRecord.legacy_bulk_insert(model.table_name, slice) # rubocop:disable Gitlab/BulkInsert @@ -40,6 +53,23 @@ module Gitlab raise NotImplementedError end + def bulk_insert_failures(validation_errors) + rows = validation_errors.map do |error| + correlation_id_value = Labkit::Correlation::CorrelationId.current_or_new_id + + { + source: self.class.name, + exception_class: 'ActiveRecord::RecordInvalid', + exception_message: error.full_messages.first.truncate(255), + correlation_id_value: correlation_id_value, + retry_count: nil, + created_at: Time.zone.now + } + end + + project.import_failures.insert_all(rows) + end + private def log_and_increment_counter(value, operation) @@ -57,6 +87,16 @@ module Gitlab value: value ) end + + def log_error(object_id, messages) + Gitlab::Import::Logger.error( + import_type: :github, + project_id: project.id, + importer: self.class.name, + message: messages, + github_identifier: object_id + ) + end end end end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index d6060141bce..065410693e5 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -15,6 +15,7 @@ module Gitlab # end class Client include ::Gitlab::Utils::StrongMemoize + include ::Gitlab::GithubImport::Clients::SearchRepos attr_reader :octokit @@ -182,19 +183,6 @@ module Gitlab end end - def search_repos_by_name(name, options = {}) - with_retry { octokit.search_repositories(search_query(str: name, type: :name), options).to_h } - end - - def search_query(str:, type:, include_collaborations: true, include_orgs: true) - query = "#{str} in:#{type} is:public,private user:#{octokit.user.to_h[:login]}" - - query = [query, collaborations_subquery].join(' ') if include_collaborations - query = [query, organizations_subquery].join(' ') if include_orgs - - query - end - # Returns `true` if we're still allowed to perform API calls. # Search API has rate limit of 30, use lowered threshold when search is used. def requests_remaining? diff --git a/lib/gitlab/github_import/clients/proxy.rb b/lib/gitlab/github_import/clients/proxy.rb new file mode 100644 index 00000000000..f6d1c8ed23c --- /dev/null +++ b/lib/gitlab/github_import/clients/proxy.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Clients + class Proxy + attr_reader :client + + def initialize(access_token, client_options) + @client = pick_client(access_token, client_options) + end + + def repos(search_text, pagination_options) + return { repos: filtered(client.repos, search_text) } if use_legacy? + + if use_graphql? + fetch_repos_via_graphql(search_text, pagination_options) + else + fetch_repos_via_rest(search_text, pagination_options) + end + end + + private + + def fetch_repos_via_rest(search_text, pagination_options) + { repos: client.search_repos_by_name(search_text, pagination_options)[:items] } + end + + def fetch_repos_via_graphql(search_text, pagination_options) + response = client.search_repos_by_name_graphql(search_text, pagination_options) + { + repos: response.dig(:data, :search, :nodes), + page_info: response.dig(:data, :search, :pageInfo) + } + end + + def pick_client(access_token, client_options) + return Gitlab::GithubImport::Client.new(access_token) unless use_legacy? + + Gitlab::LegacyGithubImport::Client.new(access_token, **client_options) + end + + def filtered(collection, search_text) + return collection if search_text.blank? + + collection.select { |item| item[:name].to_s.downcase.include?(search_text) } + end + + def use_legacy? + Feature.disabled?(:remove_legacy_github_client) + end + + def use_graphql? + Feature.enabled?(:github_client_fetch_repos_via_graphql) + end + end + end + end +end diff --git a/lib/gitlab/github_import/clients/search_repos.rb b/lib/gitlab/github_import/clients/search_repos.rb new file mode 100644 index 00000000000..bcd226087e7 --- /dev/null +++ b/lib/gitlab/github_import/clients/search_repos.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Clients + module SearchRepos + def search_repos_by_name_graphql(name, options = {}) + with_retry do + octokit.post( + '/graphql', + { query: graphql_search_repos_body(name, options) }.to_json + ).to_h + end + end + + def search_repos_by_name(name, options = {}) + with_retry do + octokit.search_repositories( + search_repos_query(str: name, type: :name), + options + ).to_h + end + end + + private + + def graphql_search_repos_body(name, options) + query = search_repos_query(str: name, type: :name) + query = "query: \"#{query}\"" + first = options[:first].present? ? ", first: #{options[:first]}" : '' + after = options[:after].present? ? ", after: \"#{options[:after]}\"" : '' + <<-TEXT + { + search(type: REPOSITORY, #{query}#{first}#{after}) { + nodes { + __typename + ... on Repository { + id: databaseId + name + full_name: nameWithOwner + owner { login } + } + } + pageInfo { + startCursor + endCursor + hasNextPage + hasPreviousPage + } + } + } + TEXT + end + + def search_repos_query(str:, type:, include_collaborations: true, include_orgs: true) + query = "#{str} in:#{type} is:public,private user:#{octokit.user.to_h[:login]}" + + query = [query, collaborations_subquery].join(' ') if include_collaborations + query = [query, organizations_subquery].join(' ') if include_orgs + + query + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/diff_note_importer.rb b/lib/gitlab/github_import/importer/diff_note_importer.rb index a9f8483d8c3..44ffcd7a1e4 100644 --- a/lib/gitlab/github_import/importer/diff_note_importer.rb +++ b/lib/gitlab/github_import/importer/diff_note_importer.rb @@ -18,7 +18,6 @@ module Gitlab def execute return if merge_request_id.blank? - note.project = project note.merge_request = merge_request build_author_attributes @@ -65,7 +64,7 @@ module Gitlab # To work around this we're using bulk_insert with a single row. This # allows us to efficiently insert data (even if it's just 1 row) # without having to use all sorts of hacks to disable callbacks. - ApplicationRecord.legacy_bulk_insert(LegacyDiffNote.table_name, [{ + attributes = { noteable_type: note.noteable_type, system: false, type: 'LegacyDiffNote', @@ -79,7 +78,12 @@ module Gitlab created_at: note.created_at, updated_at: note.updated_at, st_diff: note.diff_hash.to_yaml - }]) + } + + diff_note = LegacyDiffNote.new(attributes.merge(importing: true)) + diff_note.validate! + + ApplicationRecord.legacy_bulk_insert(LegacyDiffNote.table_name, [attributes]) end # rubocop:enabled Gitlab/BulkInsert diff --git a/lib/gitlab/github_import/importer/issue_importer.rb b/lib/gitlab/github_import/importer/issue_importer.rb index d964bae3dd2..b477468d327 100644 --- a/lib/gitlab/github_import/importer/issue_importer.rb +++ b/lib/gitlab/github_import/importer/issue_importer.rb @@ -60,6 +60,9 @@ module Gitlab work_item_type_id: issue.work_item_type_id } + issue = project.issues.new(attributes.merge(importing: true)) + issue.validate! + insert_and_return_id(attributes, project.issues) rescue ActiveRecord::InvalidForeignKey # It's possible the project has been deleted since scheduling this diff --git a/lib/gitlab/github_import/importer/label_links_importer.rb b/lib/gitlab/github_import/importer/label_links_importer.rb index 5e248c7cfc5..52c87dda347 100644 --- a/lib/gitlab/github_import/importer/label_links_importer.rb +++ b/lib/gitlab/github_import/importer/label_links_importer.rb @@ -22,7 +22,7 @@ module Gitlab def create_labels time = Time.zone.now - rows = [] + items = [] target_id = find_target_id issue.label_names.each do |label_name| @@ -31,16 +31,16 @@ module Gitlab # the project's labels. next unless (label_id = label_finder.id_for(label_name)) - rows << { + items << LabelLink.new( label_id: label_id, target_id: target_id, target_type: issue.issuable_type, created_at: time, updated_at: time - } + ) end - ApplicationRecord.legacy_bulk_insert(LabelLink.table_name, rows) # rubocop:disable Gitlab/BulkInsert + LabelLink.bulk_insert!(items) end def find_target_id diff --git a/lib/gitlab/github_import/importer/labels_importer.rb b/lib/gitlab/github_import/importer/labels_importer.rb index 9a011f17a18..d5d1cd28b7c 100644 --- a/lib/gitlab/github_import/importer/labels_importer.rb +++ b/lib/gitlab/github_import/importer/labels_importer.rb @@ -13,7 +13,10 @@ module Gitlab # rubocop: enable CodeReuse/ActiveRecord def execute - bulk_insert(Label, build_labels) + rows, validation_errors = build_labels + + bulk_insert(rows) + bulk_insert_failures(validation_errors) if validation_errors.any? build_labels_cache end @@ -29,7 +32,7 @@ module Gitlab LabelFinder.new(project).build_cache end - def build(label) + def build_attributes(label) time = Time.zone.now { @@ -49,6 +52,10 @@ module Gitlab def object_type :label end + + def model + Label + end end end end diff --git a/lib/gitlab/github_import/importer/lfs_objects_importer.rb b/lib/gitlab/github_import/importer/lfs_objects_importer.rb index 775afd5f53a..d064278e4a0 100644 --- a/lib/gitlab/github_import/importer/lfs_objects_importer.rb +++ b/lib/gitlab/github_import/importer/lfs_objects_importer.rb @@ -27,9 +27,9 @@ module Gitlab end def each_object_to_import - lfs_objects = Projects::LfsPointers::LfsObjectDownloadListService.new(project).execute + download_service = Projects::LfsPointers::LfsObjectDownloadListService.new(project) - lfs_objects.each do |object| + download_service.each_list_item do |object| Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) yield object diff --git a/lib/gitlab/github_import/importer/milestones_importer.rb b/lib/gitlab/github_import/importer/milestones_importer.rb index 1a3a54d0053..560fbdc66e3 100644 --- a/lib/gitlab/github_import/importer/milestones_importer.rb +++ b/lib/gitlab/github_import/importer/milestones_importer.rb @@ -13,7 +13,10 @@ module Gitlab # rubocop: enable CodeReuse/ActiveRecord def execute - bulk_insert(Milestone, build_milestones) + rows, validation_errors = build_milestones + + bulk_insert(rows) + bulk_insert_failures(validation_errors) if validation_errors.any? build_milestones_cache end @@ -29,7 +32,7 @@ module Gitlab MilestoneFinder.new(project).build_cache end - def build(milestone) + def build_attributes(milestone) { iid: milestone[:number], title: milestone[:title], @@ -53,6 +56,10 @@ module Gitlab def object_type :milestone end + + def model + Milestone + end end end end diff --git a/lib/gitlab/github_import/importer/note_importer.rb b/lib/gitlab/github_import/importer/note_importer.rb index 69b7b2c2a38..04da015a33f 100644 --- a/lib/gitlab/github_import/importer/note_importer.rb +++ b/lib/gitlab/github_import/importer/note_importer.rb @@ -33,6 +33,9 @@ module Gitlab updated_at: note.updated_at } + note = Note.new(attributes.merge(importing: true)) + note.validate! + # We're using bulk_insert here so we can bypass any validations and # callbacks. Running these would result in a lot of unnecessary SQL # queries being executed when importing large projects. diff --git a/lib/gitlab/github_import/importer/pull_request_importer.rb b/lib/gitlab/github_import/importer/pull_request_importer.rb index 3c17ea1195e..5690a2cc997 100644 --- a/lib/gitlab/github_import/importer/pull_request_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_importer.rb @@ -61,6 +61,9 @@ module Gitlab updated_at: pull_request.updated_at } + mr = project.merge_requests.new(attributes.merge(importing: true)) + mr.validate! + create_merge_request_without_hooks(project, attributes, pull_request.iid) end @@ -93,7 +96,7 @@ module Gitlab return if project.repository.branch_exists?(source_branch) project.repository.add_branch(project.creator, source_branch, pull_request.source_branch_sha) - rescue Gitlab::Git::CommandError => e + rescue Gitlab::Git::PreReceiveError, Gitlab::Git::CommandError => e Gitlab::ErrorTracking.track_exception(e, source_branch: source_branch, project_id: merge_request.project.id, diff --git a/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb b/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb index 640914acf4d..f05aa26a449 100644 --- a/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb @@ -4,42 +4,62 @@ module Gitlab module GithubImport module Importer class PullRequestMergedByImporter + # pull_request - An instance of + # `Gitlab::GithubImport::Representation::PullRequest` + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` def initialize(pull_request, project, client) - @project = project @pull_request = pull_request + @project = project @client = client end def execute - merge_request = project.merge_requests.find_by_iid(pull_request.iid) - timestamp = Time.new.utc - merged_at = pull_request.merged_at user_finder = GithubImport::UserFinder.new(project, client) - gitlab_user_id = user_finder.user_id_for(pull_request.merged_by) + gitlab_user_id = begin + user_finder.user_id_for(pull_request.merged_by) + rescue ::Octokit::NotFound + nil + end + + metrics_upsert(gitlab_user_id) + + add_note! + end + + private + + attr_reader :project, :pull_request, :client + + def metrics_upsert(gitlab_user_id) MergeRequest::Metrics.upsert({ target_project_id: project.id, merge_request_id: merge_request.id, merged_by_id: gitlab_user_id, - merged_at: merged_at, + merged_at: pull_request.merged_at, created_at: timestamp, updated_at: timestamp }, unique_by: :merge_request_id) + end - unless gitlab_user_id - merge_request.notes.create!( - importing: true, - note: missing_author_note, - author_id: project.creator_id, - project: project, - created_at: merged_at - ) - end + def add_note! + merge_request.notes.create!( + importing: true, + note: missing_author_note, + author_id: project.creator_id, + project: project, + created_at: pull_request.merged_at + ) end - private + def merge_request + @merge_request ||= project.merge_requests.find_by_iid(pull_request.iid) + end - attr_reader :project, :pull_request, :client + def timestamp + @timestamp ||= Time.new.utc + end def missing_author_note s_("GitHubImporter|*Merged by: %{author} at %{timestamp}*") % { diff --git a/lib/gitlab/github_import/importer/pull_request_review_importer.rb b/lib/gitlab/github_import/importer/pull_request_review_importer.rb index b11af90aa6f..de66f310edf 100644 --- a/lib/gitlab/github_import/importer/pull_request_review_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_review_importer.rb @@ -4,6 +4,9 @@ module Gitlab module GithubImport module Importer class PullRequestReviewImporter + # review - An instance of `Gitlab::GithubImport::Representation::PullRequestReview` + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` def initialize(review, project, client) @review = review @project = project @@ -13,7 +16,12 @@ module Gitlab def execute user_finder = GithubImport::UserFinder.new(project, client) - gitlab_user_id = user_finder.user_id_for(review.author) + + gitlab_user_id = begin + user_finder.user_id_for(review.author) + rescue ::Octokit::NotFound + nil + end if gitlab_user_id add_review_note!(gitlab_user_id) diff --git a/lib/gitlab/github_import/importer/releases_importer.rb b/lib/gitlab/github_import/importer/releases_importer.rb index fe6da30bbf8..62d579fda08 100644 --- a/lib/gitlab/github_import/importer/releases_importer.rb +++ b/lib/gitlab/github_import/importer/releases_importer.rb @@ -16,7 +16,10 @@ module Gitlab # to generate HTML version - you also need to regenerate it in # Gitlab::GithubImport::Importer::NoteAttachmentsImporter. def execute - bulk_insert(Release, build_releases) + rows, validation_errors = build_releases + + bulk_insert(rows) + bulk_insert_failures(validation_errors) if validation_errors.any? end def build_releases @@ -27,7 +30,7 @@ module Gitlab existing_tags.include?(release[:tag_name]) || release[:tag_name].nil? end - def build(release) + def build_attributes(release) existing_tags.add(release[:tag_name]) { @@ -66,6 +69,10 @@ module Gitlab def user_finder @user_finder ||= GithubImport::UserFinder.new(project, client) end + + def model + Release + end end end end diff --git a/lib/gitlab/github_import/markdown/attachment.rb b/lib/gitlab/github_import/markdown/attachment.rb index a5cf5ffa60e..1c814e34a39 100644 --- a/lib/gitlab/github_import/markdown/attachment.rb +++ b/lib/gitlab/github_import/markdown/attachment.rb @@ -28,6 +28,7 @@ module Gitlab def from_markdown_image(markdown_node) url = markdown_node.url + return unless url return unless github_url?(url, media: true) return unless whitelisted_type?(url, media: true) @@ -37,6 +38,7 @@ module Gitlab def from_markdown_link(markdown_node) url = markdown_node.url + return unless url return unless github_url?(url, docs: true) return unless whitelisted_type?(url, docs: true) @@ -46,7 +48,7 @@ module Gitlab def from_inline_html(markdown_node) img = Nokogiri::HTML.parse(markdown_node.string_content).xpath('//img')[0] - return unless img + return if img.nil? || img[:src].blank? return unless github_url?(img[:src], media: true) return unless whitelisted_type?(img[:src], media: true) diff --git a/lib/gitlab/github_import/page_counter.rb b/lib/gitlab/github_import/page_counter.rb index 3face4c794b..c238ccb8932 100644 --- a/lib/gitlab/github_import/page_counter.rb +++ b/lib/gitlab/github_import/page_counter.rb @@ -9,10 +9,10 @@ module Gitlab attr_reader :cache_key # The base cache key to use for storing the last page number. - CACHE_KEY = 'github-importer/page-counter/%{project}/%{collection}' + CACHE_KEY = '%{import_type}/page-counter/%{object}/%{collection}' - def initialize(project, collection) - @cache_key = CACHE_KEY % { project: project.id, collection: collection } + def initialize(object, collection, import_type = 'github-importer') + @cache_key = CACHE_KEY % { import_type: import_type, object: object.id, collection: collection } end # Sets the page number to the given value. diff --git a/lib/gitlab/github_import/representation/diff_note.rb b/lib/gitlab/github_import/representation/diff_note.rb index f3be90834c7..9259d0295d5 100644 --- a/lib/gitlab/github_import/representation/diff_note.rb +++ b/lib/gitlab/github_import/representation/diff_note.rb @@ -4,18 +4,15 @@ module Gitlab module GithubImport module Representation class DiffNote - include Gitlab::Utils::StrongMemoize include ToHash include ExposeAttribute - NOTEABLE_TYPE = 'MergeRequest' NOTEABLE_ID_REGEX = %r{/pull/(?<iid>\d+)}i.freeze - DISCUSSION_CACHE_KEY = 'github-importer/discussion-id-map/%{project_id}/%{noteable_id}/%{original_note_id}' expose_attribute :noteable_id, :commit_id, :file_path, :diff_hunk, :author, :created_at, :updated_at, :original_commit_id, :note_id, :end_line, :start_line, - :side, :in_reply_to_id + :side, :in_reply_to_id, :discussion_id # Builds a diff note from a GitHub API response. # @@ -45,7 +42,8 @@ module Gitlab end_line: note[:line], start_line: note[:start_line], side: note[:side], - in_reply_to_id: note[:in_reply_to_id] + in_reply_to_id: note[:in_reply_to_id], + discussion_id: DiffNotes::DiscussionId.new(note).find_or_generate } new(hash) @@ -59,7 +57,7 @@ module Gitlab new(hash) end - attr_accessor :merge_request, :project + attr_accessor :merge_request # attributes - A Hash containing the raw note details. The keys of this # Hash must be Symbols. @@ -74,7 +72,7 @@ module Gitlab end def noteable_type - NOTEABLE_TYPE + DiffNotes::DiscussionId::NOTEABLE_TYPE end def contains_suggestion? @@ -127,12 +125,6 @@ module Gitlab } end - def discussion_id - strong_memoize(:discussion_id) do - (in_reply_to_id.present? && current_discussion_id) || generate_discussion_id - end - end - private # Required by ExposeAttribute @@ -149,32 +141,6 @@ module Gitlab def addition? side == 'RIGHT' end - - def generate_discussion_id - Discussion.discussion_id( - Struct - .new(:noteable_id, :noteable_type) - .new(merge_request.id, NOTEABLE_TYPE) - ).tap do |discussion_id| - cache_discussion_id(discussion_id) - end - end - - def cache_discussion_id(discussion_id) - Gitlab::Cache::Import::Caching.write(discussion_id_cache_key(note_id), discussion_id) - end - - def current_discussion_id - Gitlab::Cache::Import::Caching.read(discussion_id_cache_key(in_reply_to_id)) - end - - def discussion_id_cache_key(id) - DISCUSSION_CACHE_KEY % { - project_id: project.id, - noteable_id: merge_request.id, - original_note_id: id - } - end end end end diff --git a/lib/gitlab/github_import/representation/diff_notes/discussion_id.rb b/lib/gitlab/github_import/representation/diff_notes/discussion_id.rb new file mode 100644 index 00000000000..38b560f21c0 --- /dev/null +++ b/lib/gitlab/github_import/representation/diff_notes/discussion_id.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + module DiffNotes + class DiscussionId + NOTEABLE_TYPE = 'MergeRequest' + DISCUSSION_CACHE_REGEX = %r{/(?<repo>[^/]*)/pull/(?<iid>\d+)}i.freeze + DISCUSSION_CACHE_KEY = 'github-importer/discussion-id-map/%{project}/%{noteable_id}/%{original_note_id}' + + def initialize(note) + @note = note + @matches = note[:html_url].match(DISCUSSION_CACHE_REGEX) + end + + def find_or_generate + (note[:in_reply_to_id].present? && current_discussion_id) || generate_discussion_id + end + + private + + attr_reader :note, :matches + + def generate_discussion_id + discussion_id = Discussion.discussion_id( + Struct + .new(:noteable_id, :noteable_type) + .new(matches[:iid].to_i, NOTEABLE_TYPE) + ) + cache_discussion_id(discussion_id) + end + + def cache_discussion_id(discussion_id) + Gitlab::Cache::Import::Caching.write( + discussion_id_cache_key(note[:id]), discussion_id + ) + end + + def current_discussion_id + Gitlab::Cache::Import::Caching.read( + discussion_id_cache_key(note[:in_reply_to_id]) + ) + end + + def discussion_id_cache_key(id) + format(DISCUSSION_CACHE_KEY, + project: matches[:repo], + noteable_id: matches[:iid].to_i, + original_note_id: id + ) + end + end + end + end + end +end diff --git a/lib/gitlab/gl_repository/repo_type.rb b/lib/gitlab/gl_repository/repo_type.rb index 05278b2dd35..7792ef55b28 100644 --- a/lib/gitlab/gl_repository/repo_type.rb +++ b/lib/gitlab/gl_repository/repo_type.rb @@ -66,10 +66,10 @@ module Gitlab def valid?(repository_path) repository_path.end_with?(path_suffix) && - ( - !snippet? || - repository_path.match?(Gitlab::PathRegex.full_snippets_repository_path_regex) - ) + ( + !snippet? || + repository_path.match?(Gitlab::PathRegex.full_snippets_repository_path_regex) + ) end private diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index ecb57bfc1a2..12cdcf445f7 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -65,8 +65,10 @@ module Gitlab push_frontend_feature_flag(:security_auto_fix) push_frontend_feature_flag(:new_header_search) push_frontend_feature_flag(:source_editor_toolbar) + push_frontend_feature_flag(:vscode_web_ide, current_user) push_frontend_feature_flag(:integration_slack_app_notifications) push_frontend_feature_flag(:vue_group_select) + push_frontend_feature_flag(:new_fonts, current_user) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/graphql/expose_permissions.rb b/lib/gitlab/graphql/expose_permissions.rb index ab9ed354673..070d3c188a4 100644 --- a/lib/gitlab/graphql/expose_permissions.rb +++ b/lib/gitlab/graphql/expose_permissions.rb @@ -5,11 +5,15 @@ module Gitlab module ExposePermissions extend ActiveSupport::Concern prepended do - def self.expose_permissions(permission_type, description: 'Permissions for the current user on the resource') + def self.expose_permissions( + permission_type, + description: 'Permissions for the current user on the resource', + &block) field :user_permissions, permission_type, description: description, null: false, - method: :itself + method: :itself, + &block end end end diff --git a/lib/gitlab/graphql/extensions/forward_only_externally_paginated_array_extension.rb b/lib/gitlab/graphql/extensions/forward_only_externally_paginated_array_extension.rb new file mode 100644 index 00000000000..651b4266756 --- /dev/null +++ b/lib/gitlab/graphql/extensions/forward_only_externally_paginated_array_extension.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true +module Gitlab + module Graphql + module Extensions + # This extension is meant for resolvers that only support forward looking pagination. So in order to limit + # confusion for allowed GraphQL pagination arguments on the field, we limit this to just `first` and `after`. + class ForwardOnlyExternallyPaginatedArrayExtension < ExternallyPaginatedArrayExtension + def apply + field.argument :after, GraphQL::Types::String, + description: "Returns the elements in the list that come after the specified cursor.", + required: false + field.argument :first, GraphQL::Types::Int, + description: "Returns the first _n_ elements from the list.", + required: false + end + end + end + end +end diff --git a/lib/gitlab/graphql/limit/field_call_count.rb b/lib/gitlab/graphql/limit/field_call_count.rb index 4165970a2a6..3a02e8abbb5 100644 --- a/lib/gitlab/graphql/limit/field_call_count.rb +++ b/lib/gitlab/graphql/limit/field_call_count.rb @@ -14,9 +14,18 @@ module Gitlab private def increment_call_count(context) + query_id = fetch_query_id(context) + context[:call_count] ||= {} - context[:call_count][field] ||= 0 - context[:call_count][field] += 1 + context[:call_count][query_id] ||= {} + context[:call_count][query_id][field] ||= 0 + context[:call_count][query_id][field] += 1 + end + + def fetch_query_id(context) + context.query.operation_fingerprint + rescue TypeError + '' end def limit diff --git a/lib/gitlab/graphql/pagination/keyset/connection.rb b/lib/gitlab/graphql/pagination/keyset/connection.rb index eca4d42fb9a..208ca5f2d24 100644 --- a/lib/gitlab/graphql/pagination/keyset/connection.rb +++ b/lib/gitlab/graphql/pagination/keyset/connection.rb @@ -59,16 +59,7 @@ module Gitlab if before true elsif first - if Feature.enabled?(:graphql_keyset_pagination_without_next_page_query) - limited_nodes.size > limit_value - else - case sliced_nodes - when Array - sliced_nodes.size > limit_value - else - sliced_nodes.limit(1).offset(limit_value).exists? # rubocop: disable CodeReuse/ActiveRecord - end - end + limited_nodes.size > limit_value else false end @@ -126,15 +117,9 @@ module Gitlab @has_previous_page = paginated_nodes.count > limit_value @has_previous_page ? paginated_nodes.last(limit_value) : paginated_nodes elsif loaded?(sliced_nodes) - if Feature.enabled?(:graphql_keyset_pagination_without_next_page_query) - sliced_nodes.take(limit_value + 1) # rubocop: disable CodeReuse/ActiveRecord - else - sliced_nodes.take(limit_value) # rubocop: disable CodeReuse/ActiveRecord - end - elsif Feature.enabled?(:graphql_keyset_pagination_without_next_page_query) - sliced_nodes.limit(limit_value + 1).to_a + sliced_nodes.take(limit_value + 1) # rubocop: disable CodeReuse/ActiveRecord else - sliced_nodes.limit(limit_value) + sliced_nodes.limit(limit_value + 1).to_a end end end diff --git a/lib/gitlab/group_search_results.rb b/lib/gitlab/group_search_results.rb index 4eea96f8344..b112740c4ad 100644 --- a/lib/gitlab/group_search_results.rb +++ b/lib/gitlab/group_search_results.rb @@ -12,23 +12,11 @@ module Gitlab # rubocop:disable CodeReuse/ActiveRecord def users - # get all groups the current user has access to - # ignore order inherited from GroupsFinder to improve performance - current_user_groups = GroupsFinder.new(current_user).execute.unscope(:order) + groups = group.self_and_hierarchy_intersecting_with_user_groups(current_user) + members = GroupMember.where(group: groups).non_invite - # the hierarchy of the current group - group_groups = @group.self_and_hierarchy.unscope(:order) - - # the groups where the above hierarchies intersect - intersect_groups = group_groups.where(id: current_user_groups) - - # members of @group hierarchy where the user has access to the groups - members = GroupMember.where(group: intersect_groups).non_invite - - # get all users the current user has access to (-> `SearchResults#users`), which also applies the query users = super - # filter users that belong to the previously selected groups users.where(id: members.select(:user_id)) end # rubocop:enable CodeReuse/ActiveRecord diff --git a/lib/gitlab/http.rb b/lib/gitlab/http.rb index 567c4dc899f..b05767c7ed4 100644 --- a/lib/gitlab/http.rb +++ b/lib/gitlab/http.rb @@ -59,7 +59,7 @@ module Gitlab raise ReadTotalTimeout, "Request timed out after #{elapsed} seconds" end - block.call fragment if block + yield fragment if block end rescue HTTParty::RedirectionTooDeep raise RedirectionTooDeep diff --git a/lib/gitlab/http_connection_adapter.rb b/lib/gitlab/http_connection_adapter.rb index 7b1657d3854..3ef60be67a9 100644 --- a/lib/gitlab/http_connection_adapter.rb +++ b/lib/gitlab/http_connection_adapter.rb @@ -44,7 +44,8 @@ module Gitlab Gitlab::UrlBlocker.validate!(url, allow_local_network: allow_local_requests?, allow_localhost: allow_local_requests?, allow_object_storage: allow_object_storage?, - dns_rebind_protection: dns_rebind_protection?) + dns_rebind_protection: dns_rebind_protection?, + schemes: %w[http https]) rescue Gitlab::UrlBlocker::BlockedUrlError => e raise Gitlab::HTTP::BlockedUrlError, "URL '#{url}' is blocked: #{e.message}" end diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index a42cac61a55..7a42ffca779 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -50,30 +50,30 @@ module Gitlab 'eo' => 0, 'es' => 35, 'fil_PH' => 0, - 'fr' => 85, + 'fr' => 94, 'gl_ES' => 0, 'id_ID' => 0, 'it' => 1, 'ja' => 30, - 'ko' => 21, - 'nb_NO' => 25, + 'ko' => 20, + 'nb_NO' => 24, 'nl_NL' => 0, 'pl_PL' => 3, - 'pt_BR' => 58, - 'ro_RO' => 98, - 'ru' => 25, + 'pt_BR' => 57, + 'ro_RO' => 96, + 'ru' => 26, 'si_LK' => 11, 'tr_TR' => 11, 'uk' => 52, - 'zh_CN' => 98, + 'zh_CN' => 97, 'zh_HK' => 1, - 'zh_TW' => 100 + 'zh_TW' => 99 }.freeze private_constant :TRANSLATION_LEVELS - def selectable_locales + def selectable_locales(minimum_translation_level = MINIMUM_TRANSLATION_LEVEL) AVAILABLE_LANGUAGES.reject do |code, _name| - percentage_translated_for(code) < MINIMUM_TRANSLATION_LEVEL + percentage_translated_for(code) < minimum_translation_level end end diff --git a/lib/gitlab/import_export/base/relation_factory.rb b/lib/gitlab/import_export/base/relation_factory.rb index b05d9cb2489..d1fd45882d3 100644 --- a/lib/gitlab/import_export/base/relation_factory.rb +++ b/lib/gitlab/import_export/base/relation_factory.rb @@ -211,23 +211,21 @@ module Gitlab def existing_or_new_object # Only find existing records to avoid mapping tables such as milestones # Otherwise always create the record, skipping the extra SELECT clause. - @existing_or_new_object ||= begin - if existing_object? - attribute_hash = attribute_hash_for(['events']) - - existing_object.assign_attributes(attribute_hash) if attribute_hash.any? - - existing_object - else - # Because of single-type inheritance, we need to be careful to use the `type` field - # See https://gitlab.com/gitlab-org/gitlab/issues/34860#note_235321497 - inheritance_column = relation_class.try(:inheritance_column) - inheritance_attributes = parsed_relation_hash.slice(inheritance_column) - object = relation_class.new(inheritance_attributes) - object.assign_attributes(parsed_relation_hash) - object - end - end + @existing_or_new_object ||= if existing_object? + attribute_hash = attribute_hash_for(['events']) + + existing_object.assign_attributes(attribute_hash) if attribute_hash.any? + + existing_object + else + # Because of single-type inheritance, we need to be careful to use the `type` field + # See https://gitlab.com/gitlab-org/gitlab/issues/34860#note_235321497 + inheritance_column = relation_class.try(:inheritance_column) + inheritance_attributes = parsed_relation_hash.slice(inheritance_column) + object = relation_class.new(inheritance_attributes) + object.assign_attributes(parsed_relation_hash) + object + end end def attribute_hash_for(attributes) diff --git a/lib/gitlab/import_export/decompressed_archive_size_validator.rb b/lib/gitlab/import_export/decompressed_archive_size_validator.rb index aa66fe8a5ae..564008e7a73 100644 --- a/lib/gitlab/import_export/decompressed_archive_size_validator.rb +++ b/lib/gitlab/import_export/decompressed_archive_size_validator.rb @@ -35,7 +35,7 @@ module Gitlab Timeout.timeout(TIMEOUT_LIMIT) do stderr_r, stderr_w = IO.pipe - stdout, wait_threads = Open3.pipeline_r(*command, pgroup: true, err: stderr_w ) + stdout, wait_threads = Open3.pipeline_r(*command, pgroup: true, err: stderr_w) # When validation is performed on a small archive (e.g. 100 bytes) # `wait_thr` finishes before we can get process group id. Do not diff --git a/lib/gitlab/import_export/group/import_export.yml b/lib/gitlab/import_export/group/import_export.yml index a08efdf400b..7f3254be3e8 100644 --- a/lib/gitlab/import_export/group/import_export.yml +++ b/lib/gitlab/import_export/group/import_export.yml @@ -132,3 +132,23 @@ ee: - :push_event_payload - iterations_cadences: - :iterations + +# When associated resources are from outside the group, you might need to +# validate that a user who is exporting the group can access these +# associations. `include_if_exportable` accepts an array of associations for a +# resource. During export, the `exportable_association?` method on the +# resource is called with the association's name and user to validate if +# associated resource can be included in the export. +# +# This definition will call epic's `exportable_association?(:parent, +# current_user: current_user)` method and include epic's parent association +# for each epic only if the method returns true: +# +# include_if_exportable: +# group: +# epics: +# - :parent +include_if_exportable: + group: + epics: + - :parent diff --git a/lib/gitlab/import_export/json/legacy_reader.rb b/lib/gitlab/import_export/json/legacy_reader.rb index dc80c92f507..ee360020556 100644 --- a/lib/gitlab/import_export/json/legacy_reader.rb +++ b/lib/gitlab/import_export/json/legacy_reader.rb @@ -27,7 +27,7 @@ module Gitlab end def read_hash - Gitlab::Json.parse(IO.read(@path)) + Gitlab::Json.parse(::File.read(@path)) rescue StandardError => e Gitlab::ErrorTracking.log_exception(e) raise Gitlab::ImportExport::Error, 'Incorrect JSON format' diff --git a/lib/gitlab/import_export/lfs_restorer.rb b/lib/gitlab/import_export/lfs_restorer.rb index 9931b09e9ca..83aab6d031e 100644 --- a/lib/gitlab/import_export/lfs_restorer.rb +++ b/lib/gitlab/import_export/lfs_restorer.rb @@ -71,7 +71,7 @@ module Gitlab @lfs_json ||= begin - json = IO.read(lfs_json_path) + json = File.read(lfs_json_path) Gitlab::Json.parse(json) rescue StandardError raise Gitlab::ImportExport::Error, 'Incorrect JSON format' diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index c94549a2b3f..0ad19f82e71 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -15,15 +15,13 @@ module Gitlab def map @map ||= - begin - @exported_members.each_with_object(missing_keys_tracking_hash) do |member, hash| - if member['user'] - old_user_id = member['user']['id'] - existing_user_id = existing_users_email_map[get_email(member)] - hash[old_user_id] = existing_user_id if existing_user_id && add_team_member(member, existing_user_id) - else - add_team_member(member) - end + @exported_members.each_with_object(missing_keys_tracking_hash) do |member, hash| + if member['user'] + old_user_id = member['user']['id'] + existing_user_id = existing_users_email_map[get_email(member)] + hash[old_user_id] = existing_user_id if existing_user_id && add_team_member(member, existing_user_id) + else + add_team_member(member) end end end diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 2d9c8d1108e..cc69ed55744 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -763,6 +763,19 @@ excluded_attributes: - :import_type - :import_source - :integrations + - :push_hooks_integrations + - :tag_push_hooks_integrations + - :issue_hooks_integrations + - :confidential_issue_hooks_integrations + - :merge_request_hooks_integrations + - :note_hooks_integrations + - :confidential_note_hooks_integrations + - :job_hooks_integrations + - :archive_trace_hooks_integrations + - :pipeline_hooks_integrations + - :wiki_page_hooks_integrations + - :deployment_hooks_integrations + - :alert_hooks_integrations - :mirror - :runners_token - :runners_token_encrypted @@ -1209,7 +1222,9 @@ ee: - :description iterations_cadence: - :title - + excluded_attributes: + project: + - :vulnerability_hooks_integrations preloads: issues: epic: diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb index bd34cd3ff6e..05b96f7e8ce 100644 --- a/lib/gitlab/import_export/project/tree_saver.rb +++ b/lib/gitlab/import_export/project/tree_saver.rb @@ -81,15 +81,13 @@ module Gitlab end def json_writer - @json_writer ||= begin - if ::Feature.enabled?(:project_export_as_ndjson, @project.namespace) - full_path = File.join(@shared.export_path, 'tree') - Gitlab::ImportExport::Json::NdjsonWriter.new(full_path) - else - full_path = File.join(@shared.export_path, ImportExport.project_filename) - Gitlab::ImportExport::Json::LegacyWriter.new(full_path, allowed_path: 'project') - end - end + @json_writer ||= if ::Feature.enabled?(:project_export_as_ndjson, @project.namespace) + full_path = File.join(@shared.export_path, 'tree') + Gitlab::ImportExport::Json::NdjsonWriter.new(full_path) + else + full_path = File.join(@shared.export_path, ImportExport.project_filename) + Gitlab::ImportExport::Json::LegacyWriter.new(full_path, allowed_path: 'project') + end end end end diff --git a/lib/gitlab/import_export/remote_stream_upload.rb b/lib/gitlab/import_export/remote_stream_upload.rb index f3bd241c0bd..1fb3faf0767 100644 --- a/lib/gitlab/import_export/remote_stream_upload.rb +++ b/lib/gitlab/import_export/remote_stream_upload.rb @@ -25,6 +25,7 @@ module Gitlab end end end + class StreamError < StandardError attr_reader :response_body @@ -33,6 +34,7 @@ module Gitlab @response_body = response_body end end + class ChunkStream DEFAULT_BUFFER_SIZE = 128.kilobytes diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index 1c6629cf942..cc214d730fe 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -46,6 +46,11 @@ module Gitlab ) Repositories::DestroyService.new(repository).execute + + # Because Gitlab::Git::Repository#remove happens inside a run_after_commit + # callback in the Repositories::DestroyService#execute we need to trigger + # the callback. + repository.project.touch end end end diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index 5d3a6b0c6e1..a94ea6f595b 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -15,7 +15,6 @@ module Gitlab ImportSource.new('bitbucket', 'Bitbucket Cloud', Gitlab::BitbucketImport::Importer), ImportSource.new('bitbucket_server', 'Bitbucket Server', Gitlab::BitbucketServerImport::Importer), ImportSource.new('gitlab', 'GitLab.com', Gitlab::GitlabImport::Importer), - ImportSource.new('google_code', 'Google Code', nil), ImportSource.new('fogbugz', 'FogBugz', Gitlab::FogbugzImport::Importer), ImportSource.new('git', 'Repository by URL', nil), ImportSource.new('gitlab_project', 'GitLab export', Gitlab::ImportExport::Importer), diff --git a/lib/gitlab/incident_management/pager_duty/incident_issue_description.rb b/lib/gitlab/incident_management/pager_duty/incident_issue_description.rb index 6aeeb1d31aa..cbc4f126293 100644 --- a/lib/gitlab/incident_management/pager_duty/incident_issue_description.rb +++ b/lib/gitlab/incident_management/pager_duty/incident_issue_description.rb @@ -19,7 +19,7 @@ module Gitlab "**Incident key:** #{incident_payload['incident_key']}", "**Created at:** #{markdown_incident_created_at}", "**Assignees:** #{markdown_assignees.join(', ')}", - "**Impacted services:** #{markdown_impacted_services.join(', ')}" + "**Impacted service:** #{markdown_impacted_service}" ].join(markdown_line_break) end @@ -47,10 +47,9 @@ module Gitlab end end - def markdown_impacted_services - Array(incident_payload['impacted_services']).map do |is| - markdown_link(is['summary'], is['url']) - end + def markdown_impacted_service + service = incident_payload['impacted_service'] + markdown_link(service['summary'], service['url']) unless service.nil? end def markdown_link(label, url) diff --git a/lib/gitlab/instrumentation/redis.rb b/lib/gitlab/instrumentation/redis.rb index a371930621d..a664656c467 100644 --- a/lib/gitlab/instrumentation/redis.rb +++ b/lib/gitlab/instrumentation/redis.rb @@ -39,7 +39,8 @@ module Gitlab end end - %i[get_request_count query_time read_bytes write_bytes].each do |method| + %i[get_request_count get_cross_slot_request_count get_allowed_cross_slot_request_count query_time read_bytes + write_bytes].each do |method| define_method method do STORAGES.sum(&method) end diff --git a/lib/gitlab/instrumentation/redis_base.rb b/lib/gitlab/instrumentation/redis_base.rb index 268c6cdf459..de24132a28e 100644 --- a/lib/gitlab/instrumentation/redis_base.rb +++ b/lib/gitlab/instrumentation/redis_base.rb @@ -45,6 +45,16 @@ module Gitlab ::RequestStore[write_bytes_key] += num_bytes end + def increment_cross_slot_request_count(amount = 1) + ::RequestStore[cross_slots_key] ||= 0 + ::RequestStore[cross_slots_key] += amount + end + + def increment_allowed_cross_slot_request_count(amount = 1) + ::RequestStore[allowed_cross_slots_key] ||= 0 + ::RequestStore[allowed_cross_slots_key] += amount + end + def get_request_count ::RequestStore[request_count_key] || 0 end @@ -61,13 +71,32 @@ module Gitlab ::RequestStore[call_details_key] ||= [] end + def get_cross_slot_request_count + ::RequestStore[cross_slots_key] || 0 + end + + def get_allowed_cross_slot_request_count + ::RequestStore[allowed_cross_slots_key] || 0 + end + def query_time query_time = ::RequestStore[call_duration_key] || 0 query_time.round(::Gitlab::InstrumentationHelper::DURATION_PRECISION) end def redis_cluster_validate!(commands) - ::Gitlab::Instrumentation::RedisClusterValidator.validate!(commands) if @redis_cluster_validation + return true unless @redis_cluster_validation + + result = ::Gitlab::Instrumentation::RedisClusterValidator.validate(commands) + return true if result.nil? + + if !result[:valid] && !result[:allowed] && (Rails.env.development? || Rails.env.test?) + raise RedisClusterValidator::CrossSlotError, "Redis command #{result[:command_name]} arguments hash to different slots. See https://docs.gitlab.com/ee/development/redis.html#multi-key-commands" + end + + increment_allowed_cross_slot_request_count if result[:allowed] + + result[:valid] end def enable_redis_cluster_validation @@ -122,6 +151,14 @@ module Gitlab strong_memoize(:call_details_key) { build_key(:redis_call_details) } end + def cross_slots_key + strong_memoize(:cross_slots_key) { build_key(:redis_cross_slot_request_count) } + end + + def allowed_cross_slots_key + strong_memoize(:allowed_cross_slots_key) { build_key(:redis_allowed_cross_slot_request_count) } + end + def build_key(namespace) "#{storage_key}_#{namespace}" end diff --git a/lib/gitlab/instrumentation/redis_cluster_validator.rb b/lib/gitlab/instrumentation/redis_cluster_validator.rb index 36d3e088956..1567e54d8da 100644 --- a/lib/gitlab/instrumentation/redis_cluster_validator.rb +++ b/lib/gitlab/instrumentation/redis_cluster_validator.rb @@ -183,19 +183,22 @@ module Gitlab CrossSlotError = Class.new(StandardError) class << self - def validate!(commands) - return unless Rails.env.development? || Rails.env.test? - return if allow_cross_slot_commands? + def validate(commands) return if commands.empty? # early exit for single-command (non-pipelined) if it is a single-key-command command_name = commands.size > 1 ? "PIPELINE/MULTI" : commands.first.first.to_s.upcase return if commands.size == 1 && REDIS_COMMANDS.dig(command_name, :single_key) - key_slots = commands.map { |command| key_slots(command) }.flatten - if key_slots.uniq.many? # rubocop: disable CodeReuse/ActiveRecord - raise CrossSlotError, "Redis command #{command_name} arguments hash to different slots. See https://docs.gitlab.com/ee/development/redis.html#multi-key-commands" - end + keys = commands.map { |command| extract_keys(command) }.flatten + + { + # calculate key-slots only if not allowed + valid: allow_cross_slot_commands? || !has_cross_slot_keys?(keys), + command_name: command_name, + key_count: keys.size, + allowed: allow_cross_slot_commands? + } end # Keep track of the call stack to allow nested calls to work. @@ -210,15 +213,17 @@ module Gitlab private - def key_slots(command) + def extract_keys(command) argument_positions = REDIS_COMMANDS[command.first.to_s.upcase] return [] unless argument_positions arguments = command.flatten[argument_positions[:first]..argument_positions[:last]] - arguments.each_slice(argument_positions[:step]).map do |args| - key_slot(args.first) - end + arguments.each_slice(argument_positions[:step]).map(&:first) + end + + def has_cross_slot_keys?(keys) + keys.map { |key| key_slot(key) }.uniq.many? # rubocop: disable CodeReuse/ActiveRecord end def allow_cross_slot_commands? diff --git a/lib/gitlab/instrumentation/redis_interceptor.rb b/lib/gitlab/instrumentation/redis_interceptor.rb index f19279df2fe..35dd7cbfeb8 100644 --- a/lib/gitlab/instrumentation/redis_interceptor.rb +++ b/lib/gitlab/instrumentation/redis_interceptor.rb @@ -33,7 +33,10 @@ module Gitlab def instrument_call(commands) start = Gitlab::Metrics::System.monotonic_time # must come first so that 'start' is always defined instrumentation_class.instance_count_request(commands.size) - instrumentation_class.redis_cluster_validate!(commands) + + if !instrumentation_class.redis_cluster_validate!(commands) && ::RequestStore.active? + instrumentation_class.increment_cross_slot_request_count + end yield rescue ::Redis::BaseError => ex @@ -62,13 +65,11 @@ module Gitlab # This count is an approximation that omits the Redis protocol overhead # of type prefixes, length prefixes and line endings. command.each do |x| - size += begin - if x.is_a? Array - x.inject(0) { |sum, y| sum + y.to_s.bytesize } - else - x.to_s.bytesize - end - end + size += if x.is_a? Array + x.inject(0) { |sum, y| sum + y.to_s.bytesize } + else + x.to_s.bytesize + end end instrumentation_class.increment_write_bytes(size) diff --git a/lib/gitlab/instrumentation/redis_payload.rb b/lib/gitlab/instrumentation/redis_payload.rb index 86a6525c8d0..62a4d1a846f 100644 --- a/lib/gitlab/instrumentation/redis_payload.rb +++ b/lib/gitlab/instrumentation/redis_payload.rb @@ -20,6 +20,8 @@ module Gitlab { "#{key_prefix}_calls": -> { get_request_count }, + "#{key_prefix}_cross_slot_calls": -> { get_cross_slot_request_count }, + "#{key_prefix}_allowed_cross_slot_calls": -> { get_allowed_cross_slot_request_count }, "#{key_prefix}_duration_s": -> { query_time }, "#{key_prefix}_read_bytes": -> { read_bytes }, "#{key_prefix}_write_bytes": -> { write_bytes } diff --git a/lib/gitlab/instrumentation_helper.rb b/lib/gitlab/instrumentation_helper.rb index b8a2567b775..15a760fada0 100644 --- a/lib/gitlab/instrumentation_helper.rb +++ b/lib/gitlab/instrumentation_helper.rb @@ -35,6 +35,7 @@ module Gitlab instrument_uploads(payload) instrument_rate_limiting_gates(payload) instrument_global_search_api(payload) + instrument_ldap(payload) end def instrument_gitaly(payload) @@ -136,6 +137,14 @@ module Gitlab payload.merge!(::Gitlab::Instrumentation::GlobalSearchApi.payload) end + def instrument_ldap(payload) + ldap_count = Gitlab::Metrics::Subscribers::Ldap.count + + return if ldap_count == 0 + + payload.merge! Gitlab::Metrics::Subscribers::Ldap.payload + end + # Returns the queuing duration for a Sidekiq job in seconds, as a float, if the # `enqueued_at` field or `created_at` field is available. # diff --git a/lib/gitlab/issuable_metadata.rb b/lib/gitlab/issuable_metadata.rb index d0702fb5c7d..12ec6447251 100644 --- a/lib/gitlab/issuable_metadata.rb +++ b/lib/gitlab/issuable_metadata.rb @@ -27,8 +27,8 @@ module Gitlab def data return {} if issuable_ids.empty? - issuable_ids.each_with_object({}) do |id, issuable_meta| - issuable_meta[id] = metadata_for_issuable(id) + issuable_ids.index_with do |id| + metadata_for_issuable(id) end end diff --git a/lib/gitlab/jira/http_client.rb b/lib/gitlab/jira/http_client.rb index 02b0c902a70..7abfe8e38e8 100644 --- a/lib/gitlab/jira/http_client.rb +++ b/lib/gitlab/jira/http_client.rb @@ -35,12 +35,6 @@ module Gitlab request_params[:base_uri] = uri.to_s request_params.merge!(auth_params) - if Feature.enabled?(:jira_raise_timeouts, type: :ops) - request_params[:open_timeout] = 2.minutes - request_params[:read_timeout] = 2.minutes - request_params[:write_timeout] = 2.minutes - end - result = Gitlab::HTTP.public_send(http_method, path, **request_params) # rubocop:disable GitlabSecurity/PublicSend @authenticated = result.response.is_a?(Net::HTTPOK) store_cookies(result) if options[:use_cookies] diff --git a/lib/gitlab/jira_import/issues_importer.rb b/lib/gitlab/jira_import/issues_importer.rb index 5057317ae01..7b031c26b72 100644 --- a/lib/gitlab/jira_import/issues_importer.rb +++ b/lib/gitlab/jira_import/issues_importer.rb @@ -16,7 +16,7 @@ module Gitlab @start_at = Gitlab::JiraImport.get_issues_next_start_at(project.id) @imported_items_cache_key = JiraImport.already_imported_cache_key(:issues, project.id) @job_waiter = JobWaiter.new - @issue_type_id = WorkItems::Type.default_issue_type.id + @issue_type_id = ::WorkItems::Type.default_issue_type.id end def execute diff --git a/lib/gitlab/jwt_authenticatable.rb b/lib/gitlab/jwt_authenticatable.rb index 08d9f69497e..7c36bbf3426 100644 --- a/lib/gitlab/jwt_authenticatable.rb +++ b/lib/gitlab/jwt_authenticatable.rb @@ -3,7 +3,7 @@ module Gitlab module JwtAuthenticatable # Supposedly the effective key size for HMAC-SHA256 is 256 bits, i.e. 32 - # bytes https://tools.ietf.org/html/rfc4868#section-2.6 + # bytes https://www.rfc-editor.org/rfc/rfc4868#section-2.6 SECRET_LENGTH = 32 def self.included(base) diff --git a/lib/gitlab/jwt_token.rb b/lib/gitlab/jwt_token.rb index 11bc5479b6e..83aa7fa4a15 100644 --- a/lib/gitlab/jwt_token.rb +++ b/lib/gitlab/jwt_token.rb @@ -42,7 +42,7 @@ module Gitlab def ==(other) self.id == other.id && - self.payload == other.payload + self.payload == other.payload end def issued_at=(value) diff --git a/lib/gitlab/kubernetes/helm/v2/install_command.rb b/lib/gitlab/kubernetes/helm/v2/install_command.rb index 10e16723e45..c50db6bf177 100644 --- a/lib/gitlab/kubernetes/helm/v2/install_command.rb +++ b/lib/gitlab/kubernetes/helm/v2/install_command.rb @@ -36,13 +36,13 @@ module Gitlab # installation and uprade of applications def install_command command = ['helm', 'upgrade', name, chart] + - install_flag + - rollback_support_flag + - reset_values_flag + - optional_version_flag + - rbac_create_flag + - namespace_flag + - value_flag + install_flag + + rollback_support_flag + + reset_values_flag + + optional_version_flag + + rbac_create_flag + + namespace_flag + + value_flag command.shelljoin end diff --git a/lib/gitlab/kubernetes/helm/v2/patch_command.rb b/lib/gitlab/kubernetes/helm/v2/patch_command.rb index 2855e6444b1..40e56771e47 100644 --- a/lib/gitlab/kubernetes/helm/v2/patch_command.rb +++ b/lib/gitlab/kubernetes/helm/v2/patch_command.rb @@ -37,10 +37,10 @@ module Gitlab def upgrade_command command = ['helm', 'upgrade', name, chart] + - reuse_values_flag + - version_flag + - namespace_flag + - value_flag + reuse_values_flag + + version_flag + + namespace_flag + + value_flag command.shelljoin end diff --git a/lib/gitlab/kubernetes/helm/v3/install_command.rb b/lib/gitlab/kubernetes/helm/v3/install_command.rb index 20d17f49115..8d521f0dcd4 100644 --- a/lib/gitlab/kubernetes/helm/v3/install_command.rb +++ b/lib/gitlab/kubernetes/helm/v3/install_command.rb @@ -33,13 +33,13 @@ module Gitlab # installation and uprade of applications def install_command command = ['helm', 'upgrade', name, chart] + - install_flag + - rollback_support_flag + - reset_values_flag + - optional_version_flag + - rbac_create_flag + - namespace_flag + - value_flag + install_flag + + rollback_support_flag + + reset_values_flag + + optional_version_flag + + rbac_create_flag + + namespace_flag + + value_flag command.shelljoin end diff --git a/lib/gitlab/kubernetes/helm/v3/patch_command.rb b/lib/gitlab/kubernetes/helm/v3/patch_command.rb index 00f340591e7..1278e524bd2 100644 --- a/lib/gitlab/kubernetes/helm/v3/patch_command.rb +++ b/lib/gitlab/kubernetes/helm/v3/patch_command.rb @@ -34,10 +34,10 @@ module Gitlab def upgrade_command command = ['helm', 'upgrade', name, chart] + - reuse_values_flag + - version_flag + - namespace_flag + - value_flag + reuse_values_flag + + version_flag + + namespace_flag + + value_flag command.shelljoin end diff --git a/lib/gitlab/kubernetes/kube_client.rb b/lib/gitlab/kubernetes/kube_client.rb index 92ffa65fe74..44e53e9ec70 100644 --- a/lib/gitlab/kubernetes/kube_client.rb +++ b/lib/gitlab/kubernetes/kube_client.rb @@ -161,7 +161,7 @@ module Gitlab def validate_url! return if Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services? - Gitlab::UrlBlocker.validate!(api_prefix, allow_local_network: false) + Gitlab::UrlBlocker.validate!(api_prefix, allow_local_network: false, schemes: %w[http https]) end def service_account_exists?(resource) diff --git a/lib/gitlab/memory/jemalloc.rb b/lib/gitlab/memory/jemalloc.rb index e20e186cab9..6025e6ab6f2 100644 --- a/lib/gitlab/memory/jemalloc.rb +++ b/lib/gitlab/memory/jemalloc.rb @@ -14,41 +14,22 @@ module Gitlab STATS_DEFAULT_FORMAT = :json - FILENAME_PREFIX = 'jemalloc_stats' - # Return jemalloc stats as a string. def stats(format: STATS_DEFAULT_FORMAT) - verify_format!(format) - - with_malloc_stats_print do |stats_print| - StringIO.new.tap { |io| write_stats(stats_print, io, STATS_FORMATS[format]) }.string - end + dump_stats(StringIO.new, format: format).string end - # Write jemalloc stats to the given directory - # @param [String] path Directory path the dump will be put into - # @param [String] tmp_dir Directory path the dump will be streaming to. It is moved to `path` when finished. - # @param [String] format `json` or `txt` - # @param [String] filename_label Optional custom string that will be injected into the file name, e.g. `worker_0` - # @return [String] Full path to the resulting dump file - def dump_stats(path:, tmp_dir: Dir.tmpdir, format: STATS_DEFAULT_FORMAT, filename_label: nil) + # Streams jemalloc stats to the given IO object. + def dump_stats(io, format: STATS_DEFAULT_FORMAT) verify_format!(format) format_settings = STATS_FORMATS[format] - tmp_file_path = File.join(tmp_dir, file_name(format_settings[:extension], filename_label)) - file_path = File.join(path, file_name(format_settings[:extension], filename_label)) with_malloc_stats_print do |stats_print| - File.open(tmp_file_path, 'wb') do |io| - write_stats(stats_print, io, format_settings) - end + write_stats(stats_print, io, format_settings) end - # On OSX, `with_malloc_stats_print` is no-op, and, as result, no file will be written - return unless File.exist?(tmp_file_path) - - FileUtils.mv(tmp_file_path, file_path) - file_path + io end private @@ -95,10 +76,6 @@ module Gitlab stats_print.call(callback, nil, format[:options]) end - - def file_name(extension, filename_label) - [FILENAME_PREFIX, $$, filename_label, Time.current.to_i, extension].reject(&:blank?).join('.') - end end end end diff --git a/lib/gitlab/memory/reporter.rb b/lib/gitlab/memory/reporter.rb new file mode 100644 index 00000000000..710c89c6216 --- /dev/null +++ b/lib/gitlab/memory/reporter.rb @@ -0,0 +1,130 @@ +# frozen_string_literal: true + +module Gitlab + module Memory + class Reporter + attr_reader :reports_path + + def initialize(reports_path: nil, logger: Gitlab::AppLogger) + @reports_path = reports_path || ENV["GITLAB_DIAGNOSTIC_REPORTS_PATH"] || Dir.mktmpdir + @logger = logger + + @worker_id = ::Prometheus::PidProvider.worker_id + @worker_uuid = SecureRandom.uuid + + init_prometheus_metrics + end + + def run_report(report) + return false unless report.active? + + @logger.info( + log_labels( + message: 'started', + perf_report: report.name + )) + + start_monotonic_time = Gitlab::Metrics::System.monotonic_time + start_thread_cpu_time = Gitlab::Metrics::System.thread_cpu_time + + report_file = store_report(report) + + cpu_s = Gitlab::Metrics::System.thread_cpu_duration(start_thread_cpu_time) + duration_s = Gitlab::Metrics::System.monotonic_time - start_monotonic_time + + @logger.info( + log_labels( + message: 'finished', + perf_report: report.name, + cpu_s: cpu_s.round(2), + duration_s: duration_s.round(2), + perf_report_file: report_file, + perf_report_size_bytes: file_size(report_file) + )) + + @report_duration_counter.increment({ report: report.name }, duration_s) + + true + rescue StandardError => e + @logger.error( + log_labels( + message: 'failed', + perf_report: report.name, + error: e.inspect + )) + + false + end + + private + + def store_report(report) + # Store report in tmp subdir while it is still streaming. + # This will clearly separate finished reports from the files we are still writing to. + tmp_dir = File.join(@reports_path, 'tmp') + FileUtils.mkdir_p(tmp_dir) + + report_file = file_name(report) + tmp_file_path = File.join(tmp_dir, report_file) + + io_r, io_w = IO.pipe + pid = nil + File.open(tmp_file_path, 'wb') do |file| + extras = { + in: io_r, + out: file, + err: $stderr + } + pid = Process.spawn('gzip', '--fast', **extras) + io_r.close + + report.run(io_w) + io_w.close + + Process.waitpid(pid) + end + + File.join(@reports_path, report_file).tap do |report_file_path| + FileUtils.mv(tmp_file_path, report_file_path) + end + ensure + [io_r, io_w].each(&:close) + + # Make sure we don't leave any running processes behind. + Gitlab::ProcessManagement.signal(pid, :KILL) if pid + end + + def log_labels(**extra_labels) + { + pid: $$, + worker_id: @worker_id, + perf_report_worker_uuid: @worker_uuid + }.merge(extra_labels) + end + + def file_name(report) + timestamp = Time.current.strftime('%Y-%m-%d.%H:%M:%S:%L') + + report_id = [@worker_id, @worker_uuid].join(".") + + [report.name, timestamp, report_id, 'gz'].compact_blank.join('.') + end + + def file_size(file_path) + File.size(file_path.to_s) + rescue Errno::ENOENT + 0 + end + + def init_prometheus_metrics + default_labels = { pid: @worker_id } + + @report_duration_counter = Gitlab::Metrics.counter( + :gitlab_diag_report_duration_seconds_total, + 'Total time elapsed for running diagnostic report', + default_labels + ) + end + end + end +end diff --git a/lib/gitlab/memory/reports/heap_dump.rb b/lib/gitlab/memory/reports/heap_dump.rb new file mode 100644 index 00000000000..95779407f12 --- /dev/null +++ b/lib/gitlab/memory/reports/heap_dump.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module Memory + module Reports + class HeapDump + class << self + def enqueue! + @write_heap_dump = true + end + + def enqueued? + !!@write_heap_dump + end + end + + def name + 'heap_dump' + end + + def active? + Feature.enabled?(:report_heap_dumps, type: :ops) + end + + def run(writer) + return false unless self.class.enqueued? + + ObjectSpace.dump_all(output: writer) + + true + end + end + end + end +end diff --git a/lib/gitlab/memory/reports/jemalloc_stats.rb b/lib/gitlab/memory/reports/jemalloc_stats.rb index 05f0717d7c3..cfda409594f 100644 --- a/lib/gitlab/memory/reports/jemalloc_stats.rb +++ b/lib/gitlab/memory/reports/jemalloc_stats.rb @@ -4,70 +4,19 @@ module Gitlab module Memory module Reports class JemallocStats - # On prod, Jemalloc reports sizes were ~2.5 MB: - # https://gitlab.com/gitlab-com/gl-infra/reliability/-/issues/15993#note_1014767214 - # We configured 1GB emptyDir per pod: - # https://gitlab.com/gitlab-com/gl-infra/k8s-workloads/gitlab-com/-/merge_requests/1949 - # The pod will be evicted when the size limit is exceeded. We never want this to happen, for availability. - # - # With the default, we have a headroom (250*2.5MB=625<1000 MB) to fit into configured emptyDir. - # It would allow us to keep 3+ days worth of reports for 6 workers running every 2 hours: 3*6*12=216<250 - # - # The cleanup logic will be redundant after we'll implement the uploads, which would perform the cleanup. - DEFAULT_MAX_REPORTS_STORED = 250 - - def initialize(reports_path:) - @reports_path = reports_path - - # Store report in tmp subdir while it is still streaming. - # This will clearly separate finished reports from the files we are still writing to. - @tmp_dir = File.join(@reports_path, 'tmp') - FileUtils.mkdir_p(@tmp_dir) + def name + 'jemalloc_stats' end - def run + def run(writer) return unless active? - Gitlab::Memory::Jemalloc.dump_stats(path: reports_path, tmp_dir: @tmp_dir, filename_label: worker_id).tap do - cleanup - end + Gitlab::Memory::Jemalloc.dump_stats(writer) end def active? Feature.enabled?(:report_jemalloc_stats, type: :ops) end - - private - - attr_reader :reports_path - - def cleanup - reports_files_modified_order[0...-max_reports_stored].each do |f| - File.unlink(f) if File.exist?(f) - rescue Errno::ENOENT - # Path does not exist: Ignore. We already check `File.exist?` - # Rescue to be extra safe, because each worker could perform a cleanup - end - end - - def reports_files_modified_order - pattern = File.join(reports_path, "#{Gitlab::Memory::Jemalloc::FILENAME_PREFIX}*") - - Dir.glob(pattern).sort_by do |f| - test('M', f) - rescue Errno::ENOENT - # Path does not exist: Return any timestamp to proceed with the sort - Time.current - end - end - - def worker_id - ::Prometheus::PidProvider.worker_id - end - - def max_reports_stored - ENV["GITLAB_DIAGNOSTIC_REPORTS_JEMALLOC_MAX_REPORTS_STORED"] || DEFAULT_MAX_REPORTS_STORED - end end end end diff --git a/lib/gitlab/memory/reports_daemon.rb b/lib/gitlab/memory/reports_daemon.rb index 0dfc31235e7..9bbfe81116d 100644 --- a/lib/gitlab/memory/reports_daemon.rb +++ b/lib/gitlab/memory/reports_daemon.rb @@ -7,9 +7,7 @@ module Gitlab DEFAULT_SLEEP_MAX_DELTA_S = 600 # 0..10 minutes DEFAULT_SLEEP_BETWEEN_REPORTS_S = 120 # 2 minutes - DEFAULT_REPORTS_PATH = Dir.tmpdir - - def initialize(**options) + def initialize(reporter: nil, reports: nil, **options) super @alive = true @@ -21,31 +19,20 @@ module Gitlab @sleep_between_reports_s = ENV['GITLAB_DIAGNOSTIC_REPORTS_SLEEP_BETWEEN_REPORTS_S']&.to_i || DEFAULT_SLEEP_BETWEEN_REPORTS_S - @reports_path = - ENV["GITLAB_DIAGNOSTIC_REPORTS_PATH"] || DEFAULT_REPORTS_PATH - - @reports = [Gitlab::Memory::Reports::JemallocStats.new(reports_path: reports_path)] - - init_prometheus_metrics + @reporter = reporter || Reporter.new + @reports = reports || [ + Gitlab::Memory::Reports::JemallocStats.new + ] end - attr_reader :sleep_s, :sleep_max_delta_s, :sleep_between_reports_s, :reports_path + attr_reader :sleep_s, :sleep_max_delta_s, :sleep_between_reports_s def run_thread while alive sleep interval_with_jitter reports.select(&:active?).each do |report| - start_monotonic_time = Gitlab::Metrics::System.monotonic_time - start_thread_cpu_time = Gitlab::Metrics::System.thread_cpu_time - - file_path = report.run - - cpu_s = Gitlab::Metrics::System.thread_cpu_duration(start_thread_cpu_time) - duration_s = Gitlab::Metrics::System.monotonic_time - start_monotonic_time - - log_report(label: report_label(report), cpu_s: cpu_s, duration_s: duration_s, size: file_size(file_path)) - @report_duration_counter.increment({ report: report_label(report) }, duration_s) + @reporter.run_report(report) sleep sleep_between_reports_s end @@ -62,45 +49,9 @@ module Gitlab sleep_s + rand(sleep_max_delta_s) end - def log_report(label:, duration_s:, cpu_s:, size:) - Gitlab::AppLogger.info( - message: 'finished', - pid: $$, - worker_id: worker_id, - perf_report: label, - duration_s: duration_s.round(2), - cpu_s: cpu_s.round(2), - perf_report_size_bytes: size - ) - end - - def worker_id - ::Prometheus::PidProvider.worker_id - end - - def report_label(report) - report.class.to_s.demodulize.underscore - end - def stop_working @alive = false end - - def init_prometheus_metrics - default_labels = { pid: worker_id } - - @report_duration_counter = Gitlab::Metrics.counter( - :gitlab_diag_report_duration_seconds_total, - 'Total time elapsed for running diagnostic report', - default_labels - ) - end - - def file_size(file_path) - File.size(file_path.to_s) - rescue Errno::ENOENT - 0 - end end end end diff --git a/lib/gitlab/memory/watchdog.rb b/lib/gitlab/memory/watchdog.rb index 19dfc640b5d..25af5bd781a 100644 --- a/lib/gitlab/memory/watchdog.rb +++ b/lib/gitlab/memory/watchdog.rb @@ -50,27 +50,17 @@ module Gitlab def initialize @configuration = Configuration.new @alive = true - - init_prometheus_metrics end ## - # Configuration for Watchdog, use like: - # - # watchdog.configure do |config| - # config.handler = Gitlab::Memory::Watchdog::TermProcessHandler - # config.sleep_time_seconds = 60 - # config.logger = Gitlab::AppLogger - # config.monitors do |stack| - # stack.push MyMonitorClass, args*, max_strikes:, kwargs**, &block - # end - # end + # Configuration for Watchdog, see Gitlab::Memory::Watchdog::Configurator + # for examples. def configure - yield @configuration + yield configuration end def call - logger.info(log_labels.merge(message: 'started')) + event_reporter.started(log_labels) while @alive sleep(sleep_time_seconds) @@ -78,35 +68,45 @@ module Gitlab monitor if Feature.enabled?(:gitlab_memory_watchdog, type: :ops) end - logger.info(log_labels.merge(message: 'stopped')) + event_reporter.stopped(log_labels(memwd_reason: @reason).compact) end - def stop + def stop(reason: nil) + @reason = reason @alive = false end private + attr_reader :configuration + + delegate :event_reporter, :monitors, :sleep_time_seconds, to: :configuration + def monitor - @configuration.monitors.call_each do |result| + if monitors.empty? + stop(reason: 'monitors are not configured') + return + end + + monitors.call_each do |result| break unless @alive next unless result.threshold_violated? - @counter_violations.increment(reason: result.monitor_name) + event_reporter.threshold_violated(result.monitor_name) next unless result.strikes_exceeded? - @alive = !memory_limit_exceeded_callback(result.monitor_name, result.payload) + strike_exceeded_callback(result.monitor_name, result.payload) end end - def memory_limit_exceeded_callback(monitor_name, monitor_payload) - all_labels = log_labels.merge(monitor_payload) - logger.warn(all_labels) - @counter_violations_handled.increment(reason: monitor_name) + def strike_exceeded_callback(monitor_name, monitor_payload) + event_reporter.strikes_exceeded(monitor_name, log_labels(monitor_payload)) + + Gitlab::Memory::Reports::HeapDump.enqueue! - handler.call + stop(reason: 'successfully handled') if handler.call end def handler @@ -114,46 +114,13 @@ module Gitlab # all that happens is we collect logs and Prometheus events for fragmentation violations. return NullHandler.instance unless Feature.enabled?(:enforce_memory_watchdog, type: :ops) - @configuration.handler - end - - def logger - @configuration.logger + configuration.handler end - def sleep_time_seconds - @configuration.sleep_time_seconds - end - - def log_labels - { - pid: $$, - worker_id: worker_id, + def log_labels(extra = {}) + extra.merge( memwd_handler_class: handler.class.name, - memwd_sleep_time_s: sleep_time_seconds, - memwd_rss_bytes: process_rss_bytes - } - end - - def process_rss_bytes - Gitlab::Metrics::System.memory_usage_rss[:total] - end - - def worker_id - ::Prometheus::PidProvider.worker_id - end - - def init_prometheus_metrics - default_labels = { pid: worker_id } - @counter_violations = Gitlab::Metrics.counter( - :gitlab_memwd_violations_total, - 'Total number of times a Ruby process violated a memory threshold', - default_labels - ) - @counter_violations_handled = Gitlab::Metrics.counter( - :gitlab_memwd_violations_handled_total, - 'Total number of times Ruby process memory violations were handled', - default_labels + memwd_sleep_time_s: sleep_time_seconds ) end end diff --git a/lib/gitlab/memory/watchdog/configuration.rb b/lib/gitlab/memory/watchdog/configuration.rb index 793f75adf59..5c459220be8 100644 --- a/lib/gitlab/memory/watchdog/configuration.rb +++ b/lib/gitlab/memory/watchdog/configuration.rb @@ -10,7 +10,6 @@ module Gitlab end def push(monitor_class, *args, **kwargs, &block) - remove(monitor_class) @monitors.push(build_monitor_state(monitor_class, *args, **kwargs, &block)) end @@ -20,16 +19,17 @@ module Gitlab end end - private - - def remove(monitor_class) - @monitors.delete_if { |monitor| monitor.monitor_class == monitor_class } + def empty? + @monitors.empty? end - def build_monitor_state(monitor_class, *args, max_strikes:, **kwargs, &block) + private + + def build_monitor_state(monitor_class, *args, max_strikes:, monitor_name: nil, **kwargs, &block) monitor = build_monitor(monitor_class, *args, **kwargs, &block) + monitor_name ||= monitor_class.name.demodulize.underscore - Gitlab::Memory::Watchdog::MonitorState.new(monitor, max_strikes: max_strikes) + Gitlab::Memory::Watchdog::MonitorState.new(monitor, max_strikes: max_strikes, monitor_name: monitor_name) end def build_monitor(monitor_class, *args, **kwargs, &block) @@ -39,7 +39,7 @@ module Gitlab DEFAULT_SLEEP_TIME_SECONDS = 60 - attr_writer :logger, :handler, :sleep_time_seconds + attr_writer :event_reporter, :handler, :sleep_time_seconds def monitors @monitor_stack ||= MonitorStack.new @@ -51,8 +51,8 @@ module Gitlab @handler ||= NullHandler.instance end - def logger - @logger ||= Gitlab::Logger.new($stdout) + def event_reporter + @event_reporter ||= EventReporter.new end # Used to control the frequency with which the watchdog will wake up and poll the GC. diff --git a/lib/gitlab/memory/watchdog/configurator.rb b/lib/gitlab/memory/watchdog/configurator.rb index 82b1b02b63f..04c04cbde02 100644 --- a/lib/gitlab/memory/watchdog/configurator.rb +++ b/lib/gitlab/memory/watchdog/configurator.rb @@ -4,36 +4,43 @@ module Gitlab module Memory class Watchdog class Configurator + DEFAULT_PUMA_WORKER_RSS_LIMIT_MB = 1200 + DEFAULT_SLEEP_INTERVAL_S = 60 + DEFAULT_SIDEKIQ_SLEEP_INTERVAL_S = 3 + MIN_SIDEKIQ_SLEEP_INTERVAL_S = 2 + DEFAULT_MAX_STRIKES = 5 + DEFAULT_MAX_HEAP_FRAG = 0.5 + DEFAULT_MAX_MEM_GROWTH = 3.0 + # grace_time / sleep_interval = max_strikes allowed for Sidekiq process to violate defined limits. + DEFAULT_SIDEKIQ_GRACE_TIME_S = 300 + class << self def configure_for_puma - lambda do |config| - sleep_time_seconds = ENV.fetch('GITLAB_MEMWD_SLEEP_TIME_SEC', 60).to_i - config.logger = Gitlab::AppLogger + ->(config) do config.handler = Gitlab::Memory::Watchdog::PumaHandler.new - config.sleep_time_seconds = sleep_time_seconds + config.sleep_time_seconds = ENV.fetch('GITLAB_MEMWD_SLEEP_TIME_SEC', DEFAULT_SLEEP_INTERVAL_S).to_i config.monitors(&configure_monitors_for_puma) end end def configure_for_sidekiq - lambda do |config| - sleep_time_seconds = [ENV.fetch('SIDEKIQ_MEMORY_KILLER_CHECK_INTERVAL', 3).to_i, 2].max - config.logger = Sidekiq.logger + ->(config) do config.handler = Gitlab::Memory::Watchdog::TermProcessHandler.new - config.sleep_time_seconds = sleep_time_seconds + config.sleep_time_seconds = sidekiq_sleep_time config.monitors(&configure_monitors_for_sidekiq) + config.event_reporter = SidekiqEventReporter.new end end private def configure_monitors_for_puma - lambda do |stack| - max_strikes = ENV.fetch('GITLAB_MEMWD_MAX_STRIKES', 5).to_i + ->(stack) do + max_strikes = ENV.fetch('GITLAB_MEMWD_MAX_STRIKES', DEFAULT_MAX_STRIKES).to_i if Gitlab::Utils.to_boolean(ENV['DISABLE_PUMA_WORKER_KILLER']) - max_heap_frag = ENV.fetch('GITLAB_MEMWD_MAX_HEAP_FRAG', 0.5).to_f - max_mem_growth = ENV.fetch('GITLAB_MEMWD_MAX_MEM_GROWTH', 3.0).to_f + max_heap_frag = ENV.fetch('GITLAB_MEMWD_MAX_HEAP_FRAG', DEFAULT_MAX_HEAP_FRAG).to_f + max_mem_growth = ENV.fetch('GITLAB_MEMWD_MAX_MEM_GROWTH', DEFAULT_MAX_MEM_GROWTH).to_f # stack.push MonitorClass, args*, max_strikes:, kwargs**, &block stack.push Gitlab::Memory::Watchdog::Monitor::HeapFragmentation, @@ -44,17 +51,44 @@ module Gitlab max_mem_growth: max_mem_growth, max_strikes: max_strikes else - memory_limit = ENV.fetch('PUMA_WORKER_MAX_MEMORY', 1200).to_i + memory_limit = ENV.fetch('PUMA_WORKER_MAX_MEMORY', DEFAULT_PUMA_WORKER_RSS_LIMIT_MB).to_i stack.push Gitlab::Memory::Watchdog::Monitor::RssMemoryLimit, - memory_limit: memory_limit.megabytes, + memory_limit_bytes: memory_limit.megabytes, max_strikes: max_strikes end end end + def sidekiq_sleep_time + [ + ENV.fetch('SIDEKIQ_MEMORY_KILLER_CHECK_INTERVAL', DEFAULT_SIDEKIQ_SLEEP_INTERVAL_S).to_i, + MIN_SIDEKIQ_SLEEP_INTERVAL_S + ].max + end + def configure_monitors_for_sidekiq - # NOP - At the moment we don't run watchdog for Sidekiq + ->(stack) do + if ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'].to_i.nonzero? + soft_limit_bytes = ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'].to_i.kilobytes + grace_time = ENV.fetch('SIDEKIQ_MEMORY_KILLER_GRACE_TIME', DEFAULT_SIDEKIQ_GRACE_TIME_S).to_i + max_strikes = grace_time / sidekiq_sleep_time + + stack.push Gitlab::Memory::Watchdog::Monitor::RssMemoryLimit, + memory_limit_bytes: soft_limit_bytes, + max_strikes: max_strikes.to_i, + monitor_name: :rss_memory_soft_limit + end + + if ENV['SIDEKIQ_MEMORY_KILLER_HARD_LIMIT_RSS'].to_i.nonzero? + hard_limit_bytes = ENV['SIDEKIQ_MEMORY_KILLER_HARD_LIMIT_RSS'].to_i.kilobytes + + stack.push Gitlab::Memory::Watchdog::Monitor::RssMemoryLimit, + memory_limit_bytes: hard_limit_bytes, + max_strikes: 0, + monitor_name: :rss_memory_hard_limit + end + end end end end diff --git a/lib/gitlab/memory/watchdog/event_reporter.rb b/lib/gitlab/memory/watchdog/event_reporter.rb new file mode 100644 index 00000000000..c37426cb660 --- /dev/null +++ b/lib/gitlab/memory/watchdog/event_reporter.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module Gitlab + module Memory + class Watchdog + class EventReporter + include ::Gitlab::Utils::StrongMemoize + + attr_reader :logger + + def initialize(logger: Gitlab::AppLogger) + @logger = logger + init_prometheus_metrics + end + + def started(labels = {}) + logger.info(message: 'started', **log_labels(labels)) + end + + def stopped(labels = {}) + logger.info(message: 'stopped', **log_labels(labels)) + end + + def threshold_violated(monitor_name) + @counter_violations.increment(reason: monitor_name) + end + + def strikes_exceeded(monitor_name, labels = {}) + logger.warn(log_labels(labels)) + + @counter_violations_handled.increment(reason: monitor_name) + end + + private + + def log_labels(extra = {}) + extra.merge( + pid: $$, + worker_id: worker_id, + memwd_rss_bytes: process_rss_bytes + ) + end + + def process_rss_bytes + Gitlab::Metrics::System.memory_usage_rss[:total] + end + + def worker_id + ::Prometheus::PidProvider.worker_id + end + + def init_prometheus_metrics + default_labels = { pid: worker_id } + @counter_violations = Gitlab::Metrics.counter( + :gitlab_memwd_violations_total, + 'Total number of times a Ruby process violated a memory threshold', + default_labels + ) + @counter_violations_handled = Gitlab::Metrics.counter( + :gitlab_memwd_violations_handled_total, + 'Total number of times Ruby process memory violations were handled', + default_labels + ) + end + end + end + end +end diff --git a/lib/gitlab/memory/watchdog/monitor/heap_fragmentation.rb b/lib/gitlab/memory/watchdog/monitor/heap_fragmentation.rb index 8f230980eac..ce99b68464e 100644 --- a/lib/gitlab/memory/watchdog/monitor/heap_fragmentation.rb +++ b/lib/gitlab/memory/watchdog/monitor/heap_fragmentation.rb @@ -4,10 +4,7 @@ module Gitlab module Memory class Watchdog module Monitor - # A monitor that observes Ruby heap fragmentation and calls - # memory_violation_callback when the Ruby heap has been fragmented for an extended - # period of time. - # + # A monitor that observes Ruby heap fragmentation. # See Gitlab::Metrics::Memory for how heap fragmentation is defined. class HeapFragmentation attr_reader :max_heap_fragmentation diff --git a/lib/gitlab/memory/watchdog/monitor/rss_memory_limit.rb b/lib/gitlab/memory/watchdog/monitor/rss_memory_limit.rb index 3e7de024630..ac71592294c 100644 --- a/lib/gitlab/memory/watchdog/monitor/rss_memory_limit.rb +++ b/lib/gitlab/memory/watchdog/monitor/rss_memory_limit.rb @@ -5,29 +5,38 @@ module Gitlab class Watchdog module Monitor class RssMemoryLimit - attr_reader :memory_limit + attr_reader :memory_limit_bytes - def initialize(memory_limit:) - @memory_limit = memory_limit + def initialize(memory_limit_bytes:) + @memory_limit_bytes = memory_limit_bytes + init_memory_limit_metrics end def call - worker_rss = Gitlab::Metrics::System.memory_usage_rss[:total] + worker_rss_bytes = Gitlab::Metrics::System.memory_usage_rss[:total] - return { threshold_violated: false, payload: {} } if worker_rss <= memory_limit + return { threshold_violated: false, payload: {} } if worker_rss_bytes <= memory_limit_bytes - { threshold_violated: true, payload: payload(worker_rss, memory_limit) } + { threshold_violated: true, payload: payload(worker_rss_bytes, memory_limit_bytes) } end private - def payload(worker_rss, memory_limit) + def payload(worker_rss_bytes, memory_limit_bytes) { message: 'rss memory limit exceeded', - memwd_rss_bytes: worker_rss, - memwd_max_rss_bytes: memory_limit + memwd_rss_bytes: worker_rss_bytes, + memwd_max_rss_bytes: memory_limit_bytes } end + + def init_memory_limit_metrics + rss_memory_limit = Gitlab::Metrics.gauge( + :gitlab_memwd_max_memory_limit, + 'The configured fixed limit for rss memory' + ) + rss_memory_limit.set({}, memory_limit_bytes) + end end end end diff --git a/lib/gitlab/memory/watchdog/monitor_state.rb b/lib/gitlab/memory/watchdog/monitor_state.rb index 73be5de3e45..bb083fedf2c 100644 --- a/lib/gitlab/memory/watchdog/monitor_state.rb +++ b/lib/gitlab/memory/watchdog/monitor_state.rb @@ -5,12 +5,12 @@ module Gitlab class Watchdog class MonitorState class Result - attr_reader :payload + attr_reader :payload, :monitor_name - def initialize(strikes_exceeded:, threshold_violated:, monitor_class:, payload: ) + def initialize(strikes_exceeded:, threshold_violated:, monitor_name:, payload:) @strikes_exceeded = strikes_exceeded @threshold_violated = threshold_violated - @monitor_class = monitor_class + @monitor_name = monitor_name.to_s.to_sym @payload = payload end @@ -21,15 +21,12 @@ module Gitlab def threshold_violated? @threshold_violated end - - def monitor_name - @monitor_class.name.demodulize.underscore.to_sym - end end - def initialize(monitor, max_strikes:) + def initialize(monitor, max_strikes:, monitor_name:) @monitor = monitor @max_strikes = max_strikes + @monitor_name = monitor_name @strikes = 0 end @@ -47,16 +44,12 @@ module Gitlab build_result(monitor_result) end - def monitor_class - @monitor.class - end - private def build_result(monitor_result) Result.new( strikes_exceeded: strikes_exceeded?, - monitor_class: monitor_class, + monitor_name: @monitor_name, threshold_violated: monitor_result[:threshold_violated], payload: payload.merge(monitor_result[:payload])) end diff --git a/lib/gitlab/memory/watchdog/sidekiq_event_reporter.rb b/lib/gitlab/memory/watchdog/sidekiq_event_reporter.rb new file mode 100644 index 00000000000..473ed1b8094 --- /dev/null +++ b/lib/gitlab/memory/watchdog/sidekiq_event_reporter.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module Gitlab + module Memory + class Watchdog + class SidekiqEventReporter + include ::Gitlab::Utils::StrongMemoize + + delegate :threshold_violated, :started, :stopped, :logger, to: :event_reporter + + def initialize(logger: ::Sidekiq.logger) + @event_reporter = EventReporter.new(logger: logger) + @sidekiq_daemon_monitor = Gitlab::SidekiqDaemon::Monitor.instance + init_prometheus_metrics + end + + def strikes_exceeded(monitor_name, labels = {}) + running_jobs = fetch_running_jobs + labels[:running_jobs] = running_jobs + increment_worker_counters(running_jobs) + + event_reporter.strikes_exceeded(monitor_name, labels) + end + + private + + attr_reader :event_reporter + + def fetch_running_jobs + @sidekiq_daemon_monitor.jobs.map do |jid, job| + { + jid: jid, + worker_class: job[:worker_class].name + } + end + end + + def increment_worker_counters(running_jobs) + running_jobs.each do |job| + @sidekiq_watchdog_running_jobs_counter.increment({ worker_class: job[:worker_class] }) + end + end + + def init_prometheus_metrics + @sidekiq_watchdog_running_jobs_counter = ::Gitlab::Metrics.counter( + :sidekiq_watchdog_running_jobs_total, + 'Current running jobs when limit was reached' + ) + end + end + end + end +end diff --git a/lib/gitlab/merge_requests/commit_message_generator.rb b/lib/gitlab/merge_requests/message_generator.rb index ef5c63925c2..5113fbdcd7b 100644 --- a/lib/gitlab/merge_requests/commit_message_generator.rb +++ b/lib/gitlab/merge_requests/message_generator.rb @@ -1,28 +1,41 @@ # frozen_string_literal: true module Gitlab module MergeRequests - class CommitMessageGenerator + class MessageGenerator def initialize(merge_request:, current_user:) @merge_request = merge_request @current_user = @merge_request.metrics&.merged_by || @merge_request.merge_user || current_user end - def merge_message + def merge_commit_message return unless @merge_request.target_project.merge_commit_template.present? - replace_placeholders(@merge_request.target_project.merge_commit_template) + replace_placeholders(@merge_request.target_project.merge_commit_template, allowed_placeholders: PLACEHOLDERS) end - def squash_message + def squash_commit_message return unless @merge_request.target_project.squash_commit_template.present? - replace_placeholders(@merge_request.target_project.squash_commit_template, squash: true) + replace_placeholders( + @merge_request.target_project.squash_commit_template, + allowed_placeholders: PLACEHOLDERS, + squash: true + ) + end + + def new_mr_description + return unless @merge_request.description.present? + + replace_placeholders( + @merge_request.description, + allowed_placeholders: ALLOWED_NEW_MR_PLACEHOLDERS, + keep_carriage_return: true + ) end private - attr_reader :merge_request - attr_reader :current_user + attr_reader :merge_request, :current_user PLACEHOLDERS = { 'source_branch' => ->(merge_request, _, _) { merge_request.source_branch.to_s }, @@ -38,10 +51,25 @@ module Gitlab end, 'description' => ->(merge_request, _, _) { merge_request.description }, 'reference' => ->(merge_request, _, _) { merge_request.to_reference(full: true) }, - 'first_commit' => -> (merge_request, _, _) { merge_request.first_commit&.safe_message&.strip }, - 'first_multiline_commit' => -> (merge_request, _, _) { merge_request.first_multiline_commit&.safe_message&.strip.presence || merge_request.title }, + 'first_commit' => -> (merge_request, _, _) { + return unless merge_request.persisted? || merge_request.compare_commits.present? + + merge_request.first_commit&.safe_message&.strip + }, + 'first_multiline_commit' => -> (merge_request, _, _) { + merge_request.first_multiline_commit&.safe_message&.strip.presence || merge_request.title + }, 'url' => ->(merge_request, _, _) { Gitlab::UrlBuilder.build(merge_request) }, - 'approved_by' => ->(merge_request, _, _) { merge_request.approved_by_users.map { |user| "Approved-by: #{user.name} <#{user.commit_email_or_default}>" }.join("\n") }, + 'reviewed_by' => ->(merge_request, _, _) { + merge_request.reviewed_by_users + .map { |user| "Reviewed-by: #{user.name} <#{user.commit_email_or_default}>" } + .join("\n") + }, + 'approved_by' => ->(merge_request, _, _) { + merge_request.approved_by_users + .map { |user| "Approved-by: #{user.name} <#{user.commit_email_or_default}>" } + .join("\n") + }, 'merged_by' => ->(_, user, _) { "#{user&.name} <#{user&.commit_email_or_default}>" }, 'co_authored_by' => ->(merge_request, merged_by, squash) do commit_author = squash ? merge_request.author : merged_by @@ -66,15 +94,34 @@ module Gitlab end }.freeze + # A new merge request that is in the process of being created and hasn't + # been persisted to the database. + # + # Limit the placeholders to a subset of the available ones where the + # placeholders wouldn't make sense in context. Disallowed placeholders + # will be replaced with an empty string. + ALLOWED_NEW_MR_PLACEHOLDERS = %w[ + source_branch + target_branch + first_commit + first_multiline_commit + co_authored_by + all_commits + ].freeze + PLACEHOLDERS_COMBINED_REGEX = /%{(#{Regexp.union(PLACEHOLDERS.keys)})}/.freeze - def replace_placeholders(message, squash: false) + def replace_placeholders(message, allowed_placeholders: [], squash: false, keep_carriage_return: false) # Convert CRLF to LF. - message = message.delete("\r") + message = message.delete("\r") unless keep_carriage_return used_variables = message.scan(PLACEHOLDERS_COMBINED_REGEX).map { |value| value[0] }.uniq values = used_variables.to_h do |variable_name| - ["%{#{variable_name}}", PLACEHOLDERS[variable_name].call(merge_request, current_user, squash)] + replacement = if allowed_placeholders.include?(variable_name) + PLACEHOLDERS[variable_name].call(merge_request, current_user, squash) + end + + ["%{#{variable_name}}", replacement] end names_of_empty_variables = values.filter_map { |name, value| name if value.blank? } diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index 6d7ecb53ec3..e99761a0459 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -20,6 +20,10 @@ module Gitlab status.to_i.between?(200, 499) end + def self.server_error?(status) + status.to_i >= 500 + end + # Tracks an event. # # See `Gitlab::Metrics::Transaction#add_event` for more details. diff --git a/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb b/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb index e2b43798b22..531e4079632 100644 --- a/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb +++ b/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb @@ -63,13 +63,11 @@ module Gitlab end def prometheus_metrics_attributes - @prometheus_metrics_attributes ||= begin - Dashboard::Transformers::Yml::V1::PrometheusMetrics.new( - dashboard_hash, + @prometheus_metrics_attributes ||= Dashboard::Transformers::Yml::V1::PrometheusMetrics.new( + dashboard_hash, project: project, dashboard_path: dashboard_path - ).execute - end + ).execute end end end diff --git a/lib/gitlab/metrics/dashboard/validator.rb b/lib/gitlab/metrics/dashboard/validator.rb index 1e8dc059968..57b4b5c068d 100644 --- a/lib/gitlab/metrics/dashboard/validator.rb +++ b/lib/gitlab/metrics/dashboard/validator.rb @@ -16,6 +16,8 @@ module Gitlab errors.empty? || raise(errors.first) end + private + def errors(content, schema_path = DASHBOARD_SCHEMA_PATH, dashboard_path: nil, project: nil) Validator::Client .new(content, schema_path, dashboard_path: dashboard_path, project: project) diff --git a/lib/gitlab/metrics/global_search_slis.rb b/lib/gitlab/metrics/global_search_slis.rb index 200c6eb4043..fc2e805047a 100644 --- a/lib/gitlab/metrics/global_search_slis.rb +++ b/lib/gitlab/metrics/global_search_slis.rb @@ -14,9 +14,6 @@ module Gitlab def initialize_slis! Gitlab::Metrics::Sli::Apdex.initialize_sli(:global_search, possible_labels) - - return unless Feature.enabled?(:global_search_error_rate_sli) - Gitlab::Metrics::Sli::ErrorRate.initialize_sli(:global_search, possible_labels) end @@ -28,8 +25,6 @@ module Gitlab end def record_error_rate(error:, search_type:, search_level:, search_scope:) - return unless Feature.enabled?(:global_search_error_rate_sli) - Gitlab::Metrics::Sli::ErrorRate[:global_search].increment( labels: labels(search_type: search_type, search_level: search_level, search_scope: search_scope), error: error diff --git a/lib/gitlab/metrics/rails_slis.rb b/lib/gitlab/metrics/rails_slis.rb index 71da0085c8c..9fd4eec479e 100644 --- a/lib/gitlab/metrics/rails_slis.rb +++ b/lib/gitlab/metrics/rails_slis.rb @@ -6,6 +6,7 @@ module Gitlab class << self def initialize_request_slis! Gitlab::Metrics::Sli::Apdex.initialize_sli(:rails_request, possible_request_labels) + initialize_rails_request_error_rate Gitlab::Metrics::Sli::Apdex.initialize_sli(:graphql_query, possible_graphql_query_labels) end @@ -13,6 +14,10 @@ module Gitlab Gitlab::Metrics::Sli::Apdex[:rails_request] end + def request_error_rate + Gitlab::Metrics::Sli::ErrorRate[:rails_request] + end + def graphql_query_apdex Gitlab::Metrics::Sli::Apdex[:graphql_query] end @@ -58,6 +63,12 @@ module Gitlab } end end + + def initialize_rails_request_error_rate + return unless Feature.enabled?(:gitlab_metrics_error_rate_sli, type: :development) + + Gitlab::Metrics::Sli::ErrorRate.initialize_sli(:rails_request, possible_request_labels) + end end end end diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb index d7fe983c553..0172de8731d 100644 --- a/lib/gitlab/metrics/requests_rack_middleware.rb +++ b/lib/gitlab/metrics/requests_rack_middleware.rb @@ -75,15 +75,17 @@ module Gitlab begin status, headers, body = @app.call(env) + return [status, headers, body] if health_endpoint - elapsed = ::Gitlab::Metrics::System.monotonic_time - started - - if !health_endpoint && ::Gitlab::Metrics.record_duration_for_status?(status) + urgency = urgency_for_env(env) + if ::Gitlab::Metrics.record_duration_for_status?(status) + elapsed = ::Gitlab::Metrics::System.monotonic_time - started self.class.http_request_duration_seconds.observe({ method: method }, elapsed) - - record_apdex(env, elapsed) + record_apdex(urgency, elapsed) end + record_error(urgency, status) + [status, headers, body] rescue StandardError self.class.rack_uncaught_errors_count.increment @@ -115,15 +117,22 @@ module Gitlab ::Gitlab::ApplicationContext.current_context_attribute(:caller_id) end - def record_apdex(env, elapsed) - urgency = urgency_for_env(env) - + def record_apdex(urgency, elapsed) Gitlab::Metrics::RailsSlis.request_apdex.increment( labels: labels_from_context.merge(request_urgency: urgency.name), success: elapsed < urgency.duration ) end + def record_error(urgency, status) + return unless Feature.enabled?(:gitlab_metrics_error_rate_sli, type: :development) + + Gitlab::Metrics::RailsSlis.request_error_rate.increment( + labels: labels_from_context.merge(request_urgency: urgency.name), + error: ::Gitlab::Metrics.server_error?(status) + ) + end + def labels_from_context { feature_category: feature_category.presence || FEATURE_CATEGORY_DEFAULT, diff --git a/lib/gitlab/metrics/subscribers/ldap.rb b/lib/gitlab/metrics/subscribers/ldap.rb new file mode 100644 index 00000000000..9cac5f41090 --- /dev/null +++ b/lib/gitlab/metrics/subscribers/ldap.rb @@ -0,0 +1,103 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Subscribers + class Ldap < ActiveSupport::Subscriber + # This namespace is configured in the Net::LDAP library, and appears + # at the end of the event key, e.g. `open.net_ldap` + attach_to :net_ldap + + COUNTER = :net_ldap_count + DURATION = :net_ldap_duration_s + + # Assembled from methods that are instrumented inside Net::LDAP + OBSERVABLE_EVENTS = %i[ + open + bind + add + modify + modify_password + rename + delete + search + ].freeze + + class << self + # @return [Integer] the total number of LDAP requests + def count + Gitlab::SafeRequestStore[COUNTER].to_i + end + + # @return [Float] the total duration spent on LDAP requests + def duration + Gitlab::SafeRequestStore[DURATION].to_f + end + + # Used in Gitlab::InstrumentationHelper to merge the LDAP stats + # into the log output + # + # @return [Hash<Integer, Float>] a hash of the stored statistics + def payload + { + net_ldap_count: count, + net_ldap_duration_s: duration + } + end + end + + # Called when an event is triggered in ActiveSupport::Notifications + # + # This method is aliased to the various events triggered by the + # Net::LDAP library, as the method will be called by those names + # when triggered. + # + # It stores statistics in the request for output to logs, and also + # resubmits the event data into Prometheus for monitoring purposes. + def observe_event(event) + add_to_request_store(event) + expose_metrics(event) + end + + OBSERVABLE_EVENTS.each do |event| + alias_method event, :observe_event + end + + private + + def current_transaction + ::Gitlab::Metrics::WebTransaction.current || ::Gitlab::Metrics::BackgroundTransaction.current + end + + # Track these events as statistics for the current requests, for logging purposes + def add_to_request_store(event) + return unless Gitlab::SafeRequestStore.active? + + Gitlab::SafeRequestStore[COUNTER] = Gitlab::SafeRequestStore[COUNTER].to_i + 1 + Gitlab::SafeRequestStore[DURATION] = Gitlab::SafeRequestStore[DURATION].to_f + event.duration.to_f + end + + # Converts the observed events into Prometheus metrics + def expose_metrics(event) + return unless current_transaction + + # event.name will be, for example, `search.net_ldap` + # and so we only want the first part, which is the + # true name of the event + labels = { name: event.name.split(".").first } + + current_transaction.increment(:gitlab_net_ldap_total, 1, labels) do + docstring 'Net::LDAP calls' + label_keys labels.keys + end + + current_transaction.observe(:gitlab_net_ldap_duration_seconds, event.duration, labels) do + docstring 'Net::LDAP time' + buckets [0.001, 0.01, 0.1, 1.0, 2.0, 5.0] + label_keys labels.keys + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/subscribers/rails_cache.rb b/lib/gitlab/metrics/subscribers/rails_cache.rb index b5e087d107b..b12db9df66d 100644 --- a/lib/gitlab/metrics/subscribers/rails_cache.rb +++ b/lib/gitlab/metrics/subscribers/rails_cache.rb @@ -8,6 +8,17 @@ module Gitlab class RailsCache < ActiveSupport::Subscriber attach_to :active_support + def cache_read_multi(event) + observe(:read_multi, event.duration) + + return unless current_transaction + + current_transaction.observe(:gitlab_cache_read_multikey_count, event.payload[:key].size) do + buckets [10, 50, 100, 1000] + docstring 'Number of keys for mget in read_multi/fetch_multi' + end + end + def cache_read(event) observe(:read, event.duration) diff --git a/lib/gitlab/middleware/compressed_json.rb b/lib/gitlab/middleware/compressed_json.rb index f66dfe44054..80916eab5ac 100644 --- a/lib/gitlab/middleware/compressed_json.rb +++ b/lib/gitlab/middleware/compressed_json.rb @@ -4,7 +4,18 @@ module Gitlab module Middleware class CompressedJson COLLECTOR_PATH = '/api/v4/error_tracking/collector' + PACKAGES_PATH = %r{ + \A/api/v4/ (?# prefix) + (?:projects/ + (?<project_id> + .+ (?# at least one character) + )/ + )? (?# projects segment) + packages/npm/-/npm/v1/security/ + (?:(?:advisories/bulk)|(?:audits/quick))\z (?# end) + }xi.freeze MAXIMUM_BODY_SIZE = 200.kilobytes.to_i + UNSAFE_CHARACTERS = %r{[!"#&'()*+,./:;<>=?@\[\]^`{}|~$]}xi.freeze def initialize(app) @app = app @@ -60,7 +71,21 @@ module Gitlab end def match_path?(env) - env['PATH_INFO'].start_with?((File.join(relative_url, COLLECTOR_PATH))) + env['PATH_INFO'].start_with?((File.join(relative_url, COLLECTOR_PATH))) || + match_packages_path?(env) + end + + def match_packages_path?(env) + match_data = env['PATH_INFO'].delete_prefix(relative_url).match(PACKAGES_PATH) + return false unless match_data + + return true unless match_data[:project_id] # instance level endpoint was matched + + url_encoded?(match_data[:project_id]) + end + + def url_encoded?(project_id) + project_id !~ UNSAFE_CHARACTERS end end end diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb index dcbb4557377..13f7ab36823 100644 --- a/lib/gitlab/middleware/go.rb +++ b/lib/gitlab/middleware/go.rb @@ -72,8 +72,8 @@ module Gitlab "#{project_url}.git" end - meta_import_tag = tag :meta, name: 'go-import', content: "#{import_prefix} git #{repository_url}" - meta_source_tag = tag :meta, name: 'go-source', content: "#{import_prefix} #{project_url} #{project_url}/-/tree/#{branch}{/dir} #{project_url}/-/blob/#{branch}{/dir}/{file}#L{line}" + meta_import_tag = tag.meta(name: 'go-import', content: "#{import_prefix} git #{repository_url}") + meta_source_tag = tag.meta(name: 'go-source', content: "#{import_prefix} #{project_url} #{project_url}/-/tree/#{branch}{/dir} #{project_url}/-/blob/#{branch}{/dir}/{file}#L{line}") head_tag = content_tag :head, meta_import_tag + meta_source_tag html_tag = content_tag :html, head_tag + body_tag [html_tag, 200] diff --git a/lib/gitlab/other_markup.rb b/lib/gitlab/other_markup.rb index 0dd6b8a809c..2368ea3ad28 100644 --- a/lib/gitlab/other_markup.rb +++ b/lib/gitlab/other_markup.rb @@ -3,18 +3,34 @@ module Gitlab # Parser/renderer for markups without other special support code. module OtherMarkup + RENDER_TIMEOUT = 10.seconds + # Public: Converts the provided markup into HTML. # # input - the source text in a markup format # def self.render(file_name, input, context) - html = GitHub::Markup.render(file_name, input) - .force_encoding(input.encoding) + html = render_markup(file_name, input, context).force_encoding(input.encoding) + context[:pipeline] ||= :markup html = Banzai.render(html, context) - html.html_safe end + + def self.render_markup(file_name, input, context) + Gitlab::RenderTimeout.timeout(foreground: RENDER_TIMEOUT) { GitHub::Markup.render(file_name, input) } + rescue Timeout::Error => e + class_name = name.demodulize + timeout_counter.increment(source: class_name) + Gitlab::ErrorTracking.track_exception(e, project_id: context[:project]&.id, class_name: class_name, + file_name: file_name) + + ActionController::Base.helpers.simple_format(input) + end + + def self.timeout_counter + Gitlab::Metrics.counter(:banzai_filter_timeouts_total, 'Count of the Banzai filters that time out') + end end end diff --git a/lib/gitlab/pages/cache_control.rb b/lib/gitlab/pages/cache_control.rb index 187d5f907e4..be39e52b342 100644 --- a/lib/gitlab/pages/cache_control.rb +++ b/lib/gitlab/pages/cache_control.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'set' + module Gitlab module Pages class CacheControl @@ -9,7 +11,9 @@ module Gitlab # To avoid delivering expired deployment URL in the cached payload, # use a longer expiration time in the deployment URL DEPLOYMENT_EXPIRATION = (EXPIRE + 12.hours) - CACHE_KEY_FORMAT = 'pages_domain_for_%{type}_%{id}_%{settings}' + + SETTINGS_CACHE_KEY = 'pages_domain_for_%{type}_%{id}' + PAYLOAD_CACHE_KEY = '%{settings_cache_key}_%{settings_hash}' class << self def for_project(project_id) @@ -29,29 +33,63 @@ module Gitlab end def cache_key - strong_memoize(:cache_key) do - CACHE_KEY_FORMAT % { - type: @type, - id: @id, - settings: settings - } + strong_memoize(:payload_cache_key) do + cache_settings_hash! + + payload_cache_key_for(settings_hash) end end + # Invalidates the cache. + # + # Since rails nodes and sidekiq nodes have different application settings, + # and the invalidation happens in a sidekiq node, we have to use the + # cached settings hash to build the payload cache key to be invalidated. def clear_cache - Rails.cache.delete(cache_key) + keys = cached_settings_hashes + .map { |hash| payload_cache_key_for(hash) } + .push(settings_cache_key) + + Rails.cache.delete_multi(keys) end private - def settings - values = ::Gitlab.config.pages.dup + # Since rails nodes and sidekiq nodes have different application settings, + # we cache the application settings hash when creating the payload cache + # so we can use these values to invalidate the cache in a sidekiq node later. + def cache_settings_hash! + cached = cached_settings_hashes.to_set + Rails.cache.write(settings_cache_key, cached.add(settings_hash)) + end + + def cached_settings_hashes + Rails.cache.read(settings_cache_key) || [] + end + + def payload_cache_key_for(settings_hash) + PAYLOAD_CACHE_KEY % { + settings_cache_key: settings_cache_key, + settings_hash: settings_hash + } + end - values['app_settings'] = ::Gitlab::CurrentSettings.attributes.slice( - 'force_pages_access_control' - ) + def settings_cache_key + strong_memoize(:settings_cache_key) do + SETTINGS_CACHE_KEY % { type: @type, id: @id } + end + end - ::Digest::SHA256.hexdigest(values.inspect) + def settings_hash + strong_memoize(:settings_hash) do + values = ::Gitlab.config.pages.dup + + values['app_settings'] = ::Gitlab::CurrentSettings.attributes.slice( + 'force_pages_access_control' + ) + + ::Digest::SHA256.hexdigest(values.inspect) + end end end end diff --git a/lib/gitlab/pagination/cursor_based_keyset.rb b/lib/gitlab/pagination/cursor_based_keyset.rb index 2d9fb0a50fc..199ec16d4df 100644 --- a/lib/gitlab/pagination/cursor_based_keyset.rb +++ b/lib/gitlab/pagination/cursor_based_keyset.rb @@ -22,7 +22,7 @@ module Gitlab def self.available?(cursor_based_request_context, relation) available_for_type?(relation) && - order_satisfied?(relation, cursor_based_request_context) + order_satisfied?(relation, cursor_based_request_context) end def self.enforced_for_type?(relation) diff --git a/lib/gitlab/pagination/offset_pagination.rb b/lib/gitlab/pagination/offset_pagination.rb index 00304f48dc5..a98199dae2e 100644 --- a/lib/gitlab/pagination/offset_pagination.rb +++ b/lib/gitlab/pagination/offset_pagination.rb @@ -11,15 +11,17 @@ module Gitlab @request_context = request_context end - def paginate(relation, exclude_total_headers: false, skip_default_order: false) - paginate_with_limit_optimization(add_default_order(relation, skip_default_order: skip_default_order)).tap do |data| + def paginate(relation, exclude_total_headers: false, skip_default_order: false, without_count: false) + ordered_relation = add_default_order(relation, skip_default_order: skip_default_order) + + paginate_with_limit_optimization(ordered_relation, without_count: without_count).tap do |data| add_pagination_headers(data, exclude_total_headers) end end private - def paginate_with_limit_optimization(relation) + def paginate_with_limit_optimization(relation, without_count:) pagination_data = if needs_pagination?(relation) relation.page(params[:page]).per(params[:per_page]) else @@ -28,8 +30,7 @@ module Gitlab return pagination_data unless pagination_data.is_a?(ActiveRecord::Relation) - limited_total_count = pagination_data.total_count_with_limit - if limited_total_count > Kaminari::ActiveRecordRelationMethods::MAX_COUNT_LIMIT + if without_count || exceeeds_count?(pagination_data) # The call to `total_count_with_limit` memoizes `@arel` because of a call to `references_eager_loaded_tables?` # We need to call `reset` because `without_count` relies on `@arel` being unmemoized pagination_data.reset.without_count @@ -78,6 +79,12 @@ module Gitlab # Ensure there is in total at least 1 page [paginated_data.total_pages, 1].max end + + def exceeeds_count?(paginated_data) + limited_total_count = paginated_data.total_count_with_limit + + limited_total_count > Kaminari::ActiveRecordRelationMethods::MAX_COUNT_LIMIT + end end end end diff --git a/lib/gitlab/patch/prependable.rb b/lib/gitlab/patch/prependable.rb index 1ed341e1c26..b974c0b2c7f 100644 --- a/lib/gitlab/patch/prependable.rb +++ b/lib/gitlab/patch/prependable.rb @@ -26,7 +26,7 @@ module Gitlab # https://github.com/rails/rails/pull/42067 # # Let's keep our own implementation, until the issue is fixed - Module.instance_method(:prepend_features).bind(self).call(base) + Module.instance_method(:prepend_features).bind_call(self, base) if const_defined?(:ClassMethods) klass_methods = const_get(:ClassMethods, false) diff --git a/lib/gitlab/phabricator_import/project_creator.rb b/lib/gitlab/phabricator_import/project_creator.rb index c842798ca74..4de9eaa9500 100644 --- a/lib/gitlab/phabricator_import/project_creator.rb +++ b/lib/gitlab/phabricator_import/project_creator.rb @@ -55,13 +55,11 @@ module Gitlab end def project_feature_attributes + # everything disabled except for issues @project_features_attributes ||= - begin - # everything disabled except for issues - ProjectFeature::FEATURES.to_h do |feature| - [ProjectFeature.access_level_attribute(feature), ProjectFeature::DISABLED] - end.merge(ProjectFeature.access_level_attribute(:issues) => ProjectFeature::ENABLED) - end + ProjectFeature::FEATURES.to_h do |feature| + [ProjectFeature.access_level_attribute(feature), ProjectFeature::DISABLED] + end.merge(ProjectFeature.access_level_attribute(:issues) => ProjectFeature::ENABLED) end def import_data diff --git a/lib/gitlab/process_management.rb b/lib/gitlab/process_management.rb index f8a1a3a97de..89ffd71c2d8 100644 --- a/lib/gitlab/process_management.rb +++ b/lib/gitlab/process_management.rb @@ -40,15 +40,6 @@ module Gitlab pids.each { |pid| signal(pid, signal) } end - # Waits for the given process to complete using a separate thread. - def self.wait_async(pid) - Thread.new do - Process.wait(pid) - rescue StandardError - nil # There is no reason to return `Errno::ECHILD` if it catches a `TypeError` - end - end - # Returns true if all the processes are alive. def self.all_alive?(pids) pids.each do |pid| diff --git a/lib/gitlab/process_supervisor.rb b/lib/gitlab/process_supervisor.rb index 714034f043d..09e923d1449 100644 --- a/lib/gitlab/process_supervisor.rb +++ b/lib/gitlab/process_supervisor.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require_relative './daemon' + module Gitlab # Given a set of process IDs, the supervisor can monitor processes # for being alive and invoke a callback if some or all should go away. diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb index f8a85f693bc..5af06e82c55 100644 --- a/lib/gitlab/profiler.rb +++ b/lib/gitlab/profiler.rb @@ -123,18 +123,18 @@ module Gitlab def self.with_custom_logger(logger) original_colorize_logging = ActiveSupport::LogSubscriber.colorize_logging - original_activerecord_logger = ApplicationRecord.logger + original_activerecord_logger = ActiveRecord::Base.logger original_actioncontroller_logger = ActionController::Base.logger if logger ActiveSupport::LogSubscriber.colorize_logging = false - ApplicationRecord.logger = logger + ActiveRecord::Base.logger = logger ActionController::Base.logger = logger end yield.tap do ActiveSupport::LogSubscriber.colorize_logging = original_colorize_logging - ApplicationRecord.logger = original_activerecord_logger + ActiveRecord::Base.logger = original_activerecord_logger ActionController::Base.logger = original_actioncontroller_logger end end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 8cc96970ebd..13718e63b25 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -125,17 +125,15 @@ module Gitlab def wiki_blobs(limit: count_limit) return [] unless Ability.allowed?(@current_user, :read_wiki, @project) - @wiki_blobs ||= begin - if project.wiki_enabled? && query.present? - if project.wiki.empty? - [] - else - Gitlab::WikiFileFinder.new(project, repository_wiki_ref).find(query, content_match_cutoff: limit) - end - else - [] - end - end + @wiki_blobs ||= if project.wiki_enabled? && query.present? + if project.wiki.empty? + [] + else + Gitlab::WikiFileFinder.new(project, repository_wiki_ref).find(query, content_match_cutoff: limit) + end + else + [] + end end def notes @@ -195,3 +193,5 @@ module Gitlab end end end + +Gitlab::ProjectSearchResults.prepend_mod_with('Gitlab::ProjectSearchResults') diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb index 51a5bedc44b..9bc0001be81 100644 --- a/lib/gitlab/project_template.rb +++ b/lib/gitlab/project_template.rb @@ -60,6 +60,7 @@ module Gitlab ProjectTemplate.new('dotnetcore', '.NET Core', _('A .NET Core console application template, customizable for any .NET Core project'), 'https://gitlab.com/gitlab-org/project-templates/dotnetcore', 'illustrations/third-party-logos/dotnet.svg'), ProjectTemplate.new('android', 'Android', _('A ready-to-go template for use with Android apps'), 'https://gitlab.com/gitlab-org/project-templates/android', 'illustrations/logos/android.svg'), ProjectTemplate.new('gomicro', 'Go Micro', _('Go Micro is a framework for micro service development'), 'https://gitlab.com/gitlab-org/project-templates/go-micro', 'illustrations/logos/gomicro.svg'), + ProjectTemplate.new('bridgetown', 'Pages/Bridgetown', _('Everything you need to create a GitLab Pages site using Bridgetown'), 'https://gitlab.com/gitlab-org/project-templates/bridgetown'), ProjectTemplate.new('gatsby', 'Pages/Gatsby', _('Everything you need to create a GitLab Pages site using Gatsby'), 'https://gitlab.com/pages/gatsby', 'illustrations/third-party-logos/gatsby.svg'), ProjectTemplate.new('hugo', 'Pages/Hugo', _('Everything you need to create a GitLab Pages site using Hugo'), 'https://gitlab.com/pages/hugo', 'illustrations/logos/hugo.svg'), ProjectTemplate.new('pelican', 'Pages/Pelican', _('Everything you need to create a GitLab Pages site using Pelican'), 'https://gitlab.com/pages/pelican', 'illustrations/third-party-logos/pelican.svg'), @@ -79,7 +80,8 @@ module Gitlab ProjectTemplate.new('tencent_serverless_framework', 'Tencent Serverless Framework/NextjsSSR', _('A project boilerplate for Tencent Serverless Framework that uses Next.js SSR'), 'https://gitlab.com/gitlab-org/project-templates/nextjsssr_demo', 'illustrations/logos/tencent_serverless_framework.svg'), ProjectTemplate.new('jsonnet', 'Jsonnet for Dynamic Child Pipelines', _('An example showing how to use Jsonnet with GitLab dynamic child pipelines'), 'https://gitlab.com/gitlab-org/project-templates/jsonnet'), ProjectTemplate.new('cluster_management', 'GitLab Cluster Management', _('An example project for managing Kubernetes clusters integrated with GitLab'), 'https://gitlab.com/gitlab-org/project-templates/cluster-management'), - ProjectTemplate.new('kotlin_native_linux', 'Kotlin Native Linux', _('A basic template for developing Linux programs using Kotlin Native'), 'https://gitlab.com/gitlab-org/project-templates/kotlin-native-linux') + ProjectTemplate.new('kotlin_native_linux', 'Kotlin Native Linux', _('A basic template for developing Linux programs using Kotlin Native'), 'https://gitlab.com/gitlab-org/project-templates/kotlin-native-linux'), + ProjectTemplate.new('typo3_distribution', 'TYPO3 Distribution', _('A template for starting a new TYPO3 project'), 'https://gitlab.com/ochorocho/typo3-distribution', 'illustrations/logos/typo3.svg') ].freeze end # rubocop:enable Metrics/AbcSize diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb index dda28ffdf90..6a5613ddd98 100644 --- a/lib/gitlab/prometheus_client.rb +++ b/lib/gitlab/prometheus_client.rb @@ -150,7 +150,7 @@ module Gitlab end def get(path, args) - Gitlab::HTTP.get(path, { query: args }.merge(http_options) ) + Gitlab::HTTP.get(path, { query: args }.merge(http_options)) rescue *Gitlab::HTTP::HTTP_ERRORS => e raise PrometheusClient::ConnectionError, e.message end diff --git a/lib/gitlab/quick_actions/issuable_actions.rb b/lib/gitlab/quick_actions/issuable_actions.rb index 0b37c80dc5f..a12457d89c9 100644 --- a/lib/gitlab/quick_actions/issuable_actions.rb +++ b/lib/gitlab/quick_actions/issuable_actions.rb @@ -84,7 +84,7 @@ module Gitlab current_user.can?(:"set_#{quick_action_target.to_ability_name}_metadata", quick_action_target) && find_labels.any? end - command :label do |labels_param| + command :label, :labels do |labels_param| run_label_command(labels: find_labels(labels_param), command: :label, updates_key: :add_label_ids) end diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb index e74c58e45b1..14e9e66e037 100644 --- a/lib/gitlab/quick_actions/issue_actions.rb +++ b/lib/gitlab/quick_actions/issue_actions.rb @@ -80,7 +80,7 @@ module Gitlab desc { _('Mark this issue as a duplicate of another issue') } explanation do |duplicate_reference| - _("Marks this issue as a duplicate of %{duplicate_reference}.") % { duplicate_reference: duplicate_reference } + _("Closes this issue. Marks as related to, and a duplicate of, %{duplicate_reference}.") % { duplicate_reference: duplicate_reference } end params '#issue' types Issue @@ -94,7 +94,7 @@ module Gitlab if canonical_issue.present? @updates[:canonical_issue_id] = canonical_issue.id - message = _("Marked this issue as a duplicate of %{duplicate_param}.") % { duplicate_param: duplicate_param } + message = _("Closed this issue. Marked as related to, and a duplicate of, %{duplicate_param}.") % { duplicate_param: duplicate_param } else message = _('Failed to mark this issue as a duplicate because referenced issue was not found.') end diff --git a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb index 8b1ff5d298a..e549ee2e43a 100644 --- a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb +++ b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb @@ -92,7 +92,7 @@ module Gitlab types Issue, MergeRequest condition do quick_action_target.supports_milestone? && - current_user.can?(:"set_#{quick_action_target.to_ability_name}_metadata", quick_action_target) && + current_user.can?(:"set_#{quick_action_target.to_ability_name}_metadata", quick_action_target) && find_milestones(project, state: 'active').any? end parse_params do |milestone_param| @@ -156,7 +156,7 @@ module Gitlab types Issue, MergeRequest condition do quick_action_target.supports_time_tracking? && - current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) + current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) end parse_params do |raw_duration| Gitlab::TimeTrackingFormatter.parse(raw_duration) @@ -179,7 +179,7 @@ module Gitlab types Issue, MergeRequest condition do quick_action_target.supports_time_tracking? && - current_user.can?(:"admin_#{quick_action_target.to_ability_name}", quick_action_target) + current_user.can?(:"admin_#{quick_action_target.to_ability_name}", quick_action_target) end parse_params do |raw_time_date| Gitlab::QuickActions::SpendTimeAndDateSeparator.new(raw_time_date).execute diff --git a/lib/gitlab/rack_attack.rb b/lib/gitlab/rack_attack.rb index f5fb6b5af3d..bedbe9c0bff 100644 --- a/lib/gitlab/rack_attack.rb +++ b/lib/gitlab/rack_attack.rb @@ -49,7 +49,7 @@ module Gitlab # # - Retry-After: the remaining duration in seconds until the quota is # reset. This is a standardized HTTP header: - # https://tools.ietf.org/html/rfc7231#page-69 + # https://www.rfc-editor.org/rfc/rfc7231#page-69 # # - RateLimit-Reset: the point of time that the request quota is reset, in Unix time # diff --git a/lib/gitlab/rack_attack/request.rb b/lib/gitlab/rack_attack/request.rb index 08a5ddb6ad1..d7abacb5b67 100644 --- a/lib/gitlab/rack_attack/request.rb +++ b/lib/gitlab/rack_attack/request.rb @@ -79,96 +79,96 @@ module Gitlab def throttle_unauthenticated_api? api_request? && - !should_be_skipped? && - !frontend_request? && - !throttle_unauthenticated_packages_api? && - !throttle_unauthenticated_files_api? && - !throttle_unauthenticated_deprecated_api? && - Gitlab::Throttle.settings.throttle_unauthenticated_api_enabled && - unauthenticated? + !should_be_skipped? && + !frontend_request? && + !throttle_unauthenticated_packages_api? && + !throttle_unauthenticated_files_api? && + !throttle_unauthenticated_deprecated_api? && + Gitlab::Throttle.settings.throttle_unauthenticated_api_enabled && + unauthenticated? end def throttle_unauthenticated_web? (web_request? || frontend_request?) && - !should_be_skipped? && - # TODO: Column will be renamed in https://gitlab.com/gitlab-org/gitlab/-/issues/340031 - Gitlab::Throttle.settings.throttle_unauthenticated_enabled && - unauthenticated? + !should_be_skipped? && + # TODO: Column will be renamed in https://gitlab.com/gitlab-org/gitlab/-/issues/340031 + Gitlab::Throttle.settings.throttle_unauthenticated_enabled && + unauthenticated? end def throttle_authenticated_api? api_request? && - !frontend_request? && - !throttle_authenticated_packages_api? && - !throttle_authenticated_files_api? && - !throttle_authenticated_deprecated_api? && - Gitlab::Throttle.settings.throttle_authenticated_api_enabled + !frontend_request? && + !throttle_authenticated_packages_api? && + !throttle_authenticated_files_api? && + !throttle_authenticated_deprecated_api? && + Gitlab::Throttle.settings.throttle_authenticated_api_enabled end def throttle_authenticated_web? (web_request? || frontend_request?) && - !throttle_authenticated_git_lfs? && - Gitlab::Throttle.settings.throttle_authenticated_web_enabled + !throttle_authenticated_git_lfs? && + Gitlab::Throttle.settings.throttle_authenticated_web_enabled end def throttle_unauthenticated_protected_paths? post? && - !should_be_skipped? && - protected_path? && - Gitlab::Throttle.protected_paths_enabled? && - unauthenticated? + !should_be_skipped? && + protected_path? && + Gitlab::Throttle.protected_paths_enabled? && + unauthenticated? end def throttle_authenticated_protected_paths_api? post? && - api_request? && - protected_path? && - Gitlab::Throttle.protected_paths_enabled? + api_request? && + protected_path? && + Gitlab::Throttle.protected_paths_enabled? end def throttle_authenticated_protected_paths_web? post? && - web_request? && - protected_path? && - Gitlab::Throttle.protected_paths_enabled? + web_request? && + protected_path? && + Gitlab::Throttle.protected_paths_enabled? end def throttle_unauthenticated_packages_api? packages_api_path? && - Gitlab::Throttle.settings.throttle_unauthenticated_packages_api_enabled && - unauthenticated? + Gitlab::Throttle.settings.throttle_unauthenticated_packages_api_enabled && + unauthenticated? end def throttle_authenticated_packages_api? packages_api_path? && - Gitlab::Throttle.settings.throttle_authenticated_packages_api_enabled + Gitlab::Throttle.settings.throttle_authenticated_packages_api_enabled end def throttle_authenticated_git_lfs? git_lfs_path? && - Gitlab::Throttle.settings.throttle_authenticated_git_lfs_enabled + Gitlab::Throttle.settings.throttle_authenticated_git_lfs_enabled end def throttle_unauthenticated_files_api? files_api_path? && - Gitlab::Throttle.settings.throttle_unauthenticated_files_api_enabled && - unauthenticated? + Gitlab::Throttle.settings.throttle_unauthenticated_files_api_enabled && + unauthenticated? end def throttle_authenticated_files_api? files_api_path? && - Gitlab::Throttle.settings.throttle_authenticated_files_api_enabled + Gitlab::Throttle.settings.throttle_authenticated_files_api_enabled end def throttle_unauthenticated_deprecated_api? deprecated_api_request? && - Gitlab::Throttle.settings.throttle_unauthenticated_deprecated_api_enabled && - unauthenticated? + Gitlab::Throttle.settings.throttle_unauthenticated_deprecated_api_enabled && + unauthenticated? end def throttle_authenticated_deprecated_api? deprecated_api_request? && - Gitlab::Throttle.settings.throttle_authenticated_deprecated_api_enabled + Gitlab::Throttle.settings.throttle_authenticated_deprecated_api_enabled end private diff --git a/lib/gitlab/redis/multi_store.rb b/lib/gitlab/redis/multi_store.rb index 12cb1fc6153..4f58bee49d0 100644 --- a/lib/gitlab/redis/multi_store.rb +++ b/lib/gitlab/redis/multi_store.rb @@ -10,6 +10,7 @@ module Gitlab 'Value not found on the redis primary store. Read from the redis secondary store successful.' end end + class PipelinedDiffError < StandardError def initialize(result_primary, result_secondary) @result_primary = result_primary @@ -22,6 +23,7 @@ module Gitlab "Result from the secondary: #{@result_secondary.inspect}." end end + class MethodMissingError < StandardError def message 'Method missing. Falling back to execute method on the redis secondary store.' diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 75dbccb965d..0e5389dc995 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -144,11 +144,20 @@ module Gitlab def redis_store_options config = raw_config_hash + config[:instrumentation_class] ||= self.class.instrumentation_class + + if config[:cluster].present? + config[:db] = 0 # Redis Cluster only supports db 0 + config + else + parse_redis_url(config) + end + end + + def parse_redis_url(config) redis_url = config.delete(:url) redis_uri = URI.parse(redis_url) - config[:instrumentation_class] ||= self.class.instrumentation_class - if redis_uri.scheme == 'unix' # Redis::Store does not handle Unix sockets well, so let's do it for them config[:path] = redis_uri.path @@ -178,7 +187,7 @@ module Gitlab { url: '' } end - if config_hash[:url].blank? + if config_hash[:url].blank? && config_hash[:cluster].blank? config_hash[:url] = legacy_fallback_urls[self.class.store_name] || legacy_fallback_urls[self.class.config_fallback.store_name] end diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index c5798bec0d7..540394f04bd 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -4,7 +4,8 @@ module Gitlab # Extract possible GFM references from an arbitrary String for further processing. class ReferenceExtractor < Banzai::ReferenceExtractor REFERABLES = %i(user issue label milestone mentioned_user mentioned_group mentioned_project - merge_request snippet commit commit_range directly_addressed_user epic iteration vulnerability).freeze + merge_request snippet commit commit_range directly_addressed_user epic iteration vulnerability + alert).freeze attr_accessor :project, :current_user, :author def initialize(project, current_user = nil) @@ -71,7 +72,7 @@ module Gitlab return @pattern if @pattern patterns = REFERABLES.map do |type| - Banzai::ReferenceParser[type].reference_type.to_s.classify.constantize.try(:reference_pattern) + Banzai::ReferenceParser[type].reference_class.try(:reference_pattern) end.uniq @pattern = Regexp.union(patterns.compact) diff --git a/lib/gitlab/repository_size_error_message.rb b/lib/gitlab/repository_size_error_message.rb index 8da840779c9..f5d82e61187 100644 --- a/lib/gitlab/repository_size_error_message.rb +++ b/lib/gitlab/repository_size_error_message.rb @@ -6,7 +6,7 @@ module Gitlab delegate :current_size, :limit, :exceeded_size, :additional_repo_storage_available?, to: :@checker - # @param checher [RepositorySizeChecker] + # @param checker [RepositorySizeChecker] def initialize(checker) @checker = checker end diff --git a/lib/gitlab/safe_request_store.rb b/lib/gitlab/safe_request_store.rb index 664afd1cc21..203d7d10532 100644 --- a/lib/gitlab/safe_request_store.rb +++ b/lib/gitlab/safe_request_store.rb @@ -40,7 +40,7 @@ module Gitlab def self.delete_if(&block) return unless RequestStore.active? - storage.delete_if { |k, v| block.call(k) } + storage.delete_if { |k, v| yield(k) } end end end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index bc59d4ce943..ba822955133 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -24,9 +24,7 @@ module Gitlab # # @return [String] secret token def secret_token - @secret_token ||= begin - File.read(Gitlab.config.gitlab_shell.secret_file).chomp - end + @secret_token ||= File.read(Gitlab.config.gitlab_shell.secret_file).chomp end # Ensure gitlab shell has a secret token stored in the secret_file diff --git a/lib/gitlab/sidekiq_daemon/memory_killer.rb b/lib/gitlab/sidekiq_daemon/memory_killer.rb index d5227e7a007..4bf9fd8470a 100644 --- a/lib/gitlab/sidekiq_daemon/memory_killer.rb +++ b/lib/gitlab/sidekiq_daemon/memory_killer.rb @@ -35,6 +35,7 @@ module Gitlab @enabled = true @metrics = init_metrics + @sidekiq_daemon_monitor = Gitlab::SidekiqDaemon::Monitor.instance end private @@ -78,7 +79,7 @@ module Gitlab rescue StandardError => e log_exception(e, __method__) rescue Exception => e # rubocop:disable Lint/RescueException - log_exception(e, __method__ ) + log_exception(e, __method__) raise e end end @@ -188,22 +189,17 @@ module Gitlab def increment_worker_counters(running_jobs, deadline_exceeded) running_jobs.each do |job| - @metrics[:sidekiq_memory_killer_running_jobs].increment( { worker_class: job[:worker_class], deadline_exceeded: deadline_exceeded } ) + @metrics[:sidekiq_memory_killer_running_jobs].increment({ worker_class: job[:worker_class], deadline_exceeded: deadline_exceeded }) end end def fetch_running_jobs - jobs = [] - Gitlab::SidekiqDaemon::Monitor.instance.jobs_mutex.synchronize do - jobs = Gitlab::SidekiqDaemon::Monitor.instance.jobs.map do |jid, job| - { - jid: jid, - worker_class: job[:worker_class].name - } - end + @sidekiq_daemon_monitor.jobs.map do |jid, job| + { + jid: jid, + worker_class: job[:worker_class].name + } end - - jobs end def out_of_range_description(rss, hard_limit, soft_limit, deadline_exceeded) @@ -269,10 +265,8 @@ module Gitlab end def rss_increase_by_jobs - Gitlab::SidekiqDaemon::Monitor.instance.jobs_mutex.synchronize do - Gitlab::SidekiqDaemon::Monitor.instance.jobs.sum do |job| - rss_increase_by_job(job) - end + @sidekiq_daemon_monitor.jobs.sum do |_, job| + rss_increase_by_job(job) end end @@ -297,7 +291,7 @@ module Gitlab end def any_jobs? - Gitlab::SidekiqDaemon::Monitor.instance.jobs.any? + @sidekiq_daemon_monitor.jobs.any? end end end diff --git a/lib/gitlab/sidekiq_daemon/monitor.rb b/lib/gitlab/sidekiq_daemon/monitor.rb index 655e95c82d3..125402c1e4b 100644 --- a/lib/gitlab/sidekiq_daemon/monitor.rb +++ b/lib/gitlab/sidekiq_daemon/monitor.rb @@ -15,9 +15,6 @@ module Gitlab # that should not be caught by application CancelledError = Class.new(Exception) # rubocop:disable Lint/InheritException - attr_reader :jobs - attr_reader :jobs_mutex - def initialize super @@ -31,8 +28,8 @@ module Gitlab end def within_job(worker_class, jid, queue) - jobs_mutex.synchronize do - jobs[jid] = { worker_class: worker_class, thread: Thread.current, started_at: Gitlab::Metrics::System.monotonic_time } + @jobs_mutex.synchronize do + @jobs[jid] = { worker_class: worker_class, thread: Thread.current, started_at: Gitlab::Metrics::System.monotonic_time } end if cancelled?(jid) @@ -48,8 +45,8 @@ module Gitlab yield ensure - jobs_mutex.synchronize do - jobs.delete(jid) + @jobs_mutex.synchronize do + @jobs.delete(jid) end end @@ -65,6 +62,12 @@ module Gitlab end end + def jobs + @jobs_mutex.synchronize do + @jobs.dup + end + end + private def run_thread @@ -166,14 +169,14 @@ module Gitlab # This is why it passes thread in block, # to ensure that we do process this thread def find_thread_unsafe(jid) - jobs.dig(jid, :thread) + @jobs.dig(jid, :thread) end def find_thread_with_lock(jid) # don't try to lock if we cannot find the thread return unless find_thread_unsafe(jid) - jobs_mutex.synchronize do + @jobs_mutex.synchronize do find_thread_unsafe(jid).tap do |thread| yield(thread) if thread end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb index 357e9d41187..4f7cd340461 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb @@ -253,7 +253,7 @@ module Gitlab def with_redis if Feature.enabled?(:use_primary_and_secondary_stores_for_duplicate_jobs) || - Feature.enabled?(:use_primary_store_as_default_for_duplicate_jobs) + Feature.enabled?(:use_primary_store_as_default_for_duplicate_jobs) # TODO: Swap for Gitlab::Redis::SharedState after store transition # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/923 Gitlab::Redis::DuplicateJobs.with { |redis| yield redis } diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies.rb index 63e8bee4443..fc6a849da95 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies.rb @@ -7,9 +7,9 @@ module Gitlab UnknownStrategyError = Class.new(StandardError) STRATEGIES = { - until_executing: UntilExecuting, - until_executed: UntilExecuted, - none: None + until_executing: ::Gitlab::SidekiqMiddleware::DuplicateJobs::Strategies::UntilExecuting, + until_executed: ::Gitlab::SidekiqMiddleware::DuplicateJobs::Strategies::UntilExecuted, + none: ::Gitlab::SidekiqMiddleware::DuplicateJobs::Strategies::None }.freeze def self.for(name) diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb index 17234bdf519..778d278146d 100644 --- a/lib/gitlab/sidekiq_status.rb +++ b/lib/gitlab/sidekiq_status.rb @@ -120,7 +120,7 @@ module Gitlab def self.with_redis if Feature.enabled?(:use_primary_and_secondary_stores_for_sidekiq_status) || - Feature.enabled?(:use_primary_store_as_default_for_sidekiq_status) + Feature.enabled?(:use_primary_store_as_default_for_sidekiq_status) # TODO: Swap for Gitlab::Redis::SharedState after store transition # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/923 Gitlab::Redis::SidekiqStatus.with { |redis| yield redis } diff --git a/lib/gitlab/slash_commands/application_help.rb b/lib/gitlab/slash_commands/application_help.rb index bfdb65a816d..94abc8b4508 100644 --- a/lib/gitlab/slash_commands/application_help.rb +++ b/lib/gitlab/slash_commands/application_help.rb @@ -3,6 +3,11 @@ module Gitlab module SlashCommands class ApplicationHelp < BaseCommand + def initialize(project, params) + @project = project + @params = params + end + def execute Gitlab::SlashCommands::Presenters::Help .new(project, commands, params) @@ -16,11 +21,7 @@ module Gitlab end def commands - Gitlab::SlashCommands::Command.new( - project, - chat_name, - params - ).commands + Gitlab::SlashCommands::Command.commands end end end diff --git a/lib/gitlab/slash_commands/command.rb b/lib/gitlab/slash_commands/command.rb index 265eda46489..f8b55f1a91d 100644 --- a/lib/gitlab/slash_commands/command.rb +++ b/lib/gitlab/slash_commands/command.rb @@ -3,7 +3,7 @@ module Gitlab module SlashCommands class Command < BaseCommand - def commands + def self.commands commands = [ Gitlab::SlashCommands::IssueShow, Gitlab::SlashCommands::IssueNew, @@ -15,7 +15,7 @@ module Gitlab Gitlab::SlashCommands::Run ] - if Feature.enabled?(:incident_declare_slash_command, current_user) + if Feature.enabled?(:incident_declare_slash_command) commands << Gitlab::SlashCommands::IncidentManagement::IncidentNew end @@ -50,7 +50,7 @@ module Gitlab private def available_commands - commands.keep_if do |klass| + self.class.commands.keep_if do |klass| klass.available?(project) end end diff --git a/lib/gitlab/slash_commands/deploy.rb b/lib/gitlab/slash_commands/deploy.rb index 9fcefd99f81..16a4875be91 100644 --- a/lib/gitlab/slash_commands/deploy.rb +++ b/lib/gitlab/slash_commands/deploy.rb @@ -54,7 +54,7 @@ module Gitlab return unless environment actions = environment.actions_for(to).select do |action| - action.starts_environment? + action.deployment_job? end if actions.many? diff --git a/lib/gitlab/sql/pattern.rb b/lib/gitlab/sql/pattern.rb index d13ccde8576..cd5587bbaef 100644 --- a/lib/gitlab/sql/pattern.rb +++ b/lib/gitlab/sql/pattern.rb @@ -51,16 +51,14 @@ module Gitlab if words.any? words.map { |word| arel_column.matches(to_pattern(word, use_minimum_char_limit: use_minimum_char_limit)) }.reduce(:and) - else + elsif lower_exact_match # No words of at least 3 chars, but we can search for an exact # case insensitive match with the query as a whole - if lower_exact_match - Arel::Nodes::NamedFunction + Arel::Nodes::NamedFunction .new('LOWER', [arel_column]) .eq(query) - else - arel_column.matches(sanitize_sql_like(query)) - end + else + arel_column.matches(sanitize_sql_like(query)) end end diff --git a/lib/gitlab/ssh/signature.rb b/lib/gitlab/ssh/signature.rb index 3b4df9a8d0c..a654d5b2ff1 100644 --- a/lib/gitlab/ssh/signature.rb +++ b/lib/gitlab/ssh/signature.rb @@ -9,6 +9,8 @@ module Gitlab class Signature include Gitlab::Utils::StrongMemoize + GIT_NAMESPACE = 'git' + def initialize(signature_text, signed_text, committer_email) @signature_text = signature_text @signed_text = signed_text @@ -18,11 +20,16 @@ module Gitlab def verification_status strong_memoize(:verification_status) do next :unverified unless all_attributes_present? - next :unverified unless valid_signature_blob? && committer + next :unverified unless valid_signature_blob? next :unknown_key unless signed_by_key + next :other_user unless committer next :other_user unless signed_by_key.user == committer - :verified + if signed_by_user_email_verified? + :verified + else + :unverified + end end end @@ -30,7 +37,7 @@ module Gitlab strong_memoize(:signed_by_key) do next unless key_fingerprint - Key.find_by_fingerprint_sha256(key_fingerprint) + Key.signing.find_by_fingerprint_sha256(key_fingerprint) end end @@ -48,6 +55,7 @@ module Gitlab # still need to check that the key belongs to the committer. def valid_signature_blob? return false unless signature + return false unless signature.namespace == GIT_NAMESPACE signature.verify(@signed_text) end @@ -55,7 +63,11 @@ module Gitlab def committer # Lookup by email because users can push verified commits that were made # by someone else. For example: Doing a rebase. - strong_memoize(:committer) { User.find_by_any_email(@committer_email, confirmed: true) } + strong_memoize(:committer) { User.find_by_any_email(@committer_email) } + end + + def signed_by_user_email_verified? + signed_by_key.user.verified_emails.include?(@committer_email) end def signature diff --git a/lib/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb index 54db31ffd6c..9dba8c99b99 100644 --- a/lib/gitlab/task_helpers.rb +++ b/lib/gitlab/task_helpers.rb @@ -149,18 +149,6 @@ module Gitlab end end - def all_repos - Gitlab::GitalyClient::StorageSettings.allow_disk_access do - Gitlab.config.repositories.storages.each_value do |repository_storage| - IO.popen(%W(find #{repository_storage.legacy_disk_path} -mindepth 2 -type d -name *.git)) do |find| - find.each_line do |path| - yield path.chomp - end - end - end - end - end - def repository_storage_paths_args Gitlab::GitalyClient::StorageSettings.allow_disk_access do Gitlab.config.repositories.storages.values.map { |rs| rs.legacy_disk_path } diff --git a/lib/gitlab/template/base_template.rb b/lib/gitlab/template/base_template.rb index ededc3db18e..223e3d40751 100644 --- a/lib/gitlab/template/base_template.rb +++ b/lib/gitlab/template/base_template.rb @@ -121,8 +121,8 @@ module Gitlab grouped = items.group_by(&:category) categories = grouped.keys - categories.each_with_object({}) do |category, hash| - hash[category] = grouped[category].map do |item| + categories.index_with do |category| + grouped[category].map do |item| { name: item.name, id: item.key, key: item.key, project_id: item.try(:project_id) } end end diff --git a/lib/gitlab/timeless.rb b/lib/gitlab/timeless.rb index b72d33113dd..ed0b7b4ed87 100644 --- a/lib/gitlab/timeless.rb +++ b/lib/gitlab/timeless.rb @@ -8,9 +8,9 @@ module Gitlab # negative arity means arguments are optional if block.arity == 1 || block.arity < 0 - block.call(model) + yield(model) else - block.call + yield end ensure diff --git a/lib/gitlab/tracking/destinations/snowplow.rb b/lib/gitlab/tracking/destinations/snowplow.rb index ddcd4693738..fd877bc0137 100644 --- a/lib/gitlab/tracking/destinations/snowplow.rb +++ b/lib/gitlab/tracking/destinations/snowplow.rb @@ -14,7 +14,15 @@ module Gitlab def event(category, action, label: nil, property: nil, value: nil, context: nil) return unless enabled? - tracker.track_struct_event(category, action, label, property, value, context, (Time.now.to_f * 1000).to_i) + tracker.track_struct_event( + category: category, + action: action, + label: label, + property: property, + value: value, + context: context, + tstamp: (Time.now.to_f * 1000).to_i + ) increment_total_events_counter end @@ -54,19 +62,21 @@ module Gitlab def tracker @tracker ||= SnowplowTracker::Tracker.new( - emitter, - SnowplowTracker::Subject.new, - SNOWPLOW_NAMESPACE, - app_id + emitters: [emitter], + subject: SnowplowTracker::Subject.new, + namespace: SNOWPLOW_NAMESPACE, + app_id: app_id ) end def emitter SnowplowTracker::AsyncEmitter.new( - hostname, - protocol: protocol, - on_success: method(:increment_successful_events_emissions), - on_failure: method(:failure_callback) + endpoint: hostname, + options: { + protocol: protocol, + on_success: method(:increment_successful_events_emissions), + on_failure: method(:failure_callback) + } ) end diff --git a/lib/gitlab/tracking/incident_management.rb b/lib/gitlab/tracking/incident_management.rb index df2a0658b36..a912fdbaeca 100644 --- a/lib/gitlab/tracking/incident_management.rb +++ b/lib/gitlab/tracking/incident_management.rb @@ -17,7 +17,7 @@ module Gitlab details = label ? { label: label, property: v } : {} - ::Gitlab::Tracking.event('IncidentManagement::Settings', "#{prefix}_#{key}", **details ) + ::Gitlab::Tracking.event('IncidentManagement::Settings', "#{prefix}_#{key}", **details) end end diff --git a/lib/gitlab/tracking/service_ping_context.rb b/lib/gitlab/tracking/service_ping_context.rb index 393cd647e7f..d31ca69a10c 100644 --- a/lib/gitlab/tracking/service_ping_context.rb +++ b/lib/gitlab/tracking/service_ping_context.rb @@ -4,21 +4,51 @@ module Gitlab module Tracking class ServicePingContext SCHEMA_URL = 'iglu:com.gitlab/gitlab_service_ping/jsonschema/1-0-0' - ALLOWED_SOURCES = %i[redis_hll].freeze + REDISHLL_SOURCE = :redis_hll + REDIS_SOURCE = :redis - def initialize(data_source:, event:) + ALLOWED_SOURCES = [REDISHLL_SOURCE, REDIS_SOURCE].freeze + + def initialize(data_source:, event: nil, key_path: nil) + check_configuration(data_source, event, key_path) + + @payload = { data_source: data_source } + + payload[:event_name] = event if data_source.eql? REDISHLL_SOURCE + payload[:key_path] = key_path if data_source.eql? REDIS_SOURCE + end + + def to_context + SnowplowTracker::SelfDescribingJson.new(SCHEMA_URL, payload) + end + + def to_h + { + schema: SCHEMA_URL, + data: @payload + } + end + + private + + attr_reader :payload + + def check_configuration(data_source, event, key_path) unless ALLOWED_SOURCES.include?(data_source) - raise ArgumentError, "#{data_source} is not acceptable data source for ServicePingContext" + configuration_error("#{data_source} is not acceptable data source for ServicePingContext") end - @payload = { - data_source: data_source, - event_name: event - } + if REDISHLL_SOURCE.eql?(data_source) && event.nil? + configuration_error("event attribute can not be missing for #{REDISHLL_SOURCE} data source") + end + + return unless REDIS_SOURCE.eql?(data_source) && key_path.nil? + + configuration_error("key_path attribute can not be missing for #{REDIS_SOURCE} data source") end - def to_context - SnowplowTracker::SelfDescribingJson.new(SCHEMA_URL, @payload) + def configuration_error(message) + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(ArgumentError.new(message)) end end end diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index 1e447923a39..00e609511f2 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -22,8 +22,8 @@ module Gitlab # rubocop:disable Metrics/ParameterLists def validate!( url, + schemes:, ports: [], - schemes: [], allow_localhost: false, allow_local_network: true, allow_object_storage: false, @@ -35,6 +35,8 @@ module Gitlab return [nil, nil] if url.nil? + raise ArgumentError, 'The schemes is a required argument' if schemes.blank? + # Param url can be a string, URI or Addressable::URI uri = parse_url(url) @@ -204,7 +206,7 @@ module Gitlab end def validate_scheme(scheme, schemes) - if scheme.blank? || (schemes.any? && !schemes.include?(scheme)) + if scheme.blank? || (schemes.any? && schemes.exclude?(scheme)) raise BlockedUrlError, "Only allowed schemes are #{schemes.join(', ')}" end end diff --git a/lib/gitlab/usage/metrics/aggregates.rb b/lib/gitlab/usage/metrics/aggregates.rb index 02d9fa74289..4b38809dde4 100644 --- a/lib/gitlab/usage/metrics/aggregates.rb +++ b/lib/gitlab/usage/metrics/aggregates.rb @@ -11,6 +11,7 @@ module Gitlab UnknownAggregationOperator = Class.new(AggregatedMetricError) UnknownAggregationSource = Class.new(AggregatedMetricError) DisallowedAggregationTimeFrame = Class.new(AggregatedMetricError) + UndefinedEvents = Class.new(AggregatedMetricError) DATABASE_SOURCE = 'database' REDIS_SOURCE = 'redis_hll' diff --git a/lib/gitlab/usage/metrics/aggregates/aggregate.rb b/lib/gitlab/usage/metrics/aggregates/aggregate.rb index 78f1ddc8a29..8d816c8d902 100644 --- a/lib/gitlab/usage/metrics/aggregates/aggregate.rb +++ b/lib/gitlab/usage/metrics/aggregates/aggregate.rb @@ -76,3 +76,5 @@ module Gitlab end end end + +Gitlab::Usage::Metrics::Aggregates::Aggregate.prepend_mod diff --git a/lib/gitlab/usage/metrics/aggregates/sources/calculations/intersection.rb b/lib/gitlab/usage/metrics/aggregates/sources/calculations/intersection.rb index dabf757c8a7..c8c248905f7 100644 --- a/lib/gitlab/usage/metrics/aggregates/sources/calculations/intersection.rb +++ b/lib/gitlab/usage/metrics/aggregates/sources/calculations/intersection.rb @@ -18,10 +18,8 @@ module Gitlab subset_powers_data = subsets_intersection_powers(metric_names, start_date, end_date, recorded_at, subset_powers_cache) # calculate last component of the equation |A & B & C & D| = .... - |A + B + C + D| - power_of_union_of_all_metrics = begin - subset_powers_cache[metric_names.size][metric_names.join('_+_')] ||= \ - calculate_metrics_union(metric_names: metric_names, start_date: start_date, end_date: end_date, recorded_at: recorded_at) - end + power_of_union_of_all_metrics = subset_powers_cache[metric_names.size][metric_names.join('_+_')] ||= \ + calculate_metrics_union(metric_names: metric_names, start_date: start_date, end_date: end_date, recorded_at: recorded_at) # in order to determine if part of equation (|A & B & C|, |A & B & C & D|), that represents the intersection that we need to calculate, # is positive or negative in particular equation we need to determine if number of subsets is even or odd. Please take a look at two examples below diff --git a/lib/gitlab/usage/metrics/instrumentations/count_merge_request_authors_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_merge_request_authors_metric.rb deleted file mode 100644 index a7f8bca8e08..00000000000 --- a/lib/gitlab/usage/metrics/instrumentations/count_merge_request_authors_metric.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Usage - module Metrics - module Instrumentations - class CountMergeRequestAuthorsMetric < DatabaseMetric - operation :distinct_count, column: :author_id - - relation { MergeRequest } - end - end - end - end -end diff --git a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb index f0d5298870c..f731057309e 100644 --- a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb @@ -117,6 +117,8 @@ module Gitlab case time_frame when '28d' monthly_time_range_db_params(column: self.class.metric_timestamp_column) + when '7d' + weekly_time_range_db_params(column: self.class.metric_timestamp_column) when 'all' {} when 'none' diff --git a/lib/gitlab/usage/service_ping/payload_keys_processor.rb b/lib/gitlab/usage/service_ping/payload_keys_processor.rb index ea2043ffb83..89931d8c012 100644 --- a/lib/gitlab/usage/service_ping/payload_keys_processor.rb +++ b/lib/gitlab/usage/service_ping/payload_keys_processor.rb @@ -28,8 +28,8 @@ module Gitlab payload.map do |key, value| if has_metric_definition?(key, parents) parents.dup.append(key).join('.') - else - payload_keys(value, parents.dup << key) if value.is_a?(Hash) + elsif value.is_a?(Hash) + payload_keys(value, parents.dup << key) end end end diff --git a/lib/gitlab/usage/time_frame.rb b/lib/gitlab/usage/time_frame.rb index 39b0855b917..e2eed969200 100644 --- a/lib/gitlab/usage/time_frame.rb +++ b/lib/gitlab/usage/time_frame.rb @@ -21,6 +21,10 @@ module Gitlab def monthly_time_range_db_params(column: nil) { (column || DEFAULT_TIMESTAMP_COLUMN) => 30.days.ago..2.days.ago } end + + def weekly_time_range_db_params(column: nil) + { (column || DEFAULT_TIMESTAMP_COLUMN) => 9.days.ago..2.days.ago } + end end end end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 5021dac453f..24f6cc725f6 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -157,7 +157,6 @@ module Gitlab runners_usage, integrations_usage, user_preferences_usage, - container_expiration_policies_usage, service_desk_counts ).tap do |data| data[:snippets] = add(data[:personal_snippets], data[:project_snippets]) @@ -300,7 +299,7 @@ module Gitlab object_store: { enabled: alt_usage_data { config['enabled'] }, direct_upload: alt_usage_data { config['direct_upload'] }, - background_upload: alt_usage_data { config['background_upload'] }, + background_upload: alt_usage_data { false }, # This setting no longer exists provider: alt_usage_data { config['connection']['provider'] } } } @@ -328,26 +327,6 @@ module Gitlab end # rubocop: disable CodeReuse/ActiveRecord - def container_expiration_policies_usage - results = {} - start = minimum_id(Project) - finish = maximum_id(Project) - - # rubocop: disable UsageData/LargeTable - base = ::ContainerExpirationPolicy.active - # rubocop: enable UsageData/LargeTable - - # rubocop: disable UsageData/LargeTable - ::ContainerExpirationPolicy.older_than_options.keys.each do |value| - results["projects_with_expiration_policy_enabled_with_older_than_set_to_#{value}".to_sym] = distinct_count(base.where(older_than: value), :project_id, start: start, finish: finish) - end - # rubocop: enable UsageData/LargeTable - - results[:projects_with_expiration_policy_enabled_with_older_than_unset] = distinct_count(base.where(older_than: nil), :project_id, start: start, finish: finish) - - results - end - def integrations_usage # rubocop: disable UsageData/LargeTable: Integration.available_integration_names(include_dev: false).each_with_object({}) do |name, response| @@ -611,10 +590,6 @@ module Gitlab {} end - def redis_hll_counters - { redis_hll_counters: ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events_data } - end - def action_monthly_active_users(time_period) counter = Gitlab::UsageDataCounters::EditorUniqueCounter date_range = { date_from: time_period[:created_at].first, date_to: time_period[:created_at].last } @@ -665,7 +640,6 @@ module Gitlab .merge(topology_usage_data) .merge(usage_activity_by_stage) .merge(usage_activity_by_stage(:usage_activity_by_stage_monthly, monthly_time_range_db_params)) - .merge(redis_hll_counters) end def metric_time_period(time_period) @@ -794,8 +768,8 @@ module Gitlab # rubocop:disable CodeReuse/ActiveRecord def distinct_count_user_auth_by_provider(time_period) - counts = auth_providers_except_ldap.each_with_object({}) do |provider, hash| - hash[provider] = distinct_count( + counts = auth_providers_except_ldap.index_with do |provider| + distinct_count( ::AuthenticationEvent.success.for_provider(provider).where(time_period), :user_id) end diff --git a/lib/gitlab/usage_data_counters/editor_unique_counter.rb b/lib/gitlab/usage_data_counters/editor_unique_counter.rb index 5ede840661a..0b448f68153 100644 --- a/lib/gitlab/usage_data_counters/editor_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/editor_unique_counter.rb @@ -45,20 +45,23 @@ module Gitlab private - def track_unique_action(action, author, time, project = nil) + def track_unique_action(event_name, author, time, project = nil) return unless author if Feature.enabled?(:route_hll_to_snowplow_phase2) Gitlab::Tracking.event( + name, 'ide_edit', - action.to_s, + property: event_name.to_s, project: project, namespace: project&.namespace, - user: author + user: author, + label: 'usage_activity_by_stage_monthly.create.action_monthly_active_users_ide_edit', + context: [Gitlab::Tracking::ServicePingContext.new(data_source: :redis_hll, event: event_name).to_context] ) end - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(action, values: author.id, time: time) + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name, values: author.id, time: time) end def count_unique(actions, date_from, date_to) diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index 24a87ae01f4..992cec2d174 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -25,32 +25,6 @@ module Gitlab pipeline_authoring ].freeze - CATEGORIES_COLLECTED_FROM_METRICS_DEFINITIONS = %w[ - analytics - ci_users - deploy_token_packages - code_review - ecosystem - error_tracking - ide_edit - importer - incident_management - incident_management_alerts - issues_edit - kubernetes_agent - manage - pipeline_authoring - quickactions - search - secure - snippets - source_code - terraform - testing - user_packages - work_items - ].freeze - # Track event on entity_id # Increment a Redis HLL counter for unique event_name and entity_id # @@ -114,41 +88,12 @@ module Gitlab @categories ||= known_events.map { |event| event[:category] }.uniq end - def categories_collected_from_metrics_definitions - CATEGORIES_COLLECTED_FROM_METRICS_DEFINITIONS - end - # @param category [String] the category name # @return [Array<String>] list of event names for given category def events_for_category(category) known_events.select { |event| event[:category] == category.to_s }.map { |event| event[:name] } end - # Recent 7 or 28 days unique events data for events defined in /lib/gitlab/usage_data_counters/known_events/ - # - # - For metrics for which we store a key per day, we have the last 7 days or last 28 days of data. - # - For metrics for which we store a key per week, we have the last complete week or last 4 complete weeks - # daily or weekly information is in the file we have for events definition /lib/gitlab/usage_data_counters/known_events/ - # - Most of the metrics have weekly aggregation. We recommend this as it generates fewer keys in Redis to store. - # - The aggregation used doesn't affect data granulation. - def unique_events_data - categories_pending_migration.each_with_object({}) do |category, category_results| - events_names = events_for_category(category) - - event_results = events_names.each_with_object({}) do |event, hash| - hash["#{event}_weekly"] = unique_events(**weekly_time_range.merge(event_names: [event])) unless event == "i_package_composer_deploy_token" - hash["#{event}_monthly"] = unique_events(**monthly_time_range.merge(event_names: [event])) - end - - if eligible_for_totals?(events_names) && CATEGORIES_FOR_TOTALS.include?(category) - event_results["#{category}_total_unique_counts_weekly"] = unique_events(**weekly_time_range.merge(event_names: events_names)) - event_results["#{category}_total_unique_counts_monthly"] = unique_events(**monthly_time_range.merge(event_names: events_names)) - end - - category_results["#{category}"] = event_results - end - end - def known_event?(event_name) event_for(event_name).present? end @@ -166,16 +111,13 @@ module Gitlab private - def categories_pending_migration - (categories - categories_collected_from_metrics_definitions) - end - def track(values, event_name, context: '', time: Time.zone.now) return unless ::ServicePing::ServicePingSettings.enabled? event = event_for(event_name) Gitlab::ErrorTracking.track_and_raise_for_dev_exception(UnknownEvent.new("Unknown event #{event_name}")) unless event.present? + return if event.blank? return unless feature_enabled?(event) Gitlab::Redis::HLL.add(key: redis_key(event, time, context), value: values, expiry: expiry(event)) diff --git a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb index dda72f7fa3b..477fa288874 100644 --- a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb @@ -173,7 +173,7 @@ module Gitlab private - def track_snowplow_action(action, author, project) + def track_snowplow_action(event_name, author, project) return unless Feature.enabled?(:route_hll_to_snowplow_phase2, project.namespace) return unless author @@ -181,17 +181,18 @@ module Gitlab ISSUE_CATEGORY, ISSUE_ACTION, label: ISSUE_LABEL, - property: action, + property: event_name, project: project, namespace: project.namespace, - user: author + user: author, + context: [Gitlab::Tracking::ServicePingContext.new(data_source: :redis_hll, event: event_name).to_context] ) end - def track_unique_action(action, author) + def track_unique_action(event_name, author) return unless author - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(action, values: author.id) + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name, values: author.id) end end end diff --git a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml index 5b80f6c6c0d..b9f143a3a56 100644 --- a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml +++ b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml @@ -347,6 +347,14 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_jobs_container_scanning + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_jobs_container_scanning_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_jobs_dependency_scanning_latest category: ci_templates redis_slot: ci_templates @@ -519,6 +527,10 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_implicit_jobs_container_scanning + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_implicit_jobs_dast_default_branch_deploy category: ci_templates redis_slot: ci_templates diff --git a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml index 0bd809f8aa5..3bb6655d762 100644 --- a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml @@ -39,6 +39,10 @@ redis_slot: code_review category: code_review aggregation: weekly +- name: i_code_review_create_mr + redis_slot: code_review + category: code_review + aggregation: weekly - name: i_code_review_user_create_mr redis_slot: code_review category: code_review @@ -256,7 +260,6 @@ redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_diff_searches - name: i_code_review_total_suggestions_applied redis_slot: code_review category: code_review @@ -427,3 +430,28 @@ redis_slot: code_review category: code_review aggregation: weekly +## Security Reports +- name: i_code_review_merge_request_widget_security_reports_view + redis_slot: code_review + category: code_review + aggregation: weekly +- name: i_code_review_merge_request_widget_security_reports_full_report_clicked + redis_slot: code_review + category: code_review + aggregation: weekly +- name: i_code_review_merge_request_widget_security_reports_expand + redis_slot: code_review + category: code_review + aggregation: weekly +- name: i_code_review_merge_request_widget_security_reports_expand_success + redis_slot: code_review + category: code_review + aggregation: weekly +- name: i_code_review_merge_request_widget_security_reports_expand_warning + redis_slot: code_review + category: code_review + aggregation: weekly +- name: i_code_review_merge_request_widget_security_reports_expand_failed + redis_slot: code_review + category: code_review + aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index c1720b26a22..a64b7c4032b 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -131,7 +131,6 @@ category: testing redis_slot: testing aggregation: weekly - feature_flag: usage_data_ci_i_testing_coverage_report_uploaded # Project Management group - name: g_project_management_issue_title_changed category: issues_edit diff --git a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb index 93137b762ec..10dae35d0bf 100644 --- a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb @@ -6,7 +6,8 @@ module Gitlab MR_DIFFS_ACTION = 'i_code_review_mr_diffs' MR_DIFFS_SINGLE_FILE_ACTION = 'i_code_review_mr_single_file_diffs' MR_DIFFS_USER_SINGLE_FILE_ACTION = 'i_code_review_user_single_file_diffs' - MR_CREATE_ACTION = 'i_code_review_user_create_mr' + MR_CREATE_ACTION = 'i_code_review_create_mr' + MR_USER_CREATE_ACTION = 'i_code_review_user_create_mr' MR_CLOSE_ACTION = 'i_code_review_user_close_mr' MR_REOPEN_ACTION = 'i_code_review_user_reopen_mr' MR_MERGE_ACTION = 'i_code_review_user_merge_mr' @@ -62,8 +63,24 @@ module Gitlab track_unique_action_by_user(MR_DIFFS_USER_SINGLE_FILE_ACTION, user) end - def track_create_mr_action(user:) - track_unique_action_by_user(MR_CREATE_ACTION, user) + def track_create_mr_action(user:, merge_request:) + track_unique_action_by_user(MR_USER_CREATE_ACTION, user) + track_unique_action_by_merge_request(MR_CREATE_ACTION, merge_request) + + project = merge_request.target_project + return unless Feature.enabled?(:route_hll_to_snowplow_phase2, project.namespace) + + Gitlab::Tracking.event( + name, + :create, + project: project, + namespace: project.namespace, + user: user, + property: MR_CREATE_ACTION, + label: 'redis_hll_counters.code_review.i_code_review_create_mr_monthly', + context: [Gitlab::Tracking::ServicePingContext.new(data_source: :redis_hll, + event: MR_CREATE_ACTION).to_context] + ) end def track_close_mr_action(user:) @@ -85,11 +102,15 @@ module Gitlab return unless Feature.enabled?(:route_hll_to_snowplow_phase2, project.namespace) Gitlab::Tracking.event( - 'merge_requests', - MR_APPROVE_ACTION, + name, + :approve, project: project, namespace: project.namespace, - user: user + user: user, + property: MR_APPROVE_ACTION, + label: 'redis_hll_counters.code_review.i_code_review_user_approve_mr_monthly', + context: [Gitlab::Tracking::ServicePingContext.new(data_source: :redis_hll, + event: MR_APPROVE_ACTION).to_context] ) end diff --git a/lib/gitlab/usage_data_counters/merge_request_widget_extension_counter.rb b/lib/gitlab/usage_data_counters/merge_request_widget_extension_counter.rb index f88bbc41c70..639da9bfee0 100644 --- a/lib/gitlab/usage_data_counters/merge_request_widget_extension_counter.rb +++ b/lib/gitlab/usage_data_counters/merge_request_widget_extension_counter.rb @@ -5,7 +5,16 @@ module Gitlab class MergeRequestWidgetExtensionCounter < BaseCounter KNOWN_EVENTS = %w[view full_report_clicked expand expand_success expand_warning expand_failed].freeze PREFIX = 'i_code_review_merge_request_widget' - WIDGETS = %w[accessibility code_quality license_compliance status_checks terraform test_summary metrics].freeze + WIDGETS = %w[ + accessibility + code_quality + license_compliance + status_checks + terraform + test_summary + metrics + security_reports + ].freeze class << self private diff --git a/lib/gitlab/utils/delegator_override/validator.rb b/lib/gitlab/utils/delegator_override/validator.rb index 4449fa75877..7fa5cc4deef 100644 --- a/lib/gitlab/utils/delegator_override/validator.rb +++ b/lib/gitlab/utils/delegator_override/validator.rb @@ -67,7 +67,7 @@ module Gitlab (delegator_class.instance_methods - allowlist).each do |method_name| target_classes.each do |target_class| - next unless target_class.instance_methods.include?(method_name) + next unless target_class.method_defined?(method_name) errors << generate_error(method_name, target_class, delegator_class) end diff --git a/lib/gitlab/utils/override.rb b/lib/gitlab/utils/override.rb index 39670a835a6..f83ebba7c3f 100644 --- a/lib/gitlab/utils/override.rb +++ b/lib/gitlab/utils/override.rb @@ -67,8 +67,8 @@ module Gitlab private def instance_method_defined?(klass, name) - klass.instance_methods(false).include?(name) || - klass.private_instance_methods(false).include?(name) + klass.method_defined?(name, false) || + klass.private_method_defined?(name, false) end def find_direct_method(klass, name) diff --git a/lib/gitlab/utils/sanitize_node_link.rb b/lib/gitlab/utils/sanitize_node_link.rb index b0dfa087fcf..9c34302f75e 100644 --- a/lib/gitlab/utils/sanitize_node_link.rb +++ b/lib/gitlab/utils/sanitize_node_link.rb @@ -30,7 +30,7 @@ module Gitlab # Remove all invalid scheme characters before checking against the # list of unsafe protocols. # - # See https://tools.ietf.org/html/rfc3986#section-3.1 + # See https://www.rfc-editor.org/rfc/rfc3986#section-3.1 # def safe_protocol?(scheme) return false unless scheme diff --git a/lib/gitlab/utils/strong_memoize.rb b/lib/gitlab/utils/strong_memoize.rb index 6456ad08924..7e78363dae5 100644 --- a/lib/gitlab/utils/strong_memoize.rb +++ b/lib/gitlab/utils/strong_memoize.rb @@ -16,16 +16,6 @@ module Gitlab # include Gitlab::Utils::StrongMemoize # # def trigger_from_token - # strong_memoize(:trigger) do - # Ci::Trigger.find_by_token(params[:token].to_s) - # end - # end - # - # Or like: - # - # include Gitlab::Utils::StrongMemoize - # - # def trigger_from_token # Ci::Trigger.find_by_token(params[:token].to_s) # end # strong_memoize_attr :trigger_from_token @@ -99,6 +89,15 @@ module Gitlab def do_strong_memoize(klass, method_name, member_name) method = klass.instance_method(method_name) + unless method.arity == 0 + raise <<~ERROR + Using `strong_memoize_attr` on methods with parameters is not supported. + + Use `strong_memoize_with` instead. + See https://docs.gitlab.com/ee/development/utilities.html#strongmemoize + ERROR + end + # Methods defined within a class method are already public by default, so we don't need to # explicitly make them public. scope = %i[private protected].find do |scope| @@ -106,9 +105,9 @@ module Gitlab .include? method_name end - klass.define_method(method_name) do |*args, &block| + klass.define_method(method_name) do |&block| strong_memoize(member_name) do - method.bind_call(self, *args, &block) + method.bind_call(self, &block) end end diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index 7360585df43..8b016a09889 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -11,7 +11,7 @@ module Gitlab included do scope :public_only, -> { where(visibility_level: PUBLIC) } - scope :public_and_internal_only, -> { where(visibility_level: [PUBLIC, INTERNAL] ) } + scope :public_and_internal_only, -> { where(visibility_level: [PUBLIC, INTERNAL]) } scope :private_only, -> { where(visibility_level: PRIVATE) } scope :non_public_only, -> { where.not(visibility_level: PUBLIC) } diff --git a/lib/gitlab/work_items/work_item_hierarchy.rb b/lib/gitlab/work_items/work_item_hierarchy.rb new file mode 100644 index 00000000000..e71bf2bce6a --- /dev/null +++ b/lib/gitlab/work_items/work_item_hierarchy.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module WorkItems + class WorkItemHierarchy < ObjectHierarchy + extend ::Gitlab::Utils::Override + + private + + def middle_table + ::WorkItems::ParentLink.arel_table + end + + def from_tables(cte) + [objects_table, cte.table, middle_table] + end + + override :parent_id_column + def parent_id_column(cte) + middle_table[:work_item_parent_id] + end + + override :ancestor_conditions + def ancestor_conditions(cte) + conditions = middle_table[:work_item_parent_id].eq(objects_table[:id]).and( + middle_table[:work_item_id].eq(cte.table[:id]) + ) + + with_type_filter(conditions, cte) + end + + override :descendant_conditions + def descendant_conditions(cte) + conditions = middle_table[:work_item_id].eq(objects_table[:id]).and( + middle_table[:work_item_parent_id].eq(cte.table[:id]) + ) + + with_type_filter(conditions, cte) + end + + def with_type_filter(conditions, cte) + return conditions unless options[:same_type] + + conditions.and(objects_table[:work_item_type_id].eq(cte.table[:work_item_type_id])) + end + end + end +end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 0d5daeefe90..02418c45e73 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -33,7 +33,7 @@ module Gitlab GitalyServer: { address: Gitlab::GitalyClient.address(repository.storage), token: Gitlab::GitalyClient.token(repository.storage), - features: Feature::Gitaly.server_feature_flags( + call_metadata: Feature::Gitaly.server_feature_flags( user: ::Feature::Gitaly.user_actor(user), repository: repository, project: ::Feature::Gitaly.project_actor(repository.container), @@ -48,6 +48,12 @@ module Gitlab attrs[:GitConfigOptions] << "receive.maxInputSize=#{receive_max_input_size.megabytes}" end + attrs[:GitalyServer][:call_metadata].merge!( + 'user_id' => attrs[:GL_ID].presence, + 'username' => attrs[:GL_USERNAME].presence, + 'remote_ip' => Gitlab::ApplicationContext.current_context_attribute(:remote_ip).presence + ).compact! + attrs end @@ -257,7 +263,7 @@ module Gitlab { address: Gitlab::GitalyClient.address(repository.shard), token: Gitlab::GitalyClient.token(repository.shard), - features: Feature::Gitaly.server_feature_flags( + call_metadata: Feature::Gitaly.server_feature_flags( user: ::Feature::Gitaly.user_actor, repository: repository, project: ::Feature::Gitaly.project_actor(repository.container), diff --git a/lib/gitlab/x509/signature.rb b/lib/gitlab/x509/signature.rb index f8a6980f208..d6bbb8bb2cb 100644 --- a/lib/gitlab/x509/signature.rb +++ b/lib/gitlab/x509/signature.rb @@ -6,6 +6,7 @@ module Gitlab module X509 class Signature include Gitlab::Utils::StrongMemoize + include SignatureType attr_reader :signature_text, :signed_text, :created_at @@ -16,14 +17,18 @@ module Gitlab @created_at = created_at end + def type + :x509 + end + def x509_certificate return if certificate_attributes.nil? X509Certificate.safe_create!(certificate_attributes) unless verified_signature.nil? end - def user - strong_memoize(:user) { User.find_by_any_email(@email) } + def signed_by_user + strong_memoize(:signed_by_user) { User.find_by_any_email(@email) } end def verified_signature @@ -33,11 +38,11 @@ module Gitlab def verification_status return :unverified if x509_certificate.nil? || - x509_certificate.revoked? || - !verified_signature || - user.nil? + x509_certificate.revoked? || + !verified_signature || + signed_by_user.nil? - if user.verified_emails.include?(@email.downcase) && certificate_email.casecmp?(@email) + if signed_by_user.verified_emails.include?(@email.downcase) && certificate_email.casecmp?(@email) :verified else :unverified |