diff options
Diffstat (limited to 'lib/gitlab')
197 files changed, 1858 insertions, 981 deletions
diff --git a/lib/gitlab/analytics/cycle_analytics/aggregated/data_collector.rb b/lib/gitlab/analytics/cycle_analytics/aggregated/data_collector.rb index 22d8874db57..3abf380d461 100644 --- a/lib/gitlab/analytics/cycle_analytics/aggregated/data_collector.rb +++ b/lib/gitlab/analytics/cycle_analytics/aggregated/data_collector.rb @@ -5,7 +5,7 @@ module Gitlab module CycleAnalytics module Aggregated # Arguments: - # stage - an instance of CycleAnalytics::ProjectStage or CycleAnalytics::GroupStage + # stage - an instance of CycleAnalytics::ProjectStage or CycleAnalytics::Stage # params: # current_user: an instance of User # from: DateTime diff --git a/lib/gitlab/analytics/cycle_analytics/data_collector.rb b/lib/gitlab/analytics/cycle_analytics/data_collector.rb index ae675b6ad27..0db027b9861 100644 --- a/lib/gitlab/analytics/cycle_analytics/data_collector.rb +++ b/lib/gitlab/analytics/cycle_analytics/data_collector.rb @@ -4,7 +4,7 @@ module Gitlab module Analytics module CycleAnalytics # Arguments: - # stage - an instance of CycleAnalytics::ProjectStage or CycleAnalytics::GroupStage + # stage - an instance of CycleAnalytics::ProjectStage or CycleAnalytics::Stage # params: # current_user: an instance of User # from: DateTime diff --git a/lib/gitlab/analytics/cycle_analytics/request_params.rb b/lib/gitlab/analytics/cycle_analytics/request_params.rb index ac9c465bf7d..d058782ae87 100644 --- a/lib/gitlab/analytics/cycle_analytics/request_params.rb +++ b/lib/gitlab/analytics/cycle_analytics/request_params.rb @@ -106,7 +106,7 @@ module Gitlab def use_aggregated_backend? # for now it's only available on the group-level - group.present? && aggregation.enabled + group.present? end def aggregation_attributes @@ -118,14 +118,14 @@ module Gitlab end def aggregation - @aggregation ||= ::Analytics::CycleAnalytics::Aggregation.safe_create_for_group(group) + @aggregation ||= ::Analytics::CycleAnalytics::Aggregation.safe_create_for_namespace(group) end def group_data_attributes { id: group.id, + namespace_id: group.id, name: group.name, - parent_id: group.parent_id, full_path: group.full_path, avatar_url: group.avatar_url } diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index 5b1bf99e297..a788586ebec 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -115,6 +115,38 @@ module Gitlab value > threshold_value end + # Similar to #throttled? above but checks for the bypass header in the request and logs the request when it is over the rate limit + # + # @param request [Http::Request] - Web request used to check the header and log + # @param current_user [User] Current user of the request, it can be nil + # @param key [Symbol] Key attribute registered in `.rate_limits` + # @param scope [Array<ActiveRecord>] Array of ActiveRecord models, Strings + # or Symbols to scope throttling to a specific request (e.g. per user + # per project) + # @param resource [ActiveRecord] An ActiveRecord model to count an action + # for (e.g. limit unique project (resource) downloads (action) to five + # per user (scope)) + # @param threshold [Integer] Optional threshold value to override default + # one registered in `.rate_limits` + # @param interval [Integer] Optional interval value to override default + # one registered in `.rate_limits` + # @param users_allowlist [Array<String>] Optional list of usernames to + # exclude from the limit. This param will only be functional if Scope + # includes a current user. + # @param peek [Boolean] Optional. When true the key will not be + # incremented but the current throttled state will be returned. + # + # @return [Boolean] Whether or not a request should be throttled + def throttled_request?(request, current_user, key, scope:, **options) + if ::Gitlab::Throttle.bypass_header.present? && request.get_header(Gitlab::Throttle.bypass_header) == '1' + return false + end + + throttled?(key, scope: scope, **options).tap do |throttled| + log_request(request, "#{key}_request_limit".to_sym, current_user) if throttled + end + end + # Returns the current rate limited state without incrementing the count. # # @param key [Symbol] Key attribute registered in `.rate_limits` diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 7e8f9c76dea..c97ef5a10ef 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -31,6 +31,7 @@ module Gitlab # Scopes used for GitLab as admin SUDO_SCOPE = :sudo + ADMIN_MODE_SCOPE = :admin_mode ADMIN_SCOPES = [SUDO_SCOPE].freeze # Default scopes for OAuth applications that don't define their own @@ -366,6 +367,7 @@ module Gitlab def available_scopes_for(current_user) scopes = non_admin_available_scopes scopes += ADMIN_SCOPES if current_user.admin? + scopes end diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index 26be7c8aa60..242390c3e89 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -260,7 +260,7 @@ module Gitlab if sync_profile_from_provider? UserSyncedAttributesMetadata::SYNCABLE_ATTRIBUTES.each do |key| if auth_hash.has_attribute?(key) && gl_user.sync_attribute?(key) - gl_user[key] = auth_hash.public_send(key) # rubocop:disable GitlabSecurity/PublicSend + gl_user.public_send("#{key}=".to_sym, auth_hash.public_send(key)) # rubocop:disable GitlabSecurity/PublicSend metadata.set_attribute_synced(key, true) else metadata.set_attribute_synced(key, false) diff --git a/lib/gitlab/background_migration/add_primary_email_to_emails_if_user_confirmed.rb b/lib/gitlab/background_migration/add_primary_email_to_emails_if_user_confirmed.rb deleted file mode 100644 index b39c0953fb1..00000000000 --- a/lib/gitlab/background_migration/add_primary_email_to_emails_if_user_confirmed.rb +++ /dev/null @@ -1,58 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Add user primary email to emails table if confirmed - class AddPrimaryEmailToEmailsIfUserConfirmed - INNER_BATCH_SIZE = 1_000 - - # Stubbed class to access the User table - class User < ActiveRecord::Base - include ::EachBatch - - self.table_name = 'users' - self.inheritance_column = :_type_disabled - - scope :confirmed, -> { where.not(confirmed_at: nil) } - - has_many :emails - end - - # Stubbed class to access the Emails table - class Email < ActiveRecord::Base - self.table_name = 'emails' - self.inheritance_column = :_type_disabled - - belongs_to :user - end - - def perform(start_id, end_id) - User.confirmed.where(id: start_id..end_id).select(:id, :email, :confirmed_at).each_batch(of: INNER_BATCH_SIZE) do |users| - current_time = Time.now.utc - - attributes = users.map do |user| - { - user_id: user.id, - email: user.email, - confirmed_at: user.confirmed_at, - created_at: current_time, - updated_at: current_time - } - end - - Email.insert_all(attributes) - end - mark_job_as_succeeded(start_id, end_id) - end - - private - - def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - 'AddPrimaryEmailToEmailsIfUserConfirmed', - arguments - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens.rb b/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens.rb new file mode 100644 index 00000000000..82e607ac7a7 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfill `admin_mode` scope for a range of personal access tokens + class BackfillAdminModeScopeForPersonalAccessTokens < ::Gitlab::BackgroundMigration::BatchedMigrationJob + scope_to ->(relation) do + relation.joins('INNER JOIN users ON personal_access_tokens.user_id = users.id') + .where(users: { admin: true }) + .where(revoked: [false, nil]) + .where.not('expires_at IS NOT NULL AND expires_at <= ?', Time.current) + end + + operation_name :update_all + feature_category :authentication_and_authorization + + ADMIN_MODE_SCOPE = ['admin_mode'].freeze + + def perform + each_sub_batch do |sub_batch| + sub_batch.each do |token| + token.update!(scopes: (YAML.safe_load(token.scopes) + ADMIN_MODE_SCOPE).uniq.to_yaml) + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities.rb b/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities.rb index 249c9d7af57..1dca82486ac 100644 --- a/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities.rb +++ b/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities.rb @@ -17,6 +17,7 @@ module Gitlab end operation_name :update_all + feature_category :database def perform each_sub_batch(batching_scope: RELATION) do |sub_batch| diff --git a/lib/gitlab/background_migration/backfill_environment_tiers.rb b/lib/gitlab/background_migration/backfill_environment_tiers.rb index 6f381577274..ebfabf1b28e 100644 --- a/lib/gitlab/background_migration/backfill_environment_tiers.rb +++ b/lib/gitlab/background_migration/backfill_environment_tiers.rb @@ -7,6 +7,7 @@ module Gitlab # See https://gitlab.com/gitlab-org/gitlab/-/issues/300741 for more information. class BackfillEnvironmentTiers < BatchedMigrationJob operation_name :backfill_environment_tiers + feature_category :database # Equivalent to `Environment#guess_tier` pattern matching. PRODUCTION_TIER = 0 diff --git a/lib/gitlab/background_migration/backfill_epic_cache_counts.rb b/lib/gitlab/background_migration/backfill_epic_cache_counts.rb index bd61d1a0f07..ee64a8ca2d5 100644 --- a/lib/gitlab/background_migration/backfill_epic_cache_counts.rb +++ b/lib/gitlab/background_migration/backfill_epic_cache_counts.rb @@ -4,6 +4,8 @@ module Gitlab module BackgroundMigration # rubocop: disable Style/Documentation class BackfillEpicCacheCounts < Gitlab::BackgroundMigration::BatchedMigrationJob + feature_category :database + def perform; end end # rubocop: enable Style/Documentation diff --git a/lib/gitlab/background_migration/backfill_group_features.rb b/lib/gitlab/background_migration/backfill_group_features.rb index 4ea664e2529..c45dcad5b2d 100644 --- a/lib/gitlab/background_migration/backfill_group_features.rb +++ b/lib/gitlab/background_migration/backfill_group_features.rb @@ -6,6 +6,7 @@ module Gitlab class BackfillGroupFeatures < ::Gitlab::BackgroundMigration::BatchedMigrationJob job_arguments :batch_size operation_name :upsert_group_features + feature_category :database def perform each_sub_batch( diff --git a/lib/gitlab/background_migration/backfill_imported_issue_search_data.rb b/lib/gitlab/background_migration/backfill_imported_issue_search_data.rb index c95fed512c9..8c151bc36ac 100644 --- a/lib/gitlab/background_migration/backfill_imported_issue_search_data.rb +++ b/lib/gitlab/background_migration/backfill_imported_issue_search_data.rb @@ -10,6 +10,7 @@ module Gitlab SUB_BATCH_SIZE = 1_000 operation_name :update_search_data + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/backfill_internal_on_notes.rb b/lib/gitlab/background_migration/backfill_internal_on_notes.rb index fe05b4ec3c1..2202cbb2f85 100644 --- a/lib/gitlab/background_migration/backfill_internal_on_notes.rb +++ b/lib/gitlab/background_migration/backfill_internal_on_notes.rb @@ -6,6 +6,7 @@ module Gitlab class BackfillInternalOnNotes < BatchedMigrationJob scope_to -> (relation) { relation.where(confidential: true) } operation_name :update_all + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/backfill_namespace_details.rb b/lib/gitlab/background_migration/backfill_namespace_details.rb index 640d9379351..57254c09f78 100644 --- a/lib/gitlab/background_migration/backfill_namespace_details.rb +++ b/lib/gitlab/background_migration/backfill_namespace_details.rb @@ -5,6 +5,7 @@ module Gitlab # Backfill namespace_details for a range of namespaces class BackfillNamespaceDetails < ::Gitlab::BackgroundMigration::BatchedMigrationJob operation_name :backfill_namespace_details + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/backfill_namespace_id_of_vulnerability_reads.rb b/lib/gitlab/background_migration/backfill_namespace_id_of_vulnerability_reads.rb index dca7f9fa921..8600510b6ef 100644 --- a/lib/gitlab/background_migration/backfill_namespace_id_of_vulnerability_reads.rb +++ b/lib/gitlab/background_migration/backfill_namespace_id_of_vulnerability_reads.rb @@ -5,6 +5,7 @@ module Gitlab # Sets the `namespace_id` of the existing `vulnerability_reads` records class BackfillNamespaceIdOfVulnerabilityReads < BatchedMigrationJob operation_name :set_namespace_id + feature_category :database UPDATE_SQL = <<~SQL UPDATE diff --git a/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level.rb b/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level.rb index 6520cd63711..ff20a7ed177 100644 --- a/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level.rb +++ b/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level.rb @@ -18,6 +18,7 @@ module Gitlab end operation_name :update_all + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/backfill_project_import_level.rb b/lib/gitlab/background_migration/backfill_project_import_level.rb index 21c239e0070..1a4b1e6731f 100644 --- a/lib/gitlab/background_migration/backfill_project_import_level.rb +++ b/lib/gitlab/background_migration/backfill_project_import_level.rb @@ -4,6 +4,7 @@ module Gitlab module BackgroundMigration class BackfillProjectImportLevel < BatchedMigrationJob operation_name :update_import_level + feature_category :database LEVEL = { Gitlab::Access::NO_ACCESS => [0], diff --git a/lib/gitlab/background_migration/backfill_project_member_namespace_id.rb b/lib/gitlab/background_migration/backfill_project_member_namespace_id.rb index c2e37269b5e..1bf029f5001 100644 --- a/lib/gitlab/background_migration/backfill_project_member_namespace_id.rb +++ b/lib/gitlab/background_migration/backfill_project_member_namespace_id.rb @@ -4,6 +4,8 @@ module Gitlab module BackgroundMigration # Backfills the `members.member_namespace_id` column for `type=ProjectMember` class BackfillProjectMemberNamespaceId < Gitlab::BackgroundMigration::BatchedMigrationJob + feature_category :database + def perform parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id) diff --git a/lib/gitlab/background_migration/backfill_project_namespace_details.rb b/lib/gitlab/background_migration/backfill_project_namespace_details.rb index 9bee3cf21e8..4f4db50321d 100644 --- a/lib/gitlab/background_migration/backfill_project_namespace_details.rb +++ b/lib/gitlab/background_migration/backfill_project_namespace_details.rb @@ -4,6 +4,7 @@ module Gitlab # Backfill project namespace_details for a range of projects class BackfillProjectNamespaceDetails < ::Gitlab::BackgroundMigration::BatchedMigrationJob operation_name :backfill_project_namespace_details + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/backfill_project_namespace_on_issues.rb b/lib/gitlab/background_migration/backfill_project_namespace_on_issues.rb index 34dd3321125..0c4953486f4 100644 --- a/lib/gitlab/background_migration/backfill_project_namespace_on_issues.rb +++ b/lib/gitlab/background_migration/backfill_project_namespace_on_issues.rb @@ -7,6 +7,7 @@ module Gitlab MAX_UPDATE_RETRIES = 3 operation_name :update_all + feature_category :database def perform each_sub_batch( diff --git a/lib/gitlab/background_migration/backfill_project_statistics_container_repository_size.rb b/lib/gitlab/background_migration/backfill_project_statistics_container_repository_size.rb index ec813022b8f..01cae3e2d50 100644 --- a/lib/gitlab/background_migration/backfill_project_statistics_container_repository_size.rb +++ b/lib/gitlab/background_migration/backfill_project_statistics_container_repository_size.rb @@ -4,6 +4,8 @@ module Gitlab module BackgroundMigration # Back-fill container_registry_size for project_statistics class BackfillProjectStatisticsContainerRepositorySize < Gitlab::BackgroundMigration::BatchedMigrationJob + feature_category :database + def perform # no-op end diff --git a/lib/gitlab/background_migration/backfill_project_statistics_storage_size_without_uploads_size.rb b/lib/gitlab/background_migration/backfill_project_statistics_storage_size_without_uploads_size.rb index 1a3dd88ea31..da865ed935a 100644 --- a/lib/gitlab/background_migration/backfill_project_statistics_storage_size_without_uploads_size.rb +++ b/lib/gitlab/background_migration/backfill_project_statistics_storage_size_without_uploads_size.rb @@ -4,6 +4,8 @@ module Gitlab module BackgroundMigration # Back-fill storage_size for project_statistics class BackfillProjectStatisticsStorageSizeWithoutUploadsSize < Gitlab::BackgroundMigration::BatchedMigrationJob + feature_category :database + def perform # no-op end diff --git a/lib/gitlab/background_migration/backfill_releases_author_id.rb b/lib/gitlab/background_migration/backfill_releases_author_id.rb new file mode 100644 index 00000000000..8982fe1acca --- /dev/null +++ b/lib/gitlab/background_migration/backfill_releases_author_id.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfills releases with empty release authors. + # More details on: + # 1) https://gitlab.com/groups/gitlab-org/-/epics/8375 + # 2) https://gitlab.com/gitlab-org/gitlab/-/issues/367522#note_1156503600 + class BackfillReleasesAuthorId < BatchedMigrationJob + operation_name :backfill_releases_author_id + job_arguments :ghost_user_id + feature_category :database + + scope_to ->(relation) { relation.where(author_id: nil) } + + def perform + each_sub_batch do |sub_batch| + sub_batch.update_all(author_id: ghost_user_id) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_user_details_fields.rb b/lib/gitlab/background_migration/backfill_user_details_fields.rb index 8d8619256b0..26489d06a85 100644 --- a/lib/gitlab/background_migration/backfill_user_details_fields.rb +++ b/lib/gitlab/background_migration/backfill_user_details_fields.rb @@ -11,6 +11,7 @@ module Gitlab # * organization class BackfillUserDetailsFields < BatchedMigrationJob operation_name :backfill_user_details_fields + feature_category :database def perform query = <<~SQL diff --git a/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent.rb b/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent.rb index 37b1a37569b..20c3c68ec40 100644 --- a/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent.rb +++ b/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent.rb @@ -5,6 +5,7 @@ module Gitlab # Backfills the `vulnerability_reads.casted_cluster_agent_id` column class BackfillVulnerabilityReadsClusterAgent < Gitlab::BackgroundMigration::BatchedMigrationJob operation_name :update_all + feature_category :database CLUSTER_AGENTS_JOIN = <<~SQL INNER JOIN cluster_agents diff --git a/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb b/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb index a020cabd1f4..fc0d0ce3a57 100644 --- a/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb +++ b/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb @@ -5,6 +5,8 @@ module Gitlab # Backfills the `issues.work_item_type_id` column, replacing any # instances of `NULL` with the appropriate `work_item_types.id` based on `issues.issue_type` class BackfillWorkItemTypeIdForIssues < BatchedMigrationJob + feature_category :database + # Basic AR model for issues table class MigrationIssue < ApplicationRecord self.table_name = 'issues' diff --git a/lib/gitlab/background_migration/batched_migration_job.rb b/lib/gitlab/background_migration/batched_migration_job.rb index 973ab20f547..4039a79cfa7 100644 --- a/lib/gitlab/background_migration/batched_migration_job.rb +++ b/lib/gitlab/background_migration/batched_migration_job.rb @@ -27,7 +27,7 @@ module Gitlab end def operation_name(operation) - define_method('operation_name') do + define_method(:operation_name) do operation end end diff --git a/lib/gitlab/background_migration/cleanup_orphaned_routes.rb b/lib/gitlab/background_migration/cleanup_orphaned_routes.rb index 0cd19dc5df9..5c0ddf0ba8b 100644 --- a/lib/gitlab/background_migration/cleanup_orphaned_routes.rb +++ b/lib/gitlab/background_migration/cleanup_orphaned_routes.rb @@ -8,6 +8,8 @@ module Gitlab class CleanupOrphanedRoutes < Gitlab::BackgroundMigration::BatchedMigrationJob include Gitlab::Database::DynamicModelHelpers + feature_category :database + def perform # there should really be no records to fix, there is none gitlab.com, but taking the safer route, just in case. fix_missing_namespace_id_routes diff --git a/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb b/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb index 136293242b2..033b2c87152 100644 --- a/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb +++ b/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb @@ -16,6 +16,7 @@ module Gitlab class CopyColumnUsingBackgroundMigrationJob < BatchedMigrationJob job_arguments :copy_from, :copy_to operation_name :update_all + feature_category :database def perform assignment_clauses = build_assignment_clauses(copy_from, copy_to) diff --git a/lib/gitlab/background_migration/delete_approval_rules_with_vulnerability.rb b/lib/gitlab/background_migration/delete_approval_rules_with_vulnerability.rb index 739197898d9..c7c063e8ccf 100644 --- a/lib/gitlab/background_migration/delete_approval_rules_with_vulnerability.rb +++ b/lib/gitlab/background_migration/delete_approval_rules_with_vulnerability.rb @@ -5,6 +5,8 @@ module Gitlab # This class doesn't delete approval rules # as this feature exists only in EE class DeleteApprovalRulesWithVulnerability < BatchedMigrationJob + feature_category :database + def perform end end diff --git a/lib/gitlab/background_migration/delete_invalid_epic_issues.rb b/lib/gitlab/background_migration/delete_invalid_epic_issues.rb index 3af59ab4931..6c0eb6b1950 100644 --- a/lib/gitlab/background_migration/delete_invalid_epic_issues.rb +++ b/lib/gitlab/background_migration/delete_invalid_epic_issues.rb @@ -4,6 +4,8 @@ module Gitlab module BackgroundMigration # rubocop: disable Style/Documentation class DeleteInvalidEpicIssues < BatchedMigrationJob + feature_category :database + def perform end end diff --git a/lib/gitlab/background_migration/delete_orphaned_operational_vulnerabilities.rb b/lib/gitlab/background_migration/delete_orphaned_operational_vulnerabilities.rb index f93dcf83c49..6953ae65651 100644 --- a/lib/gitlab/background_migration/delete_orphaned_operational_vulnerabilities.rb +++ b/lib/gitlab/background_migration/delete_orphaned_operational_vulnerabilities.rb @@ -17,6 +17,8 @@ module Gitlab SQL operation_name :delete_orphaned_operational_vulnerabilities + feature_category :database + scope_to ->(relation) do relation .where(report_type: [REPORT_TYPES[:cluster_image_scanning], REPORT_TYPES[:custom]]) diff --git a/lib/gitlab/background_migration/delete_orphans_approval_merge_request_rules.rb b/lib/gitlab/background_migration/delete_orphans_approval_merge_request_rules.rb index 4b7b7d42c77..e77d56d68cb 100644 --- a/lib/gitlab/background_migration/delete_orphans_approval_merge_request_rules.rb +++ b/lib/gitlab/background_migration/delete_orphans_approval_merge_request_rules.rb @@ -7,6 +7,7 @@ module Gitlab scope_to ->(relation) { relation.where(report_type: 4) } operation_name :delete_all + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/delete_orphans_approval_project_rules.rb b/lib/gitlab/background_migration/delete_orphans_approval_project_rules.rb index 33aa1a8d29d..28809df8694 100644 --- a/lib/gitlab/background_migration/delete_orphans_approval_project_rules.rb +++ b/lib/gitlab/background_migration/delete_orphans_approval_project_rules.rb @@ -5,6 +5,7 @@ module Gitlab # Deletes orphans records whenever report_type equals to scan_finding (i.e., 4) class DeleteOrphansApprovalProjectRules < BatchedMigrationJob operation_name :delete_all + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/destroy_invalid_group_members.rb b/lib/gitlab/background_migration/destroy_invalid_group_members.rb index 9eb0d4489d6..79aae719d03 100644 --- a/lib/gitlab/background_migration/destroy_invalid_group_members.rb +++ b/lib/gitlab/background_migration/destroy_invalid_group_members.rb @@ -10,6 +10,7 @@ module Gitlab end operation_name :delete_all + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/destroy_invalid_members.rb b/lib/gitlab/background_migration/destroy_invalid_members.rb index b274c71f24f..9a70dc39960 100644 --- a/lib/gitlab/background_migration/destroy_invalid_members.rb +++ b/lib/gitlab/background_migration/destroy_invalid_members.rb @@ -5,6 +5,7 @@ module Gitlab class DestroyInvalidMembers < Gitlab::BackgroundMigration::BatchedMigrationJob # rubocop:disable Style/Documentation scope_to ->(relation) { relation.where(member_namespace_id: nil) } operation_name :delete_all + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/destroy_invalid_project_members.rb b/lib/gitlab/background_migration/destroy_invalid_project_members.rb index 53b4712ef6e..5f6bb840f77 100644 --- a/lib/gitlab/background_migration/destroy_invalid_project_members.rb +++ b/lib/gitlab/background_migration/destroy_invalid_project_members.rb @@ -5,6 +5,7 @@ module Gitlab class DestroyInvalidProjectMembers < Gitlab::BackgroundMigration::BatchedMigrationJob # rubocop:disable Style/Documentation scope_to ->(relation) { relation.where(source_type: 'Project') } operation_name :delete_all + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_licence_for_recent_public_projects.rb b/lib/gitlab/background_migration/disable_legacy_open_source_licence_for_recent_public_projects.rb index b32e88581dd..c4ce88b9404 100644 --- a/lib/gitlab/background_migration/disable_legacy_open_source_licence_for_recent_public_projects.rb +++ b/lib/gitlab/background_migration/disable_legacy_open_source_licence_for_recent_public_projects.rb @@ -8,6 +8,7 @@ module Gitlab THRESHOLD_DATE = '2022-02-17 09:00:00' operation_name :disable_legacy_open_source_licence_for_recent_public_projects + feature_category :database # Migration only version of `project_settings` table class ProjectSetting < ApplicationRecord diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_inactive_public_projects.rb b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_inactive_public_projects.rb index 5685b782a71..6114aa33a43 100644 --- a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_inactive_public_projects.rb +++ b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_inactive_public_projects.rb @@ -9,6 +9,7 @@ module Gitlab LAST_ACTIVITY_DATE = '2021-07-01' operation_name :disable_legacy_open_source_license_available + feature_category :database # Migration only version of `project_settings` table class ProjectSetting < ApplicationRecord diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_no_issues_no_repo_projects.rb b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_no_issues_no_repo_projects.rb index b5e5555bd2d..2eb7c5230ba 100644 --- a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_no_issues_no_repo_projects.rb +++ b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_no_issues_no_repo_projects.rb @@ -7,6 +7,7 @@ module Gitlab PUBLIC = 20 operation_name :disable_legacy_open_source_license_for_no_issues_no_repo_projects + feature_category :database # Migration only version of `project_settings` table class ProjectSetting < ApplicationRecord diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_one_member_no_repo_projects.rb b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_one_member_no_repo_projects.rb index 89863458676..8953836c705 100644 --- a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_one_member_no_repo_projects.rb +++ b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_one_member_no_repo_projects.rb @@ -7,6 +7,7 @@ module Gitlab PUBLIC = 20 operation_name :disable_legacy_open_source_license_for_one_member_no_repo_projects + feature_category :database # Migration only version of `project_settings` table class ProjectSetting < ApplicationRecord diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_five_mb.rb b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_five_mb.rb index dcef4f086e2..b2805289b30 100644 --- a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_five_mb.rb +++ b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_five_mb.rb @@ -10,6 +10,7 @@ module Gitlab end operation_name :disable_legacy_open_source_license_for_projects_less_than_five_mb + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_one_mb.rb b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_one_mb.rb index 7d93f2d4fda..15c80a6cac2 100644 --- a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_one_mb.rb +++ b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_one_mb.rb @@ -6,6 +6,7 @@ module Gitlab class DisableLegacyOpenSourceLicenseForProjectsLessThanOneMb < ::Gitlab::BackgroundMigration::BatchedMigrationJob scope_to ->(relation) { relation.where(legacy_open_source_license_available: true) } operation_name :disable_legacy_open_source_license_for_projects_less_than_one_mb + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/expire_o_auth_tokens.rb b/lib/gitlab/background_migration/expire_o_auth_tokens.rb index 08bcdb8a789..20dacd642de 100644 --- a/lib/gitlab/background_migration/expire_o_auth_tokens.rb +++ b/lib/gitlab/background_migration/expire_o_auth_tokens.rb @@ -4,21 +4,15 @@ module Gitlab module BackgroundMigration # Add expiry to all OAuth access tokens class ExpireOAuthTokens < ::Gitlab::BackgroundMigration::BatchedMigrationJob - operation_name :update_oauth_tokens + scope_to ->(relation) { relation.where(expires_in: nil) } + operation_name :update_all + feature_category :database def perform - each_sub_batch( - batching_scope: ->(relation) { relation.where(expires_in: nil) } - ) do |sub_batch| - update_oauth_tokens(sub_batch) + each_sub_batch do |sub_batch| + sub_batch.update_all(expires_in: 2.hours) end end - - private - - def update_oauth_tokens(relation) - relation.update_all(expires_in: 7_200) - end end end end diff --git a/lib/gitlab/background_migration/fix_approval_project_rules_without_protected_branches.rb b/lib/gitlab/background_migration/fix_approval_project_rules_without_protected_branches.rb index 4b283bae79d..bfbed0408e1 100644 --- a/lib/gitlab/background_migration/fix_approval_project_rules_without_protected_branches.rb +++ b/lib/gitlab/background_migration/fix_approval_project_rules_without_protected_branches.rb @@ -5,6 +5,8 @@ module Gitlab # This class doesn't update approval project rules # as this feature exists only in EE class FixApprovalProjectRulesWithoutProtectedBranches < BatchedMigrationJob + feature_category :database + def perform; end end end diff --git a/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb b/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb index 97a9913fa74..452167d4d61 100644 --- a/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb +++ b/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb @@ -186,7 +186,7 @@ module Gitlab end def migrate_instance_cluster? - if instance_variable_defined?('@migrate_instance_cluster') + if instance_variable_defined?(:@migrate_instance_cluster) @migrate_instance_cluster else @migrate_instance_cluster = Migratable::Cluster.instance_type.has_prometheus_application? diff --git a/lib/gitlab/background_migration/fix_security_scan_statuses.rb b/lib/gitlab/background_migration/fix_security_scan_statuses.rb index b60e739f870..1cfc9a278b7 100644 --- a/lib/gitlab/background_migration/fix_security_scan_statuses.rb +++ b/lib/gitlab/background_migration/fix_security_scan_statuses.rb @@ -4,6 +4,8 @@ module Gitlab module BackgroundMigration # Fixes the `status` attribute of `security_scans` records class FixSecurityScanStatuses < BatchedMigrationJob + feature_category :database + def perform # no-op. The logic is defined in EE module. end diff --git a/lib/gitlab/background_migration/migrate_shared_vulnerability_scanners.rb b/lib/gitlab/background_migration/migrate_shared_vulnerability_scanners.rb index bea0120f093..d1acb8ca2d2 100644 --- a/lib/gitlab/background_migration/migrate_shared_vulnerability_scanners.rb +++ b/lib/gitlab/background_migration/migrate_shared_vulnerability_scanners.rb @@ -4,6 +4,8 @@ module Gitlab module BackgroundMigration # rubocop: disable Style/Documentation class MigrateSharedVulnerabilityScanners < BatchedMigrationJob + feature_category :database + def perform end end diff --git a/lib/gitlab/background_migration/migrate_vulnerabilities_feedback_to_vulnerabilities_state_transition.rb b/lib/gitlab/background_migration/migrate_vulnerabilities_feedback_to_vulnerabilities_state_transition.rb index 81b29b5a6cd..84f7462e6b8 100644 --- a/lib/gitlab/background_migration/migrate_vulnerabilities_feedback_to_vulnerabilities_state_transition.rb +++ b/lib/gitlab/background_migration/migrate_vulnerabilities_feedback_to_vulnerabilities_state_transition.rb @@ -4,6 +4,8 @@ module Gitlab module BackgroundMigration class MigrateVulnerabilitiesFeedbackToVulnerabilitiesStateTransition < BatchedMigrationJob + feature_category :database + def perform; end end end diff --git a/lib/gitlab/background_migration/populate_approval_merge_request_rules_with_security_orchestration.rb b/lib/gitlab/background_migration/populate_approval_merge_request_rules_with_security_orchestration.rb index 2257dc016be..00d7b1b9664 100644 --- a/lib/gitlab/background_migration/populate_approval_merge_request_rules_with_security_orchestration.rb +++ b/lib/gitlab/background_migration/populate_approval_merge_request_rules_with_security_orchestration.rb @@ -5,6 +5,8 @@ module Gitlab # This class doesn't delete merge request level rules # as this feature exists only in EE class PopulateApprovalMergeRequestRulesWithSecurityOrchestration < BatchedMigrationJob + feature_category :database + def perform; end end end diff --git a/lib/gitlab/background_migration/populate_approval_project_rules_with_security_orchestration.rb b/lib/gitlab/background_migration/populate_approval_project_rules_with_security_orchestration.rb index 1d0c0010551..e5f283db926 100644 --- a/lib/gitlab/background_migration/populate_approval_project_rules_with_security_orchestration.rb +++ b/lib/gitlab/background_migration/populate_approval_project_rules_with_security_orchestration.rb @@ -5,6 +5,8 @@ module Gitlab # This class doesn't delete merge request level rules # as this feature exists only in EE class PopulateApprovalProjectRulesWithSecurityOrchestration < BatchedMigrationJob + feature_category :database + def perform; end end end diff --git a/lib/gitlab/background_migration/populate_operation_visibility_permissions_from_operations.rb b/lib/gitlab/background_migration/populate_operation_visibility_permissions_from_operations.rb index 3dd867fa1fe..46758bc8fed 100644 --- a/lib/gitlab/background_migration/populate_operation_visibility_permissions_from_operations.rb +++ b/lib/gitlab/background_migration/populate_operation_visibility_permissions_from_operations.rb @@ -7,6 +7,7 @@ module Gitlab # The operations_access_level setting is being split into three seperate toggles. class PopulateOperationVisibilityPermissionsFromOperations < BatchedMigrationJob operation_name :populate_operations_visibility + feature_category :database def perform each_sub_batch do |batch| diff --git a/lib/gitlab/background_migration/populate_projects_star_count.rb b/lib/gitlab/background_migration/populate_projects_star_count.rb index 085d576637e..8417dc91b1b 100644 --- a/lib/gitlab/background_migration/populate_projects_star_count.rb +++ b/lib/gitlab/background_migration/populate_projects_star_count.rb @@ -7,6 +7,7 @@ module Gitlab MAX_UPDATE_RETRIES = 3 operation_name :update_all + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/prune_stale_project_export_jobs.rb b/lib/gitlab/background_migration/prune_stale_project_export_jobs.rb index a91cda2c427..3b4b55276fa 100644 --- a/lib/gitlab/background_migration/prune_stale_project_export_jobs.rb +++ b/lib/gitlab/background_migration/prune_stale_project_export_jobs.rb @@ -8,6 +8,7 @@ module Gitlab scope_to ->(relation) { relation.where("updated_at < ?", EXPIRES_IN.ago) } operation_name :delete_all + feature_category :database def perform each_sub_batch(&:delete_all) diff --git a/lib/gitlab/background_migration/re_expire_o_auth_tokens.rb b/lib/gitlab/background_migration/re_expire_o_auth_tokens.rb new file mode 100644 index 00000000000..c327b14669d --- /dev/null +++ b/lib/gitlab/background_migration/re_expire_o_auth_tokens.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Style/Documentation + class ReExpireOAuthTokens < Gitlab::BackgroundMigration::ExpireOAuthTokens # rubocop:disable Migration/BackgroundMigrationBaseClass + end + # rubocop: enable Style/Documentation + end +end diff --git a/lib/gitlab/background_migration/recount_epic_cache_counts.rb b/lib/gitlab/background_migration/recount_epic_cache_counts.rb index 42f84a33a5a..cec17ef7cff 100644 --- a/lib/gitlab/background_migration/recount_epic_cache_counts.rb +++ b/lib/gitlab/background_migration/recount_epic_cache_counts.rb @@ -4,6 +4,8 @@ module Gitlab module BackgroundMigration # rubocop: disable Style/Documentation class RecountEpicCacheCounts < Gitlab::BackgroundMigration::BatchedMigrationJob + feature_category :database + def perform; end end # rubocop: enable Style/Documentation diff --git a/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at.rb b/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at.rb index dc7c16d7947..7b88e10f39c 100644 --- a/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at.rb +++ b/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at.rb @@ -7,6 +7,7 @@ module Gitlab # These job artifacts will not be deleted and will have their `expire_at` removed. class RemoveBackfilledJobArtifactsExpireAt < BatchedMigrationJob operation_name :update_all + feature_category :database # The migration would have backfilled `expire_at` # to midnight on the 22nd of the month of the local timezone, diff --git a/lib/gitlab/background_migration/remove_self_managed_wiki_notes.rb b/lib/gitlab/background_migration/remove_self_managed_wiki_notes.rb index a284c04d4f5..cf3897208b8 100644 --- a/lib/gitlab/background_migration/remove_self_managed_wiki_notes.rb +++ b/lib/gitlab/background_migration/remove_self_managed_wiki_notes.rb @@ -5,6 +5,7 @@ module Gitlab # Removes obsolete wiki notes class RemoveSelfManagedWikiNotes < BatchedMigrationJob operation_name :delete_all + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/rename_task_system_note_to_checklist_item.rb b/lib/gitlab/background_migration/rename_task_system_note_to_checklist_item.rb index 1b13c2ab7ef..0615d8a6783 100644 --- a/lib/gitlab/background_migration/rename_task_system_note_to_checklist_item.rb +++ b/lib/gitlab/background_migration/rename_task_system_note_to_checklist_item.rb @@ -14,6 +14,7 @@ module Gitlab } operation_name :update_all + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values.rb b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values.rb index 832385fd662..64eae1e934e 100644 --- a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values.rb +++ b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values.rb @@ -5,6 +5,7 @@ module Gitlab # A job to nullify duplicate token_encrypted values in ci_runners table in batches class ResetDuplicateCiRunnersTokenEncryptedValues < BatchedMigrationJob operation_name :nullify_duplicate_ci_runner_token_encrypted_values + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values.rb b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values.rb index 5f552accd8d..fd15caa5644 100644 --- a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values.rb +++ b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values.rb @@ -5,6 +5,7 @@ module Gitlab # A job to nullify duplicate token values in ci_runners table in batches class ResetDuplicateCiRunnersTokenValues < BatchedMigrationJob operation_name :nullify_duplicate_ci_runner_token_values + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/reset_status_on_container_repositories.rb b/lib/gitlab/background_migration/reset_status_on_container_repositories.rb index 09cd3b1895f..0dbe2781327 100644 --- a/lib/gitlab/background_migration/reset_status_on_container_repositories.rb +++ b/lib/gitlab/background_migration/reset_status_on_container_repositories.rb @@ -13,6 +13,7 @@ module Gitlab scope_to ->(relation) { relation.where(status: DELETE_SCHEDULED_STATUS) } operation_name :reset_status_on_container_repositories + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/sanitize_confidential_todos.rb b/lib/gitlab/background_migration/sanitize_confidential_todos.rb index d3ef6ac3019..2df0b8a4d93 100644 --- a/lib/gitlab/background_migration/sanitize_confidential_todos.rb +++ b/lib/gitlab/background_migration/sanitize_confidential_todos.rb @@ -13,6 +13,7 @@ module Gitlab scope_to ->(relation) { relation.where(confidential: true) } operation_name :delete_invalid_todos + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/second_recount_epic_cache_counts.rb b/lib/gitlab/background_migration/second_recount_epic_cache_counts.rb new file mode 100644 index 00000000000..4d7c4a682a9 --- /dev/null +++ b/lib/gitlab/background_migration/second_recount_epic_cache_counts.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Style/Documentation + class SecondRecountEpicCacheCounts < Gitlab::BackgroundMigration::BatchedMigrationJob + feature_category :database + + def perform; end + end + # rubocop: enable Style/Documentation + end +end + +# rubocop: disable Layout/LineLength +# we just want to re-enqueue the previous BackfillEpicCacheCounts migration, +# because it's a EE-only migation and it's a module, we just prepend new +# RecountEpicCacheCounts with existing batched migration module (which is same in both cases) +Gitlab::BackgroundMigration::SecondRecountEpicCacheCounts.prepend_mod_with('Gitlab::BackgroundMigration::BackfillEpicCacheCounts') +# rubocop: enable Layout/LineLength diff --git a/lib/gitlab/background_migration/set_correct_vulnerability_state.rb b/lib/gitlab/background_migration/set_correct_vulnerability_state.rb index dfd71bb8b5f..49ef75d7ba8 100644 --- a/lib/gitlab/background_migration/set_correct_vulnerability_state.rb +++ b/lib/gitlab/background_migration/set_correct_vulnerability_state.rb @@ -8,6 +8,7 @@ module Gitlab scope_to ->(relation) { relation.where.not(dismissed_at: nil) } operation_name :update_vulnerabilities_state + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/set_legacy_open_source_license_available_for_non_public_projects.rb b/lib/gitlab/background_migration/set_legacy_open_source_license_available_for_non_public_projects.rb index 4ae7ad897cf..86fcfa18dc3 100644 --- a/lib/gitlab/background_migration/set_legacy_open_source_license_available_for_non_public_projects.rb +++ b/lib/gitlab/background_migration/set_legacy_open_source_license_available_for_non_public_projects.rb @@ -7,6 +7,7 @@ module Gitlab PUBLIC = 20 operation_name :set_legacy_open_source_license_available + feature_category :database # Migration only version of `project_settings` table class ProjectSetting < ApplicationRecord diff --git a/lib/gitlab/background_migration/truncate_overlong_vulnerability_html_titles.rb b/lib/gitlab/background_migration/truncate_overlong_vulnerability_html_titles.rb new file mode 100644 index 00000000000..5ae1698b910 --- /dev/null +++ b/lib/gitlab/background_migration/truncate_overlong_vulnerability_html_titles.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Truncate the Vulnerability html_title if it exceeds 800 chars + class TruncateOverlongVulnerabilityHtmlTitles < BatchedMigrationJob + feature_category :vulnerability_management + scope_to ->(relation) { relation.where("LENGTH(title_html) > 800") } + operation_name :truncate_vulnerability_title_htmls + + class Vulnerability < ApplicationRecord # rubocop:disable Style/Documentation + self.table_name = "vulnerabilities" + end + + def perform + each_sub_batch do |sub_batch| + sub_batch.update_all("title_html = left(title_html, 800)") + end + end + end + end +end diff --git a/lib/gitlab/background_migration/update_ci_pipeline_artifacts_unknown_locked_status.rb b/lib/gitlab/background_migration/update_ci_pipeline_artifacts_unknown_locked_status.rb index 84183753158..77b4a9ab7e4 100644 --- a/lib/gitlab/background_migration/update_ci_pipeline_artifacts_unknown_locked_status.rb +++ b/lib/gitlab/background_migration/update_ci_pipeline_artifacts_unknown_locked_status.rb @@ -9,6 +9,8 @@ module Gitlab # value of the associated `ci_pipelines.locked` value. This class # does an UPDATE join to make the values match. class UpdateCiPipelineArtifactsUnknownLockedStatus < BatchedMigrationJob + feature_category :database + def perform connection.exec_query(<<~SQL) UPDATE ci_pipeline_artifacts diff --git a/lib/gitlab/background_migration/update_delayed_project_removal_to_null_for_user_namespaces.rb b/lib/gitlab/background_migration/update_delayed_project_removal_to_null_for_user_namespaces.rb index b2cf8298e4f..a7faa5703da 100644 --- a/lib/gitlab/background_migration/update_delayed_project_removal_to_null_for_user_namespaces.rb +++ b/lib/gitlab/background_migration/update_delayed_project_removal_to_null_for_user_namespaces.rb @@ -11,6 +11,7 @@ module Gitlab end operation_name :set_delayed_project_removal_to_null_for_user_namespace + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb b/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb index 8aab7d13b45..6d59a5c8651 100644 --- a/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb +++ b/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb @@ -4,6 +4,8 @@ module Gitlab module BackgroundMigration class UpdateJiraTrackerDataDeploymentTypeBasedOnUrl < Gitlab::BackgroundMigration::BatchedMigrationJob + feature_category :database + # rubocop: disable Gitlab/NamespacedClass class JiraTrackerData < ActiveRecord::Base self.table_name = "jira_tracker_data" diff --git a/lib/gitlab/chat_name_token.rb b/lib/gitlab/chat_name_token.rb index 76f2a4ae38c..9b4cb9d0134 100644 --- a/lib/gitlab/chat_name_token.rb +++ b/lib/gitlab/chat_name_token.rb @@ -16,9 +16,7 @@ module Gitlab def get Gitlab::Redis::SharedState.with do |redis| data = redis.get(redis_shared_state_key) - params = Gitlab::Json.parse(data, symbolize_names: true) if data - params[:integration_id] ||= params.delete(:service_id) if params && params[:service_id] - params + Gitlab::Json.parse(data, symbolize_names: true) if data end end diff --git a/lib/gitlab/ci/config/entry/cache.rb b/lib/gitlab/ci/config/entry/cache.rb index a5481071fc5..a635f409109 100644 --- a/lib/gitlab/ci/config/entry/cache.rb +++ b/lib/gitlab/ci/config/entry/cache.rb @@ -9,7 +9,7 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[key untracked paths when policy].freeze + ALLOWED_KEYS = %i[key untracked paths when policy unprotect].freeze ALLOWED_POLICY = %w[pull-push push pull].freeze DEFAULT_POLICY = 'pull-push' ALLOWED_WHEN = %w[on_success on_failure always].freeze @@ -33,18 +33,22 @@ module Gitlab entry :key, Entry::Key, description: 'Cache key used to define a cache affinity.' + entry :unprotect, ::Gitlab::Config::Entry::Boolean, + description: 'Unprotect the cache from a protected ref.' + entry :untracked, ::Gitlab::Config::Entry::Boolean, description: 'Cache all untracked files.' entry :paths, Entry::Paths, description: 'Specify which paths should be cached across builds.' - attributes :policy, :when + attributes :policy, :when, :unprotect def value result = super result[:key] = key_value + result[:unprotect] = unprotect || false result[:policy] = policy || DEFAULT_POLICY # Use self.when to avoid conflict with reserved word result[:when] = self.when || DEFAULT_WHEN diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index e0a052ffdfd..e0f0903174c 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -27,9 +27,9 @@ module Gitlab validates :config, disallowed_keys: { in: %i[only except start_in], - message: 'key may not be used with `rules`' - }, - if: :has_rules? + message: 'key may not be used with `rules`', + ignore_nil: true + }, if: :has_rules_value? with_options allow_nil: true do validates :extends, array_of_strings_or_string: true diff --git a/lib/gitlab/ci/config/entry/product/parallel.rb b/lib/gitlab/ci/config/entry/product/parallel.rb index 5c78a8f68c7..e91714e3f5c 100644 --- a/lib/gitlab/ci/config/entry/product/parallel.rb +++ b/lib/gitlab/ci/config/entry/product/parallel.rb @@ -12,7 +12,7 @@ module Gitlab strategy :ParallelBuilds, if: -> (config) { config.is_a?(Numeric) } strategy :MatrixBuilds, if: -> (config) { config.is_a?(Hash) } - PARALLEL_LIMIT = 50 + PARALLEL_LIMIT = 200 class ParallelBuilds < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb index 16844fa88db..6408f412e6f 100644 --- a/lib/gitlab/ci/config/entry/reports.rb +++ b/lib/gitlab/ci/config/entry/reports.rb @@ -54,7 +54,7 @@ module Gitlab end def value - @config.transform_values do |value| + @config.compact.transform_values do |value| if value.is_a?(Hash) value else diff --git a/lib/gitlab/ci/config/entry/variable.rb b/lib/gitlab/ci/config/entry/variable.rb index decb568ffc9..a5c6aaa1e3a 100644 --- a/lib/gitlab/ci/config/entry/variable.rb +++ b/lib/gitlab/ci/config/entry/variable.rb @@ -54,9 +54,7 @@ module Gitlab validates :key, alphanumeric: true validates :config_value, alphanumeric: true, allow_nil: true validates :config_description, alphanumeric: true, allow_nil: true - validates :config_expand, boolean: true, allow_nil: true, if: -> { - ci_raw_variables_in_yaml_config_enabled? - } + validates :config_expand, boolean: true, allow_nil: true validates :config_options, array_of_strings: true, allow_nil: true validate do @@ -82,16 +80,10 @@ module Gitlab end def value_with_data - if ci_raw_variables_in_yaml_config_enabled? - { - value: config_value.to_s, - raw: (!config_expand if has_config_expand?) - }.compact - else - { - value: config_value.to_s - }.compact - end + { + value: config_value.to_s, + raw: (!config_expand if has_config_expand?) + }.compact end def value_with_prefill_data @@ -100,10 +92,6 @@ module Gitlab options: config_options ).compact end - - def ci_raw_variables_in_yaml_config_enabled? - YamlProcessor::FeatureFlags.enabled?(:ci_raw_variables_in_yaml_config) - end end class UnknownStrategy < ::Gitlab::Config::Entry::Node diff --git a/lib/gitlab/ci/config/external/file/artifact.rb b/lib/gitlab/ci/config/external/file/artifact.rb index 21a57640aee..140cbfac5c1 100644 --- a/lib/gitlab/ci/config/external/file/artifact.rb +++ b/lib/gitlab/ci/config/external/file/artifact.rb @@ -38,10 +38,6 @@ module Gitlab private - def project - context&.parent_pipeline&.project - end - def validate_context! context.logger.instrument(:config_file_artifact_validate_context) do if !creating_child_pipeline? diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb index 65caf4ac47d..7899fe0ff73 100644 --- a/lib/gitlab/ci/config/external/file/base.rb +++ b/lib/gitlab/ci/config/external/file/base.rb @@ -47,7 +47,6 @@ module Gitlab end def validate! - context.check_execution_time! if ::Feature.disabled?(:ci_refactoring_external_mapper, context.project) validate_location! validate_context! if valid? fetch_and_validate_content! if valid? diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index a41bc2b39f2..61b4d1ada10 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -7,18 +7,6 @@ module Gitlab class Mapper include Gitlab::Utils::StrongMemoize - # Will be removed with FF ci_refactoring_external_mapper - FILE_CLASSES = [ - External::File::Local, - External::File::Project, - External::File::Remote, - External::File::Template, - External::File::Artifact - ].freeze - - # Will be removed with FF ci_refactoring_external_mapper - FILE_SUBKEYS = FILE_CLASSES.map { |f| f.name.demodulize.downcase }.freeze - Error = Class.new(StandardError) AmbigiousSpecificationError = Class.new(Error) TooManyIncludesError = Class.new(Error) @@ -32,11 +20,7 @@ module Gitlab return [] if @locations.empty? context.logger.instrument(:config_mapper_process) do - if ::Feature.enabled?(:ci_refactoring_external_mapper, context.project) - process_without_instrumentation - else - legacy_process_without_instrumentation - end + process_without_instrumentation end end @@ -57,138 +41,6 @@ module Gitlab files end - - # This and the following methods will be removed with FF ci_refactoring_external_mapper - def legacy_process_without_instrumentation - @locations - .map(&method(:normalize_location)) - .filter_map(&method(:verify_rules)) - .flat_map(&method(:expand_project_files)) - .flat_map(&method(:expand_wildcard_paths)) - .map(&method(:expand_variables)) - .map(&method(:select_first_matching)) - .each(&method(:verify!)) - end - - # convert location if String to canonical form - def normalize_location(location) - if location.is_a?(String) - expanded_location = expand_variables(location) - normalize_location_string(expanded_location) - else - location.deep_symbolize_keys - end - end - - def verify_rules(location) - logger.instrument(:config_mapper_rules) do - verify_rules_without_instrumentation(location) - end - end - - def verify_rules_without_instrumentation(location) - return unless Rules.new(location[:rules]).evaluate(context).pass? - - location - end - - def expand_project_files(location) - return location unless location[:project] - - Array.wrap(location[:file]).map do |file| - location.merge(file: file) - end - end - - def expand_wildcard_paths(location) - logger.instrument(:config_mapper_wildcards) do - expand_wildcard_paths_without_instrumentation(location) - end - end - - def expand_wildcard_paths_without_instrumentation(location) - # We only support local files for wildcard paths - return location unless location[:local] && location[:local].include?('*') - - context.project.repository.search_files_by_wildcard_path(location[:local], context.sha).map do |path| - { local: path } - end - end - - def normalize_location_string(location) - if ::Gitlab::UrlSanitizer.valid?(location) - { remote: location } - else - { local: location } - end - end - - def select_first_matching(location) - logger.instrument(:config_mapper_select) do - select_first_matching_without_instrumentation(location) - end - end - - def select_first_matching_without_instrumentation(location) - matching = FILE_CLASSES.map do |file_class| - file_class.new(location, context) - end.select(&:matching?) - - if matching.one? - matching.first - elsif matching.empty? - raise AmbigiousSpecificationError, "`#{masked_location(location.to_json)}` does not have a valid subkey for include. Valid subkeys are: `#{FILE_SUBKEYS.join('`, `')}`" - else - raise AmbigiousSpecificationError, "Each include must use only one of: `#{FILE_SUBKEYS.join('`, `')}`" - end - end - - def verify!(location_object) - verify_max_includes! - location_object.validate! - expandset.add(location_object) - end - - def verify_max_includes! - if expandset.count >= context.max_includes - raise TooManyIncludesError, "Maximum of #{context.max_includes} nested includes are allowed!" - end - end - - def expand_variables(data) - logger.instrument(:config_mapper_variables) do - expand_variables_without_instrumentation(data) - end - end - - def expand_variables_without_instrumentation(data) - if data.is_a?(String) - expand(data) - else - transform(data) - end - end - - def transform(data) - data.transform_values do |values| - case values - when Array - values.map { |value| expand(value.to_s) } - when String - expand(values) - else - values - end - end - end - - def expand(data) - ExpandVariables.expand(data, -> { context.variables_hash }) - end - - def masked_location(location) - context.mask_variables_from(location) - end end end end diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb index ab5203252a2..e6a2e5c3b33 100644 --- a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb +++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb @@ -17,7 +17,7 @@ module Gitlab secret_detection: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4] }.freeze - VERSIONS_TO_REMOVE_IN_16_0 = [].freeze + VERSIONS_TO_REMOVE_IN_16_0 = %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3].freeze DEPRECATED_VERSIONS = { cluster_image_scanning: VERSIONS_TO_REMOVE_IN_16_0, @@ -30,6 +30,8 @@ module Gitlab secret_detection: VERSIONS_TO_REMOVE_IN_16_0 }.freeze + CURRENT_VERSIONS = SUPPORTED_VERSIONS.to_h { |k, v| [k, v - DEPRECATED_VERSIONS[k]] } + class Schema def root_path File.join(__dir__, 'schemas') @@ -129,6 +131,11 @@ module Gitlab end def report_uses_deprecated_schema_version? + # Avoid deprecation warnings for GitLab security scanners + # To be removed via https://gitlab.com/gitlab-org/gitlab/-/issues/386798 + return if report_data.dig('scan', 'scanner', 'vendor', 'name')&.downcase == 'gitlab' + return if report_data.dig('scan', 'analyzer', 'vendor', 'name')&.downcase == 'gitlab' + DEPRECATED_VERSIONS[report_type].include?(report_version) end @@ -182,11 +189,15 @@ module Gitlab def add_deprecated_report_version_message log_warnings(problem_type: 'using_deprecated_schema_version') - template = _("Version %{report_version} for report type %{report_type} has been deprecated,"\ - " supported versions for this report type are: %{supported_schema_versions}."\ - " GitLab will attempt to parse and ingest this report if valid.") + template = _("version %{report_version} for report type %{report_type} is deprecated. "\ + "However, GitLab will still attempt to parse and ingest this report. "\ + "Upgrade the security report to one of the following versions: %{current_schema_versions}.") - message = format(template, report_version: report_version, report_type: report_type, supported_schema_versions: supported_schema_versions) + message = format( + template, + report_version: report_version, + report_type: report_type, + current_schema_versions: current_schema_versions) add_message_as(level: :deprecation_warning, message: message) end @@ -207,6 +218,10 @@ module Gitlab ) end + def current_schema_versions + CURRENT_VERSIONS[report_type].join(", ") + end + def supported_schema_versions SUPPORTED_VERSIONS[report_type].join(", ") end diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 31b130b5ab7..d2dc712e366 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -127,6 +127,10 @@ module Gitlab .observe({ plan: project.actual_plan_name }, jobs_count) end + def observe_pipeline_includes_count(pipeline) + logger.observe(:pipeline_includes_count, pipeline.config_metadata&.[](:includes)&.count, once: true) + end + def increment_pipeline_failure_reason_counter(reason) metrics.pipeline_failure_reason_counter .increment(reason: (reason || :unknown_failure).to_s) diff --git a/lib/gitlab/ci/pipeline/chain/create_deployments.rb b/lib/gitlab/ci/pipeline/chain/create_deployments.rb index a8276d84b87..99e438ddbae 100644 --- a/lib/gitlab/ci/pipeline/chain/create_deployments.rb +++ b/lib/gitlab/ci/pipeline/chain/create_deployments.rb @@ -6,7 +6,7 @@ module Gitlab module Chain class CreateDeployments < Chain::Base def perform! - create_deployments! + create_deployments! if Feature.disabled?(:move_create_deployments_to_worker, pipeline.project) end def break? diff --git a/lib/gitlab/ci/pipeline/chain/populate.rb b/lib/gitlab/ci/pipeline/chain/populate.rb index 654e24be8e1..c59ef2ba6a4 100644 --- a/lib/gitlab/ci/pipeline/chain/populate.rb +++ b/lib/gitlab/ci/pipeline/chain/populate.rb @@ -18,7 +18,8 @@ module Gitlab pipeline.stages = @command.pipeline_seed.stages if stage_names.empty? - return error('No stages / jobs for this pipeline.') + return error('Pipeline will not run for the selected trigger. ' \ + 'The rules configuration prevented any jobs from being added to the pipeline.') end if pipeline.invalid? diff --git a/lib/gitlab/ci/pipeline/chain/populate_metadata.rb b/lib/gitlab/ci/pipeline/chain/populate_metadata.rb index 89befb2a65b..e7a9009f8f4 100644 --- a/lib/gitlab/ci/pipeline/chain/populate_metadata.rb +++ b/lib/gitlab/ci/pipeline/chain/populate_metadata.rb @@ -22,8 +22,7 @@ module Gitlab private def set_pipeline_name - return if Feature.disabled?(:pipeline_name, pipeline.project) || - @command.yaml_processor_result.workflow_name.blank? + return if @command.yaml_processor_result.workflow_name.blank? name = @command.yaml_processor_result.workflow_name name = ExpandVariables.expand(name, -> { global_context.variables.sort_and_expand_all }) diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb index de147914850..dd097187955 100644 --- a/lib/gitlab/ci/pipeline/chain/sequence.rb +++ b/lib/gitlab/ci/pipeline/chain/sequence.rb @@ -30,6 +30,7 @@ module Gitlab @command.observe_creation_duration(current_monotonic_time - @start) @command.observe_pipeline_size(@pipeline) @command.observe_jobs_count_in_alive_pipelines + @command.observe_pipeline_includes_count(@pipeline) @pipeline end diff --git a/lib/gitlab/ci/pipeline/logger.rb b/lib/gitlab/ci/pipeline/logger.rb index f393406b549..8286dfc6560 100644 --- a/lib/gitlab/ci/pipeline/logger.rb +++ b/lib/gitlab/ci/pipeline/logger.rb @@ -121,7 +121,7 @@ module Gitlab def enabled? ::Feature.enabled?(:ci_pipeline_creation_logger, project, type: :ops) end - strong_memoize_attr :enabled?, :enabled + strong_memoize_attr :enabled? def observations @observations ||= {} diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index b0b79b994c1..684b58474ad 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -53,7 +53,7 @@ module Gitlab end end end - strong_memoize_attr :included?, :inclusion + strong_memoize_attr :included? def errors logger.instrument(:pipeline_seed_build_errors) do @@ -261,7 +261,7 @@ module Gitlab def reuse_build_in_seed_context? Feature.enabled?(:ci_reuse_build_in_seed_context, @pipeline.project) end - strong_memoize_attr :reuse_build_in_seed_context?, :reuse_build_in_seed_context + strong_memoize_attr :reuse_build_in_seed_context? end end end diff --git a/lib/gitlab/ci/pipeline/seed/build/cache.rb b/lib/gitlab/ci/pipeline/seed/build/cache.rb index 781065a63db..409b6658cc0 100644 --- a/lib/gitlab/ci/pipeline/seed/build/cache.rb +++ b/lib/gitlab/ci/pipeline/seed/build/cache.rb @@ -14,6 +14,7 @@ module Gitlab @policy = local_cache.delete(:policy) @untracked = local_cache.delete(:untracked) @when = local_cache.delete(:when) + @unprotect = local_cache.delete(:unprotect) @custom_key_prefix = custom_key_prefix raise ArgumentError, "unknown cache keys: #{local_cache.keys}" if local_cache.any? @@ -25,7 +26,8 @@ module Gitlab paths: @paths, policy: @policy, untracked: @untracked, - when: @when + when: @when, + unprotect: @unprotect }.compact end diff --git a/lib/gitlab/ci/status/build/manual.rb b/lib/gitlab/ci/status/build/manual.rb index 0074f3675e0..5e77db3d336 100644 --- a/lib/gitlab/ci/status/build/manual.rb +++ b/lib/gitlab/ci/status/build/manual.rb @@ -22,14 +22,26 @@ module Gitlab def illustration_content if can?(user, :update_build, subject) - _('This job requires manual intervention to start. Before starting this job, you can add variables below for last-minute configuration changes.') + manual_job_action_message else generic_permission_failure_message end end + def manual_job_action_message + if subject.retryable? + _("You can modify this job's CI/CD variables before running it again.") + else + _('This job does not start automatically and must be started manually. You can add CI/CD variables below for last-minute configuration changes before starting the job.') + end + end + def generic_permission_failure_message - _("This job does not run automatically and must be started manually, but you do not have access to it.") + if subject.outdated_deployment? + _("This deployment job does not run automatically and must be started manually, but it's older than the latest deployment, and therefore can't run.") + else + _("This job does not run automatically and must be started manually, but you do not have access to it.") + end end end end diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index b4beeb60dfd..47b79302828 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -8,7 +8,7 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE_TAG: "0.87.3" + CODE_QUALITY_IMAGE_TAG: "0.89.0" CODE_QUALITY_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/gitlab-org/ci-cd/codequality:$CODE_QUALITY_IMAGE_TAG" needs: [] script: diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index 7a208584c4c..6884a9556b4 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.42.1' + DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.45.0' .dast-auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 292b0a0036d..dc7e5f445d2 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.42.1' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.45.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index ba03ad6304f..9e15b07f5d1 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.42.1' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.45.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml index 2c5027cdb43..8b49d2de8cf 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml @@ -255,7 +255,7 @@ sobelow-sast: when: never - if: $CI_COMMIT_BRANCH exists: - - 'mix.exs' + - '**/mix.exs' spotbugs-sast: extends: .sast-analyzer diff --git a/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml index 58709d3ab62..1c4dbe6cd0f 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml @@ -332,12 +332,12 @@ sobelow-sast: when: never - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. exists: - - 'mix.exs' + - '**/mix.exs' - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. when: never - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. exists: - - 'mix.exs' + - '**/mix.exs' spotbugs-sast: extends: .sast-analyzer diff --git a/lib/gitlab/ci/variables/collection.rb b/lib/gitlab/ci/variables/collection.rb index e9766061072..9960a6fbdf5 100644 --- a/lib/gitlab/ci/variables/collection.rb +++ b/lib/gitlab/ci/variables/collection.rb @@ -72,8 +72,36 @@ module Gitlab Collection.new(@variables.reject(&block)) end - # `expand_raw_refs` will be deleted with the FF `ci_raw_variables_in_yaml_config`. - def expand_value(value, keep_undefined: false, expand_file_refs: true, expand_raw_refs: true, project: nil) + def sort_and_expand_all(keep_undefined: false, expand_file_refs: true, expand_raw_refs: true) + sorted = Sort.new(self) + return self.class.new(self, sorted.errors) unless sorted.valid? + + new_collection = self.class.new + + sorted.tsort.each do |item| + unless item.depends_on + new_collection.append(item) + next + end + + # expand variables as they are added + variable = item.to_runner_variable + variable[:value] = new_collection.expand_value(variable[:value], keep_undefined: keep_undefined, + expand_file_refs: expand_file_refs, + expand_raw_refs: expand_raw_refs) + new_collection.append(variable) + end + + new_collection + end + + def to_s + "#{@variables_by_key.keys}, @errors='#{@errors}'" + end + + protected + + def expand_value(value, keep_undefined: false, expand_file_refs: true, expand_raw_refs: true) value.gsub(Item::VARIABLES_REGEXP) do match = Regexp.last_match # it is either a valid variable definition or a ($$ / %%) full_match = match[0] @@ -88,19 +116,20 @@ module Gitlab if variable # VARIABLE_NAME is an existing variable if variable.file? - # Will be cleaned up with https://gitlab.com/gitlab-org/gitlab/-/issues/378266 - if project - # We only log if `project` exists to make sure it is called from `Ci::BuildRunnerPresenter` - # when the variables are sent to Runner. - Gitlab::AppJsonLogger.info(event: 'file_variable_is_referenced_in_another_variable', - project_id: project.id, - variable: variable_name) - end - expand_file_refs ? variable.value : full_match elsif variable.raw? - # With `full_match`, we defer the expansion of raw variables to the runner. If we expand them here, - # the runner will not know the expanded value is a raw variable and it tries to expand it again. + # Normally, it's okay to expand a raw variable if it's referenced in another variable because + # its rawness is not broken. However, the runner also tries to expand variables. + # Here, with `full_match`, we defer the expansion of raw variables to the runner. + # If we expand them here, the runner will not know that the expanded value is a raw variable + # and it tries to expand it again. + # Example: `A` is a normal variable with value `normal`. + # `B` is a raw variable with value `raw-$A`. + # `C` is a normal variable with value `$B`. + # If we expanded `C` here, the runner would receive `C` as `raw-$A`. And since `A` is a normal + # variable, the runner would expand it. So, the result would be `raw-normal`. + # With `full_match`, the runner receives `C` as `$B`. And since `B` is a raw variable, the + # runner expanded it as `raw-$A`, which is what we want. # Discussion: https://gitlab.com/gitlab-org/gitlab/-/issues/353991#note_1103274951 expand_raw_refs ? variable.value : full_match else @@ -115,36 +144,7 @@ module Gitlab end end - # `expand_raw_refs` will be deleted with the FF `ci_raw_variables_in_yaml_config`. - def sort_and_expand_all(keep_undefined: false, expand_file_refs: true, expand_raw_refs: true, project: nil) - sorted = Sort.new(self) - return self.class.new(self, sorted.errors) unless sorted.valid? - - new_collection = self.class.new - - sorted.tsort.each do |item| - unless item.depends_on - new_collection.append(item) - next - end - - # expand variables as they are added - variable = item.to_runner_variable - variable[:value] = new_collection.expand_value(variable[:value], keep_undefined: keep_undefined, - expand_file_refs: expand_file_refs, - expand_raw_refs: expand_raw_refs, - project: project) - new_collection.append(variable) - end - - new_collection - end - - def to_s - "#{@variables_by_key.keys}, @errors='#{@errors}'" - end - - protected + private attr_reader :variables end diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index f2c1ad0575d..d867439b10b 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -64,12 +64,7 @@ module Gitlab private def assign_valid_attributes - @root_variables = if YamlProcessor::FeatureFlags.enabled?(:ci_raw_variables_in_yaml_config) - transform_to_array(@ci_config.variables_with_data) - else - transform_to_array(@ci_config.variables) - end - + @root_variables = transform_to_array(@ci_config.variables_with_data) @root_variables_with_prefill_data = @ci_config.variables_with_prefill_data @stages = @ci_config.stages diff --git a/lib/gitlab/config/entry/attributable.rb b/lib/gitlab/config/entry/attributable.rb index c8ad2521574..2e5b226678a 100644 --- a/lib/gitlab/config/entry/attributable.rb +++ b/lib/gitlab/config/entry/attributable.rb @@ -24,6 +24,10 @@ module Gitlab define_method("has_#{attribute_method}?") do config.is_a?(Hash) && config.key?(attribute) end + + define_method("has_#{attribute_method}_value?") do + config.is_a?(Hash) && config.key?(attribute) && !config[attribute].nil? + end end end end diff --git a/lib/gitlab/config/entry/validators.rb b/lib/gitlab/config/entry/validators.rb index b88a6766d92..9e6a3d86e92 100644 --- a/lib/gitlab/config/entry/validators.rb +++ b/lib/gitlab/config/entry/validators.rb @@ -17,6 +17,7 @@ module Gitlab class DisallowedKeysValidator < ActiveModel::EachValidator def validate_each(record, attribute, value) + value = value.try(:compact) if options[:ignore_nil] present_keys = value.try(:keys).to_a & options[:in] if present_keys.any? diff --git a/lib/gitlab/counters.rb b/lib/gitlab/counters.rb new file mode 100644 index 00000000000..5ff664f53bd --- /dev/null +++ b/lib/gitlab/counters.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +module Gitlab + module Counters + Increment = Struct.new(:amount, :ref, keyword_init: true) + end +end diff --git a/lib/gitlab/counters/buffered_counter.rb b/lib/gitlab/counters/buffered_counter.rb index 56593b642a9..3e232c78e45 100644 --- a/lib/gitlab/counters/buffered_counter.rb +++ b/lib/gitlab/counters/buffered_counter.rb @@ -8,6 +8,17 @@ module Gitlab WORKER_DELAY = 10.minutes WORKER_LOCK_TTL = 10.minutes + # Refresh keys are set to expire after a very long time, + # so that they do not occupy Redis memory indefinitely, + # if for any reason they are not deleted. + # In practice, a refresh is not expected to take longer than this TTL. + REFRESH_KEYS_TTL = 14.days + CLEANUP_BATCH_SIZE = 50 + CLEANUP_INTERVAL_SECONDS = 0.1 + + # Limit size of bitmap key to 2^26-1 (~8MB) + MAX_BITMAP_OFFSET = 67108863 + LUA_FLUSH_INCREMENT_SCRIPT = <<~LUA local increment_key, flushed_key = KEYS[1], KEYS[2] local increment_value = redis.call("get", increment_key) or 0 @@ -31,9 +42,47 @@ module Gitlab end end - def increment(amount) + LUA_INCREMENT_WITH_DEDUPLICATION_SCRIPT = <<~LUA + local counter_key, refresh_key, refresh_indicator_key = KEYS[1], KEYS[2], KEYS[3] + local tracking_shard_key, opposing_tracking_shard_key, shards_key = KEYS[4], KEYS[5], KEYS[6] + + local amount, tracking_offset = tonumber(ARGV[1]), tonumber(ARGV[2]) + + -- increment to the counter key when not refreshing + if redis.call("exists", refresh_indicator_key) == 0 then + return redis.call("incrby", counter_key, amount) + end + + -- deduplicate and increment to the refresh counter key while refreshing + local found_duplicate = redis.call("getbit", tracking_shard_key, tracking_offset) + if found_duplicate == 1 then + return redis.call("get", refresh_key) + end + + redis.call("setbit", tracking_shard_key, tracking_offset, 1) + redis.call("expire", tracking_shard_key, #{REFRESH_KEYS_TTL.seconds}) + redis.call("sadd", shards_key, tracking_shard_key) + redis.call("expire", shards_key, #{REFRESH_KEYS_TTL.seconds}) + + local found_opposing_change = redis.call("getbit", opposing_tracking_shard_key, tracking_offset) + local increment_without_previous_decrement = amount > 0 and found_opposing_change == 0 + local decrement_with_previous_increment = amount < 0 and found_opposing_change == 1 + local net_change = 0 + + if increment_without_previous_decrement or decrement_with_previous_increment then + net_change = amount + end + + return redis.call("incrby", refresh_key, net_change) + LUA + + def increment(increment) result = redis_state do |redis| - redis.incrby(key, amount) + if Feature.enabled?(:project_statistics_bulk_increment, type: :development) + redis.eval(LUA_INCREMENT_WITH_DEDUPLICATION_SCRIPT, **increment_args(increment)).to_i + else + redis.incrby(key, increment.amount) + end end FlushCounterIncrementsWorker.perform_in(WORKER_DELAY, counter_record.class.name, counter_record.id, attribute) @@ -41,11 +90,63 @@ module Gitlab result end - def reset! + def bulk_increment(increments) + result = redis_state do |redis| + redis.pipelined do |pipeline| + increments.each do |increment| + if Feature.enabled?(:project_statistics_bulk_increment, type: :development) + pipeline.eval(LUA_INCREMENT_WITH_DEDUPLICATION_SCRIPT, **increment_args(increment)) + else + pipeline.incrby(key, increment.amount) + end + end + end + end + + FlushCounterIncrementsWorker.perform_in(WORKER_DELAY, counter_record.class.name, counter_record.id, attribute) + + result.last.to_i + end + + LUA_INITIATE_REFRESH_SCRIPT = <<~LUA + local counter_key, refresh_indicator_key = KEYS[1], KEYS[2] + redis.call("del", counter_key) + redis.call("set", refresh_indicator_key, 1, "ex", #{REFRESH_KEYS_TTL.seconds}) + LUA + + def initiate_refresh! counter_record.update!(attribute => 0) redis_state do |redis| - redis.del(key) + redis.eval(LUA_INITIATE_REFRESH_SCRIPT, keys: [key, refresh_indicator_key]) + end + end + + LUA_FINALIZE_REFRESH_SCRIPT = <<~LUA + local counter_key, refresh_key, refresh_indicator_key = KEYS[1], KEYS[2], KEYS[3] + local refresh_amount = redis.call("get", refresh_key) or 0 + + redis.call("incrby", counter_key, refresh_amount) + redis.call("del", refresh_indicator_key, refresh_key) + LUA + + def finalize_refresh + redis_state do |redis| + redis.eval(LUA_FINALIZE_REFRESH_SCRIPT, keys: [key, refresh_key, refresh_indicator_key]) + end + + FlushCounterIncrementsWorker.perform_in(WORKER_DELAY, counter_record.class.name, counter_record.id, attribute) + ::Counters::CleanupRefreshWorker.perform_async(counter_record.class.name, counter_record.id, attribute) + end + + def cleanup_refresh + redis_state do |redis| + while (shards = redis.spop(shards_key, CLEANUP_BATCH_SIZE)) + redis.del(*shards) + break if shards.size < CLEANUP_BATCH_SIZE + + sleep CLEANUP_INTERVAL_SECONDS + end end end @@ -87,10 +188,67 @@ module Gitlab "#{key}:flushed" end + def refresh_indicator_key + "#{key}:refresh-in-progress" + end + + def refresh_key + "#{key}:refresh" + end + private attr_reader :counter_record, :attribute + def increment_args(increment) + { + keys: [ + key, + refresh_key, + refresh_indicator_key, + tracking_shard_key(increment), + opposing_tracking_shard_key(increment), + shards_key + ], + argv: [ + increment.amount, + tracking_offset(increment) + ] + } + end + + def tracking_shard_key(increment) + positive?(increment) ? positive_shard_key(increment.ref.to_i) : negative_shard_key(increment.ref.to_i) + end + + def opposing_tracking_shard_key(increment) + positive?(increment) ? negative_shard_key(increment.ref.to_i) : positive_shard_key(increment.ref.to_i) + end + + def shards_key + "#{refresh_key}:shards" + end + + def positive_shard_key(ref) + "#{refresh_key}:+:#{shard_number(ref)}" + end + + def negative_shard_key(ref) + "#{refresh_key}:-:#{shard_number(ref)}" + end + + def shard_number(ref) + ref / MAX_BITMAP_OFFSET + end + + def tracking_offset(increment) + increment.ref.to_i % MAX_BITMAP_OFFSET + end + + def positive?(increment) + increment.amount > 0 + end + def remove_flushed_key redis_state do |redis| redis.del(flushed_key) diff --git a/lib/gitlab/counters/legacy_counter.rb b/lib/gitlab/counters/legacy_counter.rb index 06951514ec3..823f9955168 100644 --- a/lib/gitlab/counters/legacy_counter.rb +++ b/lib/gitlab/counters/legacy_counter.rb @@ -11,23 +11,36 @@ module Gitlab @current_value = counter_record.method(attribute).call end - def increment(amount) - updated = counter_record.class.update_counters(counter_record.id, { attribute => amount }) + def increment(increment) + updated = update_counter_record_attribute(increment.amount) if updated == 1 counter_record.execute_after_commit_callbacks - @current_value += amount + @current_value += increment.amount end @current_value end - def reset! - counter_record.update!(attribute => 0) + def bulk_increment(increments) + total_increment = increments.sum(&:amount) + + updated = update_counter_record_attribute(total_increment) + + if updated == 1 + counter_record.execute_after_commit_callbacks + @current_value += total_increment + end + + @current_value end private + def update_counter_record_attribute(amount) + counter_record.class.update_counters(counter_record.id, { attribute => amount }) + end + attr_reader :counter_record, :attribute end end diff --git a/lib/gitlab/data_builder/build.rb b/lib/gitlab/data_builder/build.rb index 8eda871770b..8fec5cf3303 100644 --- a/lib/gitlab/data_builder/build.rb +++ b/lib/gitlab/data_builder/build.rb @@ -45,6 +45,7 @@ module Gitlab commit: { # note: commit.id is actually the pipeline id id: commit.id, + name: commit.name, sha: commit.sha, message: commit.git_commit_message, author_name: commit.git_author_name, diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 51d5bfcee38..40e2e637114 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -84,7 +84,7 @@ module Gitlab # # TODO: https://gitlab.com/gitlab-org/geo-team/discussions/-/issues/5032 def self.database_base_models_using_load_balancing - @database_base_models_with_gitlab_shared ||= { + @database_base_models_using_load_balancing ||= { # Note that we use ActiveRecord::Base here and not ApplicationRecord. # This is deliberate, as we also use these classes to apply load # balancing to, and the load balancer must be enabled for _all_ models diff --git a/lib/gitlab/database/as_with_materialized.rb b/lib/gitlab/database/as_with_materialized.rb index a04ea97117d..417e9f211b9 100644 --- a/lib/gitlab/database/as_with_materialized.rb +++ b/lib/gitlab/database/as_with_materialized.rb @@ -25,7 +25,7 @@ module Gitlab # Note: to be deleted after the minimum PG version is set to 12.0 # Update the documentation together when deleting the method - # https://docs.gitlab.com/ee/development/merge_request_performance_guidelines.html#use-ctes-wisely + # https://docs.gitlab.com/ee/development/merge_request_concepts/performance.html#use-ctes-wisely def self.materialized_if_supported materialized_supported? ? 'MATERIALIZED' : '' end diff --git a/lib/gitlab/database/async_indexes/index_creator.rb b/lib/gitlab/database/async_indexes/index_creator.rb index 2fb4cc8f675..3ae2bb7b3e5 100644 --- a/lib/gitlab/database/async_indexes/index_creator.rb +++ b/lib/gitlab/database/async_indexes/index_creator.rb @@ -4,10 +4,10 @@ module Gitlab module Database module AsyncIndexes class IndexCreator - include ExclusiveLeaseGuard + include IndexingExclusiveLeaseGuard TIMEOUT_PER_ACTION = 1.day - STATEMENT_TIMEOUT = 9.hours + STATEMENT_TIMEOUT = 20.hours def initialize(async_index) @async_index = async_index @@ -47,10 +47,6 @@ module Gitlab TIMEOUT_PER_ACTION end - def lease_key - [super, async_index.connection_db_config.name].join('/') - end - def set_statement_timeout connection.execute("SET statement_timeout TO '%ds'" % STATEMENT_TIMEOUT) yield diff --git a/lib/gitlab/database/async_indexes/index_destructor.rb b/lib/gitlab/database/async_indexes/index_destructor.rb index fe05872b87a..66955df9d04 100644 --- a/lib/gitlab/database/async_indexes/index_destructor.rb +++ b/lib/gitlab/database/async_indexes/index_destructor.rb @@ -4,7 +4,7 @@ module Gitlab module Database module AsyncIndexes class IndexDestructor - include ExclusiveLeaseGuard + include IndexingExclusiveLeaseGuard TIMEOUT_PER_ACTION = 1.day @@ -53,10 +53,6 @@ module Gitlab TIMEOUT_PER_ACTION end - def lease_key - [super, async_index.connection_db_config.name].join('/') - end - def log_index_info(message) Gitlab::AppLogger.info(message: message, table_name: async_index.table_name, index_name: async_index.name) end diff --git a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb index ad747a8131d..f1fc3efae9e 100644 --- a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb +++ b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb @@ -49,6 +49,8 @@ module Gitlab def execute_job(tracking_record) job_class = tracking_record.migration_job_class + ApplicationContext.push(feature_category: fetch_feature_category(job_class)) + if job_class < Gitlab::BackgroundMigration::BatchedMigrationJob execute_batched_migration_job(job_class, tracking_record) else @@ -86,6 +88,14 @@ module Gitlab job_instance end + + def fetch_feature_category(job_class) + if job_class.respond_to?(:feature_category) + job_class.feature_category.to_s + else + Gitlab::BackgroundMigration::BatchedMigrationJob::DEFAULT_FEATURE_CATEGORY + end + end end end end diff --git a/lib/gitlab/database/gitlab_schema.rb b/lib/gitlab/database/gitlab_schema.rb index 0f848ed40fb..38558512b6a 100644 --- a/lib/gitlab/database/gitlab_schema.rb +++ b/lib/gitlab/database/gitlab_schema.rb @@ -15,46 +15,12 @@ module Gitlab module Database module GitlabSchema + UnknownSchemaError = Class.new(StandardError) + DICTIONARY_PATH = 'db/docs/' - # These tables are deleted/renamed, but still referenced by migrations. - # This is needed for now, but should be removed in the future - DELETED_TABLES = { - # main tables - 'alerts_service_data' => :gitlab_main, - 'analytics_devops_adoption_segment_selections' => :gitlab_main, - 'analytics_repository_file_commits' => :gitlab_main, - 'analytics_repository_file_edits' => :gitlab_main, - 'analytics_repository_files' => :gitlab_main, - 'audit_events_archived' => :gitlab_main, - 'backup_labels' => :gitlab_main, - 'clusters_applications_fluentd' => :gitlab_main, - 'forked_project_links' => :gitlab_main, - 'issue_milestones' => :gitlab_main, - 'merge_request_milestones' => :gitlab_main, - 'namespace_onboarding_actions' => :gitlab_main, - 'services' => :gitlab_main, - 'terraform_state_registry' => :gitlab_main, - 'tmp_fingerprint_sha256_migration' => :gitlab_main, # used by lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb - 'web_hook_logs_archived' => :gitlab_main, - 'vulnerability_export_registry' => :gitlab_main, - 'vulnerability_finding_fingerprints' => :gitlab_main, - 'vulnerability_export_verification_status' => :gitlab_main, - - # CI tables - 'ci_build_trace_sections' => :gitlab_ci, - 'ci_build_trace_section_names' => :gitlab_ci, - 'ci_daily_report_results' => :gitlab_ci, - 'ci_test_cases' => :gitlab_ci, - 'ci_test_case_failures' => :gitlab_ci, - - # leftovers from early implementation of partitioning - 'audit_events_part_5fc467ac26' => :gitlab_main, - 'web_hook_logs_part_0c5294f417' => :gitlab_main - }.freeze - - def self.table_schemas(tables) - tables.map { |table| table_schema(table) }.to_set + def self.table_schemas(tables, undefined: true) + tables.map { |table| table_schema(table, undefined: undefined) }.to_set end def self.table_schema(name, undefined: true) @@ -69,13 +35,13 @@ module Gitlab # strip partition number of a form `loose_foreign_keys_deleted_records_1` table_name.gsub!(/_[0-9]+$/, '') - # Tables that are properly mapped + # Tables and views that are properly mapped if gitlab_schema = views_and_tables_to_schema[table_name] return gitlab_schema end - # Tables that are deleted, but we still need to reference them - if gitlab_schema = DELETED_TABLES[table_name] + # Tables and views that are deleted, but we still need to reference them + if gitlab_schema = deleted_views_and_tables_to_schema[table_name] return gitlab_schema end @@ -106,29 +72,58 @@ module Gitlab [Rails.root.join(DICTIONARY_PATH, 'views', '*.yml')] end + def self.deleted_views_path_globs + [Rails.root.join(DICTIONARY_PATH, 'deleted_views', '*.yml')] + end + + def self.deleted_tables_path_globs + [Rails.root.join(DICTIONARY_PATH, 'deleted_tables', '*.yml')] + end + def self.views_and_tables_to_schema @views_and_tables_to_schema ||= self.tables_to_schema.merge(self.views_to_schema) end - def self.tables_to_schema - @tables_to_schema ||= Dir.glob(self.dictionary_path_globs).each_with_object({}) do |file_path, dic| - data = YAML.load_file(file_path) + def self.table_schema!(name) + self.table_schema(name, undefined: false) || raise( + UnknownSchemaError, + "Could not find gitlab schema for table #{name}: Any new tables must be added to the database dictionary" + ) + end - dic[data['table_name']] = data['gitlab_schema'].to_sym - end + def self.deleted_views_and_tables_to_schema + @deleted_views_and_tables_to_schema ||= self.deleted_tables_to_schema.merge(self.deleted_views_to_schema) end - def self.views_to_schema - @views_to_schema ||= Dir.glob(self.view_path_globs).each_with_object({}) do |file_path, dic| - data = YAML.load_file(file_path) + def self.deleted_tables_to_schema + @deleted_tables_to_schema ||= self.build_dictionary(self.deleted_tables_path_globs) + end - dic[data['view_name']] = data['gitlab_schema'].to_sym - end + def self.deleted_views_to_schema + @deleted_views_to_schema ||= self.build_dictionary(self.deleted_views_path_globs) + end + + def self.tables_to_schema + @tables_to_schema ||= self.build_dictionary(self.dictionary_path_globs) + end + + def self.views_to_schema + @views_to_schema ||= self.build_dictionary(self.view_path_globs) end def self.schema_names @schema_names ||= self.views_and_tables_to_schema.values.to_set end + + private_class_method def self.build_dictionary(path_globs) + Dir.glob(path_globs).each_with_object({}) do |file_path, dic| + data = YAML.load_file(file_path) + + key_name = data['table_name'] || data['view_name'] + + dic[key_name] = data['gitlab_schema'].to_sym + end + end end end end diff --git a/lib/gitlab/database/indexing_exclusive_lease_guard.rb b/lib/gitlab/database/indexing_exclusive_lease_guard.rb new file mode 100644 index 00000000000..fb45de347e6 --- /dev/null +++ b/lib/gitlab/database/indexing_exclusive_lease_guard.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module IndexingExclusiveLeaseGuard + extend ActiveSupport::Concern + include ExclusiveLeaseGuard + + def lease_key + @lease_key ||= "gitlab/database/indexing/actions/#{database_config_name}" + end + + def database_config_name + Gitlab::Database.db_config_name(connection) + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/resolver.rb b/lib/gitlab/database/load_balancing/resolver.rb index a291080cc3d..3e3998cae92 100644 --- a/lib/gitlab/database/load_balancing/resolver.rb +++ b/lib/gitlab/database/load_balancing/resolver.rb @@ -7,8 +7,21 @@ module Gitlab module Database module LoadBalancing class Resolver + FAR_FUTURE_TTL = 100.years.from_now + UnresolvableNameserverError = Class.new(StandardError) + Response = Class.new do + attr_reader :address, :ttl + + def initialize(address:, ttl:) + raise ArgumentError unless ttl.present? && address.present? + + @address = address + @ttl = ttl + end + end + def initialize(nameserver) @nameserver = nameserver end @@ -28,13 +41,14 @@ module Gitlab private def ip_address - IPAddr.new(@nameserver) + # IP addresses are valid forever + Response.new(address: IPAddr.new(@nameserver), ttl: FAR_FUTURE_TTL) rescue IPAddr::InvalidAddressError end def ip_address_from_hosts_file ip = Resolv::Hosts.new.getaddress(@nameserver) - IPAddr.new(ip) + Response.new(address: IPAddr.new(ip), ttl: FAR_FUTURE_TTL) rescue Resolv::ResolvError end @@ -42,7 +56,12 @@ module Gitlab answer = Net::DNS::Resolver.start(@nameserver, Net::DNS::A).answer return if answer.empty? - answer.first.address + raw_response = answer.first + + # Defaults to 30 seconds if there is no TTL present + ttl_in_seconds = raw_response.ttl.presence || 30 + + Response.new(address: answer.first.address, ttl: ttl_in_seconds.seconds.from_now) rescue Net::DNS::Resolver::NoResponseError raise UnresolvableNameserverError, "no response from DNS server(s)" end diff --git a/lib/gitlab/database/load_balancing/service_discovery.rb b/lib/gitlab/database/load_balancing/service_discovery.rb index 3295301a2d7..5059b3b5c93 100644 --- a/lib/gitlab/database/load_balancing/service_discovery.rb +++ b/lib/gitlab/database/load_balancing/service_discovery.rb @@ -69,6 +69,7 @@ module Gitlab @use_tcp = use_tcp @load_balancer = load_balancer @max_replica_pools = max_replica_pools + @nameserver_ttl = 1.second.ago # Begin with an expired ttl to trigger a nameserver dns lookup end # rubocop:enable Metrics/ParameterLists @@ -191,8 +192,14 @@ module Gitlab end def resolver - @resolver ||= Net::DNS::Resolver.new( - nameservers: Resolver.new(@nameserver).resolve, + return @resolver if defined?(@resolver) && @nameserver_ttl.future? + + response = Resolver.new(@nameserver).resolve + + @nameserver_ttl = response.ttl + + @resolver = Net::DNS::Resolver.new( + nameservers: response.address, port: @port, use_tcp: @use_tcp ) diff --git a/lib/gitlab/database/lock_writes_manager.rb b/lib/gitlab/database/lock_writes_manager.rb index e3ae2892668..2e08e1ffb42 100644 --- a/lib/gitlab/database/lock_writes_manager.rb +++ b/lib/gitlab/database/lock_writes_manager.rb @@ -22,37 +22,38 @@ module Gitlab end end - def initialize(table_name:, connection:, database_name:, logger: nil, dry_run: false) + def initialize(table_name:, connection:, database_name:, with_retries: true, logger: nil, dry_run: false) @table_name = table_name @connection = connection @database_name = database_name @logger = logger @dry_run = dry_run + @with_retries = with_retries @table_name_without_schema = ActiveRecord::ConnectionAdapters::PostgreSQL::Utils .extract_schema_qualified_name(table_name) .identifier end - def table_locked_for_writes?(table_name) + def table_locked_for_writes? query = <<~SQL SELECT COUNT(*) from information_schema.triggers WHERE event_object_table = '#{table_name_without_schema}' - AND trigger_name = '#{write_trigger_name(table_name)}' + AND trigger_name = '#{write_trigger_name}' SQL connection.select_value(query) == EXPECTED_TRIGGER_RECORD_COUNT end def lock_writes - if table_locked_for_writes?(table_name) + if table_locked_for_writes? logger&.info "Skipping lock_writes, because #{table_name} is already locked for writes" return end logger&.info "Database: '#{database_name}', Table: '#{table_name}': Lock Writes".color(:yellow) sql_statement = <<~SQL - CREATE TRIGGER #{write_trigger_name(table_name)} + CREATE TRIGGER #{write_trigger_name} BEFORE INSERT OR UPDATE OR DELETE OR TRUNCATE ON #{table_name} FOR EACH STATEMENT EXECUTE FUNCTION #{TRIGGER_FUNCTION_NAME}(); @@ -64,7 +65,7 @@ module Gitlab def unlock_writes logger&.info "Database: '#{database_name}', Table: '#{table_name}': Allow Writes".color(:green) sql_statement = <<~SQL - DROP TRIGGER IF EXISTS #{write_trigger_name(table_name)} ON #{table_name}; + DROP TRIGGER IF EXISTS #{write_trigger_name} ON #{table_name}; SQL execute_sql_statement(sql_statement) @@ -72,19 +73,23 @@ module Gitlab private - attr_reader :table_name, :connection, :database_name, :logger, :dry_run, :table_name_without_schema + attr_reader :table_name, :connection, :database_name, :logger, :dry_run, :table_name_without_schema, :with_retries def execute_sql_statement(sql) if dry_run logger&.info sql - else - with_retries(connection) do + elsif with_retries + raise "Cannot call lock_retries_helper if a transaction is already open" if connection.transaction_open? + + run_with_retries(connection) do connection.execute(sql) end + else + connection.execute(sql) end end - def with_retries(connection, &block) + def run_with_retries(connection, &block) with_statement_timeout_retries do with_lock_retries(connection) do yield @@ -110,11 +115,12 @@ module Gitlab Gitlab::Database::WithLockRetries.new( klass: "gitlab:db:lock_writes", logger: logger || Gitlab::AppLogger, - connection: connection + connection: connection, + allow_savepoints: false # this causes the WithLockRetries to fail if sub-transaction has been detected. ).run(&block) end - def write_trigger_name(table_name) + def write_trigger_name "gitlab_schema_write_trigger_for_#{table_name_without_schema}" end end diff --git a/lib/gitlab/database/loose_foreign_keys.rb b/lib/gitlab/database/loose_foreign_keys.rb index 1338b18a099..6512c672965 100644 --- a/lib/gitlab/database/loose_foreign_keys.rb +++ b/lib/gitlab/database/loose_foreign_keys.rb @@ -22,7 +22,7 @@ module Gitlab { column: config.fetch('column'), on_delete: config.fetch('on_delete').to_sym, - gitlab_schema: GitlabSchema.table_schema(child_table_name) + gitlab_schema: GitlabSchema.table_schema!(child_table_name) } ) end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 4858a96c173..e41107370ec 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -281,6 +281,9 @@ module Gitlab # target_column - The name of the referenced column, defaults to "id". # on_delete - The action to perform when associated data is removed, # defaults to "CASCADE". + # on_update - The action to perform when associated data is updated, + # defaults to nil. This is useful for multi column FKs if + # it's desirable to update one of the columns. # name - The name of the foreign key. # validate - Flag that controls whether the new foreign key will be validated after creation. # If the flag is not set, the constraint will only be enforced for new data. @@ -288,7 +291,8 @@ module Gitlab # order of the ALTER TABLE. This can be useful in situations where the foreign # key creation could deadlock with another process. # - def add_concurrent_foreign_key(source, target, column:, on_delete: :cascade, target_column: :id, name: nil, validate: true, reverse_lock_order: false) + # rubocop: disable Metrics/ParameterLists + def add_concurrent_foreign_key(source, target, column:, on_delete: :cascade, on_update: nil, target_column: :id, name: nil, validate: true, reverse_lock_order: false) # Transactions would result in ALTER TABLE locks being held for the # duration of the transaction, defeating the purpose of this method. if transaction_open? @@ -298,6 +302,7 @@ module Gitlab options = { column: column, on_delete: on_delete, + on_update: on_update, name: name.presence || concurrent_foreign_key_name(source, column), primary_key: target_column } @@ -306,7 +311,8 @@ module Gitlab warning_message = "Foreign key not created because it exists already " \ "(this may be due to an aborted migration or similar): " \ "source: #{source}, target: #{target}, column: #{options[:column]}, "\ - "name: #{options[:name]}, on_delete: #{options[:on_delete]}" + "name: #{options[:name]}, on_update: #{options[:on_update]}, "\ + "on_delete: #{options[:on_delete]}" Gitlab::AppLogger.warn warning_message else @@ -322,6 +328,7 @@ module Gitlab ADD CONSTRAINT #{options[:name]} FOREIGN KEY (#{multiple_columns(options[:column])}) REFERENCES #{target} (#{multiple_columns(target_column)}) + #{on_update_statement(options[:on_update])} #{on_delete_statement(options[:on_delete])} NOT VALID; EOF @@ -343,6 +350,7 @@ module Gitlab end end end + # rubocop: enable Metrics/ParameterLists def validate_foreign_key(source, column, name: nil) fk_name = name || concurrent_foreign_key_name(source, column) @@ -357,10 +365,28 @@ module Gitlab end def foreign_key_exists?(source, target = nil, **options) - foreign_keys(source).any? do |foreign_key| - tables_match?(target.to_s, foreign_key.to_table.to_s) && - options_match?(foreign_key.options, options) + # This if block is necessary because foreign_key_exists? is called in down migrations that may execute before + # the postgres_foreign_keys view had necessary columns added, or even before the view existed. + # In that case, we revert to the previous behavior of this method. + # The behavior in the if block has a bug: it always returns false if the fk being checked has multiple columns. + # This can be removed after init_schema.rb passes 20221122210711_add_columns_to_postgres_foreign_keys.rb + # Tracking issue: https://gitlab.com/gitlab-org/gitlab/-/issues/386796 + if ActiveRecord::Migrator.current_version < 20221122210711 + return foreign_keys(source).any? do |foreign_key| + tables_match?(target.to_s, foreign_key.to_table.to_s) && + options_match?(foreign_key.options, options) + end end + + fks = Gitlab::Database::PostgresForeignKey.by_constrained_table_name(source) + + fks = fks.by_referenced_table_name(target) if target + fks = fks.by_name(options[:name]) if options[:name] + fks = fks.by_constrained_columns(options[:column]) if options[:column] + fks = fks.by_referenced_columns(options[:primary_key]) if options[:primary_key] + fks = fks.by_on_delete_action(options[:on_delete]) if options[:on_delete] + + fks.exists? end # Returns the name for a concurrent foreign key. @@ -1278,6 +1304,13 @@ into similar problems in the future (e.g. when new tables are created). "ON DELETE #{on_delete.upcase}" end + def on_update_statement(on_update) + return '' if on_update.blank? + return 'ON UPDATE SET NULL' if on_update == :nullify + + "ON UPDATE #{on_update.upcase}" + end + def create_column_from(table, old, new, type: nil, batch_column_name: :id, type_cast_function: nil, limit: nil) old_col = column_for(table, old) new_type = type || old_col.type diff --git a/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables.rb b/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables.rb index 0aa4b0d01c4..c59139344ea 100644 --- a/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables.rb +++ b/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables.rb @@ -42,7 +42,7 @@ module Gitlab def should_lock_writes_on_table?(table_name) # currently gitlab_schema represents only present existing tables, this is workaround for deleted tables # that should be skipped as they will be removed in a future migration. - return false if Gitlab::Database::GitlabSchema::DELETED_TABLES[table_name] + return false if Gitlab::Database::GitlabSchema.deleted_tables_to_schema[table_name] table_schema = Gitlab::Database::GitlabSchema.table_schema(table_name.to_s, undefined: false) @@ -60,12 +60,15 @@ module Gitlab Gitlab::Database.gitlab_schemas_for_connection(connection).exclude?(table_schema) end + # with_retries creates new a transaction. So we set it to false if the connection is + # already has an open transaction, to avoid sub-transactions. def lock_writes_on_table(connection, table_name) database_name = Gitlab::Database.db_config_name(connection) LockWritesManager.new( table_name: table_name, connection: connection, database_name: database_name, + with_retries: !connection.transaction_open?, logger: Logger.new($stdout) ).lock_writes end diff --git a/lib/gitlab/database/migrations/base_background_runner.rb b/lib/gitlab/database/migrations/base_background_runner.rb index dbb85bad95c..8975c04e33a 100644 --- a/lib/gitlab/database/migrations/base_background_runner.rb +++ b/lib/gitlab/database/migrations/base_background_runner.rb @@ -44,13 +44,20 @@ module Gitlab jobs.each do |j| break if run_until <= Time.current + meta = migration_meta(j) + instrumentation.observe(version: nil, name: batch_names.next, - connection: connection) do + connection: connection, + meta: meta) do run_job(j) end end end + + def migration_meta(_job) + {} + end end end end diff --git a/lib/gitlab/database/migrations/instrumentation.rb b/lib/gitlab/database/migrations/instrumentation.rb index 7c21346007a..8c479d7eda2 100644 --- a/lib/gitlab/database/migrations/instrumentation.rb +++ b/lib/gitlab/database/migrations/instrumentation.rb @@ -11,8 +11,8 @@ module Gitlab @result_dir = result_dir end - def observe(version:, name:, connection:, &block) - observation = Observation.new(version: version, name: name, success: false) + def observe(version:, name:, connection:, meta: {}, &block) + observation = Observation.new(version: version, name: name, success: false, meta: meta) per_migration_result_dir = File.join(@result_dir, name) diff --git a/lib/gitlab/database/migrations/observation.rb b/lib/gitlab/database/migrations/observation.rb index 228eea3393c..80388c4dbbb 100644 --- a/lib/gitlab/database/migrations/observation.rb +++ b/lib/gitlab/database/migrations/observation.rb @@ -10,6 +10,7 @@ module Gitlab :walltime, :success, :total_database_size_change, + :meta, :query_statistics, keyword_init: true ) diff --git a/lib/gitlab/database/migrations/test_batched_background_runner.rb b/lib/gitlab/database/migrations/test_batched_background_runner.rb index a16103f452c..c123d01f327 100644 --- a/lib/gitlab/database/migrations/test_batched_background_runner.rb +++ b/lib/gitlab/database/migrations/test_batched_background_runner.rb @@ -13,7 +13,7 @@ module Gitlab end def jobs_by_migration_name - Gitlab::Database::SharedModel.using_connection(connection) do + set_shared_model_connection do Gitlab::Database::BackgroundMigration::BatchedMigration .executable .where('id > ?', from_id) @@ -70,7 +70,7 @@ module Gitlab end def run_job(job) - Gitlab::Database::SharedModel.using_connection(connection) do + set_shared_model_connection do Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper.new(connection: connection).perform(job) end end @@ -107,6 +107,16 @@ module Gitlab private attr_reader :from_id + + def set_shared_model_connection(&block) + Gitlab::Database::SharedModel.using_connection(connection, &block) + end + + def migration_meta(job) + set_shared_model_connection do + job.batched_migration.slice(:max_batch_size, :total_tuple_count, :interval) + end + end end end end diff --git a/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb index bd8ed677d77..8849191f356 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb @@ -5,6 +5,7 @@ module Gitlab module PartitioningMigrationHelpers module ForeignKeyHelpers include ::Gitlab::Database::SchemaHelpers + include ::Gitlab::Database::Migrations::LockRetriesHelpers ERROR_SCOPE = 'foreign keys' diff --git a/lib/gitlab/database/postgres_foreign_key.rb b/lib/gitlab/database/postgres_foreign_key.rb index 241b6f009f7..d3ede45fe86 100644 --- a/lib/gitlab/database/postgres_foreign_key.rb +++ b/lib/gitlab/database/postgres_foreign_key.rb @@ -5,17 +5,44 @@ module Gitlab class PostgresForeignKey < SharedModel self.primary_key = :oid + # These values come from the possible confdeltype values in pg_constraint + enum on_delete_action: { + restrict: 'r', + cascade: 'c', + nullify: 'n', + set_default: 'd', + no_action: 'a' + } + scope :by_referenced_table_identifier, ->(identifier) do raise ArgumentError, "Referenced table name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/ where(referenced_table_identifier: identifier) end + scope :by_referenced_table_name, ->(name) { where(referenced_table_name: name) } + scope :by_constrained_table_identifier, ->(identifier) do raise ArgumentError, "Constrained table name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/ where(constrained_table_identifier: identifier) end + + scope :by_constrained_table_name, ->(name) { where(constrained_table_name: name) } + + scope :not_inherited, -> { where(is_inherited: false) } + + scope :by_name, ->(name) { where(name: name) } + + scope :by_constrained_columns, ->(cols) { where(constrained_columns: Array.wrap(cols)) } + + scope :by_referenced_columns, ->(cols) { where(referenced_columns: Array.wrap(cols)) } + + scope :by_on_delete_action, ->(on_delete) do + raise ArgumentError, "Invalid on_delete action #{on_delete}" unless on_delete_actions.key?(on_delete) + + where(on_delete_action: on_delete) + end end end end diff --git a/lib/gitlab/database/postgres_partition.rb b/lib/gitlab/database/postgres_partition.rb index eda11fd8382..e4f70ee1745 100644 --- a/lib/gitlab/database/postgres_partition.rb +++ b/lib/gitlab/database/postgres_partition.rb @@ -17,7 +17,9 @@ module Gitlab for_identifier(identifier).first! end - scope :for_parent_table, ->(name) { where("parent_identifier = concat(current_schema(), '.', ?)", name).order(:name) } + scope :for_parent_table, ->(name) do + where("parent_identifier = concat(current_schema(), '.', ?)", name).order(:name) + end def self.partition_exists?(table_name) where("identifier = concat(current_schema(), '.', ?)", table_name).exists? diff --git a/lib/gitlab/database/query_analyzer.rb b/lib/gitlab/database/query_analyzer.rb index 1280789b30c..6f64d04270f 100644 --- a/lib/gitlab/database/query_analyzer.rb +++ b/lib/gitlab/database/query_analyzer.rb @@ -86,11 +86,7 @@ module Gitlab analyzers.each do |analyzer| next if analyzer.suppressed? && !analyzer.requires_tracking?(parsed) - if analyzer.raw? - analyzer.analyze(sql) - else - analyzer.analyze(parsed) - end + analyzer.analyze(parsed) rescue StandardError, ::Gitlab::Database::QueryAnalyzers::Base::QueryAnalyzerError => e # We catch all standard errors to prevent validation errors to introduce fatal errors in production Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) diff --git a/lib/gitlab/database/query_analyzers/base.rb b/lib/gitlab/database/query_analyzers/base.rb index 9c2c228f869..9a52a4f6e23 100644 --- a/lib/gitlab/database/query_analyzers/base.rb +++ b/lib/gitlab/database/query_analyzers/base.rb @@ -53,10 +53,6 @@ module Gitlab Thread.current[self.context_key] end - def self.raw? - false - end - def self.enabled? raise NotImplementedError end diff --git a/lib/gitlab/database/query_analyzers/gitlab_schemas_validate_connection.rb b/lib/gitlab/database/query_analyzers/gitlab_schemas_validate_connection.rb index 3de9e8011fb..c966ae0e105 100644 --- a/lib/gitlab/database/query_analyzers/gitlab_schemas_validate_connection.rb +++ b/lib/gitlab/database/query_analyzers/gitlab_schemas_validate_connection.rb @@ -22,13 +22,16 @@ module Gitlab return unless allowed_schemas invalid_schemas = table_schemas - allowed_schemas - if invalid_schemas.any? - message = "The query tried to access #{tables} (of #{table_schemas.to_a}) " - message += "which is outside of allowed schemas (#{allowed_schemas}) " - message += "for the current connection '#{Gitlab::Database.db_config_name(parsed.connection)}'" - raise CrossSchemaAccessError, message - end + return if invalid_schemas.empty? + + schema_list = table_schemas.sort.join(',') + + message = "The query tried to access #{tables} (of #{schema_list}) " + message += "which is outside of allowed schemas (#{allowed_schemas}) " + message += "for the current connection '#{Gitlab::Database.db_config_name(parsed.connection)}'" + + raise CrossSchemaAccessError, message end end end diff --git a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb index dd10e0d7992..713e1f772e3 100644 --- a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb +++ b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb @@ -87,15 +87,15 @@ module Gitlab return if tables == ['schema_migrations'] context[:modified_tables_by_db][database].merge(tables) - all_tables = context[:modified_tables_by_db].values.map(&:to_a).flatten + all_tables = context[:modified_tables_by_db].values.flat_map(&:to_a) schemas = ::Gitlab::Database::GitlabSchema.table_schemas(all_tables) schemas += ApplicationRecord.gitlab_transactions_stack if schemas.many? message = "Cross-database data modification of '#{schemas.to_a.join(", ")}' were detected within " \ - "a transaction modifying the '#{all_tables.to_a.join(", ")}' tables. " \ - "Please refer to https://docs.gitlab.com/ee/development/database/multiple_databases.html#removing-cross-database-transactions for details on how to resolve this exception." + "a transaction modifying the '#{all_tables.to_a.join(", ")}' tables. " \ + "Please refer to https://docs.gitlab.com/ee/development/database/multiple_databases.html#removing-cross-database-transactions for details on how to resolve this exception." if schemas.any? { |s| s.to_s.start_with?("undefined") } message += " The gitlab_schema was undefined for one or more of the tables in this transaction. Any new tables must be added to lib/gitlab/database/gitlab_schemas.yml ." diff --git a/lib/gitlab/database/query_analyzers/query_recorder.rb b/lib/gitlab/database/query_analyzers/query_recorder.rb index b54f3442512..63b4fbb8c1d 100644 --- a/lib/gitlab/database/query_analyzers/query_recorder.rb +++ b/lib/gitlab/database/query_analyzers/query_recorder.rb @@ -5,21 +5,19 @@ module Gitlab module QueryAnalyzers class QueryRecorder < Base LOG_PATH = 'query_recorder/' + LIST_PARAMETER_REGEX = %r{\$\d+(?:\s*,\s*\$\d+)+}.freeze + SINGLE_PARAMETER_REGEX = %r{\$\d+}.freeze class << self - def raw? - true - end - def enabled? # Only enable QueryRecorder in CI on database MRs or default branch ENV['CI_MERGE_REQUEST_LABELS']&.include?('database') || (ENV['CI_COMMIT_REF_NAME'].present? && ENV['CI_COMMIT_REF_NAME'] == ENV['CI_DEFAULT_BRANCH']) end - def analyze(sql) + def analyze(parsed) payload = { - sql: sql + normalized: normalize_query(parsed.sql) } log_query(payload) @@ -42,6 +40,12 @@ module Gitlab File.write(log_file, log_line, mode: 'a') end + + def normalize_query(query) + query + .gsub(LIST_PARAMETER_REGEX, '?,?,?') # Replace list parameters with ?,?,? + .gsub(SINGLE_PARAMETER_REGEX, '?') # Replace single parameters with ? + end end end end diff --git a/lib/gitlab/database/reindexing/coordinator.rb b/lib/gitlab/database/reindexing/coordinator.rb index b4f7da999df..eca118a4ff2 100644 --- a/lib/gitlab/database/reindexing/coordinator.rb +++ b/lib/gitlab/database/reindexing/coordinator.rb @@ -4,7 +4,7 @@ module Gitlab module Database module Reindexing class Coordinator - include ExclusiveLeaseGuard + include IndexingExclusiveLeaseGuard # Maximum lease time for the global Redis lease # This should be higher than the maximum time for any @@ -20,6 +20,8 @@ module Gitlab end def perform + return if too_late_for_reindexing? + # This obtains a global lease such that there's # only one live reindexing process at a time. try_obtain_lease do @@ -32,26 +34,28 @@ module Gitlab end def drop + return if too_late_for_reindexing? + try_obtain_lease do Gitlab::AppLogger.info("Removing index #{index.identifier} which is a leftover, temporary index from previous reindexing activity") retries = Gitlab::Database::WithLockRetriesOutsideTransaction.new( - connection: index.connection, + connection: connection, timing_configuration: REMOVE_INDEX_RETRY_CONFIG, klass: self.class, logger: Gitlab::AppLogger ) retries.run(raise_on_exhaustion: false) do - index.connection.tap do |conn| - conn.execute("DROP INDEX CONCURRENTLY IF EXISTS #{conn.quote_table_name(index.schema)}.#{conn.quote_table_name(index.name)}") - end + connection.execute("DROP INDEX CONCURRENTLY IF EXISTS #{full_index_name}") end end end private + delegate :connection, to: :index + def with_notifications(action) notifier.notify_start(action) yield @@ -73,8 +77,18 @@ module Gitlab TIMEOUT_PER_ACTION end - def lease_key - [super, index.connection_db_config.name].join('/') + def full_index_name + [ + connection.quote_table_name(index.schema), + connection.quote_table_name(index.name) + ].join('.') + end + + # We need to check the time explicitly because we execute 4 reindexing + # action per rake invocation and one action can take up to 24 hours. + # This means that it can span for more than the weekend. + def too_late_for_reindexing? + !Time.current.on_weekend? end end end diff --git a/lib/gitlab/database/reindexing/grafana_notifier.rb b/lib/gitlab/database/reindexing/grafana_notifier.rb index ece9327b658..e43eddbefc0 100644 --- a/lib/gitlab/database/reindexing/grafana_notifier.rb +++ b/lib/gitlab/database/reindexing/grafana_notifier.rb @@ -60,7 +60,9 @@ module Gitlab "Authorization": "Bearer #{@api_key}" } - success = Gitlab::HTTP.post("#{@api_url}/api/annotations", body: payload.to_json, headers: headers, allow_local_requests: true).success? + success = Gitlab::HTTP.post( + "#{@api_url}/api/annotations", body: payload.to_json, headers: headers, allow_local_requests: true + ).success? log_error("Response code #{response.code}") unless success diff --git a/lib/gitlab/database/reindexing/index_selection.rb b/lib/gitlab/database/reindexing/index_selection.rb index 2d384f2f9e2..ebe245bfadb 100644 --- a/lib/gitlab/database/reindexing/index_selection.rb +++ b/lib/gitlab/database/reindexing/index_selection.rb @@ -12,6 +12,10 @@ module Gitlab # Only consider indexes beyond this size (before reindexing) INDEX_SIZE_MINIMUM = 1.gigabyte + VERY_LARGE_TABLES = %i[ + ci_builds + ].freeze + delegate :each, to: :indexes def initialize(candidates) @@ -30,13 +34,24 @@ module Gitlab # we force a N+1 pattern here and estimate bloat on a per-index # basis. - @indexes ||= candidates - .not_recently_reindexed - .where('ondisk_size_bytes >= ?', INDEX_SIZE_MINIMUM) + @indexes ||= relations_that_need_cleaning_before_deadline .sort_by(&:relative_bloat_level) # forced N+1 .reverse .select { |candidate| candidate.relative_bloat_level >= MINIMUM_RELATIVE_BLOAT } end + + def relations_that_need_cleaning_before_deadline + relation = candidates.not_recently_reindexed.where('ondisk_size_bytes >= ?', INDEX_SIZE_MINIMUM) + relation = relation.where.not(tablename: VERY_LARGE_TABLES) if too_late_for_very_large_table? + relation + end + + # The reindexing process takes place during the weekends and starting a + # reindexing action on a large table late on Sunday could span during + # Monday. We don't want this because it prevents vacuum from running. + def too_late_for_very_large_table? + !Date.today.saturday? + end end end end diff --git a/lib/gitlab/database/schema_helpers.rb b/lib/gitlab/database/schema_helpers.rb index 2c7ca28942e..d81ff4ff1ae 100644 --- a/lib/gitlab/database/schema_helpers.rb +++ b/lib/gitlab/database/schema_helpers.rb @@ -71,19 +71,6 @@ module Gitlab "#{type}_#{hashed_identifier}" end - def with_lock_retries(*args, **kwargs, &block) - raise_on_exhaustion = !!kwargs.delete(:raise_on_exhaustion) - merged_args = { - connection: connection, - klass: self.class, - logger: Gitlab::BackgroundMigration::Logger, - allow_savepoints: true - }.merge(kwargs) - - Gitlab::Database::WithLockRetries.new(**merged_args) - .run(raise_on_exhaustion: raise_on_exhaustion, &block) - end - def assert_not_in_transaction_block(scope:) return unless transaction_open? diff --git a/lib/gitlab/database/tables_truncate.rb b/lib/gitlab/database/tables_truncate.rb index 807ecdb862a..daef0402742 100644 --- a/lib/gitlab/database/tables_truncate.rb +++ b/lib/gitlab/database/tables_truncate.rb @@ -40,11 +40,12 @@ module Gitlab table_name: table_name, connection: connection, database_name: database_name, + with_retries: true, logger: logger, dry_run: dry_run ) - unless lock_writes_manager.table_locked_for_writes?(table_name) + unless lock_writes_manager.table_locked_for_writes? raise "Table '#{table_name}' is not locked for writes. Run the rake task gitlab:db:lock_writes first" end end @@ -81,6 +82,22 @@ module Gitlab sql_statement = "SELECT set_config('lock_writes.#{table_name_without_schema}', 'false', false)" logger&.info(sql_statement) connection.execute(sql_statement) unless dry_run + + # Temporarily unlocking writes on the attached partitions of the table. + # Because in some cases they might have been locked for writes as well, when they used to be + # normal tables before being converted into attached partitions. + Gitlab::Database::SharedModel.using_connection(connection) do + table_partitions = Gitlab::Database::PostgresPartition.for_parent_table(table_name_without_schema) + table_partitions.each do |table_partition| + partition_name_without_schema = ActiveRecord::ConnectionAdapters::PostgreSQL::Utils + .extract_schema_qualified_name(table_partition.identifier) + .identifier + + sql_statement = "SELECT set_config('lock_writes.#{partition_name_without_schema}', 'false', false)" + logger&.info(sql_statement) + connection.execute(sql_statement) unless dry_run + end + end end # We do the truncation in stages to avoid high IO diff --git a/lib/gitlab/diff/file_collection/merge_request_diff_base.rb b/lib/gitlab/diff/file_collection/merge_request_diff_base.rb index c6ab56e783a..801c1967e0a 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff_base.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff_base.rb @@ -23,7 +23,9 @@ module Gitlab strong_memoize(:diff_files) do diff_files = super - diff_files.each { |diff_file| highlight_cache.decorate(diff_file) } + Gitlab::Metrics.measure(:diffs_highlight_cache_decorate) do + diff_files.each { |diff_file| highlight_cache.decorate(diff_file) } + end diff_files end diff --git a/lib/gitlab/error_tracking.rb b/lib/gitlab/error_tracking.rb index 582c3380869..876a1cbb183 100644 --- a/lib/gitlab/error_tracking.rb +++ b/lib/gitlab/error_tracking.rb @@ -116,6 +116,20 @@ module Gitlab process_exception(exception, extra: extra, trackers: [Logger]) end + # This should be used when you want to log the exception and passthrough + # exception handling: rescue and raise to be catched in upper layers of + # the application. + # + # If the exception implements the method `sentry_extra_data` and that method + # returns a Hash, then the return value of that method will be merged into + # `extra`. Exceptions can use this mechanism to provide structured data + # to sentry in addition to their message and back-trace. + def log_and_raise_exception(exception, extra = {}) + process_exception(exception, extra: extra, trackers: [Logger]) + + raise exception + end + private def before_send_raven(event, hint) diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 344dd27589c..35b330fa089 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -49,7 +49,7 @@ module Gitlab def self.error_message(key) self.ancestors.each do |cls| - return cls.const_get('ERROR_MESSAGES', false).fetch(key) + return cls.const_get(:ERROR_MESSAGES, false).fetch(key) rescue NameError, KeyError next end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 735c7fcf80c..199257f767d 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -37,9 +37,8 @@ module Gitlab @stubs[storage] ||= {} @stubs[storage][name] ||= begin klass = stub_class(name) - addr = stub_address(storage) - creds = stub_creds(storage) - klass.new(addr, creds, interceptors: interceptors, channel_args: channel_args) + channel = create_channel(storage) + klass.new(channel.target, nil, interceptors: interceptors, channel_override: channel) end end end @@ -52,11 +51,29 @@ module Gitlab private_class_method :interceptors def self.channel_args - # These values match the go Gitaly client - # https://gitlab.com/gitlab-org/gitaly/-/blob/bf9f52bc/client/dial.go#L78 { + # These keepalive values match the go Gitaly client + # https://gitlab.com/gitlab-org/gitaly/-/blob/bf9f52bc/client/dial.go#L78 'grpc.keepalive_time_ms': 20000, - 'grpc.keepalive_permit_without_calls': 1 + 'grpc.keepalive_permit_without_calls': 1, + # Enable client-side automatic retry. After enabled, gRPC requests will be retried when there are connectivity + # problems with the target host. Only transparent failures, which mean requests fail before leaving clients, are + # eligible. Other cases are configurable via retry policy in service config (below). In theory, we can auto-retry + # read-only RPCs. Gitaly defines a custom field in service proto. Unfortunately, gRPC ruby doesn't support + # descriptor reflection. + # For more information please visit https://github.com/grpc/proposal/blob/master/A6-client-retries.md + 'grpc.enable_retries': 1, + # Service config is a mechanism for grpc to control the behavior of gRPC client. It defines the client-side + # balancing strategy and retry policy. The config receives a raw JSON string. The format is defined here: + # https://github.com/grpc/grpc-proto/blob/master/grpc/service_config/service_config.proto + 'grpc.service_config': { + # By default, gRPC uses pick_first strategy. This strategy establishes one single connection to the first + # target returned by the name resolver. We would like to use round_robin load-balancing strategy so that + # grpc creates multiple subchannels to all targets retrurned by the resolver. Requests are distributed to + # those subchannels in a round-robin fashion. + # More about client-side load-balancing: https://gitlab.com/groups/gitlab-org/-/epics/8971#note_1207008162 + "loadBalancingConfig": [{ "round_robin": {} }] + }.to_json } end private_class_method :channel_args @@ -81,9 +98,20 @@ module Gitlab address(storage).sub(%r{^tcp://|^tls://}, '') end + # Cache gRPC servers by storage. All the client stubs in the same process can share the underlying connection to the + # same host thanks to HTTP2 framing protocol that gRPC is built on top. This method is not thread-safe. It is + # intended to be a part of `stub`, method behind a mutex protection. + def self.create_channel(storage) + @channels ||= {} + @channels[storage] ||= GRPC::ClientStub.setup_channel( + nil, stub_address(storage), stub_creds(storage), channel_args + ) + end + def self.clear_stubs! MUTEX.synchronize do @stubs = nil + @channels = nil end end diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index 98b1d3dceef..74034c4e717 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -215,12 +215,6 @@ module Gitlab consume_list_refs_response(response) end - def pack_refs - request = Gitaly::PackRefsRequest.new(repository: @gitaly_repo) - - gitaly_client_call(@storage, :ref_service, :pack_refs, request, timeout: GitalyClient.long_timeout) - end - def find_refs_by_oid(oid:, limit:, ref_patterns: nil) request = Gitaly::FindRefsByOIDRequest.new(repository: @gitaly_repo, sort_field: :refname, oid: oid, limit: limit, ref_patterns: ref_patterns) diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index daaf18c711d..203854264ce 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -34,21 +34,6 @@ module Gitlab gitaly_client_call(@storage, :repository_service, :prune_unreachable_objects, request, timeout: GitalyClient.long_timeout) end - def garbage_collect(create_bitmap, prune:) - request = Gitaly::GarbageCollectRequest.new(repository: @gitaly_repo, create_bitmap: create_bitmap, prune: prune) - gitaly_client_call(@storage, :repository_service, :garbage_collect, request, timeout: GitalyClient.long_timeout) - end - - def repack_full(create_bitmap) - request = Gitaly::RepackFullRequest.new(repository: @gitaly_repo, create_bitmap: create_bitmap) - gitaly_client_call(@storage, :repository_service, :repack_full, request, timeout: GitalyClient.long_timeout) - end - - def repack_incremental - request = Gitaly::RepackIncrementalRequest.new(repository: @gitaly_repo) - gitaly_client_call(@storage, :repository_service, :repack_incremental, request, timeout: GitalyClient.long_timeout) - end - def repository_size request = Gitaly::RepositorySizeRequest.new(repository: @gitaly_repo) response = gitaly_client_call(@storage, :repository_service, :repository_size, request, timeout: GitalyClient.long_timeout) diff --git a/lib/gitlab/github_gists_import/importer/gist_importer.rb b/lib/gitlab/github_gists_import/importer/gist_importer.rb index a5e87d3cf7d..4018f425e7c 100644 --- a/lib/gitlab/github_gists_import/importer/gist_importer.rb +++ b/lib/gitlab/github_gists_import/importer/gist_importer.rb @@ -7,6 +7,7 @@ module Gitlab attr_reader :gist, :user FileCountLimitError = Class.new(StandardError) + FILE_COUNT_LIMIT_MESSAGE = 'Snippet maximum file count exceeded' # gist - An instance of `Gitlab::GithubGistsImport::Representation::Gist`. def initialize(gist, user_id) @@ -76,7 +77,7 @@ module Gitlab def fail_and_track(snippet) remove_snippet_and_repository(snippet) - ServiceResponse.error(message: 'Snippet max file count exceeded').track_exception(as: FileCountLimitError) + ServiceResponse.error(message: FILE_COUNT_LIMIT_MESSAGE).track_exception(as: FileCountLimitError) end end end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 065410693e5..1c9ca9f43a8 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -264,18 +264,6 @@ module Gitlab private - def collaborations_subquery - each_object(:repos, nil, { affiliation: 'collaborator' }) - .map { |repo| "repo:#{repo[:full_name]}" } - .join(' ') - end - - def organizations_subquery - each_object(:organizations) - .map { |org| "org:#{org[:login]}" } - .join(' ') - end - def with_retry Retriable.retriable(on: CLIENT_CONNECTION_ERROR, on_retry: on_retry) do yield diff --git a/lib/gitlab/github_import/clients/proxy.rb b/lib/gitlab/github_import/clients/proxy.rb index f6d1c8ed23c..b12df404640 100644 --- a/lib/gitlab/github_import/clients/proxy.rb +++ b/lib/gitlab/github_import/clients/proxy.rb @@ -10,24 +10,24 @@ module Gitlab @client = pick_client(access_token, client_options) end - def repos(search_text, pagination_options) + def repos(search_text, options) return { repos: filtered(client.repos, search_text) } if use_legacy? if use_graphql? - fetch_repos_via_graphql(search_text, pagination_options) + fetch_repos_via_graphql(search_text, options) else - fetch_repos_via_rest(search_text, pagination_options) + fetch_repos_via_rest(search_text, options) end end private - def fetch_repos_via_rest(search_text, pagination_options) - { repos: client.search_repos_by_name(search_text, pagination_options)[:items] } + def fetch_repos_via_rest(search_text, options) + { repos: client.search_repos_by_name(search_text, options)[:items] } end - def fetch_repos_via_graphql(search_text, pagination_options) - response = client.search_repos_by_name_graphql(search_text, pagination_options) + def fetch_repos_via_graphql(search_text, options) + response = client.search_repos_by_name_graphql(search_text, options) { repos: response.dig(:data, :search, :nodes), page_info: response.dig(:data, :search, :pageInfo) diff --git a/lib/gitlab/github_import/clients/search_repos.rb b/lib/gitlab/github_import/clients/search_repos.rb index bcd226087e7..b72e5ac7751 100644 --- a/lib/gitlab/github_import/clients/search_repos.rb +++ b/lib/gitlab/github_import/clients/search_repos.rb @@ -14,18 +14,17 @@ module Gitlab end def search_repos_by_name(name, options = {}) + search_query = search_repos_query(name, options) + with_retry do - octokit.search_repositories( - search_repos_query(str: name, type: :name), - options - ).to_h + octokit.search_repositories(search_query, options).to_h end end private def graphql_search_repos_body(name, options) - query = search_repos_query(str: name, type: :name) + query = search_repos_query(name, options) query = "query: \"#{query}\"" first = options[:first].present? ? ", first: #{options[:first]}" : '' after = options[:after].present? ? ", after: \"#{options[:after]}\"" : '' @@ -52,13 +51,49 @@ module Gitlab TEXT end - def search_repos_query(str:, type:, include_collaborations: true, include_orgs: true) - query = "#{str} in:#{type} is:public,private user:#{octokit.user.to_h[:login]}" + def search_repos_query(string, options = {}) + base = "#{string} in:name is:public,private" + + case options[:relation_type] + when 'organization' then organization_repos_query(base, options) + when 'collaborated' then collaborated_repos_query(base) + when 'owned' then owned_repos_query(base) + # TODO: remove after https://gitlab.com/gitlab-org/gitlab/-/issues/385113 get done + else legacy_all_repos_query(base) + end + end + + def organization_repos_query(search_string, options) + "#{search_string} org:#{options[:organization_login]}" + end + + def collaborated_repos_query(search_string) + "#{search_string} #{collaborations_subquery}" + end + + def owned_repos_query(search_string) + "#{search_string} user:#{octokit.user.to_h[:login]}" + end - query = [query, collaborations_subquery].join(' ') if include_collaborations - query = [query, organizations_subquery].join(' ') if include_orgs + def legacy_all_repos_query(search_string) + [ + search_string, + "user:#{octokit.user.to_h[:login]}", + collaborations_subquery, + organizations_subquery + ].join(' ') + end + + def collaborations_subquery + each_object(:repos, nil, { affiliation: 'collaborator' }) + .map { |repo| "repo:#{repo[:full_name]}" } + .join(' ') + end - query + def organizations_subquery + each_object(:organizations) + .map { |org| "org:#{org[:login]}" } + .join(' ') end end end diff --git a/lib/gitlab/github_import/importer/protected_branch_importer.rb b/lib/gitlab/github_import/importer/protected_branch_importer.rb index 801a0840c52..2077e0c6b11 100644 --- a/lib/gitlab/github_import/importer/protected_branch_importer.rb +++ b/lib/gitlab/github_import/importer/protected_branch_importer.rb @@ -4,7 +4,7 @@ module Gitlab module GithubImport module Importer class ProtectedBranchImporter - attr_reader :protected_branch, :project, :client + attr_reader :project # By default on GitHub, both developers and maintainers can merge # a PR into the protected branch @@ -18,6 +18,7 @@ module Gitlab @protected_branch = protected_branch @project = project @client = client + @user_finder = GithubImport::UserFinder.new(project, client) end def execute @@ -32,11 +33,13 @@ module Gitlab private + attr_reader :protected_branch, :user_finder + def params { name: protected_branch.id, - push_access_levels_attributes: [{ access_level: push_access_level }], - merge_access_levels_attributes: [{ access_level: merge_access_level }], + push_access_levels_attributes: push_access_levels_attributes, + merge_access_levels_attributes: merge_access_levels_attributes, allow_force_push: allow_force_push?, code_owner_approval_required: code_owner_approval_required? } @@ -55,7 +58,7 @@ module Gitlab end def code_owner_approval_required? - return false unless project.licensed_feature_available?(:code_owner_approval_required) + return false unless licensed_feature_available?(:code_owner_approval_required) return protected_branch.require_code_owner_reviews unless protected_on_gitlab? @@ -83,7 +86,7 @@ module Gitlab end def update_project_push_rule - return unless project.licensed_feature_available?(:push_rules) + return unless licensed_feature_available?(:push_rules) return unless protected_branch.required_signatures push_rule = project.push_rule || project.build_push_rule @@ -91,12 +94,34 @@ module Gitlab project.project_setting.update!(push_rule_id: push_rule.id) end - def push_access_level - if protected_branch.required_pull_request_reviews - Gitlab::Access::NO_ACCESS + def push_access_levels_attributes + if allowed_to_push_gitlab_user_ids.present? + @allowed_to_push_gitlab_user_ids.map { |user_id| { user_id: user_id } } + elsif protected_branch.required_pull_request_reviews + [{ access_level: Gitlab::Access::NO_ACCESS }] else - gitlab_access_level_for(:push) + [{ access_level: gitlab_access_level_for(:push) }] + end + end + + def merge_access_levels_attributes + [{ access_level: merge_access_level }] + end + + def allowed_to_push_gitlab_user_ids + return if protected_branch.allowed_to_push_users.empty? || + !licensed_feature_available?(:protected_refs_for_users) + + @allowed_to_push_gitlab_user_ids = [] + + protected_branch.allowed_to_push_users.each do |github_user_data| + gitlab_user_id = user_finder.user_id_for(github_user_data) + next unless gitlab_user_id + + @allowed_to_push_gitlab_user_ids << gitlab_user_id end + + @allowed_to_push_gitlab_user_ids &= project_member_ids end # Gets the strictest merge_access_level between GitHub and GitLab @@ -155,6 +180,14 @@ module Gitlab ProtectedBranch::MergeAccessLevel::GITLAB_DEFAULT_ACCESS_LEVEL end + + def licensed_feature_available?(feature) + project.licensed_feature_available?(feature) + end + + def project_member_ids + project.authorized_users.map(&:id) + end end end end diff --git a/lib/gitlab/github_import/representation/protected_branch.rb b/lib/gitlab/github_import/representation/protected_branch.rb index d2a52b64bbf..eb9dd3bc247 100644 --- a/lib/gitlab/github_import/representation/protected_branch.rb +++ b/lib/gitlab/github_import/representation/protected_branch.rb @@ -10,7 +10,7 @@ module Gitlab attr_reader :attributes expose_attribute :id, :allow_force_pushes, :required_conversation_resolution, :required_signatures, - :required_pull_request_reviews, :require_code_owner_reviews + :required_pull_request_reviews, :require_code_owner_reviews, :allowed_to_push_users # Builds a Branch Protection info from a GitHub API response. # Resource structure details: @@ -19,6 +19,12 @@ module Gitlab def self.from_api_response(branch_protection, _additional_object_data = {}) branch_name = branch_protection[:url].match(%r{/branches/(\S{1,255})/protection$})[1] + allowed_to_push_users = branch_protection.dig(:required_pull_request_reviews, + :bypass_pull_request_allowances, + :users) + allowed_to_push_users &&= allowed_to_push_users.map do |u| + Representation::User.from_api_response(u) + end hash = { id: branch_name, allow_force_pushes: branch_protection.dig(:allow_force_pushes, :enabled), @@ -26,7 +32,8 @@ module Gitlab required_signatures: branch_protection.dig(:required_signatures, :enabled), required_pull_request_reviews: branch_protection[:required_pull_request_reviews].present?, require_code_owner_reviews: branch_protection.dig(:required_pull_request_reviews, - :require_code_owner_reviews).present? + :require_code_owner_reviews).present?, + allowed_to_push_users: allowed_to_push_users.to_a } new(hash) @@ -34,7 +41,13 @@ module Gitlab # Builds a new Protection using a Hash that was built from a JSON payload. def self.from_json_hash(raw_hash) - new(Representation.symbolize_hash(raw_hash)) + hash = Representation.symbolize_hash(raw_hash) + + hash[:allowed_to_push_users].map! do |u| + Representation::User.from_json_hash(u) + end + + new(hash) end # attributes - A Hash containing the raw Protection details. The keys of this diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 12cdcf445f7..ceef072a710 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -57,6 +57,7 @@ module Gitlab gon.current_user_fullname = current_user.name gon.current_user_avatar_url = current_user.avatar_url gon.time_display_relative = current_user.time_display_relative + gon.use_new_navigation = Feature.enabled?(:super_sidebar_nav, current_user) && current_user&.use_new_navigation end # Initialize gon.features with any flags that should be @@ -67,7 +68,6 @@ module Gitlab push_frontend_feature_flag(:source_editor_toolbar) push_frontend_feature_flag(:vscode_web_ide, current_user) push_frontend_feature_flag(:integration_slack_app_notifications) - push_frontend_feature_flag(:vue_group_select) push_frontend_feature_flag(:new_fonts, current_user) end diff --git a/lib/gitlab/graphql/deprecations_base.rb b/lib/gitlab/graphql/deprecations_base.rb index 2ee14620907..8a5f07b6ee9 100644 --- a/lib/gitlab/graphql/deprecations_base.rb +++ b/lib/gitlab/graphql/deprecations_base.rb @@ -9,11 +9,11 @@ module Gitlab def self.included(klass) klass.extend(ClassMethods) - klass.const_set('OLD_GRAPHQL_NAME_MAP', klass::DEPRECATIONS.index_by do |d| + klass.const_set(:OLD_GRAPHQL_NAME_MAP, klass::DEPRECATIONS.index_by do |d| klass.map_graphql_name(d.old_name) end.freeze) - klass.const_set('OLD_NAME_MAP', klass::DEPRECATIONS.index_by(&:old_name).freeze) - klass.const_set('NEW_NAME_MAP', klass::DEPRECATIONS.index_by(&:new_name).freeze) + klass.const_set(:OLD_NAME_MAP, klass::DEPRECATIONS.index_by(&:old_name).freeze) + klass.const_set(:NEW_NAME_MAP, klass::DEPRECATIONS.index_by(&:new_name).freeze) end module ClassMethods diff --git a/lib/gitlab/graphql/errors.rb b/lib/gitlab/graphql/errors.rb index 657364abfdf..319c05d6e23 100644 --- a/lib/gitlab/graphql/errors.rb +++ b/lib/gitlab/graphql/errors.rb @@ -8,6 +8,8 @@ module Gitlab ResourceNotAvailable = Class.new(BaseError) MutationError = Class.new(BaseError) LimitError = Class.new(BaseError) + InvalidMembersError = Class.new(StandardError) + InvalidMemberCountError = Class.new(StandardError) end end end diff --git a/lib/gitlab/hook_data/merge_request_builder.rb b/lib/gitlab/hook_data/merge_request_builder.rb index 96128f432c5..a6ca8323a20 100644 --- a/lib/gitlab/hook_data/merge_request_builder.rb +++ b/lib/gitlab/hook_data/merge_request_builder.rb @@ -64,7 +64,7 @@ module Gitlab assignee_id: merge_request.assignee_ids.first, # This key is deprecated reviewer_ids: merge_request.reviewer_ids, labels: merge_request.labels_hook_attrs, - state: merge_request.state, # This key is deprecated + state: merge_request.state, blocking_discussions_resolved: merge_request.mergeable_discussions_state?, first_contribution: merge_request.first_contribution?, detailed_merge_status: detailed_merge_status diff --git a/lib/gitlab/hotlinking_detector.rb b/lib/gitlab/hotlinking_detector.rb index dd58f6aca26..b5000777010 100644 --- a/lib/gitlab/hotlinking_detector.rb +++ b/lib/gitlab/hotlinking_detector.rb @@ -12,8 +12,6 @@ module Gitlab def intercept_hotlinking?(request) request_accepts = parse_request_accepts(request) - return false unless Feature.enabled?(:repository_archive_hotlinking_interception) - # Block attempts to embed as JS return true if sec_fetch_invalid?(request) diff --git a/lib/gitlab/http.rb b/lib/gitlab/http.rb index b05767c7ed4..c6cd5fbfced 100644 --- a/lib/gitlab/http.rb +++ b/lib/gitlab/http.rb @@ -17,7 +17,8 @@ module Gitlab HTTP_ERRORS = HTTP_TIMEOUT_ERRORS + [ EOFError, SocketError, OpenSSL::SSL::SSLError, OpenSSL::OpenSSLError, Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ENETUNREACH, - Gitlab::HTTP::BlockedUrlError, Gitlab::HTTP::RedirectionTooDeep + Gitlab::HTTP::BlockedUrlError, Gitlab::HTTP::RedirectionTooDeep, + Net::HTTPBadResponse ].freeze DEFAULT_TIMEOUT_OPTIONS = { diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 7a42ffca779..31952f75006 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -44,28 +44,28 @@ module Gitlab TRANSLATION_LEVELS = { 'bg' => 0, 'cs_CZ' => 0, - 'da_DK' => 36, - 'de' => 17, + 'da_DK' => 35, + 'de' => 16, 'en' => 100, 'eo' => 0, - 'es' => 35, + 'es' => 34, 'fil_PH' => 0, - 'fr' => 94, + 'fr' => 98, 'gl_ES' => 0, 'id_ID' => 0, 'it' => 1, - 'ja' => 30, + 'ja' => 29, 'ko' => 20, 'nb_NO' => 24, 'nl_NL' => 0, 'pl_PL' => 3, 'pt_BR' => 57, - 'ro_RO' => 96, + 'ro_RO' => 94, 'ru' => 26, 'si_LK' => 11, - 'tr_TR' => 11, - 'uk' => 52, - 'zh_CN' => 97, + 'tr_TR' => 10, + 'uk' => 54, + 'zh_CN' => 98, 'zh_HK' => 1, 'zh_TW' => 99 }.freeze diff --git a/lib/gitlab/import_export/base/relation_object_saver.rb b/lib/gitlab/import_export/base/relation_object_saver.rb index ed3858d0bf4..77b85fc9f15 100644 --- a/lib/gitlab/import_export/base/relation_object_saver.rb +++ b/lib/gitlab/import_export/base/relation_object_saver.rb @@ -71,6 +71,8 @@ module Gitlab invalid_subrelations << invalid_record unless invalid_record.persisted? end + + relation_object.save end end end diff --git a/lib/gitlab/import_export/error.rb b/lib/gitlab/import_export/error.rb index af0026b8864..fa179f584eb 100644 --- a/lib/gitlab/import_export/error.rb +++ b/lib/gitlab/import_export/error.rb @@ -17,6 +17,10 @@ module Gitlab def self.file_compression_error self.new('File compression/decompression failed') end + + def self.incompatible_import_file_error + self.new('The import file is incompatible') + end end end end diff --git a/lib/gitlab/import_export/group/legacy_tree_restorer.rb b/lib/gitlab/import_export/group/legacy_tree_restorer.rb deleted file mode 100644 index fa9e765b33a..00000000000 --- a/lib/gitlab/import_export/group/legacy_tree_restorer.rb +++ /dev/null @@ -1,132 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - module Group - class LegacyTreeRestorer - include Gitlab::Utils::StrongMemoize - - attr_reader :user - attr_reader :shared - attr_reader :group - - def initialize(user:, shared:, group:, group_hash:) - @user = user - @shared = shared - @group = group - @group_hash = group_hash - end - - def restore - @group_attributes = relation_reader.consume_attributes(nil) - @group_members = relation_reader.consume_relation(nil, 'members') - .map(&:first) - - # We need to remove `name` and `path` as we did consume it in previous pass - @group_attributes.delete('name') - @group_attributes.delete('path') - - @children = @group_attributes.delete('children') - - if members_mapper.map && restorer.restore - @children&.each do |group_hash| - group = create_group(group_hash: group_hash, parent_group: @group) - shared = Gitlab::ImportExport::Shared.new(group) - - self.class.new( - user: @user, - shared: shared, - group: group, - group_hash: group_hash - ).restore - end - end - - return false if @shared.errors.any? - - true - rescue StandardError => e - @shared.error(e) - false - end - - private - - def relation_reader - strong_memoize(:relation_reader) do - if @group_hash.present? - ImportExport::Json::LegacyReader::Hash.new( - @group_hash, - relation_names: reader.group_relation_names) - else - ImportExport::Json::LegacyReader::File.new( - File.join(shared.export_path, 'group.json'), - relation_names: reader.group_relation_names) - end - end - end - - def restorer - @relation_tree_restorer ||= RelationTreeRestorer.new( - user: @user, - shared: @shared, - relation_reader: relation_reader, - members_mapper: members_mapper, - object_builder: object_builder, - relation_factory: relation_factory, - reader: reader, - importable: @group, - importable_attributes: @group_attributes, - importable_path: nil - ) - end - - def create_group(group_hash:, parent_group:) - group_params = { - name: group_hash['name'], - path: group_hash['path'], - parent_id: parent_group&.id, - visibility_level: sub_group_visibility_level(group_hash, parent_group) - } - - ::Groups::CreateService.new(@user, group_params).execute - end - - def sub_group_visibility_level(group_hash, parent_group) - original_visibility_level = group_hash['visibility_level'] || Gitlab::VisibilityLevel::PRIVATE - - if parent_group && parent_group.visibility_level < original_visibility_level - Gitlab::VisibilityLevel.closest_allowed_level(parent_group.visibility_level) - else - original_visibility_level - end - end - - def members_mapper - @members_mapper ||= Gitlab::ImportExport::MembersMapper.new( - exported_members: @group_members, - user: @user, - importable: @group - ) - end - - def relation_factory - Gitlab::ImportExport::Group::RelationFactory - end - - def object_builder - Gitlab::ImportExport::Group::ObjectBuilder - end - - def reader - @reader ||= Gitlab::ImportExport::Reader.new( - shared: @shared, - config: Gitlab::ImportExport::Config.new( - config: Gitlab::ImportExport.legacy_group_config_file - ).to_h - ) - end - end - end - end -end diff --git a/lib/gitlab/import_export/group/legacy_tree_saver.rb b/lib/gitlab/import_export/group/legacy_tree_saver.rb deleted file mode 100644 index 0f74fabeac3..00000000000 --- a/lib/gitlab/import_export/group/legacy_tree_saver.rb +++ /dev/null @@ -1,57 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - module Group - class LegacyTreeSaver - attr_reader :full_path, :shared - - def initialize(group:, current_user:, shared:, params: {}) - @params = params - @current_user = current_user - @shared = shared - @group = group - @full_path = File.join(@shared.export_path, ImportExport.group_filename) - end - - def save - group_tree = serialize(@group, reader.group_tree) - tree_saver.save(group_tree, @shared.export_path, ImportExport.group_filename) - - true - rescue StandardError => e - @shared.error(e) - false - end - - private - - def serialize(group, relations_tree) - group_tree = tree_saver.serialize(group, relations_tree) - - group.children.each do |child| - group_tree['children'] ||= [] - group_tree['children'] << serialize(child, relations_tree) - end - - group_tree - rescue StandardError => e - @shared.error(e) - end - - def reader - @reader ||= Gitlab::ImportExport::Reader.new( - shared: @shared, - config: Gitlab::ImportExport::Config.new( - config: Gitlab::ImportExport.legacy_group_config_file - ).to_h - ) - end - - def tree_saver - @tree_saver ||= LegacyRelationTreeSaver.new - end - end - end - end -end diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index cc69ed55744..99364996864 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -137,6 +137,7 @@ included_attributes: ci_cd_settings: - :group_runners_enabled - :runner_token_expiration_interval + - :default_git_depth metrics_setting: - :dashboard_timezone - :external_dashboard_url @@ -719,7 +720,6 @@ included_attributes: - :feature_flags_access_level - :releases_access_level - :infrastructure_access_level - - :allow_merge_on_skipped_pipeline - :auto_devops_deploy_strategy - :auto_devops_enabled - :container_registry_enabled @@ -728,13 +728,14 @@ included_attributes: - :merge_method - :merge_requests_enabled - :snippets_enabled - - :squash_option - :topics - :visibility - :wiki_enabled - :build_git_strategy - :build_enabled - :security_and_compliance_enabled + - :allow_merge_on_skipped_pipeline + - :squash_option resource_milestone_events: - :user_id - :action @@ -776,6 +777,7 @@ excluded_attributes: - :wiki_page_hooks_integrations - :deployment_hooks_integrations - :alert_hooks_integrations + - :incident_hooks_integrations - :mirror - :runners_token - :runners_token_encrypted @@ -1071,6 +1073,9 @@ excluded_attributes: - :sequence methods: + project: + - :allow_merge_on_skipped_pipeline + - :squash_option notes: - :type labels: @@ -1179,6 +1184,7 @@ ee: - :reject_unsigned_commits - :commit_committer_check - :regexp_uses_re2 + - :reject_non_dco_commits unprotect_access_levels: - :access_level - :user_id diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb index 5ec9db00d0a..ad071a4cbd7 100644 --- a/lib/gitlab/import_export/version_checker.rb +++ b/lib/gitlab/import_export/version_checker.rb @@ -34,7 +34,7 @@ module Gitlab end def different_version?(version) - Gem::Version.new(version) != Gem::Version.new(Gitlab::ImportExport.version) + Gitlab::VersionInfo.parse(version) != Gitlab::VersionInfo.parse(Gitlab::ImportExport.version) rescue StandardError => e Gitlab::Import::Logger.error( message: 'Import error', diff --git a/lib/gitlab/memory/reporter.rb b/lib/gitlab/memory/reporter.rb index 710c89c6216..5effafc9f5b 100644 --- a/lib/gitlab/memory/reporter.rb +++ b/lib/gitlab/memory/reporter.rb @@ -3,6 +3,8 @@ module Gitlab module Memory class Reporter + COMPRESS_CMD = %w[gzip --fast].freeze + attr_reader :reports_path def initialize(reports_path: nil, logger: Gitlab::AppLogger) @@ -67,29 +69,39 @@ module Gitlab report_file = file_name(report) tmp_file_path = File.join(tmp_dir, report_file) + write_heap_dump_file(report, tmp_file_path) + + File.join(@reports_path, report_file).tap do |report_file_path| + FileUtils.mv(tmp_file_path, report_file_path) + end + end + + def write_heap_dump_file(report, path) io_r, io_w = IO.pipe + err_r, err_w = IO.pipe pid = nil - File.open(tmp_file_path, 'wb') do |file| + status = nil + File.open(path, 'wb') do |file| extras = { in: io_r, out: file, - err: $stderr + err: err_w } - pid = Process.spawn('gzip', '--fast', **extras) + pid = Process.spawn(*COMPRESS_CMD, **extras) io_r.close + err_w.close report.run(io_w) io_w.close - Process.waitpid(pid) + _, status = Process.wait2(pid) end - File.join(@reports_path, report_file).tap do |report_file_path| - FileUtils.mv(tmp_file_path, report_file_path) - end + errors = err_r.read&.strip + err_r.close + raise StandardError, "exit #{status.exitstatus}: #{errors}" if !status&.success? && errors.present? ensure - [io_r, io_w].each(&:close) - + [io_r, io_w, err_r, err_w].each(&:close) # Make sure we don't leave any running processes behind. Gitlab::ProcessManagement.signal(pid, :KILL) if pid end diff --git a/lib/gitlab/memory/watchdog.rb b/lib/gitlab/memory/watchdog.rb index aac70a2f6aa..c94dbed1d46 100644 --- a/lib/gitlab/memory/watchdog.rb +++ b/lib/gitlab/memory/watchdog.rb @@ -68,12 +68,11 @@ module Gitlab monitor end - event_reporter.stopped(log_labels(memwd_reason: @reason).compact) + event_reporter.stopped(log_labels(memwd_reason: @stop_reason).compact) end - def stop(reason: nil) - @reason = reason - @alive = false + def stop + stop_working(reason: 'background task stopped') end private @@ -84,7 +83,7 @@ module Gitlab def monitor if monitors.empty? - stop(reason: 'monitors are not configured') + stop_working(reason: 'monitors are not configured') return end @@ -106,7 +105,7 @@ module Gitlab Gitlab::Memory::Reports::HeapDump.enqueue! - stop(reason: 'successfully handled') if handler.call + stop_working(reason: 'successfully handled') if handler.call end def handler @@ -123,6 +122,13 @@ module Gitlab memwd_sleep_time_s: sleep_time_seconds ) end + + def stop_working(reason:) + return unless @alive + + @stop_reason = reason + @alive = false + end end end end diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb index 0172de8731d..cfdac5264e0 100644 --- a/lib/gitlab/metrics/requests_rack_middleware.rb +++ b/lib/gitlab/metrics/requests_rack_middleware.rb @@ -23,7 +23,7 @@ module Gitlab # with an explosion in unused metric combinations, but we want the # most common ones to be always present. FEATURE_CATEGORIES_TO_INITIALIZE = ['authentication_and_authorization', - 'code_review', 'continuous_integration', + 'code_review_workflow', 'continuous_integration', 'not_owned', 'source_code_management', FEATURE_CATEGORY_DEFAULT].freeze diff --git a/lib/gitlab/net_http_adapter.rb b/lib/gitlab/net_http_adapter.rb index 2f7557f2bc3..17eb07fff2b 100644 --- a/lib/gitlab/net_http_adapter.rb +++ b/lib/gitlab/net_http_adapter.rb @@ -6,7 +6,7 @@ module Gitlab # Net::HTTP#request usually calls Net::HTTP#connect but the Webmock overwrite doesn't. # This makes sure that, in a test environment, the superclass is the Webmock overwrite. parent_class = if defined?(WebMock) && Rails.env.test? - WebMock::HttpLibAdapters::NetHttpAdapter.instance_variable_get('@webMockNetHTTP') + WebMock::HttpLibAdapters::NetHttpAdapter.instance_variable_get(:@webMockNetHTTP) else Net::HTTP end diff --git a/lib/gitlab/observability.rb b/lib/gitlab/observability.rb index 8dde60a73be..8dbd2f41ccb 100644 --- a/lib/gitlab/observability.rb +++ b/lib/gitlab/observability.rb @@ -11,5 +11,9 @@ module Gitlab 'https://observe.gitlab.com' end + + def observability_enabled?(user, group) + Gitlab::Observability.observability_url.present? && Ability.allowed?(user, :read_observability, group) + end end end diff --git a/lib/gitlab/pages/cache_control.rb b/lib/gitlab/pages/cache_control.rb index be39e52b342..a24d958b7e5 100644 --- a/lib/gitlab/pages/cache_control.rb +++ b/lib/gitlab/pages/cache_control.rb @@ -16,8 +16,8 @@ module Gitlab PAYLOAD_CACHE_KEY = '%{settings_cache_key}_%{settings_hash}' class << self - def for_project(project_id) - new(type: :project, id: project_id) + def for_domain(domain_id) + new(type: :domain, id: domain_id) end def for_namespace(namespace_id) @@ -26,7 +26,7 @@ module Gitlab end def initialize(type:, id:) - raise(ArgumentError, "type must be :namespace or :project") unless %i[namespace project].include?(type) + raise(ArgumentError, "type must be :namespace or :domain") unless %i[namespace domain].include?(type) @type = type @id = id @@ -50,7 +50,9 @@ module Gitlab .map { |hash| payload_cache_key_for(hash) } .push(settings_cache_key) - Rails.cache.delete_multi(keys) + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + Rails.cache.delete_multi(keys) + end end private diff --git a/lib/gitlab/pagination/cursor_based_keyset.rb b/lib/gitlab/pagination/cursor_based_keyset.rb index 199ec16d4df..a21d0228082 100644 --- a/lib/gitlab/pagination/cursor_based_keyset.rb +++ b/lib/gitlab/pagination/cursor_based_keyset.rb @@ -5,7 +5,8 @@ module Gitlab module CursorBasedKeyset SUPPORTED_ORDERING = { Group => { name: :asc }, - AuditEvent => { id: :desc } + AuditEvent => { id: :desc }, + ::Ci::Build => { id: :desc } }.freeze # Relation types that are enforced in this list diff --git a/lib/gitlab/pagination/keyset/cursor_pager.rb b/lib/gitlab/pagination/keyset/cursor_pager.rb index 0b49aa87a02..d8fa94091ea 100644 --- a/lib/gitlab/pagination/keyset/cursor_pager.rb +++ b/lib/gitlab/pagination/keyset/cursor_pager.rb @@ -10,7 +10,7 @@ module Gitlab @cursor_based_request_context = cursor_based_request_context end - def paginate(relation) + def paginate(relation, _params = {}) @paginator ||= relation.keyset_paginate( per_page: cursor_based_request_context.per_page, cursor: cursor_based_request_context.cursor diff --git a/lib/gitlab/pagination/keyset/pager.rb b/lib/gitlab/pagination/keyset/pager.rb index 6a2ae20f3b8..3fabd454ee3 100644 --- a/lib/gitlab/pagination/keyset/pager.rb +++ b/lib/gitlab/pagination/keyset/pager.rb @@ -10,7 +10,7 @@ module Gitlab @request = request end - def paginate(relation) + def paginate(relation, _params = {}) # Validate assumption: The last two columns must match the page order_by validate_order!(relation) diff --git a/lib/gitlab/pagination/keyset/simple_order_builder.rb b/lib/gitlab/pagination/keyset/simple_order_builder.rb index 318720c77d1..cbd523389d6 100644 --- a/lib/gitlab/pagination/keyset/simple_order_builder.rb +++ b/lib/gitlab/pagination/keyset/simple_order_builder.rb @@ -11,8 +11,6 @@ module Gitlab # [transformed_scope, true] # true indicates that the new scope was successfully built # [orginal_scope, false] # false indicates that the order values are not supported in this class class SimpleOrderBuilder - NULLS_ORDER_REGEX = /(?<column_name>.*) (?<direction>\bASC\b|\bDESC\b) (?<nullable>\bNULLS LAST\b|\bNULLS FIRST\b)/.freeze - def self.build(scope) new(scope: scope).build end @@ -90,32 +88,6 @@ module Gitlab end end - # This method converts the first order value to a corresponding arel expression - # if the order value uses either NULLS LAST or NULLS FIRST ordering in raw SQL. - # - # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/356644 - # We should stop matching raw literals once we switch to using the Arel methods. - def convert_raw_nulls_order! - order_value = order_values.first - - return unless order_value.is_a?(Arel::Nodes::SqlLiteral) - - # Detect NULLS LAST or NULLS FIRST ordering by looking at the raw SQL string. - if matches = order_value.match(NULLS_ORDER_REGEX) - return unless table_column?(matches[:column_name]) - - column_attribute = arel_table[matches[:column_name]] - direction = matches[:direction].downcase.to_sym - nullable = matches[:nullable].downcase.parameterize(separator: '_').to_sym - - # Build an arel order expression for NULLS ordering. - order = direction == :desc ? column_attribute.desc : column_attribute.asc - arel_order_expression = nullable == :nulls_first ? order.nulls_first : order.nulls_last - - order_values[0] = arel_order_expression - end - end - def nullability(order_value, attribute_name) nullable = model_class.columns.find { |column| column.name == attribute_name }.null @@ -206,16 +178,12 @@ module Gitlab def ordered_by_other_column? return unless order_values.one? - convert_raw_nulls_order! - supported_column?(order_values.first) end def ordered_by_other_column_with_tie_breaker? return unless order_values.size == 2 - convert_raw_nulls_order! - return unless supported_column?(order_values.first) tie_breaker_attribute = order_values.second.try(:expr) diff --git a/lib/gitlab/phabricator_import.rb b/lib/gitlab/phabricator_import.rb index 3885a9934d5..49e01eceb5b 100644 --- a/lib/gitlab/phabricator_import.rb +++ b/lib/gitlab/phabricator_import.rb @@ -5,8 +5,7 @@ module Gitlab BaseError = Class.new(StandardError) def self.available? - Feature.enabled?(:phabricator_import) && - Gitlab::CurrentSettings.import_sources.include?('phabricator') + Gitlab::CurrentSettings.import_sources.include?('phabricator') end end end diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb index 9bc0001be81..5394cd115b1 100644 --- a/lib/gitlab/project_template.rb +++ b/lib/gitlab/project_template.rb @@ -81,7 +81,7 @@ module Gitlab ProjectTemplate.new('jsonnet', 'Jsonnet for Dynamic Child Pipelines', _('An example showing how to use Jsonnet with GitLab dynamic child pipelines'), 'https://gitlab.com/gitlab-org/project-templates/jsonnet'), ProjectTemplate.new('cluster_management', 'GitLab Cluster Management', _('An example project for managing Kubernetes clusters integrated with GitLab'), 'https://gitlab.com/gitlab-org/project-templates/cluster-management'), ProjectTemplate.new('kotlin_native_linux', 'Kotlin Native Linux', _('A basic template for developing Linux programs using Kotlin Native'), 'https://gitlab.com/gitlab-org/project-templates/kotlin-native-linux'), - ProjectTemplate.new('typo3_distribution', 'TYPO3 Distribution', _('A template for starting a new TYPO3 project'), 'https://gitlab.com/ochorocho/typo3-distribution', 'illustrations/logos/typo3.svg') + ProjectTemplate.new('typo3_distribution', 'TYPO3 Distribution', _('A template for starting a new TYPO3 project'), 'https://gitlab.com/gitlab-org/project-templates/typo3-distribution', 'illustrations/logos/typo3.svg') ].freeze end # rubocop:enable Metrics/AbcSize diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb index 14e9e66e037..f782f2802b6 100644 --- a/lib/gitlab/quick_actions/issue_actions.rb +++ b/lib/gitlab/quick_actions/issue_actions.rb @@ -252,23 +252,14 @@ module Gitlab desc { _('Promote issue to incident') } explanation { _('Promotes issue to incident') } + execution_message { _('Issue has been promoted to incident') } types Issue condition do - quick_action_target.persisted? && - !quick_action_target.incident? && - current_user.can?(:update_issue, quick_action_target) + !quick_action_target.incident? && + current_user.can?(:"set_#{quick_action_target.issue_type}_metadata", quick_action_target) end command :promote_to_incident do - issue = ::Issues::UpdateService - .new(project: quick_action_target.project, current_user: current_user, params: { issue_type: 'incident' }) - .execute(quick_action_target) - - @execution_message[:promote_to_incident] = - if issue.incident? - _('Issue has been promoted to incident') - else - _('Failed to promote issue to incident') - end + @updates[:issue_type] = "incident" end desc { _('Add customer relation contacts') } diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb index 8857b544364..ed4f6015603 100644 --- a/lib/gitlab/redis.rb +++ b/lib/gitlab/redis.rb @@ -11,6 +11,7 @@ module Gitlab Gitlab::Redis::Cache, Gitlab::Redis::Queues, Gitlab::Redis::RateLimiting, + Gitlab::Redis::RepositoryCache, Gitlab::Redis::Sessions, Gitlab::Redis::SharedState, Gitlab::Redis::TraceChunks diff --git a/lib/gitlab/redis/multi_store.rb b/lib/gitlab/redis/multi_store.rb index 4f58bee49d0..aa8f390ac10 100644 --- a/lib/gitlab/redis/multi_store.rb +++ b/lib/gitlab/redis/multi_store.rb @@ -26,7 +26,7 @@ module Gitlab class MethodMissingError < StandardError def message - 'Method missing. Falling back to execute method on the redis secondary store.' + 'Method missing. Falling back to execute method on the redis default store in Rails.env.production.' end end @@ -36,31 +36,64 @@ module Gitlab FAILED_TO_WRITE_ERROR_MESSAGE = 'Failed to write to the redis primary_store.' FAILED_TO_RUN_PIPELINE = 'Failed to execute pipeline on the redis primary_store.' - SKIP_LOG_METHOD_MISSING_FOR_COMMANDS = %i(info).freeze + SKIP_LOG_METHOD_MISSING_FOR_COMMANDS = %i[info].freeze - READ_COMMANDS = %i( - get - mget - smembers - scard - ).freeze - - WRITE_COMMANDS = %i( - set - setnx - setex - sadd - srem + # For ENUMERATOR_CACHE_HIT_VALIDATOR and READ_CACHE_HIT_VALIDATOR, + # we define procs to validate cache hit. The only other acceptable value is nil, + # in the case of errors being raised. + # + # If a command has no empty response, set ->(val) { true } + # + # Ref: https://www.rubydoc.info/github/redis/redis-rb/Redis/Commands + # + ENUMERATOR_CACHE_HIT_VALIDATOR = { + scan_each: ->(val) { val.is_a?(Enumerator) && !val.first.nil? }, + hscan_each: ->(val) { val.is_a?(Enumerator) && !val.first.nil? }, + sscan_each: ->(val) { val.is_a?(Enumerator) && !val.first.nil? }, + zscan_each: ->(val) { val.is_a?(Enumerator) && !val.first.nil? } + }.freeze + + READ_CACHE_HIT_VALIDATOR = { + exists: ->(val) { val != 0 }, + exists?: ->(val) { val }, + get: ->(val) { !val.nil? }, + hexists: ->(val) { val }, + hget: ->(val) { !val.nil? }, + hgetall: ->(val) { val.is_a?(Hash) && !val.empty? }, + hlen: ->(val) { val != 0 }, + hmget: ->(val) { val.is_a?(Array) && !val.compact.empty? }, + mapped_hmget: ->(val) { val.is_a?(Hash) && !val.compact.empty? }, + mget: ->(val) { val.is_a?(Array) && !val.compact.empty? }, + scard: ->(val) { val != 0 }, + sismember: ->(val) { val }, + smembers: ->(val) { val.is_a?(Array) && !val.empty? }, + sscan: ->(val) { val != ['0', []] }, + ttl: ->(val) { val != 0 && val != -2 } + }.freeze + + WRITE_COMMANDS = %i[ del + eval + expire flushdb + hdel + hset + incr + incrby + mapped_hmset rpush - eval - ).freeze + sadd + set + setex + setnx + srem + unlink + ].freeze - PIPELINED_COMMANDS = %i( + PIPELINED_COMMANDS = %i[ pipelined multi - ).freeze + ].freeze # To transition between two Redis store, `primary_store` should be the target store, # and `secondary_store` should be the current store. Transition is controlled with feature flags: @@ -81,12 +114,12 @@ module Gitlab end # rubocop:disable GitlabSecurity/PublicSend - READ_COMMANDS.each do |name| - define_method(name) do |*args, &block| + READ_CACHE_HIT_VALIDATOR.each_key do |name| + define_method(name) do |*args, **kwargs, &block| if use_primary_and_secondary_stores? - read_command(name, *args, &block) + read_command(name, *args, **kwargs, &block) else - default_store.send(name, *args, &block) + default_store.send(name, *args, **kwargs, &block) end end end @@ -101,6 +134,20 @@ module Gitlab end end + ENUMERATOR_CACHE_HIT_VALIDATOR.each_key do |name| + define_method(name) do |*args, **kwargs, &block| + enumerator = if use_primary_and_secondary_stores? + read_command(name, *args, **kwargs) + else + default_store.send(name, *args, **kwargs) + end + + return enumerator if block.nil? + + enumerator.each(&block) + end + end + PIPELINED_COMMANDS.each do |name| define_method(name) do |*args, **kwargs, &block| if use_primary_and_secondary_stores? @@ -170,12 +217,23 @@ module Gitlab extra.merge(command_name: command_name, instance_name: instance_name)) end + def ping(message = nil) + if use_primary_and_secondary_stores? + # Both stores have to response success for the ping to be considered success. + # We assume both stores cannot return different responses (only both "PONG" or both echo the message). + # If either store is not reachable, an Error will be raised anyway thus taking any response works. + [primary_store, secondary_store].map { |store| store.ping(message) }.first + else + default_store.ping(message) + end + end + private # @return [Boolean] def feature_enabled?(prefix) feature_table_exists? && - Feature.enabled?("#{prefix}_#{instance_name.underscore}") && + Feature.enabled?("#{prefix}_#{instance_name.underscore}") && # rubocop:disable Cop/FeatureFlagUsage !same_redis_store? end @@ -193,15 +251,17 @@ module Gitlab def log_method_missing(command_name, *_args) return if SKIP_LOG_METHOD_MISSING_FOR_COMMANDS.include?(command_name) + raise MethodMissingError if Rails.env.test? || Rails.env.development? + log_error(MethodMissingError.new, command_name) increment_method_missing_count(command_name) end - def read_command(command_name, *args, &block) + def read_command(command_name, *args, **kwargs, &block) if @instance - send_command(@instance, command_name, *args, &block) + send_command(@instance, command_name, *args, **kwargs, &block) else - read_one_with_fallback(command_name, *args, &block) + read_one_with_fallback(command_name, *args, **kwargs, &block) end end @@ -213,19 +273,28 @@ module Gitlab end end - def read_one_with_fallback(command_name, *args, &block) + def read_one_with_fallback(command_name, *args, **kwargs, &block) begin - value = send_command(primary_store, command_name, *args, &block) + value = send_command(primary_store, command_name, *args, **kwargs, &block) rescue StandardError => e log_error(e, command_name, multi_store_error_message: FAILED_TO_READ_ERROR_MESSAGE) end - value || fallback_read(command_name, *args, &block) + return value if cache_hit?(command_name, value) + + fallback_read(command_name, *args, **kwargs, &block) + end + + def cache_hit?(command, value) + validator = READ_CACHE_HIT_VALIDATOR[command] || ENUMERATOR_CACHE_HIT_VALIDATOR[command] + return false unless validator + + !value.nil? && validator.call(value) end - def fallback_read(command_name, *args, &block) - value = send_command(secondary_store, command_name, *args, &block) + def fallback_read(command_name, *args, **kwargs, &block) + value = send_command(secondary_store, command_name, *args, **kwargs, &block) if value log_error(ReadFromPrimaryError.new, command_name) diff --git a/lib/gitlab/redis/repository_cache.rb b/lib/gitlab/redis/repository_cache.rb new file mode 100644 index 00000000000..8bfbfcfea60 --- /dev/null +++ b/lib/gitlab/redis/repository_cache.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Redis + class RepositoryCache < ::Gitlab::Redis::Wrapper + class << self + # The data we store on RepositoryCache used to be stored on Cache. + def config_fallback + Cache + end + + def cache_store + @cache_store ||= ActiveSupport::Cache::RedisCacheStore.new( + redis: pool, + compress: Gitlab::Utils.to_boolean(ENV.fetch('ENABLE_REDIS_CACHE_COMPRESSION', '1')), + namespace: Cache::CACHE_NAMESPACE, + # Cache should not grow forever + expires_in: ENV.fetch('GITLAB_RAILS_CACHE_DEFAULT_TTL_SECONDS', 8.hours).to_i + ) + end + + private + + def redis + primary_store = ::Redis.new(params) + secondary_store = ::Redis.new(config_fallback.params) + + MultiStore.new(primary_store, secondary_store, store_name) + end + end + end + end +end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 0e5389dc995..e5e1e1d4165 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -41,21 +41,6 @@ module Gitlab size end - def _raw_config - return @_raw_config if defined?(@_raw_config) - - @_raw_config = - begin - if filename = config_file_name - ERB.new(File.read(filename)).result.freeze - else - false - end - rescue Errno::ENOENT - false - end - end - def config_file_path(filename) path = File.join(rails_root, 'config', filename) return path if File.file?(path) @@ -67,10 +52,6 @@ module Gitlab File.expand_path('../../..', __dir__) end - def config_fallback? - config_file_name == config_fallback&.config_file_name - end - def config_file_name [ # Instance specific config sources: @@ -91,6 +72,10 @@ module Gitlab ].compact.first end + def redis_yml_path + File.join(rails_root, 'config/redis.yml') + end + def store_name name.demodulize end @@ -212,16 +197,20 @@ module Gitlab end def fetch_config - return false unless self.class._raw_config - - yaml = YAML.safe_load(self.class._raw_config, aliases: true) + redis_yml = read_yaml(self.class.redis_yml_path).fetch(@rails_env, {}) + instance_config_yml = read_yaml(self.class.config_file_name)[@rails_env] + + [ + redis_yml[self.class.store_name.underscore], + instance_config_yml, + self.class.config_fallback && redis_yml[self.class.config_fallback.store_name.underscore] + ].compact.first + end - # If the file has content but it's invalid YAML, `load` returns false - if yaml - yaml.fetch(@rails_env, false) - else - false - end + def read_yaml(path) + YAML.safe_load(ERB.new(File.read(path.to_s)).result, aliases: true) || {} + rescue Errno::ENOENT + {} end end end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 4f76cce2c7d..828cf65fb82 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -251,6 +251,26 @@ module Gitlab extend self extend Packages + def bulk_import_namespace_path_regex + # This regexp validates the string conforms to rules for a namespace path: + # i.e does not start with a non-alphanueric character except for periods or underscores, + # contains only alphanumeric characters, forward slashes, periods, and underscores, + # does not end with a period or forward slash, and has a relative path structure + # with no http protocol chars or leading or trailing forward slashes + # eg 'source/full/path' or 'destination_namespace' not 'https://example.com/source/full/path' + @bulk_import_namespace_path_regex ||= %r/^([.]?)[^\W](\/?[.]?[0-9a-z][-_]*)+$/i + end + + def group_path_regex + # This regexp validates the string conforms to rules for a group slug: + # i.e does not start with a non-alphanueric character except for periods or underscores, + # contains only alphanumeric characters, periods, and underscores, + # does not end with a period or forward slash, and has a relative path structure + # with no http protocol chars or leading or trailing forward slashes + # eg 'source/full/path' or 'destination_namespace' not 'https://example.com/source/full/path' + @group_path_regex ||= %r/^[.]?[^\W]([.]?[0-9a-z][-_]*)+$/i + end + def project_name_regex # The character range \p{Alnum} overlaps with \u{00A9}-\u{1f9ff} # hence the Ruby warning. diff --git a/lib/gitlab/repository_cache.rb b/lib/gitlab/repository_cache.rb index dc8b2467f72..8de2c2fe772 100644 --- a/lib/gitlab/repository_cache.rb +++ b/lib/gitlab/repository_cache.rb @@ -5,7 +5,7 @@ module Gitlab class RepositoryCache attr_reader :repository, :namespace, :backend - def initialize(repository, extra_namespace: nil, backend: Rails.cache) + def initialize(repository, extra_namespace: nil, backend: self.class.store) @repository = repository @namespace = "#{repository.full_path}" @namespace += ":#{repository.project.id}" if repository.project @@ -48,5 +48,14 @@ module Gitlab value end + + def self.store + if Feature.enabled?(:use_primary_and_secondary_stores_for_repository_cache) || + Feature.enabled?(:use_primary_store_as_default_for_repository_cache) + Gitlab::Redis::RepositoryCache.cache_store + else + Rails.cache + end + end end end diff --git a/lib/gitlab/repository_hash_cache.rb b/lib/gitlab/repository_hash_cache.rb index 1ecdf506208..ea90a341b1e 100644 --- a/lib/gitlab/repository_hash_cache.rb +++ b/lib/gitlab/repository_hash_cache.rb @@ -139,8 +139,17 @@ module Gitlab private + def cache + if Feature.enabled?(:use_primary_and_secondary_stores_for_repository_cache) || + Feature.enabled?(:use_primary_store_as_default_for_repository_cache) + Gitlab::Redis::RepositoryCache + else + Gitlab::Redis::Cache + end + end + def with(&blk) - Gitlab::Redis::Cache.with(&blk) # rubocop:disable CodeReuse/ActiveRecord + cache.with(&blk) # rubocop:disable CodeReuse/ActiveRecord end # Take a hash and convert both keys and values to strings, for insertion into Redis. diff --git a/lib/gitlab/repository_set_cache.rb b/lib/gitlab/repository_set_cache.rb index baf48fd0dc1..c67ca92af40 100644 --- a/lib/gitlab/repository_set_cache.rb +++ b/lib/gitlab/repository_set_cache.rb @@ -64,5 +64,20 @@ module Gitlab redis.sscan_each(full_key, match: pattern) end end + + private + + def cache + if Feature.enabled?(:use_primary_and_secondary_stores_for_repository_cache) || + Feature.enabled?(:use_primary_store_as_default_for_repository_cache) + Gitlab::Redis::RepositoryCache + else + Gitlab::Redis::Cache + end + end + + def with(&blk) + cache.with(&blk) # rubocop:disable CodeReuse/ActiveRecord + end end end diff --git a/lib/gitlab/seeders/ci/runner/runner_fleet_pipeline_seeder.rb b/lib/gitlab/seeders/ci/runner/runner_fleet_pipeline_seeder.rb new file mode 100644 index 00000000000..c77db02061c --- /dev/null +++ b/lib/gitlab/seeders/ci/runner/runner_fleet_pipeline_seeder.rb @@ -0,0 +1,202 @@ +# frozen_string_literal: true + +module Gitlab + module Seeders + module Ci + module Runner + class RunnerFleetPipelineSeeder + DEFAULT_JOB_COUNT = 400 + + MAX_QUEUE_TIME_IN_SECONDS = 5.minutes.to_i + PIPELINE_CREATION_RANGE_MIN_IN_SECONDS = 2.hours.to_i + PIPELINE_CREATION_RANGE_MAX_IN_SECONDS = 30.days.to_i + PIPELINE_START_RANGE_MAX_IN_SECONDS = 5.minutes.to_i + PIPELINE_FINISH_RANGE_MAX_IN_SECONDS = 1.hour.to_i + + PROJECT_JOB_DISTRIBUTION = [ + { allocation: 70, job_count_default: 10 }, + { allocation: 15, job_count_default: 10 }, + { allocation: 15, job_count_default: 100 } + # remaining jobs on 4th project + ].freeze + + attr_reader :logger + + # Initializes the class + # + # @param [Gitlab::Logger] logger + # @param [Integer] job_count the number of jobs to create across the runners + # @param [Array<Hash>] projects_to_runners list of project IDs to respective runner IDs + def initialize(logger = Gitlab::AppLogger, projects_to_runners:, job_count:) + @logger = logger + @projects_to_runners = projects_to_runners.map do |v| + { project_id: v[:project_id], runners: ::Ci::Runner.id_in(v[:runner_ids]).to_a } + end + @job_count = job_count || DEFAULT_JOB_COUNT + end + + def seed + logger.info(message: 'Starting seed of runner fleet pipelines', job_count: @job_count) + + remaining_job_count = @job_count + PROJECT_JOB_DISTRIBUTION.each_with_index do |d, index| + remaining_job_count = create_pipelines_and_distribute_jobs(remaining_job_count, project_index: index, **d) + end + + while remaining_job_count > 0 + remaining_job_count -= create_pipeline( + job_count: remaining_job_count, + **@projects_to_runners[PROJECT_JOB_DISTRIBUTION.length], + status: random_pipeline_status + ) + end + + logger.info( + message: 'Completed seeding of runner fleet', + job_count: @job_count - remaining_job_count + ) + + nil + end + + private + + def create_pipelines_and_distribute_jobs(remaining_job_count, project_index:, allocation:, job_count_default:) + max_jobs_per_pipeline = [1, @job_count / 3].max + + create_pipelines( + remaining_job_count, + **@projects_to_runners[project_index], + total_jobs: @job_count * allocation / 100, + pipeline_job_count: job_count_default.clamp(1, max_jobs_per_pipeline) + ) + end + + def create_pipelines(remaining_job_count, project_id:, runners:, total_jobs:, pipeline_job_count:) + pipeline_job_count = remaining_job_count if pipeline_job_count > remaining_job_count + return 0 if pipeline_job_count == 0 + + pipeline_count = [1, total_jobs / pipeline_job_count].max + + (1..pipeline_count).each do + remaining_job_count -= create_pipeline( + job_count: pipeline_job_count, + project_id: project_id, + runners: runners, + status: random_pipeline_status + ) + end + + remaining_job_count + end + + def create_pipeline(job_count:, runners:, project_id:, status: 'success', **attrs) + logger.info(message: 'Creating pipeline with builds on project', + status: status, job_count: job_count, project_id: project_id, **attrs) + + raise ArgumentError('runners') unless runners + raise ArgumentError('project_id') unless project_id + + sha = '00000000' + if ::Ci::HasStatus::ALIVE_STATUSES.include?(status) || ::Ci::HasStatus::COMPLETED_STATUSES.include?(status) + created_at = Random.rand(PIPELINE_CREATION_RANGE_MIN_IN_SECONDS..PIPELINE_CREATION_RANGE_MAX_IN_SECONDS) + .seconds.ago + + if ::Ci::HasStatus::STARTED_STATUSES.include?(status) || + ::Ci::HasStatus::COMPLETED_STATUSES.include?(status) + started_at = created_at + Random.rand(1..PIPELINE_START_RANGE_MAX_IN_SECONDS) + if ::Ci::HasStatus::COMPLETED_STATUSES.include?(status) + finished_at = started_at + Random.rand(1..PIPELINE_FINISH_RANGE_MAX_IN_SECONDS) + end + end + end + + pipeline = ::Ci::Pipeline.new( + project_id: project_id, + ref: 'main', + sha: sha, + source: 'api', + status: status, + created_at: created_at, + started_at: started_at, + finished_at: finished_at, + **attrs + ) + pipeline.ensure_project_iid! # allocate an internal_id outside of pipeline creation transaction + pipeline.save! + + if created_at.present? + (1..job_count).each do |index| + create_build(pipeline, runners.sample, job_status(pipeline.status, index, job_count), index) + end + end + + job_count + end + + def create_build(pipeline, runner, job_status, index) + started_at = pipeline.started_at + finished_at = pipeline.finished_at + + max_job_duration = [MAX_QUEUE_TIME_IN_SECONDS, 5, 2].sample + max_job_duration = (finished_at - started_at) if finished_at && max_job_duration > finished_at - started_at + + job_created_at = pipeline.created_at + job_started_at = job_created_at + Random.rand(1..max_job_duration) if started_at + if finished_at + job_finished_at = Random.rand(job_started_at..finished_at) + elsif job_status == 'running' + job_finished_at = job_started_at + Random.rand(1 * 60..PIPELINE_FINISH_RANGE_MAX_IN_SECONDS) + end + + # Do not use the first 2 runner tags ('runner-fleet', "#{registration_prefix}runner"). + # See Gitlab::Seeders::Ci::Runner::RunnerFleetSeeder#additional_runner_args + tags = runner.tags.offset(2).sample(Random.rand(1..5)) # rubocop: disable CodeReuse/ActiveRecord + + build_attrs = { + name: "Fake job #{index}", + scheduling_type: 'dag', + ref: 'main', + status: job_status, + pipeline_id: pipeline.id, + runner_id: runner.id, + project_id: pipeline.project_id, + tag_list: tags, + created_at: job_created_at, + queued_at: job_created_at, + started_at: job_started_at, + finished_at: job_finished_at + } + logger.info(message: 'Creating build', **build_attrs) + + ::Ci::Build.new(importing: true, **build_attrs).tap(&:save!) + end + + def random_pipeline_status + if Random.rand(1..4) == 4 + %w[created pending canceled running].sample + elsif Random.rand(1..3) == 1 + 'success' + else + 'failed' + end + end + + def job_status(pipeline_status, job_index, job_count) + return pipeline_status if %w[created pending success].include?(pipeline_status) + + # Ensure that a failed/canceled pipeline has at least 1 failed/canceled job + if job_index == job_count && ::Ci::HasStatus::PASSED_WITH_WARNINGS_STATUSES.include?(pipeline_status) + return pipeline_status + end + + possible_statuses = %w[failed success] + possible_statuses << pipeline_status if %w[canceled running].include?(pipeline_status) + + possible_statuses.sample + end + end + end + end + end +end diff --git a/lib/gitlab/seeders/ci/runner/runner_fleet_seeder.rb b/lib/gitlab/seeders/ci/runner/runner_fleet_seeder.rb new file mode 100644 index 00000000000..082d267442c --- /dev/null +++ b/lib/gitlab/seeders/ci/runner/runner_fleet_seeder.rb @@ -0,0 +1,246 @@ +# frozen_string_literal: true + +module Gitlab + module Seeders + module Ci + module Runner + class RunnerFleetSeeder + DEFAULT_USERNAME = 'root' + DEFAULT_PREFIX = 'rf-' + DEFAULT_RUNNER_COUNT = 40 + DEFAULT_JOB_COUNT = DEFAULT_RUNNER_COUNT * 10 + + TAG_LIST = %w[gitlab-org docker ruby 2gb mysql linux shared shell deploy hhvm windows build postgres ios stage android stz front back review-apps pc java scraper test kubernetes staging no-priority osx php nodejs production nvm x86_64 gcc nginx dev unity odoo node sbt amazon xamarin debian gcloud e2e clang composer npm energiency dind flake8 cordova x64 private aws solution ruby2.2 python xcode kube compute mongo runner docker-compose phpunit t-matix docker-machine win server docker-in-docker redis go dotnet win7 area51-1 testing chefdk light osx_10-11 ubuntu gulp jertis gitlab-runner frontendv2 capifony centos7 mac gradle golang docker-builder runrepeat maven centos6 msvc14 amd64 xcode_8-2 macos VS2015 mono osx_10-12 azure-contend-docker msbuild git deployer local development python2.7 eezeeit release ios_9-3 fastlane selenium integration tests review cabinet-dev vs2015 ios_10-2 latex odoo_test quantum-ci prod sqlite heavy icc html-test labs feature alugha ps appivo-server fast web ios_9-2 c# python3 home js xcode_7-3 drupal 7 arm headless php70 gce x86 msvc builder Windows bower mssql pagetest wpf ssh inmobiliabeta.com xcode_7-2 repo laravel testonly gcp online-auth powershell ila-preprod ios_10-1 lossless sharesies backbone javascript fusonic-review autoscale ci ubuntu1604 rails windows10 xcode_8-1 php56 drupal embedded readyselect xamarin.ios XCode-8.1 iOS-10.1 macOS-10.12.1 develop taggun koumoul-internal docker-build iOS angular2 deployment xcode8 lcov test-cluster priv api bundler freebsd x86-64 BOB xcode_8 nuget vinome-backend cq_check fusonic-perf django php7 dy-manager-shell DEV mongodb neadev meteor ANSIBLE ftp master exerica-build server01 exerica-test mother-of-god nodejs-app ansible Golang mpi exploragen shootr Android macos_10-12 win64 ngsrunner @docker images script-maven ayk makepkg Linux ecolint wix xcode_8-0 coverage dreamhost multi ubuntu1404 eyeka jow3an-site repository politibot qt haskellstack arch priviti backend Sisyphus gm-dev dotNet internal support rpi .net buildbot-01 quay.io BOB2 codebnb vs2013 no-reset live 192.168.100.209 failfast-ci ios_10 crm_master_builds Qt packer selenium hub ci-shell rust dyscount-ci-manager-shell kubespray vagrant deployAutomobileBuild 1md k8s behat vinome-frontend development-nanlabs build-backend libvirt build-frontend contend-server windows-x64 chimpAPI ec2-runner kubectl linux-x64 epitech portals kvm ucaya-docker scala desktop buildmacbinaries ghc buildwinbinaries sonarqube deploySteelDistributorsBuild macOS r cpran rubocop binarylane r-packages alpha SIGAC tester area51-2 customer Build qa acegames_central mTaxNativeShell c++ cloveapp-ios smallville portal root lemmy nightly buildlinuxbinaries rundeck taxonic ios_10-0 n0004 data fedora rr-test seedai_master_builds geofence_master_builds].freeze # rubocop:disable Layout/LineLength + + attr_reader :logger + + # Initializes the class + # + # @param [Gitlab::Logger] logger + # @param [Hash] options + # @option options [String] :username username of the user that will create the fleet + # @option options [String] :registration_prefix string to use as prefix in group, project, and runner names + # @option options [Integer] :runner_count number of runners to create across the groups and projects + # @return [Array<Hash>] list of project IDs to respective runner IDs + def initialize(logger = Gitlab::AppLogger, **options) + username = options[:username] || DEFAULT_USERNAME + + @logger = logger + @user = User.find_by_username(username) + @registration_prefix = options[:registration_prefix] || DEFAULT_PREFIX + @runner_count = options[:runner_count] || DEFAULT_RUNNER_COUNT + @groups = {} + @projects = {} + end + + # seed returns an array of hashes of projects to its assigned runners + def seed + return unless within_plan_limits? + + logger.info( + message: 'Starting seed of runner fleet', + user_id: @user.id, + registration_prefix: @registration_prefix, + runner_count: @runner_count + ) + + groups_and_projects = create_groups_and_projects + runner_ids = create_runners(groups_and_projects) + + logger.info( + message: 'Completed seeding of runner fleet', + registration_prefix: @registration_prefix, + groups: @groups.count, + projects: @projects.count, + runner_count: @runner_count + ) + + %i[project_1_1_1_1 project_1_1_2_1 project_2_1_1].map do |project_key| + { project_id: groups_and_projects[project_key].id, runner_ids: runner_ids[project_key] } + end + end + + private + + def within_plan_limits? + plan_limits = Plan.default.actual_limits + + if plan_limits.ci_registered_group_runners < @runner_count + logger.error('The plan limits for group runners is set to ' \ + "#{plan_limits.ci_registered_group_runners} runners. " \ + 'You should raise the plan limits to avoid errors during runner creation') + return false + elsif plan_limits.ci_registered_project_runners < @runner_count + logger.error('The plan limits for project runners is set to ' \ + "#{plan_limits.ci_registered_project_runners} runners. " \ + 'You should raise the plan limits to avoid errors during runner creation') + return false + end + + true + end + + def create_groups_and_projects + root_group_1 = ensure_group(name: 'top-level group 1') + root_group_2 = ensure_group(name: 'top-level group 2') + group_1_1 = ensure_group(name: 'group 1.1', parent_id: root_group_1.id) + group_1_1_1 = ensure_group(name: 'group 1.1.1', parent_id: group_1_1.id) + group_1_1_2 = ensure_group(name: 'group 1.1.2', parent_id: group_1_1.id) + group_2_1 = ensure_group(name: 'group 2.1', parent_id: root_group_2.id) + + { + root_group_1: root_group_1, + root_group_2: root_group_2, + group_1_1: group_1_1, + group_1_1_1: group_1_1_1, + group_1_1_2: group_1_1_2, + project_1_1_1_1: ensure_project(name: 'project 1.1.1.1', namespace_id: group_1_1_1.id), + project_1_1_2_1: ensure_project(name: 'project 1.1.2.1', namespace_id: group_1_1_2.id), + group_2_1: group_2_1, + project_2_1_1: ensure_project(name: 'project 2.1.1', namespace_id: group_2_1.id) + } + end + + def create_runners(gp) + instance_runners = [] + group_1_1_1_runners = [] + group_2_1_runners = [] + project_1_1_1_1_runners = [] + project_1_1_2_1_runners = [] + project_2_1_1_runners = [] + instance_runners << create_runner(name: 'instance runner 1') + project_1_1_1_1_shared_runner_1 = + create_runner(name: 'project 1.1.1.1 shared runner 1', scope: gp[:project_1_1_1_1]) + project_1_1_1_1_runners << project_1_1_1_1_shared_runner_1 + project_1_1_2_1_runners << assign_runner(project_1_1_1_1_shared_runner_1, gp[:project_1_1_2_1]) + project_2_1_1_runners << assign_runner(project_1_1_1_1_shared_runner_1, gp[:project_2_1_1]) + + (3..@runner_count).each do + case Random.rand(0..100) + when 0..30 + runner_name = "group 1.1.1 runner #{1 + group_1_1_1_runners.count}" + group_1_1_1_runners << create_runner(name: runner_name, scope: gp[:group_1_1_1]) + when 31..50 + runner_name = "project 1.1.1.1 runner #{1 + project_1_1_1_1_runners.count}" + project_1_1_1_1_runners << create_runner(name: runner_name, scope: gp[:project_1_1_1_1]) + when 51..99 + runner_name = "project 1.1.2.1 runner #{1 + project_1_1_2_1_runners.count}" + project_1_1_2_1_runners << create_runner(name: runner_name, scope: gp[:project_1_1_2_1]) + else + runner_name = "group 2.1 runner #{1 + group_2_1_runners.count}" + group_2_1_runners << create_runner(name: runner_name, scope: gp[:group_2_1]) + end + end + + { # use only the first 5 runners to assign CI jobs + project_1_1_1_1: + ((instance_runners + project_1_1_1_1_runners).map(&:id) + group_1_1_1_runners.map(&:id)).first(5), + project_1_1_2_1: (instance_runners + project_1_1_2_1_runners).map(&:id).first(5), + project_2_1_1: + ((instance_runners + project_2_1_1_runners).map(&:id) + group_2_1_runners.map(&:id)).first(5) + } + end + + def ensure_group(name:, parent_id: nil, **args) + args[:description] ||= "Runner fleet #{name}" + name = generate_name(name) + + group = ::Group.by_parent(parent_id).find_by_name(name) + group ||= create_group(name: name, path: name.tr(' ', '-'), parent_id: parent_id, **args) + + register_record(group, @groups) + end + + def generate_name(name) + "#{@registration_prefix}#{name}" + end + + def create_group(**args) + logger.info(message: 'Creating group', **args) + + ensure_success(::Groups::CreateService.new(@user, **args).execute) + end + + def ensure_project(name:, namespace_id:, **args) + args[:description] ||= "Runner fleet #{name}" + name = generate_name(name) + + project = ::Project.in_namespace(namespace_id).find_by_name(name) + project ||= create_project(name: name, namespace_id: namespace_id, **args) + + register_record(project, @projects) + end + + def create_project(**args) + logger.info(message: 'Creating project', **args) + + ensure_success(::Projects::CreateService.new(@user, **args).execute) + end + + def register_record(record, records) + return record if record.errors.any? + + records[record.id] = record + end + + def ensure_success(record) + return record unless record.errors.any? + + logger.error(record.errors.full_messages.to_sentence) + raise RuntimeError + end + + def create_runner(name:, scope: nil, **args) + name = generate_name(name) + + scope_name = scope.class.name if scope + logger.info(message: 'Creating runner', scope: scope_name, name: name) + + executor = ::Ci::Runner::EXECUTOR_NAME_TO_TYPES.keys.sample + args.merge!(additional_runner_args(name, executor)) + + runners_token = if scope.nil? + Gitlab::CurrentSettings.runners_registration_token + else + scope.runners_token + end + + response = ::Ci::Runners::RegisterRunnerService.new.execute(runners_token, name: name, **args) + runner = response.payload[:runner] + + ::Ci::Runners::ProcessRunnerVersionUpdateWorker.new.perform(args[:version]) + + if runner && runner.errors.empty? && + Random.rand(0..100) < 70 # % of runners having contacted GitLab instance + runner.heartbeat(args.merge(executor: executor)) + runner.save! + end + + ensure_success(runner) + end + + def additional_runner_args(name, executor) + base_tags = ['runner-fleet', "#{@registration_prefix}runner", executor] + tag_limit = ::Ci::Runner::TAG_LIST_MAX_LENGTH - base_tags.length + + { + tag_list: base_tags + TAG_LIST.sample(Random.rand(1..tag_limit)), + description: "Runner fleet #{name}", + run_untagged: false, + active: Random.rand(1..3) != 1, + version: ::Gitlab::Ci::RunnerReleases.instance.releases.sample.to_s, + ip_address: '127.0.0.1' + } + end + + def assign_runner(runner, project) + result = ::Ci::Runners::AssignRunnerService.new(runner, project, @user).execute + result.track_and_raise_exception + + runner + end + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_config/cli_methods.rb b/lib/gitlab/sidekiq_config/cli_methods.rb index 70798f8c3e8..c49180a6c1c 100644 --- a/lib/gitlab/sidekiq_config/cli_methods.rb +++ b/lib/gitlab/sidekiq_config/cli_methods.rb @@ -57,8 +57,8 @@ module Gitlab end def clear_memoization! - if instance_variable_defined?('@worker_metadatas') - remove_instance_variable('@worker_metadatas') + if instance_variable_defined?(:@worker_metadatas) + remove_instance_variable(:@worker_metadatas) end end diff --git a/lib/gitlab/sidekiq_daemon/memory_killer.rb b/lib/gitlab/sidekiq_daemon/memory_killer.rb index 4bf9fd8470a..1682d62d782 100644 --- a/lib/gitlab/sidekiq_daemon/memory_killer.rb +++ b/lib/gitlab/sidekiq_daemon/memory_killer.rb @@ -241,9 +241,8 @@ module Gitlab deadline = Gitlab::Metrics::System.monotonic_time + time - # we try to finish as early as all jobs finished - # so we retest that in loop - sleep(CHECK_INTERVAL_SECONDS) while enabled? && any_jobs? && Gitlab::Metrics::System.monotonic_time < deadline + # Sleep until thread killed or timeout reached + sleep(CHECK_INTERVAL_SECONDS) while enabled? && Gitlab::Metrics::System.monotonic_time < deadline end def signal_pgroup(signal, explanation) @@ -289,10 +288,6 @@ module Gitlab def pid Process.pid end - - def any_jobs? - @sidekiq_daemon_monitor.jobs.any? - end end end end diff --git a/lib/gitlab/sql/pattern.rb b/lib/gitlab/sql/pattern.rb index cd5587bbaef..6563968f315 100644 --- a/lib/gitlab/sql/pattern.rb +++ b/lib/gitlab/sql/pattern.rb @@ -81,3 +81,6 @@ module Gitlab end end end + +Gitlab::SQL::Pattern.prepend_mod +Gitlab::SQL::Pattern::ClassMethods.prepend_mod_with('Gitlab::SQL::Pattern::ClassMethods') diff --git a/lib/gitlab/ssh/commit.rb b/lib/gitlab/ssh/commit.rb index bfeefc47f13..d9ac8c1b881 100644 --- a/lib/gitlab/ssh/commit.rb +++ b/lib/gitlab/ssh/commit.rb @@ -16,6 +16,8 @@ module Gitlab commit_sha: @commit.sha, project: @commit.project, key_id: signature.signed_by_key&.id, + key_fingerprint_sha256: signature.key_fingerprint, + user_id: signature.signed_by_key&.user_id, verification_status: signature.verification_status } end diff --git a/lib/gitlab/ssh/signature.rb b/lib/gitlab/ssh/signature.rb index a654d5b2ff1..763d89116f1 100644 --- a/lib/gitlab/ssh/signature.rb +++ b/lib/gitlab/ssh/signature.rb @@ -41,6 +41,10 @@ module Gitlab end end + def key_fingerprint + strong_memoize(:key_fingerprint) { signature&.public_key&.fingerprint } + end + private def all_attributes_present? @@ -77,10 +81,6 @@ module Gitlab nil end end - - def key_fingerprint - strong_memoize(:key_fingerprint) { signature&.public_key&.fingerprint } - end end end end diff --git a/lib/gitlab/ssh_public_key.rb b/lib/gitlab/ssh_public_key.rb index e9c8e816f18..707f7f3fc0a 100644 --- a/lib/gitlab/ssh_public_key.rb +++ b/lib/gitlab/ssh_public_key.rb @@ -2,6 +2,8 @@ module Gitlab class SSHPublicKey + include Gitlab::Utils::StrongMemoize + Technology = Struct.new(:name, :key_class, :supported_sizes, :supported_algorithms) # See https://man.openbsd.org/sshd#AUTHORIZED_KEYS_FILE_FORMAT for the list of @@ -15,29 +17,6 @@ module Gitlab Technology.new(:ed25519_sk, SSHData::PublicKey::SKED25519, [256], %w(sk-ssh-ed25519@openssh.com)) ].freeze - BANNED_SSH_KEY_FINGERPRINTS = [ - # https://github.com/rapid7/ssh-badkeys/tree/master/authorized - # banned ssh rsa keys - "SHA256:Z+q4XhSwWY7q0BIDVPR1v/S306FjGBsid7tLq/8kIxM", - "SHA256:uy5wXyEgbRCGsk23+J6f85om7G55Cu3UIPwC7oMZhNQ", - "SHA256:9prMbqhS4QteoFQ1ZRJDqSBLWoHXPyKB0iWR05Ghro4", - "SHA256:1M4RzhMyWuFS/86uPY/ce2prh/dVTHW7iD2RhpquOZA", - - # banned ssh dsa keys - "SHA256:/JLp6z6uGE3BPcs70RQob6QOdEWQ6nDC0xY7ejPOCc0", - "SHA256:whDP3xjKBEettbDuecxtGsfWBST+78gb6McdB9P7jCU", - "SHA256:MEc4HfsOlMqJ3/9QMTmrKn5Xj/yfnMITMW8EwfUfTww", - "SHA256:aPoYT2nPIfhqv6BIlbCCpbDjirBxaDFOtPfZ2K20uWw", - "SHA256:VtjqZ5fiaeoZ3mXOYi49Lk9aO31iT4pahKFP9JPiQPc", - - # other banned ssh keys - # https://github.com/BenBE/kompromat/commit/c8d9a05ea155a1ed609c617d4516f0ac978e8559 - "SHA256:Z+q4XhSwWY7q0BIDVPR1v/S306FjGBsid7tLq/8kIxM", - - # https://www.ctrlu.net/vuln/0006.html - "SHA256:2ewGtK7Dc8XpnfNKShczdc8HSgoEGpoX+MiJkfH2p5I" - ].to_set.freeze - def self.technologies if Gitlab::FIPS.enabled? Gitlab::FIPS::SSH_KEY_TECHNOLOGIES @@ -139,11 +118,21 @@ module Gitlab end def banned? - BANNED_SSH_KEY_FINGERPRINTS.include?(fingerprint_sha256) + return false unless valid? + + banned_ssh_keys.fetch(type.to_s, []).include?(fingerprint_sha256) end private + def banned_ssh_keys + path = Rails.root.join('config/security/banned_ssh_keys.yml') + config = YAML.load_file(path) if File.exist?(path) + + config || {} + end + strong_memoize_attr :banned_ssh_keys + def technology @technology ||= self.class.technology_for_key(key) || raise_unsupported_key_type_error diff --git a/lib/gitlab/usage/metrics/aggregates/aggregate.rb b/lib/gitlab/usage/metrics/aggregates/aggregate.rb index 8d816c8d902..b68e1ace658 100644 --- a/lib/gitlab/usage/metrics/aggregates/aggregate.rb +++ b/lib/gitlab/usage/metrics/aggregates/aggregate.rb @@ -7,6 +7,11 @@ module Gitlab class Aggregate include Gitlab::Usage::TimeFrame + # TODO: define this missing event https://gitlab.com/gitlab-org/gitlab/-/issues/385080 + EVENTS_NOT_DEFINED_YET = %w[ + i_code_review_merge_request_widget_license_compliance_warning + ].freeze + def initialize(recorded_at) @recorded_at = recorded_at end @@ -14,11 +19,12 @@ module Gitlab def calculate_count_for_aggregation(aggregation:, time_frame:) with_validate_configuration(aggregation, time_frame) do source = SOURCES[aggregation[:source]] + events = select_defined_events(aggregation[:events], aggregation[:source]) if aggregation[:operator] == UNION_OF_AGGREGATED_METRICS - source.calculate_metrics_union(**time_constraints(time_frame).merge(metric_names: aggregation[:events], recorded_at: recorded_at)) + source.calculate_metrics_union(**time_constraints(time_frame).merge(metric_names: events, recorded_at: recorded_at)) else - source.calculate_metrics_intersections(**time_constraints(time_frame).merge(metric_names: aggregation[:events], recorded_at: recorded_at)) + source.calculate_metrics_intersections(**time_constraints(time_frame).merge(metric_names: events, recorded_at: recorded_at)) end end rescue Gitlab::UsageDataCounters::HLLRedisCounter::EventError, AggregatedMetricError => error @@ -71,6 +77,16 @@ module Gitlab { start_date: nil, end_date: nil } end end + + def select_defined_events(events, source) + # Database source metrics get validated inside the PostgresHll class: + # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb#L16 + return events if source != ::Gitlab::Usage::Metrics::Aggregates::REDIS_SOURCE + + events.select do |event| + ::Gitlab::UsageDataCounters::HLLRedisCounter.known_event?(event) || EVENTS_NOT_DEFINED_YET.include?(event) + end + end end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/base_metric.rb b/lib/gitlab/usage/metrics/instrumentations/base_metric.rb index 55da2315e45..0c102f0f386 100644 --- a/lib/gitlab/usage/metrics/instrumentations/base_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/base_metric.rb @@ -15,7 +15,7 @@ module Gitlab def available?(&block) return @metric_available = block if block - return @metric_available.call if instance_variable_defined?('@metric_available') + return @metric_available.call if instance_variable_defined?(:@metric_available) true end diff --git a/lib/gitlab/usage/service_ping/legacy_metric_metadata_decorator.rb b/lib/gitlab/usage/service_ping/legacy_metric_metadata_decorator.rb new file mode 100644 index 00000000000..db313bc1fbe --- /dev/null +++ b/lib/gitlab/usage/service_ping/legacy_metric_metadata_decorator.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module ServicePing + class LegacyMetricMetadataDecorator < SimpleDelegator + attr_reader :duration, :error + + delegate :class, :is_a?, :kind_of?, :nil?, to: :__getobj__ + + def initialize(value, duration, error: nil) + @duration = duration + @error = error + super(value) + end + end + end + end +end diff --git a/lib/gitlab/usage/service_ping/legacy_metric_timing_decorator.rb b/lib/gitlab/usage/service_ping/legacy_metric_timing_decorator.rb deleted file mode 100644 index e32dcd3777b..00000000000 --- a/lib/gitlab/usage/service_ping/legacy_metric_timing_decorator.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Usage - module ServicePing - class LegacyMetricTimingDecorator < SimpleDelegator - attr_reader :duration - - delegate :class, :is_a?, :kind_of?, to: :__getobj__ - - def initialize(value, duration) - @duration = duration - super(value) - end - end - end - end -end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 24f6cc725f6..c105288fff0 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -344,6 +344,11 @@ module Gitlab def jira_usage # Jira Cloud does not support custom domains as per https://jira.atlassian.com/browse/CLOUD-6999 # so we can just check for subdomains of atlassian.net + jira_integration_data_hash = jira_integration_data + if jira_integration_data_hash.nil? + return { projects_jira_server_active: FALLBACK, projects_jira_cloud_active: FALLBACK } + end + results = { projects_jira_server_active: 0, projects_jira_cloud_active: 0, @@ -351,14 +356,10 @@ module Gitlab projects_jira_dvcs_server_active: count(ProjectFeatureUsage.with_jira_dvcs_integration_enabled(cloud: false)) } - jira_integration_data_hash = jira_integration_data results[:projects_jira_server_active] = jira_integration_data_hash[:projects_jira_server_active] results[:projects_jira_cloud_active] = jira_integration_data_hash[:projects_jira_cloud_active] results - rescue ActiveRecord::StatementInvalid => error - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) - { projects_jira_server_active: FALLBACK, projects_jira_cloud_active: FALLBACK } end # rubocop: enable CodeReuse/ActiveRecord @@ -602,13 +603,18 @@ module Gitlab } end - def with_duration + def with_metadata result = nil + error = nil + duration = Benchmark.realtime do result = yield + rescue StandardError => e + error = e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) end - ::Gitlab::Usage::ServicePing::LegacyMetricTimingDecorator.new(result, duration) + ::Gitlab::Usage::ServicePing::LegacyMetricMetadataDecorator.new(result, duration, error: error) end private diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb index 4486ca53966..0d15475eebb 100644 --- a/lib/gitlab/usage_data_queries.rb +++ b/lib/gitlab/usage_data_queries.rb @@ -5,7 +5,7 @@ module Gitlab # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41091 class UsageDataQueries < UsageData class << self - def with_duration + def with_metadata yield end diff --git a/lib/gitlab/utils/strong_memoize.rb b/lib/gitlab/utils/strong_memoize.rb index 7e78363dae5..eb44b7ddd95 100644 --- a/lib/gitlab/utils/strong_memoize.rb +++ b/lib/gitlab/utils/strong_memoize.rb @@ -23,7 +23,7 @@ module Gitlab # def enabled? # Feature.enabled?(:some_feature) # end - # strong_memoize_attr :enabled?, :enabled + # strong_memoize_attr :enabled? # def strong_memoize(name) key = ivar(name) @@ -46,20 +46,20 @@ module Gitlab end def strong_memoized?(name) - instance_variable_defined?(ivar(name)) + key = ivar(StrongMemoize.normalize_key(name)) + instance_variable_defined?(key) end def clear_memoization(name) - key = ivar(name) + key = ivar(StrongMemoize.normalize_key(name)) remove_instance_variable(key) if instance_variable_defined?(key) end module StrongMemoizeClassMethods - def strong_memoize_attr(method_name, member_name = nil) - member_name ||= method_name + def strong_memoize_attr(method_name) + member_name = StrongMemoize.normalize_key(method_name) - StrongMemoize.send( # rubocop:disable GitlabSecurity/PublicSend - :do_strong_memoize, self, method_name, member_name) + StrongMemoize.send(:do_strong_memoize, self, method_name, member_name) # rubocop:disable GitlabSecurity/PublicSend end end @@ -83,7 +83,14 @@ module Gitlab end end - class <<self + class << self + def normalize_key(key) + return key unless key.end_with?('!', '?') + + # Replace invalid chars like `!` and `?` with allowed Unicode codeparts. + key.to_s.tr('!?', "\uFF01\uFF1F") + end + private def do_strong_memoize(klass, method_name, member_name) diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb index 0b818b99ac7..fab8617bcda 100644 --- a/lib/gitlab/utils/usage_data.rb +++ b/lib/gitlab/utils/usage_data.rb @@ -44,7 +44,7 @@ module Gitlab DISTRIBUTED_HLL_FALLBACK = -2 MAX_BUCKET_SIZE = 100 - def with_duration + def with_metadata yield end @@ -55,7 +55,7 @@ module Gitlab end def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil, start_at: Time.current) - with_duration do + with_metadata do if batch Gitlab::Database::BatchCount.batch_count(relation, column, batch_size: batch_size, start: start, finish: finish) else @@ -68,7 +68,7 @@ module Gitlab end def distinct_count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) - with_duration do + with_metadata do if batch Gitlab::Database::BatchCount.batch_distinct_count(relation, column, batch_size: batch_size, start: start, finish: finish) else @@ -81,7 +81,7 @@ module Gitlab end def estimate_batch_distinct_count(relation, column = nil, batch_size: nil, start: nil, finish: nil) - with_duration do + with_metadata do buckets = Gitlab::Database::PostgresHll::BatchDistinctCounter .new(relation, column) .execute(batch_size: batch_size, start: start, finish: finish) @@ -96,7 +96,7 @@ module Gitlab end def sum(relation, column, batch_size: nil, start: nil, finish: nil) - with_duration do + with_metadata do Gitlab::Database::BatchCount.batch_sum(relation, column, batch_size: batch_size, start: start, finish: finish) rescue ActiveRecord::StatementInvalid => error Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) @@ -105,7 +105,7 @@ module Gitlab end def average(relation, column, batch_size: nil, start: nil, finish: nil) - with_duration do + with_metadata do Gitlab::Database::BatchCount.batch_average(relation, column, batch_size: batch_size, start: start, finish: finish) rescue ActiveRecord::StatementInvalid => error Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) @@ -119,7 +119,7 @@ module Gitlab # # rubocop: disable CodeReuse/ActiveRecord def histogram(relation, column, buckets:, bucket_size: buckets.size) - with_duration do + with_metadata do # Using lambda to avoid exposing histogram specific methods parameters_valid = lambda do error_message = @@ -184,7 +184,7 @@ module Gitlab # rubocop: enable CodeReuse/ActiveRecord def add(*args) - with_duration do + with_metadata do break -1 if args.any?(&:negative?) args.sum @@ -195,7 +195,7 @@ module Gitlab end def alt_usage_data(value = nil, fallback: FALLBACK, &block) - with_duration do + with_metadata do if block yield else @@ -208,7 +208,7 @@ module Gitlab end def redis_usage_data(counter = nil, &block) - with_duration do + with_metadata do if block redis_usage_counter(&block) elsif counter.present? @@ -218,7 +218,7 @@ module Gitlab end def with_prometheus_client(fallback: {}, verify: true) - with_duration do + with_metadata do client = prometheus_client(verify: verify) break fallback unless client @@ -257,7 +257,7 @@ module Gitlab # rubocop: disable UsageData/LargeTable: def jira_integration_data - with_duration do + with_metadata do data = { projects_jira_server_active: 0, projects_jira_cloud_active: 0 diff --git a/lib/gitlab/version_info.rb b/lib/gitlab/version_info.rb index 61de003c28d..0351c9b30b3 100644 --- a/lib/gitlab/version_info.rb +++ b/lib/gitlab/version_info.rb @@ -7,11 +7,14 @@ module Gitlab attr_reader :major, :minor, :patch VERSION_REGEX = /(\d+)\.(\d+)\.(\d+)/.freeze + # To mitigate ReDoS, limit the length of the version string we're + # willing to check + MAX_VERSION_LENGTH = 128 def self.parse(str, parse_suffix: false) if str.is_a?(self) str - elsif str && m = str.match(VERSION_REGEX) + elsif str && str.length <= MAX_VERSION_LENGTH && m = str.match(VERSION_REGEX) VersionInfo.new(m[1].to_i, m[2].to_i, m[3].to_i, parse_suffix ? m.post_match : nil) else VersionInfo.new |