diff options
Diffstat (limited to 'lib/gitlab')
244 files changed, 3340 insertions, 1848 deletions
diff --git a/lib/gitlab/access/branch_protection.rb b/lib/gitlab/access/branch_protection.rb index 81f02c004af..f6d0f8b04b3 100644 --- a/lib/gitlab/access/branch_protection.rb +++ b/lib/gitlab/access/branch_protection.rb @@ -74,7 +74,11 @@ module Gitlab end def protection_partial - protection_none.merge(allow_force_push: false) + { + allowed_to_push: [{ 'access_level' => Gitlab::Access::DEVELOPER }], + allowed_to_merge: [{ 'access_level' => Gitlab::Access::MAINTAINER }], + allow_force_push: false + } end def protected_fully @@ -89,15 +93,15 @@ module Gitlab { allowed_to_push: [{ 'access_level' => Gitlab::Access::MAINTAINER }], allowed_to_merge: [{ 'access_level' => Gitlab::Access::DEVELOPER }], - allow_force_push: true + allow_force_push: false } end def protected_after_initial_push { allowed_to_push: [{ 'access_level' => Gitlab::Access::MAINTAINER }], - allowed_to_merge: [{ 'access_level' => Gitlab::Access::DEVELOPER }], - allow_force_push: true, + allowed_to_merge: [{ 'access_level' => Gitlab::Access::MAINTAINER }], + allow_force_push: false, developer_can_initial_push: true } end diff --git a/lib/gitlab/analytics/cycle_analytics/aggregated/records_fetcher.rb b/lib/gitlab/analytics/cycle_analytics/aggregated/records_fetcher.rb index 2143497f084..6a1529ade92 100644 --- a/lib/gitlab/analytics/cycle_analytics/aggregated/records_fetcher.rb +++ b/lib/gitlab/analytics/cycle_analytics/aggregated/records_fetcher.rb @@ -14,12 +14,14 @@ module Gitlab Issue => { serializer_class: AnalyticsIssueSerializer, includes_for_query: { project: { namespace: [:route] }, author: [] }, - columns_for_select: %I[title iid id created_at author_id project_id] + columns_for_select: %I[title iid id created_at author_id project_id], + finder_class: IssuesFinder }, MergeRequest => { serializer_class: AnalyticsMergeRequestSerializer, includes_for_query: { target_project: [:namespace], author: [] }, - columns_for_select: %I[title iid id created_at author_id state_id target_project_id] + columns_for_select: %I[title iid id created_at author_id state_id target_project_id], + finder_class: MergeRequestsFinder } }.freeze @@ -80,14 +82,17 @@ module Gitlab def load_issuables(stage_event_records) stage_event_records_by_issuable_id = stage_event_records.index_by(&:issuable_id) - issuable_model = stage_event_model.issuable_model - issuables_by_id = issuable_model.id_in(stage_event_records_by_issuable_id.keys).index_by(&:id) + issuables_by_id = finder.execute.id_in(stage_event_records_by_issuable_id.keys).index_by(&:id) stage_event_records_by_issuable_id.map do |issuable_id, record| [issuables_by_id[issuable_id], record] if issuables_by_id[issuable_id] end.compact end + def finder + MAPPINGS.fetch(subject_class).fetch(:finder_class).new(params[:current_user]) + end + def serializer MAPPINGS.fetch(subject_class).fetch(:serializer_class).new end diff --git a/lib/gitlab/analytics/cycle_analytics/request_params.rb b/lib/gitlab/analytics/cycle_analytics/request_params.rb index 0c4a0afa1d5..4a444b06500 100644 --- a/lib/gitlab/analytics/cycle_analytics/request_params.rb +++ b/lib/gitlab/analytics/cycle_analytics/request_params.rb @@ -119,7 +119,9 @@ module Gitlab attrs[:namespace] = namespace_attributes attrs[:enable_tasks_by_type_chart] = 'false' attrs[:enable_customizable_stages] = 'false' + attrs[:can_edit] = 'false' attrs[:enable_projects_filter] = 'false' + attrs[:enable_vsd_link] = 'false' attrs[:default_stages] = Gitlab::Analytics::CycleAnalytics::DefaultStages.all.map do |stage_params| ::Analytics::CycleAnalytics::StagePresenter.new(stage_params) end.to_json @@ -151,8 +153,8 @@ module Gitlab helpers = ActionController::Base.helpers {}.tap do |paths| - paths[:empty_state_svg_path] = helpers.image_path("illustrations/analytics/cycle-analytics-empty-chart.svg") - paths[:no_data_svg_path] = helpers.image_path("illustrations/analytics/cycle-analytics-empty-chart.svg") + paths[:empty_state_svg_path] = helpers.image_path("illustrations/empty-state/empty-dashboard-md.svg") + paths[:no_data_svg_path] = helpers.image_path("illustrations/empty-state/empty-dashboard-md.svg") paths[:no_access_svg_path] = helpers.image_path("illustrations/analytics/no-access.svg") if project diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb index 67fc2ae2fcc..e46bbc2cfda 100644 --- a/lib/gitlab/application_context.rb +++ b/lib/gitlab/application_context.rb @@ -26,7 +26,8 @@ module Gitlab :artifacts_dependencies_size, :artifacts_dependencies_count, :root_caller_id, - :merge_action_status + :merge_action_status, + :bulk_import_entity_id ].freeze private_constant :KNOWN_KEYS @@ -45,7 +46,8 @@ module Gitlab Attribute.new(:artifacts_dependencies_size, Integer), Attribute.new(:artifacts_dependencies_count, Integer), Attribute.new(:root_caller_id, String), - Attribute.new(:merge_action_status, String) + Attribute.new(:merge_action_status, String), + Attribute.new(:bulk_import_entity_id, Integer) ].freeze private_constant :APPLICATION_ATTRIBUTES @@ -95,6 +97,7 @@ module Gitlab # rubocop: disable Metrics/CyclomaticComplexity # rubocop: disable Metrics/PerceivedComplexity + # rubocop: disable Metrics/AbcSize def to_lazy_hash {}.tap do |hash| assign_hash_if_value(hash, :caller_id) @@ -106,6 +109,7 @@ module Gitlab assign_hash_if_value(hash, :artifacts_dependencies_size) assign_hash_if_value(hash, :artifacts_dependencies_count) assign_hash_if_value(hash, :merge_action_status) + assign_hash_if_value(hash, :bulk_import_entity_id) hash[:user] = -> { username } if include_user? hash[:user_id] = -> { user_id } if include_user? @@ -115,10 +119,12 @@ module Gitlab hash[:pipeline_id] = -> { job&.pipeline_id } if set_values.include?(:job) hash[:job_id] = -> { job&.id } if set_values.include?(:job) hash[:artifact_size] = -> { artifact&.size } if set_values.include?(:artifact) + hash[:bulk_import_entity_id] = -> { bulk_import_entity_id } if set_values.include?(:bulk_import_entity_id) end end # rubocop: enable Metrics/CyclomaticComplexity # rubocop: enable Metrics/PerceivedComplexity + # rubocop: enable Metrics/AbcSize def use Labkit::Context.with_context(to_lazy_hash) { yield } diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index 469927b8a53..3d2f13af9dc 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -52,8 +52,9 @@ module Gitlab project_testing_integration: { threshold: 5, interval: 1.minute }, email_verification: { threshold: 10, interval: 10.minutes }, email_verification_code_send: { threshold: 10, interval: 1.hour }, - phone_verification_send_code: { threshold: 10, interval: 1.hour }, - phone_verification_verify_code: { threshold: 10, interval: 10.minutes }, + phone_verification_challenge: { threshold: 3, interval: 1.day }, + phone_verification_send_code: { threshold: 5, interval: 1.day }, + phone_verification_verify_code: { threshold: 5, interval: 1.day }, namespace_exists: { threshold: 20, interval: 1.minute }, update_namespace_name: { threshold: -> { application_settings.update_namespace_name_rate_limit }, interval: 1.hour }, fetch_google_ip_list: { threshold: 10, interval: 1.minute }, diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 578cfb52714..8e894be4fc4 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -436,8 +436,7 @@ module Gitlab end def unavailable_scopes_for_resource(resource) - unavailable_observability_scopes_for_resource(resource) + - unavailable_ai_features_scopes_for_resource(resource) + unavailable_observability_scopes_for_resource(resource) end def unavailable_observability_scopes_for_resource(resource) @@ -447,10 +446,6 @@ module Gitlab OBSERVABILITY_SCOPES end - def unavailable_ai_features_scopes_for_resource(_resource) - AI_FEATURES_SCOPES - end - def non_admin_available_scopes API_SCOPES + REPOSITORY_SCOPES + registry_scopes + OBSERVABILITY_SCOPES + AI_FEATURES_SCOPES end diff --git a/lib/gitlab/auth/saml/config.rb b/lib/gitlab/auth/saml/config.rb index 7524d8b9f85..235c472d292 100644 --- a/lib/gitlab/auth/saml/config.rb +++ b/lib/gitlab/auth/saml/config.rb @@ -4,10 +4,39 @@ module Gitlab module Auth module Saml class Config + DEFAULT_NICKNAME_ATTRS = %w[username nickname].freeze + DEFAULT_NAME_ATTRS = %w[ + http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name + http://schemas.microsoft.com/ws/2008/06/identity/claims/name + ].freeze + DEFAULT_EMAIL_ATTRS = %w[ + http://schemas.xmlsoap.org/ws/2005/05/identity/claims/emailaddress + http://schemas.microsoft.com/ws/2008/06/identity/claims/emailaddress + ].freeze + DEFAULT_FIRST_NAME_ATTRS = %w[ + http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname + http://schemas.microsoft.com/ws/2008/06/identity/claims/givenname + ].freeze + DEFAULT_LAST_NAME_ATTRS = %w[ + http://schemas.xmlsoap.org/ws/2005/05/identity/claims/surname + http://schemas.microsoft.com/ws/2008/06/identity/claims/surname + ].freeze + class << self def enabled? ::AuthHelper.saml_providers.any? end + + def default_attribute_statements + defaults = OmniAuth::Strategies::SAML.default_options[:attribute_statements].to_hash.deep_symbolize_keys + defaults[:nickname] = DEFAULT_NICKNAME_ATTRS.dup + defaults[:name].concat(DEFAULT_NAME_ATTRS) + defaults[:email].concat(DEFAULT_EMAIL_ATTRS) + defaults[:first_name].concat(DEFAULT_FIRST_NAME_ATTRS) + defaults[:last_name].concat(DEFAULT_LAST_NAME_ATTRS) + + defaults + end end DEFAULT_PROVIDER_NAME = 'saml' diff --git a/lib/gitlab/auth/unique_ips_limiter.rb b/lib/gitlab/auth/unique_ips_limiter.rb index 74f7fdfc180..341edbed9c2 100644 --- a/lib/gitlab/auth/unique_ips_limiter.rb +++ b/lib/gitlab/auth/unique_ips_limiter.rb @@ -30,13 +30,11 @@ module Gitlab key = "#{USER_UNIQUE_IPS_PREFIX}:#{user_id}" Gitlab::Redis::SharedState.with do |redis| - unique_ips_count = nil redis.multi do |r| r.zadd(key, time, ip) r.zremrangebyscore(key, 0, time - config.unique_ips_limit_time_window) - unique_ips_count = r.zcard(key) - end - unique_ips_count.value + r.zcard(key) + end.last end end end diff --git a/lib/gitlab/background_migration/.rubocop.yml b/lib/gitlab/background_migration/.rubocop.yml index 9424686340f..e9d54ca3359 100644 --- a/lib/gitlab/background_migration/.rubocop.yml +++ b/lib/gitlab/background_migration/.rubocop.yml @@ -40,12 +40,6 @@ Metrics/BlockLength: Long blocks can be hard to read. Consider splitting the code into separate methods. -Style/Documentation: - Enabled: true - Details: > - Adding documentation makes it easier to figure out what a migration is - supposed to do. - Style/FrozenStringLiteralComment: Enabled: true Details: >- diff --git a/lib/gitlab/background_migration/backfill_branch_protection_namespace_setting.rb b/lib/gitlab/background_migration/backfill_branch_protection_namespace_setting.rb new file mode 100644 index 00000000000..c063a990188 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_branch_protection_namespace_setting.rb @@ -0,0 +1,135 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class is used to update the default_branch_protection_defaults column + # for user namespaces of the namespace_settings table. + class BackfillBranchProtectionNamespaceSetting < BatchedMigrationJob + operation_name :set_default_branch_protection_defaults + feature_category :source_code_management + + # Migration only version of `namespaces` table + class Namespace < ::ApplicationRecord + self.table_name = 'namespaces' + self.inheritance_column = :_type_disabled + + has_one :namespace_setting, + class_name: '::Gitlab::BackgroundMigration::BackfillDefaultBranchProtectionNamespaceSetting::NamespaceSetting' + end + + # Migration only version of `namespace_settings` table + class NamespaceSetting < ::ApplicationRecord + self.table_name = 'namespace_settings' + belongs_to :namespace, + class_name: '::Gitlab::BackgroundMigration::BackfillDefaultBranchProtectionNamespaceSetting::Namespace' + end + + # Migration only version of Gitlab::Access:BranchProtection application code. + class BranchProtection + attr_reader :level + + def initialize(level) + @level = level + end + + PROTECTION_NONE = 0 + PROTECTION_DEV_CAN_PUSH = 1 + PROTECTION_FULL = 2 + PROTECTION_DEV_CAN_MERGE = 3 + PROTECTION_DEV_CAN_INITIAL_PUSH = 4 + + DEVELOPER = 30 + MAINTAINER = 40 + + def to_hash + case level + when PROTECTION_NONE + self.class.protection_none + when PROTECTION_DEV_CAN_PUSH + self.class.protection_partial + when PROTECTION_FULL + self.class.protected_fully + when PROTECTION_DEV_CAN_MERGE + self.class.protected_against_developer_pushes + when PROTECTION_DEV_CAN_INITIAL_PUSH + self.class.protected_after_initial_push + end + end + + class << self + def protection_none + { + allowed_to_push: [{ 'access_level' => Gitlab::Access::DEVELOPER }], + allowed_to_merge: [{ 'access_level' => Gitlab::Access::DEVELOPER }], + allow_force_push: true + } + end + + def protection_partial + { + allowed_to_push: [{ 'access_level' => Gitlab::Access::DEVELOPER }], + allowed_to_merge: [{ 'access_level' => Gitlab::Access::MAINTAINER }], + allow_force_push: false + } + end + + def protected_fully + { + allowed_to_push: [{ 'access_level' => Gitlab::Access::MAINTAINER }], + allowed_to_merge: [{ 'access_level' => Gitlab::Access::MAINTAINER }], + allow_force_push: false + } + end + + def protected_against_developer_pushes + { + allowed_to_push: [{ 'access_level' => Gitlab::Access::MAINTAINER }], + allowed_to_merge: [{ 'access_level' => Gitlab::Access::DEVELOPER }], + allow_force_push: false + } + end + + def protected_after_initial_push + { + allowed_to_push: [{ 'access_level' => Gitlab::Access::MAINTAINER }], + allowed_to_merge: [{ 'access_level' => Gitlab::Access::MAINTAINER }], + allow_force_push: false, + developer_can_initial_push: true + } + end + end + end + + def perform + each_sub_batch do |sub_batch| + update_default_protection_branch_defaults(sub_batch) + end + end + + private + + def update_default_protection_branch_defaults(batch) + namespace_settings = NamespaceSetting.where(namespace_id: batch.pluck(:namespace_id)).includes(:namespace) + + values_list = namespace_settings.map do |namespace_setting| + level = namespace_setting.namespace.default_branch_protection.to_i + value = BranchProtection.new(level).to_hash.to_json + "(#{namespace_setting.namespace_id}, '#{value}'::jsonb)" + end.join(", ") + + sql = <<~SQL + WITH new_values (namespace_id, default_branch_protection_defaults) AS ( + VALUES + #{values_list} + ) + UPDATE namespace_settings + SET default_branch_protection_defaults = new_values.default_branch_protection_defaults + FROM new_values + WHERE namespace_settings.namespace_id = new_values.namespace_id; + SQL + + connection.execute(sql) + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_has_remediations_of_vulnerability_reads.rb b/lib/gitlab/background_migration/backfill_has_remediations_of_vulnerability_reads.rb index 83acd8a27f7..84b0f5c97df 100644 --- a/lib/gitlab/background_migration/backfill_has_remediations_of_vulnerability_reads.rb +++ b/lib/gitlab/background_migration/backfill_has_remediations_of_vulnerability_reads.rb @@ -20,6 +20,8 @@ module Gitlab def perform each_sub_batch do |sub_batch| + reset_has_remediations_attribute(sub_batch) + update_query = update_query_for(sub_batch) connection.execute(update_query) @@ -28,6 +30,10 @@ module Gitlab private + def reset_has_remediations_attribute(sub_batch) + sub_batch.update_all(has_remediations: false) + end + def update_query_for(sub_batch) subquery = sub_batch.joins(" INNER JOIN vulnerability_occurrences ON diff --git a/lib/gitlab/background_migration/backfill_imported_issue_search_data.rb b/lib/gitlab/background_migration/backfill_imported_issue_search_data.rb index 8c151bc36ac..e230fe46466 100644 --- a/lib/gitlab/background_migration/backfill_imported_issue_search_data.rb +++ b/lib/gitlab/background_migration/backfill_imported_issue_search_data.rb @@ -15,7 +15,7 @@ module Gitlab def perform each_sub_batch do |sub_batch| update_search_data(sub_batch) - rescue ActiveRecord::StatementInvalid => e + rescue ActiveRecord::StatementInvalid => e # rubocop:todo BackgroundMigration/AvoidSilentRescueExceptions -- https://gitlab.com/gitlab-org/gitlab/-/issues/431592 raise unless e.cause.is_a?(PG::ProgramLimitExceeded) && e.message.include?('string is too long for tsvector') update_search_data_individually(sub_batch) @@ -44,7 +44,7 @@ module Gitlab relation.pluck(:id).each do |issue_id| update_search_data(relation.klass.where(id: issue_id)) sleep(pause_ms * 0.001) - rescue ActiveRecord::StatementInvalid => e + rescue ActiveRecord::StatementInvalid => e # rubocop:todo BackgroundMigration/AvoidSilentRescueExceptions -- https://gitlab.com/gitlab-org/gitlab/-/issues/431592 raise unless e.cause.is_a?(PG::ProgramLimitExceeded) && e.message.include?('string is too long for tsvector') logger.error( diff --git a/lib/gitlab/background_migration/backfill_merge_request_diffs_project_id.rb b/lib/gitlab/background_migration/backfill_merge_request_diffs_project_id.rb new file mode 100644 index 00000000000..881716b5cc0 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_merge_request_diffs_project_id.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This migration populates the new `merge_request_diffs.project_id` column from joining with `merge_requests` table + class BackfillMergeRequestDiffsProjectId < BatchedMigrationJob + operation_name :update_all + scope_to ->(relation) { relation.where(project_id: nil) } + + feature_category :code_review_workflow + + def perform + each_sub_batch do |sub_batch| + ApplicationRecord.connection.execute <<-SQL + UPDATE merge_request_diffs + SET project_id = merge_requests.target_project_id + FROM merge_requests + WHERE merge_requests.id = merge_request_diffs.merge_request_id + AND merge_request_diffs.id IN (#{sub_batch.select(:id).to_sql}) + SQL + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_vs_code_settings_uuid.rb b/lib/gitlab/background_migration/backfill_vs_code_settings_uuid.rb new file mode 100644 index 00000000000..2bb0e0b6d98 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_vs_code_settings_uuid.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class BackfillVsCodeSettingsUuid < BatchedMigrationJob + operation_name :backfill_vs_code_settings_uuid + scope_to ->(relation) { relation.where(uuid: nil) } + feature_category :web_ide + + def perform + each_sub_batch do |sub_batch| + vs_code_settings = sub_batch.map do |vs_code_setting| + vs_code_setting.attributes.merge(uuid: SecureRandom.uuid) + end + + VsCode::Settings::VsCodeSetting.upsert_all(vs_code_settings) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/fix_allow_descendants_override_disabled_shared_runners.rb b/lib/gitlab/background_migration/fix_allow_descendants_override_disabled_shared_runners.rb index 44bda3fe2b6..618944e1653 100644 --- a/lib/gitlab/background_migration/fix_allow_descendants_override_disabled_shared_runners.rb +++ b/lib/gitlab/background_migration/fix_allow_descendants_override_disabled_shared_runners.rb @@ -7,7 +7,7 @@ module Gitlab # This combination fails validation and doesn't make sense: # we always allow descendants to disable shared runners class FixAllowDescendantsOverrideDisabledSharedRunners < BatchedMigrationJob - feature_category :runner_fleet + feature_category :fleet_visibility operation_name :fix_allow_descendants_override_disabled_shared_runners def perform diff --git a/lib/gitlab/background_migration/migrate_links_for_vulnerability_findings.rb b/lib/gitlab/background_migration/migrate_links_for_vulnerability_findings.rb index 0b79bc143db..4f1f70f3337 100644 --- a/lib/gitlab/background_migration/migrate_links_for_vulnerability_findings.rb +++ b/lib/gitlab/background_migration/migrate_links_for_vulnerability_findings.rb @@ -36,7 +36,7 @@ module Gitlab def migrate_remediations(findings, existing_links) findings.each do |finding| create_links(build_links_from(finding, existing_links)) - rescue ActiveRecord::StatementInvalid => e + rescue ActiveRecord::StatementInvalid => e # rubocop:todo BackgroundMigration/AvoidSilentRescueExceptions -- https://gitlab.com/gitlab-org/gitlab/-/issues/431592 logger.error( message: e.message, class: self.class.name, @@ -76,7 +76,7 @@ module Gitlab return [] if parsed_links.blank? parsed_links.select { |link| link.try(:[], 'url').present? }.uniq - rescue JSON::ParserError => e + rescue JSON::ParserError => e # rubocop:todo BackgroundMigration/AvoidSilentRescueExceptions -- https://gitlab.com/gitlab-org/gitlab/-/issues/431592 logger.warn( message: e.message, class: self.class.name diff --git a/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings.rb b/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings.rb index 9eadef96db6..c6a41bc1c65 100644 --- a/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings.rb +++ b/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings.rb @@ -74,7 +74,7 @@ module Gitlab create_finding_remediations(finding.id, result_ids) end - rescue StandardError => e + rescue StandardError => e # rubocop:todo BackgroundMigration/AvoidSilentRescueExceptions -- https://gitlab.com/gitlab-org/gitlab/-/issues/431592 logger.error( message: e.message, class: self.class.name, diff --git a/lib/gitlab/background_migration/populate_vulnerability_dismissal_fields.rb b/lib/gitlab/background_migration/populate_vulnerability_dismissal_fields.rb index ee0f73cc3de..c310c10d7fa 100644 --- a/lib/gitlab/background_migration/populate_vulnerability_dismissal_fields.rb +++ b/lib/gitlab/background_migration/populate_vulnerability_dismissal_fields.rb @@ -58,7 +58,7 @@ module Gitlab def populate_for(vulnerability) log_warning(vulnerability) unless vulnerability.copy_dismissal_information - rescue StandardError => error + rescue StandardError => error # rubocop:todo BackgroundMigration/AvoidSilentRescueExceptions -- https://gitlab.com/gitlab-org/gitlab/-/issues/431592 log_error(error, vulnerability) end diff --git a/lib/gitlab/background_migration/reset_status_on_container_repositories.rb b/lib/gitlab/background_migration/reset_status_on_container_repositories.rb index 56506814dc0..a83c4625cb4 100644 --- a/lib/gitlab/background_migration/reset_status_on_container_repositories.rb +++ b/lib/gitlab/background_migration/reset_status_on_container_repositories.rb @@ -117,7 +117,7 @@ module Gitlab return DUMMY_TAGS unless response response['tags'] || [] - rescue StandardError + rescue StandardError # rubocop:todo BackgroundMigration/AvoidSilentRescueExceptions -- https://gitlab.com/gitlab-org/gitlab/-/issues/431592 DUMMY_TAGS end end diff --git a/lib/gitlab/base_doorkeeper_controller.rb b/lib/gitlab/base_doorkeeper_controller.rb index c8520993b8e..91994c2fa95 100644 --- a/lib/gitlab/base_doorkeeper_controller.rb +++ b/lib/gitlab/base_doorkeeper_controller.rb @@ -3,8 +3,7 @@ # This is a base controller for doorkeeper. # It adds the `can?` helper used in the views. module Gitlab - # rubocop:disable Rails/ApplicationController - class BaseDoorkeeperController < ActionController::Base + class BaseDoorkeeperController < BaseActionController include Gitlab::Allowable include EnforcesTwoFactorAuthentication include SessionsHelper @@ -13,5 +12,4 @@ module Gitlab helper_method :can? end - # rubocop:enable Rails/ApplicationController end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb deleted file mode 100644 index 9f87bb2347c..00000000000 --- a/lib/gitlab/bitbucket_import/importer.rb +++ /dev/null @@ -1,339 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BitbucketImport - class Importer - LABELS = [{ title: 'bug', color: '#FF0000' }, - { title: 'enhancement', color: '#428BCA' }, - { title: 'proposal', color: '#69D100' }, - { title: 'task', color: '#7F8C8D' }].freeze - - attr_reader :project, :client, :errors, :users - - ALREADY_IMPORTED_CACHE_KEY = 'bitbucket_cloud-importer/already-imported/%{project}/%{collection}' - - def initialize(project) - @project = project - @client = Bitbucket::Client.new(project.import_data.credentials) - @formatter = Gitlab::ImportFormatter.new - @ref_converter = Gitlab::BitbucketImport::RefConverter.new(project) - @labels = {} - @errors = [] - @users = {} - end - - def execute - import_wiki - import_issues - import_pull_requests - handle_errors - metrics.track_finished_import - - true - end - - def create_labels - LABELS.each do |label_params| - label = ::Labels::FindOrCreateService.new(nil, project, label_params).execute(skip_authorization: true) - if label.valid? - @labels[label_params[:title]] = label - else - raise "Failed to create label \"#{label_params[:title]}\" for project \"#{project.full_name}\"" - end - end - end - - def import_pull_request_comments(pull_request, merge_request) - comments = client.pull_request_comments(repo, pull_request.iid) - - inline_comments, pr_comments = comments.partition(&:inline?) - - import_inline_comments(inline_comments, pull_request, merge_request) - import_standalone_pr_comments(pr_comments, merge_request) - end - - private - - def already_imported?(collection, iid) - Gitlab::Cache::Import::Caching.set_includes?(cache_key(collection), iid) - end - - def mark_as_imported(collection, iid) - Gitlab::Cache::Import::Caching.set_add(cache_key(collection), iid) - end - - def cache_key(collection) - format(ALREADY_IMPORTED_CACHE_KEY, project: project.id, collection: collection) - end - - def handle_errors - return unless errors.any? - - project.import_state.update_column(:last_error, { - message: 'The remote data could not be fully imported.', - errors: errors - }.to_json) - end - - def store_pull_request_error(pull_request, ex) - backtrace = Gitlab::BacktraceCleaner.clean_backtrace(ex.backtrace) - error = { type: :pull_request, iid: pull_request.iid, errors: ex.message, trace: backtrace, raw_response: pull_request.raw&.to_json } - - Gitlab::ErrorTracking.log_exception(ex, error) - - # Omit the details from the database to avoid blowing up usage in the error column - error.delete(:trace) - error.delete(:raw_response) - - errors << error - end - - def gitlab_user_id(project, username) - find_user_id(username) || project.creator_id - end - - # rubocop: disable CodeReuse/ActiveRecord - def find_user_id(username) - return unless username - - return users[username] if users.key?(username) - - users[username] = User.by_provider_and_extern_uid(:bitbucket, username).select(:id).first&.id - end - # rubocop: enable CodeReuse/ActiveRecord - - def allocate_issues_internal_id!(project, client) - last_bitbucket_issue = client.last_issue(repo) - - return unless last_bitbucket_issue - - Issue.track_namespace_iid!(project.project_namespace, last_bitbucket_issue.iid) - end - - def repo - @repo ||= client.repo(project.import_source) - end - - def import_wiki - return if project.wiki.repository_exists? - - wiki = WikiFormatter.new(project) - - project.wiki.repository.import_repository(wiki.import_url) - rescue StandardError => e - errors << { type: :wiki, errors: e.message } - end - - def import_issues - return unless repo.issues_enabled? - - create_labels - - issue_type_id = ::WorkItems::Type.default_issue_type.id - - client.issues(repo).each_with_index do |issue, index| - next if already_imported?(:issues, issue.iid) - - # If a user creates an issue while the import is in progress, this can lead to an import failure. - # The workaround is to allocate IIDs before starting the importer. - allocate_issues_internal_id!(project, client) if index == 0 - - import_issue(issue, issue_type_id) - end - end - - # rubocop: disable CodeReuse/ActiveRecord - def import_issue(issue, issue_type_id) - description = '' - description += @formatter.author_line(issue.author) unless find_user_id(issue.author) - description += issue.description - - label_name = issue.kind - milestone = issue.milestone ? project.milestones.find_or_create_by(title: issue.milestone) : nil - - gitlab_issue = project.issues.create!( - iid: issue.iid, - title: issue.title, - description: description, - state_id: Issue.available_states[issue.state], - author_id: gitlab_user_id(project, issue.author), - namespace_id: project.project_namespace_id, - milestone: milestone, - work_item_type_id: issue_type_id, - created_at: issue.created_at, - updated_at: issue.updated_at - ) - - mark_as_imported(:issues, issue.iid) - - metrics.issues_counter.increment - - gitlab_issue.labels << @labels[label_name] - - import_issue_comments(issue, gitlab_issue) if gitlab_issue.persisted? - rescue StandardError => e - errors << { type: :issue, iid: issue.iid, errors: e.message } - end - # rubocop: enable CodeReuse/ActiveRecord - - def import_issue_comments(issue, gitlab_issue) - client.issue_comments(repo, issue.iid).each do |comment| - # The note can be blank for issue service messages like "Changed title: ..." - # We would like to import those comments as well but there is no any - # specific parameter that would allow to process them, it's just an empty comment. - # To prevent our importer from just crashing or from creating useless empty comments - # we do this check. - next unless comment.note.present? - - note = '' - note += @formatter.author_line(comment.author) unless find_user_id(comment.author) - note += @ref_converter.convert_note(comment.note.to_s) - - begin - gitlab_issue.notes.create!( - project: project, - note: note, - author_id: gitlab_user_id(project, comment.author), - created_at: comment.created_at, - updated_at: comment.updated_at - ) - rescue StandardError => e - errors << { type: :issue_comment, iid: issue.iid, errors: e.message } - end - end - end - - def import_pull_requests - pull_requests = client.pull_requests(repo) - - pull_requests.each do |pull_request| - next if already_imported?(:pull_requests, pull_request.iid) - - import_pull_request(pull_request) - end - end - - def import_pull_request(pull_request) - description = '' - description += @formatter.author_line(pull_request.author) unless find_user_id(pull_request.author) - description += pull_request.description - - source_branch_sha = pull_request.source_branch_sha - target_branch_sha = pull_request.target_branch_sha - - source_sha_from_commit_sha = project.repository.commit(source_branch_sha)&.sha - source_sha_from_merge_sha = project.repository.commit(pull_request.merge_commit_sha)&.sha - - source_branch_sha = source_sha_from_commit_sha || source_sha_from_merge_sha || source_branch_sha - target_branch_sha = project.repository.commit(target_branch_sha)&.sha || target_branch_sha - - merge_request = project.merge_requests.create!( - iid: pull_request.iid, - title: pull_request.title, - description: description, - source_project: project, - source_branch: pull_request.source_branch_name, - source_branch_sha: source_branch_sha, - target_project: project, - target_branch: pull_request.target_branch_name, - target_branch_sha: target_branch_sha, - state: pull_request.state, - author_id: gitlab_user_id(project, pull_request.author), - created_at: pull_request.created_at, - updated_at: pull_request.updated_at - ) - - mark_as_imported(:pull_requests, pull_request.iid) - - metrics.merge_requests_counter.increment - - import_pull_request_comments(pull_request, merge_request) if merge_request.persisted? - rescue StandardError => e - store_pull_request_error(pull_request, e) - end - - def import_inline_comments(inline_comments, pull_request, merge_request) - position_map = {} - discussion_map = {} - - children, parents = inline_comments.partition(&:has_parent?) - - # The Bitbucket API returns threaded replies as parent-child - # relationships. We assume that the child can appear in any order in - # the JSON. - parents.each do |comment| - position_map[comment.iid] = build_position(merge_request, comment) - end - - children.each do |comment| - position_map[comment.iid] = position_map.fetch(comment.parent_id, nil) - end - - inline_comments.each do |comment| - attributes = pull_request_comment_attributes(comment) - attributes[:discussion_id] = discussion_map[comment.parent_id] if comment.has_parent? - - attributes.merge!( - position: position_map[comment.iid], - type: 'DiffNote') - - note = merge_request.notes.create!(attributes) - - # We can't store a discussion ID until a note is created, so if - # replies are created before the parent the discussion ID won't be - # linked properly. - discussion_map[comment.iid] = note.discussion_id - rescue StandardError => e - errors << { type: :pull_request, iid: comment.iid, errors: e.message } - end - end - - def build_position(merge_request, pr_comment) - params = { - diff_refs: merge_request.diff_refs, - old_path: pr_comment.file_path, - new_path: pr_comment.file_path, - old_line: pr_comment.old_pos, - new_line: pr_comment.new_pos - } - - Gitlab::Diff::Position.new(params) - end - - def import_standalone_pr_comments(pr_comments, merge_request) - pr_comments.each do |comment| - merge_request.notes.create!(pull_request_comment_attributes(comment)) - rescue StandardError => e - errors << { type: :pull_request, iid: comment.iid, errors: e.message } - end - end - - def pull_request_comment_attributes(comment) - { - project: project, - author_id: gitlab_user_id(project, comment.author), - note: comment_note(comment), - created_at: comment.created_at, - updated_at: comment.updated_at - } - end - - def comment_note(comment) - author = @formatter.author_line(comment.author) unless find_user_id(comment.author) - author.to_s + @ref_converter.convert_note(comment.note.to_s) - end - - def log_base_data - { - class: self.class.name, - project_id: project.id, - project_path: project.full_path - } - end - - def metrics - @metrics ||= Gitlab::Import::Metrics.new(:bitbucket_importer, @project) - end - end - end -end diff --git a/lib/gitlab/bitbucket_import/importers/issues_importer.rb b/lib/gitlab/bitbucket_import/importers/issues_importer.rb index 8ab82ddb0be..678cb4e129d 100644 --- a/lib/gitlab/bitbucket_import/importers/issues_importer.rb +++ b/lib/gitlab/bitbucket_import/importers/issues_importer.rb @@ -33,6 +33,7 @@ module Gitlab job_waiter rescue StandardError => e track_import_failure!(project, exception: e) + job_waiter end private diff --git a/lib/gitlab/bitbucket_import/importers/issues_notes_importer.rb b/lib/gitlab/bitbucket_import/importers/issues_notes_importer.rb index 03dcc645f07..ecc41cc5436 100644 --- a/lib/gitlab/bitbucket_import/importers/issues_notes_importer.rb +++ b/lib/gitlab/bitbucket_import/importers/issues_notes_importer.rb @@ -22,6 +22,7 @@ module Gitlab job_waiter rescue StandardError => e track_import_failure!(project, exception: e) + job_waiter end private diff --git a/lib/gitlab/bitbucket_import/importers/pull_request_importer.rb b/lib/gitlab/bitbucket_import/importers/pull_request_importer.rb index f7b1753a9f9..37bbc1d0c78 100644 --- a/lib/gitlab/bitbucket_import/importers/pull_request_importer.rb +++ b/lib/gitlab/bitbucket_import/importers/pull_request_importer.rb @@ -15,6 +15,8 @@ module Gitlab end def execute + return if skip + log_info(import_stage: 'import_pull_request', message: 'starting', iid: object[:iid]) description = '' @@ -58,6 +60,15 @@ module Gitlab attr_reader :object, :project, :formatter, :user_finder + def skip + return false unless object[:source_and_target_project_different] + + message = 'skipping because source and target projects are different' + log_info(import_stage: 'import_pull_request', message: message, iid: object[:iid]) + + true + end + def author_line return '' if find_user_id diff --git a/lib/gitlab/bitbucket_import/importers/pull_requests_importer.rb b/lib/gitlab/bitbucket_import/importers/pull_requests_importer.rb index 1c7ce7f2f3a..eedb89c2d49 100644 --- a/lib/gitlab/bitbucket_import/importers/pull_requests_importer.rb +++ b/lib/gitlab/bitbucket_import/importers/pull_requests_importer.rb @@ -26,6 +26,7 @@ module Gitlab job_waiter rescue StandardError => e track_import_failure!(project, exception: e) + job_waiter end private diff --git a/lib/gitlab/bitbucket_import/importers/pull_requests_notes_importer.rb b/lib/gitlab/bitbucket_import/importers/pull_requests_notes_importer.rb index a1b0c2a5afe..1dc3c6fbfc1 100644 --- a/lib/gitlab/bitbucket_import/importers/pull_requests_notes_importer.rb +++ b/lib/gitlab/bitbucket_import/importers/pull_requests_notes_importer.rb @@ -22,6 +22,7 @@ module Gitlab job_waiter rescue StandardError => e track_import_failure!(project, exception: e) + job_waiter end private diff --git a/lib/gitlab/bitbucket_import/importers/repository_importer.rb b/lib/gitlab/bitbucket_import/importers/repository_importer.rb index 9be7ed99436..cc950bbe13d 100644 --- a/lib/gitlab/bitbucket_import/importers/repository_importer.rb +++ b/lib/gitlab/bitbucket_import/importers/repository_importer.rb @@ -6,6 +6,11 @@ module Gitlab class RepositoryImporter include Loggable + LABELS = [{ title: 'bug', color: '#FF0000' }, + { title: 'enhancement', color: '#428BCA' }, + { title: 'proposal', color: '#69D100' }, + { title: 'task', color: '#7F8C8D' }].freeze + def initialize(project) @project = project end @@ -62,8 +67,9 @@ module Gitlab end def create_labels - importer = Gitlab::BitbucketImport::Importer.new(project) - importer.create_labels + LABELS.each do |label_params| + ::Labels::FindOrCreateService.new(nil, project, label_params).execute(skip_authorization: true) + end end def wiki diff --git a/lib/gitlab/bitbucket_import/ref_converter.rb b/lib/gitlab/bitbucket_import/ref_converter.rb index 1159159a76d..1763bd26d61 100644 --- a/lib/gitlab/bitbucket_import/ref_converter.rb +++ b/lib/gitlab/bitbucket_import/ref_converter.rb @@ -4,7 +4,7 @@ module Gitlab module BitbucketImport class RefConverter REPO_MATCHER = 'https://bitbucket.org/%s' - PR_NOTE_ISSUE_NAME_REGEX = '(?<=/)[^/\)]+(?=\)[^/]*$)' + PR_NOTE_ISSUE_NAME_REGEX = "(issues\/.*\/(.*)\\))" UNWANTED_NOTE_REF_HTML = "{: data-inline-card='' }" attr_reader :project @@ -24,7 +24,7 @@ module Gitlab if note.match?('issues') note.gsub!('issues', '-/issues') - note.gsub!(issue_name(note), '') + note.gsub!("/#{issue_name(note)}", '') if issue_name(note) else note.gsub!('pull-requests', '-/merge_requests') note.gsub!('src', '-/blob') @@ -41,7 +41,11 @@ module Gitlab end def issue_name(note) - note.match(PR_NOTE_ISSUE_NAME_REGEX)[0] + match_data = note.match(PR_NOTE_ISSUE_NAME_REGEX) + + return unless match_data + + match_data[2] end end end diff --git a/lib/gitlab/bitbucket_server_import/importers/pull_request_notes_importer.rb b/lib/gitlab/bitbucket_server_import/importers/pull_request_notes_importer.rb index 69de47e2006..d58f7cec8ff 100644 --- a/lib/gitlab/bitbucket_server_import/importers/pull_request_notes_importer.rb +++ b/lib/gitlab/bitbucket_server_import/importers/pull_request_notes_importer.rb @@ -4,21 +4,19 @@ module Gitlab module BitbucketServerImport module Importers class PullRequestNotesImporter + include ::Gitlab::Import::MergeRequestHelpers include Loggable def initialize(project, hash) @project = project - @formatter = Gitlab::ImportFormatter.new - @client = BitbucketServer::Client.new(project.import_data.credentials) - @project_key = project.import_data.data['project_key'] - @repository_slug = project.import_data.data['repo_slug'] @user_finder = UserFinder.new(project) - - # TODO: Convert object into a object instead of using it as a hash + @formatter = Gitlab::ImportFormatter.new @object = hash.with_indifferent_access end def execute + return unless import_data_valid? + log_info(import_stage: 'import_pull_request_notes', message: 'starting', iid: object[:iid]) merge_request = project.merge_requests.find_by(iid: object[:iid]) # rubocop: disable CodeReuse/ActiveRecord @@ -35,6 +33,9 @@ module Gitlab import_inline_comments(inline_comments.map(&:comment), merge_request) import_standalone_pr_comments(pr_comments.map(&:comment), merge_request) + + approved_events = other_activities.select(&:approved_event?) + approved_events.each { |event| import_approved_event(merge_request, event) } end log_info(import_stage: 'import_pull_request_notes', message: 'finished', iid: object[:iid]) @@ -42,7 +43,11 @@ module Gitlab private - attr_reader :object, :project, :formatter, :client, :project_key, :repository_slug, :user_finder + attr_reader :object, :project, :formatter, :user_finder + + def import_data_valid? + project.import_data&.credentials && project.import_data&.data + end # rubocop: disable CodeReuse/ActiveRecord def import_merge_event(merge_request, merge_event) @@ -60,6 +65,32 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord + def import_approved_event(merge_request, approved_event) + log_info( + import_stage: 'import_approved_event', + message: 'starting', + iid: merge_request.iid, + event_id: approved_event.id + ) + + user_id = user_finder.find_user_id(by: :username, value: approved_event.approver_username) || + user_finder.find_user_id(by: :email, value: approved_event.approver_email) + + return unless user_id + + submitted_at = approved_event.created_at || merge_request.updated_at + + create_approval!(project.id, merge_request.id, user_id, submitted_at) + create_reviewer!(merge_request.id, user_id, submitted_at) + + log_info( + import_stage: 'import_approved_event', + message: 'finished', + iid: merge_request.iid, + event_id: approved_event.id + ) + end + def import_inline_comments(inline_comments, merge_request) log_info(import_stage: 'import_inline_comments', message: 'starting', iid: merge_request.iid) @@ -177,6 +208,18 @@ module Gitlab updated_at: comment.updated_at } end + + def client + BitbucketServer::Client.new(project.import_data.credentials) + end + + def project_key + project.import_data.data['project_key'] + end + + def repository_slug + project.import_data.data['repo_slug'] + end end end end diff --git a/lib/gitlab/bitbucket_server_import/importers/pull_requests_importer.rb b/lib/gitlab/bitbucket_server_import/importers/pull_requests_importer.rb index ae73681f7f8..61c31fb9644 100644 --- a/lib/gitlab/bitbucket_server_import/importers/pull_requests_importer.rb +++ b/lib/gitlab/bitbucket_server_import/importers/pull_requests_importer.rb @@ -6,16 +6,20 @@ module Gitlab class PullRequestsImporter include ParallelScheduling + # Reduce fetch limit (from 100) to avoid Gitlab::Git::ResourceExhaustedError + PULL_REQUESTS_BATCH_SIZE = 50 + def execute page = 1 loop do log_info( - import_stage: 'import_pull_requests', message: "importing page #{page} using batch-size #{BATCH_SIZE}" + import_stage: 'import_pull_requests', + message: "importing page #{page} using batch-size #{PULL_REQUESTS_BATCH_SIZE}" ) pull_requests = client.pull_requests( - project_key, repository_slug, page_offset: page, limit: BATCH_SIZE + project_key, repository_slug, page_offset: page, limit: PULL_REQUESTS_BATCH_SIZE ).to_a break if pull_requests.empty? @@ -24,7 +28,15 @@ module Gitlab next if already_processed?(pull_request) next unless pull_request.merged? || pull_request.closed? - [pull_request.source_branch_sha, pull_request.target_branch_sha] + [].tap do |commits| + source_sha = pull_request.source_branch_sha + target_sha = pull_request.target_branch_sha + + existing_commits = repo.commits_by(oids: [source_sha, target_sha]).map(&:sha) + + commits << source_branch_commit(source_sha, pull_request) unless existing_commits.include?(source_sha) + commits << target_branch_commit(target_sha) unless existing_commits.include?(target_sha) + end end.flatten # Bitbucket Server keeps tracks of references for open pull requests in @@ -78,6 +90,22 @@ module Gitlab def id_for_already_processed_cache(object) object.iid end + + def repo + @repo ||= project.repository + end + + def ref_path(pull_request) + "refs/#{Repository::REF_MERGE_REQUEST}/#{pull_request.iid}/head" + end + + def source_branch_commit(source_branch_sha, pull_request) + [source_branch_sha, ':', ref_path(pull_request)].join + end + + def target_branch_commit(target_branch_sha) + [target_branch_sha, ':refs/keep-around/', target_branch_sha].join + end end end end diff --git a/lib/gitlab/bitbucket_server_import/importers/users_importer.rb b/lib/gitlab/bitbucket_server_import/importers/users_importer.rb new file mode 100644 index 00000000000..f8d0521afb2 --- /dev/null +++ b/lib/gitlab/bitbucket_server_import/importers/users_importer.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketServerImport + module Importers + class UsersImporter + include Loggable + include UserCaching + + BATCH_SIZE = 100 + + def initialize(project) + @project = project + @project_id = project.id + end + + attr_reader :project, :project_id + + def execute + log_info(import_stage: 'import_users', message: 'starting') + + page = 1 + + loop do + log_info( + import_stage: 'import_users', + message: "importing page #{page} using batch size #{BATCH_SIZE}" + ) + + users = client.users(project_key, page_offset: page, limit: BATCH_SIZE).to_a + + break if users.empty? + + cache_users(users) + + page += 1 + end + + log_info(import_stage: 'import_users', message: 'finished') + end + + private + + def cache_users(users) + users_hash = users.each_with_object({}) do |user, hash| + cache_key = source_user_cache_key(project_id, user.username) + hash[cache_key] = user.email + end + + ::Gitlab::Cache::Import::Caching.write_multiple(users_hash) + end + + def client + @client ||= BitbucketServer::Client.new(project.import_data.credentials) + end + + def project_key + project.import_data.data['project_key'] + end + end + end + end +end diff --git a/lib/gitlab/bitbucket_server_import/user_caching.rb b/lib/gitlab/bitbucket_server_import/user_caching.rb new file mode 100644 index 00000000000..0f0169122c5 --- /dev/null +++ b/lib/gitlab/bitbucket_server_import/user_caching.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketServerImport + module UserCaching + SOURCE_USER_CACHE_KEY = 'bitbucket_server/project/%s/source/username/%s' + + def source_user_cache_key(project_id, username) + format(SOURCE_USER_CACHE_KEY, project_id, username) + end + end + end +end diff --git a/lib/gitlab/cache/import/caching.rb b/lib/gitlab/cache/import/caching.rb index 8f2df29c320..e81a90831f7 100644 --- a/lib/gitlab/cache/import/caching.rb +++ b/lib/gitlab/cache/import/caching.rb @@ -138,7 +138,7 @@ module Gitlab key = cache_key_for(raw_key) - Redis::Cache.with do |redis| + with_redis do |redis| redis.sismember(key, value) end end @@ -244,7 +244,14 @@ module Gitlab end def self.with_redis(&block) - Redis::Cache.with(&block) # rubocop:disable CodeReuse/ActiveRecord + block_result = Gitlab::Redis::Cache.with(&block) # rubocop:disable CodeReuse/ActiveRecord -- This is not AR + cache_identity = Gitlab::Redis::Cache.with(&:inspect) # rubocop:disable CodeReuse/ActiveRecord -- This is not AR + + Gitlab::Redis::SharedState.with do |redis| + yield redis unless cache_identity == redis.default_store.inspect + end + + block_result end def self.validate_redis_value!(value) diff --git a/lib/gitlab/checks/global_file_size_check.rb b/lib/gitlab/checks/global_file_size_check.rb index ff24467e9cc..5dc41b2a4cc 100644 --- a/lib/gitlab/checks/global_file_size_check.rb +++ b/lib/gitlab/checks/global_file_size_check.rb @@ -3,6 +3,8 @@ module Gitlab module Checks class GlobalFileSizeCheck < BaseBulkChecker + include ActionView::Helpers::NumberHelper + LOG_MESSAGE = 'Checking for blobs over the file size limit' def validate! @@ -17,31 +19,24 @@ module Gitlab ).find if oversized_blobs.present? - - blob_details = {} - blob_id_size_msg = "" - oversized_blobs.each do |blob| - blob_details[blob.id] = { "size" => blob.size } - - # blob size is in byte, divide it by "/ 1024.0 / 1024.0" to get MiB - blob_id_size_msg += "- #{blob.id} (#{(blob.size / 1024.0 / 1024.0).round(2)} MiB) \n" - end + blob_id_size_msg = oversized_blobs.map do |blob| + "- #{blob.id} (#{number_to_human_size(blob.size)})" + end.join("\n") oversize_err_msg = <<~OVERSIZE_ERR_MSG - You are attempting to check in one or more blobs which exceed the #{file_size_limit}MiB limit: - - #{blob_id_size_msg} - To resolve this error, you must either reduce the size of the above blobs, or utilize LFS. - You may use "git ls-tree -r HEAD | grep $BLOB_ID" to see the file path. - Please refer to #{Rails.application.routes.url_helpers.help_page_url('user/free_push_limit')} and - #{Rails.application.routes.url_helpers.help_page_url('administration/settings/account_and_limit_settings')} - for further information. + You are attempting to check in one or more blobs which exceed the #{file_size_limit}MiB limit: + + #{blob_id_size_msg} + To resolve this error, you must either reduce the size of the above blobs, or utilize LFS. + You may use "git ls-tree -r HEAD | grep $BLOB_ID" to see the file path. + Please refer to #{Rails.application.routes.url_helpers.help_page_url('user/free_push_limit')} and + #{Rails.application.routes.url_helpers.help_page_url('administration/settings/account_and_limit_settings')} + for further information. OVERSIZE_ERR_MSG Gitlab::AppJsonLogger.info( message: 'Found blob over global limit', - blob_sizes: oversized_blobs.map(&:size), - blob_details: blob_details + blob_details: oversized_blobs.map { |blob| { "id" => blob.id, "size" => blob.size } } ) raise ::Gitlab::GitAccess::ForbiddenError, oversize_err_msg if enforce_global_file_size_limit? diff --git a/lib/gitlab/checks/tag_check.rb b/lib/gitlab/checks/tag_check.rb index cdc648bf005..cb0e60a096a 100644 --- a/lib/gitlab/checks/tag_check.rb +++ b/lib/gitlab/checks/tag_check.rb @@ -6,8 +6,8 @@ module Gitlab ERROR_MESSAGES = { change_existing_tags: 'You are not allowed to change existing tags on this project.', update_protected_tag: 'Protected tags cannot be updated.', - delete_protected_tag: 'You are not allowed to delete protected tags from this project. '\ - 'Only a project maintainer or owner can delete a protected tag.', + delete_protected_tag: 'You are not allowed to delete protected tags from this project. ' \ + 'Only a project maintainer or owner can delete a protected tag.', delete_protected_tag_non_web: 'You can only delete protected tags using the web interface.', create_protected_tag: 'You are not allowed to create this tag as it is protected.', default_branch_collision: 'You cannot use default branch name to create a tag', @@ -27,70 +27,86 @@ module Gitlab def validate! return unless tag_name - logger.log_timed(LOG_MESSAGES[:tag_checks]) do - if tag_exists? && user_access.cannot_do_action?(:admin_tag) - raise GitAccess::ForbiddenError, ERROR_MESSAGES[:change_existing_tags] - end - end - - default_branch_collision_check + logger.log_timed(LOG_MESSAGES[:tag_checks]) { tag_checks } + logger.log_timed(LOG_MESSAGES[:default_branch_collision_check]) { default_branch_collision_check } prohibited_tag_checks - protected_tag_checks + logger.log_timed(LOG_MESSAGES[:protected_tag_checks]) { protected_tag_checks } end private - def prohibited_tag_checks - return if deletion? + def tag_checks + return unless tag_exists? && user_access.cannot_do_action?(:admin_tag) - unless Gitlab::GitRefValidator.validate(tag_name) - raise GitAccess::ForbiddenError, ERROR_MESSAGES[:prohibited_tag_name] - end + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:change_existing_tags] + end - if tag_name.start_with?("refs/tags/") # rubocop: disable Style/GuardClause - raise GitAccess::ForbiddenError, ERROR_MESSAGES[:prohibited_tag_name] - end + def default_branch_collision_check + return unless creation? && tag_name == project.default_branch - # rubocop: disable Style/GuardClause - # rubocop: disable Style/SoleNestedConditional - if Feature.enabled?(:prohibited_tag_name_encoding_check, project) - unless Gitlab::EncodingHelper.force_encode_utf8(tag_name).valid_encoding? - raise GitAccess::ForbiddenError, ERROR_MESSAGES[:prohibited_tag_name_encoding] - end - end - # rubocop: enable Style/SoleNestedConditional - # rubocop: enable Style/GuardClause + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:default_branch_collision] + end + + def prohibited_tag_checks + return if deletion? + + # Incorrectly encoded tags names may raise during other checks so we + # need to validate the encoding first + validate_encoding! + validate_valid_tag_name! + validate_tag_name_not_fully_qualified! validate_tag_name_not_sha_like! end def protected_tag_checks - logger.log_timed(LOG_MESSAGES[__method__]) do - return unless ProtectedTag.protected?(project, tag_name) # rubocop:disable Cop/AvoidReturnFromBlocks + return unless ProtectedTag.protected?(project, tag_name) - raise(GitAccess::ForbiddenError, ERROR_MESSAGES[:update_protected_tag]) if update? + validate_protected_tag_update! + validate_protected_tag_deletion! + validate_protected_tag_creation! + end - if deletion? - unless user_access.user.can?(:maintainer_access, project) - raise(GitAccess::ForbiddenError, ERROR_MESSAGES[:delete_protected_tag]) - end + def validate_encoding! + return unless Feature.enabled?(:prohibited_tag_name_encoding_check, project) + return if Gitlab::EncodingHelper.force_encode_utf8(tag_name).valid_encoding? - unless updated_from_web? - raise GitAccess::ForbiddenError, ERROR_MESSAGES[:delete_protected_tag_non_web] - end - end + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:prohibited_tag_name_encoding] + end - unless user_access.can_create_tag?(tag_name) - raise GitAccess::ForbiddenError, ERROR_MESSAGES[:create_protected_tag] - end - end + def validate_valid_tag_name! + return if Gitlab::GitRefValidator.validate(tag_name) + + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:prohibited_tag_name] end - def default_branch_collision_check - logger.log_timed(LOG_MESSAGES[:default_branch_collision_check]) do - if creation? && tag_name == project.default_branch - raise GitAccess::ForbiddenError, ERROR_MESSAGES[:default_branch_collision] - end + def validate_tag_name_not_fully_qualified! + return unless tag_name.start_with?("refs/tags/") + + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:prohibited_tag_name] + end + + def validate_protected_tag_update! + return unless update? + + raise(GitAccess::ForbiddenError, ERROR_MESSAGES[:update_protected_tag]) + end + + def validate_protected_tag_deletion! + return unless deletion? + + unless user_access.user.can?(:maintainer_access, project) + raise(GitAccess::ForbiddenError, ERROR_MESSAGES[:delete_protected_tag]) end + + return if updated_from_web? + + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:delete_protected_tag_non_web] + end + + def validate_protected_tag_creation! + return if user_access.can_create_tag?(tag_name) + + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:create_protected_tag] end def validate_tag_name_not_sha_like! diff --git a/lib/gitlab/ci/build/image.rb b/lib/gitlab/ci/build/image.rb index 84f8eae8deb..660d7701a8f 100644 --- a/lib/gitlab/ci/build/image.rb +++ b/lib/gitlab/ci/build/image.rb @@ -4,7 +4,7 @@ module Gitlab module Ci module Build class Image - attr_reader :alias, :command, :entrypoint, :name, :ports, :variables, :pull_policy + attr_reader :alias, :command, :entrypoint, :name, :ports, :variables, :executor_opts, :pull_policy class << self def from_image(job) @@ -28,6 +28,7 @@ module Gitlab when String @name = image @ports = [] + @executor_opts = {} when Hash @alias = image[:alias] @command = image[:command] @@ -35,6 +36,7 @@ module Gitlab @name = image[:name] @ports = build_ports(image).select(&:valid?) @variables = build_variables(image) + @executor_opts = image.fetch(:executor_opts, {}) @pull_policy = image[:pull_policy] end end diff --git a/lib/gitlab/ci/components/instance_path.rb b/lib/gitlab/ci/components/instance_path.rb index 50731d54fc0..607eff902ea 100644 --- a/lib/gitlab/ci/components/instance_path.rb +++ b/lib/gitlab/ci/components/instance_path.rb @@ -45,14 +45,31 @@ module Gitlab private - attr_reader :version + attr_reader :version, :component_name - # Given a path like "my-org/sub-group/the-project/path/to/component" - # find the project "my-org/sub-group/the-project" by looking at all possible paths. def find_project_by_component_path(path) + if Feature.enabled?(:ci_redirect_component_project, Feature.current_request) + project_full_path = extract_project_path(path) + + Project.find_by_full_path(project_full_path, follow_redirects: true).tap do |project| + next unless project + + @component_name = extract_component_name(project_full_path) + end + else + legacy_finder(path).tap do |project| + next unless project + + @component_name = extract_component_name(project.full_path) + end + end + end + + def legacy_finder(path) return if path.start_with?('/') # exit early if path starts with `/` or it will loop forever. possible_paths = [path] + index = nil loop_until(limit: 20) do @@ -68,17 +85,32 @@ module Gitlab ::Project.where_full_path_in(possible_paths).take # rubocop: disable CodeReuse/ActiveRecord end + # Given a path like "my-org/sub-group/the-project/the-component" + # we expect that the last `/` is the separator between the project full path and the + # component name. + def extract_project_path(path) + return if path.start_with?('/') # invalid project full path. + + index = path.rindex('/') # find index of last `/` in the path + return unless index + + path[0..index - 1] + end + def instance_path @full_path.delete_prefix(host) end - def component_name - instance_path.delete_prefix(project.full_path).delete_prefix('/') + def extract_component_name(project_path) + instance_path.delete_prefix(project_path).delete_prefix('/') end - strong_memoize_attr :component_name def latest_version_sha - project.releases.latest&.sha + if project.catalog_resource + project.catalog_resource.versions.latest&.sha + else + project.releases.latest&.sha + end end end end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 73d329930a5..16e4e473928 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -19,13 +19,14 @@ module Gitlab Config::Yaml::Tags::TagError ].freeze - attr_reader :root, :context, :source_ref_path, :source, :logger + attr_reader :root, :context, :source_ref_path, :source, :logger, :inject_edge_stages # rubocop: disable Metrics/ParameterLists - def initialize(config, project: nil, pipeline: nil, sha: nil, user: nil, parent_pipeline: nil, source: nil, pipeline_config: nil, logger: nil) + def initialize(config, project: nil, pipeline: nil, sha: nil, user: nil, parent_pipeline: nil, source: nil, pipeline_config: nil, logger: nil, inject_edge_stages: true) @logger = logger || ::Gitlab::Ci::Pipeline::Logger.new(project: project) @source_ref_path = pipeline&.source_ref_path @project = project + @inject_edge_stages = inject_edge_stages @context = self.logger.instrument(:config_build_context, once: true) do pipeline ||= ::Ci::Pipeline.new(project: project, sha: sha, user: user, source: source) @@ -99,6 +100,10 @@ module Gitlab root.workflow_entry.name end + def workflow_auto_cancel + root.workflow_entry.auto_cancel_value + end + def normalized_jobs @normalized_jobs ||= Ci::Config::Normalizer.new(jobs).normalize_jobs end @@ -145,6 +150,8 @@ module Gitlab Config::Yaml::Tags::Resolver.new(initial_config).to_hash end + return initial_config unless inject_edge_stages + logger.instrument(:config_stages_inject, once: true) do Config::EdgeStagesInjector.new(initial_config).to_hash end diff --git a/lib/gitlab/ci/config/entry/auto_cancel.rb b/lib/gitlab/ci/config/entry/auto_cancel.rb new file mode 100644 index 00000000000..2c51ab82214 --- /dev/null +++ b/lib/gitlab/ci/config/entry/auto_cancel.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + class AutoCancel < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Attributable + include ::Gitlab::Config::Entry::Validatable + + ALLOWED_KEYS = %i[on_new_commit on_job_failure].freeze + ALLOWED_ON_NEW_COMMIT_OPTIONS = ::Ci::PipelineMetadata.auto_cancel_on_new_commits.keys.freeze + ALLOWED_ON_JOB_FAILURE_OPTIONS = ::Ci::PipelineMetadata.auto_cancel_on_job_failures.keys.freeze + + attributes ALLOWED_KEYS + + validations do + validates :config, type: Hash, allowed_keys: ALLOWED_KEYS + validates :on_new_commit, allow_nil: true, type: String, inclusion: { + in: ALLOWED_ON_NEW_COMMIT_OPTIONS, + message: format(_("must be one of: %{values}"), values: ALLOWED_ON_NEW_COMMIT_OPTIONS.join(', ')) + } + validates :on_job_failure, allow_nil: true, type: String, inclusion: { + in: ALLOWED_ON_JOB_FAILURE_OPTIONS, + message: format(_("must be one of: %{values}"), values: ALLOWED_ON_JOB_FAILURE_OPTIONS.join(', ')) + } + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/image.rb b/lib/gitlab/ci/config/entry/image.rb index 84e31ca1fc6..58ab488d833 100644 --- a/lib/gitlab/ci/config/entry/image.rb +++ b/lib/gitlab/ci/config/entry/image.rb @@ -13,21 +13,6 @@ module Gitlab validations do validates :config, allowed_keys: IMAGEABLE_ALLOWED_KEYS end - - def value - if string? - { name: @config } - elsif hash? - { - name: @config[:name], - entrypoint: @config[:entrypoint], - ports: (ports_value if ports_defined?), - pull_policy: pull_policy_value - }.compact - else - {} - end - end end end end diff --git a/lib/gitlab/ci/config/entry/imageable.rb b/lib/gitlab/ci/config/entry/imageable.rb index 1aecfee9ab9..53b810b3037 100644 --- a/lib/gitlab/ci/config/entry/imageable.rb +++ b/lib/gitlab/ci/config/entry/imageable.rb @@ -12,7 +12,9 @@ module Gitlab include ::Gitlab::Config::Entry::Attributable include ::Gitlab::Config::Entry::Configurable - IMAGEABLE_ALLOWED_KEYS = %i[name entrypoint ports pull_policy].freeze + EXECUTOR_OPTS_KEYS = %i[docker].freeze + + IMAGEABLE_ALLOWED_KEYS = EXECUTOR_OPTS_KEYS + %i[name entrypoint ports pull_policy].freeze included do include ::Gitlab::Config::Entry::Validatable @@ -23,9 +25,15 @@ module Gitlab validates :name, type: String, presence: true validates :entrypoint, array_of_strings: true, allow_nil: true + validates :executor_opts, json_schema: { + base_directory: "lib/gitlab/ci/config/entry/schemas/imageable", + detail_errors: true, + filename: "executor_opts", + hash_conversion: true + }, allow_nil: true end - attributes :ports, :pull_policy + attributes :docker, :ports, :pull_policy entry :ports, Entry::Ports, description: 'Ports used to expose the image/service' @@ -49,6 +57,28 @@ module Gitlab def skip_config_hash_validation? true end + + def executor_opts + return unless config.is_a?(Hash) + + config.slice(*EXECUTOR_OPTS_KEYS).compact.presence + end + + def value + if string? + { name: config } + elsif hash? + { + name: config[:name], + entrypoint: config[:entrypoint], + executor_opts: executor_opts, + ports: (ports_value if ports_defined?), + pull_policy: pull_policy_value + }.compact + else + {} + end + end end end end diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 5fcafcba829..7ea4b460640 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -13,7 +13,7 @@ module Gitlab ALLOWED_WHEN = %w[on_success on_failure always manual delayed].freeze ALLOWED_KEYS = %i[tags script image services start_in artifacts cache dependencies before_script after_script hooks - coverage retry parallel interruptible timeout + coverage retry parallel timeout release id_tokens publish pages].freeze validations do @@ -83,10 +83,6 @@ module Gitlab description: 'Services that will be used to execute this job.', inherit: true - entry :interruptible, ::Gitlab::Config::Entry::Boolean, - description: 'Set jobs interruptible value.', - inherit: true - entry :timeout, Entry::Timeout, description: 'Timeout duration of this job.', inherit: true @@ -139,7 +135,7 @@ module Gitlab attributes :script, :tags, :when, :dependencies, :needs, :retry, :parallel, :start_in, - :interruptible, :timeout, :release, + :timeout, :release, :allow_failure, :publish, :pages def self.matching?(name, config) @@ -169,7 +165,6 @@ module Gitlab coverage: coverage_defined? ? coverage_value : nil, retry: retry_defined? ? retry_value : nil, parallel: has_parallel? ? parallel_value : nil, - interruptible: interruptible_defined? ? interruptible_value : nil, timeout: parsed_timeout, artifacts: artifacts_value, release: release_value, diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index d0e9a9afc51..0b322fd433c 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -15,7 +15,8 @@ module Gitlab include ::Gitlab::Config::Entry::Inheritable PROCESSABLE_ALLOWED_KEYS = %i[extends stage only except rules variables - inherit allow_failure when needs resource_group environment].freeze + inherit allow_failure when needs resource_group environment + interruptible].freeze MAX_NESTING_LEVEL = 10 included do @@ -74,6 +75,10 @@ module Gitlab description: 'Environment configuration for this job.', inherit: false + entry :interruptible, ::Gitlab::Config::Entry::Boolean, + description: 'Set jobs interruptible value.', + inherit: true + attributes :extends, :rules, :resource_group end @@ -133,7 +138,8 @@ module Gitlab except: except_value, environment: environment_defined? ? environment_value : nil, environment_name: environment_defined? ? environment_value[:name] : nil, - resource_group: resource_group }.compact + resource_group: resource_group, + interruptible: interruptible_defined? ? interruptible_value : nil }.compact end def root_variables_inheritance diff --git a/lib/gitlab/ci/config/entry/release.rb b/lib/gitlab/ci/config/entry/release.rb index 2be0eae120b..113e6fefd6a 100644 --- a/lib/gitlab/ci/config/entry/release.rb +++ b/lib/gitlab/ci/config/entry/release.rb @@ -16,12 +16,6 @@ module Gitlab attributes %i[tag_name tag_message name ref milestones assets].freeze attr_reader :released_at - # Attributable description conflicts with - # ::Gitlab::Config::Entry::Node.description - def has_description? - true - end - def description config[:description] end diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb index 3c180674f2a..16755ac320c 100644 --- a/lib/gitlab/ci/config/entry/reports.rb +++ b/lib/gitlab/ci/config/entry/reports.rb @@ -17,7 +17,7 @@ module Gitlab dast performance browser_performance load_performance license_scanning metrics lsif dotenv terraform accessibility coverage_fuzzing api_fuzzing cluster_image_scanning - requirements requirements_v2 coverage_report cyclonedx annotations].freeze + requirements requirements_v2 coverage_report cyclonedx annotations repository_xray].freeze attributes ALLOWED_KEYS @@ -51,6 +51,7 @@ module Gitlab validates :requirements_v2, array_of_strings_or_string: true validates :cyclonedx, array_of_strings_or_string: true validates :annotations, array_of_strings_or_string: true + validates :repository_xray, array_of_strings_or_string: true end end diff --git a/lib/gitlab/ci/config/entry/retry.rb b/lib/gitlab/ci/config/entry/retry.rb index e9cbcb31e21..f225ed4caf4 100644 --- a/lib/gitlab/ci/config/entry/retry.rb +++ b/lib/gitlab/ci/config/entry/retry.rb @@ -35,8 +35,8 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[max when].freeze - attributes :max, :when + ALLOWED_KEYS = %i[max when exit_codes].freeze + attributes ALLOWED_KEYS validations do validates :config, allowed_keys: ALLOWED_KEYS @@ -53,6 +53,7 @@ module Gitlab validates :when, inclusion: { in: FullRetry.possible_retry_when_values }, if: -> (config) { config.when.is_a?(String) } + validates :exit_codes, array_of_integers_or_integer: true end end @@ -62,9 +63,14 @@ module Gitlab def value super.tap do |config| - # make sure that `when` is an array, because we allow it to - # be passed as a String in config for simplicity + # make sure that `when` and `exit_codes` are arrays, because we allow them to + # be passed as a String/Integer in config for simplicity config[:when] = Array.wrap(config[:when]) if config[:when] + if config[:exit_codes] && Feature.enabled?(:ci_retry_on_exit_codes, Feature.current_request) + config[:exit_codes] = Array.wrap(config[:exit_codes]) + else + config.delete(:exit_codes) + end end end diff --git a/lib/gitlab/ci/config/entry/schemas/imageable/executor_opts.json b/lib/gitlab/ci/config/entry/schemas/imageable/executor_opts.json new file mode 100644 index 00000000000..a31374650e6 --- /dev/null +++ b/lib/gitlab/ci/config/entry/schemas/imageable/executor_opts.json @@ -0,0 +1,19 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "Describe `image:` and `service:` options like `docker:`", + "type": "object", + "properties": { + "docker": { + "type": "object", + "properties": { + "platform": { + "type": "string", + "minLength": 1, + "maxLength": 64 + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false +} diff --git a/lib/gitlab/ci/config/entry/service.rb b/lib/gitlab/ci/config/entry/service.rb index 4b3a9990df4..94fd28badb7 100644 --- a/lib/gitlab/ci/config/entry/service.rb +++ b/lib/gitlab/ci/config/entry/service.rb @@ -34,14 +34,14 @@ module Gitlab end def value - if string? - { name: @config } - elsif hash? - @config.merge( - pull_policy: pull_policy_value + if hash? + super.merge( + command: @config[:command], + alias: @config[:alias], + variables: (variables_value if variables_defined?) ).compact else - {} + super end end end diff --git a/lib/gitlab/ci/config/entry/workflow.rb b/lib/gitlab/ci/config/entry/workflow.rb index 691d9e2d48b..5b81c74fe4d 100644 --- a/lib/gitlab/ci/config/entry/workflow.rb +++ b/lib/gitlab/ci/config/entry/workflow.rb @@ -9,7 +9,7 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[rules name].freeze + ALLOWED_KEYS = %i[rules name auto_cancel].freeze attributes :name @@ -23,6 +23,9 @@ module Gitlab description: 'List of evaluable Rules to determine Pipeline status.', metadata: { allowed_when: %w[always never] } + entry :auto_cancel, Entry::AutoCancel, + description: 'Auto-cancel configuration for this pipeline.' + def has_rules? @config.try(:key?, :rules) end diff --git a/lib/gitlab/ci/config/external/file/remote.rb b/lib/gitlab/ci/config/external/file/remote.rb index bc8cebb8c3e..fc90b497f85 100644 --- a/lib/gitlab/ci/config/external/file/remote.rb +++ b/lib/gitlab/ci/config/external/file/remote.rb @@ -14,9 +14,20 @@ module Gitlab super end + def preload_content + fetch_async_content + end + def content - strong_memoize(:content) { fetch_remote_content } + fetch_with_error_handling do + if fetch_async_content + fetch_async_content.value + else + fetch_sync_content + end + end end + strong_memoize_attr :content def metadata super.merge( @@ -42,11 +53,23 @@ module Gitlab private - def fetch_remote_content + def fetch_async_content + return if ::Feature.disabled?(:ci_parallel_remote_includes, context.project) + + # It starts fetching the remote content in a separate thread and returns a promise immediately. + Gitlab::HTTP.get(location, async: true).execute + end + strong_memoize_attr :fetch_async_content + + def fetch_sync_content + context.logger.instrument(:config_file_fetch_remote_content) do + Gitlab::HTTP.get(location) + end + end + + def fetch_with_error_handling begin - response = context.logger.instrument(:config_file_fetch_remote_content) do - Gitlab::HTTP.get(location) - end + response = yield rescue SocketError errors.push("Remote file `#{masked_location}` could not be fetched because of a socket error!") rescue Timeout::Error diff --git a/lib/gitlab/ci/config/external/mapper/verifier.rb b/lib/gitlab/ci/config/external/mapper/verifier.rb index 0e296aa0b5b..3bb0df88803 100644 --- a/lib/gitlab/ci/config/external/mapper/verifier.rb +++ b/lib/gitlab/ci/config/external/mapper/verifier.rb @@ -25,7 +25,7 @@ module Gitlab file.preload_context if file.valid? end - # We do not combine the loops because we need to load the context of all files via `BatchLoader`. + # We do not combine the loops because we need to preload the context of all files via `BatchLoader`. files.each do |file| # rubocop:disable Style/CombinableLoops verify_execution_time! @@ -33,7 +33,8 @@ module Gitlab file.preload_content if file.valid? end - # We do not combine the loops because we need to load the content of all files via `BatchLoader`. + # We do not combine the loops because we need to preload the content of all files via `BatchLoader` + # or `Concurrent::Promise`. files.each do |file| # rubocop:disable Style/CombinableLoops verify_execution_time! diff --git a/lib/gitlab/ci/config/interpolation/inputs/base_input.rb b/lib/gitlab/ci/config/interpolation/inputs/base_input.rb index 987268b0525..e506645df11 100644 --- a/lib/gitlab/ci/config/interpolation/inputs/base_input.rb +++ b/lib/gitlab/ci/config/interpolation/inputs/base_input.rb @@ -10,9 +10,8 @@ module Gitlab class BaseInput ArgumentNotValidError = Class.new(StandardError) - # Checks whether the class matches the type in the specification def self.matches?(spec) - raise NotImplementedError + spec.is_a?(Hash) && spec[:type] == type_name end # Human readable type used in error messages diff --git a/lib/gitlab/ci/config/interpolation/inputs/boolean_input.rb b/lib/gitlab/ci/config/interpolation/inputs/boolean_input.rb index 4c34f7e7fdd..51845a2fea8 100644 --- a/lib/gitlab/ci/config/interpolation/inputs/boolean_input.rb +++ b/lib/gitlab/ci/config/interpolation/inputs/boolean_input.rb @@ -8,10 +8,6 @@ module Gitlab class BooleanInput < BaseInput extend ::Gitlab::Utils::Override - def self.matches?(spec) - spec.is_a?(Hash) && spec[:type] == type_name - end - def self.type_name 'boolean' end diff --git a/lib/gitlab/ci/config/interpolation/inputs/number_input.rb b/lib/gitlab/ci/config/interpolation/inputs/number_input.rb index 59bc057749a..bb023a8a85b 100644 --- a/lib/gitlab/ci/config/interpolation/inputs/number_input.rb +++ b/lib/gitlab/ci/config/interpolation/inputs/number_input.rb @@ -8,10 +8,6 @@ module Gitlab class NumberInput < BaseInput extend ::Gitlab::Utils::Override - def self.matches?(spec) - spec.is_a?(Hash) && spec[:type] == type_name - end - def self.type_name 'number' end diff --git a/lib/gitlab/ci/config/interpolation/inputs/string_input.rb b/lib/gitlab/ci/config/interpolation/inputs/string_input.rb index 01b9d34a883..3c4868b299c 100644 --- a/lib/gitlab/ci/config/interpolation/inputs/string_input.rb +++ b/lib/gitlab/ci/config/interpolation/inputs/string_input.rb @@ -17,7 +17,7 @@ module Gitlab # inputs: # foo: # ``` - spec.nil? || (spec.is_a?(Hash) && [nil, type_name].include?(spec[:type])) + spec.nil? || super || (spec.is_a?(Hash) && !spec.key?(:type)) end def self.type_name diff --git a/lib/gitlab/ci/config/interpolation/text_interpolator.rb b/lib/gitlab/ci/config/interpolation/text_interpolator.rb new file mode 100644 index 00000000000..5c4953f8bbe --- /dev/null +++ b/lib/gitlab/ci/config/interpolation/text_interpolator.rb @@ -0,0 +1,102 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Interpolation + ## + # Performs CI config file interpolation and either returns the interpolated result or interpolation errors. + # + class TextInterpolator + attr_reader :errors + + def initialize(config, input_args, variables) + @config = config + @input_args = input_args.to_h + @variables = variables + @errors = [] + @interpolated = false + end + + def valid? + errors.none? + end + + def to_result + @result + end + + def error_message + # Interpolator can have multiple error messages, like: ["interpolation interrupted by errors", "unknown + # interpolation key: `abc`"] ? + # + # We are joining them together into a single one, because only one error can be surfaced when an external + # file gets included and is invalid. The limit to three error messages combined is more than required. + # + errors.first(3).join(', ') + end + + def interpolate! + return errors.push(config.error) unless config.valid? + + if inputs_without_header? + return errors.push( + _('Given inputs not defined in the `spec` section of the included configuration file')) + end + + return @result ||= config.content unless config.has_header? + + return errors.concat(header.errors) unless header.valid? + return errors.concat(inputs.errors) unless inputs.valid? + return errors.concat(context.errors) unless context.valid? + return errors.concat(template.errors) unless template.valid? + + @interpolated = true + + @result ||= template.interpolated + end + + def interpolated? + @interpolated + end + + private + + attr_reader :config, :input_args, :variables + + def inputs_without_header? + input_args.any? && !config.has_header? + end + + def header + @header ||= Header::Root.new(config.header).tap do |header| + header.key = 'header' + + header.compose! + end + end + + def content + @content ||= config.content + end + + def spec + @spec ||= header.inputs_value + end + + def inputs + @inputs ||= Inputs.new(spec, input_args) + end + + def context + @context ||= Context.new({ inputs: inputs.to_hash }, variables: variables) + end + + def template + @template ||= TextTemplate.new(content, context) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/interpolation/text_template.rb b/lib/gitlab/ci/config/interpolation/text_template.rb new file mode 100644 index 00000000000..e1f5d368e88 --- /dev/null +++ b/lib/gitlab/ci/config/interpolation/text_template.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Interpolation + class TextTemplate + MAX_BLOCKS = 10_000 + + def initialize(content, ctx) + @content = content + @ctx = Interpolation::Context.fabricate(ctx) + @errors = [] + @blocks = {} + + interpolate! if valid? + end + + def valid? + errors.none? + end + + def errors + @errors + ctx.errors + blocks.values.flat_map(&:errors) + end + + def interpolated + @result if valid? + end + + private + + attr_reader :blocks, :content, :ctx + + def interpolate! + return @errors.push('config too large') if content.bytesize > max_total_yaml_size_bytes + + @result = Interpolation::Block.match(content) do |matched, data| + block = (blocks[matched] ||= Interpolation::Block.new(matched, data, ctx)) + + break @errors.push('too many interpolation blocks') if blocks.count > MAX_BLOCKS + break unless block.valid? + + if block.value.is_a?(String) + block.value + else + block.value.to_json + end + end + end + + def max_total_yaml_size_bytes + Gitlab::CurrentSettings.current_application_settings.ci_max_total_yaml_size_bytes + end + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers/sbom/cyclonedx.rb b/lib/gitlab/ci/parsers/sbom/cyclonedx.rb index 1e5200e8682..79c1c14dc4e 100644 --- a/lib/gitlab/ci/parsers/sbom/cyclonedx.rb +++ b/lib/gitlab/ci/parsers/sbom/cyclonedx.rb @@ -5,8 +5,6 @@ module Gitlab module Parsers module Sbom class Cyclonedx - SUPPORTED_SPEC_VERSIONS = %w[1.4].freeze - def parse!(blob, sbom_report) @report = sbom_report @data = Gitlab::Json.parse(blob) @@ -27,18 +25,7 @@ module Gitlab end def valid? - valid_schema? && supported_spec_version? - end - - def supported_spec_version? - return true if SUPPORTED_SPEC_VERSIONS.include?(data['specVersion']) - - report.add_error( - "Unsupported CycloneDX spec version. Must be one of: %{versions}" \ - % { versions: SUPPORTED_SPEC_VERSIONS.join(', ') } - ) - - false + valid_schema? end def valid_schema? diff --git a/lib/gitlab/ci/parsers/sbom/validators/cyclonedx_schema_validator.rb b/lib/gitlab/ci/parsers/sbom/validators/cyclonedx_schema_validator.rb index 9d56e001c2f..a8d3ef1d6b5 100644 --- a/lib/gitlab/ci/parsers/sbom/validators/cyclonedx_schema_validator.rb +++ b/lib/gitlab/ci/parsers/sbom/validators/cyclonedx_schema_validator.rb @@ -6,7 +6,9 @@ module Gitlab module Sbom module Validators class CyclonedxSchemaValidator - SCHEMA_PATH = Rails.root.join('app', 'validators', 'json_schemas', 'cyclonedx_report.json').freeze + SUPPORTED_SPEC_VERSIONS = %w[1.4 1.5].freeze + + SCHEMA_BASE_PATH = Rails.root.join('app', 'validators', 'json_schemas', 'cyclonedx').freeze def initialize(report_data) @report_data = report_data @@ -17,13 +19,30 @@ module Gitlab end def errors - @errors ||= pretty_errors + @errors ||= validate! end private + def validate! + if spec_version_valid? + pretty_errors + else + [format("Unsupported CycloneDX spec version. Must be one of: %{versions}", + versions: SUPPORTED_SPEC_VERSIONS.join(', '))] + end + end + + def spec_version_valid? + SUPPORTED_SPEC_VERSIONS.include?(spec_version) + end + + def spec_version + @report_data['specVersion'] + end + def raw_errors - JSONSchemer.schema(SCHEMA_PATH).validate(@report_data) + JSONSchemer.schema(SCHEMA_BASE_PATH.join("bom-#{spec_version}.schema.json")).validate(@report_data) end def pretty_errors diff --git a/lib/gitlab/ci/pipeline/chain/assign_partition.rb b/lib/gitlab/ci/pipeline/chain/assign_partition.rb index 4b8efe13d44..0740226ac9b 100644 --- a/lib/gitlab/ci/pipeline/chain/assign_partition.rb +++ b/lib/gitlab/ci/pipeline/chain/assign_partition.rb @@ -21,7 +21,7 @@ module Gitlab if @command.creates_child_pipeline? @command.parent_pipeline_partition_id else - ::Ci::Pipeline.current_partition_value + ::Ci::Pipeline.current_partition_value(project) end end end diff --git a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb index dcaaefee98f..14ec86c5d62 100644 --- a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb +++ b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb @@ -6,7 +6,11 @@ module Gitlab module Chain class CancelPendingPipelines < Chain::Base def perform! - ::Ci::CancelRedundantPipelinesWorker.perform_async(pipeline.id) + if pipeline.schedule? + ::Ci::LowUrgencyCancelRedundantPipelinesWorker.perform_async(pipeline.id) + else + ::Ci::CancelRedundantPipelinesWorker.perform_async(pipeline.id) + end end def break? diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index d4c4f94c7d3..d1153a0990e 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -34,7 +34,7 @@ module Gitlab pipeline .stages .flat_map(&:statuses) - .select { |status| status.respond_to?(:tag_list) } + .select { |status| status.respond_to?(:tag_list=) } end end end diff --git a/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb b/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb index cceaa52de16..ab37eb93f18 100644 --- a/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb +++ b/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb @@ -11,7 +11,16 @@ module Gitlab def perform! @command.workflow_rules_result = workflow_rules_result - error('Pipeline filtered out by workflow rules.') unless workflow_passed? + return if workflow_passed? + + if Feature.enabled?(:always_set_pipeline_failure_reason, @command.project) + drop_reason = :filtered_by_workflow_rules + end + + error( + 'Pipeline filtered out by workflow rules.', + drop_reason: drop_reason + ) end def break? diff --git a/lib/gitlab/ci/pipeline/chain/helpers.rb b/lib/gitlab/ci/pipeline/chain/helpers.rb index 343a189f773..0e55928ff80 100644 --- a/lib/gitlab/ci/pipeline/chain/helpers.rb +++ b/lib/gitlab/ci/pipeline/chain/helpers.rb @@ -35,7 +35,7 @@ module Gitlab def drop_pipeline!(drop_reason) return if pipeline.readonly? - if drop_reason && command.save_incompleted + if Enums::Ci::Pipeline.persistable_failure_reason?(drop_reason) && command.save_incompleted # Project iid must be called outside a transaction, so we ensure it is set here # otherwise it may be set within the state transition transaction of the drop! call # which it will lock the InternalId row for the whole transaction @@ -44,6 +44,8 @@ module Gitlab pipeline.drop!(drop_reason) else command.increment_pipeline_failure_reason_counter(drop_reason) + + pipeline.set_failed(drop_reason) if Feature.enabled?(:always_set_pipeline_failure_reason, command.project) end end end diff --git a/lib/gitlab/ci/pipeline/chain/populate.rb b/lib/gitlab/ci/pipeline/chain/populate.rb index c59ef2ba6a4..f73addcd098 100644 --- a/lib/gitlab/ci/pipeline/chain/populate.rb +++ b/lib/gitlab/ci/pipeline/chain/populate.rb @@ -18,8 +18,15 @@ module Gitlab pipeline.stages = @command.pipeline_seed.stages if stage_names.empty? - return error('Pipeline will not run for the selected trigger. ' \ - 'The rules configuration prevented any jobs from being added to the pipeline.') + if Feature.enabled?(:always_set_pipeline_failure_reason, @command.project) + drop_reason = :filtered_by_rules + end + + return error( + 'Pipeline will not run for the selected trigger. ' \ + 'The rules configuration prevented any jobs from being added to the pipeline.', + drop_reason: drop_reason + ) end if pipeline.invalid? diff --git a/lib/gitlab/ci/pipeline/chain/populate_metadata.rb b/lib/gitlab/ci/pipeline/chain/populate_metadata.rb index e7a9009f8f4..3ac910da752 100644 --- a/lib/gitlab/ci/pipeline/chain/populate_metadata.rb +++ b/lib/gitlab/ci/pipeline/chain/populate_metadata.rb @@ -9,6 +9,8 @@ module Gitlab def perform! set_pipeline_name + set_auto_cancel + return if pipeline.pipeline_metadata.nil? || pipeline.pipeline_metadata.valid? message = pipeline.pipeline_metadata.errors.full_messages.join(', ') @@ -29,13 +31,45 @@ module Gitlab return if name.blank? - pipeline.build_pipeline_metadata(project: pipeline.project, name: name.strip) + assign_to_metadata(name: name.strip) + end + + def set_auto_cancel + auto_cancel = @command.yaml_processor_result.workflow_auto_cancel + + return if auto_cancel.blank? + + set_auto_cancel_on_new_commit(auto_cancel) + set_auto_cancel_on_job_failure(auto_cancel) + end + + def set_auto_cancel_on_new_commit(auto_cancel) + auto_cancel_on_new_commit = auto_cancel[:on_new_commit] + + return if auto_cancel_on_new_commit.blank? + + assign_to_metadata(auto_cancel_on_new_commit: auto_cancel_on_new_commit) + end + + def set_auto_cancel_on_job_failure(auto_cancel) + return if Feature.disabled?(:auto_cancel_pipeline_on_job_failure, pipeline.project) + + auto_cancel_on_job_failure = auto_cancel[:on_job_failure] + + return if auto_cancel_on_job_failure.blank? + + assign_to_metadata(auto_cancel_on_job_failure: auto_cancel_on_job_failure) end def global_context Gitlab::Ci::Build::Context::Global.new( pipeline, yaml_variables: @command.pipeline_seed.root_variables) end + + def assign_to_metadata(attributes) + metadata = pipeline.pipeline_metadata || pipeline.build_pipeline_metadata(project: pipeline.project) + metadata.assign_attributes(attributes) + end end end end diff --git a/lib/gitlab/ci/reports/sbom/source.rb b/lib/gitlab/ci/reports/sbom/source.rb index b7af6ea17c3..7d284b5babf 100644 --- a/lib/gitlab/ci/reports/sbom/source.rb +++ b/lib/gitlab/ci/reports/sbom/source.rb @@ -5,28 +5,14 @@ module Gitlab module Reports module Sbom class Source + include SourceHelper + attr_reader :source_type, :data def initialize(type:, data:) @source_type = type @data = data end - - def source_file_path - data.dig('source_file', 'path') - end - - def input_file_path - data.dig('input_file', 'path') - end - - def packager - data.dig('package_manager', 'name') - end - - def language - data.dig('language', 'name') - end end end end diff --git a/lib/gitlab/ci/reports/sbom/source_helper.rb b/lib/gitlab/ci/reports/sbom/source_helper.rb new file mode 100644 index 00000000000..49b606f658b --- /dev/null +++ b/lib/gitlab/ci/reports/sbom/source_helper.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Sbom + module SourceHelper + def source_file_path + data.dig('source_file', 'path') + end + + def input_file_path + data.dig('input_file', 'path') + end + + def packager + data.dig('package_manager', 'name') + end + + def language + data.dig('language', 'name') + end + + def image_name + data.dig('image', 'name') + end + + def image_tag + data.dig('image', 'tag') + end + + def operating_system_name + data.dig('operating_system', 'name') + end + + def operating_system_version + data.dig('operating_system', 'version') + end + end + end + end + end +end diff --git a/lib/gitlab/ci/templates/Diffblue-Cover.gitlab-ci.yml b/lib/gitlab/ci/templates/Diffblue-Cover.gitlab-ci.yml new file mode 100644 index 00000000000..c8b3aa1d705 --- /dev/null +++ b/lib/gitlab/ci/templates/Diffblue-Cover.gitlab-ci.yml @@ -0,0 +1,88 @@ +# This template is provided and maintained by Diffblue. +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# This template is designed to be used with the Cover Pipeline for GitLab integration from Diffblue. +# It will download the latest version of Diffblue Cover, build the associated project, and +# automatically write Java unit tests for the project. +# Note that additional config is required: +# https://docs.diffblue.com/features/cover-pipeline/cover-pipeline-for-gitlab +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Diffblue-Cover.gitlab-ci.yml + +variables: + # Configure the following via the Diffblue Cover integration config for your project, or by + # using CI/CD masked Variables. + # For details, see https://docs.diffblue.com/features/cover-pipeline/cover-pipeline-for-gitlab + + # Diffblue Cover license key: DIFFBLUE_LICENSE_KEY + # Refer to your welcome email or you can obtain a free trial key from + # https://www.diffblue.com/try-cover/gitlab + + # GitLab access token: DIFFBLUE_ACCESS_TOKEN, DIFFBLUE_ACCESS_TOKEN_NAME + # The access token should have a role of Developer or better and should have + # api and write_repository permissions. + + # Diffblue Cover requires a minimum of 4GB of memory. + JVM_ARGS: -Xmx4g + +stages: + - build + +diffblue-cover: + stage: build + + # Select the Cover CLI docker image to use with your CI tool. + # Tag variations are produced for each supported JDK version. + # Go to https://hub.docker.com/r/diffblue/cover-cli for details. + # Note: To use the latest version of Diffblue Cover, use one of the latest-jdk<nn> tags. + # To use a specific release version, use one of the yyyy.mm.dd-jdk<nn> tags. + image: diffblue/cover-cli:latest-jdk17 + + # Diffblue Cover currently only supports running on merge_request_events. + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + + # Diffblue Cover log files are saved to a .diffblue/ directory in the pipeline artifacts, + # and are available for download once the pipeline completes. + artifacts: + paths: + - "**/.diffblue/" + + script: + + # Diffblue Cover requires the project to be built before creating any tests. + # Either specify the build command here (one of the following), or provide + # prebuilt artifacts via a job dependency. + + # Maven project example (comment out the Gradle version if used): + - mvn test-compile --batch-mode --no-transfer-progress + + # Gradle project example (comment out the Maven version if used): + # - gradle testClasses + + # Diffblue Cover commands and options to run. + # dcover – the core Diffblue Cover command + # ci – enable the GitLab CI/CD integration via environment variables + # activate - activate the license key + # validate - remove non-compiling and failing tests + # create - create new tests for your project + # --maven – use the maven build tool + # For detailed information on Cover CLI commands and options, see + # https://docs.diffblue.com/features/cover-cli/commands-and-arguments + - dcover + ci + activate + validate --maven + create --maven + + # Diffblue Cover will also respond to specific project labels: + # Diffblue Cover: Baseline + # Used to mark a merge request as requiring a full suite of tests to be written. + # This overrides the default behaviour where Cover will only write tests related + # to the code changes already in the merge request. This is useful when running Diffblue + # Cover for the first time on a project and when new product enhancements are released. + # Diffblue Cover: Skip + # Used to mark a merge request as requiring no tests to be written. diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index 6898923bc53..111df0af67a 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.49.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.51.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml index 6898923bc53..111df0af67a 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.49.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.51.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index 7d923245d79..a5cddf5d2d7 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.60.0' + DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.71.0' .dast-auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 0f8d5bf6d8f..0a899f3bb74 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.60.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.71.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index e29d18ea45a..87a7f79c0ce 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.60.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.71.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml index 488b035d189..c698bd49140 100644 --- a/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml @@ -3,7 +3,7 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml -# Read more about the feature here: https://docs.gitlab.com/ee/user/project/merge_requests/accessibility_testing.html +# Read more about the feature here: https://docs.gitlab.com/ee/ci/testing/accessibility_testing.html stages: - build - test diff --git a/lib/gitlab/ci/variables/builder.rb b/lib/gitlab/ci/variables/builder.rb index c279af6acfc..a1c6437bf84 100644 --- a/lib/gitlab/ci/variables/builder.rb +++ b/lib/gitlab/ci/variables/builder.rb @@ -34,6 +34,25 @@ module Gitlab end end + def unprotected_scoped_variables(job, expose_project_variables:, expose_group_variables:, environment:, dependencies:) + Gitlab::Ci::Variables::Collection.new.tap do |variables| + variables.concat(predefined_variables(job, environment)) + variables.concat(project.predefined_variables) + variables.concat(pipeline_variables_builder.predefined_variables) + variables.concat(job.runner.predefined_variables) if job.runnable? && job.runner + variables.concat(kubernetes_variables(environment: environment, job: job)) + variables.concat(job.yaml_variables) + variables.concat(user_variables(job.user)) + variables.concat(job.dependency_variables) if dependencies + variables.concat(secret_instance_variables) + variables.concat(secret_group_variables(environment: environment, include_protected_vars: expose_group_variables)) + variables.concat(secret_project_variables(environment: environment, include_protected_vars: expose_project_variables)) + variables.concat(pipeline.variables) + variables.concat(pipeline_schedule_variables) + variables.concat(release_variables) + end + end + def config_variables Gitlab::Ci::Variables::Collection.new.tap do |variables| break variables unless project @@ -91,21 +110,21 @@ module Gitlab end end - def secret_group_variables(environment:) - strong_memoize_with(:secret_group_variables, environment) do + def secret_group_variables(environment:, include_protected_vars: protected_ref?) + strong_memoize_with(:secret_group_variables, environment, include_protected_vars) do group_variables_builder .secret_variables( environment: environment, - protected_ref: protected_ref?) + protected_ref: include_protected_vars) end end - def secret_project_variables(environment:) - strong_memoize_with(:secret_project_variables, environment) do + def secret_project_variables(environment:, include_protected_vars: protected_ref?) + strong_memoize_with(:secret_project_variables, environment, include_protected_vars) do project_variables_builder .secret_variables( environment: environment, - protected_ref: protected_ref?) + protected_ref: include_protected_vars) end end @@ -183,3 +202,5 @@ module Gitlab end end end + +Gitlab::Ci::Variables::Builder.prepend_mod_with('Gitlab::Ci::Variables::Builder') diff --git a/lib/gitlab/ci/variables/downstream/generator.rb b/lib/gitlab/ci/variables/downstream/generator.rb index 350d29958cf..e1fd8200dd6 100644 --- a/lib/gitlab/ci/variables/downstream/generator.rb +++ b/lib/gitlab/ci/variables/downstream/generator.rb @@ -5,8 +5,6 @@ module Gitlab module Variables module Downstream class Generator - include Gitlab::Utils::StrongMemoize - Context = Struct.new(:all_bridge_variables, :expand_file_refs, keyword_init: true) def initialize(bridge) @@ -33,6 +31,7 @@ module Gitlab # The order of this list refers to the priority of the variables # The variables added later takes priority. downstream_yaml_variables + + downstream_pipeline_dotenv_variables + downstream_pipeline_variables + downstream_pipeline_schedule_variables end @@ -57,6 +56,13 @@ module Gitlab build_downstream_variables_from(pipeline_schedule_variables) end + def downstream_pipeline_dotenv_variables + return [] unless bridge.forward_pipeline_variables? + + pipeline_dotenv_variables = bridge.dependency_variables.to_a + build_downstream_variables_from(pipeline_dotenv_variables) + end + def build_downstream_variables_from(variables) Gitlab::Ci::Variables::Collection.fabricate(variables).flat_map do |item| if item.raw? diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index 2435d128bf2..5933b537098 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -8,7 +8,8 @@ module Gitlab class Result attr_reader :errors, :warnings, :root_variables, :root_variables_with_prefill_data, - :stages, :jobs, :workflow_rules, :workflow_name + :stages, :jobs, + :workflow_rules, :workflow_name, :workflow_auto_cancel def initialize(ci_config: nil, errors: [], warnings: []) @ci_config = ci_config @@ -71,6 +72,7 @@ module Gitlab @workflow_rules = @ci_config.workflow_rules @workflow_name = @ci_config.workflow_name&.strip + @workflow_auto_cancel = @ci_config.workflow_auto_cancel end def stage_builds_attributes(stage) diff --git a/lib/gitlab/circuit_breaker.rb b/lib/gitlab/circuit_breaker.rb new file mode 100644 index 00000000000..2b3a6187f27 --- /dev/null +++ b/lib/gitlab/circuit_breaker.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +# A configurable circuit breaker to protect the application from external service failures. +# The circuit measures the amount of failures and if the threshold is exceeded, stops sending requests. +module Gitlab + module CircuitBreaker + InternalServerError = Class.new(StandardError) + + DEFAULT_ERROR_THRESHOLD = 50 + DEFAULT_VOLUME_THRESHOLD = 10 + + class << self + include ::Gitlab::Utils::StrongMemoize + + # @param [String] unique name for the circuit + # @param options [Hash] an options hash setting optional values per circuit + def run_with_circuit(service_name, options = {}, &block) + circuit(service_name, options).run(exception: false, &block) + end + + private + + def circuit(service_name, options) + strong_memoize_with(:circuit, service_name, options) do + circuit_options = { + exceptions: [InternalServerError], + error_threshold: DEFAULT_ERROR_THRESHOLD, + volume_threshold: DEFAULT_VOLUME_THRESHOLD + }.merge(options) + + Circuitbox.circuit(service_name, circuit_options) + end + end + end + end +end diff --git a/lib/gitlab/circuit_breaker/notifier.rb b/lib/gitlab/circuit_breaker/notifier.rb new file mode 100644 index 00000000000..b555158ee48 --- /dev/null +++ b/lib/gitlab/circuit_breaker/notifier.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module CircuitBreaker + class Notifier + CircuitBreakerError = Class.new(RuntimeError) + + def notify(service_name, event) + return unless event == 'failure' + + exception = CircuitBreakerError.new("Service #{service_name}: #{event}") + exception.set_backtrace(Gitlab::BacktraceCleaner.clean_backtrace(caller)) + + Gitlab::ErrorTracking.track_exception(exception) + end + + def notify_warning(_service_name, _message) + # no-op + end + + def notify_run(_service_name, &_block) + # This gets called by Circuitbox::CircuitBreaker#run to actually execute + # the block passed. + yield + end + end + end +end diff --git a/lib/gitlab/circuit_breaker/store.rb b/lib/gitlab/circuit_breaker/store.rb new file mode 100644 index 00000000000..0ba4f08d5e1 --- /dev/null +++ b/lib/gitlab/circuit_breaker/store.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Gitlab + module CircuitBreaker + class Store + def key?(key) + with { |redis| redis.exists?(key) } + end + + def store(key, value, opts = {}) + with do |redis| + redis.set(key, value, ex: opts[:expires]) + value + end + end + + def increment(key, amount = 1, opts = {}) + expires = opts[:expires] + + with do |redis| + redis.multi do |multi| + multi.incrby(key, amount) + multi.expire(key, expires) if expires + end + end + end + + def load(key, _opts = {}) + with { |redis| redis.get(key) } + end + + def values_at(*keys, **_opts) + keys.map! { |key| load(key) } + end + + def delete(key) + with { |redis| redis.del(key) } + end + + private + + def with(&block) + Gitlab::Redis::RateLimiting.with(&block) + rescue ::Redis::BaseConnectionError + # Do not raise an error if we cannot connect to Redis. If + # Redis::RateLimiting is unavailable it should not take the site down. + nil + end + end + end +end diff --git a/lib/gitlab/cleanup/project_uploads.rb b/lib/gitlab/cleanup/project_uploads.rb index 6feaab2a791..918f723cd60 100644 --- a/lib/gitlab/cleanup/project_uploads.rb +++ b/lib/gitlab/cleanup/project_uploads.rb @@ -122,7 +122,7 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def project_id - @project_id ||= Project.where_full_path_in([full_path]).pluck(:id) + @project_id ||= Project.where_full_path_in([full_path], use_includes: false).pluck(:id) end # rubocop: enable CodeReuse/ActiveRecord end diff --git a/lib/gitlab/content_security_policy/directives.rb b/lib/gitlab/content_security_policy/directives.rb index e293e5653c7..3df0ec76839 100644 --- a/lib/gitlab/content_security_policy/directives.rb +++ b/lib/gitlab/content_security_policy/directives.rb @@ -12,11 +12,11 @@ module Gitlab end def self.frame_src - "https://www.google.com/recaptcha/ https://www.recaptcha.net/ https://content.googleapis.com https://content-compute.googleapis.com https://content-cloudbilling.googleapis.com https://content-cloudresourcemanager.googleapis.com https://www.googletagmanager.com/ns.html" + "https://www.google.com/recaptcha/ https://www.recaptcha.net/ https://www.googletagmanager.com/ns.html" end def self.script_src - "'strict-dynamic' 'self' 'unsafe-inline' 'unsafe-eval' https://www.google.com/recaptcha/ https://www.recaptcha.net https://apis.google.com" + "'strict-dynamic' 'self' 'unsafe-inline' 'unsafe-eval' https://www.google.com/recaptcha/ https://www.recaptcha.net" end def self.style_src diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index 2068a9ae7d5..a0bb37fb097 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -3,6 +3,7 @@ module Gitlab class ContributionsCalendar include TimeZoneHelper + include ::Gitlab::Utils::StrongMemoize attr_reader :contributor attr_reader :current_user @@ -16,93 +17,89 @@ module Gitlab .execute(current_user, ignore_visibility: @contributor.include_private_contributions?) end - # rubocop: disable CodeReuse/ActiveRecord def activity_dates - return {} if @projects.empty? - return @activity_dates if @activity_dates.present? + return {} if projects.empty? start_time = @contributor_time_instance.years_ago(1).beginning_of_day end_time = @contributor_time_instance.end_of_day date_interval = "INTERVAL '#{@contributor_time_instance.utc_offset} seconds'" - # Can't use Event.contributions here because we need to check 3 different - # project_features for the (currently) 3 different contribution types - repo_events = events_created_between(start_time, end_time, :repository) - .where(action: :pushed) - issue_events = events_created_between(start_time, end_time, :issues) - .where(action: [:created, :closed], target_type: %w[Issue WorkItem]) - mr_events = events_created_between(start_time, end_time, :merge_requests) - .where(action: [:merged, :created, :closed], target_type: "MergeRequest") - note_events = events_created_between(start_time, end_time, :merge_requests) - .where(action: :commented) - - events = Event - .select("date(created_at + #{date_interval}) AS date", 'COUNT(*) AS num_events') - .from_union([repo_events, issue_events, mr_events, note_events], remove_duplicates: false) - .group(:date) - .map(&:attributes) - - @activity_dates = events.each_with_object(Hash.new { |h, k| h[k] = 0 }) do |event, activities| - activities[event["date"]] += event["num_events"] - end + contributions_between(start_time, end_time).count_by_dates(date_interval) end - # rubocop: enable CodeReuse/ActiveRecord - # rubocop: disable CodeReuse/ActiveRecord def events_by_date(date) return Event.none unless can_read_cross_project? date_in_time_zone = date.in_time_zone(@contributor_time_instance.time_zone) - Event.contributions.where(author_id: contributor.id) - .where(created_at: date_in_time_zone.beginning_of_day..date_in_time_zone.end_of_day) - .where(project_id: projects) - .with_associations + contributions_between(date_in_time_zone.beginning_of_day, date_in_time_zone.end_of_day).with_associations end - # rubocop: enable CodeReuse/ActiveRecord - def starting_year - @contributor_time_instance.years_ago(1).year - end + private - def starting_month - @contributor_time_instance.month + def contributions_between(start_time, end_time) + # Can't use Event.contributions here because we need to check 3 different + # project_features for the (currently) 4 different contribution types + repo_events = + project_events_created_between(start_time, end_time, features: :repository) + .for_action(:pushed) + + issue_events = + project_events_created_between(start_time, end_time, features: :issues) + .for_issue + .for_action(%i[created closed]) + + mr_events = + project_events_created_between(start_time, end_time, features: :merge_requests) + .for_merge_request + .for_action(%i[merged created closed approved]) + + note_events = + project_events_created_between(start_time, end_time, features: %i[issues merge_requests]) + .for_action(:commented) + + Event.from_union([repo_events, issue_events, mr_events, note_events], remove_duplicates: false) end - private - def can_read_cross_project? Ability.allowed?(current_user, :read_cross_project) end - # rubocop: disable CodeReuse/ActiveRecord - def events_created_between(start_time, end_time, feature) + # rubocop: disable CodeReuse/ActiveRecord -- no need to move this to ActiveRecord model + def project_events_created_between(start_time, end_time, features:) + Array(features).reduce(Event.none) do |events, feature| + events.or(contribution_events(start_time, end_time).where(project_id: authed_projects(feature))) + end + end + # rubocop: enable CodeReuse/ActiveRecord + + def authed_projects(feature) + strong_memoize("#{feature}_projects") do + # no need to check features access of current user, if the contributor opted-in + # to show all private events anyway - otherwise they would get filtered out again + next contributed_project_ids if contributor.include_private_contributions? + + # rubocop: disable CodeReuse/ActiveRecord -- no need to move this to ActiveRecord model + ProjectFeature + .with_feature_available_for_user(feature, current_user) + .where(project_id: contributed_project_ids) + .pluck(:project_id) + # rubocop: enable CodeReuse/ActiveRecord + end + end + + # rubocop: disable CodeReuse/ActiveRecord -- no need to move this to ActiveRecord model + def contributed_project_ids # re-running the contributed projects query in each union is expensive, so # use IN(project_ids...) instead. It's the intersection of two users so # the list will be (relatively) short @contributed_project_ids ||= projects.distinct.pluck(:id) - - # no need to check feature access of current user, if the contributor opted-in - # to show all private events anyway - otherwise they would get filtered out again - authed_projects = if @contributor.include_private_contributions? - @contributed_project_ids - else - ProjectFeature - .with_feature_available_for_user(feature, current_user) - .where(project_id: @contributed_project_ids) - .reorder(nil) - .select(:project_id) - end - - Event.reorder(nil) - .select(:created_at) - .where( - author_id: contributor.id, - created_at: start_time..end_time, - events: { project_id: authed_projects } - ) end # rubocop: enable CodeReuse/ActiveRecord + + def contribution_events(start_time, end_time) + contributor.events.created_between(start_time, end_time) + end end end diff --git a/lib/gitlab/counters/buffered_counter.rb b/lib/gitlab/counters/buffered_counter.rb index 258ada864c8..9d704f5613c 100644 --- a/lib/gitlab/counters/buffered_counter.rb +++ b/lib/gitlab/counters/buffered_counter.rb @@ -248,7 +248,7 @@ module Gitlab end def redis_state(&block) - Gitlab::Redis::SharedState.with(&block) + Gitlab::Redis::BufferedCounter.with(&block) end def with_exclusive_lease(&block) diff --git a/lib/gitlab/database/background_migration/batched_background_migration_dictionary.rb b/lib/gitlab/database/background_migration/batched_background_migration_dictionary.rb new file mode 100644 index 00000000000..a6efa09afda --- /dev/null +++ b/lib/gitlab/database/background_migration/batched_background_migration_dictionary.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module BackgroundMigration + class BatchedBackgroundMigrationDictionary + def self.entry(migration_job_name) + entries_by_migration_job_name[migration_job_name] + end + + private_class_method def self.entries_by_migration_job_name + @entries_by_migration_job_name ||= Dir.glob(dict_path).to_h do |file_path| + entry = Entry.new(file_path) + [entry.migration_job_name, entry] + end + end + + private_class_method def self.dict_path + Rails.root.join('db/docs/batched_background_migrations/*.yml') + end + + class Entry + def initialize(file_path) + @file_path = file_path + @data = YAML.load_file(file_path) + end + + def migration_job_name + data['migration_job_name'] + end + + def finalized_by + data['finalized_by'] + end + + private + + attr_reader :file_path, :data + end + end + end + end +end diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb index 83beee091f1..d0655fa4564 100644 --- a/lib/gitlab/database/background_migration/batched_migration.rb +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -264,6 +264,13 @@ module Gitlab 100 * migrated_tuple_count / total_tuple_count end + def finalize_command + <<~SCRIPT.delete("\n").squeeze(' ').strip + sudo gitlab-rake gitlab:background_migrations:finalize + [#{job_class_name},#{table_name},#{column_name},'#{job_arguments.to_json.gsub(',', '\,')}'] + SCRIPT + end + private def validate_batched_jobs_status diff --git a/lib/gitlab/database/decomposition/migrate.rb b/lib/gitlab/database/decomposition/migrate.rb new file mode 100644 index 00000000000..b6ca5adf857 --- /dev/null +++ b/lib/gitlab/database/decomposition/migrate.rb @@ -0,0 +1,181 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Decomposition + MigrateError = Class.new(RuntimeError) + + class Migrate + TABLE_SIZE_QUERY = <<-SQL + select sum(pg_table_size(concat(table_schema,'.',table_name))) as total + from information_schema.tables + where table_catalog = :table_catalog and table_type = 'BASE TABLE' + SQL + + TABLE_COUNT_QUERY = <<-SQL + select count(*) as total + from information_schema.tables + where table_catalog = :table_catalog and table_type = 'BASE TABLE' + and table_schema not in ('information_schema', 'pg_catalog') + SQL + + DISKSPACE_HEADROOM_FACTOR = 1.25 + + attr_reader :backup_location + + def initialize(backup_base_location: nil) + random_post_fix = SecureRandom.alphanumeric(10) + @backup_base_location = backup_base_location || Gitlab.config.backup.path + @backup_location = File.join(@backup_base_location, "migration_#{random_post_fix}") + end + + def process! + return unless can_migrate? + + dump_main_db + import_dump_to_ci_db + + FileUtils.remove_entry_secure(@backup_location, true) + end + + private + + def valid_backup_location? + FileUtils.mkdir_p(@backup_base_location) + + true + rescue StandardError => e + raise MigrateError, "Failed to create directory #{@backup_base_location}: #{e.message}" + end + + def main_table_sizes + ApplicationRecord.connection.execute( + ApplicationRecord.sanitize_sql([ + TABLE_SIZE_QUERY, + { table_catalog: main_config.dig(:activerecord, :database) } + ]) + ).first["total"].to_f + end + + def diskspace_free + Sys::Filesystem.stat( + File.expand_path("#{@backup_location}/../") + ).bytes_free + end + + def required_diskspace_available? + needed = main_table_sizes * DISKSPACE_HEADROOM_FACTOR + available = diskspace_free + + if needed > available + raise MigrateError, + "Not enough diskspace available on #{@backup_location}: " \ + "Available: #{ActiveSupport::NumberHelper.number_to_human_size(available)}, " \ + "Needed: #{ActiveSupport::NumberHelper.number_to_human_size(needed)}" + end + + true + end + + def single_database_setup? + if Gitlab::Database.database_mode == Gitlab::Database::MODE_MULTIPLE_DATABASES + raise MigrateError, "GitLab is already configured to run on multiple databases" + end + + true + end + + def ci_database_connect_ok? + _, status = with_transient_pg_env(ci_config[:pg_env]) do + psql_args = ["--dbname=#{ci_database_name}", "-tAc", "select 1"] + + Open3.capture2e('psql', *psql_args) + end + + unless status.success? + raise MigrateError, + "Can't connect to database '#{ci_database_name} on host '#{ci_config[:pg_env]['PGHOST']}'. " \ + "Ensure the database has been created." + end + + true + end + + def ci_database_empty? + sql = ApplicationRecord.sanitize_sql([ + TABLE_COUNT_QUERY, + { table_catalog: ci_database_name } + ]) + + output, status = with_transient_pg_env(ci_config[:pg_env]) do + psql_args = ["--dbname=#{ci_database_name}", "-tAc", sql] + + Open3.capture2e('psql', *psql_args) + end + + unless status.success? && output.chomp.to_i == 0 + raise MigrateError, + "Database '#{ci_database_name}' is not empty" + end + + true + end + + def background_migrations_done? + unfinished_count = Gitlab::Database::BackgroundMigration::BatchedMigration.without_status(:finished).count + if unfinished_count > 0 + raise MigrateError, + "Found #{unfinished_count} unfinished Background Migration(s). Please wait until they are finished." + end + + true + end + + def can_migrate? + valid_backup_location? && + single_database_setup? && + ci_database_connect_ok? && + ci_database_empty? && + required_diskspace_available? && + background_migrations_done? + end + + def with_transient_pg_env(extended_env) + ENV.merge!(extended_env) + result = yield + ENV.reject! { |k, _| extended_env.key?(k) } + + result + end + + def import_dump_to_ci_db + with_transient_pg_env(ci_config[:pg_env]) do + restore_args = ["--jobs=4", "--dbname=#{ci_database_name}"] + + Open3.capture2e('pg_restore', *restore_args, @backup_location) + end + end + + def dump_main_db + with_transient_pg_env(main_config[:pg_env]) do + args = ['--format=d', '--jobs=4', "--file=#{@backup_location}"] + + Open3.capture2e('pg_dump', *args, main_config.dig(:activerecord, :database)) + end + end + + def main_config + @main_config ||= ::Backup::DatabaseModel.new('main').config + end + + def ci_config + @ci_config ||= ::Backup::DatabaseModel.new('ci').config + end + + def ci_database_name + @ci_database_name ||= "#{main_config.dig(:activerecord, :database)}_ci" + end + end + end + end +end diff --git a/lib/gitlab/database/dictionary.rb b/lib/gitlab/database/dictionary.rb index 7b0c8560a26..4ef392a4e44 100644 --- a/lib/gitlab/database/dictionary.rb +++ b/lib/gitlab/database/dictionary.rb @@ -3,57 +3,99 @@ module Gitlab module Database class Dictionary - def initialize(file_path) - @file_path = file_path - @data = YAML.load_file(file_path) + def self.entries(scope = '') + @entries ||= {} + @entries[scope] ||= Dir.glob(dictionary_path_globs(scope)).map do |file_path| + dictionary = Entry.new(file_path) + dictionary.validate! + dictionary + end end - def name_and_schema - [key_name, gitlab_schema.to_sym] + def self.entry(name, scope = '') + entries(scope).find do |entry| + entry.key_name == name + end end - def table_name - data['table_name'] + private_class_method def self.dictionary_path_globs(scope) + dictionary_paths.map { |path| Rails.root.join(path, scope, '*.yml') } end - def view_name - data['view_name'] + private_class_method def self.dictionary_paths + ::Gitlab::Database.all_database_connections + .values.map(&:db_docs_dir).uniq end - def milestone - data['milestone'] - end + class Entry + def initialize(file_path) + @file_path = file_path + @data = YAML.load_file(file_path) + end - def gitlab_schema - data['gitlab_schema'] - end + def name_and_schema + [key_name, gitlab_schema.to_sym] + end - def schema?(schema_name) - gitlab_schema == schema_name.to_s - end + def table_name + data['table_name'] + end - def key_name - table_name || view_name - end + def feature_categories + data['feature_categories'] + end - def validate! - return true unless gitlab_schema.nil? + def view_name + data['view_name'] + end - raise( - GitlabSchema::UnknownSchemaError, - "#{file_path} must specify a valid gitlab_schema for #{key_name}. " \ - "See #{help_page_url}" - ) - end + def milestone + data['milestone'] + end + + def gitlab_schema + data['gitlab_schema'] + end + + def sharding_key + data['sharding_key'] + end + + def desired_sharding_key + data['desired_sharding_key'] + end + + def classes + data['classes'] + end + + def schema?(schema_name) + gitlab_schema == schema_name.to_s + end + + def key_name + table_name || view_name + end + + def validate! + return true unless gitlab_schema.nil? + + raise( + GitlabSchema::UnknownSchemaError, + "#{file_path} must specify a valid gitlab_schema for #{key_name}. " \ + "See #{help_page_url}" + ) + end - private + private - attr_reader :file_path, :data + attr_reader :file_path, :data - def help_page_url - # rubocop:disable Gitlab/DocUrl -- link directly to docs.gitlab.com, always - 'https://docs.gitlab.com/ee/development/database/database_dictionary.html' - # rubocop:enable Gitlab/DocUrl + def help_page_url + # rubocop:disable Gitlab/DocUrl -- link directly to docs.gitlab.com, always + 'https://docs.gitlab.com/ee/development/database/database_dictionary.html' + # rubocop:enable Gitlab/DocUrl + end end end end diff --git a/lib/gitlab/database/gitlab_schema.rb b/lib/gitlab/database/gitlab_schema.rb index ecb45622061..e6f7dbec69c 100644 --- a/lib/gitlab/database/gitlab_schema.rb +++ b/lib/gitlab/database/gitlab_schema.rb @@ -88,6 +88,10 @@ module Gitlab # rubocop:enable Gitlab/DocUrl end + def self.cell_local?(schema) + Gitlab::Database.all_gitlab_schemas[schema.to_s].cell_local + end + def self.cross_joins_allowed?(table_schemas, all_tables) return true unless table_schemas.many? @@ -121,15 +125,6 @@ module Gitlab end end - def self.dictionary_paths - Gitlab::Database.all_database_connections - .values.map(&:db_docs_dir).uniq - end - - def self.dictionary_path_globs(scope) - self.dictionary_paths.map { |path| Rails.root.join(path, scope, '*.yml') } - end - def self.views_and_tables_to_schema @views_and_tables_to_schema ||= self.tables_to_schema.merge(self.views_to_schema) end @@ -139,32 +134,24 @@ module Gitlab end def self.deleted_tables_to_schema - @deleted_tables_to_schema ||= self.build_dictionary('deleted_tables').map(&:name_and_schema).to_h + @deleted_tables_to_schema ||= ::Gitlab::Database::Dictionary.entries('deleted_tables').map(&:name_and_schema).to_h end def self.deleted_views_to_schema - @deleted_views_to_schema ||= self.build_dictionary('deleted_views').map(&:name_and_schema).to_h + @deleted_views_to_schema ||= ::Gitlab::Database::Dictionary.entries('deleted_views').map(&:name_and_schema).to_h end def self.tables_to_schema - @tables_to_schema ||= self.build_dictionary('').map(&:name_and_schema).to_h + @tables_to_schema ||= ::Gitlab::Database::Dictionary.entries.map(&:name_and_schema).to_h end def self.views_to_schema - @views_to_schema ||= self.build_dictionary('views').map(&:name_and_schema).to_h + @views_to_schema ||= ::Gitlab::Database::Dictionary.entries('views').map(&:name_and_schema).to_h end def self.schema_names @schema_names ||= self.views_and_tables_to_schema.values.to_set end - - def self.build_dictionary(scope) - Dir.glob(dictionary_path_globs(scope)).map do |file_path| - dictionary = Dictionary.new(file_path) - dictionary.validate! - dictionary - end - end end end end diff --git a/lib/gitlab/database/gitlab_schema_info.rb b/lib/gitlab/database/gitlab_schema_info.rb index 20d2b31a65c..b7ec3dfc893 100644 --- a/lib/gitlab/database/gitlab_schema_info.rb +++ b/lib/gitlab/database/gitlab_schema_info.rb @@ -14,6 +14,7 @@ module Gitlab :allow_cross_transactions, :allow_cross_foreign_keys, :file_path, + :cell_local, keyword_init: true ) do def initialize(*) diff --git a/lib/gitlab/database/health_status/indicators/autovacuum_active_on_table.rb b/lib/gitlab/database/health_status/indicators/autovacuum_active_on_table.rb index 6bf2bbf0c70..f3aa03657c7 100644 --- a/lib/gitlab/database/health_status/indicators/autovacuum_active_on_table.rb +++ b/lib/gitlab/database/health_status/indicators/autovacuum_active_on_table.rb @@ -26,6 +26,10 @@ module Gitlab attr_reader :tables def enabled? + if tables.include?('ci_builds') && Feature.enabled?(:skip_autovacuum_health_check_for_ci_builds, type: :ops) + return false + end + Feature.enabled?(:batched_migrations_health_status_autovacuum, type: :ops) end diff --git a/lib/gitlab/database/load_balancing/load_balancer.rb b/lib/gitlab/database/load_balancing/load_balancer.rb index 9495648d069..55a27f89b36 100644 --- a/lib/gitlab/database/load_balancing/load_balancer.rb +++ b/lib/gitlab/database/load_balancing/load_balancer.rb @@ -51,6 +51,8 @@ module Gitlab # If no secondaries were available this method will use the primary # instead. def read(&block) + raise_if_concurrent_ruby! + service_discovery&.log_refresh_thread_interruption conflict_retried = 0 @@ -111,6 +113,8 @@ module Gitlab # Yields a connection that can be used for both reads and writes. def read_write + raise_if_concurrent_ruby! + service_discovery&.log_refresh_thread_interruption connection = nil @@ -372,6 +376,12 @@ module Gitlab row = ar_connection.select_all(sql).first row['location'] if row end + + def raise_if_concurrent_ruby! + Gitlab::Utils.raise_if_concurrent_ruby!(:db) + rescue StandardError => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) + end end end end diff --git a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb index 3d4ac113bf6..39706582e3c 100644 --- a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb @@ -38,10 +38,6 @@ module Gitlab # batch_class_name - The name of the class that will be called to find the range of each next batch # batch_size - The maximum number of rows per job # sub_batch_size - The maximum number of rows processed per "iteration" within the job - # queued_migration_version - Version of the migration that queues the BBM, this is used to establish dependecies - # - # queued_migration_version is made optional temporarily to allow prior migrations to not fail, - # https://gitlab.com/gitlab-org/gitlab/-/issues/426417 will make it mandatory. # # *Returns the created BatchedMigration record* # @@ -67,7 +63,6 @@ module Gitlab batch_column_name, *job_arguments, job_interval:, - queued_migration_version: nil, batch_min_value: BATCH_MIN_VALUE, batch_max_value: nil, batch_class_name: BATCH_CLASS_NAME, @@ -80,6 +75,8 @@ module Gitlab Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_dml_mode! gitlab_schema ||= gitlab_schema_from_context + # Version of the migration that queued the BBM, this is used to establish dependencies + queued_migration_version = version Gitlab::Database::BackgroundMigration::BatchedMigration.reset_column_information @@ -120,7 +117,7 @@ module Gitlab "(given #{job_arguments.count}, expected #{migration.job_class.job_arguments_count})" end - assign_attribtues_safely( + assign_attributes_safely( migration, max_batch_size, batch_table_name, @@ -231,7 +228,7 @@ module Gitlab "\n\n" \ "Finalize it manually by running the following command in a `bash` or `sh` shell:" \ "\n\n" \ - "\tsudo gitlab-rake gitlab:background_migrations:finalize[#{job_class_name},#{table_name},#{column_name},'#{job_arguments.to_json.gsub(',', '\,')}']" \ + "\t#{migration.finalize_command}" \ "\n\n" \ "For more information, check the documentation" \ "\n\n" \ @@ -246,7 +243,7 @@ module Gitlab # about columns introduced later on because this model is not # isolated in migrations, which is why we need to check for existence # of these columns first. - def assign_attribtues_safely(migration, max_batch_size, batch_table_name, gitlab_schema, queued_migration_version) + def assign_attributes_safely(migration, max_batch_size, batch_table_name, gitlab_schema, queued_migration_version) # We keep track of the estimated number of tuples in 'total_tuple_count' to reason later # about the overall progress of a migration. safe_attributes_value = { diff --git a/lib/gitlab/database/migrations/pg_backend_pid.rb b/lib/gitlab/database/migrations/pg_backend_pid.rb index b59eb55cc6e..52f309e4058 100644 --- a/lib/gitlab/database/migrations/pg_backend_pid.rb +++ b/lib/gitlab/database/migrations/pg_backend_pid.rb @@ -13,7 +13,7 @@ module Gitlab Gitlab::Database::Migrations::PgBackendPid.say(conn) yield(conn) - + ensure Gitlab::Database::Migrations::PgBackendPid.say(conn) end end diff --git a/lib/gitlab/database/partitioning/ci_sliding_list_strategy.rb b/lib/gitlab/database/partitioning/ci_sliding_list_strategy.rb index 69a69091b5c..de6319582cb 100644 --- a/lib/gitlab/database/partitioning/ci_sliding_list_strategy.rb +++ b/lib/gitlab/database/partitioning/ci_sliding_list_strategy.rb @@ -12,6 +12,13 @@ module Gitlab partition_for(active_partition.value + 1) end + def missing_partitions + partitions = [] + partitions << initial_partition if no_partitions_exist? + partitions << next_partition if next_partition_if.call(active_partition) + partitions + end + def validate_and_fix; end def after_adding_partitions; end @@ -20,6 +27,10 @@ module Gitlab [] end + def active_partition + super || initial_partition + end + private def ensure_partitioning_column_ignored_or_readonly!; end diff --git a/lib/gitlab/database/postgres_index.rb b/lib/gitlab/database/postgres_index.rb index 1c775482e7e..f52785c1e56 100644 --- a/lib/gitlab/database/postgres_index.rb +++ b/lib/gitlab/database/postgres_index.rb @@ -23,7 +23,7 @@ module Gitlab # Indexes with reindexing support scope :reindexing_support, -> do - where(partitioned: false, exclusion: false, expression: false, type: Gitlab::Database::Reindexing::SUPPORTED_TYPES) + where(exclusion: false, expression: false, type: Gitlab::Database::Reindexing::SUPPORTED_TYPES) .not_match("#{Gitlab::Database::Reindexing::ReindexConcurrently::TEMPORARY_INDEX_PATTERN}$") end diff --git a/lib/gitlab/database/postgres_sequence.rb b/lib/gitlab/database/postgres_sequence.rb new file mode 100644 index 00000000000..bf394d80e12 --- /dev/null +++ b/lib/gitlab/database/postgres_sequence.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Gitlab + module Database + # Backed by the postgres_sequences view + class PostgresSequence < SharedModel + self.primary_key = :seq_name + + scope :by_table_name, ->(table_name) { where(table_name: table_name) } + end + end +end diff --git a/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch.rb b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch.rb index 583aceba098..847f7064ad4 100644 --- a/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch.rb +++ b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch.rb @@ -6,7 +6,7 @@ module Gitlab class PreventSetOperatorMismatch < Base SetOperatorStarError = Class.new(QueryAnalyzerError) - DETECT_REGEX = /.*SELECT.+(UNION|EXCEPT|INTERSECT)/i + DETECT_REGEX = /.*SELECT.+\b(UNION|EXCEPT|INTERSECT)\b/i class << self def enabled? @@ -36,9 +36,8 @@ module Gitlab node.stmt.select_stmt end - # This not entirely correct and will run true on `SELECT union_station, ...` def requires_detection?(sql) - sql.match DETECT_REGEX + DETECT_REGEX.match?(sql) end end end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index de7be6efd72..6ddd8a208bc 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -9,7 +9,8 @@ module Gitlab delegate :new_file?, :deleted_file?, :renamed_file?, :unidiff, :old_path, :new_path, :a_mode, :b_mode, :mode_changed?, - :submodule?, :expanded?, :too_large?, :collapsed?, :line_count, :has_binary_notice?, to: :diff, prefix: false + :submodule?, :expanded?, :too_large?, :collapsed?, :line_count, :has_binary_notice?, + :generated?, to: :diff, prefix: false # Finding a viewer for a diff file happens based only on extension and whether the # diff file blobs are binary or text, which means 1 diff file should only be matched by 1 viewer, diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb index 63a437b021d..dc5f4e1b324 100644 --- a/lib/gitlab/diff/highlight_cache.rb +++ b/lib/gitlab/diff/highlight_cache.rb @@ -185,18 +185,15 @@ module Gitlab def read_cache return {} unless file_paths.any? - results = [] cache_key = key # Moving out redis calls for feature flags out of redis.pipelined - with_redis do |redis| + results, _ = with_redis do |redis| redis.pipelined do |pipeline| - results = pipeline.hmget(cache_key, file_paths) + pipeline.hmget(cache_key, file_paths) pipeline.expire(key, EXPIRATION) end end - results = results.value - record_hit_ratio(results) results.map! do |result| diff --git a/lib/gitlab/email/handler/create_note_handler.rb b/lib/gitlab/email/handler/create_note_handler.rb index 7daa1bb96a1..817956831e3 100644 --- a/lib/gitlab/email/handler/create_note_handler.rb +++ b/lib/gitlab/email/handler/create_note_handler.rb @@ -33,6 +33,8 @@ module Gitlab record: create_note, invalid_exception: InvalidNoteError, record_name: 'comment') + + reopen_issue_on_external_participant_note end def metrics_event @@ -71,6 +73,35 @@ module Gitlab raise UserNotFoundError unless from_address && author.verified_email?(from_address) end + + def reopen_issue_on_external_participant_note + return unless noteable.respond_to?(:closed?) + return unless noteable.closed? + return unless author == Users::Internal.support_bot + return unless project.service_desk_setting&.reopen_issue_on_external_participant_note? + + ::Notes::CreateService.new( + project, + Users::Internal.support_bot, + noteable: noteable, + note: build_reopen_message, + confidential: true + ).execute + end + + def build_reopen_message + translated_text = s_( + "ServiceDesk|This issue has been reopened because it received a new comment from an external participant." + ) + + "#{assignees_references} :wave: #{translated_text}\n/reopen".lstrip + end + + def assignees_references + return unless noteable.assignees.any? + + noteable.assignees.map(&:to_reference).join(' ') + end end end end diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb index e3249b143c8..b507af3024e 100644 --- a/lib/gitlab/email/handler/service_desk_handler.rb +++ b/lib/gitlab/email/handler/service_desk_handler.rb @@ -74,7 +74,6 @@ module Gitlab attr_reader :project_id, :project_path, :service_desk_key def contains_custom_email_address_verification_subaddress? - return false unless Feature.enabled?(:service_desk_custom_email, project) return false unless to_address.present? # Verification email only has one recipient @@ -230,6 +229,9 @@ module Gitlab def add_email_participants return if reply_email? && !Feature.enabled?(:issue_email_participants, @issue.project) + # Migrate this to ::IssueEmailParticipants::CreateService once the + # feature flag issue_email_participants has been enabled globally + # or removed: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/137147#note_1652104416 @issue.issue_email_participants.create(email: from_address) add_external_participants_from_cc @@ -239,11 +241,11 @@ module Gitlab return if project.service_desk_setting.nil? return unless project.service_desk_setting.add_external_participants_from_cc? - cc_addresses.each do |email| - next if service_desk_addresses.include?(email) - - @issue.issue_email_participants.create!(email: email) - end + ::IssueEmailParticipants::CreateService.new( + target: @issue, + current_user: Users::Internal.support_bot, + emails: cc_addresses.excluding(service_desk_addresses) + ).execute end def service_desk_addresses diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index d5877234c3a..e36b07da801 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -85,7 +85,13 @@ module Gitlab def mail_key strong_memoize(:mail_key) do - find_first_key_from(to) || key_from_additional_headers + find_most_concrete_key_from(to) || key_from_additional_headers + end + end + + def find_most_concrete_key_from(items) + find_first_key_from(items) do |email| + Gitlab::Email::ServiceDesk::CustomEmail.key_from_reply_address(email) || email_class.key_from_address(email) end end @@ -93,7 +99,8 @@ module Gitlab items.each do |item| email = item.is_a?(Mail::Field) ? item.value : item - key = email_class.key_from_address(email) + key = block_given? ? yield(email) : email_class.key_from_address(email) + return key if key end nil diff --git a/lib/gitlab/email/service_desk/custom_email.rb b/lib/gitlab/email/service_desk/custom_email.rb index 30ae435a6ec..1828f71984b 100644 --- a/lib/gitlab/email/service_desk/custom_email.rb +++ b/lib/gitlab/email/service_desk/custom_email.rb @@ -7,6 +7,9 @@ module Gitlab # support all features and methods of ingestable email addresses like # incoming_email and service_desk_email. module CustomEmail + REPLY_ADDRESS_KEY_REGEXP = /\+([0-9a-f]{32})@/ + EMAIL_REGEXP = /\A[\w\-._]+@[\w\-.]+\.{1}[a-zA-Z]{2,}\z/ + class << self def reply_address(issue, reply_key) return if reply_key.nil? @@ -18,6 +21,29 @@ module Gitlab # We don't have a placeholder. custom_email.sub('@', "+#{reply_key}@") end + + def key_from_reply_address(email) + match_data = REPLY_ADDRESS_KEY_REGEXP.match(email) + return unless match_data + + key = match_data[1] + + settings = find_service_desk_setting_from_reply_address(email, key) + # We intentionally don't check whether custom email is enabled + # so we don't lose emails that are addressed to a disabled custom email address + return unless settings + + key + end + + private + + def find_service_desk_setting_from_reply_address(email, key) + potential_custom_email = email.sub("+#{key}", '') + return unless EMAIL_REGEXP.match?(potential_custom_email) + + ServiceDeskSetting.find_by_custom_email(potential_custom_email) + end end end end diff --git a/lib/gitlab/encrypted_command_base.rb b/lib/gitlab/encrypted_command_base.rb index 679d9d8e31a..5e483fa2b15 100644 --- a/lib/gitlab/encrypted_command_base.rb +++ b/lib/gitlab/encrypted_command_base.rb @@ -41,7 +41,11 @@ module Gitlab encrypted.change do |contents| contents = encrypted_file_template unless File.exist?(encrypted.content_path) File.write(temp_file.path, contents) - system(ENV['EDITOR'], temp_file.path) + + edit_success = system(*editor_args, temp_file.path) + + raise "Unable to run $EDITOR: #{editor_args}" unless edit_success + changes = File.read(temp_file.path) contents_changed = contents != changes validate_contents(changes) @@ -99,6 +103,10 @@ module Gitlab def encrypted_file_template raise NotImplementedError end + + def editor_args + ENV['EDITOR']&.split + end end end end diff --git a/lib/gitlab/error_tracking.rb b/lib/gitlab/error_tracking.rb index 2b00fe48951..239aee97378 100644 --- a/lib/gitlab/error_tracking.rb +++ b/lib/gitlab/error_tracking.rb @@ -70,8 +70,8 @@ module Gitlab # returns a Hash, then the return value of that method will be merged into # `extra`. Exceptions can use this mechanism to provide structured data # to sentry in addition to their message and back-trace. - def track_and_raise_exception(exception, extra = {}) - process_exception(exception, extra: extra) + def track_and_raise_exception(exception, extra = {}, tags = {}) + process_exception(exception, extra: extra, tags: tags) raise exception end @@ -90,8 +90,8 @@ module Gitlab # # Provide an issue URL for follow up. # as `issue_url: 'http://gitlab.com/gitlab-org/gitlab/issues/111'` - def track_and_raise_for_dev_exception(exception, extra = {}) - process_exception(exception, extra: extra) + def track_and_raise_for_dev_exception(exception, extra = {}, tags = {}) + process_exception(exception, extra: extra, tags: tags) raise exception if should_raise_for_dev? end @@ -102,8 +102,8 @@ module Gitlab # returns a Hash, then the return value of that method will be merged into # `extra`. Exceptions can use this mechanism to provide structured data # to sentry in addition to their message and back-trace. - def track_exception(exception, extra = {}) - process_exception(exception, extra: extra) + def track_exception(exception, extra = {}, tags = {}) + process_exception(exception, extra: extra, tags: tags) end # This should be used when you only want to log the exception, @@ -157,8 +157,8 @@ module Gitlab end end - def process_exception(exception, extra:, trackers: default_trackers) - context_payload = Gitlab::ErrorTracking::ContextPayloadGenerator.generate(exception, extra) + def process_exception(exception, extra:, tags: {}, trackers: default_trackers) + context_payload = Gitlab::ErrorTracking::ContextPayloadGenerator.generate(exception, extra, tags) trackers.each do |tracker| tracker.capture_exception(exception, **context_payload) diff --git a/lib/gitlab/error_tracking/context_payload_generator.rb b/lib/gitlab/error_tracking/context_payload_generator.rb index 3d0a707608f..23dd2e33a58 100644 --- a/lib/gitlab/error_tracking/context_payload_generator.rb +++ b/lib/gitlab/error_tracking/context_payload_generator.rb @@ -3,14 +3,14 @@ module Gitlab module ErrorTracking class ContextPayloadGenerator - def self.generate(exception, extra = {}) - new.generate(exception, extra) + def self.generate(exception, extra = {}, tags = {}) + new.generate(exception, extra, tags) end - def generate(exception, extra = {}) + def generate(exception, extra = {}, tags = {}) { extra: extra_payload(exception, extra), - tags: tags_payload, + tags: tags_payload(tags), user: user_payload } end @@ -31,12 +31,14 @@ module Gitlab filter.filter(parameters) end - def tags_payload - extra_tags_from_env.merge!( - program: Gitlab.process_name, - locale: I18n.locale, - feature_category: current_context['meta.feature_category'], - Labkit::Correlation::CorrelationId::LOG_KEY.to_sym => Labkit::Correlation::CorrelationId.current_id + def tags_payload(tags) + tags.merge( + extra_tags_from_env.merge!( + program: Gitlab.process_name, + locale: I18n.locale, + feature_category: current_context['meta.feature_category'], + Labkit::Correlation::CorrelationId::LOG_KEY.to_sym => Labkit::Correlation::CorrelationId.current_id + ) ) end diff --git a/lib/gitlab/event_store.rb b/lib/gitlab/event_store.rb index 1e397b52ddf..b422fd061ff 100644 --- a/lib/gitlab/event_store.rb +++ b/lib/gitlab/event_store.rb @@ -17,6 +17,10 @@ module Gitlab instance.publish(event) end + def self.publish_group(events) + instance.publish_group(events) + end + def self.instance @instance ||= Store.new { |store| configure!(store) } end @@ -40,7 +44,9 @@ module Gitlab store.subscribe ::MergeRequests::CreateApprovalNoteWorker, to: ::MergeRequests::ApprovedEvent store.subscribe ::MergeRequests::ResolveTodosAfterApprovalWorker, to: ::MergeRequests::ApprovedEvent store.subscribe ::MergeRequests::ExecuteApprovalHooksWorker, to: ::MergeRequests::ApprovedEvent - store.subscribe ::MergeRequests::SetReviewerReviewedWorker, to: ::MergeRequests::ApprovedEvent + store.subscribe ::MergeRequests::SetReviewerReviewedWorker, + to: ::MergeRequests::ApprovedEvent, + if: -> (event) { ::Feature.disabled?(:mr_request_changes, User.find_by_id(event.data[:current_user_id])) } store.subscribe ::Ml::ExperimentTracking::AssociateMlCandidateToPackageWorker, to: ::Packages::PackageCreatedEvent, if: -> (event) { ::Ml::ExperimentTracking::AssociateMlCandidateToPackageWorker.handles_event?(event) } diff --git a/lib/gitlab/event_store/event.rb b/lib/gitlab/event_store/event.rb index ee0c329b8e8..ba82ae6dd6a 100644 --- a/lib/gitlab/event_store/event.rb +++ b/lib/gitlab/event_store/event.rb @@ -29,8 +29,13 @@ module Gitlab class Event attr_reader :data + class << self + attr_accessor :json_schema_valid + end + def initialize(data:) - validate_schema!(data) + validate_schema! + validate_data!(data) @data = data end @@ -40,7 +45,17 @@ module Gitlab private - def validate_schema!(data) + def validate_schema! + if self.class.json_schema_valid.nil? + self.class.json_schema_valid = JSONSchemer.schema(self.class.json_schema).valid?(schema) + end + + return if self.class.json_schema_valid == true + + raise Gitlab::EventStore::InvalidEvent, "Schema for event #{self.class} is invalid" + end + + def validate_data!(data) unless data.is_a?(Hash) raise Gitlab::EventStore::InvalidEvent, "Event data must be a Hash" end @@ -49,6 +64,10 @@ module Gitlab raise Gitlab::EventStore::InvalidEvent, "Data for event #{self.class} does not match the defined schema: #{schema}" end end + + def self.json_schema + @json_schema ||= Gitlab::Json.parse(File.read(File.join(__dir__, 'json_schema_draft07.json'))) + end end end end diff --git a/lib/gitlab/event_store/json_schema_draft07.json b/lib/gitlab/event_store/json_schema_draft07.json new file mode 100644 index 00000000000..aea0a29c4dc --- /dev/null +++ b/lib/gitlab/event_store/json_schema_draft07.json @@ -0,0 +1,250 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "http://json-schema.org/draft-07/schema#", + "title": "Core schema meta-schema", + "definitions": { + "schemaArray": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#" + } + }, + "nonNegativeInteger": { + "type": "integer", + "minimum": 0 + }, + "nonNegativeIntegerDefault0": { + "allOf": [ + { + "$ref": "#/definitions/nonNegativeInteger" + }, + { + "default": 0 + } + ] + }, + "simpleTypes": { + "enum": [ + "array", + "boolean", + "integer", + "null", + "number", + "object", + "string" + ] + }, + "stringArray": { + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true, + "default": [ + + ] + } + }, + "type": [ + "object", + "boolean" + ], + "properties": { + "$id": { + "type": "string", + "format": "uri-reference" + }, + "$schema": { + "type": "string", + "format": "uri" + }, + "$ref": { + "type": "string", + "format": "uri-reference" + }, + "$comment": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "default": true, + "readOnly": { + "type": "boolean", + "default": false + }, + "writeOnly": { + "type": "boolean", + "default": false + }, + "examples": { + "type": "array", + "items": true + }, + "multipleOf": { + "type": "number", + "exclusiveMinimum": 0 + }, + "maximum": { + "type": "number" + }, + "exclusiveMaximum": { + "type": "number" + }, + "minimum": { + "type": "number" + }, + "exclusiveMinimum": { + "type": "number" + }, + "maxLength": { + "$ref": "#/definitions/nonNegativeInteger" + }, + "minLength": { + "$ref": "#/definitions/nonNegativeIntegerDefault0" + }, + "pattern": { + "type": "string", + "format": "regex" + }, + "additionalItems": { + "$ref": "#" + }, + "items": { + "anyOf": [ + { + "$ref": "#" + }, + { + "$ref": "#/definitions/schemaArray" + } + ], + "default": true + }, + "maxItems": { + "$ref": "#/definitions/nonNegativeInteger" + }, + "minItems": { + "$ref": "#/definitions/nonNegativeIntegerDefault0" + }, + "uniqueItems": { + "type": "boolean", + "default": false + }, + "contains": { + "$ref": "#" + }, + "maxProperties": { + "$ref": "#/definitions/nonNegativeInteger" + }, + "minProperties": { + "$ref": "#/definitions/nonNegativeIntegerDefault0" + }, + "required": { + "$ref": "#/definitions/stringArray" + }, + "additionalProperties": { + "$ref": "#" + }, + "definitions": { + "type": "object", + "additionalProperties": { + "$ref": "#" + }, + "default": { + } + }, + "properties": { + "type": "object", + "additionalProperties": { + "$ref": "#" + }, + "default": { + } + }, + "patternProperties": { + "type": "object", + "additionalProperties": { + "$ref": "#" + }, + "propertyNames": { + "format": "regex" + }, + "default": { + } + }, + "dependencies": { + "type": "object", + "additionalProperties": { + "anyOf": [ + { + "$ref": "#" + }, + { + "$ref": "#/definitions/stringArray" + } + ] + } + }, + "propertyNames": { + "$ref": "#" + }, + "const": true, + "enum": { + "type": "array", + "items": true, + "minItems": 1, + "uniqueItems": true + }, + "type": { + "anyOf": [ + { + "$ref": "#/definitions/simpleTypes" + }, + { + "type": "array", + "items": { + "$ref": "#/definitions/simpleTypes" + }, + "minItems": 1, + "uniqueItems": true + } + ] + }, + "format": { + "type": "string" + }, + "contentMediaType": { + "type": "string" + }, + "contentEncoding": { + "type": "string" + }, + "if": { + "$ref": "#" + }, + "then": { + "$ref": "#" + }, + "else": { + "$ref": "#" + }, + "allOf": { + "$ref": "#/definitions/schemaArray" + }, + "anyOf": { + "$ref": "#/definitions/schemaArray" + }, + "oneOf": { + "$ref": "#/definitions/schemaArray" + }, + "not": { + "$ref": "#" + } + }, + "default": true +} diff --git a/lib/gitlab/event_store/store.rb b/lib/gitlab/event_store/store.rb index 318745cc192..c558362122b 100644 --- a/lib/gitlab/event_store/store.rb +++ b/lib/gitlab/event_store/store.rb @@ -15,12 +15,12 @@ module Gitlab lock! end - def subscribe(worker, to:, if: nil, delay: nil) + def subscribe(worker, to:, if: nil, delay: nil, group_size: nil) condition = binding.local_variable_get('if') Array(to).each do |event| validate_subscription!(worker, event) - subscriptions[event] << Gitlab::EventStore::Subscription.new(worker, condition, delay) + subscriptions[event] << Gitlab::EventStore::Subscription.new(worker, condition, delay, group_size) end end @@ -34,6 +34,18 @@ module Gitlab end end + def publish_group(events) + event_class = events.first.class + + unless events.all? { |e| e.class < Event && e.instance_of?(event_class) } + raise InvalidEvent, "Not all events being published are valid" + end + + subscriptions.fetch(event_class, []).each do |subscription| + subscription.consume_events(events) + end + end + private def lock! diff --git a/lib/gitlab/event_store/subscriber.rb b/lib/gitlab/event_store/subscriber.rb index da95d3cfcfa..81770624cd9 100644 --- a/lib/gitlab/event_store/subscriber.rb +++ b/lib/gitlab/event_store/subscriber.rb @@ -29,16 +29,22 @@ module Gitlab def perform(event_type, data) raise InvalidEvent, event_type unless self.class.const_defined?(event_type) - event = event_type.constantize.new( - data: data.with_indifferent_access - ) + event_type_class = event_type.constantize - handle_event(event) + Array.wrap(data).each do |single_event_data| + handle_event(construct_event(event_type_class, single_event_data)) + end end def handle_event(event) raise NotImplementedError, 'you must implement this methods in order to handle events' end + + private + + def construct_event(event_type, event_data) + event_type.new(data: event_data.with_indifferent_access) + end end end end diff --git a/lib/gitlab/event_store/subscription.rb b/lib/gitlab/event_store/subscription.rb index 81a65f9a8ff..f39bbc2aaf0 100644 --- a/lib/gitlab/event_store/subscription.rb +++ b/lib/gitlab/event_store/subscription.rb @@ -3,12 +3,17 @@ module Gitlab module EventStore class Subscription - attr_reader :worker, :condition, :delay + DEFAULT_GROUP_SIZE = 10 + SCHEDULING_BATCH_SIZE = 100 + SCHEDULING_BATCH_DELAY = 10.seconds - def initialize(worker, condition, delay) + attr_reader :worker, :condition, :delay, :group_size + + def initialize(worker, condition, delay, group_size) @worker = worker @condition = condition @delay = delay + @group_size = group_size || DEFAULT_GROUP_SIZE end def consume_event(event) @@ -29,6 +34,30 @@ module Gitlab Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, event_class: event.class.name, event_data: event.data) end + def consume_events(events) + event_class = events.first.class + unless events.all? { |e| e.class < Event && e.instance_of?(event_class) } + raise InvalidEvent, "Events being published are not an instance of Gitlab::EventStore::Event" + end + + matched_events = events.select { |event| condition_met?(event) } + worker_args = events_worker_args(event_class, matched_events) + + # rubocop:disable Scalability/BulkPerformWithContext -- Context info is already available in `ApplicationContext` here. + if worker_args.size > SCHEDULING_BATCH_SIZE + # To reduce the number of concurrent jobs, we batch the group of events and add delay between each batch. + # We add a delay of 1s as bulk_perform_in does not support 0s delay. + worker.bulk_perform_in(delay || 1.second, worker_args, batch_size: SCHEDULING_BATCH_SIZE, batch_delay: SCHEDULING_BATCH_DELAY) + elsif delay + worker.bulk_perform_in(delay, worker_args) + else + worker.bulk_perform_async(worker_args) + end + # rubocop:enable Scalability/BulkPerformWithContext + rescue StandardError => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, event_class: event_class, events: events.map(&:data)) + end + private def condition_met?(event) @@ -36,6 +65,13 @@ module Gitlab condition.call(event) end + + def events_worker_args(event_class, events) + events + .map { |event| event.data.deep_stringify_keys } + .each_slice(group_size) + .map { |events_data_group| [event_class.name, events_data_group] } + end end end end diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index e887e455792..0b18a337707 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -31,7 +31,7 @@ module Gitlab EOS def self.get_uuid(key) - Gitlab::Redis::ClusterSharedState.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.get(redis_shared_state_key(key)) || false end end @@ -61,7 +61,7 @@ module Gitlab def self.cancel(key, uuid) return unless key.present? - Gitlab::Redis::ClusterSharedState.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.eval(LUA_CANCEL_SCRIPT, keys: [ensure_prefixed_key(key)], argv: [uuid]) end end @@ -79,7 +79,7 @@ module Gitlab # Removes any existing exclusive_lease from redis # Don't run this in a live system without making sure no one is using the leases def self.reset_all!(scope = '*') - Gitlab::Redis::ClusterSharedState.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.scan_each(match: redis_shared_state_key(scope)).each do |key| redis.del(key) end @@ -96,7 +96,7 @@ module Gitlab # false if the lease is already taken. def try_obtain # Performing a single SET is atomic - Gitlab::Redis::ClusterSharedState.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.set(@redis_shared_state_key, @uuid, nx: true, ex: @timeout) && @uuid end end @@ -109,7 +109,7 @@ module Gitlab # Try to renew an existing lease. Return lease UUID on success, # false if the lease is taken by a different UUID or inexistent. def renew - Gitlab::Redis::ClusterSharedState.with do |redis| + Gitlab::Redis::SharedState.with do |redis| result = redis.eval(LUA_RENEW_SCRIPT, keys: [@redis_shared_state_key], argv: [@uuid, @timeout]) result == @uuid end @@ -117,7 +117,7 @@ module Gitlab # Returns true if the key for this lease is set. def exists? - Gitlab::Redis::ClusterSharedState.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.exists?(@redis_shared_state_key) # rubocop:disable CodeReuse/ActiveRecord end end @@ -126,7 +126,7 @@ module Gitlab # # This method will return `nil` if no TTL could be obtained. def ttl - Gitlab::Redis::ClusterSharedState.with do |redis| + Gitlab::Redis::SharedState.with do |redis| ttl = redis.ttl(@redis_shared_state_key) ttl if ttl > 0 diff --git a/lib/gitlab/experiment/rollout/feature.rb b/lib/gitlab/experiment/rollout/feature.rb index 4ff61aa3551..0f2a1b9fb1d 100644 --- a/lib/gitlab/experiment/rollout/feature.rb +++ b/lib/gitlab/experiment/rollout/feature.rb @@ -13,7 +13,7 @@ module Gitlab # no inclusions, etc.) def enabled? return false unless feature_flag_defined? - return false unless Gitlab.com? + return false unless available? return false unless ::Feature.enabled?(:gitlab_experiment, type: :ops) feature_flag_instance.state != :off @@ -57,8 +57,12 @@ module Gitlab private + def available? + ApplicationExperiment.available? + end + def feature_flag_instance - ::Feature.get(feature_flag_name) # rubocop:disable Gitlab/AvoidFeatureGet + ::Feature.get(feature_flag_name) # rubocop:disable Gitlab/AvoidFeatureGet -- We are using at a lower layer here in experiment framework end def feature_flag_defined? diff --git a/lib/gitlab/file_detector.rb b/lib/gitlab/file_detector.rb index ef8f2d4d61b..b586c4b5892 100644 --- a/lib/gitlab/file_detector.rb +++ b/lib/gitlab/file_detector.rb @@ -22,6 +22,7 @@ module Gitlab # Configuration files gitignore: '.gitignore', gitlab_ci: ::Ci::Pipeline::DEFAULT_CONFIG_PATH, + jenkinsfile: 'jenkinsfile', route_map: '.gitlab/route-map.yml', # Dependency files diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 37f593ed551..8cbd1a4ce72 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -12,6 +12,13 @@ module Gitlab TAG_REF_PREFIX = "refs/tags/" BRANCH_REF_PREFIX = "refs/heads/" + # NOTE: We don't use linguist anymore, but we'd still want to support it + # to be backward/GitHub compatible. Using `gitlab-*` prefixed overrides + # going forward would give us a better control and flexibility. + ATTRIBUTE_OVERRIDES = { + generated: %w[gitlab-generated linguist-generated] + }.freeze + CommandError = Class.new(BaseError) CommitError = Class.new(BaseError) OSError = Class.new(BaseError) diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 3744c81f51d..aa59caa4268 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -149,7 +149,7 @@ module Gitlab return if @data == '' # don't mess with submodule blobs # Even if we return early, recalculate whether this blob is binary in - # case a blob was initialized as text but the full data isn't + # case a blob was initialized as text but the full data isn'tspec/requests/api/graphql/mutations/branch_rules/update_spec.rb: @binary = nil return if @loaded_all_data diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 1086ea45a7a..d899ed3ba25 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -28,7 +28,8 @@ module Gitlab SERIALIZE_KEYS = [ :id, :message, :parent_ids, :authored_date, :author_name, :author_email, - :committed_date, :committer_name, :committer_email, :trailers, :referenced_by + :committed_date, :committer_name, :committer_email, + :trailers, :extended_trailers, :referenced_by ].freeze attr_accessor(*SERIALIZE_KEYS) @@ -432,9 +433,17 @@ module Gitlab @committer_email = commit.committer.email.dup @parent_ids = Array(commit.parent_ids) @trailers = commit.trailers.to_h { |t| [t.key, t.value] } + @extended_trailers = parse_commit_trailers(commit.trailers) @referenced_by = Array(commit.referenced_by) end + # Turn the commit trailers into a hash of key: [value, value] arrays + def parse_commit_trailers(trailers) + trailers.each_with_object({}) do |trailer, hash| + (hash[trailer.key] ||= []) << trailer.value + end + end + # Gitaly provides a UNIX timestamp in author.date.seconds, and a timezone # offset in author.timezone. If the latter isn't present, assume UTC. def init_date_from_gitaly(author) diff --git a/lib/gitlab/git/compare.rb b/lib/gitlab/git/compare.rb index ab5245ba7cb..c6d678c9432 100644 --- a/lib/gitlab/git/compare.rb +++ b/lib/gitlab/git/compare.rb @@ -42,6 +42,16 @@ module Gitlab options[:straight] = @straight Gitlab::Git::Diff.between(@repository, @head.id, @base.id, options, *paths) end + + def generated_files + return Set.new unless @base && @head + + changed_paths = @repository + .find_changed_paths([Gitlab::Git::DiffTree.new(@base.id, @head.id)]) + .map(&:path) + + @repository.detect_generated_files(@base.id, changed_paths) + end end end end diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index 743bac62764..e753d356bc6 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -10,7 +10,7 @@ module Gitlab attr_accessor :old_path, :new_path, :a_mode, :b_mode, :diff # Stats properties - attr_accessor :new_file, :renamed_file, :deleted_file + attr_accessor :new_file, :renamed_file, :deleted_file, :generated alias_method :new_file?, :new_file alias_method :deleted_file?, :deleted_file @@ -20,6 +20,7 @@ module Gitlab attr_writer :too_large alias_method :expanded?, :expanded + alias_method :generated?, :generated # The default maximum content size to display a diff patch. # @@ -31,7 +32,18 @@ module Gitlab # persisting limits over that. MAX_PATCH_BYTES_UPPER_BOUND = 500.kilobytes - SERIALIZE_KEYS = %i[diff new_path old_path a_mode b_mode new_file renamed_file deleted_file too_large].freeze + SERIALIZE_KEYS = %i[ + diff + new_path + old_path + a_mode + b_mode + new_file + renamed_file + deleted_file + too_large + generated + ].freeze BINARY_NOTICE_PATTERN = %r{Binary files (.*) and (.*) differ} @@ -79,9 +91,12 @@ module Gitlab # If false, patch raw data will not be included in the diff after # `max_files`, `max_lines` or any of the limits in `limits` are # exceeded + # :generated_files :: + # If the list of generated files is given, those files will be marked + # as generated. def filter_diff_options(options, default_options = {}) allowed_options = [:ignore_whitespace_change, :max_files, :max_lines, - :limits, :expanded, :collect_all_paths] + :limits, :expanded, :collect_all_paths, :generated_files] if default_options actual_defaults = default_options.dup @@ -144,8 +159,9 @@ module Gitlab text.start_with?(BINARY_NOTICE_PATTERN) end end - def initialize(raw_diff, expanded: true, replace_invalid_utf8_chars: true) + def initialize(raw_diff, expanded: true, replace_invalid_utf8_chars: true, generated: nil) @expanded = expanded + @generated = generated case raw_diff when Hash @@ -255,6 +271,10 @@ module Gitlab private + def collapse_generated_file? + generated? && !expanded + end + def encode_diff_to_utf8(replace_invalid_utf8_chars) return unless replace_invalid_utf8_chars && diff_should_be_converted? @@ -300,7 +320,7 @@ module Gitlab ::Gitlab::Metrics.add_event(:patch_hard_limit_bytes_hit) too_large! - elsif collapsed? + elsif collapsed? || collapse_generated_file? collapse! end end diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index c021268a62a..e8b6e5fc181 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -35,6 +35,8 @@ module Gitlab def initialize(iterator, options = {}) @iterator = iterator + @generated_files = options.fetch(:generated_files, nil) + @collapse_generated = options.fetch(:collapse_generated, false) @limits = self.class.limits(options) @enforce_limits = !!options.fetch(:limits, true) @expanded = !!options.fetch(:expanded, true) @@ -164,7 +166,10 @@ module Gitlab i = @array.length @iterator.each do |raw| - diff = Gitlab::Git::Diff.new(raw, expanded: expand_diff?) + options = { expanded: expand_diff? } + options[:generated] = @generated_files.include?(raw.from_path) if @generated_files + + diff = Gitlab::Git::Diff.new(raw, **options) if raw.overflow_marker @overflow = true @@ -193,7 +198,10 @@ module Gitlab break end - diff = Gitlab::Git::Diff.new(raw, expanded: expand_diff?) + # Discard generated field if it is already set when FF is disabled + raw_data = @collapse_generated ? raw : raw.except(:generated) + + diff = Gitlab::Git::Diff.new(raw_data, expanded: expand_diff?) if !expand_diff? && over_safe_limits?(i) && diff.line_count > 0 diff.collapse! diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index db6e6b4d00b..312e05b5f54 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -46,7 +46,7 @@ module Gitlab attr_reader :storage, :gl_repository, :gl_project_path, :container - delegate :list_all_blobs, to: :gitaly_blob_client + delegate :list_all_blobs, :list_blobs, to: :gitaly_blob_client # This remote name has to be stable for all types of repositories that # can join an object pool. If it's structure ever changes, a migration @@ -84,13 +84,6 @@ module Gitlab [self.class, storage, relative_path].hash end - # This method will be removed when Gitaly reaches v1.1. - def path - File.join( - Gitlab.config.repositories.storages[@storage].legacy_disk_path, @relative_path - ) - end - # Default branch in the repository def root_ref(head_only: false) wrapped_gitaly_errors do @@ -102,9 +95,9 @@ module Gitlab gitaly_repository_client.exists? end - def create_repository(default_branch = nil) + def create_repository(default_branch = nil, object_format: nil) wrapped_gitaly_errors do - gitaly_repository_client.create_repository(default_branch) + gitaly_repository_client.create_repository(default_branch, object_format: object_format) rescue GRPC::AlreadyExists => e raise RepositoryExists, e.message end @@ -1214,9 +1207,26 @@ module Gitlab gitaly_repository_client .get_file_attributes(revision, file_paths, attributes) .attribute_infos + .map(&:to_h) end end + def object_format + wrapped_gitaly_errors do + gitaly_repository_client.object_format.format + end + end + + # rubocop: disable CodeReuse/ActiveRecord -- not an active record operation + def detect_generated_files(revision, paths) + return Set.new if paths.blank? + + get_file_attributes(revision, paths, Gitlab::Git::ATTRIBUTE_OVERRIDES[:generated]) + .pluck(:path) + .to_set + end + # rubocop: enable CodeReuse/ActiveRecord + private def repository_info_size_megabytes diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index da38c11ebca..6dee9a404f4 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -283,6 +283,7 @@ module Gitlab def self.execute(storage, service, rpc, request, remote_storage:, timeout:) enforce_gitaly_request_limits(:call) Gitlab::RequestContext.instance.ensure_deadline_not_exceeded! + raise_if_concurrent_ruby! kwargs = request_kwargs(storage, timeout: timeout.to_f, remote_storage: remote_storage) kwargs = yield(kwargs) if block_given? @@ -547,43 +548,10 @@ module Gitlab end end - def self.storage_metadata_file_path(storage) - Gitlab::GitalyClient::StorageSettings.allow_disk_access do - File.join( - Gitlab.config.repositories.storages[storage].legacy_disk_path, GITALY_METADATA_FILENAME - ) - end - end - - def self.can_use_disk?(storage) - cached_value = MUTEX.synchronize do - @can_use_disk ||= {} - @can_use_disk[storage] - end - - return cached_value unless cached_value.nil? - - gitaly_filesystem_id = filesystem_id(storage) - direct_filesystem_id = filesystem_id_from_disk(storage) - - MUTEX.synchronize do - @can_use_disk[storage] = gitaly_filesystem_id.present? && - gitaly_filesystem_id == direct_filesystem_id - end - end - def self.filesystem_id(storage) Gitlab::GitalyClient::ServerService.new(storage).storage_info&.filesystem_id end - def self.filesystem_id_from_disk(storage) - metadata_file = File.read(storage_metadata_file_path(storage)) - metadata_hash = Gitlab::Json.parse(metadata_file) - metadata_hash['gitaly_filesystem_id'] - rescue Errno::ENOENT, Errno::EACCES, JSON::ParserError - nil - end - def self.filesystem_disk_available(storage) Gitlab::GitalyClient::ServerService.new(storage).storage_disk_statistics&.available end @@ -669,5 +637,12 @@ module Gitlab Thread.current[:gitaly_feature_flag_actors] ||= {} end end + + def self.raise_if_concurrent_ruby! + Gitlab::Utils.raise_if_concurrent_ruby!(:gitaly) + rescue StandardError => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) + end + private_class_method :raise_if_concurrent_ruby! end end diff --git a/lib/gitlab/gitaly_client/conflicts_service.rb b/lib/gitlab/gitaly_client/conflicts_service.rb index ffe65307c80..831c5ca1305 100644 --- a/lib/gitlab/gitaly_client/conflicts_service.rb +++ b/lib/gitlab/gitaly_client/conflicts_service.rb @@ -30,8 +30,8 @@ module Gitlab end def conflicts? - skip_content = Feature.enabled?(:skip_conflict_files_in_gitaly, type: :experiment) - list_conflict_files(skip_content: skip_content).any? + list_conflict_files(skip_content: true).any? + rescue GRPC::FailedPrecondition, GRPC::Unknown # The server raises FailedPrecondition when it encounters # ConflictSideMissing, which means a conflict exists but its `theirs` or diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 905588c2afc..882982b3cde 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -288,8 +288,6 @@ module Gitlab def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:, push_options: []) request_enum = QueueEnumerator.new - rebase_sha = nil - response_enum = gitaly_client_call( @repository.storage, :operation_service, @@ -316,16 +314,14 @@ module Gitlab ) ) - perform_next_gitaly_rebase_request(response_enum) do |response| - rebase_sha = response.rebase_sha - end + response = response_enum.next + rebase_sha = response.rebase_sha yield rebase_sha # Second request confirms with gitaly to finalize the rebase request_enum.push(Gitaly::UserRebaseConfirmableRequest.new(apply: true)) - - perform_next_gitaly_rebase_request(response_enum) + response_enum.next rebase_sha rescue GRPC::BadStatus => e @@ -528,20 +524,6 @@ module Gitlab private - def perform_next_gitaly_rebase_request(response_enum) - response = response_enum.next - - if response.pre_receive_error.present? - raise Gitlab::Git::PreReceiveError, response.pre_receive_error - elsif response.git_error.present? - raise Gitlab::Git::Repository::GitError, response.git_error - end - - yield response if block_given? - - response - end - def call_cherry_pick_or_revert(rpc, user:, commit:, branch_name:, message:, start_branch_name:, start_repository:, dry_run:) request_class = "Gitaly::User#{rpc.to_s.camelcase}Request".constantize diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 457380615f7..60d14d18f62 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -114,8 +114,8 @@ module Gitlab end # rubocop: enable Metrics/ParameterLists - def create_repository(default_branch = nil) - request = Gitaly::CreateRepositoryRequest.new(repository: @gitaly_repo, default_branch: encode_binary(default_branch)) + def create_repository(default_branch = nil, object_format: nil) + request = Gitaly::CreateRepositoryRequest.new(repository: @gitaly_repo, default_branch: encode_binary(default_branch), object_format: gitaly_object_format(object_format)) gitaly_client_call(@storage, :repository_service, :create_repository, request, timeout: GitalyClient.fast_timeout) end @@ -363,6 +363,12 @@ module Gitlab gitaly_client_call(@repository.storage, :repository_service, :get_file_attributes, request, timeout: GitalyClient.fast_timeout) end + def object_format + request = Gitaly::ObjectFormatRequest.new(repository: @gitaly_repo) + + gitaly_client_call(@storage, :repository_service, :object_format, request, timeout: GitalyClient.fast_timeout) + end + private def search_results_from_response(gitaly_response, options = {}) @@ -449,6 +455,15 @@ module Gitlab entry end + + def gitaly_object_format(format) + case format + when Repository::FORMAT_SHA1 + Gitaly::ObjectFormat::OBJECT_FORMAT_SHA1 + when Repository::FORMAT_SHA256 + Gitaly::ObjectFormat::OBJECT_FORMAT_SHA256 + end + end end end end diff --git a/lib/gitlab/gitaly_client/storage_settings.rb b/lib/gitlab/gitaly_client/storage_settings.rb index adf0c811274..253d7c4a93e 100644 --- a/lib/gitlab/gitaly_client/storage_settings.rb +++ b/lib/gitlab/gitaly_client/storage_settings.rb @@ -12,7 +12,7 @@ module Gitlab InvalidConfigurationError = Class.new(StandardError) INVALID_STORAGE_MESSAGE = <<~MSG - Storage is invalid because it has no `path` key. + Storage is invalid because it has no `gitaly_address` key. For source installations, update your config/gitlab.yml Refer to gitlab.yml.example for an updated example. If you're using the GitLab Development Kit, you can update your configuration running `gdk reconfigure`. @@ -38,13 +38,15 @@ module Gitlab def initialize(storage) raise InvalidConfigurationError, "expected a Hash, got a #{storage.class.name}" unless storage.is_a?(Hash) - raise InvalidConfigurationError, INVALID_STORAGE_MESSAGE unless storage.has_key?('path') + + @hash = ActiveSupport::HashWithIndifferentAccess.new(storage) + + raise InvalidConfigurationError, INVALID_STORAGE_MESSAGE unless @hash.has_key?('gitaly_address') # Support a nil 'path' field because some of the circuit breaker tests use it. - @legacy_disk_path = File.expand_path(storage['path'], Rails.root) if storage['path'] + @legacy_disk_path = File.expand_path(@hash['path'], Rails.root) if @hash['path'] && @hash['path'] != Deprecated - storage['path'] = Deprecated - @hash = ActiveSupport::HashWithIndifferentAccess.new(storage) + @hash['path'] = Deprecated end def gitaly_address diff --git a/lib/gitlab/github_import.rb b/lib/gitlab/github_import.rb index d48b25842b3..31fe2461e86 100644 --- a/lib/gitlab/github_import.rb +++ b/lib/gitlab/github_import.rb @@ -8,18 +8,12 @@ module Gitlab def self.new_client_for(project, token: nil, host: nil, parallel: true) token_to_use = token || project.import_data&.credentials&.fetch(:user) - token_pool = project.import_data&.credentials&.dig(:additional_access_tokens) - options = { + Client.new( + token_to_use, host: host.presence || self.formatted_import_url(project), per_page: self.per_page(project), parallel: parallel - } - - if token_pool - ClientPool.new(token_pool: token_pool.append(token_to_use), **options) - else - Client.new(token_to_use, **options) - end + ) end # Returns the ID of the ghost user. diff --git a/lib/gitlab/github_import/client_pool.rb b/lib/gitlab/github_import/client_pool.rb deleted file mode 100644 index e8414942d1b..00000000000 --- a/lib/gitlab/github_import/client_pool.rb +++ /dev/null @@ -1,39 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module GithubImport - class ClientPool - delegate_missing_to :best_client - - def initialize(token_pool:, per_page:, parallel:, host: nil) - @token_pool = token_pool - @host = host - @per_page = per_page - @parallel = parallel - end - - # Returns the client with the most remaining requests, or the client with - # the closest rate limit reset time, if all clients are rate limited. - def best_client - clients_with_requests_remaining = clients.select(&:requests_remaining?) - - return clients_with_requests_remaining.max_by(&:remaining_requests) if clients_with_requests_remaining.any? - - clients.min_by(&:rate_limit_resets_in) - end - - private - - def clients - @clients ||= @token_pool.map do |token| - Client.new( - token, - host: @host, - per_page: @per_page, - parallel: @parallel - ) - end - end - end - end -end diff --git a/lib/gitlab/github_import/importer/collaborator_importer.rb b/lib/gitlab/github_import/importer/collaborator_importer.rb index 9a90ea5a4ed..a5e3373bacb 100644 --- a/lib/gitlab/github_import/importer/collaborator_importer.rb +++ b/lib/gitlab/github_import/importer/collaborator_importer.rb @@ -53,6 +53,7 @@ module Gitlab def create_membership!(user_id, access_level) ::ProjectMember.create!( + importing: true, source: project, access_level: access_level, user_id: user_id, diff --git a/lib/gitlab/github_import/importer/events/changed_assignee.rb b/lib/gitlab/github_import/importer/events/changed_assignee.rb index bcf9cd94ad9..23e3f4f4dfa 100644 --- a/lib/gitlab/github_import/importer/events/changed_assignee.rb +++ b/lib/gitlab/github_import/importer/events/changed_assignee.rb @@ -18,6 +18,7 @@ module Gitlab def create_note(issue_event, note_body, author_id) Note.create!( + importing: true, system: true, noteable_type: issuable_type(issue_event), noteable_id: issuable_db_id(issue_event), diff --git a/lib/gitlab/github_import/importer/events/changed_milestone.rb b/lib/gitlab/github_import/importer/events/changed_milestone.rb index 39b92d88b58..f002cfb6478 100644 --- a/lib/gitlab/github_import/importer/events/changed_milestone.rb +++ b/lib/gitlab/github_import/importer/events/changed_milestone.rb @@ -17,10 +17,14 @@ module Gitlab private def create_event(issue_event) + milestone = project.milestones.find_by_title(issue_event.milestone_title) + return unless milestone + attrs = { + importing: true, user_id: author_id(issue_event), created_at: issue_event.created_at, - milestone_id: project.milestones.find_by_title(issue_event.milestone_title)&.id, + milestone_id: milestone.id, action: action(issue_event.event), state: DEFAULT_STATE }.merge(resource_event_belongs_to(issue_event)) diff --git a/lib/gitlab/github_import/importer/events/changed_reviewer.rb b/lib/gitlab/github_import/importer/events/changed_reviewer.rb index 17b1fa4ab45..eb142478b16 100644 --- a/lib/gitlab/github_import/importer/events/changed_reviewer.rb +++ b/lib/gitlab/github_import/importer/events/changed_reviewer.rb @@ -18,6 +18,7 @@ module Gitlab def create_note(issue_event, note_body, review_requester_id) Note.create!( + importing: true, system: true, noteable_type: issuable_type(issue_event), noteable_id: issuable_db_id(issue_event), diff --git a/lib/gitlab/github_import/importer/events/closed.rb b/lib/gitlab/github_import/importer/events/closed.rb index 58d9dbf826c..6058ccda1b5 100644 --- a/lib/gitlab/github_import/importer/events/closed.rb +++ b/lib/gitlab/github_import/importer/events/closed.rb @@ -26,6 +26,7 @@ module Gitlab def create_state_event(issue_event) attrs = { + importing: true, user_id: author_id(issue_event), source_commit: issue_event.commit_id, state: 'closed', diff --git a/lib/gitlab/github_import/importer/events/cross_referenced.rb b/lib/gitlab/github_import/importer/events/cross_referenced.rb index 4fe371e5900..9a67fa1c6fe 100644 --- a/lib/gitlab/github_import/importer/events/cross_referenced.rb +++ b/lib/gitlab/github_import/importer/events/cross_referenced.rb @@ -32,6 +32,7 @@ module Gitlab def create_note(issue_event, note_body, user_id) Note.create!( + importing: true, system: true, noteable_type: issuable_type(issue_event), noteable_id: issuable_db_id(issue_event), diff --git a/lib/gitlab/github_import/importer/events/merged.rb b/lib/gitlab/github_import/importer/events/merged.rb new file mode 100644 index 00000000000..6189fa8f429 --- /dev/null +++ b/lib/gitlab/github_import/importer/events/merged.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module Events + class Merged < BaseImporter + def execute(issue_event) + create_event(issue_event) + create_state_event(issue_event) + end + + private + + def create_event(issue_event) + Event.create!( + project_id: project.id, + author_id: author_id(issue_event), + action: 'merged', + target_type: issuable_type(issue_event), + target_id: issuable_db_id(issue_event), + created_at: issue_event.created_at, + updated_at: issue_event.created_at + ) + end + + def create_state_event(issue_event) + attrs = { + importing: true, + user_id: author_id(issue_event), + source_commit: issue_event.commit_id, + state: 'merged', + close_after_error_tracking_resolve: false, + close_auto_resolve_prometheus_alert: false, + created_at: issue_event.created_at + }.merge(resource_event_belongs_to(issue_event)) + + ResourceStateEvent.create!(attrs) + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/events/renamed.rb b/lib/gitlab/github_import/importer/events/renamed.rb index fb9e08116ba..5d306f9dce7 100644 --- a/lib/gitlab/github_import/importer/events/renamed.rb +++ b/lib/gitlab/github_import/importer/events/renamed.rb @@ -13,6 +13,7 @@ module Gitlab def note_params(issue_event) { + importing: true, noteable_id: issuable_db_id(issue_event), noteable_type: issuable_type(issue_event), project_id: project.id, diff --git a/lib/gitlab/github_import/importer/issue_event_importer.rb b/lib/gitlab/github_import/importer/issue_event_importer.rb index 80749aae93c..d20482eca6f 100644 --- a/lib/gitlab/github_import/importer/issue_event_importer.rb +++ b/lib/gitlab/github_import/importer/issue_event_importer.rb @@ -6,6 +6,22 @@ module Gitlab class IssueEventImporter attr_reader :issue_event, :project, :client + SUPPORTED_EVENTS = %w[ + assigned + closed + cross-referenced + demilestoned + labeled + merged + milestoned + renamed + reopened + review_request_removed + review_requested + unassigned + unlabeled + ].freeze + # issue_event - An instance of `Gitlab::GithubImport::Representation::IssueEvent`. # project - An instance of `Project`. # client - An instance of `Gitlab::GithubImport::Client`. @@ -47,6 +63,8 @@ module Gitlab Gitlab::GithubImport::Importer::Events::ChangedAssignee when 'review_requested', 'review_request_removed' Gitlab::GithubImport::Importer::Events::ChangedReviewer + when 'merged' + Gitlab::GithubImport::Importer::Events::Merged end end end diff --git a/lib/gitlab/github_import/importer/pull_requests/review_importer.rb b/lib/gitlab/github_import/importer/pull_requests/review_importer.rb index b250a42a53c..6df130eb6e8 100644 --- a/lib/gitlab/github_import/importer/pull_requests/review_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests/review_importer.rb @@ -5,6 +5,8 @@ module Gitlab module Importer module PullRequests class ReviewImporter + include ::Gitlab::Import::MergeRequestHelpers + # review - An instance of `Gitlab::GithubImport::Representation::PullRequestReview` # project - An instance of `Project` # client - An instance of `Gitlab::GithubImport::Client` @@ -83,52 +85,11 @@ module Gitlab def add_approval!(user_id) return unless review.review_type == 'APPROVED' - approval_attribues = { - merge_request_id: merge_request.id, - user_id: user_id, - created_at: submitted_at, - updated_at: submitted_at - } - - result = ::Approval.insert( - approval_attribues, - returning: [:id], - unique_by: [:user_id, :merge_request_id] - ) - - add_approval_system_note!(user_id) if result.rows.present? + create_approval!(project.id, merge_request.id, user_id, submitted_at) end def add_reviewer!(user_id) - return if review_re_requested?(user_id) - - ::MergeRequestReviewer.create!( - merge_request_id: merge_request.id, - user_id: user_id, - state: ::MergeRequestReviewer.states['reviewed'], - created_at: submitted_at - ) - rescue ActiveRecord::RecordNotUnique - # multiple reviews from single person could make a SQL concurrency issue here - nil - end - - # rubocop:disable CodeReuse/ActiveRecord - def review_re_requested?(user_id) - # records that were imported on previous stage with "unreviewed" status - MergeRequestReviewer.where(merge_request_id: merge_request.id, user_id: user_id).exists? - end - # rubocop:enable CodeReuse/ActiveRecord - - def add_approval_system_note!(user_id) - attributes = note_attributes( - user_id, - 'approved this merge request', - system: true, - system_note_metadata: SystemNoteMetadata.new(action: 'approved') - ) - - Note.create!(attributes) + create_reviewer!(merge_request.id, user_id, submitted_at) end def submitted_at diff --git a/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb b/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb index e0a7e6479f5..d7fa098a775 100644 --- a/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb +++ b/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb @@ -29,7 +29,8 @@ module Gitlab associated = associated.to_h compose_associated_id!(parent_record, associated) - return if already_imported?(associated) + + return if already_imported?(associated) || importer_class::SUPPORTED_EVENTS.exclude?(associated[:event]) Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) diff --git a/lib/gitlab/github_import/issuable_finder.rb b/lib/gitlab/github_import/issuable_finder.rb index 0780ba6119f..5ce50e5b4e7 100644 --- a/lib/gitlab/github_import/issuable_finder.rb +++ b/lib/gitlab/github_import/issuable_finder.rb @@ -26,8 +26,6 @@ module Gitlab def database_id val = Gitlab::Cache::Import::Caching.read_integer(cache_key, timeout: timeout) - return val if Feature.disabled?(:import_fallback_to_db_empty_cache, project) - return if val == CACHE_OBJECT_NOT_FOUND return val if val.present? diff --git a/lib/gitlab/github_import/job_delay_calculator.rb b/lib/gitlab/github_import/job_delay_calculator.rb index 077a27df16c..50cad1aae19 100644 --- a/lib/gitlab/github_import/job_delay_calculator.rb +++ b/lib/gitlab/github_import/job_delay_calculator.rb @@ -7,7 +7,9 @@ module Gitlab module JobDelayCalculator # Default batch settings for parallel import (can be redefined in Importer/Worker classes) def parallel_import_batch - { size: 1000, delay: 1.minute } + batch_size = Feature.enabled?(:github_import_increased_concurrent_workers, project.creator) ? 5000 : 1000 + + { size: batch_size, delay: 1.minute } end private diff --git a/lib/gitlab/github_import/label_finder.rb b/lib/gitlab/github_import/label_finder.rb index d0bbd2bc7cf..87d3195eb93 100644 --- a/lib/gitlab/github_import/label_finder.rb +++ b/lib/gitlab/github_import/label_finder.rb @@ -19,8 +19,6 @@ module Gitlab cache_key = cache_key_for(name) val = Gitlab::Cache::Import::Caching.read_integer(cache_key) - return val if Feature.disabled?(:import_fallback_to_db_empty_cache, project) - return if val == CACHE_OBJECT_NOT_FOUND return val if val.present? diff --git a/lib/gitlab/github_import/milestone_finder.rb b/lib/gitlab/github_import/milestone_finder.rb index dcb679fda6d..fd60fa86e82 100644 --- a/lib/gitlab/github_import/milestone_finder.rb +++ b/lib/gitlab/github_import/milestone_finder.rb @@ -24,8 +24,6 @@ module Gitlab val = Gitlab::Cache::Import::Caching.read_integer(cache_key) - return val if Feature.disabled?(:import_fallback_to_db_empty_cache, project) - return if val == CACHE_OBJECT_NOT_FOUND return val if val.present? diff --git a/lib/gitlab/github_import/representation/collaborator.rb b/lib/gitlab/github_import/representation/collaborator.rb index fb58a572151..3e3706f05b5 100644 --- a/lib/gitlab/github_import/representation/collaborator.rb +++ b/lib/gitlab/github_import/representation/collaborator.rb @@ -4,10 +4,7 @@ module Gitlab module GithubImport module Representation class Collaborator - include ToHash - include ExposeAttribute - - attr_reader :attributes + include Representable expose_attribute :id, :login, :role_name diff --git a/lib/gitlab/github_import/representation/diff_note.rb b/lib/gitlab/github_import/representation/diff_note.rb index e8e515d1f87..f678fe38688 100644 --- a/lib/gitlab/github_import/representation/diff_note.rb +++ b/lib/gitlab/github_import/representation/diff_note.rb @@ -4,8 +4,7 @@ module Gitlab module GithubImport module Representation class DiffNote - include ToHash - include ExposeAttribute + include Representable NOTEABLE_ID_REGEX = %r{/pull/(?<iid>\d+)}i @@ -134,9 +133,6 @@ module Gitlab private - # Required by ExposeAttribute - attr_reader :attributes - def diff_line_params if addition? { new_line: end_line, old_line: nil } diff --git a/lib/gitlab/github_import/representation/issue.rb b/lib/gitlab/github_import/representation/issue.rb index 95a7c5ebf4b..8c072c0ed06 100644 --- a/lib/gitlab/github_import/representation/issue.rb +++ b/lib/gitlab/github_import/representation/issue.rb @@ -4,10 +4,7 @@ module Gitlab module GithubImport module Representation class Issue - include ToHash - include ExposeAttribute - - attr_reader :attributes + include Representable expose_attribute :iid, :title, :description, :milestone_number, :created_at, :updated_at, :state, :assignees, diff --git a/lib/gitlab/github_import/representation/issue_event.rb b/lib/gitlab/github_import/representation/issue_event.rb index 068d5cf9482..30608112f85 100644 --- a/lib/gitlab/github_import/representation/issue_event.rb +++ b/lib/gitlab/github_import/representation/issue_event.rb @@ -4,10 +4,7 @@ module Gitlab module GithubImport module Representation class IssueEvent - include ToHash - include ExposeAttribute - - attr_reader :attributes + include Representable expose_attribute :id, :actor, :event, :commit_id, :label_title, :old_title, :new_title, :milestone_title, :issue, :source, :assignee, :review_requester, diff --git a/lib/gitlab/github_import/representation/lfs_object.rb b/lib/gitlab/github_import/representation/lfs_object.rb index 716e77bf401..153a1680577 100644 --- a/lib/gitlab/github_import/representation/lfs_object.rb +++ b/lib/gitlab/github_import/representation/lfs_object.rb @@ -4,10 +4,7 @@ module Gitlab module GithubImport module Representation class LfsObject - include ToHash - include ExposeAttribute - - attr_reader :attributes + include Representable expose_attribute :oid, :link, :size diff --git a/lib/gitlab/github_import/representation/note.rb b/lib/gitlab/github_import/representation/note.rb index 76adbb651af..308cab08dea 100644 --- a/lib/gitlab/github_import/representation/note.rb +++ b/lib/gitlab/github_import/representation/note.rb @@ -4,10 +4,7 @@ module Gitlab module GithubImport module Representation class Note - include ToHash - include ExposeAttribute - - attr_reader :attributes + include Representable expose_attribute :noteable_id, :noteable_type, :author, :note, :created_at, :updated_at, :note_id diff --git a/lib/gitlab/github_import/representation/note_text.rb b/lib/gitlab/github_import/representation/note_text.rb index 70dd242303a..43e18a923d6 100644 --- a/lib/gitlab/github_import/representation/note_text.rb +++ b/lib/gitlab/github_import/representation/note_text.rb @@ -8,14 +8,11 @@ module Gitlab module GithubImport module Representation class NoteText - include ToHash - include ExposeAttribute + include Representable MODELS_ALLOWLIST = [::Release, ::Note, ::Issue, ::MergeRequest].freeze ModelNotSupported = Class.new(StandardError) - attr_reader :attributes - expose_attribute :record_db_id, :record_type, :text, :iid, :tag, :noteable_type # Builds a note text representation from DB record of Note or Release. diff --git a/lib/gitlab/github_import/representation/protected_branch.rb b/lib/gitlab/github_import/representation/protected_branch.rb index eb9dd3bc247..0b755f0c79d 100644 --- a/lib/gitlab/github_import/representation/protected_branch.rb +++ b/lib/gitlab/github_import/representation/protected_branch.rb @@ -4,10 +4,7 @@ module Gitlab module GithubImport module Representation class ProtectedBranch - include ToHash - include ExposeAttribute - - attr_reader :attributes + include Representable expose_attribute :id, :allow_force_pushes, :required_conversation_resolution, :required_signatures, :required_pull_request_reviews, :require_code_owner_reviews, :allowed_to_push_users diff --git a/lib/gitlab/github_import/representation/pull_request.rb b/lib/gitlab/github_import/representation/pull_request.rb index f26fa953773..370d3b541f0 100644 --- a/lib/gitlab/github_import/representation/pull_request.rb +++ b/lib/gitlab/github_import/representation/pull_request.rb @@ -4,10 +4,7 @@ module Gitlab module GithubImport module Representation class PullRequest - include ToHash - include ExposeAttribute - - attr_reader :attributes + include Representable expose_attribute :iid, :title, :description, :source_branch, :source_branch_sha, :target_branch, :target_branch_sha, diff --git a/lib/gitlab/github_import/representation/pull_request_review.rb b/lib/gitlab/github_import/representation/pull_request_review.rb index 0c6e281cd6d..86e32bbab7b 100644 --- a/lib/gitlab/github_import/representation/pull_request_review.rb +++ b/lib/gitlab/github_import/representation/pull_request_review.rb @@ -4,10 +4,7 @@ module Gitlab module GithubImport module Representation class PullRequestReview - include ToHash - include ExposeAttribute - - attr_reader :attributes + include Representable expose_attribute :author, :note, :review_type, :submitted_at, :merge_request_id, :merge_request_iid, :review_id diff --git a/lib/gitlab/github_import/representation/pull_requests/review_requests.rb b/lib/gitlab/github_import/representation/pull_requests/review_requests.rb index a6ec1d3178b..a3ca5cb644d 100644 --- a/lib/gitlab/github_import/representation/pull_requests/review_requests.rb +++ b/lib/gitlab/github_import/representation/pull_requests/review_requests.rb @@ -5,10 +5,7 @@ module Gitlab module Representation module PullRequests class ReviewRequests - include ToHash - include ExposeAttribute - - attr_reader :attributes + include Representable expose_attribute :merge_request_id, :merge_request_iid, :users diff --git a/lib/gitlab/github_import/representation/representable.rb b/lib/gitlab/github_import/representation/representable.rb new file mode 100644 index 00000000000..49095d4c819 --- /dev/null +++ b/lib/gitlab/github_import/representation/representable.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + module Representable + extend ActiveSupport::Concern + + included do + include ToHash + include ExposeAttribute + + def github_identifiers + error = NotImplementedError.new('Subclasses must implement #github_identifiers') + + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) + + {} + end + + private + + attr_reader :attributes + end + end + end + end +end diff --git a/lib/gitlab/github_import/representation/user.rb b/lib/gitlab/github_import/representation/user.rb index 02cbe037384..6f172d8fb91 100644 --- a/lib/gitlab/github_import/representation/user.rb +++ b/lib/gitlab/github_import/representation/user.rb @@ -4,10 +4,7 @@ module Gitlab module GithubImport module Representation class User - include ToHash - include ExposeAttribute - - attr_reader :attributes + include Representable expose_attribute :id, :login diff --git a/lib/gitlab/github_import/settings.rb b/lib/gitlab/github_import/settings.rb index a4170f4147f..3947ae3c63d 100644 --- a/lib/gitlab/github_import/settings.rb +++ b/lib/gitlab/github_import/settings.rb @@ -57,16 +57,13 @@ module Gitlab user_settings = user_settings.to_h.with_indifferent_access optional_stages = fetch_stages_from_params(user_settings[:optional_stages]) - credentials = project.import_data&.credentials&.merge( - additional_access_tokens: user_settings[:additional_access_tokens] - ) import_data = project.build_or_assign_import_data( data: { optional_stages: optional_stages, timeout_strategy: user_settings[:timeout_strategy] }, - credentials: credentials + credentials: project.import_data&.credentials ) import_data.save! diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 59813e4f5a0..caf7cfb3f76 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -56,7 +56,6 @@ module Gitlab gon.dot_com = Gitlab.com? gon.uf_error_prefix = ::Gitlab::Utils::ErrorMessage::UF_ERROR_PREFIX gon.pat_prefix = Gitlab::CurrentSettings.current_application_settings.personal_access_token_prefix - gon.use_new_navigation = NavHelper.show_super_sidebar?(current_user) gon.keyboard_shortcuts_enabled = current_user ? current_user.keyboard_shortcuts_enabled : true gon.diagramsnet_url = Gitlab::CurrentSettings.diagramsnet_url if Gitlab::CurrentSettings.diagramsnet_enabled @@ -77,9 +76,12 @@ module Gitlab push_frontend_feature_flag(:security_auto_fix) push_frontend_feature_flag(:source_editor_toolbar) push_frontend_feature_flag(:vscode_web_ide, current_user) + push_frontend_feature_flag(:key_contacts_management, current_user) # To be removed with https://gitlab.com/gitlab-org/gitlab/-/issues/399248 push_frontend_feature_flag(:remove_monitor_metrics) push_frontend_feature_flag(:custom_emoji) + push_frontend_feature_flag(:encoding_logs_tree) + push_frontend_feature_flag(:group_user_saml) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/graphql/loaders/full_path_model_loader.rb b/lib/gitlab/graphql/loaders/full_path_model_loader.rb index a99b8c81930..7de4956a668 100644 --- a/lib/gitlab/graphql/loaders/full_path_model_loader.rb +++ b/lib/gitlab/graphql/loaders/full_path_model_loader.rb @@ -19,9 +19,7 @@ module Gitlab scope = args[:key] # this logic cannot be placed in the NamespaceResolver due to N+1 scope = scope.without_project_namespaces if scope == Namespace - # `with_route` avoids an N+1 calculating full_path - scope = scope.where_full_path_in(full_paths).with_route - .allow_cross_joins_across_databases(url: "https://gitlab.com/gitlab-org/gitlab/-/issues/420046") + scope = scope.where_full_path_in(full_paths) scope.each do |model_instance| loader.call(model_instance.full_path.downcase, model_instance) diff --git a/lib/gitlab/group_search_results.rb b/lib/gitlab/group_search_results.rb index 6fe7a0030f0..b112740c4ad 100644 --- a/lib/gitlab/group_search_results.rb +++ b/lib/gitlab/group_search_results.rb @@ -13,7 +13,6 @@ module Gitlab # rubocop:disable CodeReuse/ActiveRecord def users groups = group.self_and_hierarchy_intersecting_with_user_groups(current_user) - groups = groups.allow_cross_joins_across_databases(url: "https://gitlab.com/gitlab-org/gitlab/-/issues/427108") members = GroupMember.where(group: groups).non_invite users = super diff --git a/lib/gitlab/hook_data/project_builder.rb b/lib/gitlab/hook_data/project_builder.rb index aec842e061f..56b8b842a78 100644 --- a/lib/gitlab/hook_data/project_builder.rb +++ b/lib/gitlab/hook_data/project_builder.rb @@ -33,7 +33,6 @@ module Gitlab private def project_data - owners = project.owners.compact # When this is removed, also remove the `deprecated_owner` method # See https://gitlab.com/gitlab-org/gitlab/-/issues/350603 owner = project.deprecated_owner @@ -45,13 +44,27 @@ module Gitlab project_id: project.id, owner_name: owner.try(:name), owner_email: user_email(owner), - owners: owners.map do |owner| - owner_data(owner) - end, + owners: owners_data, project_visibility: project.visibility.downcase } end + def owners_data + # Extracted code from ProjectTeam#owners, but works without creating cross joins queries + # Can be consolidate again once https://gitlab.com/gitlab-org/gitlab/-/issues/432606 is addressed + if project.group + project.group.all_owner_members.select(:id, :user_id) + .preload_user.find_each.map { |member| owner_data(member.user) } + else + data = [] + project.project_authorizations.owners.preload_user.each_batch(column: :user_id) do |relation| + data.concat(relation.map { |member| owner_data(member.user) }) + end + data |= Array.wrap(owner_data(project.owner)) if project.owner + data + end + end + def owner_data(user) { name: user.name, email: user_email(user) } end diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 96e3d90c139..02afdedb4be 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -44,8 +44,8 @@ module Gitlab TRANSLATION_LEVELS = { 'bg' => 0, 'cs_CZ' => 0, - 'da_DK' => 29, - 'de' => 97, + 'da_DK' => 28, + 'de' => 95, 'en' => 100, 'eo' => 0, 'es' => 28, @@ -56,18 +56,18 @@ module Gitlab 'it' => 1, 'ja' => 98, 'ko' => 23, - 'nb_NO' => 21, + 'nb_NO' => 20, 'nl_NL' => 0, 'pl_PL' => 3, - 'pt_BR' => 57, - 'ro_RO' => 76, + 'pt_BR' => 60, + 'ro_RO' => 74, 'ru' => 21, - 'si_LK' => 12, + 'si_LK' => 11, 'tr_TR' => 8, - 'uk' => 52, + 'uk' => 51, 'zh_CN' => 99, 'zh_HK' => 1, - 'zh_TW' => 100 + 'zh_TW' => 99 }.freeze private_constant :TRANSLATION_LEVELS diff --git a/lib/gitlab/import/merge_request_helpers.rb b/lib/gitlab/import/merge_request_helpers.rb index 9fd393c61a0..bcaae530927 100644 --- a/lib/gitlab/import/merge_request_helpers.rb +++ b/lib/gitlab/import/merge_request_helpers.rb @@ -69,6 +69,52 @@ module Gitlab rows = reviewers.map { |reviewer_id| { merge_request_id: merge_request.id, user_id: reviewer_id } } MergeRequestReviewer.insert_all(rows) end + + def create_approval!(project_id, merge_request_id, user_id, submitted_at) + approval_attributes = { + merge_request_id: merge_request_id, + user_id: user_id, + created_at: submitted_at, + updated_at: submitted_at + } + + result = ::Approval.insert( + approval_attributes, + returning: [:id], + unique_by: [:user_id, :merge_request_id] + ) + + add_approval_system_note!(project_id, merge_request_id, user_id, submitted_at) if result.rows.present? + end + + def add_approval_system_note!(project_id, merge_request_id, user_id, submitted_at) + attributes = { + importing: true, + noteable_id: merge_request_id, + noteable_type: 'MergeRequest', + project_id: project_id, + author_id: user_id, + note: 'approved this merge request', + system: true, + system_note_metadata: SystemNoteMetadata.new(action: 'approved'), + created_at: submitted_at, + updated_at: submitted_at + } + + Note.create!(attributes) + end + + def create_reviewer!(merge_request_id, user_id, submitted_at) + ::MergeRequestReviewer.create!( + merge_request_id: merge_request_id, + user_id: user_id, + state: ::MergeRequestReviewer.states['reviewed'], + created_at: submitted_at + ) + rescue ActiveRecord::RecordNotUnique + # multiple reviews from single person could make a SQL concurrency issue here + nil + end end end end diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 6f3601e9a21..e38930ed548 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -318,6 +318,7 @@ included_attributes: - :releases_access_level - :infrastructure_access_level - :model_experiments_access_level + - :model_registry_access_level prometheus_metrics: - :created_at - :updated_at @@ -738,6 +739,7 @@ included_attributes: - :releases_access_level - :infrastructure_access_level - :model_experiments_access_level + - :model_registry_access_level - :auto_devops_deploy_strategy - :auto_devops_enabled - :container_registry_enabled diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index fec8b3a7708..6e507142e88 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -11,7 +11,7 @@ module Gitlab IMPORT_TABLE = [ ImportSource.new('github', 'GitHub', Gitlab::GithubImport::ParallelImporter), - ImportSource.new('bitbucket', 'Bitbucket Cloud', Gitlab::BitbucketImport::Importer), + ImportSource.new('bitbucket', 'Bitbucket Cloud', Gitlab::BitbucketImport::ParallelImporter), ImportSource.new('bitbucket_server', 'Bitbucket Server', Gitlab::BitbucketServerImport::ParallelImporter), ImportSource.new('fogbugz', 'FogBugz', Gitlab::FogbugzImport::Importer), ImportSource.new('git', 'Repository by URL', nil), @@ -44,15 +44,7 @@ module Gitlab end def import_table - bitbucket_parallel_enabled = Feature.enabled?(:bitbucket_parallel_importer) - - return IMPORT_TABLE unless bitbucket_parallel_enabled - - import_table = IMPORT_TABLE.deep_dup - - import_table[1].importer = Gitlab::BitbucketImport::ParallelImporter if bitbucket_parallel_enabled - - import_table + IMPORT_TABLE end end end diff --git a/lib/gitlab/inactive_projects_deletion_warning_tracker.rb b/lib/gitlab/inactive_projects_deletion_warning_tracker.rb index 3fdb34d42b7..560c113fb5f 100644 --- a/lib/gitlab/inactive_projects_deletion_warning_tracker.rb +++ b/lib/gitlab/inactive_projects_deletion_warning_tracker.rb @@ -36,7 +36,7 @@ module Gitlab def mark_notified Gitlab::Redis::SharedState.with do |redis| - redis.hset(DELETION_TRACKING_REDIS_KEY, "project:#{project_id}", Date.current) + redis.hset(DELETION_TRACKING_REDIS_KEY, "project:#{project_id}", Date.current.to_s) end end @@ -47,8 +47,9 @@ module Gitlab end def scheduled_deletion_date - if notification_date.present? - (notification_date.to_date + grace_period_after_notification).to_s + notif_date = notification_date + if notif_date.present? + (notif_date.to_date + grace_period_after_notification).to_s else grace_period_after_notification.from_now.to_date.to_s end diff --git a/lib/gitlab/instrumentation/connection_pool.rb b/lib/gitlab/instrumentation/connection_pool.rb new file mode 100644 index 00000000000..76e6af34054 --- /dev/null +++ b/lib/gitlab/instrumentation/connection_pool.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module Instrumentation + # rubocop:disable Gitlab/ModuleWithInstanceVariables -- this module patches ConnectionPool to instrument it + module ConnectionPool + def initialize(options = {}, &block) + @name = options.fetch(:name, 'unknown') + + super + end + + def checkout(options = {}) + conn = super + + connection_class = conn.class.to_s + track_available_connections(connection_class) + track_pool_size(connection_class) + + conn + end + + def track_pool_size(connection_class) + # this means that the size metric for this pool key has been sent + return if @size_gauge + + @size_gauge ||= ::Gitlab::Metrics.gauge(:gitlab_connection_pool_size, 'Size of connection pool', {}, :all) + @size_gauge.set({ pool_name: @name, pool_key: @key, connection_class: connection_class }, @size) + end + + def track_available_connections(connection_class) + @available_gauge ||= ::Gitlab::Metrics.gauge(:gitlab_connection_pool_available_count, + 'Number of available connections in the pool', {}, :all) + + @available_gauge.set({ pool_name: @name, pool_key: @key, connection_class: connection_class }, available) + end + end + # rubocop:enable Gitlab/ModuleWithInstanceVariables + end +end diff --git a/lib/gitlab/instrumentation/redis_base.rb b/lib/gitlab/instrumentation/redis_base.rb index 88991495a10..1e117172c3a 100644 --- a/lib/gitlab/instrumentation/redis_base.rb +++ b/lib/gitlab/instrumentation/redis_base.rb @@ -128,6 +128,11 @@ module Gitlab @exception_counter.increment({ storage: storage_key, exception: ex.class.to_s }) end + def instance_count_connection_exception(ex) + @connection_exception_counter ||= Gitlab::Metrics.counter(:gitlab_redis_client_connection_exceptions_total, 'Client side Redis connection exception count, per Redis server, per exception class') + @connection_exception_counter.increment({ storage: storage_key, exception: ex.class.to_s }) + end + def instance_count_cluster_redirection(ex) # This metric is meant to give a client side view of how often are commands # redirected to the right node, especially during resharding.. diff --git a/lib/gitlab/instrumentation/redis_helper.rb b/lib/gitlab/instrumentation/redis_helper.rb new file mode 100644 index 00000000000..ba1c8132250 --- /dev/null +++ b/lib/gitlab/instrumentation/redis_helper.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +module Gitlab + module Instrumentation + module RedisHelper + APDEX_EXCLUDE = %w[brpop blpop brpoplpush bzpopmin bzpopmax command xread xreadgroup].freeze + + def instrument_call(commands, instrumentation_class, pipelined = false) + start = Gitlab::Metrics::System.monotonic_time # must come first so that 'start' is always defined + instrumentation_class.instance_count_request(commands.size) + instrumentation_class.instance_count_pipelined_request(commands.size) if pipelined + + if !instrumentation_class.redis_cluster_validate!(commands) && ::RequestStore.active? + instrumentation_class.increment_cross_slot_request_count + end + + yield + rescue ::Redis::BaseError => ex + if ex.message.start_with?('MOVED', 'ASK') + instrumentation_class.instance_count_cluster_redirection(ex) + else + instrumentation_class.instance_count_exception(ex) + end + + instrumentation_class.log_exception(ex) + raise ex + ensure + duration = Gitlab::Metrics::System.monotonic_time - start + + unless exclude_from_apdex?(commands) + commands.each { instrumentation_class.instance_observe_duration(duration / commands.size) } + end + + if ::RequestStore.active? + # These metrics measure total Redis usage per Rails request / job. + instrumentation_class.increment_request_count(commands.size) + instrumentation_class.add_duration(duration) + instrumentation_class.add_call_details(duration, commands) + end + end + + def measure_write_size(command, instrumentation_class) + size = 0 + + # Mimic what happens in + # https://github.com/redis/redis-rb/blob/f597f21a6b954b685cf939febbc638f6c803e3a7/lib/redis/connection/command_helper.rb#L8. + # This count is an approximation that omits the Redis protocol overhead + # of type prefixes, length prefixes and line endings. + command.each do |x| + size += if x.is_a? Array + x.inject(0) { |sum, y| sum + y.to_s.bytesize } + else + x.to_s.bytesize + end + end + + instrumentation_class.increment_write_bytes(size) + end + + def measure_read_size(result, instrumentation_class) + # The Connection::Ruby#read class can return one of four types of results from read: + # https://github.com/redis/redis-rb/blob/f597f21a6b954b685cf939febbc638f6c803e3a7/lib/redis/connection/ruby.rb#L406 + # + # 1. Error (exception, will not reach this line) + # 2. Status (string) + # 3. Integer (will be converted to string by to_s.bytesize and thrown away) + # 4. "Binary" string (i.e. may contain zero byte) + # 5. Array of binary string + + if result.is_a? Array + # Redis can return nested arrays, e.g. from XRANGE or GEOPOS, so we use recursion here. + result.each { |x| measure_read_size(x, instrumentation_class) } + else + # This count is an approximation that omits the Redis protocol overhead + # of type prefixes, length prefixes and line endings. + instrumentation_class.increment_read_bytes(result.to_s.bytesize) + end + end + + def exclude_from_apdex?(commands) + commands.any? { |command| APDEX_EXCLUDE.include?(command.first.to_s.downcase) } + end + end + end +end diff --git a/lib/gitlab/instrumentation/redis_interceptor.rb b/lib/gitlab/instrumentation/redis_interceptor.rb index 5934204bd0f..9c89af6a0dc 100644 --- a/lib/gitlab/instrumentation/redis_interceptor.rb +++ b/lib/gitlab/instrumentation/redis_interceptor.rb @@ -3,103 +3,45 @@ module Gitlab module Instrumentation module RedisInterceptor - APDEX_EXCLUDE = %w[brpop blpop brpoplpush bzpopmin bzpopmax command xread xreadgroup].freeze + include RedisHelper def call(command) - instrument_call([command]) do + instrument_call([command], instrumentation_class) do super end end def call_pipeline(pipeline) - instrument_call(pipeline.commands, true) do + instrument_call(pipeline.commands, instrumentation_class, true) do super end end def write(command) - measure_write_size(command) if ::RequestStore.active? + measure_write_size(command, instrumentation_class) if ::RequestStore.active? super end def read result = super - measure_read_size(result) if ::RequestStore.active? + measure_read_size(result, instrumentation_class) if ::RequestStore.active? result end - private - - def instrument_call(commands, pipelined = false) - start = ::Gitlab::Metrics::System.monotonic_time # must come first so that 'start' is always defined - instrumentation_class.instance_count_request(commands.size) - instrumentation_class.instance_count_pipelined_request(commands.size) if pipelined - - if !instrumentation_class.redis_cluster_validate!(commands) && ::RequestStore.active? - instrumentation_class.increment_cross_slot_request_count + def ensure_connected + super do + instrument_reconnection_errors do + yield + end end + end + def instrument_reconnection_errors yield - rescue ::Redis::BaseError => ex - if ex.message.start_with?('MOVED', 'ASK') - instrumentation_class.instance_count_cluster_redirection(ex) - else - instrumentation_class.instance_count_exception(ex) - end + rescue ::Redis::BaseConnectionError => ex + instrumentation_class.instance_count_connection_exception(ex) - instrumentation_class.log_exception(ex) raise ex - ensure - duration = ::Gitlab::Metrics::System.monotonic_time - start - - unless exclude_from_apdex?(commands) - commands.each { instrumentation_class.instance_observe_duration(duration / commands.size) } - end - - if ::RequestStore.active? - # These metrics measure total Redis usage per Rails request / job. - instrumentation_class.increment_request_count(commands.size) - instrumentation_class.add_duration(duration) - instrumentation_class.add_call_details(duration, commands) - end - end - - def measure_write_size(command) - size = 0 - - # Mimic what happens in - # https://github.com/redis/redis-rb/blob/f597f21a6b954b685cf939febbc638f6c803e3a7/lib/redis/connection/command_helper.rb#L8. - # This count is an approximation that omits the Redis protocol overhead - # of type prefixes, length prefixes and line endings. - command.each do |x| - size += if x.is_a? Array - x.inject(0) { |sum, y| sum + y.to_s.bytesize } - else - x.to_s.bytesize - end - end - - instrumentation_class.increment_write_bytes(size) - end - - def measure_read_size(result) - # The Connection::Ruby#read class can return one of four types of results from read: - # https://github.com/redis/redis-rb/blob/f597f21a6b954b685cf939febbc638f6c803e3a7/lib/redis/connection/ruby.rb#L406 - # - # 1. Error (exception, will not reach this line) - # 2. Status (string) - # 3. Integer (will be converted to string by to_s.bytesize and thrown away) - # 4. "Binary" string (i.e. may contain zero byte) - # 5. Array of binary string - - if result.is_a? Array - # Redis can return nested arrays, e.g. from XRANGE or GEOPOS, so we use recursion here. - result.each { |x| measure_read_size(x) } - else - # This count is an approximation that omits the Redis protocol overhead - # of type prefixes, length prefixes and line endings. - instrumentation_class.increment_read_bytes(result.to_s.bytesize) - end end # That's required so it knows which GitLab Redis instance @@ -108,10 +50,6 @@ module Gitlab def instrumentation_class @options[:instrumentation_class] # rubocop:disable Gitlab/ModuleWithInstanceVariables end - - def exclude_from_apdex?(commands) - commands.any? { |command| APDEX_EXCLUDE.include?(command.first.to_s.downcase) } - end end end end diff --git a/lib/gitlab/internal_events.rb b/lib/gitlab/internal_events.rb index e2e4ea75dbf..eb2ba3449fb 100644 --- a/lib/gitlab/internal_events.rb +++ b/lib/gitlab/internal_events.rb @@ -4,30 +4,61 @@ module Gitlab module InternalEvents UnknownEventError = Class.new(StandardError) InvalidPropertyError = Class.new(StandardError) - InvalidMethodError = Class.new(StandardError) + InvalidPropertyTypeError = Class.new(StandardError) class << self include Gitlab::Tracking::Helpers + include Gitlab::Utils::StrongMemoize def track_event(event_name, send_snowplow_event: true, **kwargs) raise UnknownEventError, "Unknown event: #{event_name}" unless EventDefinitions.known_event?(event_name) + validate_property!(kwargs, :user, User) + validate_property!(kwargs, :namespace, Namespaces::UserNamespace, Group) + validate_property!(kwargs, :project, Project) + + project = kwargs[:project] + kwargs[:namespace] ||= project.namespace if project + increase_total_counter(event_name) + increase_weekly_total_counter(event_name) update_unique_counter(event_name, kwargs) trigger_snowplow_event(event_name, kwargs) if send_snowplow_event + + if Feature.enabled?(:internal_events_for_product_analytics) + send_application_instrumentation_event(event_name, kwargs) + end rescue StandardError => e - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, event_name: event_name, kwargs: kwargs) + extra = {} + kwargs.each_key do |k| + extra[k] = kwargs[k].is_a?(::ApplicationRecord) ? kwargs[k].try(:id) : kwargs[k] + end + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, event_name: event_name, kwargs: extra) nil end private + def validate_property!(kwargs, property_name, *class_names) + return unless kwargs.has_key?(property_name) + return if kwargs[property_name].nil? + return if class_names.include?(kwargs[property_name].class) + + raise InvalidPropertyTypeError, "#{property_name} should be an instance of #{class_names.join(', ')}" + end + def increase_total_counter(event_name) redis_counter_key = Gitlab::Usage::Metrics::Instrumentations::TotalCountMetric.redis_key(event_name) Gitlab::Redis::SharedState.with { |redis| redis.incr(redis_counter_key) } end + def increase_weekly_total_counter(event_name) + redis_counter_key = + Gitlab::Usage::Metrics::Instrumentations::TotalCountMetric.redis_key(event_name, Date.today) + Gitlab::Redis::SharedState.with { |redis| redis.incr(redis_counter_key) } + end + def update_unique_counter(event_name, kwargs) unique_property = EventDefinitions.unique_property(event_name) return unless unique_property @@ -35,11 +66,9 @@ module Gitlab unique_method = :id unless kwargs.has_key?(unique_property) - raise InvalidPropertyError, "#{event_name} should be triggered with a named parameter '#{unique_property}'." - end - - unless kwargs[unique_property].respond_to?(unique_method) - raise InvalidMethodError, "'#{unique_property}' should have a '#{unique_method}' method." + message = "#{event_name} should be triggered with a named parameter '#{unique_property}'." + Gitlab::AppJsonLogger.warn(message: message) + return end unique_value = kwargs[unique_property].public_send(unique_method) # rubocop:disable GitlabSecurity/PublicSend @@ -75,6 +104,25 @@ module Gitlab Gitlab::ErrorTracking .track_and_raise_for_dev_exception(error, snowplow_category: category, snowplow_action: event_name) end + + def send_application_instrumentation_event(event_name, kwargs) + return if gitlab_sdk_client.nil? + + user = kwargs[:user] + + gitlab_sdk_client.identify(user&.id) + gitlab_sdk_client.track(event_name, { project_id: kwargs[:project]&.id, namespace_id: kwargs[:namespace]&.id }) + end + + def gitlab_sdk_client + app_id = ENV['GITLAB_ANALYTICS_ID'] + host = ENV['GITLAB_ANALYTICS_URL'] + + return unless app_id.present? && host.present? + + GitlabSDK::Client.new(app_id: app_id, host: host) + end + strong_memoize_attr :gitlab_sdk_client end end end diff --git a/lib/gitlab/issuables_count_for_state.rb b/lib/gitlab/issuables_count_for_state.rb index f11dd520d2d..13909ca2ce3 100644 --- a/lib/gitlab/issuables_count_for_state.rb +++ b/lib/gitlab/issuables_count_for_state.rb @@ -124,7 +124,7 @@ module Gitlab def cache_issues_count? @store_in_redis_cache && - finder.instance_of?(IssuesFinder) && + finder.class <= IssuesFinder && parent_group.present? && !params_include_filters? end @@ -134,7 +134,7 @@ module Gitlab end def redis_cache_key - ['group', parent_group&.id, 'issues'] + ['group', parent_group&.id, finder.klass.model_name.plural] end def cache_options @@ -143,8 +143,8 @@ module Gitlab def params_include_filters? non_filtering_params = %i[ - scope state sort group_id include_subgroups - attempt_group_search_optimizations non_archived issue_types + scope state sort group_id include_subgroups namespace_id + attempt_group_search_optimizations non_archived issue_types lookahead ] finder.params.except(*non_filtering_params).values.any? diff --git a/lib/gitlab/kas/client.rb b/lib/gitlab/kas/client.rb index fe244bd88a0..464c049ee52 100644 --- a/lib/gitlab/kas/client.rb +++ b/lib/gitlab/kas/client.rb @@ -19,13 +19,13 @@ module Gitlab raise ConfigurationError, 'KAS internal URL is not configured' unless Gitlab::Kas.internal_url.present? end - def get_connected_agents(project:) - request = Gitlab::Agent::AgentTracker::Rpc::GetConnectedAgentsRequest.new(project_id: project.id) + def get_connected_agents_by_agent_ids(agent_ids:) + request = Gitlab::Agent::AgentTracker::Rpc::GetConnectedAgentsByAgentIdsRequest.new(agent_ids: agent_ids) stub_for(:agent_tracker) - .get_connected_agents(request, metadata: metadata) - .agents - .to_a + .get_connected_agents_by_agent_ids(request, metadata: metadata) + .agents + .to_a end def list_agent_config_files(project:) diff --git a/lib/gitlab/markdown_cache/redis/extension.rb b/lib/gitlab/markdown_cache/redis/extension.rb index add71fa120e..19c14faa3d6 100644 --- a/lib/gitlab/markdown_cache/redis/extension.rb +++ b/lib/gitlab/markdown_cache/redis/extension.rb @@ -27,7 +27,7 @@ module Gitlab fields = Gitlab::MarkdownCache::Redis::Store.bulk_read(objects) objects.each do |object| - fields[object.cache_key].value.each do |field_name, value| + fields[object.cache_key].each do |field_name, value| object.write_markdown_field(field_name, value) end end diff --git a/lib/gitlab/markdown_cache/redis/store.rb b/lib/gitlab/markdown_cache/redis/store.rb index f742cb82b8d..af9098c3300 100644 --- a/lib/gitlab/markdown_cache/redis/store.rb +++ b/lib/gitlab/markdown_cache/redis/store.rb @@ -9,16 +9,21 @@ module Gitlab def self.bulk_read(subjects) results = {} - Gitlab::Redis::Cache.with do |r| + data = Gitlab::Redis::Cache.with do |r| Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do Gitlab::Redis::CrossSlot::Pipeline.new(r).pipelined do |pipeline| subjects.each do |subject| - results[subject.cache_key] = new(subject).read(pipeline) + new(subject).read(pipeline) end end end end + # enumerate data + data.each_with_index do |elem, idx| + results[subjects[idx].cache_key] = elem + end + results end diff --git a/lib/gitlab/memory/watchdog.rb b/lib/gitlab/memory/watchdog.rb index cc335c00e26..ae567cb7d0e 100644 --- a/lib/gitlab/memory/watchdog.rb +++ b/lib/gitlab/memory/watchdog.rb @@ -69,10 +69,6 @@ module Gitlab end def handler - # This allows us to keep the watchdog running but turn it into "friendly mode" where - # all that happens is we collect logs and Prometheus events for fragmentation violations. - return Handlers::NullHandler.instance unless Feature.enabled?(:enforce_memory_watchdog, type: :ops) - configuration.handler end diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index 80ce155321b..92a8a2b95c4 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -1,172 +1,11 @@ # frozen_string_literal: true +require 'gitlab/utils/system' + module Gitlab module Metrics - # Module for gathering system/process statistics such as the memory usage. - # - # This module relies on the /proc filesystem being available. If /proc is - # not available the methods of this module will be stubbed. module System - extend self - - PROC_STAT_PATH = '/proc/self/stat' - PROC_STATUS_PATH = '/proc/%s/status' - PROC_SMAPS_ROLLUP_PATH = '/proc/%s/smaps_rollup' - PROC_LIMITS_PATH = '/proc/self/limits' - PROC_FD_GLOB = '/proc/self/fd/*' - PROC_MEM_INFO = '/proc/meminfo' - - PRIVATE_PAGES_PATTERN = /^(Private_Clean|Private_Dirty|Private_Hugetlb):\s+(?<value>\d+)/ - PSS_PATTERN = /^Pss:\s+(?<value>\d+)/ - RSS_TOTAL_PATTERN = /^VmRSS:\s+(?<value>\d+)/ - RSS_ANON_PATTERN = /^RssAnon:\s+(?<value>\d+)/ - RSS_FILE_PATTERN = /^RssFile:\s+(?<value>\d+)/ - MAX_OPEN_FILES_PATTERN = /Max open files\s*(?<value>\d+)/ - MEM_TOTAL_PATTERN = /^MemTotal:\s+(?<value>\d+) (.+)/ - - def summary - proportional_mem = memory_usage_uss_pss - { - version: RUBY_DESCRIPTION, - gc_stat: GC.stat, - memory_rss: memory_usage_rss[:total], - memory_uss: proportional_mem[:uss], - memory_pss: proportional_mem[:pss], - time_cputime: cpu_time, - time_realtime: real_time, - time_monotonic: monotonic_time - } - end - - # Returns the given process' RSS (resident set size) in bytes. - def memory_usage_rss(pid: 'self') - results = { total: 0, anon: 0, file: 0 } - - safe_yield_procfile(PROC_STATUS_PATH % pid) do |io| - io.each_line do |line| - if (value = parse_metric_value(line, RSS_TOTAL_PATTERN)) > 0 - results[:total] = value.kilobytes - elsif (value = parse_metric_value(line, RSS_ANON_PATTERN)) > 0 - results[:anon] = value.kilobytes - elsif (value = parse_metric_value(line, RSS_FILE_PATTERN)) > 0 - results[:file] = value.kilobytes - end - end - end - - results - end - - # Returns the given process' USS/PSS (unique/proportional set size) in bytes. - def memory_usage_uss_pss(pid: 'self') - sum_matches(PROC_SMAPS_ROLLUP_PATH % pid, uss: PRIVATE_PAGES_PATTERN, pss: PSS_PATTERN) - .transform_values(&:kilobytes) - end - - def memory_total - sum_matches(PROC_MEM_INFO, memory_total: MEM_TOTAL_PATTERN)[:memory_total].kilobytes - end - - def file_descriptor_count - Dir.glob(PROC_FD_GLOB).length - end - - def max_open_file_descriptors - sum_matches(PROC_LIMITS_PATH, max_fds: MAX_OPEN_FILES_PATTERN)[:max_fds] - end - - def cpu_time - Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID, :float_second) - end - - # Returns the current real time in a given precision. - # - # Returns the time as a Float for precision = :float_second. - def real_time(precision = :float_second) - Process.clock_gettime(Process::CLOCK_REALTIME, precision) - end - - # Returns the current monotonic clock time as seconds with microseconds precision. - # - # Returns the time as a Float. - def monotonic_time - Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_second) - end - - def thread_cpu_time - # Not all OS kernels are supporting `Process::CLOCK_THREAD_CPUTIME_ID` - # Refer: https://gitlab.com/gitlab-org/gitlab/issues/30567#note_221765627 - return unless defined?(Process::CLOCK_THREAD_CPUTIME_ID) - - Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID, :float_second) - end - - def thread_cpu_duration(start_time) - end_time = thread_cpu_time - return unless start_time && end_time - - end_time - start_time - end - - # Returns the total time the current process has been running in seconds. - def process_runtime_elapsed_seconds - # Entry 22 (1-indexed) contains the process `starttime`, see: - # https://man7.org/linux/man-pages/man5/proc.5.html - # - # This value is a fixed timestamp in clock ticks. - # To obtain an elapsed time in seconds, we divide by the number - # of ticks per second and subtract from the system uptime. - start_time_ticks = proc_stat_entries[21].to_f - clock_ticks_per_second = Etc.sysconf(Etc::SC_CLK_TCK) - uptime - (start_time_ticks / clock_ticks_per_second) - end - - private - - # Given a path to a file in /proc and a hash of (metric, pattern) pairs, - # sums up all values found for those patterns under the respective metric. - def sum_matches(proc_file, **patterns) - results = patterns.transform_values { 0 } - - safe_yield_procfile(proc_file) do |io| - io.each_line do |line| - patterns.each do |metric, pattern| - results[metric] += parse_metric_value(line, pattern) - end - end - end - - results - end - - def parse_metric_value(line, pattern) - match = line.match(pattern) - return 0 unless match - - match.named_captures.fetch('value', 0).to_i - end - - def proc_stat_entries - safe_yield_procfile(PROC_STAT_PATH) do |io| - io.read.split(' ') - end || [] - end - - def safe_yield_procfile(path, &block) - File.open(path, &block) - rescue Errno::ENOENT - # This means the procfile we're reading from did not exist; - # most likely we're on Darwin. - end - - # Equivalent to reading /proc/uptime on Linux 2.6+. - # - # Returns 0 if not supported, e.g. on Darwin. - def uptime - Process.clock_gettime(Process::CLOCK_BOOTTIME) - rescue NameError - 0 - end + extend Gitlab::Utils::System end end end diff --git a/lib/gitlab/middleware/path_traversal_check.rb b/lib/gitlab/middleware/path_traversal_check.rb index 6fef247b708..d1260c81925 100644 --- a/lib/gitlab/middleware/path_traversal_check.rb +++ b/lib/gitlab/middleware/path_traversal_check.rb @@ -32,20 +32,24 @@ module Gitlab end def call(env) - if Feature.enabled?(:check_path_traversal_middleware, Feature.current_request) - log_params = {} + return @app.call(env) unless Feature.enabled?(:check_path_traversal_middleware, Feature.current_request) - execution_time = measure_execution_time do - request = ::Rack::Request.new(env.dup) - check(request, log_params) unless excluded?(request) - end + log_params = {} - log_params[:duration_ms] = execution_time.round(5) if execution_time + execution_time = measure_execution_time do + request = ::Rack::Request.new(env.dup) + check(request, log_params) unless excluded?(request) + end + log_params[:duration_ms] = execution_time.round(5) if execution_time + + result = @app.call(env) - log(log_params) unless log_params.empty? + unless log_params.empty? + log_params[:status] = result.first + log(log_params) end - @app.call(env) + result end private diff --git a/lib/gitlab/nav/top_nav_menu_builder.rb b/lib/gitlab/nav/top_nav_menu_builder.rb deleted file mode 100644 index dca3432a6a1..00000000000 --- a/lib/gitlab/nav/top_nav_menu_builder.rb +++ /dev/null @@ -1,47 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Nav - class TopNavMenuBuilder - def initialize - @primary = [] - @secondary = [] - @last_header_added = nil - end - - def add_primary_menu_item(header: nil, **args) - if header && (header != @last_header_added) - add_menu_header(dest: @primary, title: header) - @last_header_added = header - end - - add_menu_item(dest: @primary, **args) - end - - def add_secondary_menu_item(**args) - add_menu_item(dest: @secondary, **args) - end - - def build - { - primary: @primary, - secondary: @secondary - } - end - - private - - def add_menu_item(dest:, **args) - item = ::Gitlab::Nav::TopNavMenuItem.build(**args) - - dest.push(item) - end - - def add_menu_header(dest:, **args) - header = ::Gitlab::Nav::TopNavMenuHeader.build(**args) - - dest.push(header) - end - end - end -end diff --git a/lib/gitlab/nav/top_nav_menu_header.rb b/lib/gitlab/nav/top_nav_menu_header.rb deleted file mode 100644 index 520091dbd97..00000000000 --- a/lib/gitlab/nav/top_nav_menu_header.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Nav - class TopNavMenuHeader - def self.build(title:) - { - type: :header, - title: title - } - end - end - end -end diff --git a/lib/gitlab/nav/top_nav_menu_item.rb b/lib/gitlab/nav/top_nav_menu_item.rb index e7790fd77d0..f6fea97dae9 100644 --- a/lib/gitlab/nav/top_nav_menu_item.rb +++ b/lib/gitlab/nav/top_nav_menu_item.rb @@ -21,7 +21,7 @@ module Gitlab href: href, view: view.to_s, css_class: css_class, - data: data || { testid: 'menu_item_link', qa_title: title }, + data: data || { testid: 'menu-item-link', qa_title: title }, partial: partial, component: component } diff --git a/lib/gitlab/nav/top_nav_view_model_builder.rb b/lib/gitlab/nav/top_nav_view_model_builder.rb deleted file mode 100644 index 10b841f777e..00000000000 --- a/lib/gitlab/nav/top_nav_view_model_builder.rb +++ /dev/null @@ -1,53 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Nav - class TopNavViewModelBuilder - def initialize - @menu_builder = ::Gitlab::Nav::TopNavMenuBuilder.new - @views = {} - @shortcuts = [] - end - - # Using delegate hides the stacktrace for some errors, so we choose to be explicit. - # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/62047#note_579031091 - def add_primary_menu_item(...) - @menu_builder.add_primary_menu_item(...) - end - - def add_secondary_menu_item(...) - @menu_builder.add_secondary_menu_item(...) - end - - def add_shortcut(**args) - item = ::Gitlab::Nav::TopNavMenuItem.build(**args) - - @shortcuts.push(item) - end - - def add_primary_menu_item_with_shortcut(shortcut_class:, shortcut_href: nil, **args) - add_primary_menu_item(**args) - add_shortcut( - id: "#{args.fetch(:id)}-shortcut", - title: args.fetch(:title), - href: shortcut_href || args.fetch(:href), - css_class: shortcut_class - ) - end - - def add_view(name, props) - @views[name] = props - end - - def build - menu = @menu_builder.build - - menu.merge({ - views: @views, - shortcuts: @shortcuts, - menuTooltip: _('Main menu') - }.compact) - end - end - end -end diff --git a/lib/gitlab/omniauth_initializer.rb b/lib/gitlab/omniauth_initializer.rb index 81ad7a7f9e1..1835aef755f 100644 --- a/lib/gitlab/omniauth_initializer.rb +++ b/lib/gitlab/omniauth_initializer.rb @@ -29,6 +29,8 @@ module Gitlab { authorize_params: { gl_auth_type: 'login' } } + when ->(provider_name) { AuthHelper.saml_providers.include?(provider_name.to_sym) } + { attribute_statements: ::Gitlab::Auth::Saml::Config.default_attribute_statements } else {} end @@ -61,7 +63,7 @@ module Gitlab provider_arguments.concat arguments provider_arguments << defaults unless defaults.empty? when Hash, GitlabSettings::Options - hash_arguments = arguments.deep_symbolize_keys.deep_merge(defaults) + hash_arguments = merge_hash_defaults_and_args(defaults, arguments) normalized = normalize_hash_arguments(hash_arguments) # A Hash from the configuration will be passed as is. @@ -80,6 +82,15 @@ module Gitlab provider_arguments end + def merge_hash_defaults_and_args(defaults, arguments) + return arguments.to_hash if defaults.empty? + + revert_merging = Gitlab::Utils.to_boolean(ENV['REVERT_OMNIAUTH_DEFAULT_MERGING']) + return arguments.to_hash.deep_symbolize_keys.deep_merge(defaults) if revert_merging + + defaults.deep_merge(arguments.deep_symbolize_keys) + end + def normalize_hash_arguments(args) args.deep_symbolize_keys! diff --git a/lib/gitlab/pages/deployment_update.rb b/lib/gitlab/pages/deployment_update.rb index bf6ac3a056d..a572a59b2f5 100644 --- a/lib/gitlab/pages/deployment_update.rb +++ b/lib/gitlab/pages/deployment_update.rb @@ -92,6 +92,7 @@ module Gitlab # If a newer pipeline already build a PagesDeployment def validate_outdated_sha return if latest? + return if latest_pipeline_id.blank? return if latest_pipeline_id <= build.pipeline_id errors.add(:base, 'build SHA is outdated for this ref') diff --git a/lib/gitlab/pages/url_builder.rb b/lib/gitlab/pages/url_builder.rb index 5a28a5ffd23..f01ec54b853 100644 --- a/lib/gitlab/pages/url_builder.rb +++ b/lib/gitlab/pages/url_builder.rb @@ -14,6 +14,7 @@ module Gitlab end def pages_url(with_unique_domain: false) + return namespace_in_path_url(with_unique_domain && unique_domain_enabled?) if config.namespace_in_path return unique_url if with_unique_domain && unique_domain_enabled? project_path_url = "#{config.protocol}://#{project_path}".downcase @@ -29,6 +30,7 @@ module Gitlab def unique_host return unless unique_domain_enabled? + return if config.namespace_in_path URI(unique_url).host end @@ -40,9 +42,11 @@ module Gitlab def artifact_url(artifact, job) return unless artifact_url_available?(artifact, job) + host_url = config.namespace_in_path ? "#{pages_base_url}/#{project_namespace}" : namespace_url + format( ARTIFACT_URL, - host: namespace_url, + host: host_url, project_path: project_path, job_id: job.id, artifact_path: artifact.path) @@ -67,6 +71,21 @@ module Gitlab @unique_url ||= url_for(project.project_setting.pages_unique_domain) end + def pages_base_url + @pages_url ||= URI(config.url) + .tap { |url| url.port = config.port } + .to_s + .downcase + end + + def namespace_in_path_url(with_unique_domain) + if with_unique_domain + "#{pages_base_url}/#{project.project_setting.pages_unique_domain}".downcase + else + "#{pages_base_url}/#{project_namespace}/#{project_path}".downcase + end + end + def url_for(subdomain) URI(config.url) .tap { |url| url.port = config.port } diff --git a/lib/gitlab/pagination/cursor_based_keyset.rb b/lib/gitlab/pagination/cursor_based_keyset.rb index 9e8c0c530a9..b5cc127d232 100644 --- a/lib/gitlab/pagination/cursor_based_keyset.rb +++ b/lib/gitlab/pagination/cursor_based_keyset.rb @@ -3,20 +3,6 @@ module Gitlab module Pagination module CursorBasedKeyset - SUPPORTED_MULTI_ORDERING = { - Group => { name: [:asc] }, - AuditEvent => { id: [:desc] }, - User => { - id: [:asc, :desc], - name: [:asc, :desc], - username: [:asc, :desc], - created_at: [:asc, :desc], - updated_at: [:asc, :desc] - }, - ::Ci::Build => { id: [:desc] }, - ::Packages::BuildInfo => { id: [:desc] } - }.freeze - # Relation types that are enforced in this list # enforce the use of keyset pagination, thus erroring out requests # made with offset pagination above a certain limit. @@ -26,7 +12,7 @@ module Gitlab ENFORCED_TYPES = [Group].freeze def self.available_for_type?(relation) - SUPPORTED_MULTI_ORDERING.key?(relation.klass) + relation.klass.respond_to?(:supported_keyset_orderings) end def self.available?(cursor_based_request_context, relation) @@ -44,7 +30,7 @@ module Gitlab order_by_from_request = cursor_based_request_context.order sort_from_request = cursor_based_request_context.sort - SUPPORTED_MULTI_ORDERING[relation.klass][order_by_from_request]&.include?(sort_from_request) + !!relation.klass.supported_keyset_orderings[order_by_from_request]&.include?(sort_from_request) end private_class_method :order_satisfied? end diff --git a/lib/gitlab/pagination/gitaly_keyset_pager.rb b/lib/gitlab/pagination/gitaly_keyset_pager.rb index 82d6fc64d89..a1c340baf23 100644 --- a/lib/gitlab/pagination/gitaly_keyset_pager.rb +++ b/lib/gitlab/pagination/gitaly_keyset_pager.rb @@ -64,7 +64,7 @@ module Gitlab def paginate_via_gitaly(finder) finder.execute(gitaly_pagination: true).tap do |records| - apply_headers(records) + apply_headers(records, finder.next_cursor) end end @@ -82,20 +82,18 @@ module Gitlab end end - def apply_headers(records) + def apply_headers(records, next_cursor) if records.count == params[:per_page] Gitlab::Pagination::Keyset::HeaderBuilder .new(request_context) .add_next_page_header( - query_params_for(records.last) + query_params_for(next_cursor) ) end end - def query_params_for(record) - # NOTE: page_token is name for now, but it could be dynamic if we have other gitaly finders - # that is based on something other than name - { page_token: record.name } + def query_params_for(next_cursor) + { page_token: next_cursor } end end end diff --git a/lib/gitlab/patch/sidekiq_cron_poller.rb b/lib/gitlab/patch/sidekiq_cron_poller.rb index 8f1fbf53161..3f962c47ae9 100644 --- a/lib/gitlab/patch/sidekiq_cron_poller.rb +++ b/lib/gitlab/patch/sidekiq_cron_poller.rb @@ -11,7 +11,7 @@ if Gem::Version.new(Sidekiq::VERSION) != Gem::Version.new('6.5.12') raise 'New version of sidekiq detected, please remove or update this patch' end -if Gem::Version.new(Sidekiq::Cron::VERSION) != Gem::Version.new('1.8.0') +if Gem::Version.new(Sidekiq::Cron::VERSION) != Gem::Version.new('1.12.0') raise 'New version of sidekiq-cron detected, please remove or update this patch' end diff --git a/lib/gitlab/patch/sidekiq_scheduled_enq.rb b/lib/gitlab/patch/sidekiq_scheduled_enq.rb deleted file mode 100644 index b5a40c19923..00000000000 --- a/lib/gitlab/patch/sidekiq_scheduled_enq.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -# Patch to address https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/2286 -# Using a dual-namespace poller eliminates the need for script based migration of -# schedule-related sets in Sidekiq. -module Gitlab - module Patch - module SidekiqScheduledEnq - # The patched enqueue_jobs will poll non-namespaced scheduled sets before doing the same for - # namespaced sets via super and vice-versa depending on how Sidekiq.redis was configured - def enqueue_jobs(sorted_sets = Sidekiq::Scheduled::SETS) - # checks the other namespace - if Gitlab::Utils.to_boolean(ENV['SIDEKIQ_ENABLE_DUAL_NAMESPACE_POLLING'], default: true) - # Refer to https://github.com/sidekiq/sidekiq/blob/v6.5.7/lib/sidekiq/scheduled.rb#L25 - # this portion swaps out Sidekiq.redis for Gitlab::Redis::Queues - Gitlab::Redis::Queues.with do |conn| # rubocop:disable Cop/RedisQueueUsage - sorted_sets.each do |sorted_set| - # adds namespace since `super` polls with a non-namespaced Sidekiq.redis - sorted_set = "#{Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE}:#{sorted_set}" # rubocop:disable Cop/RedisQueueUsage - - while !@done && (job = zpopbyscore(conn, keys: [sorted_set], argv: [Time.now.to_f.to_s])) # rubocop:disable Gitlab/ModuleWithInstanceVariables, Lint/AssignmentInCondition - Sidekiq::Client.push(Sidekiq.load_json(job)) # rubocop:disable Cop/SidekiqApiUsage - Sidekiq.logger.debug { "enqueued #{sorted_set}: #{job}" } - end - end - end - end - - super - end - end - end -end diff --git a/lib/gitlab/puma/error_handler.rb b/lib/gitlab/puma/error_handler.rb index 4efc4866431..9eabe0731e2 100644 --- a/lib/gitlab/puma/error_handler.rb +++ b/lib/gitlab/puma/error_handler.rb @@ -18,10 +18,11 @@ module Gitlab # https://github.com/puma/puma/pull/3094 status_code ||= 500 - if Raven.configuration.capture_allowed? - Raven.capture_exception(ex, tags: { handler: 'puma_low_level' }, - extra: { puma_env: env, status_code: status_code }) - end + Gitlab::ErrorTracking.track_exception( + ex, + { puma_env: env, status_code: status_code }, + { handler: 'puma_low_level' } + ) # note the below is just a Rack response [status_code, {}, message] diff --git a/lib/gitlab/quick_actions/extractor.rb b/lib/gitlab/quick_actions/extractor.rb index 5cf79db83af..c6a7a39a943 100644 --- a/lib/gitlab/quick_actions/extractor.rb +++ b/lib/gitlab/quick_actions/extractor.rb @@ -9,19 +9,6 @@ module Gitlab # extractor = Gitlab::QuickActions::Extractor.new([:open, :assign, :labels]) # ``` class Extractor - CODE_REGEX = %r{ - (?<code> - # Code blocks: - # ``` - # Anything, including `/cmd arg` which are ignored by this filter - # ``` - - ^``` - .+? - \n```$ - ) - }mix - INLINE_CODE_REGEX = %r{ (?<inline_code> # Inline code on separate rows: @@ -46,22 +33,7 @@ module Gitlab ) }mix - QUOTE_BLOCK_REGEX = %r{ - (?<html> - # Quote block: - # >>> - # Anything, including `/cmd arg` which are ignored by this filter - # >>> - - ^>>> - .+? - \n>>>$ - ) - }mix - - EXCLUSION_REGEX = %r{ - #{CODE_REGEX} | #{INLINE_CODE_REGEX} | #{HTML_BLOCK_REGEX} | #{QUOTE_BLOCK_REGEX} - }mix + EXCLUSION_REGEX = %r{#{INLINE_CODE_REGEX} | #{HTML_BLOCK_REGEX}}mix attr_reader :command_definitions, :keep_actions @@ -119,15 +91,40 @@ module Gitlab content = content.dup content.delete!("\r") - content.gsub!(commands_regex(names: names, sub_names: sub_names)) do - command, output = if $~[:substitution] - process_substitutions($~) - else - process_commands($~, redact) - end + # use a markdown based pipeline to grab possible paragraphs that might + # contain quick actions. This ensures they are not in HTML blocks, quote blocks, + # or code blocks. + pipeline = Banzai::Pipeline::QuickActionPipeline.html_pipeline + possible_paragraphs = pipeline.call(content, {}, {})[:quick_action_paragraphs] + + if possible_paragraphs.present? + content_lines = content.lines + + # Each paragraph that possibly contains quick actions must be searched. In order + # to use the `sourcepos` information, we need to convert into individual lines, + # and then replace the specific lines. + possible_paragraphs.each do |possible| + endpos = possible[:end_line] + endpos += 1 if content_lines[endpos + 1] == "\n" + + paragraph = content_lines[possible[:start_line]..endpos].join + + paragraph.gsub!(commands_regex(names: names, sub_names: sub_names)) do + command, output = if $~[:substitution] + process_substitutions($~) + else + process_commands($~, redact) + end + + commands << command + output + end + + content_lines.fill('', possible[:start_line]..endpos) + content_lines[possible[:start_line]] = paragraph + end - commands << command - output + content = content_lines.join end [content.rstrip, commands.reject(&:empty?)] @@ -181,7 +178,7 @@ module Gitlab #{EXCLUSION_REGEX} | (?: - # Command not in a blockquote, blockcode, or HTML tag: + # Command such as: # /close ^\/ @@ -194,7 +191,8 @@ module Gitlab ) | (?: - # Substitution not in a blockquote, blockcode, or HTML tag: + # Substitution such as: + # /shrug ^\/ (?<substitution>#{Regexp.new(Regexp.union(sub_names).source, Regexp::IGNORECASE)}) diff --git a/lib/gitlab/quick_actions/issuable_actions.rb b/lib/gitlab/quick_actions/issuable_actions.rb index 57ed6c5c35e..2f7fa89019e 100644 --- a/lib/gitlab/quick_actions/issuable_actions.rb +++ b/lib/gitlab/quick_actions/issuable_actions.rb @@ -213,7 +213,7 @@ module Gitlab match = emoji_param.match(Banzai::Filter::EmojiFilter.emoji_pattern) match[1] if match end - command :award do |name| + command :award, :react do |name| if name && quick_action_target.user_can_award?(current_user) @updates[:emoji_award] = name end diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb index ae79db723f2..c79432f36cc 100644 --- a/lib/gitlab/quick_actions/issue_actions.rb +++ b/lib/gitlab/quick_actions/issue_actions.rb @@ -226,28 +226,18 @@ module Gitlab params 'email1@example.com email2@example.com (up to 6 emails)' types Issue condition do - Feature.enabled?(:issue_email_participants, parent) && + quick_action_target.persisted? && + Feature.enabled?(:issue_email_participants, parent) && current_user.can?(:"admin_#{quick_action_target.to_ability_name}", quick_action_target) end command :invite_email do |emails = ""| - MAX_NUMBER_OF_EMAILS = 6 - - existing_emails = quick_action_target.email_participants_emails_downcase - emails_to_add = emails.split(' ').index_by { |email| [email.downcase, email] }.except(*existing_emails).each_value.first(MAX_NUMBER_OF_EMAILS) - added_emails = [] - - emails_to_add.each do |email| - new_participant = quick_action_target.issue_email_participants.create(email: email) - added_emails << email if new_participant.persisted? - end + response = ::IssueEmailParticipants::CreateService.new( + target: quick_action_target, + current_user: current_user, + emails: emails.split(' ') + ).execute - if added_emails.any? - message = _("added %{emails}") % { emails: added_emails.to_sentence } - SystemNoteService.add_email_participants(quick_action_target, quick_action_target.project, current_user, message) - @execution_message[:invite_email] = message.upcase_first << "." - else - @execution_message[:invite_email] = _("No email participants were added. Either none were provided, or they already exist.") - end + @execution_message[:invite_email] = response.message end desc { _('Promote issue to incident') } diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb index 72bec159226..fe18bc8e133 100644 --- a/lib/gitlab/quick_actions/merge_request_actions.rb +++ b/lib/gitlab/quick_actions/merge_request_actions.rb @@ -161,10 +161,25 @@ module Gitlab condition do quick_action_target.persisted? end - command :submit_review do + command :submit_review do |state = "reviewed"| next if params[:review_id] result = DraftNotes::PublishService.new(quick_action_target, current_user).execute + + if Feature.enabled?(:mr_request_changes, current_user) + reviewer_state = state.strip.presence + + if reviewer_state === 'approve' + ::MergeRequests::ApprovalService + .new(project: quick_action_target.project, current_user: current_user) + .execute(quick_action_target) + elsif MergeRequestReviewer.states.key?(reviewer_state) + ::MergeRequests::UpdateReviewerStateService + .new(project: quick_action_target.project, current_user: current_user) + .execute(quick_action_target, reviewer_state) + end + end + @execution_message[:submit_review] = if result[:status] == :success _('Submitted the current review.') else diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb index 9f7599d2500..a29c37411c3 100644 --- a/lib/gitlab/redis.rb +++ b/lib/gitlab/redis.rb @@ -8,6 +8,7 @@ module Gitlab # This will make sure the connection pool is initialized on application boot in # config/initializers/7_redis.rb, instrumented, and used in health- & readiness checks. ALL_CLASSES = [ + Gitlab::Redis::BufferedCounter, Gitlab::Redis::Cache, Gitlab::Redis::ClusterSharedState, Gitlab::Redis::DbLoadBalancing, diff --git a/lib/gitlab/redis/buffered_counter.rb b/lib/gitlab/redis/buffered_counter.rb new file mode 100644 index 00000000000..21fc4ba8034 --- /dev/null +++ b/lib/gitlab/redis/buffered_counter.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module Redis + class BufferedCounter < ::Gitlab::Redis::Wrapper + class << self + def config_fallback + SharedState + end + end + end + end +end diff --git a/lib/gitlab/redis/db_load_balancing.rb b/lib/gitlab/redis/db_load_balancing.rb index 01276445611..f6769a39397 100644 --- a/lib/gitlab/redis/db_load_balancing.rb +++ b/lib/gitlab/redis/db_load_balancing.rb @@ -8,15 +8,6 @@ module Gitlab def config_fallback SharedState end - - private - - def redis - primary_store = ::Redis.new(params) - secondary_store = ::Redis.new(config_fallback.params) - - MultiStore.new(primary_store, secondary_store, store_name) - end end end end diff --git a/lib/gitlab/redis/sidekiq_status.rb b/lib/gitlab/redis/sidekiq_status.rb deleted file mode 100644 index 9b8bbf5a0ad..00000000000 --- a/lib/gitlab/redis/sidekiq_status.rb +++ /dev/null @@ -1,24 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Redis - # Pseudo-store to transition `Gitlab::SidekiqStatus` from - # using `Sidekiq.redis` to using the `SharedState` redis store. - class SidekiqStatus < ::Gitlab::Redis::Wrapper - class << self - def store_name - 'SharedState' - end - - private - - def redis - primary_store = ::Redis.new(Gitlab::Redis::SharedState.params) - secondary_store = ::Redis.new(Gitlab::Redis::Queues.params) # rubocop:disable Cop/RedisQueueUsage - - MultiStore.new(primary_store, secondary_store, name.demodulize) - end - end - end - end -end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index c1f346ec7e4..bb231eec226 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -30,7 +30,7 @@ module Gitlab end def pool - @pool ||= ConnectionPool.new(size: pool_size) { redis } + @pool ||= ConnectionPool.new(size: pool_size, name: store_name.underscore) { redis } end def pool_size @@ -83,8 +83,6 @@ module Gitlab "::Gitlab::Instrumentation::Redis::#{store_name}".constantize end - private - def redis ::Redis.new(params) end diff --git a/lib/gitlab/registration_features/password_complexity.rb b/lib/gitlab/registration_features/password_complexity.rb deleted file mode 100644 index 6d165a7a665..00000000000 --- a/lib/gitlab/registration_features/password_complexity.rb +++ /dev/null @@ -1,12 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module RegistrationFeatures - class PasswordComplexity - def self.feature_available? - ::License.feature_available?(:password_complexity) || - ::GitlabSubscriptions::Features.usage_ping_feature?(:password_complexity) - end - end - end -end diff --git a/lib/gitlab/request_forgery_protection.rb b/lib/gitlab/request_forgery_protection.rb index d5e80053772..3a389d3363f 100644 --- a/lib/gitlab/request_forgery_protection.rb +++ b/lib/gitlab/request_forgery_protection.rb @@ -6,8 +6,7 @@ module Gitlab module RequestForgeryProtection - # rubocop:disable Rails/ApplicationController - class Controller < ActionController::Base + class Controller < BaseActionController protect_from_forgery with: :exception, prepend: true def initialize @@ -40,6 +39,5 @@ module Gitlab rescue ActionController::InvalidAuthenticityToken false end - # rubocop:enable Rails/ApplicationController end end diff --git a/lib/gitlab/seeders/ci/catalog/resource_seeder.rb b/lib/gitlab/seeders/ci/catalog/resource_seeder.rb index 2971dabe044..c29075cff32 100644 --- a/lib/gitlab/seeders/ci/catalog/resource_seeder.rb +++ b/lib/gitlab/seeders/ci/catalog/resource_seeder.rb @@ -5,25 +5,23 @@ module Gitlab module Ci module Catalog class ResourceSeeder - # This is currently disabled until it gets fixed: https://gitlab.com/gitlab-org/gitlab/-/issues/429649 # Initializes the class # # @param [String] Path of the group to find # @param [Integer] Number of resources to create - def initialize(group_path:, seed_count:) + # @param[Boolean] If the created resources should be published or not, defaults to false + def initialize(group_path:, seed_count:, publish:) @group = Group.find_by_full_path(group_path) @seed_count = seed_count + @publish = publish @current_user = @group&.first_owner end def seed - if @group.nil? - warn 'ERROR: Group was not found.' - return - end + return warn 'ERROR: Group was not found.' if @group.nil? @seed_count.times do |i| - create_ci_catalog_resource(i) + seed_catalog_resource(i) end end @@ -59,9 +57,16 @@ module Gitlab stage: $[[ inputs.stage ]] YAML + project.repository.create_dir( + @current_user, + 'templates', + message: 'Add template dir', + branch_name: project.default_branch_or_main + ) + project.repository.create_file( @current_user, - 'template.yml', + 'templates/component.yml', template_content, message: 'Add template.yml', branch_name: project.default_branch_or_main @@ -78,21 +83,22 @@ module Gitlab ) end - def create_ci_catalog(project) + def create_catalog_resource(project) result = ::Ci::Catalog::Resources::CreateService.new(project, @current_user).execute if result.success? result.payload else - warn "Project '#{project.name}' could not be converted to a Catalog resource" + warn "Catalog resource could not be created for Project '#{project.name}': #{result.errors.join}" nil end end - def create_ci_catalog_resource(index) + def seed_catalog_resource(index) name = "ci_seed_resource_#{index}" + existing_project = Project.find_by_name(name) - if Project.find_by_name(name).present? - warn "Project '#{name}' already exists!" + if existing_project.present? && existing_project.group.path == @group.path + warn "Project '#{@group.path}/#{name}' already exists!" return end @@ -103,9 +109,12 @@ module Gitlab create_readme(project, index) create_template_yml(project) - return unless create_ci_catalog(project) + new_catalog_resource = create_catalog_resource(project) + return unless new_catalog_resource + + warn "Project '#{@group.path}/#{name}' was saved successfully!" - warn "Project '#{name}' was saved successfully!" + new_catalog_resource.publish! if @publish end end end diff --git a/lib/gitlab/sidekiq_middleware.rb b/lib/gitlab/sidekiq_middleware.rb index e1c155a4848..96bda86ab08 100644 --- a/lib/gitlab/sidekiq_middleware.rb +++ b/lib/gitlab/sidekiq_middleware.rb @@ -37,6 +37,7 @@ module Gitlab chain.add ::Gitlab::SidekiqStatus::ServerMiddleware chain.add ::Gitlab::SidekiqMiddleware::WorkerContext::Server chain.add ::Gitlab::SidekiqMiddleware::PauseControl::Server + chain.add ::ClickHouse::MigrationSupport::SidekiqMiddleware # DuplicateJobs::Server should be placed at the bottom, but before the SidekiqServerMiddleware, # so we can compare the latest WAL location against replica chain.add ::Gitlab::SidekiqMiddleware::DuplicateJobs::Server diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb index 10a69acc037..883e1ba0558 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb @@ -20,8 +20,7 @@ module Gitlab class DuplicateJob include Gitlab::Utils::StrongMemoize - DEFAULT_DUPLICATE_KEY_TTL = 6.hours - SHORT_DUPLICATE_KEY_TTL = 10.minutes + DEFAULT_DUPLICATE_KEY_TTL = 10.minutes DEFAULT_STRATEGY = :until_executing STRATEGY_NONE = :none @@ -75,7 +74,8 @@ module Gitlab argv = [] job_wal_locations.each do |connection_name, location| - argv += [connection_name, pg_wal_lsn_diff(connection_name), location] + diff = pg_wal_lsn_diff(connection_name) + argv += [connection_name, diff || '', location] end with_redis { |r| r.eval(UPDATE_WAL_COOKIE_SCRIPT, keys: [cookie_key], argv: argv) } @@ -174,7 +174,7 @@ module Gitlab end def duplicate_key_ttl - options[:ttl] || default_duplicate_key_ttl + options[:ttl] || DEFAULT_DUPLICATE_KEY_TTL end private @@ -183,12 +183,6 @@ module Gitlab attr_reader :queue_name, :job attr_writer :existing_jid - def default_duplicate_key_ttl - return SHORT_DUPLICATE_KEY_TTL if Feature.enabled?(:reduce_duplicate_job_key_ttl) - - DEFAULT_DUPLICATE_KEY_TTL - end - def worker_klass @worker_klass ||= worker_class_name.to_s.safe_constantize end diff --git a/lib/gitlab/sidekiq_middleware/pause_control.rb b/lib/gitlab/sidekiq_middleware/pause_control.rb index 2f0fd0cc799..8f4da7267d7 100644 --- a/lib/gitlab/sidekiq_middleware/pause_control.rb +++ b/lib/gitlab/sidekiq_middleware/pause_control.rb @@ -8,6 +8,7 @@ module Gitlab UnknownStrategyError = Class.new(StandardError) STRATEGIES = { + click_house_migration: ::Gitlab::SidekiqMiddleware::PauseControl::Strategies::ClickHouseMigration, zoekt: ::Gitlab::SidekiqMiddleware::PauseControl::Strategies::Zoekt, none: ::Gitlab::SidekiqMiddleware::PauseControl::Strategies::None }.freeze diff --git a/lib/gitlab/sidekiq_middleware/pause_control/server.rb b/lib/gitlab/sidekiq_middleware/pause_control/server.rb index cfa02b3ec3a..7beb5f9ca5b 100644 --- a/lib/gitlab/sidekiq_middleware/pause_control/server.rb +++ b/lib/gitlab/sidekiq_middleware/pause_control/server.rb @@ -4,8 +4,8 @@ module Gitlab module SidekiqMiddleware module PauseControl class Server - def call(worker_class, job, _queue, &block) - ::Gitlab::SidekiqMiddleware::PauseControl::StrategyHandler.new(worker_class, job).perform(&block) + def call(worker, job, _queue, &block) + ::Gitlab::SidekiqMiddleware::PauseControl::StrategyHandler.new(worker, job).perform(&block) end end end diff --git a/lib/gitlab/sidekiq_middleware/pause_control/strategies/click_house_migration.rb b/lib/gitlab/sidekiq_middleware/pause_control/strategies/click_house_migration.rb new file mode 100644 index 00000000000..adeb0524567 --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/pause_control/strategies/click_house_migration.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + module PauseControl + module Strategies + class ClickHouseMigration < Base + override :should_pause? + def should_pause? + return false unless Feature.enabled?(:pause_clickhouse_workers_during_migration) + + ClickHouse::MigrationSupport::ExclusiveLock.pause_workers? + end + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/pause_control/workers_map.rb b/lib/gitlab/sidekiq_middleware/pause_control/workers_map.rb index dc6aff92f50..97080dc91fc 100644 --- a/lib/gitlab/sidekiq_middleware/pause_control/workers_map.rb +++ b/lib/gitlab/sidekiq_middleware/pause_control/workers_map.rb @@ -17,7 +17,8 @@ module Gitlab def strategy_for(worker:) return unless @workers - @workers.find { |_, v| v.include?(worker) }&.first + worker_class = worker.is_a?(Class) ? worker : worker.class + @workers.find { |_, v| v.include?(worker_class) }&.first end end end diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb index ae4aca7ff92..496ed9de828 100644 --- a/lib/gitlab/sidekiq_status.rb +++ b/lib/gitlab/sidekiq_status.rb @@ -50,6 +50,16 @@ module Gitlab end end + # Refreshes the timeout on the key if it exists + # + # jid = The Sidekiq job ID + # expire - The expiration time of the Redis key. + def self.expire(jid, expire = DEFAULT_EXPIRATION) + with_redis do |redis| + redis.expire(key_for(jid), expire) + end + end + # Returns true if all the given job have been completed. # # job_ids - The Sidekiq job IDs to check. @@ -132,7 +142,8 @@ module Gitlab Feature.enabled?(:use_primary_store_as_default_for_sidekiq_status) # TODO: Swap for Gitlab::Redis::SharedState after store transition # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/923 - Gitlab::Redis::SidekiqStatus.with { |redis| yield redis } + # For now, we use SharedState to reduce amount of spawned connection to Redis Cluster during initialisation + Gitlab::Redis::SharedState.with { |redis| yield redis } else # Keep the old behavior intact if neither feature flag is turned on Sidekiq.redis { |redis| yield redis } # rubocop:disable Cop/SidekiqRedisCall diff --git a/lib/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb index 78c0f04e07e..038808667f4 100644 --- a/lib/gitlab/task_helpers.rb +++ b/lib/gitlab/task_helpers.rb @@ -149,12 +149,6 @@ module Gitlab end end - def repository_storage_paths_args - Gitlab::GitalyClient::StorageSettings.allow_disk_access do - Gitlab.config.repositories.storages.values.map { |rs| rs.legacy_disk_path } - end - end - def user_home Rails.env.test? ? Rails.root.join('tmp/tests') : Gitlab.config.gitlab.user_home end diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb index 3bbcd59f45e..0b606b712c7 100644 --- a/lib/gitlab/tracking.rb +++ b/lib/gitlab/tracking.rb @@ -6,10 +6,16 @@ module Gitlab module Tracking class << self + delegate :flush, to: :tracker + def enabled? tracker.enabled? end + def micro_verification_enabled? + Gitlab::Utils.to_boolean(ENV['VERIFY_TRACKING'], default: false) + end + def event(category, action, label: nil, property: nil, value: nil, context: [], project: nil, user: nil, namespace: nil, **extra) # rubocop:disable Metrics/ParameterLists action = action.to_s @@ -66,7 +72,7 @@ module Gitlab end def snowplow_micro_enabled? - Rails.env.development? && Gitlab.config.snowplow_micro.enabled + (Rails.env.development? || micro_verification_enabled?) && Gitlab.config.snowplow_micro.enabled rescue GitlabSettings::MissingSetting false end diff --git a/lib/gitlab/tracking/destinations/snowplow_micro.rb b/lib/gitlab/tracking/destinations/snowplow_micro.rb index e15c03b6808..1fc4b4e6d9c 100644 --- a/lib/gitlab/tracking/destinations/snowplow_micro.rb +++ b/lib/gitlab/tracking/destinations/snowplow_micro.rb @@ -7,6 +7,8 @@ module Gitlab include ::Gitlab::Utils::StrongMemoize extend ::Gitlab::Utils::Override + delegate :flush, to: :tracker + DEFAULT_URI = 'http://localhost:9090' override :options diff --git a/lib/gitlab/tracking/event_definition.rb b/lib/gitlab/tracking/event_definition.rb index 928eb6338f6..9d197de454e 100644 --- a/lib/gitlab/tracking/event_definition.rb +++ b/lib/gitlab/tracking/event_definition.rb @@ -17,9 +17,7 @@ module Gitlab end def definitions - paths.each_with_object({}) do |glob_path, definitions| - load_all_from_path!(definitions, glob_path) - end + paths.flat_map { |glob_path| load_all_from_path(glob_path) } end private @@ -34,11 +32,8 @@ module Gitlab Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Gitlab::Tracking::InvalidEventError.new(e.message)) end - def load_all_from_path!(definitions, glob_path) - Dir.glob(glob_path).each do |path| - definition = load_from_file(path) - definitions[definition.path] = definition - end + def load_all_from_path(glob_path) + Dir.glob(glob_path).map { |path| load_from_file(path) } end end diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index 8f2dfce67bb..8164cc4524a 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -11,6 +11,7 @@ require 'ipaddress' module Gitlab class UrlBlocker + GETADDRINFO_TIMEOUT_SECONDS = 15 DENY_ALL_REQUESTS_EXCEPT_ALLOWED_DEFAULT = proc { deny_all_requests_except_allowed_app_setting }.freeze # Result stores the validation result: @@ -181,12 +182,16 @@ module Gitlab # # @return [Array<Addrinfo>] def get_address_info(uri) - Addrinfo.getaddrinfo(uri.hostname, get_port(uri), nil, :STREAM).map do |addr| - addr.ipv6_v4mapped? ? addr.ipv6_to_ipv4 : addr + Timeout.timeout(GETADDRINFO_TIMEOUT_SECONDS) do + Addrinfo.getaddrinfo(uri.hostname, get_port(uri), nil, :STREAM).map do |addr| + addr.ipv6_v4mapped? ? addr.ipv6_to_ipv4 : addr + end end - rescue ArgumentError => error + rescue Timeout::Error => e + raise Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError, e.message + rescue ArgumentError => e # Addrinfo.getaddrinfo errors if the domain exceeds 1024 characters. - raise unless error.message.include?('hostname too long') + raise unless e.message.include?('hostname too long') raise Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError, "Host is too long (maximum is 1024 characters)" end diff --git a/lib/gitlab/usage/metric.rb b/lib/gitlab/usage/metric.rb index d7e983d126a..1efd8ded77c 100644 --- a/lib/gitlab/usage/metric.rb +++ b/lib/gitlab/usage/metric.rb @@ -37,11 +37,8 @@ module Gitlab ::Gitlab::Usage::Metrics::KeyPathProcessor.process(definition.key_path, value) end - def instrumentation_class - "Gitlab::Usage::Metrics::Instrumentations::#{definition.instrumentation_class}" - end - def instrumentation_object + instrumentation_class = "Gitlab::Usage::Metrics::Instrumentations::#{definition.instrumentation_class}" @instrumentation_object ||= instrumentation_class.constantize.new(definition.attributes) end end diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb index 941c2f793c4..5eddf8da7dd 100644 --- a/lib/gitlab/usage/metric_definition.rb +++ b/lib/gitlab/usage/metric_definition.rb @@ -25,6 +25,14 @@ module Gitlab events_from_new_structure || events_from_old_structure || {} end + def instrumentation_class + if internal_events? + events.each_value.first.nil? ? "TotalCountMetric" : "RedisHLLMetric" + else + attributes[:instrumentation_class] + end + end + def to_context return unless %w[redis redis_hll].include?(data_source) @@ -77,6 +85,10 @@ module Gitlab VALID_SERVICE_PING_STATUSES.include?(attributes[:status]) end + def internal_events? + data_source == 'internal_events' + end + alias_method :to_dictionary, :to_h class << self @@ -97,7 +109,9 @@ module Gitlab end def with_instrumentation_class - all.select { |definition| definition.attributes[:instrumentation_class].present? && definition.available? } + all.select do |definition| + (definition.internal_events? || definition.attributes[:instrumentation_class].present?) && definition.available? + end end def context_for(key_path) diff --git a/lib/gitlab/usage/metrics/instrumentations/bulk_imports_users_metric.rb b/lib/gitlab/usage/metrics/instrumentations/bulk_imports_users_metric.rb new file mode 100644 index 00000000000..453e9a13765 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/bulk_imports_users_metric.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class BulkImportsUsersMetric < DatabaseMetric + operation :distinct_count, column: :user_id + + relation do + ::BulkImport + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_service_desk_custom_email_enabled_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_service_desk_custom_email_enabled_metric.rb new file mode 100644 index 00000000000..85f59f36941 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_service_desk_custom_email_enabled_metric.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountServiceDeskCustomEmailEnabledMetric < DatabaseMetric + operation :count + + relation do + ServiceDeskSetting.where(custom_email_enabled: true) + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/csv_imports_users_metric.rb b/lib/gitlab/usage/metrics/instrumentations/csv_imports_users_metric.rb new file mode 100644 index 00000000000..16f498fbc5a --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/csv_imports_users_metric.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CsvImportsUsersMetric < DatabaseMetric + operation :distinct_count, column: :user_id + + relation do + ::Issues::CsvImport + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb b/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb index 774f65da3bf..0a47045aab5 100644 --- a/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb @@ -32,9 +32,9 @@ module Gitlab super(metric_definition.reverse_merge(time_frame: 'none')) end - def value(...) + def value alt_usage_data(fallback: self.class.fallback) do - self.class.metric_value.call(...) + instance_eval(&self.class.metric_value) end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/gitlab_config_metric.rb b/lib/gitlab/usage/metrics/instrumentations/gitlab_config_metric.rb new file mode 100644 index 00000000000..daeef06e6c5 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/gitlab_config_metric.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class GitlabConfigMetric < GenericMetric + value do + method_name_array = config_hash_to_method_array(options[:config]) + + method_name_array.inject(Gitlab.config, :public_send) + end + + private + + def config_hash_to_method_array(object) + object.each_with_object([]) do |(key, value), result| + result.append(key) + + if value.is_a?(Hash) + result.concat(config_hash_to_method_array(value)) + else + result.append(value) + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/gitlab_settings_metric.rb b/lib/gitlab/usage/metrics/instrumentations/gitlab_settings_metric.rb new file mode 100644 index 00000000000..6a36b69e287 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/gitlab_settings_metric.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class GitlabSettingsMetric < GenericMetric + value do + # rubocop:disable GitlabSecurity/PublicSend -- this is on static data and not a user-controlled input + Gitlab::CurrentSettings.public_send(options[:setting_method]) + # rubocop:enable GitlabSecurity/PublicSend + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/group_imports_users_metric.rb b/lib/gitlab/usage/metrics/instrumentations/group_imports_users_metric.rb new file mode 100644 index 00000000000..a1207300c2a --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/group_imports_users_metric.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class GroupImportsUsersMetric < DatabaseMetric + operation :distinct_count, column: :user_id + + relation do + ::GroupImportState + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/in_product_marketing_email_cta_clicked_metric.rb b/lib/gitlab/usage/metrics/instrumentations/in_product_marketing_email_cta_clicked_metric.rb deleted file mode 100644 index b1a2de29fd7..00000000000 --- a/lib/gitlab/usage/metrics/instrumentations/in_product_marketing_email_cta_clicked_metric.rb +++ /dev/null @@ -1,54 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Usage - module Metrics - module Instrumentations - class InProductMarketingEmailCtaClickedMetric < DatabaseMetric - operation :count - - def initialize(metric_definition) - super - - unless track.in?(allowed_track) - raise ArgumentError, "track '#{track}' must be one of: #{allowed_track.join(', ')}" - end - - return if series.in?(allowed_series) - - raise ArgumentError, "series '#{series}' must be one of: #{allowed_series.join(', ')}" - end - - relation { Users::InProductMarketingEmail } - - private - - def relation - scope = super.where.not(cta_clicked_at: nil) - scope = scope.where(series: series) - scope.where(track: track) - end - - def track - options[:track] - end - - def series - options[:series] - end - - def allowed_track - Users::InProductMarketingEmail::ACTIVE_TRACKS.keys - end - - def allowed_series - @allowed_series ||= begin - series_amount = Namespaces::InProductMarketingEmailsService.email_count_for_track(track) - 0.upto(series_amount - 1).to_a - end - end - end - end - end - end -end diff --git a/lib/gitlab/usage/metrics/instrumentations/in_product_marketing_email_sent_metric.rb b/lib/gitlab/usage/metrics/instrumentations/in_product_marketing_email_sent_metric.rb deleted file mode 100644 index 50dec606d9b..00000000000 --- a/lib/gitlab/usage/metrics/instrumentations/in_product_marketing_email_sent_metric.rb +++ /dev/null @@ -1,54 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Usage - module Metrics - module Instrumentations - class InProductMarketingEmailSentMetric < DatabaseMetric - operation :count - - def initialize(metric_definition) - super - - unless track.in?(allowed_track) - raise ArgumentError, "track '#{track}' must be one of: #{allowed_track.join(', ')}" - end - - return if series.in?(allowed_series) - - raise ArgumentError, "series '#{series}' must be one of: #{allowed_series.join(', ')}" - end - - relation { Users::InProductMarketingEmail } - - private - - def relation - scope = super - scope = scope.where(series: series) - scope.where(track: track) - end - - def track - options[:track] - end - - def series - options[:series] - end - - def allowed_track - Users::InProductMarketingEmail::ACTIVE_TRACKS.keys - end - - def allowed_series - @allowed_series ||= begin - series_amount = Namespaces::InProductMarketingEmailsService.email_count_for_track(track) - 0.upto(series_amount - 1).to_a - end - end - end - end - end - end -end diff --git a/lib/gitlab/usage/metrics/instrumentations/index_inconsistencies_metric.rb b/lib/gitlab/usage/metrics/instrumentations/index_inconsistencies_metric.rb index 2ce7e95ce77..b5c3420b5fe 100644 --- a/lib/gitlab/usage/metrics/instrumentations/index_inconsistencies_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/index_inconsistencies_metric.rb @@ -18,26 +18,24 @@ module Gitlab end end - class << self - private + private - def database - database_model = Gitlab::Database.database_base_models[Gitlab::Database::MAIN_DATABASE_NAME] - Gitlab::Schema::Validation::Sources::Database.new(database_model.connection) - end + def database + database_model = Gitlab::Database.database_base_models[Gitlab::Database::MAIN_DATABASE_NAME] + Gitlab::Schema::Validation::Sources::Database.new(database_model.connection) + end - def structure_sql - stucture_sql_path = Rails.root.join('db/structure.sql') - Gitlab::Schema::Validation::Sources::StructureSql.new(stucture_sql_path) - end + def structure_sql + stucture_sql_path = Rails.root.join('db/structure.sql') + Gitlab::Schema::Validation::Sources::StructureSql.new(stucture_sql_path) + end - def validators - [ - Gitlab::Schema::Validation::Validators::MissingIndexes, - Gitlab::Schema::Validation::Validators::DifferentDefinitionIndexes, - Gitlab::Schema::Validation::Validators::ExtraIndexes - ] - end + def validators + [ + Gitlab::Schema::Validation::Validators::MissingIndexes, + Gitlab::Schema::Validation::Validators::DifferentDefinitionIndexes, + Gitlab::Schema::Validation::Validators::ExtraIndexes + ] end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/jira_imports_users_metric.rb b/lib/gitlab/usage/metrics/instrumentations/jira_imports_users_metric.rb new file mode 100644 index 00000000000..239df5605ae --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/jira_imports_users_metric.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class JiraImportsUsersMetric < DatabaseMetric + operation :distinct_count, column: :user_id + + relation do + ::JiraImportState + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/omniauth_enabled_metric.rb b/lib/gitlab/usage/metrics/instrumentations/omniauth_enabled_metric.rb new file mode 100644 index 00000000000..d6496da569a --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/omniauth_enabled_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class OmniauthEnabledMetric < GenericMetric + value do + Gitlab::Auth.omniauth_enabled? + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/project_imports_creators_metric.rb b/lib/gitlab/usage/metrics/instrumentations/project_imports_creators_metric.rb new file mode 100644 index 00000000000..f34bd6dbfe3 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/project_imports_creators_metric.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class ProjectImportsCreatorsMetric < DatabaseMetric + operation :distinct_count, column: :creator_id + + relation do + ::Project.where.not(import_type: nil) + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/prometheus_enabled_metric.rb b/lib/gitlab/usage/metrics/instrumentations/prometheus_enabled_metric.rb new file mode 100644 index 00000000000..d5d07637a9c --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/prometheus_enabled_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class PrometheusEnabledMetric < GenericMetric + value do + Gitlab::Prometheus::Internal.prometheus_enabled? + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/prometheus_metric.rb b/lib/gitlab/usage/metrics/instrumentations/prometheus_metric.rb index ab1298b63c3..cc6be7fb349 100644 --- a/lib/gitlab/usage/metrics/instrumentations/prometheus_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/prometheus_metric.rb @@ -18,7 +18,7 @@ module Gitlab # end def value with_prometheus_client(verify: false, fallback: FALLBACK) do |client| - super(client) + self.class.metric_value.call(client) end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/prometheus_metrics_enabled_metric.rb b/lib/gitlab/usage/metrics/instrumentations/prometheus_metrics_enabled_metric.rb new file mode 100644 index 00000000000..76f9c7d2588 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/prometheus_metrics_enabled_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class PrometheusMetricsEnabledMetric < GenericMetric + value do + Gitlab::Metrics.prometheus_metrics_enabled? + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/reply_by_email_enabled_metric.rb b/lib/gitlab/usage/metrics/instrumentations/reply_by_email_enabled_metric.rb new file mode 100644 index 00000000000..24502147352 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/reply_by_email_enabled_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class ReplyByEmailEnabledMetric < GenericMetric + value do + Gitlab::Email::IncomingEmail.enabled? + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/schema_inconsistencies_metric.rb b/lib/gitlab/usage/metrics/instrumentations/schema_inconsistencies_metric.rb index a481f7a5682..737cecccec3 100644 --- a/lib/gitlab/usage/metrics/instrumentations/schema_inconsistencies_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/schema_inconsistencies_metric.rb @@ -21,22 +21,20 @@ module Gitlab end end - class << self - private + private - def validators - Gitlab::Schema::Validation::Validators::Base.all_validators - end + def validators + Gitlab::Schema::Validation::Validators::Base.all_validators + end - def database - database_model = Gitlab::Database.database_base_models[Gitlab::Database::MAIN_DATABASE_NAME] - Gitlab::Schema::Validation::Sources::Database.new(database_model.connection) - end + def database + database_model = Gitlab::Database.database_base_models[Gitlab::Database::MAIN_DATABASE_NAME] + Gitlab::Schema::Validation::Sources::Database.new(database_model.connection) + end - def structure_sql - stucture_sql_path = Rails.root.join('db/structure.sql') - Gitlab::Schema::Validation::Sources::StructureSql.new(stucture_sql_path) - end + def structure_sql + stucture_sql_path = Rails.root.join('db/structure.sql') + Gitlab::Schema::Validation::Sources::StructureSql.new(stucture_sql_path) end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/total_count_metric.rb b/lib/gitlab/usage/metrics/instrumentations/total_count_metric.rb index d07438f4bf7..ce7b2feb745 100644 --- a/lib/gitlab/usage/metrics/instrumentations/total_count_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/total_count_metric.rb @@ -14,20 +14,56 @@ module Gitlab # class TotalCountMetric < BaseMetric include Gitlab::UsageDataCounters::RedisCounter + extend Gitlab::Usage::TimeSeriesStorable KEY_PREFIX = "{event_counters}_" - def self.redis_key(event_name) - KEY_PREFIX + event_name + def self.redis_key(event_name, date = nil) + base_key = KEY_PREFIX + event_name + return base_key unless date + + apply_time_aggregation(base_key, date) end def value - events.sum do |event| + return total_value if time_frame == 'all' + + period_value + end + + private + + def total_value + event_names.sum do |event_name| redis_usage_data do - total_count(self.class.redis_key(event[:name])) + total_count(self.class.redis_key(event_name)) end end end + + def period_value + keys = self.class.keys_for_aggregation(events: event_names, **time_constraint) + keys.sum do |key| + redis_usage_data do + total_count(key) + end + end + end + + def time_constraint + case time_frame + when '28d' + monthly_time_range + when '7d' + weekly_time_range + else + raise "Unknown time frame: #{time_frame} for #{self.class} :: #{events}" + end + end + + def event_names + events.pluck(:name) + end end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/unique_users_all_imports_metric.rb b/lib/gitlab/usage/metrics/instrumentations/unique_users_all_imports_metric.rb new file mode 100644 index 00000000000..931859bf7fa --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/unique_users_all_imports_metric.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class UniqueUsersAllImportsMetric < NumbersMetric + IMPORTS_METRICS = [ + ProjectImportsCreatorsMetric, + BulkImportsUsersMetric, + JiraImportsUsersMetric, + CsvImportsUsersMetric, + GroupImportsUsersMetric + ].freeze + + operation :add + + data do |time_frame| + IMPORTS_METRICS.map { |metric| metric.new(time_frame: time_frame).value } + end + + # overwriting instrumentation to generate the appropriate sql query + def instrumentation + metric_queries = IMPORTS_METRICS.map do |metric| + "(#{metric.new(time_frame: time_frame).instrumentation})" + end.join(' + ') + + "SELECT #{metric_queries}" + end + end + end + end + end +end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 5f819f060e4..e36bf9ff6ad 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -157,28 +157,7 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord - def features_usage_data - features_usage_data_ce - end - - def features_usage_data_ce - { - instance_auto_devops_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.auto_devops_enabled? }, - container_registry_enabled: alt_usage_data(fallback: nil) { Gitlab.config.registry.enabled }, - dependency_proxy_enabled: Gitlab.config.try(:dependency_proxy)&.enabled, - gitlab_shared_runners_enabled: alt_usage_data(fallback: nil) { Gitlab.config.gitlab_ci.shared_runners_enabled }, - gravatar_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.gravatar_enabled? }, - ldap_enabled: alt_usage_data(fallback: nil) { Gitlab.config.ldap.enabled }, - mattermost_enabled: alt_usage_data(fallback: nil) { Gitlab.config.mattermost.enabled }, - omniauth_enabled: alt_usage_data(fallback: nil) { Gitlab::Auth.omniauth_enabled? }, - prometheus_enabled: alt_usage_data(fallback: nil) { Gitlab::Prometheus::Internal.prometheus_enabled? }, - prometheus_metrics_enabled: alt_usage_data(fallback: nil) { Gitlab::Metrics.prometheus_metrics_enabled? }, - reply_by_email_enabled: alt_usage_data(fallback: nil) { Gitlab::Email::IncomingEmail.enabled? }, - signup_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.allow_signup? }, - grafana_link_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.grafana_enabled? }, - gitpod_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.gitpod_enabled? } - } - end + def features_usage_data = {} def components_usage_data { @@ -365,7 +344,6 @@ module Gitlab users_created: count(::User.where(time_period), start: minimum_id(User), finish: maximum_id(User)), omniauth_providers: filtered_omniauth_provider_names.reject { |name| name == 'group_saml' }, user_auth_by_provider: distinct_count_user_auth_by_provider(time_period), - unique_users_all_imports: unique_users_all_imports(time_period), bulk_imports: { gitlab_v1: count(::BulkImport.where(**time_period, source_type: :gitlab)) }, @@ -417,7 +395,6 @@ module Gitlab service_desk_enabled_projects: distinct_count_service_desk_enabled_projects(time_period), service_desk_issues: count(::Issue.service_desk.where(time_period)), projects_jira_active: distinct_count(::Project.with_active_integration(::Integrations::Jira).where(time_period), :creator_id), - projects_jira_dvcs_cloud_active: distinct_count(::Project.with_active_integration(::Integrations::Jira).with_jira_dvcs_cloud.where(time_period), :creator_id), projects_jira_dvcs_server_active: distinct_count(::Project.with_active_integration(::Integrations::Jira).with_jira_dvcs_server.where(time_period), :creator_id) } end @@ -565,18 +542,6 @@ module Gitlab end # rubocop:disable CodeReuse/ActiveRecord - def unique_users_all_imports(time_period) - project_imports = distinct_count(::Project.where(time_period).where.not(import_type: nil), :creator_id) - bulk_imports = distinct_count(::BulkImport.where(time_period), :user_id) - jira_issue_imports = distinct_count(::JiraImportState.where(time_period), :user_id) - csv_issue_imports = distinct_count(::Issues::CsvImport.where(time_period), :user_id) - group_imports = distinct_count(::GroupImportState.where(time_period), :user_id) - - add(project_imports, bulk_imports, jira_issue_imports, csv_issue_imports, group_imports) - end - # rubocop:enable CodeReuse/ActiveRecord - - # rubocop:disable CodeReuse/ActiveRecord def distinct_count_user_auth_by_provider(time_period) counts = auth_providers_except_ldap.index_with do |provider| distinct_count( diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index 185b49d4a68..b0444066722 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -66,7 +66,7 @@ module Gitlab rescue StandardError => e # Ignore any exceptions unless is dev or test env - # The application flow should not be blocked by erros in tracking + # The application flow should not be blocked by errors in tracking Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) end diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb index 534a08cad9a..8310c464a59 100644 --- a/lib/gitlab/usage_data_queries.rb +++ b/lib/gitlab/usage_data_queries.rb @@ -75,16 +75,6 @@ module Gitlab } end - # rubocop: disable CodeReuse/ActiveRecord - def sent_in_product_marketing_email_count(sent_emails, track, series) - count(Users::InProductMarketingEmail.where(track: track, series: series)) - end - - def clicked_in_product_marketing_email_count(clicked_emails, track, series) - count(Users::InProductMarketingEmail.where(track: track, series: series).where.not(cta_clicked_at: nil)) - end - # rubocop: enable CodeReuse/ActiveRecord - def stage_manage_events(time_period) # rubocop: disable CodeReuse/ActiveRecord # rubocop: disable UsageData/LargeTable diff --git a/lib/gitlab/web_ide/default_oauth_application.rb b/lib/gitlab/web_ide/default_oauth_application.rb new file mode 100644 index 00000000000..01b7637c1c0 --- /dev/null +++ b/lib/gitlab/web_ide/default_oauth_application.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module WebIde + module DefaultOauthApplication + class << self + def feature_enabled?(current_user) + Feature.enabled?(:vscode_web_ide, current_user) && Feature.enabled?(:web_ide_oauth, current_user) + end + + def oauth_application + application_settings.web_ide_oauth_application + end + + def oauth_callback_url + Gitlab::Routing.url_helpers.ide_oauth_redirect_url + end + + def ensure_oauth_application! + return if oauth_application + + should_expire_cache = false + + application_settings.transaction do + # note: This should run very rarely and should be safe for us to do a lock + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/132496#note_1587293087 + application_settings.lock! + + # note: `lock!`` breaks applicaiton_settings cache and will trigger another query. + # We need to double check here so that requests previously waiting on the lock can + # now just skip. + next if oauth_application + + application = Doorkeeper::Application.new( + name: 'GitLab Web IDE', + redirect_uri: oauth_callback_url, + scopes: ['api'], + trusted: true, + confidential: false) + application.save! + application_settings.update!(web_ide_oauth_application: application) + should_expire_cache = true + end + + # note: This needs to happen outside the transaction, but only if we actually changed something + ::Gitlab::CurrentSettings.expire_current_application_settings if should_expire_cache + end + + private + + def application_settings + ::Gitlab::CurrentSettings.current_application_settings + end + end + end + end +end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 057e89a2a97..715638ba0d9 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -157,7 +157,15 @@ module Gitlab ] end - def send_url(url, allow_redirects: false, method: 'GET', body: nil, headers: nil) + # response_statuses can be set for 'error' and 'timeout'. They are optional. + # Their values must be a symbol accepted by Rack::Utils::SYMBOL_TO_STATUS_CODE. + # Example: response_statuses : { error: :internal_server_error, timeout: :bad_request } + # timeouts can be given for the opening the connection and reading the response headers. + # Their values must be given in seconds. + # Example: timeouts: { open: 5, read: 5 } + def send_url( + url, allow_redirects: false, method: 'GET', body: nil, headers: nil, timeouts: {}, response_statuses: {} + ) params = { 'URL' => url, 'AllowRedirects' => allow_redirects, @@ -166,9 +174,24 @@ module Gitlab 'Method' => method }.compact + if timeouts.present? + params['DialTimeout'] = "#{timeouts[:open]}s" if timeouts[:open] + params['ResponseHeaderTimeout'] = "#{timeouts[:read]}s" if timeouts[:read] + end + + if response_statuses.present? + if response_statuses[:error] + params['ErrorResponseStatus'] = Rack::Utils::SYMBOL_TO_STATUS_CODE[response_statuses[:error]] + end + + if response_statuses[:timeout] + params['TimeoutResponseStatus'] = Rack::Utils::SYMBOL_TO_STATUS_CODE[response_statuses[:timeout]] + end + end + [ SEND_DATA_HEADER, - "send-url:#{encode(params)}" + "send-url:#{encode(params.compact)}" ] end |