diff options
Diffstat (limited to 'lib/gitlab')
258 files changed, 4309 insertions, 2488 deletions
diff --git a/lib/gitlab/analytics/cycle_analytics/aggregated/records_fetcher.rb b/lib/gitlab/analytics/cycle_analytics/aggregated/records_fetcher.rb index 07dc4c02ba8..2143497f084 100644 --- a/lib/gitlab/analytics/cycle_analytics/aggregated/records_fetcher.rb +++ b/lib/gitlab/analytics/cycle_analytics/aggregated/records_fetcher.rb @@ -94,10 +94,10 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def preload_associations(records) - ActiveRecord::Associations::Preloader.new.preload( - records, - MAPPINGS.fetch(subject_class).fetch(:includes_for_query) - ) + ActiveRecord::Associations::Preloader.new( + records: records, + associations: MAPPINGS.fetch(subject_class).fetch(:includes_for_query) + ).call records end diff --git a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb index 140c4a300ca..9deb5072112 100644 --- a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb +++ b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb @@ -67,10 +67,10 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def preload_associations(records) # using preloader instead of includes to avoid AR generating a large column list - ActiveRecord::Associations::Preloader.new.preload( - records, - MAPPINGS.fetch(subject_class).fetch(:includes_for_query) - ) + ActiveRecord::Associations::Preloader.new( + records: records, + associations: MAPPINGS.fetch(subject_class).fetch(:includes_for_query) + ).call records end diff --git a/lib/gitlab/analytics/cycle_analytics/request_params.rb b/lib/gitlab/analytics/cycle_analytics/request_params.rb index 2df3680db5f..3e70d64fea6 100644 --- a/lib/gitlab/analytics/cycle_analytics/request_params.rb +++ b/lib/gitlab/analytics/cycle_analytics/request_params.rb @@ -38,13 +38,12 @@ module Gitlab attribute :created_after, :datetime attribute :created_before, :datetime - attribute :group + attribute :namespace attribute :current_user attribute :value_stream attribute :sort attribute :direction attribute :page - attribute :project attribute :stage_id attribute :end_event_filter @@ -66,10 +65,6 @@ module Gitlab self.end_event_filter ||= Gitlab::Analytics::CycleAnalytics::BaseQueryBuilder::DEFAULT_END_EVENT_FILTER end - def project_ids - Array(@project_ids) - end - def to_data_collector_params { current_user: current_user, @@ -86,12 +81,9 @@ module Gitlab def to_data_attributes {}.tap do |attrs| - attrs[:aggregation] = aggregation_attributes if group - attrs[:group] = group_data_attributes if group attrs[:value_stream] = value_stream_data_attributes.to_json if value_stream attrs[:created_after] = created_after.to_date.iso8601 attrs[:created_before] = created_before.to_date.iso8601 - attrs[:projects] = group_projects(project_ids) if group && project_ids.present? attrs[:labels] = label_name.to_json if label_name.present? attrs[:assignees] = assignee_username.to_json if assignee_username.present? attrs[:author] = author_username if author_username.present? @@ -99,35 +91,61 @@ module Gitlab attrs[:sort] = sort if sort.present? attrs[:direction] = direction if direction.present? attrs[:stage] = stage_data_attributes.to_json if stage_id.present? + attrs[:namespace] = namespace_attributes + attrs[:enable_tasks_by_type_chart] = 'false' + attrs[:default_stages] = Gitlab::Analytics::CycleAnalytics::DefaultStages.all.map do |stage_params| + ::Analytics::CycleAnalytics::StagePresenter.new(stage_params) + end.to_json + + attrs.merge!(foss_project_level_params, resource_paths) end end + def project_ids + Array(@project_ids) + end + private - def use_aggregated_backend? - # for now it's only available on the group-level - group.present? - end + delegate :url_helpers, to: Gitlab::Routing + + def foss_project_level_params + return {} unless project - def aggregation_attributes { - enabled: aggregation.enabled.to_s, - last_run_at: aggregation.last_incremental_run_at&.iso8601, - next_run_at: aggregation.estimated_next_run_at&.iso8601 + project_id: project.id, + group_path: project.group&.path, + request_path: url_helpers.project_cycle_analytics_path(project), + full_path: project.full_path } end - def aggregation - @aggregation ||= ::Analytics::CycleAnalytics::Aggregation.safe_create_for_namespace(group) + def resource_paths + helpers = ActionController::Base.helpers + + {}.tap do |paths| + paths[:empty_state_svg_path] = helpers.image_path("illustrations/analytics/cycle-analytics-empty-chart.svg") + paths[:no_data_svg_path] = helpers.image_path("illustrations/analytics/cycle-analytics-empty-chart.svg") + paths[:no_access_svg_path] = helpers.image_path("illustrations/analytics/no-access.svg") + + if project + paths[:milestones_path] = url_helpers.project_milestones_path(project, format: :json) + paths[:labels_path] = url_helpers.project_labels_path(project, format: :json) + end + end + end + + # FOSS version doesn't use the aggregated VSA backend + def use_aggregated_backend? + false end - def group_data_attributes + def namespace_attributes + return {} unless project + { - id: group.id, - namespace_id: group.id, - name: group.name, - full_path: group.full_path, - avatar_url: group.avatar_url + name: project.name, + full_path: project.full_path } end @@ -139,28 +157,6 @@ module Gitlab } end - def group_projects(project_ids) - GroupProjectsFinder.new( - group: group, - current_user: current_user, - options: { include_subgroups: true }, - project_ids_relation: project_ids - ) - .execute - .with_route - .map { |project| project_data_attributes(project) } - .to_json - end - - def project_data_attributes(project) - { - id: project.to_gid.to_s, - name: project.name, - path_with_namespace: project.path_with_namespace, - avatar_url: project.avatar_url - } - end - def stage_data_attributes return unless stage @@ -196,10 +192,18 @@ module Gitlab return unless value_stream strong_memoize(:stage) do - ::Analytics::CycleAnalytics::StageFinder.new(parent: project&.project_namespace || group, stage_id: stage_id).execute if stage_id + ::Analytics::CycleAnalytics::StageFinder.new(parent: namespace, stage_id: stage_id).execute if stage_id + end + end + + def project + strong_memoize(:project) do + namespace.project if namespace.is_a?(Namespaces::ProjectNamespace) end end end end end end + +Gitlab::Analytics::CycleAnalytics::RequestParams.prepend_mod_with('Gitlab::Analytics::CycleAnalytics::RequestParams') diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb index 9b4cbc9090c..85dcc773e2b 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb @@ -6,7 +6,7 @@ module Gitlab module StageEvents class PlanStageStart < MetricsBasedStageEvent def self.name - s_("CycleAnalyticsEvent|Issue first associated with a milestone or issue first added to a board") + s_("CycleAnalyticsEvent|Issue first associated with a milestone or first added to a board") end def self.identifier diff --git a/lib/gitlab/app_logger.rb b/lib/gitlab/app_logger.rb index a39e7f31886..40bdc538594 100644 --- a/lib/gitlab/app_logger.rb +++ b/lib/gitlab/app_logger.rb @@ -5,7 +5,7 @@ module Gitlab LOGGERS = [Gitlab::AppTextLogger, Gitlab::AppJsonLogger].freeze def self.loggers - if Gitlab::Utils.to_boolean(ENV.fetch('UNSTRUCTURED_RAILS_LOG', 'true')) + if Gitlab::Utils.to_boolean(ENV.fetch('UNSTRUCTURED_RAILS_LOG', 'false')) LOGGERS else [Gitlab::AppJsonLogger] diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index 466538df56e..71629eb701c 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -55,8 +55,12 @@ module Gitlab phone_verification_verify_code: { threshold: 10, interval: 10.minutes }, namespace_exists: { threshold: 20, interval: 1.minute }, fetch_google_ip_list: { threshold: 10, interval: 1.minute }, + project_fork_sync: { threshold: 10, interval: 30.minutes }, jobs_index: { threshold: 600, interval: 1.minute }, - bulk_import: { threshold: 6, interval: 1.minute } + bulk_import: { threshold: 6, interval: 1.minute }, + projects_api_rate_limit_unauthenticated: { + threshold: -> { application_settings.projects_api_rate_limit_unauthenticated }, interval: 10.minutes + } }.freeze end diff --git a/lib/gitlab/audit/auditor.rb b/lib/gitlab/audit/auditor.rb index fddc1f830aa..e3d2b394404 100644 --- a/lib/gitlab/audit/auditor.rb +++ b/lib/gitlab/audit/auditor.rb @@ -5,6 +5,10 @@ module Gitlab class Auditor attr_reader :scope, :name + PERMITTED_TARGET_CLASSES = [ + ::Operations::FeatureFlag + ].freeze + # Record audit events # # @param [Hash] context @@ -113,7 +117,11 @@ module Gitlab end def audit_enabled? - authentication_event? + authentication_event? || permitted_target? + end + + def permitted_target? + @target.class.in? PERMITTED_TARGET_CLASSES end def authentication_event? diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index 01e126ec2f5..bb47b4236fb 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -233,7 +233,7 @@ module Gitlab email ||= auth_hash.email valid_username = ::Namespace.clean_path(username) - valid_username = Uniquify.new.string(valid_username) { |s| !NamespacePathValidator.valid_path?(s) } + valid_username = Gitlab::Utils::Uniquify.new.string(valid_username) { |s| !NamespacePathValidator.valid_path?(s) } { name: name.strip.presence || valid_username, diff --git a/lib/gitlab/auth/otp/duo_auth.rb b/lib/gitlab/auth/otp/duo_auth.rb new file mode 100644 index 00000000000..eeae04bc08b --- /dev/null +++ b/lib/gitlab/auth/otp/duo_auth.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module Auth + module Otp + module DuoAuth + def duo_auth_enabled?(_user) + ::Gitlab.config.duo_auth.enabled + end + end + end + end +end diff --git a/lib/gitlab/auth/otp/strategies/duo_auth/manual_otp.rb b/lib/gitlab/auth/otp/strategies/duo_auth/manual_otp.rb new file mode 100644 index 00000000000..57bc88de175 --- /dev/null +++ b/lib/gitlab/auth/otp/strategies/duo_auth/manual_otp.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Auth + module Otp + module Strategies + module DuoAuth + class ManualOtp < Base + include Gitlab::Utils::StrongMemoize + + def validate(otp_code) + params = { username: user.username, factor: "passcode", passcode: otp_code.to_i } + response = duo_client.request('POST', "/auth/v2/auth", params) + approve_or_deny(parse_response(response)) + rescue StandardError => e + Gitlab::AppLogger.error(e) + error(e.message) + end + + private + + def duo_client + DuoApi.new(::Gitlab.config.duo_auth.integration_key, + ::Gitlab.config.duo_auth.secret_key, + ::Gitlab.config.duo_auth.hostname) + end + strong_memoize_attr :duo_client + + def parse_response(response) + Gitlab::Json.parse(response.body) + end + + def approve_or_deny(parsed_response) + result_key = parsed_response.dig('response', 'result') + if result_key.to_s == "allow" + success + else + error(message: parsed_response.dig('response', 'status_msg').to_s) + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/auth/user_access_denied_reason.rb b/lib/gitlab/auth/user_access_denied_reason.rb index 322dfa74d09..3025960a8ab 100644 --- a/lib/gitlab/auth/user_access_denied_reason.rb +++ b/lib/gitlab/auth/user_access_denied_reason.rb @@ -29,7 +29,8 @@ module Gitlab "Your password expired. "\ "Please access GitLab from a web browser to update your password." else - "Your account has been blocked." + "Your request has been rejected for an unknown reason."\ + "Please contact your GitLab administrator and/or GitLab Support." end end diff --git a/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens.rb b/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens.rb index 82e607ac7a7..6f5ddec628d 100644 --- a/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens.rb +++ b/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens.rb @@ -12,7 +12,7 @@ module Gitlab end operation_name :update_all - feature_category :authentication_and_authorization + feature_category :system_access ADMIN_MODE_SCOPE = ['admin_mode'].freeze diff --git a/lib/gitlab/background_migration/backfill_compliance_violations.rb b/lib/gitlab/background_migration/backfill_compliance_violations.rb new file mode 100644 index 00000000000..131b4a05e41 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_compliance_violations.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Style/Documentation + class BackfillComplianceViolations < Gitlab::BackgroundMigration::BatchedMigrationJob + feature_category :compliance_management + + def perform + # no-op. The logic is defined in EE module. + end + end + # rubocop: enable Style/Documentation + end +end + +::Gitlab::BackgroundMigration::BackfillComplianceViolations.prepend_mod diff --git a/lib/gitlab/background_migration/backfill_namespace_ldap_settings.rb b/lib/gitlab/background_migration/backfill_namespace_ldap_settings.rb new file mode 100644 index 00000000000..1a5ad1c14a6 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_namespace_ldap_settings.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Back-fill container_registry_size for project_statistics + class BackfillNamespaceLdapSettings < Gitlab::BackgroundMigration::BatchedMigrationJob + operation_name :backfill_namespace_ldap_settings + feature_category :system_access + + def perform + # no-op in FOSS + end + end + end +end + +Gitlab::BackgroundMigration::BackfillNamespaceLdapSettings.prepend_mod diff --git a/lib/gitlab/background_migration/backfill_prepared_at_merge_requests.rb b/lib/gitlab/background_migration/backfill_prepared_at_merge_requests.rb new file mode 100644 index 00000000000..9bf503bd6e7 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_prepared_at_merge_requests.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfill prepared_at for an array of merge requests + class BackfillPreparedAtMergeRequests < ::Gitlab::BackgroundMigration::BatchedMigrationJob + scope_to ->(relation) { relation } + operation_name :update_all + feature_category :code_review_workflow + + def perform + each_sub_batch do |sub_batch| + sub_batch.where(prepared_at: nil).where.not(merge_status: 'preparing').update_all('prepared_at = created_at') + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_project_wiki_repositories.rb b/lib/gitlab/background_migration/backfill_project_wiki_repositories.rb new file mode 100644 index 00000000000..8d6df905f15 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_project_wiki_repositories.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfill project_wiki_repositories table for a range of projects + class BackfillProjectWikiRepositories < BatchedMigrationJob + operation_name :backfill_project_wiki_repositories + feature_category :geo_replication + + scope_to ->(relation) do + relation + .joins('LEFT OUTER JOIN project_wiki_repositories ON project_wiki_repositories.project_id = projects.id') + .where(project_wiki_repositories: { project_id: nil }) + end + + def perform + each_sub_batch do |sub_batch| + backfill_project_wiki_repositories(sub_batch) + end + end + + def backfill_project_wiki_repositories(relation) + connection.execute( + <<~SQL + INSERT INTO project_wiki_repositories (project_id, created_at, updated_at) + SELECT projects.id, now(), now() + FROM projects + WHERE projects.id IN(#{relation.select(:id).to_sql}) + ON CONFLICT (project_id) DO NOTHING; + SQL + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/batched_migration_job.rb b/lib/gitlab/background_migration/batched_migration_job.rb index 4039a79cfa7..952e6d01f1a 100644 --- a/lib/gitlab/background_migration/batched_migration_job.rb +++ b/lib/gitlab/background_migration/batched_migration_job.rb @@ -7,6 +7,8 @@ module Gitlab # # Job arguments needed must be defined explicitly, # see https://docs.gitlab.com/ee/development/database/batched_background_migrations.html#job-arguments. + # rubocop:disable Metrics/ClassLength + # rubocop:disable Metrics/ParameterLists class BatchedMigrationJob include Gitlab::Database::DynamicModelHelpers include Gitlab::ClassAttributes @@ -60,7 +62,8 @@ module Gitlab end def initialize( - start_id:, end_id:, batch_table:, batch_column:, sub_batch_size:, pause_ms:, job_arguments: [], connection: + start_id:, end_id:, batch_table:, batch_column:, sub_batch_size:, pause_ms:, job_arguments: [], connection:, + sub_batch_exception: nil ) @start_id = start_id @@ -71,6 +74,7 @@ module Gitlab @pause_ms = pause_ms @job_arguments = job_arguments @connection = connection + @sub_batch_exception = sub_batch_exception end def filter_batch(relation) @@ -87,7 +91,8 @@ module Gitlab private - attr_reader :start_id, :end_id, :batch_table, :batch_column, :sub_batch_size, :pause_ms, :connection + attr_reader :start_id, :end_id, :batch_table, :batch_column, :sub_batch_size, + :pause_ms, :connection, :sub_batch_exception def each_sub_batch(batching_arguments: {}, batching_scope: nil) all_batching_arguments = { column: batch_column, of: sub_batch_size }.merge(batching_arguments) @@ -98,6 +103,10 @@ module Gitlab sub_batch_relation.each_batch(**all_batching_arguments) do |relation| batch_metrics.instrument_operation(operation_name) do yield relation + rescue *Gitlab::Database::BackgroundMigration::BatchedJob::TIMEOUT_EXCEPTIONS => exception + exception_class = sub_batch_exception || exception.class + + raise exception_class, exception end sleep([pause_ms, 0].max * 0.001) @@ -137,3 +146,5 @@ module Gitlab end end end +# rubocop:enable Metrics/ClassLength +# rubocop:enable Metrics/ParameterLists diff --git a/lib/gitlab/background_migration/create_vulnerability_links.rb b/lib/gitlab/background_migration/create_vulnerability_links.rb new file mode 100644 index 00000000000..bbc71dfb392 --- /dev/null +++ b/lib/gitlab/background_migration/create_vulnerability_links.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +# rubocop:disable Style/Documentation +module Gitlab + module BackgroundMigration + class CreateVulnerabilityLinks < BatchedMigrationJob + feature_category :vulnerability_management + def perform; end + end + end +end + +Gitlab::BackgroundMigration::CreateVulnerabilityLinks.prepend_mod +# rubocop:enable Style/Documentation diff --git a/lib/gitlab/background_migration/delete_orphaned_packages_dependencies.rb b/lib/gitlab/background_migration/delete_orphaned_packages_dependencies.rb new file mode 100644 index 00000000000..a795300fa9d --- /dev/null +++ b/lib/gitlab/background_migration/delete_orphaned_packages_dependencies.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Deletes orphaned packages_dependencies records that have no packages_dependency_links + class DeleteOrphanedPackagesDependencies < BatchedMigrationJob + operation_name :delete_all + feature_category :package_registry + + scope_to ->(relation) { + relation.where( + <<~SQL.squish + NOT EXISTS ( + SELECT 1 + FROM packages_dependency_links + WHERE packages_dependency_links.dependency_id = packages_dependencies.id + ) + SQL + ) + } + + def perform + each_sub_batch(&:delete_all) + end + end + end +end diff --git a/lib/gitlab/background_migration/fix_vulnerability_reads_has_issues.rb b/lib/gitlab/background_migration/fix_vulnerability_reads_has_issues.rb new file mode 100644 index 00000000000..5b3b5642ba8 --- /dev/null +++ b/lib/gitlab/background_migration/fix_vulnerability_reads_has_issues.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This migration fixes existing `vulnerability_reads` records which did not have `has_issues` + # correctly set at the time of creation. + class FixVulnerabilityReadsHasIssues < BatchedMigrationJob + operation_name :fix_has_issues + feature_category :vulnerability_management + + # rubocop:disable Style/Documentation + class VulnerabilityRead < ::ApplicationRecord + self.table_name = 'vulnerability_reads' + + scope :with_vulnerability_ids, ->(ids) { where(vulnerability_id: ids) } + scope :without_issues, -> { where(has_issues: false) } + end + # rubocop:enable Style/Documentation + + def perform + each_sub_batch do |sub_batch| + vulnerability_reads_with_issue_links(sub_batch).update_all('has_issues = true') + end + end + + private + + def vulnerability_reads_with_issue_links(sub_batch) + VulnerabilityRead.with_vulnerability_ids(sub_batch.select(:vulnerability_id)).without_issues + end + end + end +end diff --git a/lib/gitlab/background_migration/issues_internal_id_scope_updater.rb b/lib/gitlab/background_migration/issues_internal_id_scope_updater.rb new file mode 100644 index 00000000000..21ca4392003 --- /dev/null +++ b/lib/gitlab/background_migration/issues_internal_id_scope_updater.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Migrates internal_ids records for `usage: issues` from project to namespace scope. + # For project issues it will be project namespace, for group issues it will be group namespace. + class IssuesInternalIdScopeUpdater < ::Gitlab::BackgroundMigration::BatchedMigrationJob + operation_name :issues_internal_id_scope_updater + feature_category :database + + ISSUES_USAGE = 0 # see Enums::InternalId#usage_resources[:issues] + + scope_to ->(relation) do + relation.where(usage: ISSUES_USAGE).where.not(project_id: nil) + end + + def perform + each_sub_batch do |sub_batch| + create_namespace_scoped_records(sub_batch) + delete_project_scoped_records(sub_batch) + end + end + + private + + def delete_project_scoped_records(sub_batch) + # There is no need to keep the project scoped issues usage as we move to scoping issues to namespace. + # Also in case we do decide to move back to scoping issues usage to project, we are better off if the + # project record is not present as that would result in overlapping IIDs because project scoped issues + # usage will have outdated IIDs left in the DB + log_info("Deleted internal_ids records", ids: sub_batch.pluck(:id)) + + connection.execute( + <<~SQL + DELETE FROM internal_ids WHERE id IN (#{sub_batch.select(:id).to_sql}) + SQL + ) + end + + def create_namespace_scoped_records(sub_batch) + # Creates a corresponding namespace scoped record for every `issues` usage scoped to a project. + # On conflict it means the record was already created when a new issue is created with the + # newly namespace scoped Issue model, see Issue#has_internal_id definition. In which case to + # make sure we have the namespace_id scoped record set to the greatest of the two last_values. + created_records_ids = connection.execute( + <<~SQL + INSERT INTO internal_ids (usage, last_value, namespace_id) + SELECT #{ISSUES_USAGE}, last_value, project_namespace_id + FROM internal_ids + INNER JOIN projects ON projects.id = internal_ids.project_id + WHERE internal_ids.id IN(#{sub_batch.select(:id).to_sql}) + ON CONFLICT (usage, namespace_id) WHERE namespace_id IS NOT NULL + DO UPDATE SET last_value = GREATEST(EXCLUDED.last_value, internal_ids.last_value) + RETURNING id; + SQL + ) + + log_info("Created/updated internal_ids records", ids: created_records_ids.field_values('id')) + end + + def log_info(message, **extra) + ::Gitlab::BackgroundMigration::Logger.info(migrator: self.class.to_s, message: message, **extra) + end + end + end +end diff --git a/lib/gitlab/background_migration/migrate_evidences_for_vulnerability_findings.rb b/lib/gitlab/background_migration/migrate_evidences_for_vulnerability_findings.rb new file mode 100644 index 00000000000..78a93b49c49 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_evidences_for_vulnerability_findings.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # The class to migrate the evidence data into their own records from the json attribute + class MigrateEvidencesForVulnerabilityFindings < BatchedMigrationJob + feature_category :vulnerability_management + operation_name :migrate_evidences_for_vulnerability_findings + + # The class is mimicking Vulnerabilites::Finding + class Finding < ApplicationRecord + self.table_name = 'vulnerability_occurrences' + + validates :details, json_schema: { filename: 'vulnerability_finding_details', draft: 7 }, if: false + end + + # The class is mimicking Vulnerabilites::Finding::Evidence + class Evidence < ApplicationRecord + self.table_name = 'vulnerability_finding_evidences' + + # This data has been already validated when parsed into vulnerability_occurrences.raw_metadata + # Having this validation is a requerment from: + # https://gitlab.com/gitlab-org/gitlab/-/blob/dc3262f850cbd0ac14171d3c389b1258b4749cda/spec/db/schema_spec.rb#L253-265 + validates :data, json_schema: { filename: "filename" }, if: false + end + + def perform + each_sub_batch do |sub_batch| + migrate_evidences(sub_batch) + end + end + + private + + def migrate_evidences(sub_batch) + attrs = sub_batch.filter_map do |finding| + evidence = extract_evidence(finding.raw_metadata) + + next unless evidence + + build_evidence(finding, evidence) + end.compact + + begin + create_evidences(attrs) if attrs.present? + rescue StandardError => e + logger.error( + message: e.message, + class: self.class.name + ) + end + end + + def build_evidence(finding, evidence) + current_time = Time.current + { + vulnerability_occurrence_id: finding.id, + data: evidence, + created_at: current_time, + updated_at: current_time + } + end + + def create_evidences(evidences) + Evidence.upsert_all(evidences, returning: false, unique_by: %i[vulnerability_occurrence_id]) + end + + def extract_evidence(metadata) + parsed_metadata = Gitlab::Json.parse(metadata) + + parsed_metadata['evidence'] + rescue JSON::ParserError + nil + end + + def logger + @logger ||= ::Gitlab::AppLogger + end + end + end +end diff --git a/lib/gitlab/background_migration/migrate_links_for_vulnerability_findings.rb b/lib/gitlab/background_migration/migrate_links_for_vulnerability_findings.rb new file mode 100644 index 00000000000..222ee4e524e --- /dev/null +++ b/lib/gitlab/background_migration/migrate_links_for_vulnerability_findings.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # The class to migrate the link data into their own records from the json attribute + class MigrateLinksForVulnerabilityFindings < BatchedMigrationJob + feature_category :vulnerability_management + operation_name :migrate_links_for_vulnerability_findings + + # The class is mimicking Vulnerabilites::Finding + class Finding < ApplicationRecord + self.table_name = 'vulnerability_occurrences' + + validates :details, json_schema: { filename: 'vulnerability_finding_details', draft: 7 }, if: false + end + + # The class is mimicking Vulnerabilites::FindingLink + class Link < ApplicationRecord + self.table_name = 'vulnerability_finding_links' + end + + def perform + each_sub_batch do |sub_batch| + migrate_remediations(sub_batch) + end + end + + private + + def migrate_remediations(sub_batch) + sub_batch.each do |finding| + links = extract_links(finding.raw_metadata) + + list_of_attrs = links.map do |link| + build_link(finding, link) + end + + next unless list_of_attrs.present? + + create_links(list_of_attrs) + rescue ActiveRecord::RecordNotUnique + rescue StandardError => e + logger.error( + message: e.message, + class: self.class.name, + model_id: finding.id + ) + end + end + + def build_link(finding, link) + current_time = Time.current + { + vulnerability_occurrence_id: finding.id, + name: link['name'], + url: link['url'], + created_at: current_time, + updated_at: current_time + } + end + + def create_links(attributes) + Link.upsert_all(attributes, returning: false) + end + + def extract_links(metadata) + parsed_metadata = Gitlab::Json.parse(metadata) + + return [] unless parsed_metadata['links'] + + parsed_metadata['links'].compact.uniq + end + + def logger + @logger ||= ::Gitlab::AppLogger + end + end + end +end diff --git a/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings.rb b/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings.rb new file mode 100644 index 00000000000..9eadef96db6 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings.rb @@ -0,0 +1,164 @@ +# frozen_string_literal: true + +module Vulnerabilities + # The class is mimicking Vulnerabilites::Remediation + class Remediation < ApplicationRecord + include FileStoreMounter + include ShaAttribute + + self.table_name = 'vulnerability_remediations' + + sha_attribute :checksum + + mount_file_store_uploader AttachmentUploader + + def retrieve_upload(_identifier, paths) + Upload.find_by(model: self, path: paths) + end + end +end + +module Gitlab + module BackgroundMigration + # The class to migrate the remediation data into their own records from the json attribute + class MigrateRemediationsForVulnerabilityFindings < BatchedMigrationJob + feature_category :vulnerability_management + operation_name :migrate_remediations_for_vulnerability_findings + + # The class to encapsulate checksum and file for uploading + class DiffFile < StringIO + # This method is used by the `carrierwave` gem + def original_filename + @original_filename ||= self.class.original_filename(checksum) + end + + def checksum + @checksum ||= self.class.checksum(string) + end + + def self.checksum(value) + Digest::SHA256.hexdigest(value) + end + + def self.original_filename(checksum) + "#{checksum}.diff" + end + end + + # The class is mimicking Vulnerabilites::Finding + class Finding < ApplicationRecord + self.table_name = 'vulnerability_occurrences' + + validates :details, json_schema: { filename: 'vulnerability_finding_details', draft: 7 }, if: false + end + + # The class is mimicking Vulnerabilites::FindingRemediation + class FindingRemediation < ApplicationRecord + self.table_name = 'vulnerability_findings_remediations' + end + + def perform + each_sub_batch do |sub_batch| + migrate_remediations(sub_batch) + end + end + + private + + def migrate_remediations(sub_batch) + sub_batch.each do |finding| + FindingRemediation.transaction do + remediations = append_remediations_diff_checksum(finding.raw_metadata) + + result_ids = create_remediations(finding, remediations) + + create_finding_remediations(finding.id, result_ids) + end + rescue StandardError => e + logger.error( + message: e.message, + class: self.class.name, + model_id: finding.id + ) + end + end + + def create_finding_remediations(finding_id, result_ids) + attrs = result_ids.map do |result_id| + build_finding_remediation_attrs(finding_id, result_id) + end + + return unless attrs.present? + + FindingRemediation.upsert_all( + attrs, + returning: false, + unique_by: [:vulnerability_occurrence_id, :vulnerability_remediation_id] + ) + end + + def create_remediations(finding, remediations) + attrs = remediations.map do |remediation| + build_remediation_attrs(finding, remediation) + end + + return [] unless attrs.present? + + ids_checksums = ::Vulnerabilities::Remediation.upsert_all( + attrs, + returning: %w[id checksum], + unique_by: [:project_id, :checksum] + ) + + ids_checksums.each do |id_checksum| + upload_file(id_checksum['id'], id_checksum['checksum'], remediations) + end + + ids_checksums.pluck('id') + end + + def upload_file(id, checksum, remediations) + deserialized_checksum = Gitlab::Database::ShaAttribute.new.deserialize(checksum) + diff = remediations.find { |rem| rem['checksum'] == deserialized_checksum }["diff"] + file = DiffFile.new(diff) + ::Vulnerabilities::Remediation.find_by(id: id).update!(file: file) + end + + def build_remediation_attrs(finding, remediation) + { + project_id: finding.project_id, + summary: remediation['summary'], + file: DiffFile.original_filename(remediation['checksum']), + checksum: remediation['checksum'], + created_at: Time.current, + updated_at: Time.current + } + end + + def build_finding_remediation_attrs(finding_id, remediation_id) + { + vulnerability_occurrence_id: finding_id, + vulnerability_remediation_id: remediation_id, + created_at: Time.current, + updated_at: Time.current + } + end + + def append_remediations_diff_checksum(metadata) + parsed_metadata = Gitlab::Json.parse(metadata) + + return [] unless parsed_metadata['remediations'] + + parsed_metadata['remediations'].filter_map do |remediation| + next unless remediation && remediation['diff'].present? + + remediation.merge('checksum' => DiffFile.checksum(remediation['diff'])) + end.compact.uniq + end + + def logger + @logger ||= ::Gitlab::AppLogger + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 3dafe7c8962..592e75b1430 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -72,7 +72,7 @@ module Gitlab return unless last_bitbucket_issue - Issue.track_project_iid!(project, last_bitbucket_issue.iid) + Issue.track_namespace_iid!(project.project_namespace, last_bitbucket_issue.iid) end def repo diff --git a/lib/gitlab/cache/client.rb b/lib/gitlab/cache/client.rb new file mode 100644 index 00000000000..ac710ee0adf --- /dev/null +++ b/lib/gitlab/cache/client.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module Gitlab + module Cache + # It replaces Rails.cache with metrics support + class Client + DEFAULT_BACKING_RESOURCE = :unknown + + # Build Cache client with the metadata support + # + # @param cache_identifier [String] defines the location of the cache definition + # Example: "ProtectedBranches::CacheService#fetch" + # @param feature_category [Symbol] name of the feature category (from config/feature_categories.yml) + # @param caller_id [String] caller id from labkit context + # @param backing_resource [Symbol] most affected resource by cache generation (full list: VALID_BACKING_RESOURCES) + # @return [Gitlab::Cache::Client] + def self.build_with_metadata( + cache_identifier:, + feature_category:, + caller_id: Gitlab::ApplicationContext.current_context_attribute(:caller_id), + backing_resource: DEFAULT_BACKING_RESOURCE + ) + new(Metadata.new( + caller_id: caller_id, + cache_identifier: cache_identifier, + feature_category: feature_category, + backing_resource: backing_resource + )) + end + + def initialize(metadata, backend: Rails.cache) + @metadata = metadata + @metrics = Metrics.new(metadata) + @backend = backend + end + + def read(name) + read_result = backend.read(name) + + if read_result.nil? + metrics.increment_cache_miss + else + metrics.increment_cache_hit + end + + read_result + end + + def fetch(name, options = nil, &block) + read_result = read(name) + + return read_result unless block || read_result + + backend.fetch(name, options) do + metrics.observe_cache_generation(&block) + end + end + + delegate :write, :exist?, :delete, to: :backend + + attr_reader :metadata, :metrics + + private + + attr_reader :backend + end + end +end diff --git a/lib/gitlab/cache/metadata.rb b/lib/gitlab/cache/metadata.rb index d6c89b5b2c3..224f215ef82 100644 --- a/lib/gitlab/cache/metadata.rb +++ b/lib/gitlab/cache/metadata.rb @@ -5,13 +5,18 @@ module Gitlab # Value object for cache metadata class Metadata VALID_BACKING_RESOURCES = [:cpu, :database, :gitaly, :memory, :unknown].freeze - DEFAULT_BACKING_RESOURCE = :unknown + # @param cache_identifier [String] defines the location of the cache definition + # Example: "ProtectedBranches::CacheService#fetch" + # @param feature_category [Symbol] name of the feature category (from config/feature_categories.yml) + # @param caller_id [String] caller id from labkit context + # @param backing_resource [Symbol] most affected resource by cache generation (full list: VALID_BACKING_RESOURCES) + # @return [Gitlab::Cache::Metadata] def initialize( cache_identifier:, feature_category:, caller_id: Gitlab::ApplicationContext.current_context_attribute(:caller_id), - backing_resource: DEFAULT_BACKING_RESOURCE + backing_resource: Client::DEFAULT_BACKING_RESOURCE ) @cache_identifier = cache_identifier @feature_category = Gitlab::FeatureCategories.default.get!(feature_category) @@ -28,7 +33,7 @@ module Gitlab raise "Unknown backing resource: #{resource}" if Gitlab.dev_or_test_env? - DEFAULT_BACKING_RESOURCE + Client::DEFAULT_BACKING_RESOURCE end end end diff --git a/lib/gitlab/changes_list.rb b/lib/gitlab/changes_list.rb index 999d2ee4356..ca55692ca2f 100644 --- a/lib/gitlab/changes_list.rb +++ b/lib/gitlab/changes_list.rb @@ -15,12 +15,12 @@ module Gitlab end def changes - @changes ||= @raw_changes.map do |change| + @changes ||= @raw_changes.filter_map do |change| next if change.blank? oldrev, newrev, ref = change.strip.split(' ') { oldrev: oldrev, newrev: newrev, ref: ref } - end.compact + end end end end diff --git a/lib/gitlab/chat/responder.rb b/lib/gitlab/chat/responder.rb index 478be5bd350..7ec64b7cfbf 100644 --- a/lib/gitlab/chat/responder.rb +++ b/lib/gitlab/chat/responder.rb @@ -11,21 +11,13 @@ module Gitlab # # build - A `Ci::Build` that executed a chat command. def self.responder_for(build) - if Feature.enabled?(:use_response_url_for_chat_responder) - response_url = build.pipeline.chat_data&.response_url - return unless response_url + response_url = build.pipeline.chat_data&.response_url + return unless response_url - if response_url.start_with?('https://hooks.slack.com/') - Gitlab::Chat::Responder::Slack.new(build) - else - Gitlab::Chat::Responder::Mattermost.new(build) - end + if response_url.start_with?('https://hooks.slack.com/') + Gitlab::Chat::Responder::Slack.new(build) else - integration = build.pipeline.chat_data&.chat_name&.integration - - if (responder = integration.try(:chat_responder)) - responder.new(build) - end + Gitlab::Chat::Responder::Mattermost.new(build) end end end diff --git a/lib/gitlab/checks/base_single_checker.rb b/lib/gitlab/checks/base_single_checker.rb index 435f4ccf5ba..755778efa60 100644 --- a/lib/gitlab/checks/base_single_checker.rb +++ b/lib/gitlab/checks/base_single_checker.rb @@ -5,7 +5,7 @@ module Gitlab class BaseSingleChecker < BaseChecker attr_reader :change_access - delegate(*SingleChangeAccess::ATTRIBUTES, to: :change_access) + delegate(*SingleChangeAccess::ATTRIBUTES, :branch_ref?, :tag_ref?, to: :change_access) def initialize(change_access) @change_access = change_access diff --git a/lib/gitlab/checks/changes_access.rb b/lib/gitlab/checks/changes_access.rb index 99752dc6a01..194e3f6e938 100644 --- a/lib/gitlab/checks/changes_access.rb +++ b/lib/gitlab/checks/changes_access.rb @@ -36,13 +36,13 @@ module Gitlab # any of the new revisions. def commits strong_memoize(:commits) do - newrevs = @changes.map do |change| + newrevs = @changes.filter_map do |change| newrev = change[:newrev] next if blank_rev?(newrev) newrev - end.compact + end next [] if newrevs.empty? @@ -89,7 +89,7 @@ module Gitlab @single_changes_accesses ||= changes.map do |change| commits = - if blank_rev?(change[:newrev]) + if !commitish_ref?(change[:ref]) || blank_rev?(change[:newrev]) [] else Gitlab::Lazy.new { commits_for(change[:oldrev], change[:newrev]) } @@ -122,6 +122,14 @@ module Gitlab def blank_rev?(rev) rev.blank? || Gitlab::Git.blank_ref?(rev) end + + # refs/notes/commits contains commits added via `git-notes`. We currently + # have no features that check notes so we can skip them. To future-proof + # we are skipping anything that isn't a branch or tag ref as those are + # the only refs that can contain commits. + def commitish_ref?(ref) + Gitlab::Git.branch_ref?(ref) || Gitlab::Git.tag_ref?(ref) + end end end end diff --git a/lib/gitlab/checks/diff_check.rb b/lib/gitlab/checks/diff_check.rb index d8f5cec8a4a..083c2448a0a 100644 --- a/lib/gitlab/checks/diff_check.rb +++ b/lib/gitlab/checks/diff_check.rb @@ -10,6 +10,10 @@ module Gitlab }.freeze def validate! + # git-notes stores notes history as commits in refs/notes/commits (by + # default but is configurable) so we restrict the diff checks to tag + # and branch refs + return unless tag_ref? || branch_ref? return if deletion? return unless should_run_validations? return if commits.empty? diff --git a/lib/gitlab/checks/single_change_access.rb b/lib/gitlab/checks/single_change_access.rb index 2fd48dfbfe2..9f427e98e55 100644 --- a/lib/gitlab/checks/single_change_access.rb +++ b/lib/gitlab/checks/single_change_access.rb @@ -14,7 +14,9 @@ module Gitlab protocol:, logger:, commits: nil ) @oldrev, @newrev, @ref = change.values_at(:oldrev, :newrev, :ref) + @branch_ref = Gitlab::Git.branch_ref?(@ref) @branch_name = Gitlab::Git.branch_name(@ref) + @tag_ref = Gitlab::Git.tag_ref?(@ref) @tag_name = Gitlab::Git.tag_name(@ref) @user_access = user_access @project = project @@ -38,6 +40,14 @@ module Gitlab @commits ||= project.repository.new_commits(newrev) end + def branch_ref? + @branch_ref + end + + def tag_ref? + @tag_ref + end + protected def ref_level_checks diff --git a/lib/gitlab/ci/badge/release/latest_release.rb b/lib/gitlab/ci/badge/release/latest_release.rb index e73bb2a912a..8d84a54787b 100644 --- a/lib/gitlab/ci/badge/release/latest_release.rb +++ b/lib/gitlab/ci/badge/release/latest_release.rb @@ -10,7 +10,8 @@ module Gitlab::Ci @project = project @customization = { key_width: opts[:key_width] ? opts[:key_width].to_i : nil, - key_text: opts[:key_text] + key_text: opts[:key_text], + value_width: opts[:value_width] ? opts[:value_width].to_i : nil } # In the future, we should support `order_by=semver` for showing the diff --git a/lib/gitlab/ci/badge/release/template.rb b/lib/gitlab/ci/badge/release/template.rb index 354be6276fa..549742226a1 100644 --- a/lib/gitlab/ci/badge/release/template.rb +++ b/lib/gitlab/ci/badge/release/template.rb @@ -11,9 +11,11 @@ module Gitlab::Ci }.freeze KEY_WIDTH_DEFAULT = 90 VALUE_WIDTH_DEFAULT = 54 + VALUE_WIDTH_MAXIMUM = 200 def initialize(badge) @tag = badge.tag || "none" + @value_width = badge.customization[:value_width] super end @@ -30,7 +32,11 @@ module Gitlab::Ci end def value_width - VALUE_WIDTH_DEFAULT + if @value_width && @value_width.between?(1, VALUE_WIDTH_MAXIMUM) + @value_width + else + VALUE_WIDTH_DEFAULT + end end def value_color diff --git a/lib/gitlab/ci/components/header.rb b/lib/gitlab/ci/components/header.rb new file mode 100644 index 00000000000..732874d7a88 --- /dev/null +++ b/lib/gitlab/ci/components/header.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Components + ## + # Components::Header class represents full component specification that is being prepended as first YAML document + # in the CI Component file. + # + class Header + attr_reader :errors + + def initialize(header) + @header = header + @errors = [] + end + + def empty? + inputs_spec.to_h.empty? + end + + def inputs(args) + @input ||= Ci::Input::Inputs.new(inputs_spec, args) + end + + def context(args) + inputs(args).then do |input| + raise ArgumentError unless input.valid? + + Ci::Interpolation::Context.new({ inputs: input.to_hash }) + end + end + + private + + def inputs_spec + @header.dig(:spec, :inputs) + end + end + end + end +end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 585e671ce42..534b84afc23 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -21,14 +21,15 @@ module Gitlab attr_reader :root, :context, :source_ref_path, :source, :logger - def initialize(config, project: nil, pipeline: nil, sha: nil, user: nil, parent_pipeline: nil, source: nil, logger: nil) + # rubocop: disable Metrics/ParameterLists + def initialize(config, project: nil, pipeline: nil, sha: nil, user: nil, parent_pipeline: nil, source: nil, pipeline_config: nil, logger: nil) @logger = logger || ::Gitlab::Ci::Pipeline::Logger.new(project: project) @source_ref_path = pipeline&.source_ref_path @project = project @context = self.logger.instrument(:config_build_context, once: true) do pipeline ||= ::Ci::Pipeline.new(project: project, sha: sha, user: user, source: source) - build_context(project: project, pipeline: pipeline, sha: sha, user: user, parent_pipeline: parent_pipeline) + build_context(project: project, pipeline: pipeline, sha: sha, user: user, parent_pipeline: parent_pipeline, pipeline_config: pipeline_config) end @context.set_deadline(TIMEOUT_SECONDS) @@ -49,6 +50,7 @@ module Gitlab rescue *rescue_errors => e raise Config::ConfigError, e.message end + # rubocop: enable Metrics/ParameterLists def valid? @root.valid? @@ -117,8 +119,7 @@ module Gitlab def expand_config(config) build_config(config) - rescue Gitlab::Config::Loader::Yaml::DataTooLargeError, - Gitlab::Config::Loader::MultiDocYaml::DataTooLargeError => e + rescue Gitlab::Config::Loader::Yaml::DataTooLargeError => e track_and_raise_for_dev_exception(e) raise Config::ConfigError, e.message @@ -157,13 +158,14 @@ module Gitlab end end - def build_context(project:, pipeline:, sha:, user:, parent_pipeline:) + def build_context(project:, pipeline:, sha:, user:, parent_pipeline:, pipeline_config:) Config::External::Context.new( project: project, sha: sha || find_sha(project), user: user, parent_pipeline: parent_pipeline, variables: build_variables(pipeline: pipeline), + pipeline_config: pipeline_config, logger: logger) end diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 7c49b59a7f0..2390ba05916 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -164,7 +164,7 @@ module Gitlab artifacts: artifacts_value, release: release_value, after_script: after_script_value, - hooks: hooks_pre_get_sources_script_enabled? ? hooks_value : nil, + hooks: hooks_value, ignore: ignored?, allow_failure_criteria: allow_failure_criteria, needs: needs_defined? ? needs_value : nil, @@ -194,10 +194,6 @@ module Gitlab allow_failure_value end - - def hooks_pre_get_sources_script_enabled? - YamlProcessor::FeatureFlags.enabled?(:ci_hooks_pre_get_sources_script) - end end end end diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb index 6eef279d3de..156109a084d 100644 --- a/lib/gitlab/ci/config/external/context.rb +++ b/lib/gitlab/ci/config/external/context.rb @@ -9,29 +9,30 @@ module Gitlab TimeoutError = Class.new(StandardError) - MAX_INCLUDES = 100 - NEW_MAX_INCLUDES = 150 # Update to MAX_INCLUDES when FF ci_includes_count_duplicates is removed + MAX_INCLUDES = 150 + TEMP_MAX_INCLUDES = 100 # For logging; to be removed in https://gitlab.com/gitlab-org/gitlab/-/issues/367150 include ::Gitlab::Utils::StrongMemoize - attr_reader :project, :sha, :user, :parent_pipeline, :variables + attr_reader :project, :sha, :user, :parent_pipeline, :variables, :pipeline_config attr_reader :expandset, :execution_deadline, :logger, :max_includes delegate :instrument, to: :logger def initialize( project: nil, sha: nil, user: nil, parent_pipeline: nil, variables: nil, - logger: nil + pipeline_config: nil, logger: nil ) @project = project @sha = sha @user = user @parent_pipeline = parent_pipeline @variables = variables || Ci::Variables::Collection.new - @expandset = Feature.enabled?(:ci_includes_count_duplicates, project) ? [] : Set.new + @pipeline_config = pipeline_config + @expandset = [] @execution_deadline = 0 @logger = logger || Gitlab::Ci::Pipeline::Logger.new(project: project) - @max_includes = Feature.enabled?(:ci_includes_count_duplicates, project) ? NEW_MAX_INCLUDES : MAX_INCLUDES + @max_includes = MAX_INCLUDES yield self if block_given? end @@ -91,6 +92,13 @@ module Gitlab expandset.map(&:metadata) end + # Some Ci::ProjectConfig sources prepend the config content with an "internal" `include`, which becomes + # the first included file. When running a pipeline, we pass pipeline_config into the context of the first + # included file, which we use in this method to determine if the file is an "internal" one. + def internal_include? + !!pipeline_config&.internal_include_prepended? + end + protected attr_writer :expandset, :execution_deadline, :logger, :max_includes diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb index 84f34f2584b..7060754a670 100644 --- a/lib/gitlab/ci/config/external/file/base.rb +++ b/lib/gitlab/ci/config/external/file/base.rb @@ -73,6 +73,18 @@ module Gitlab validate_hash! end + # This method is overridden to load context into the memoized result + # or to lazily load context via BatchLoader + def preload_context + # no-op + end + + def preload_content + # calling the `content` method either loads content into the memoized result + # or lazily loads it via BatchLoader + content + end + def validate_location! if invalid_location_type? errors.push("Included file `#{masked_location}` needs to be a string") @@ -93,6 +105,19 @@ module Gitlab protected + def content_result + strong_memoize(:content_hash) do + ::Gitlab::Ci::Config::Yaml + .load_result!(content, project: context.project) + end + end + + def content_hash + return unless content_result.valid? + + content_result.content + end + def expanded_content_hash return unless content_hash @@ -101,14 +126,6 @@ module Gitlab end end - def content_hash - strong_memoize(:content_hash) do - ::Gitlab::Ci::Config::Yaml.load!(content) - end - rescue Gitlab::Config::Loader::FormatError - nil - end - def validate_hash! if to_hash.blank? errors.push("Included file `#{masked_location}` does not have valid YAML syntax!") diff --git a/lib/gitlab/ci/config/external/file/component.rb b/lib/gitlab/ci/config/external/file/component.rb index 33e7724bf9b..7ab7dc3d64e 100644 --- a/lib/gitlab/ci/config/external/file/component.rb +++ b/lib/gitlab/ci/config/external/file/component.rb @@ -15,7 +15,7 @@ module Gitlab end def matching? - super && ::Feature.enabled?(:ci_include_components, context.project) + super && ::Feature.enabled?(:ci_include_components, context.project&.root_namespace) end def content diff --git a/lib/gitlab/ci/config/external/file/project.rb b/lib/gitlab/ci/config/external/file/project.rb index f8d4cb27710..5efefeeaf9d 100644 --- a/lib/gitlab/ci/config/external/file/project.rb +++ b/lib/gitlab/ci/config/external/file/project.rb @@ -15,7 +15,8 @@ module Gitlab # `Repository#blobs_at` does not support files with the `/` prefix. @location = Gitlab::Utils.remove_leading_slashes(params[:file]) - @project_name = get_project_name(params[:project]) + # We are using the same downcase in the `project` method. + @project_name = get_project_name(params[:project]).to_s.downcase @ref_name = params[:ref] || 'HEAD' super @@ -39,6 +40,15 @@ module Gitlab ) end + def preload_context + # + # calling these methods lazily loads them via BatchLoader + # + project + can_access_local_content? + sha + end + def validate_context! if !can_access_local_content? errors.push("Project `#{masked_project_name}` not found or access denied! Make sure any includes in the pipeline configuration are correctly defined.") @@ -58,21 +68,48 @@ module Gitlab private def project - strong_memoize(:project) do - ::Project.find_by_full_path(project_name) + return legacy_project if ::Feature.disabled?(:ci_batch_project_includes_context, context.project) + + # Although we use `where_full_path_in`, this BatchLoader does not reduce the number of queries to 1. + # That's because we use it in the `can_access_local_content?` and `sha` BatchLoaders + # as the `for` parameter. And this loads the project immediately. + BatchLoader.for(project_name) + .batch do |project_names, loader| + ::Project.where_full_path_in(project_names.uniq).each do |project| + # We are using the same downcase in the `initialize` method. + loader.call(project.full_path.downcase, project) + end end end def can_access_local_content? - strong_memoize(:can_access_local_content) do - context.logger.instrument(:config_file_project_validate_access) do - Ability.allowed?(context.user, :download_code, project) + if ::Feature.disabled?(:ci_batch_project_includes_context, context.project) + return legacy_can_access_local_content? + end + + BatchLoader.for(project) + .batch(key: context.user) do |projects, loader, args| + projects.uniq.each do |project| + context.logger.instrument(:config_file_project_validate_access) do + loader.call(project, Ability.allowed?(args[:key], :download_code, project)) + end + end + end + end + + def sha + return legacy_sha if ::Feature.disabled?(:ci_batch_project_includes_context, context.project) + + BatchLoader.for([project, ref_name]) + .batch do |project_ref_pairs, loader| + project_ref_pairs.uniq.each do |project, ref_name| + loader.call([project, ref_name], project.commit(ref_name).try(:sha)) end end end def fetch_local_content - BatchLoader.for([sha, location]) + BatchLoader.for([sha.to_s, location]) .batch(key: project) do |locations, loader, args| context.logger.instrument(:config_file_fetch_project_content) do args[:key].repository.blobs_at(locations).each do |blob| @@ -84,8 +121,22 @@ module Gitlab end end - def sha - strong_memoize(:sha) do + def legacy_project + strong_memoize(:legacy_project) do + ::Project.find_by_full_path(project_name) + end + end + + def legacy_can_access_local_content? + strong_memoize(:legacy_can_access_local_content) do + context.logger.instrument(:config_file_project_validate_access) do + Ability.allowed?(context.user, :download_code, project) + end + end + end + + def legacy_sha + strong_memoize(:legacy_sha) do project.commit(ref_name).try(:sha) end end @@ -94,7 +145,7 @@ module Gitlab def expand_context_attrs { project: project, - sha: sha, + sha: sha.to_s, # we need to use `.to_s` to load the value from the BatchLoader user: context.user, parent_pipeline: context.parent_pipeline, variables: context.variables diff --git a/lib/gitlab/ci/config/external/mapper/verifier.rb b/lib/gitlab/ci/config/external/mapper/verifier.rb index 2982b0efb6c..7284d2a7e01 100644 --- a/lib/gitlab/ci/config/external/mapper/verifier.rb +++ b/lib/gitlab/ci/config/external/mapper/verifier.rb @@ -9,8 +9,56 @@ module Gitlab class Verifier < Base private + # rubocop: disable Metrics/CyclomaticComplexity def process_without_instrumentation(files) + if ::Feature.disabled?(:ci_batch_project_includes_context, context.project) + return legacy_process_without_instrumentation(files) + end + files.each do |file| + if YamlProcessor::FeatureFlags.enabled?(:ci_fix_max_includes) + # When running a pipeline, some Ci::ProjectConfig sources prepend the config content with an + # "internal" `include`. We use this condition to exclude that `include` from the included file set. + context.expandset << file unless context.internal_include? + verify_max_includes! + end + + verify_execution_time! + + file.validate_location! + file.preload_context if file.valid? + end + + # We do not combine the loops because we need to load the context of all files via `BatchLoader`. + files.each do |file| # rubocop:disable Style/CombinableLoops + verify_execution_time! + + file.validate_context! if file.valid? + file.preload_content if file.valid? + end + + # We do not combine the loops because we need to load the content of all files via `BatchLoader`. + files.each do |file| # rubocop:disable Style/CombinableLoops + verify_max_includes! unless YamlProcessor::FeatureFlags.enabled?(:ci_fix_max_includes) + verify_execution_time! + + file.validate_content! if file.valid? + file.load_and_validate_expanded_hash! if file.valid? + + context.expandset << file unless YamlProcessor::FeatureFlags.enabled?(:ci_fix_max_includes) + end + end + # rubocop: enable Metrics/CyclomaticComplexity + + def legacy_process_without_instrumentation(files) + files.each do |file| + if YamlProcessor::FeatureFlags.enabled?(:ci_fix_max_includes) + # When running a pipeline, some Ci::ProjectConfig sources prepend the config content with an + # "internal" `include`. We use this condition to exclude that `include` from the included file set. + context.expandset << file unless context.internal_include? + verify_max_includes! + end + verify_execution_time! file.validate_location! @@ -21,23 +69,22 @@ module Gitlab # We do not combine the loops because we need to load the content of all files before continuing # to call `BatchLoader` for all locations. files.each do |file| # rubocop:disable Style/CombinableLoops - # Checking the max includes will be changed with https://gitlab.com/gitlab-org/gitlab/-/issues/367150 - verify_max_includes! + verify_max_includes! unless YamlProcessor::FeatureFlags.enabled?(:ci_fix_max_includes) verify_execution_time! file.validate_content! if file.valid? file.load_and_validate_expanded_hash! if file.valid? - if context.expandset.is_a?(Array) # To be removed when FF 'ci_includes_count_duplicates' is removed - context.expandset << file - else - context.expandset.add(file) - end + context.expandset << file unless YamlProcessor::FeatureFlags.enabled?(:ci_fix_max_includes) end end def verify_max_includes! - return if context.expandset.count < context.max_includes + if YamlProcessor::FeatureFlags.enabled?(:ci_fix_max_includes) + return if context.expandset.count <= context.max_includes + else + return if context.expandset.count < context.max_includes # rubocop:disable Style/IfInsideElse + end raise Mapper::TooManyIncludesError, "Maximum of #{context.max_includes} nested includes are allowed!" end diff --git a/lib/gitlab/ci/config/header/input.rb b/lib/gitlab/ci/config/header/input.rb new file mode 100644 index 00000000000..525b009afe3 --- /dev/null +++ b/lib/gitlab/ci/config/header/input.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Header + ## + # Input parameter used for interpolation with the CI configuration. + class Input < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + attributes :default, prefix: :input + + validations do + validates :config, type: Hash, allowed_keys: [:default] + validates :key, alphanumeric: true + validates :input_default, alphanumeric: true, allow_nil: true + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/header/root.rb b/lib/gitlab/ci/config/header/root.rb new file mode 100644 index 00000000000..251682d13b4 --- /dev/null +++ b/lib/gitlab/ci/config/header/root.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Header + ## + # This class represents the root entry of the GitLab CI configuration header. + # + # A header is the first document in a multi-doc YAML that contains metadata + # and specifications about the GitLab CI configuration (the second document). + # + # The header is optional. A CI configuration can also be represented with a + # YAML containing a single document. + class Root < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable + + ALLOWED_KEYS = %i[spec].freeze + + validations do + validates :config, type: Hash, allowed_keys: ALLOWED_KEYS + end + + entry :spec, Header::Spec, + description: 'Specifications of the CI configuration.', + inherit: false, + default: {} + + def inputs_value + spec_entry.inputs_value + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/header/spec.rb b/lib/gitlab/ci/config/header/spec.rb new file mode 100644 index 00000000000..98d6d0d5783 --- /dev/null +++ b/lib/gitlab/ci/config/header/spec.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Header + class Spec < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable + + ALLOWED_KEYS = %i[inputs].freeze + + validations do + validates :config, type: Hash, allowed_keys: ALLOWED_KEYS + end + + entry :inputs, ::Gitlab::Config::Entry::ComposableHash, + description: 'Allowed input parameters used for interpolation.', + inherit: false, + metadata: { composable_class: ::Gitlab::Ci::Config::Header::Input } + end + end + end + end +end diff --git a/lib/gitlab/ci/config/yaml.rb b/lib/gitlab/ci/config/yaml.rb index 94ef0afe7f9..d1b1b8caa5c 100644 --- a/lib/gitlab/ci/config/yaml.rb +++ b/lib/gitlab/ci/config/yaml.rb @@ -7,23 +7,38 @@ module Gitlab AVAILABLE_TAGS = [Config::Yaml::Tags::Reference].freeze MAX_DOCUMENTS = 2 - class << self - def load!(content) + class Loader + def initialize(content, project: nil) + @content = content + @project = project + end + + def load! ensure_custom_tags - if ::Feature.enabled?(:ci_multi_doc_yaml) - Gitlab::Config::Loader::MultiDocYaml.new( + if project.present? && ::Feature.enabled?(:ci_multi_doc_yaml, project) + ::Gitlab::Config::Loader::MultiDocYaml.new( content, max_documents: MAX_DOCUMENTS, additional_permitted_classes: AVAILABLE_TAGS - ).load!.first + ).load! else - Gitlab::Config::Loader::Yaml.new(content, additional_permitted_classes: AVAILABLE_TAGS).load! + ::Gitlab::Config::Loader::Yaml + .new(content, additional_permitted_classes: AVAILABLE_TAGS) + .load! end end + def to_result + Yaml::Result.new(config: load!, error: nil) + rescue ::Gitlab::Config::Loader::FormatError => e + Yaml::Result.new(error: e) + end + private + attr_reader :content, :project + def ensure_custom_tags @ensure_custom_tags ||= begin AVAILABLE_TAGS.each { |klass| Psych.add_tag(klass.tag, klass) } @@ -32,6 +47,23 @@ module Gitlab end end end + + class << self + def load!(content, project: nil) + Loader.new(content, project: project).to_result.then do |result| + ## + # raise an error for backwards compatibility + # + raise result.error unless result.valid? + + result.content + end + end + + def load_result!(content, project: nil) + Loader.new(content, project: project).to_result + end + end end end end diff --git a/lib/gitlab/ci/config/yaml/result.rb b/lib/gitlab/ci/config/yaml/result.rb new file mode 100644 index 00000000000..1a3ca53c161 --- /dev/null +++ b/lib/gitlab/ci/config/yaml/result.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Yaml + class Result + attr_reader :error + + def initialize(config: nil, error: nil) + @config = Array.wrap(config) + @error = error + end + + def valid? + error.nil? + end + + def has_header? + @config.size > 1 + end + + def header + raise ArgumentError unless has_header? + + @config.first + end + + def content + @config.last + end + end + end + end + end +end diff --git a/lib/gitlab/ci/input/arguments/base.rb b/lib/gitlab/ci/input/arguments/base.rb new file mode 100644 index 00000000000..a46037c40ce --- /dev/null +++ b/lib/gitlab/ci/input/arguments/base.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Input + module Arguments + ## + # Input::Arguments::Base is a common abstraction for input arguments: + # - required + # - optional + # - with a default value + # + class Base + attr_reader :key, :value, :spec, :errors + + ArgumentNotValidError = Class.new(StandardError) + + def initialize(key, spec, value) + @key = key # hash key / argument name + @value = value # user-provided value + @spec = spec # configured specification + @errors = [] + + unless value.is_a?(String) || value.nil? # rubocop:disable Style/IfUnlessModifier + @errors.push("unsupported value in input argument `#{key}`") + end + + validate! + end + + def valid? + @errors.none? + end + + def validate! + raise NotImplementedError + end + + def to_value + raise NotImplementedError + end + + def to_hash + raise ArgumentNotValidError unless valid? + + @output ||= { key => to_value } + end + + def self.matches?(spec) + raise NotImplementedError + end + + private + + def error(message) + @errors.push("`#{@key}` input: #{message}") + end + end + end + end + end +end diff --git a/lib/gitlab/ci/input/arguments/default.rb b/lib/gitlab/ci/input/arguments/default.rb new file mode 100644 index 00000000000..fd61c1ab786 --- /dev/null +++ b/lib/gitlab/ci/input/arguments/default.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Input + module Arguments + ## + # Input::Arguments::Default class represents user-provided input argument that has a default value. + # + class Default < Input::Arguments::Base + def validate! + error('invalid specification') unless default.present? + end + + ## + # User-provided value needs to be specified, but it may be an empty string: + # + # ```yaml + # inputs: + # env: + # default: development + # + # with: + # env: "" + # ``` + # + # The configuration above will result in `env` being an empty string. + # + def to_value + value.nil? ? default : value + end + + def default + spec[:default] + end + + def self.matches?(spec) + spec.count == 1 && spec.each_key.first == :default + end + end + end + end + end +end diff --git a/lib/gitlab/ci/input/arguments/options.rb b/lib/gitlab/ci/input/arguments/options.rb new file mode 100644 index 00000000000..debc89b10bd --- /dev/null +++ b/lib/gitlab/ci/input/arguments/options.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Input + module Arguments + ## + # Input::Arguments::Options class represents user-provided input argument that is an enum, and is only valid + # when the value provided is listed as an acceptable one. + # + class Options < Input::Arguments::Base + ## + # An empty value is valid if it is allowlisted: + # + # ```yaml + # inputs: + # run: + # - "" + # - tests + # + # with: + # run: "" + # ``` + # + # The configuration above will return an empty value. + # + def validate! + return error('argument specification invalid') if options.to_a.empty? + + if !value.nil? + error("argument value #{value} not allowlisted") unless options.include?(value) + else + error('argument not provided') + end + end + + def to_value + value + end + + def options + spec[:options] + end + + def self.matches?(spec) + spec.count == 1 && spec.each_key.first == :options + end + end + end + end + end +end diff --git a/lib/gitlab/ci/input/arguments/required.rb b/lib/gitlab/ci/input/arguments/required.rb new file mode 100644 index 00000000000..b4e218ed29e --- /dev/null +++ b/lib/gitlab/ci/input/arguments/required.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Input + module Arguments + ## + # Input::Arguments::Required class represents user-provided required input argument. + # + class Required < Input::Arguments::Base + ## + # The value has to be defined, but it may be empty. + # + def validate! + error('required value has not been provided') if value.nil? + end + + def to_value + value + end + + ## + # Required arguments do not have nested configuration. It has to be defined a null value. + # + # ```yaml + # spec: + # inputs: + # website: + # ``` + # + # An empty value, that has no specification is also considered as a "required" input, however we should + # never see that being used, because it will be rejected by Ci::Config::Header validation. + # + # ```yaml + # spec: + # inputs: + # website: "" + # ``` + def self.matches?(spec) + spec.to_s.empty? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/input/arguments/unknown.rb b/lib/gitlab/ci/input/arguments/unknown.rb new file mode 100644 index 00000000000..5873e6e66a6 --- /dev/null +++ b/lib/gitlab/ci/input/arguments/unknown.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Input + module Arguments + ## + # Input::Arguments::Unknown object gets fabricated when we can't match an input argument entry with any known + # specification. It is matched as the last one, and always returns an error. + # + class Unknown < Input::Arguments::Base + def validate! + if spec.is_a?(Hash) && spec.count == 1 + error("unrecognized input argument specification: `#{spec.each_key.first}`") + else + error('unrecognized input argument definition') + end + end + + def to_value + raise ArgumentError, 'unknown argument value' + end + + def self.matches?(*) + true + end + end + end + end + end +end diff --git a/lib/gitlab/ci/input/inputs.rb b/lib/gitlab/ci/input/inputs.rb new file mode 100644 index 00000000000..743ae2ecf1e --- /dev/null +++ b/lib/gitlab/ci/input/inputs.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Input + ## + # Inputs::Input class represents user-provided inputs, configured using `with:` keyword. + # + # Input arguments are only valid with an associated component's inputs specification from component's header. + # + class Inputs + UnknownSpecArgumentError = Class.new(StandardError) + + ARGUMENTS = [ + Input::Arguments::Required, # Input argument is required + Input::Arguments::Default, # Input argument has a default value + Input::Arguments::Options, # Input argument that needs to be allowlisted + Input::Arguments::Unknown # Input argument has not been recognized + ].freeze + + def initialize(spec, args) + @spec = spec + @args = args + @inputs = [] + @errors = [] + + validate! + fabricate! + end + + def errors + @errors + @inputs.flat_map(&:errors) + end + + def valid? + errors.none? + end + + def unknown + @args.keys - @spec.keys + end + + def count + @inputs.count + end + + def to_hash + @inputs.inject({}) do |hash, argument| + raise ArgumentError unless argument.valid? + + hash.merge(argument.to_hash) + end + end + + private + + def validate! + @errors.push("unknown input arguments: #{unknown.inspect}") if unknown.any? + end + + def fabricate! + @spec.each do |key, spec| + argument = ARGUMENTS.find { |klass| klass.matches?(spec) } + + raise UnknownSpecArgumentError if argument.nil? + + @inputs.push(argument.new(key, spec, @args[key])) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers/security/common.rb b/lib/gitlab/ci/parsers/security/common.rb index 1b9afc92d6b..447136df81f 100644 --- a/lib/gitlab/ci/parsers/security/common.rb +++ b/lib/gitlab/ci/parsers/security/common.rb @@ -139,6 +139,7 @@ module Gitlab details: data['details'] || {}, signatures: signatures, project_id: @project.id, + found_by_pipeline: report.pipeline, vulnerability_finding_signatures_enabled: @signatures_enabled)) end diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index d2dc712e366..4bc2f6c7be7 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -13,7 +13,8 @@ module Gitlab :seeds_block, :variables_attributes, :push_options, :chat_data, :allow_mirror_update, :bridge, :content, :dry_run, :logger, # These attributes are set by Chains during processing: - :config_content, :yaml_processor_result, :workflow_rules_result, :pipeline_seed + :config_content, :yaml_processor_result, :workflow_rules_result, :pipeline_seed, + :pipeline_config ) do include Gitlab::Utils::StrongMemoize diff --git a/lib/gitlab/ci/pipeline/chain/config/content.rb b/lib/gitlab/ci/pipeline/chain/config/content.rb index d41213ef6dd..779aac7d520 100644 --- a/lib/gitlab/ci/pipeline/chain/config/content.rb +++ b/lib/gitlab/ci/pipeline/chain/config/content.rb @@ -14,6 +14,7 @@ module Gitlab @pipeline.build_pipeline_config(content: pipeline_config.content) @command.config_content = pipeline_config.content @pipeline.config_source = pipeline_config.source + @command.pipeline_config = pipeline_config else error('Missing CI config file') end diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb index ad6b2fd3411..4976e075727 100644 --- a/lib/gitlab/ci/pipeline/chain/config/process.rb +++ b/lib/gitlab/ci/pipeline/chain/config/process.rb @@ -20,6 +20,7 @@ module Gitlab source: @pipeline.source, user: current_user, parent_pipeline: parent_pipeline, + pipeline_config: @command.pipeline_config, logger: logger } ) diff --git a/lib/gitlab/ci/pipeline/duration.rb b/lib/gitlab/ci/pipeline/duration.rb index e8a991026b5..573d4c25b91 100644 --- a/lib/gitlab/ci/pipeline/duration.rb +++ b/lib/gitlab/ci/pipeline/duration.rb @@ -82,6 +82,8 @@ module Gitlab module Duration extend self + STATUSES = %w[success failed running canceled].freeze + Period = Struct.new(:first, :last) do def duration last - first @@ -90,14 +92,15 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def from_pipeline(pipeline) - status = %w[success failed running canceled] - builds = pipeline.processables.latest - .where(status: status).where.not(started_at: nil).order(:started_at) + builds = + self_and_downstreams_builds_of_pipeline(pipeline) from_builds(builds) end # rubocop: enable CodeReuse/ActiveRecord + private + def from_builds(builds) now = Time.now @@ -113,8 +116,6 @@ module Gitlab process_duration(process_periods(periods)) end - private - def process_periods(periods) return periods if periods.empty? @@ -139,6 +140,20 @@ module Gitlab end # rubocop: disable CodeReuse/ActiveRecord + def self_and_downstreams_builds_of_pipeline(pipeline) + ::Ci::Build + .select(:id, :type, :started_at, :finished_at) + .in_pipelines( + pipeline.self_and_downstreams.select(:id) + ) + .with_status(STATUSES) + .latest + .where.not(started_at: nil) + .order(:started_at) + end + # rubocop: enable CodeReuse/ActiveRecord + + # rubocop: disable CodeReuse/ActiveRecord def process_duration(periods) periods.sum(&:duration) end diff --git a/lib/gitlab/ci/project_config.rb b/lib/gitlab/ci/project_config.rb index ded6877ef29..00b2ad58428 100644 --- a/lib/gitlab/ci/project_config.rb +++ b/lib/gitlab/ci/project_config.rb @@ -26,6 +26,7 @@ module Gitlab end delegate :content, :source, to: :@config, allow_nil: true + delegate :internal_include_prepended?, to: :@config def exists? !!@config&.exists? diff --git a/lib/gitlab/ci/project_config/auto_devops.rb b/lib/gitlab/ci/project_config/auto_devops.rb index c6905f480a2..c5f010ebaea 100644 --- a/lib/gitlab/ci/project_config/auto_devops.rb +++ b/lib/gitlab/ci/project_config/auto_devops.rb @@ -13,6 +13,10 @@ module Gitlab end end + def internal_include_prepended? + true + end + def source :auto_devops_source end diff --git a/lib/gitlab/ci/project_config/external_project.rb b/lib/gitlab/ci/project_config/external_project.rb index 0ed5d6fa226..0afdab23886 100644 --- a/lib/gitlab/ci/project_config/external_project.rb +++ b/lib/gitlab/ci/project_config/external_project.rb @@ -17,6 +17,10 @@ module Gitlab end end + def internal_include_prepended? + true + end + def source :external_project_source end diff --git a/lib/gitlab/ci/project_config/remote.rb b/lib/gitlab/ci/project_config/remote.rb index cf1292706d2..19cbf8e9c1e 100644 --- a/lib/gitlab/ci/project_config/remote.rb +++ b/lib/gitlab/ci/project_config/remote.rb @@ -12,6 +12,10 @@ module Gitlab end end + def internal_include_prepended? + true + end + def source :remote_source end diff --git a/lib/gitlab/ci/project_config/repository.rb b/lib/gitlab/ci/project_config/repository.rb index 435ad4d42fe..272425fd546 100644 --- a/lib/gitlab/ci/project_config/repository.rb +++ b/lib/gitlab/ci/project_config/repository.rb @@ -12,6 +12,10 @@ module Gitlab end end + def internal_include_prepended? + true + end + def source :repository_source end diff --git a/lib/gitlab/ci/project_config/source.rb b/lib/gitlab/ci/project_config/source.rb index ebe5728163b..9a4a6394fa1 100644 --- a/lib/gitlab/ci/project_config/source.rb +++ b/lib/gitlab/ci/project_config/source.rb @@ -24,6 +24,11 @@ module Gitlab raise NotImplementedError end + # Indicates if we are prepending the content with an "internal" `include` + def internal_include_prepended? + false + end + def source raise NotImplementedError end diff --git a/lib/gitlab/ci/reports/security/finding.rb b/lib/gitlab/ci/reports/security/finding.rb index 92a91854358..45e67528f12 100644 --- a/lib/gitlab/ci/reports/security/finding.rb +++ b/lib/gitlab/ci/reports/security/finding.rb @@ -29,12 +29,13 @@ module Gitlab attr_reader :signatures attr_reader :project_id attr_reader :original_data + attr_reader :found_by_pipeline delegate :file_path, :start_line, :end_line, to: :location alias_method :cve, :compare_key - def initialize(compare_key:, identifiers:, flags: [], links: [], remediations: [], location:, evidence:, metadata_version:, name:, original_data:, report_type:, scanner:, scan:, uuid:, confidence: nil, severity: nil, details: {}, signatures: [], project_id: nil, vulnerability_finding_signatures_enabled: false) # rubocop:disable Metrics/ParameterLists + def initialize(compare_key:, identifiers:, flags: [], links: [], remediations: [], location:, evidence:, metadata_version:, name:, original_data:, report_type:, scanner:, scan:, uuid:, confidence: nil, severity: nil, details: {}, signatures: [], project_id: nil, vulnerability_finding_signatures_enabled: false, found_by_pipeline: nil) # rubocop:disable Metrics/ParameterLists @compare_key = compare_key @confidence = confidence @identifiers = identifiers @@ -55,6 +56,7 @@ module Gitlab @signatures = signatures @project_id = project_id @vulnerability_finding_signatures_enabled = vulnerability_finding_signatures_enabled + @found_by_pipeline = found_by_pipeline @project_fingerprint = generate_project_fingerprint end diff --git a/lib/gitlab/ci/reports/security/report.rb b/lib/gitlab/ci/reports/security/report.rb index 54b21da5436..2287c397c2b 100644 --- a/lib/gitlab/ci/reports/security/report.rb +++ b/lib/gitlab/ci/reports/security/report.rb @@ -5,8 +5,9 @@ module Gitlab module Reports module Security class Report - attr_reader :created_at, :type, :pipeline, :findings, :scanners, :identifiers - attr_accessor :scan, :scanned_resources, :errors, :analyzer, :version, :schema_validation_status, :warnings + attr_reader :created_at, :type, :findings, :scanners, :identifiers + attr_accessor :scan, :pipeline, :scanned_resources, :errors, + :analyzer, :version, :schema_validation_status, :warnings delegate :project_id, to: :pipeline delegate :project, to: :pipeline diff --git a/lib/gitlab/ci/reports/security/vulnerability_reports_comparer.rb b/lib/gitlab/ci/reports/security/vulnerability_reports_comparer.rb deleted file mode 100644 index 4be4cf62e7b..00000000000 --- a/lib/gitlab/ci/reports/security/vulnerability_reports_comparer.rb +++ /dev/null @@ -1,165 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Reports - module Security - class VulnerabilityReportsComparer - include Gitlab::Utils::StrongMemoize - - attr_reader :base_report, :head_report - - ACCEPTABLE_REPORT_AGE = 1.week - - def initialize(project, base_report, head_report) - @base_report = base_report - @head_report = head_report - - @signatures_enabled = project.licensed_feature_available?(:vulnerability_finding_signatures) - - if @signatures_enabled - @added_findings = [] - @fixed_findings = [] - calculate_changes - end - end - - def base_report_created_at - @base_report.created_at - end - - def head_report_created_at - @head_report.created_at - end - - def base_report_out_of_date - return false unless @base_report.created_at - - ACCEPTABLE_REPORT_AGE.ago > @base_report.created_at - end - - def added - strong_memoize(:added) do - if @signatures_enabled - @added_findings - else - head_report.findings - base_report.findings - end - end - end - - def fixed - strong_memoize(:fixed) do - if @signatures_enabled - @fixed_findings - else - base_report.findings - head_report.findings - end - end - end - - private - - def calculate_changes - # This is a deconstructed version of the eql? method on - # Ci::Reports::Security::Finding. It: - # - # * precomputes for the head_findings (using FindingMatcher): - # * sets of signature shas grouped by priority - # * mappings of signature shas to the head finding object - # - # These are then used when iterating the base findings to perform - # fast(er) prioritized, signature-based comparisons between each base finding - # and the head findings. - # - # Both the head_findings and base_findings arrays are iterated once - - base_findings = base_report.findings - head_findings = head_report.findings - - matcher = FindingMatcher.new(head_findings) - - base_findings.each do |base_finding| - next if base_finding.requires_manual_resolution? - - matched_head_finding = matcher.find_and_remove_match!(base_finding) - - @fixed_findings << base_finding if matched_head_finding.nil? - end - - @added_findings = matcher.unmatched_head_findings.values - end - end - - class FindingMatcher - attr_reader :unmatched_head_findings, :head_findings - - include Gitlab::Utils::StrongMemoize - - def initialize(head_findings) - @head_findings = head_findings - @unmatched_head_findings = @head_findings.index_by(&:object_id) - end - - def find_and_remove_match!(base_finding) - matched_head_finding = find_matched_head_finding_for(base_finding) - - # no signatures matched, so check the normal uuids of the base and head findings - # for a match - matched_head_finding = head_signatures_shas[base_finding.uuid] if matched_head_finding.nil? - - @unmatched_head_findings.delete(matched_head_finding.object_id) unless matched_head_finding.nil? - - matched_head_finding - end - - private - - def find_matched_head_finding_for(base_finding) - base_signature = sorted_signatures_for(base_finding).find do |signature| - # at this point a head_finding exists that has a signature with a - # matching priority, and a matching sha --> lookup the actual finding - # object from head_signatures_shas - head_signatures_shas[signature.signature_sha].eql?(base_finding) - end - - base_signature.present? ? head_signatures_shas[base_signature.signature_sha] : nil - end - - def sorted_signatures_for(base_finding) - base_finding.signatures.select { |signature| head_finding_signature?(signature) } - .sort_by { |sig| -sig.priority } - end - - def head_finding_signature?(signature) - head_signatures_priorities[signature.priority].include?(signature.signature_sha) - end - - def head_signatures_priorities - strong_memoize(:head_signatures_priorities) do - signatures_priorities = Hash.new { |hash, key| hash[key] = Set.new } - - head_findings.each_with_object(signatures_priorities) do |head_finding, memo| - head_finding.signatures.each do |signature| - memo[signature.priority].add(signature.signature_sha) - end - end - end - end - - def head_signatures_shas - strong_memoize(:head_signatures_shas) do - head_findings.each_with_object({}) do |head_finding, memo| - head_finding.signatures.each do |signature| - memo[signature.signature_sha] = head_finding - end - # for the final uuid check when no signatures have matched - memo[head_finding.uuid] = head_finding - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/resource_groups/logger.rb b/lib/gitlab/ci/resource_groups/logger.rb new file mode 100644 index 00000000000..9c93ee95bc7 --- /dev/null +++ b/lib/gitlab/ci/resource_groups/logger.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module ResourceGroups + class Logger < ::Gitlab::JsonLogger + def self.file_name_noext + 'ci_resource_groups_json' + end + end + end + end +end diff --git a/lib/gitlab/ci/runner_releases.rb b/lib/gitlab/ci/runner_releases.rb index dab24bfd501..a27dd3896e1 100644 --- a/lib/gitlab/ci/runner_releases.rb +++ b/lib/gitlab/ci/runner_releases.rb @@ -15,9 +15,14 @@ module Gitlab reset_backoff! end + def enabled? + ::Gitlab::CurrentSettings.current_application_settings.update_runner_versions_enabled? + end + # Returns a sorted list of the publicly available GitLab Runner releases # def releases + return unless enabled? return if backoff_active? Rails.cache.fetch( diff --git a/lib/gitlab/ci/status/composite.rb b/lib/gitlab/ci/status/composite.rb index e854164d377..002bd846ab1 100644 --- a/lib/gitlab/ci/status/composite.rb +++ b/lib/gitlab/ci/status/composite.rb @@ -7,6 +7,7 @@ module Gitlab include Gitlab::Utils::StrongMemoize # This class accepts an array of arrays/hashes/or objects + # `with_allow_failure` will be removed when deleting ci_remove_ensure_stage_service def initialize(all_statuses, with_allow_failure: true, dag: false) unless all_statuses.respond_to?(:pluck) raise ArgumentError, "all_statuses needs to respond to `.pluck`" @@ -26,6 +27,12 @@ module Gitlab # 2. In other cases we assume that status is of that type # based on what statuses are no longer valid based on the # data set that we have + # + # This method is used for two cases: + # 1. When it is called for a stage or a pipeline (with `all_statuses` from all jobs in a stage or a pipeline), + # then, the returned status is assigned to the stage or pipeline. + # 2. When it is called for a job (with `all_statuses` from all previous jobs or all needed jobs), + # then, the returned status is used to determine if the job is processed or not. # rubocop: disable Metrics/CyclomaticComplexity # rubocop: disable Metrics/PerceivedComplexity def status @@ -101,23 +108,22 @@ module Gitlab all_statuses .pluck(*columns) # rubocop: disable CodeReuse/ActiveRecord - .each(&method(:consume_status)) + .each do |status_attrs| + consume_status(Array.wrap(status_attrs)) + end end - def consume_status(description) - # convert `"status"` into `["status"]` - description = Array(description) - - status = - if success_with_warnings?(description) + def consume_status(status_attrs) + status_result = + if success_with_warnings?(status_attrs) :success_with_warnings - elsif ignored_status?(description) + elsif ignored_status?(status_attrs) :ignored else - description[@status_key].to_sym + status_attrs[@status_key].to_sym end - @status_set.add(status) + @status_set.add(status_result) end def success_with_warnings?(status) @@ -129,7 +135,7 @@ module Gitlab def ignored_status?(status) @allow_failure_key && status[@allow_failure_key] && - ::Ci::HasStatus::EXCLUDE_IGNORED_STATUSES.include?(status[@status_key]) + ::Ci::HasStatus::IGNORED_STATUSES.include?(status[@status_key]) end end end diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index 40f5109851b..2f7c16f0904 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.28.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.30.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml index 40f5109851b..2f7c16f0904 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.28.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.30.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml index fa609afc5a8..7f8e2150c71 100644 --- a/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml @@ -23,6 +23,7 @@ variables: CS_ANALYZER_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/security-products/container-scanning:5" + CS_SCHEMA_MODEL: 15 container_scanning: image: "$CS_ANALYZER_IMAGE$CS_IMAGE_SUFFIX" diff --git a/lib/gitlab/ci/templates/Jobs/Container-Scanning.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Container-Scanning.latest.gitlab-ci.yml index f750bda2a3f..15688da71ab 100644 --- a/lib/gitlab/ci/templates/Jobs/Container-Scanning.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Container-Scanning.latest.gitlab-ci.yml @@ -23,6 +23,7 @@ variables: CS_ANALYZER_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/security-products/container-scanning:5" + CS_SCHEMA_MODEL: 15 container_scanning: image: "$CS_ANALYZER_IMAGE$CS_IMAGE_SUFFIX" diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index aa2356f6a34..61c2b468899 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.46.0' + DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.47.0' .dast-auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml index eb8e5de5b56..31d19779434 100644 --- a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml @@ -56,15 +56,15 @@ dependency_scanning: .gemnasium-shared-rule: exists: - - '{Gemfile.lock,*/Gemfile.lock,*/*/Gemfile.lock}' - - '{composer.lock,*/composer.lock,*/*/composer.lock}' - - '{gems.locked,*/gems.locked,*/*/gems.locked}' - - '{go.sum,*/go.sum,*/*/go.sum}' - - '{npm-shrinkwrap.json,*/npm-shrinkwrap.json,*/*/npm-shrinkwrap.json}' - - '{package-lock.json,*/package-lock.json,*/*/package-lock.json}' - - '{yarn.lock,*/yarn.lock,*/*/yarn.lock}' - - '{packages.lock.json,*/packages.lock.json,*/*/packages.lock.json}' - - '{conan.lock,*/conan.lock,*/*/conan.lock}' + - '**/Gemfile.lock' + - '**/composer.lock' + - '**/gems.locked' + - '**/go.sum' + - '**/npm-shrinkwrap.json' + - '**/package-lock.json' + - '**/yarn.lock' + - '**/packages.lock.json' + - '**/conan.lock' gemnasium-dependency_scanning: extends: @@ -91,10 +91,10 @@ gemnasium-dependency_scanning: .gemnasium-maven-shared-rule: exists: - - '{build.gradle,*/build.gradle,*/*/build.gradle}' - - '{build.gradle.kts,*/build.gradle.kts,*/*/build.gradle.kts}' - - '{build.sbt,*/build.sbt,*/*/build.sbt}' - - '{pom.xml,*/pom.xml,*/*/pom.xml}' + - '**/build.gradle' + - '**/build.gradle.kts' + - '**/build.sbt' + - '**/pom.xml' gemnasium-maven-dependency_scanning: extends: @@ -119,12 +119,12 @@ gemnasium-maven-dependency_scanning: .gemnasium-python-shared-rule: exists: - - '{requirements.txt,*/requirements.txt,*/*/requirements.txt}' - - '{requirements.pip,*/requirements.pip,*/*/requirements.pip}' - - '{Pipfile,*/Pipfile,*/*/Pipfile}' - - '{requires.txt,*/requires.txt,*/*/requires.txt}' - - '{setup.py,*/setup.py,*/*/setup.py}' - - '{poetry.lock,*/poetry.lock,*/*/poetry.lock}' + - '**/requirements.txt' + - '**/requirements.pip' + - '**/Pipfile' + - '**/requires.txt' + - '**/setup.py' + - '**/poetry.lock' gemnasium-python-dependency_scanning: extends: diff --git a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml index 655ac6ee712..9ab17997c27 100644 --- a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml @@ -56,15 +56,15 @@ dependency_scanning: .gemnasium-shared-rule: exists: - - '{Gemfile.lock,*/Gemfile.lock,*/*/Gemfile.lock}' - - '{composer.lock,*/composer.lock,*/*/composer.lock}' - - '{gems.locked,*/gems.locked,*/*/gems.locked}' - - '{go.sum,*/go.sum,*/*/go.sum}' - - '{npm-shrinkwrap.json,*/npm-shrinkwrap.json,*/*/npm-shrinkwrap.json}' - - '{package-lock.json,*/package-lock.json,*/*/package-lock.json}' - - '{yarn.lock,*/yarn.lock,*/*/yarn.lock}' - - '{packages.lock.json,*/packages.lock.json,*/*/packages.lock.json}' - - '{conan.lock,*/conan.lock,*/*/conan.lock}' + - '**/Gemfile.lock' + - '**/composer.lock' + - '**/gems.locked' + - '**/go.sum' + - '**/npm-shrinkwrap.json' + - '**/package-lock.json' + - '**/yarn.lock' + - '**/packages.lock.json' + - '**/conan.lock' gemnasium-dependency_scanning: extends: @@ -109,10 +109,10 @@ gemnasium-dependency_scanning: .gemnasium-maven-shared-rule: exists: - - '{build.gradle,*/build.gradle,*/*/build.gradle}' - - '{build.gradle.kts,*/build.gradle.kts,*/*/build.gradle.kts}' - - '{build.sbt,*/build.sbt,*/*/build.sbt}' - - '{pom.xml,*/pom.xml,*/*/pom.xml}' + - '**/build.gradle' + - '**/build.gradle.kts' + - '**/build.sbt' + - '**/pom.xml' gemnasium-maven-dependency_scanning: extends: @@ -155,12 +155,12 @@ gemnasium-maven-dependency_scanning: .gemnasium-python-shared-rule: exists: - - '{requirements.txt,*/requirements.txt,*/*/requirements.txt}' - - '{requirements.pip,*/requirements.pip,*/*/requirements.pip}' - - '{Pipfile,*/Pipfile,*/*/Pipfile}' - - '{requires.txt,*/requires.txt,*/*/requires.txt}' - - '{setup.py,*/setup.py,*/*/setup.py}' - - '{poetry.lock,*/poetry.lock,*/*/poetry.lock}' + - '**/requirements.txt' + - '**/requirements.pip' + - '**/Pipfile' + - '**/requires.txt' + - '**/setup.py' + - '**/poetry.lock' gemnasium-python-dependency_scanning: extends: diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 372b782c0a0..9bac82b660f 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.46.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.47.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index feba2efcf22..ec43217792f 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.46.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.47.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml index 77048037915..b4bff9d9667 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml @@ -34,7 +34,7 @@ kics-iac-sast: SAST_ANALYZER_IMAGE_TAG: 3 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kics:$SAST_ANALYZER_IMAGE_TAG$SAST_IMAGE_SUFFIX" rules: - - if: $SAST_DISABLED + - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1' when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /kics/ when: never diff --git a/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml index 1c4dbe6cd0f..e7c8356662b 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml @@ -51,7 +51,7 @@ brakeman-sast: SAST_ANALYZER_IMAGE_TAG: 3 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/brakeman:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED + - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1' when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /brakeman/ when: never @@ -83,7 +83,7 @@ flawfinder-sast: SAST_ANALYZER_IMAGE_TAG: 3 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/flawfinder:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED + - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1' when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /flawfinder/ when: never @@ -123,7 +123,7 @@ kubesec-sast: SAST_ANALYZER_IMAGE_TAG: 3 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kubesec:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED + - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1' when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /kubesec/ when: never @@ -147,7 +147,7 @@ kubesec-sast: mobsf-android-sast: extends: .mobsf-sast rules: - - if: $SAST_DISABLED + - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1' when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /mobsf/ when: never @@ -169,7 +169,7 @@ mobsf-android-sast: mobsf-ios-sast: extends: .mobsf-sast rules: - - if: $SAST_DISABLED + - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1' when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /mobsf/ when: never @@ -196,7 +196,7 @@ nodejs-scan-sast: SAST_ANALYZER_IMAGE_TAG: 3 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/nodejs-scan:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED + - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1' when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /nodejs-scan/ when: never @@ -217,7 +217,7 @@ phpcs-security-audit-sast: SAST_ANALYZER_IMAGE_TAG: 3 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/phpcs-security-audit:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED + - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1' when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /phpcs-security-audit/ when: never @@ -238,7 +238,7 @@ pmd-apex-sast: SAST_ANALYZER_IMAGE_TAG: 3 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/pmd-apex:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED + - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1' when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /pmd-apex/ when: never @@ -259,7 +259,7 @@ security-code-scan-sast: SAST_ANALYZER_IMAGE_TAG: 3 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/security-code-scan:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED + - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1' when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /security-code-scan/ when: never @@ -283,7 +283,7 @@ semgrep-sast: SAST_ANALYZER_IMAGE_TAG: 3 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/semgrep:$SAST_ANALYZER_IMAGE_TAG$SAST_IMAGE_SUFFIX" rules: - - if: $SAST_DISABLED + - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1' when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /semgrep/ when: never @@ -326,7 +326,7 @@ sobelow-sast: SAST_ANALYZER_IMAGE_TAG: 3 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/sobelow:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED + - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1' when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /sobelow/ when: never @@ -353,7 +353,7 @@ spotbugs-sast: exists: - '**/AndroidManifest.xml' when: never - - if: $SAST_DISABLED + - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1' when: never - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. exists: diff --git a/lib/gitlab/ci/templates/Jobs/Secret-Detection.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Secret-Detection.latest.gitlab-ci.yml index 6603ee4268e..f343dfaa28f 100644 --- a/lib/gitlab/ci/templates/Jobs/Secret-Detection.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Secret-Detection.latest.gitlab-ci.yml @@ -27,7 +27,7 @@ variables: secret_detection: extends: .secret-analyzer rules: - - if: $SECRET_DETECTION_DISABLED + - if: $SECRET_DETECTION_DISABLED == 'true' || $SECRET_DETECTION_DISABLED == '1' when: never - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. diff --git a/lib/gitlab/ci/templates/Security/API-Discovery.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Discovery.gitlab-ci.yml new file mode 100644 index 00000000000..d9bc76dad1e --- /dev/null +++ b/lib/gitlab/ci/templates/Security/API-Discovery.gitlab-ci.yml @@ -0,0 +1,66 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/API-Discovery.gitlab-ci.yml + +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/api_discovery/ +# +# Configure API Discovery with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/api_discovery/#available-cicd-variables + +variables: + API_DISCOVERY_PACKAGES: "$CI_API_V4_URL/projects/42503323/packages" + API_DISCOVERY_VERSION: "1" + +.api_discovery_java_spring_boot: + stage: test + allow_failure: true + script: + # + # Check configuration + - if [[ -z "$API_DISCOVERY_VERSION" ]]; then echo "Error, API_DISCOVERY_VERSION not provided. Please set this variable and re-run the pipeline."; exit 1; fi + # + # Check for required commands + - requires() { command -v "$1" >/dev/null 2>&1 || { echo "'$1' is required but it's not installed. Add the needed command to the job image and retry." >&2; exit 1; } } + - requires 'curl' + - requires 'java' + # + # Set JAVA_HOME if API_DISCOVERY_JAVA_HOME provided + - if [[ -n "$API_DISCOVERY_JAVA_HOME" ]]; then export JAVA_HOME="$API_DISCOVERY_JAVA_HOME"; export PATH="$JAVA_HOME/bin:$PATH"; fi + # + # Download jar file + - if [[ -n "$API_DISCOVERY_PACKAGE_TOKEN" ]]; then echo "Using API_DISCOVERY_PACKAGE_TOKEN"; export CURL_AUTH="-H PRIVATE-TOKEN:$API_DISCOVERY_PACKAGE_TOKEN"; else export CURL_AUTH=""; fi + - DL_URL="$API_DISCOVERY_PACKAGES/maven/com/gitlab/analyzers/api-discovery/api-discovery_spring-boot/$API_DISCOVERY_VERSION/api-discovery_spring-boot-$API_DISCOVERY_VERSION.jar" + - echo "Downloading Discovery jar from '${DL_URL}'" + - CURL_CMD="curl -L ${CURL_AUTH} --write-out "%{http_code}" --output api_discovery_java_spring_boot_${API_DISCOVERY_VERSION}.jar ${DL_URL}" + - STATUS_CODE=$(${CURL_CMD}) + - RC=$? + - if [[ $RC -ne 0 ]]; then echo "Error connecting to GitLab API, curl exit code was $RC."; echo "To diagnose, see the curl documentation- https://everything.curl.dev/usingcurl/returns"; exit 1; fi + - if [[ "$STATUS_CODE" != "200" ]]; then echo "Error, Unable to download api_discovery_java_spring_boot_${API_DISCOVERY_VERSION}.jar"; echo "Error, Status Code was $STATUS_CODE, but wanted 200"; exit 1; fi + # + # Run API Discovery + - java -jar "api_discovery_java_spring_boot_${API_DISCOVERY_VERSION}.jar" + # + # Check for expected output file + - if [[ ! -e "gl-api-discovery-openapi.json" ]]; then echo "Error, Unable to find gl-api-discovery-openapi.json"; exit 1; fi + # + artifacts: + when: always + paths: + - gl-api-discovery-openapi.json + - gl-*.log + rules: + - if: $API_DISCOVERY_DISABLED + when: never + - if: $API_DISCOVERY_DISABLED_FOR_DEFAULT_BRANCH && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + + # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + - if: $CI_OPEN_MERGE_REQUESTS + when: never + + # Add the job to branch pipelines. + - if: $CI_COMMIT_BRANCH diff --git a/lib/gitlab/color_schemes.rb b/lib/gitlab/color_schemes.rb index 3884f5f0428..1ba99e23d65 100644 --- a/lib/gitlab/color_schemes.rb +++ b/lib/gitlab/color_schemes.rb @@ -46,7 +46,7 @@ module Gitlab # # Returns a Scheme def self.default - by_id(1) + by_id(Gitlab::CurrentSettings.default_syntax_highlighting_theme) end # Iterate through each Scheme diff --git a/lib/gitlab/config/entry/validators.rb b/lib/gitlab/config/entry/validators.rb index fad2260d818..28cfb6d8fee 100644 --- a/lib/gitlab/config/entry/validators.rb +++ b/lib/gitlab/config/entry/validators.rb @@ -356,7 +356,7 @@ module Gitlab ports_size = value.count return if ports_size <= 1 - named_ports = value.select { |e| e.is_a?(Hash) }.map { |e| e[:name] }.compact.map(&:downcase) + named_ports = value.select { |e| e.is_a?(Hash) }.filter_map { |e| e[:name] }.map(&:downcase) if ports_size != named_ports.size record.errors.add(attribute, 'when there is more than one port, a unique name should be added') diff --git a/lib/gitlab/config/loader/multi_doc_yaml.rb b/lib/gitlab/config/loader/multi_doc_yaml.rb index 346adc79896..34080d26b7c 100644 --- a/lib/gitlab/config/loader/multi_doc_yaml.rb +++ b/lib/gitlab/config/loader/multi_doc_yaml.rb @@ -4,59 +4,48 @@ module Gitlab module Config module Loader class MultiDocYaml - TooManyDocumentsError = Class.new(Loader::FormatError) - DataTooLargeError = Class.new(Loader::FormatError) - NotHashError = Class.new(Loader::FormatError) + include Gitlab::Utils::StrongMemoize - MULTI_DOC_DIVIDER = /^---$/.freeze + MULTI_DOC_DIVIDER = /^---\s+/.freeze def initialize(config, max_documents:, additional_permitted_classes: []) + @config = config @max_documents = max_documents - @safe_config = load_config(config, additional_permitted_classes) + @additional_permitted_classes = additional_permitted_classes end - def load! - raise TooManyDocumentsError, 'The parsed YAML has too many documents' if too_many_documents? - raise DataTooLargeError, 'The parsed YAML is too big' if too_big? - raise NotHashError, 'Invalid configuration format' unless all_hashes? - - safe_config.map(&:deep_symbolize_keys) + def valid? + documents.all?(&:valid?) end - private - - attr_reader :safe_config, :max_documents - - def load_config(config, additional_permitted_classes) - config.split(MULTI_DOC_DIVIDER).filter_map do |document| - YAML.safe_load(document, - permitted_classes: [Symbol, *additional_permitted_classes], - permitted_symbols: [], - aliases: true - ) - end - rescue Psych::Exception => e - raise Loader::FormatError, e.message + def load_raw! + documents.map(&:load_raw!) end - def all_hashes? - safe_config.all?(Hash) + def load! + documents.map(&:load!) end - def too_many_documents? - safe_config.count > max_documents - end + private + + attr_reader :config, :max_documents, :additional_permitted_classes + + # Valid YAML files can start with either a leading delimiter or no delimiter. + # To avoid counting a leading delimiter towards the document limit, + # this method splits the file by one more than the maximum number of permitted documents. + # It then discards the first document if it is blank. + def documents + docs = config + .split(MULTI_DOC_DIVIDER, max_documents_including_leading_delimiter) + .map { |d| Yaml.new(d, additional_permitted_classes: additional_permitted_classes) } - def too_big? - !deep_sizes.all?(&:valid?) + docs.shift if docs.first.blank? + docs end + strong_memoize_attr :documents - def deep_sizes - safe_config.map do |config| - Gitlab::Utils::DeepSize.new(config, - max_size: Gitlab::CurrentSettings.current_application_settings.max_yaml_size_bytes, - max_depth: Gitlab::CurrentSettings.current_application_settings.max_yaml_depth) - end + def max_documents_including_leading_delimiter + max_documents + 1 end end end diff --git a/lib/gitlab/config/loader/yaml.rb b/lib/gitlab/config/loader/yaml.rb index 7b87b5b8f97..38f8eca3c3c 100644 --- a/lib/gitlab/config/loader/yaml.rb +++ b/lib/gitlab/config/loader/yaml.rb @@ -34,6 +34,10 @@ module Gitlab @symbolized_config ||= load_raw!.deep_symbolize_keys end + def blank? + @config.blank? + end + private def hash? diff --git a/lib/gitlab/content_security_policy/config_loader.rb b/lib/gitlab/content_security_policy/config_loader.rb index ceca206b084..477877e6a7c 100644 --- a/lib/gitlab/content_security_policy/config_loader.rb +++ b/lib/gitlab/content_security_policy/config_loader.rb @@ -50,6 +50,7 @@ module Gitlab allow_sentry(directives) if Gitlab::CurrentSettings.try(:sentry_enabled) && Gitlab::CurrentSettings.try(:sentry_clientside_dsn) allow_framed_gitlab_paths(directives) allow_customersdot(directives) if ENV['CUSTOMER_PORTAL_URL'].present? + allow_kas(directives) allow_review_apps(directives) if ENV['REVIEW_APPS_ENABLED'] # The follow section contains workarounds to patch Safari's lack of support for CSP Level 3 @@ -147,6 +148,17 @@ module Gitlab append_to_directive(directives, 'frame_src', customersdot_host) end + def self.allow_kas(directives) + return unless ::Gitlab::Kas::UserAccess.enabled? + + kas_url = ::Gitlab::Kas.tunnel_url + return if URI(kas_url).host == ::Gitlab.config.gitlab.host # already allowed, no need for exception + + kas_url += '/' unless kas_url.end_with?('/') + + append_to_directive(directives, 'connect_src', kas_url) + end + def self.allow_legacy_sentry(directives) # Support for Sentry setup via configuration files will be removed in 16.0 # in favor of Gitlab::CurrentSettings. diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index 939eaa377aa..ecb0cc20a64 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -45,8 +45,9 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def preload_builds(pipeline, association) - ActiveRecord::Associations::Preloader.new.preload(pipeline, - { + ActiveRecord::Associations::Preloader.new( + records: [pipeline], + associations: { association => { **::Ci::Pipeline::PROJECT_ROUTE_AND_NAMESPACE_ROUTE, runner: :tags, @@ -56,7 +57,7 @@ module Gitlab ci_stage: [] } } - ) + ).call end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/gitlab/database/async_foreign_keys.rb b/lib/gitlab/database/async_constraints.rb index 115ae9ba2e8..026197c7e40 100644 --- a/lib/gitlab/database/async_foreign_keys.rb +++ b/lib/gitlab/database/async_constraints.rb @@ -2,12 +2,12 @@ module Gitlab module Database - module AsyncForeignKeys + module AsyncConstraints DEFAULT_ENTRIES_PER_INVOCATION = 2 def self.validate_pending_entries!(how_many: DEFAULT_ENTRIES_PER_INVOCATION) - PostgresAsyncForeignKeyValidation.ordered.limit(how_many).each do |record| - ForeignKeyValidator.new(record).perform + PostgresAsyncConstraintValidation.ordered.limit(how_many).each do |record| + AsyncConstraints::Validators.for(record).perform end end end diff --git a/lib/gitlab/database/async_foreign_keys/migration_helpers.rb b/lib/gitlab/database/async_constraints/migration_helpers.rb index b8b9fc6d156..8b4d4ecea04 100644 --- a/lib/gitlab/database/async_foreign_keys/migration_helpers.rb +++ b/lib/gitlab/database/async_constraints/migration_helpers.rb @@ -2,7 +2,7 @@ module Gitlab module Database - module AsyncForeignKeys + module AsyncConstraints module MigrationHelpers # Prepares a foreign key for asynchronous validation. # @@ -12,7 +12,7 @@ module Gitlab def prepare_async_foreign_key_validation(table_name, column_name = nil, name: nil) Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode! - return unless async_fk_validation_available? + return unless async_constraint_validation_available? fk_name = name || concurrent_foreign_key_name(table_name, column_name) @@ -20,7 +20,8 @@ module Gitlab raise missing_schema_object_message(table_name, "foreign key", fk_name) end - async_validation = PostgresAsyncForeignKeyValidation + async_validation = PostgresAsyncConstraintValidation + .foreign_key_type .find_or_create_by!(name: fk_name, table_name: table_name) Gitlab::AppLogger.info( @@ -34,17 +35,20 @@ module Gitlab def unprepare_async_foreign_key_validation(table_name, column_name = nil, name: nil) Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode! - return unless async_fk_validation_available? + return unless async_constraint_validation_available? fk_name = name || concurrent_foreign_key_name(table_name, column_name) - PostgresAsyncForeignKeyValidation.find_by(name: fk_name).try(&:destroy) + PostgresAsyncConstraintValidation + .foreign_key_type + .find_by(name: fk_name, table_name: table_name) + .try(&:destroy!) end def prepare_partitioned_async_foreign_key_validation(table_name, column_name = nil, name: nil) Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode! - return unless async_fk_validation_available? + return unless async_constraint_validation_available? Gitlab::Database::PostgresPartitionedTable.each_partition(table_name) do |partition| prepare_async_foreign_key_validation(partition.identifier, column_name, name: name) @@ -54,17 +58,54 @@ module Gitlab def unprepare_partitioned_async_foreign_key_validation(table_name, column_name = nil, name: nil) Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode! - return unless async_fk_validation_available? + return unless async_constraint_validation_available? Gitlab::Database::PostgresPartitionedTable.each_partition(table_name) do |partition| unprepare_async_foreign_key_validation(partition.identifier, column_name, name: name) end end + # Prepares a check constraint for asynchronous validation. + # + # Stores the constraint information in the postgres_async_foreign_key_validations + # table to be executed later. + # + def prepare_async_check_constraint_validation(table_name, name:) + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode! + + return unless async_constraint_validation_available? + + unless check_constraint_exists?(table_name, name) + raise missing_schema_object_message(table_name, "check constraint", name) + end + + async_validation = PostgresAsyncConstraintValidation + .check_constraint_type + .find_or_create_by!(name: name, table_name: table_name) + + Gitlab::AppLogger.info( + message: 'Prepared check constraint for async validation', + table_name: async_validation.table_name, + constraint_name: async_validation.name) + + async_validation + end + + def unprepare_async_check_constraint_validation(table_name, name:) + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode! + + return unless async_constraint_validation_available? + + PostgresAsyncConstraintValidation + .check_constraint_type + .find_by(name: name, table_name: table_name) + .try(&:destroy!) + end + private - def async_fk_validation_available? - connection.table_exists?(:postgres_async_foreign_key_validations) + def async_constraint_validation_available? + PostgresAsyncConstraintValidation.table_available? end end end diff --git a/lib/gitlab/database/async_constraints/postgres_async_constraint_validation.rb b/lib/gitlab/database/async_constraints/postgres_async_constraint_validation.rb new file mode 100644 index 00000000000..7fb62948119 --- /dev/null +++ b/lib/gitlab/database/async_constraints/postgres_async_constraint_validation.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module AsyncConstraints + class PostgresAsyncConstraintValidation < SharedModel + include QueueErrorHandlingConcern + + self.table_name = 'postgres_async_foreign_key_validations' + + MAX_IDENTIFIER_LENGTH = Gitlab::Database::MigrationHelpers::MAX_IDENTIFIER_NAME_LENGTH + MAX_LAST_ERROR_LENGTH = 10_000 + + validates :name, presence: true, uniqueness: { scope: :table_name }, length: { maximum: MAX_IDENTIFIER_LENGTH } + validates :table_name, presence: true, length: { maximum: MAX_IDENTIFIER_LENGTH } + + enum constraint_type: { foreign_key: 0, check_constraint: 1 } + + scope :ordered, -> { order(attempts: :asc, id: :asc) } + scope :foreign_key_type, -> { constraint_type_exists? ? foreign_key : all } + scope :check_constraint_type, -> { check_constraint } + + class << self + def table_available? + connection.table_exists?(table_name) + end + + def constraint_type_exists? + connection.column_exists?(table_name, :constraint_type) + end + end + end + end + end +end diff --git a/lib/gitlab/database/async_constraints/validators.rb b/lib/gitlab/database/async_constraints/validators.rb new file mode 100644 index 00000000000..39792a5ee8e --- /dev/null +++ b/lib/gitlab/database/async_constraints/validators.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module AsyncConstraints + module Validators + MAPPING = { + foreign_key: Validators::ForeignKey, + check_constraint: Validators::CheckConstraint + }.freeze + + def self.for(record) + MAPPING + .fetch(record.constraint_type.to_sym) + .new(record) + end + end + end + end +end diff --git a/lib/gitlab/database/async_constraints/validators/base.rb b/lib/gitlab/database/async_constraints/validators/base.rb new file mode 100644 index 00000000000..39a72955d63 --- /dev/null +++ b/lib/gitlab/database/async_constraints/validators/base.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module AsyncConstraints + module Validators + class Base + include AsyncDdlExclusiveLeaseGuard + extend ::Gitlab::Utils::Override + + TIMEOUT_PER_ACTION = 1.day + STATEMENT_TIMEOUT = 12.hours + + def initialize(record) + @record = record + end + + def perform + try_obtain_lease do + if constraint_exists? + log_info('Starting to validate constraint') + validate_constraint_with_error_handling + log_info('Finished validating constraint') + else + log_info(skip_log_message) + record.destroy! + end + end + end + + private + + attr_reader :record + + delegate :connection, :name, :table_name, :connection_db_config, to: :record + + def constraint_exists?; end + + def validate_constraint_with_error_handling + validate_constraint + record.destroy! + rescue StandardError => error + record.handle_exception!(error) + + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) + Gitlab::AppLogger.error(message: error.message, **logging_options) + end + + def validate_constraint + set_statement_timeout do + connection.execute(<<~SQL.squish) + ALTER TABLE #{connection.quote_table_name(table_name)} + VALIDATE CONSTRAINT #{connection.quote_column_name(name)}; + SQL + end + end + + def set_statement_timeout + connection.execute(format("SET statement_timeout TO '%ds'", STATEMENT_TIMEOUT)) + yield + ensure + connection.execute('RESET statement_timeout') + end + + def lease_timeout + TIMEOUT_PER_ACTION + end + + def log_info(message) + Gitlab::AppLogger.info(message: message, **logging_options) + end + + def skip_log_message + "Skipping #{name} validation since it does not exist. " \ + "The queuing entry will be deleted" + end + + def logging_options + { + class: self.class.name.to_s, + connection_name: database_config_name, + constraint_name: name, + constraint_type: record.constraint_type, + table_name: table_name + } + end + end + end + end + end +end diff --git a/lib/gitlab/database/async_constraints/validators/check_constraint.rb b/lib/gitlab/database/async_constraints/validators/check_constraint.rb new file mode 100644 index 00000000000..695ecdebc9f --- /dev/null +++ b/lib/gitlab/database/async_constraints/validators/check_constraint.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module AsyncConstraints + module Validators + class CheckConstraint < Base + private + + override :constraint_exists? + def constraint_exists? + Gitlab::Database::Migrations::ConstraintsHelpers + .check_constraint_exists?(table_name, name, connection: connection) + end + end + end + end + end +end diff --git a/lib/gitlab/database/async_constraints/validators/foreign_key.rb b/lib/gitlab/database/async_constraints/validators/foreign_key.rb new file mode 100644 index 00000000000..ff6b807c982 --- /dev/null +++ b/lib/gitlab/database/async_constraints/validators/foreign_key.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module AsyncConstraints + module Validators + class ForeignKey < Base + private + + override :constraint_exists? + def constraint_exists? + Gitlab::Database::PostgresForeignKey + .by_constrained_table_name_or_identifier(table_name) + .by_name(name) + .exists? + end + end + end + end + end +end diff --git a/lib/gitlab/database/async_foreign_keys/foreign_key_validator.rb b/lib/gitlab/database/async_foreign_keys/foreign_key_validator.rb deleted file mode 100644 index 5958c56a45a..00000000000 --- a/lib/gitlab/database/async_foreign_keys/foreign_key_validator.rb +++ /dev/null @@ -1,94 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module AsyncForeignKeys - class ForeignKeyValidator - include AsyncDdlExclusiveLeaseGuard - - TIMEOUT_PER_ACTION = 1.day - STATEMENT_TIMEOUT = 12.hours - - def initialize(async_validation) - @async_validation = async_validation - end - - def perform - try_obtain_lease do - if foreign_key_exists? - log_index_info("Starting to validate foreign key") - validate_foreign_with_error_handling - log_index_info("Finished validating foreign key") - else - log_index_info(skip_log_message) - async_validation.destroy! - end - end - end - - private - - attr_reader :async_validation - - delegate :connection, :name, :table_name, :connection_db_config, to: :async_validation - - def foreign_key_exists? - relation = if table_name =~ Gitlab::Database::FULLY_QUALIFIED_IDENTIFIER - Gitlab::Database::PostgresForeignKey.by_constrained_table_identifier(table_name) - else - Gitlab::Database::PostgresForeignKey.by_constrained_table_name(table_name) - end - - relation.by_name(name).exists? - end - - def validate_foreign_with_error_handling - validate_foreign_key - async_validation.destroy! - rescue StandardError => error - async_validation.handle_exception!(error) - - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) - Gitlab::AppLogger.error(message: error.message, **logging_options) - end - - def validate_foreign_key - set_statement_timeout do - connection.execute(<<~SQL.squish) - ALTER TABLE #{connection.quote_table_name(table_name)} - VALIDATE CONSTRAINT #{connection.quote_column_name(name)}; - SQL - end - end - - def set_statement_timeout - connection.execute(format("SET statement_timeout TO '%ds'", STATEMENT_TIMEOUT)) - yield - ensure - connection.execute('RESET statement_timeout') - end - - def lease_timeout - TIMEOUT_PER_ACTION - end - - def log_index_info(message) - Gitlab::AppLogger.info(message: message, **logging_options) - end - - def skip_log_message - "Skipping #{name} validation since it does not exist. " \ - "The queuing entry will be deleted" - end - - def logging_options - { - fk_name: name, - table_name: table_name, - class: self.class.name.to_s - } - end - end - end - end -end diff --git a/lib/gitlab/database/async_foreign_keys/postgres_async_foreign_key_validation.rb b/lib/gitlab/database/async_foreign_keys/postgres_async_foreign_key_validation.rb deleted file mode 100644 index de69a3d496f..00000000000 --- a/lib/gitlab/database/async_foreign_keys/postgres_async_foreign_key_validation.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module AsyncForeignKeys - class PostgresAsyncForeignKeyValidation < SharedModel - include QueueErrorHandlingConcern - - self.table_name = 'postgres_async_foreign_key_validations' - - MAX_IDENTIFIER_LENGTH = Gitlab::Database::MigrationHelpers::MAX_IDENTIFIER_NAME_LENGTH - MAX_LAST_ERROR_LENGTH = 10_000 - - validates :name, presence: true, uniqueness: true, length: { maximum: MAX_IDENTIFIER_LENGTH } - validates :table_name, presence: true, length: { maximum: MAX_IDENTIFIER_LENGTH } - - scope :ordered, -> { order(attempts: :asc, id: :asc) } - end - end - end -end diff --git a/lib/gitlab/database/background_migration/batch_optimizer.rb b/lib/gitlab/database/background_migration/batch_optimizer.rb index c8fdf8281cd..9eb456f6e2e 100644 --- a/lib/gitlab/database/background_migration/batch_optimizer.rb +++ b/lib/gitlab/database/background_migration/batch_optimizer.rb @@ -43,11 +43,14 @@ module Gitlab def optimize! return unless Feature.enabled?(:optimize_batched_migrations, type: :ops) - if multiplier = batch_size_multiplier - max_batch = migration.max_batch_size || MAX_BATCH_SIZE - migration.batch_size = (migration.batch_size * multiplier).to_i.clamp(MIN_BATCH_SIZE, max_batch) - migration.save! - end + multiplier = batch_size_multiplier + return if multiplier.nil? + + max_batch = migration.max_batch_size || MAX_BATCH_SIZE + min_batch = [max_batch, MIN_BATCH_SIZE].min + + migration.batch_size = (migration.batch_size * multiplier).to_i.clamp(min_batch, max_batch) + migration.save! end private diff --git a/lib/gitlab/database/background_migration/batched_job.rb b/lib/gitlab/database/background_migration/batched_job.rb index 6b7ff308c7e..5147ea92291 100644 --- a/lib/gitlab/database/background_migration/batched_job.rb +++ b/lib/gitlab/database/background_migration/batched_job.rb @@ -4,6 +4,7 @@ module Gitlab module Database module BackgroundMigration SplitAndRetryError = Class.new(StandardError) + ReduceSubBatchSizeError = Class.new(StandardError) class BatchedJob < SharedModel include EachBatch @@ -12,6 +13,9 @@ module Gitlab self.table_name = :batched_background_migration_jobs MAX_ATTEMPTS = 3 + MIN_BATCH_SIZE = 1 + SUB_BATCH_SIZE_REDUCE_FACTOR = 0.75 + SUB_BATCH_SIZE_THRESHOLD = 65 STUCK_JOBS_TIMEOUT = 1.hour.freeze TIMEOUT_EXCEPTIONS = [ActiveRecord::StatementTimeout, ActiveRecord::ConnectionTimeoutError, ActiveRecord::AdapterTimeout, ActiveRecord::LockWaitTimeout, @@ -59,12 +63,12 @@ module Gitlab end after_transition any => :failed do |job, transition| - error_hash = transition.args.find { |arg| arg[:error].present? } + exception, from_sub_batch = job.class.extract_transition_options(transition.args) - exception = error_hash&.fetch(:error) + job.reduce_sub_batch_size! if from_sub_batch && job.can_reduce_sub_batch_size? job.split_and_retry! if job.can_split?(exception) - rescue SplitAndRetryError => error + rescue SplitAndRetryError, ReduceSubBatchSizeError => error Gitlab::AppLogger.error( message: error.message, batched_job_id: job.id, @@ -75,9 +79,7 @@ module Gitlab end after_transition do |job, transition| - error_hash = transition.args.find { |arg| arg[:error].present? } - - exception = error_hash&.fetch(:error) + exception, _ = job.class.extract_transition_options(transition.args) job.batched_job_transition_logs.create(previous_status: transition.from, next_status: transition.to, exception_class: exception&.class, exception_message: exception&.message) @@ -100,7 +102,16 @@ module Gitlab delegate :job_class, :table_name, :column_name, :job_arguments, :job_class_name, to: :batched_migration, prefix: :migration - attribute :pause_ms, :integer, default: 100 + def self.extract_transition_options(args) + error_hash = args.find { |arg| arg[:error].present? } + + return [] unless error_hash + + exception = error_hash.fetch(:error) + from_sub_batch = error_hash[:from_sub_batch] + + [exception, from_sub_batch] + end def time_efficiency return unless succeeded? @@ -113,10 +124,15 @@ module Gitlab end def can_split?(exception) - attempts >= MAX_ATTEMPTS && - exception&.class&.in?(TIMEOUT_EXCEPTIONS) && - batch_size > sub_batch_size && - batch_size > 1 + return if still_retryable? + + exception.class.in?(TIMEOUT_EXCEPTIONS) && within_batch_size_boundaries? + end + + def can_reduce_sub_batch_size? + return false unless Feature.enabled?(:reduce_sub_batch_size_on_timeouts) + + still_retryable? && within_batch_size_boundaries? end def split_and_retry! @@ -165,6 +181,51 @@ module Gitlab end end end + + # It reduces the size of +sub_batch_size+ by 25% + def reduce_sub_batch_size! + raise ReduceSubBatchSizeError, 'Only sub_batch_size of failed jobs can be reduced' unless failed? + + return if sub_batch_exceeds_threshold? + + with_lock do + actual_sub_batch_size = sub_batch_size + reduced_sub_batch_size = (sub_batch_size * SUB_BATCH_SIZE_REDUCE_FACTOR).to_i.clamp(1, batch_size) + + update!(sub_batch_size: reduced_sub_batch_size) + + Gitlab::AppLogger.warn( + message: 'Sub batch size reduced due to timeout', + batched_job_id: id, + sub_batch_size: actual_sub_batch_size, + reduced_sub_batch_size: reduced_sub_batch_size, + attempts: attempts, + batched_migration_id: batched_migration.id, + job_class_name: migration_job_class_name, + job_arguments: migration_job_arguments + ) + end + end + + def still_retryable? + attempts < MAX_ATTEMPTS + end + + def within_batch_size_boundaries? + batch_size > MIN_BATCH_SIZE && batch_size > sub_batch_size + end + + # It doesn't allow sub-batch size to be reduced lower than the threshold + # + # @info It will prevent the next iteration to reduce the +sub_batch_size+ lower + # than the +SUB_BATCH_SIZE_THRESHOLD+ or 65% of its original size. + def sub_batch_exceeds_threshold? + initial_sub_batch_size = batched_migration.sub_batch_size + reduced_sub_batch_size = (sub_batch_size * SUB_BATCH_SIZE_REDUCE_FACTOR).to_i + diff = initial_sub_batch_size - reduced_sub_batch_size + + (1.0 * diff / initial_sub_batch_size * 100).round(2) > SUB_BATCH_SIZE_THRESHOLD + end end end end diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb index 61a660ad14c..429dc79e170 100644 --- a/lib/gitlab/database/background_migration/batched_migration.rb +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -83,8 +83,6 @@ module Gitlab end end - attribute :pause_ms, :integer, default: 100 - def self.valid_status state_machine.states.map(&:name) end diff --git a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb index f1fc3efae9e..8fdaa685ba9 100644 --- a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb +++ b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb @@ -15,13 +15,21 @@ module Gitlab # when starting and finishing execution, and optionally saves batch_metrics # the migration provides, if any are given. # - # The job's batch_metrics are serialized to JSON for storage. + # @info The job's batch_metrics are serialized to JSON for storage. + # + # @info Track exceptions that could happen when processing sub-batches + # through +Gitlab::BackgroundMigration::SubBatchTimeoutException+ def perform(batch_tracking_record) start_tracking_execution(batch_tracking_record) execute_batch(batch_tracking_record) batch_tracking_record.succeed! + rescue SubBatchTimeoutError => exception + caused_by = exception.caused_by + batch_tracking_record.failure!(error: caused_by, from_sub_batch: true) + + raise caused_by rescue Exception => error # rubocop:disable Lint/RescueException batch_tracking_record.failure!(error: error) @@ -67,7 +75,8 @@ module Gitlab sub_batch_size: tracking_record.sub_batch_size, pause_ms: tracking_record.pause_ms, job_arguments: tracking_record.migration_job_arguments, - connection: connection) + connection: connection, + sub_batch_exception: ::Gitlab::Database::BackgroundMigration::SubBatchTimeoutError) job_instance.perform diff --git a/lib/gitlab/database/background_migration/sub_batch_timeout_error.rb b/lib/gitlab/database/background_migration/sub_batch_timeout_error.rb new file mode 100644 index 00000000000..815dff11e1f --- /dev/null +++ b/lib/gitlab/database/background_migration/sub_batch_timeout_error.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module BackgroundMigration + class SubBatchTimeoutError < StandardError + def initialize(caused_by) + @caused_by = caused_by + + super(caused_by) + end + + attr_reader :caused_by + end + end + end +end diff --git a/lib/gitlab/database/gitlab_schema.rb b/lib/gitlab/database/gitlab_schema.rb index 38558512b6a..926a4aeedf1 100644 --- a/lib/gitlab/database/gitlab_schema.rb +++ b/lib/gitlab/database/gitlab_schema.rb @@ -45,6 +45,11 @@ module Gitlab return gitlab_schema end + # Partitions that belong to the CI domain + if table_name.start_with?('ci_') && gitlab_schema = views_and_tables_to_schema["p_#{table_name}"] + return gitlab_schema + end + # All tables from `information_schema.` are marked as `internal` return :gitlab_internal if schema_name == 'information_schema' @@ -121,6 +126,16 @@ module Gitlab key_name = data['table_name'] || data['view_name'] + # rubocop:disable Gitlab/DocUrl + if data['gitlab_schema'].nil? + raise( + UnknownSchemaError, + "#{file_path} must specify a valid gitlab_schema for #{key_name}." \ + "See https://docs.gitlab.com/ee/development/database/database_dictionary.html" + ) + end + # rubocop:enable Gitlab/DocUrl + dic[key_name] = data['gitlab_schema'].to_sym end end diff --git a/lib/gitlab/database/lock_writes_manager.rb b/lib/gitlab/database/lock_writes_manager.rb index 83884e89d6e..e8f7b51955d 100644 --- a/lib/gitlab/database/lock_writes_manager.rb +++ b/lib/gitlab/database/lock_writes_manager.rb @@ -10,6 +10,8 @@ module Gitlab # See https://www.postgresql.org/message-id/16934.1568989957%40sss.pgh.pa.us EXPECTED_TRIGGER_RECORD_COUNT = 3 + # table_name can include schema name as a prefix. For example: 'gitlab_partitions_static.events_03', + # otherwise, it will default to current used schema, for example 'public'. def initialize(table_name:, connection:, database_name:, with_retries: true, logger: nil, dry_run: false) @table_name = table_name @connection = connection diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 9b041c18da4..3a342abe65d 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -14,7 +14,7 @@ module Gitlab include DynamicModelHelpers include RenameTableHelpers include AsyncIndexes::MigrationHelpers - include AsyncForeignKeys::MigrationHelpers + include AsyncConstraints::MigrationHelpers def define_batchable_model(table_name, connection: self.connection) super(table_name, connection: connection) @@ -292,23 +292,34 @@ module Gitlab # order of the ALTER TABLE. This can be useful in situations where the foreign # key creation could deadlock with another process. # - # rubocop: disable Metrics/ParameterLists - def add_concurrent_foreign_key(source, target, column:, on_delete: :cascade, on_update: nil, target_column: :id, name: nil, validate: true, reverse_lock_order: false) + def add_concurrent_foreign_key(source, target, column:, **options) + options.reverse_merge!({ + on_delete: :cascade, + on_update: nil, + target_column: :id, + validate: true, + reverse_lock_order: false, + allow_partitioned: false, + column: column + }) + # Transactions would result in ALTER TABLE locks being held for the # duration of the transaction, defeating the purpose of this method. if transaction_open? raise 'add_concurrent_foreign_key can not be run inside a transaction' end - options = { - column: column, - on_delete: on_delete, - on_update: on_update, - name: name.presence || concurrent_foreign_key_name(source, column), - primary_key: target_column - } + if !options.delete(:allow_partitioned) && table_partitioned?(source) + raise ArgumentError, 'add_concurrent_foreign_key can not be used on a partitioned ' \ + 'table. Please use add_concurrent_partitioned_foreign_key on the partitioned table ' \ + 'as we need to create foreign keys on each partition and a FK on the parent table' + end + + options[:name] ||= concurrent_foreign_key_name(source, column) + options[:primary_key] = options[:target_column] + check_options = options.slice(:column, :on_delete, :on_update, :name, :primary_key) - if foreign_key_exists?(source, target, **options) + if foreign_key_exists?(source, target, **check_options) warning_message = "Foreign key not created because it exists already " \ "(this may be due to an aborted migration or similar): " \ "source: #{source}, target: #{target}, column: #{options[:column]}, "\ @@ -317,23 +328,7 @@ module Gitlab Gitlab::AppLogger.warn warning_message else - # Using NOT VALID allows us to create a key without immediately - # validating it. This means we keep the ALTER TABLE lock only for a - # short period of time. The key _is_ enforced for any newly created - # data. - - with_lock_retries do - execute("LOCK TABLE #{target}, #{source} IN SHARE ROW EXCLUSIVE MODE") if reverse_lock_order - execute <<-EOF.strip_heredoc - ALTER TABLE #{source} - ADD CONSTRAINT #{options[:name]} - FOREIGN KEY (#{multiple_columns(options[:column])}) - REFERENCES #{target} (#{multiple_columns(target_column)}) - #{on_update_statement(options[:on_update])} - #{on_delete_statement(options[:on_delete])} - NOT VALID; - EOF - end + execute_add_concurrent_foreign_key(source, target, options) end # Validate the existing constraint. This can potentially take a very @@ -345,13 +340,12 @@ module Gitlab # # Note this is a no-op in case the constraint is VALID already - if validate + if options[:validate] disable_statement_timeout do execute("ALTER TABLE #{source} VALIDATE CONSTRAINT #{options[:name]};") end end end - # rubocop: enable Metrics/ParameterLists def validate_foreign_key(source, column, name: nil) fk_name = name || concurrent_foreign_key_name(source, column) @@ -379,7 +373,7 @@ module Gitlab end end - fks = Gitlab::Database::PostgresForeignKey.by_constrained_table_name(source) + fks = Gitlab::Database::PostgresForeignKey.by_constrained_table_name_or_identifier(source) fks = fks.by_referenced_table_name(target) if target fks = fks.by_name(options[:name]) if options[:name] @@ -1239,6 +1233,12 @@ into similar problems in the future (e.g. when new tables are created). end end + def table_partitioned?(table_name) + Gitlab::Database::PostgresPartitionedTable + .find_by_name_in_current_schema(table_name) + .present? + end + private def multiple_columns(columns, separator: ', ') @@ -1354,6 +1354,33 @@ into similar problems in the future (e.g. when new tables are created). Must end with `_at`} MESSAGE end + + def execute_add_concurrent_foreign_key(source, target, options) + # Using NOT VALID allows us to create a key without immediately + # validating it. This means we keep the ALTER TABLE lock only for a + # short period of time. The key _is_ enforced for any newly created + # data. + not_valid = 'NOT VALID' + lock_mode = 'SHARE ROW EXCLUSIVE' + + if table_partitioned?(source) + not_valid = '' + lock_mode = 'ACCESS EXCLUSIVE' + end + + with_lock_retries do + execute("LOCK TABLE #{target}, #{source} IN #{lock_mode} MODE") if options[:reverse_lock_order] + execute(<<~SQL.squish) + ALTER TABLE #{source} + ADD CONSTRAINT #{options[:name]} + FOREIGN KEY (#{multiple_columns(options[:column])}) + REFERENCES #{target} (#{multiple_columns(options[:target_column])}) + #{on_update_statement(options[:on_update])} + #{on_delete_statement(options[:on_delete])} + #{not_valid}; + SQL + end + end end end end diff --git a/lib/gitlab/database/migration_helpers/convert_to_bigint.rb b/lib/gitlab/database/migration_helpers/convert_to_bigint.rb new file mode 100644 index 00000000000..cf5640deb3d --- /dev/null +++ b/lib/gitlab/database/migration_helpers/convert_to_bigint.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module MigrationHelpers + module ConvertToBigint + # This helper is extracted for the purpose of + # https://gitlab.com/gitlab-org/gitlab/-/issues/392815 + # so that we can test all combinations just once, + # and simplify migration tests. + # + # Once we are done with the PK conversions we can remove this. + def com_or_dev_or_test_but_not_jh? + !Gitlab.jh? && (Gitlab.com? || Gitlab.dev_or_test_env?) + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb index e958ce2aba4..cb2a98b553f 100644 --- a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb @@ -12,6 +12,7 @@ module Gitlab # For now, these migrations are not considered ready for general use, for more information see the tracking epic: # https://gitlab.com/groups/gitlab-org/-/epics/6751 module BatchedBackgroundMigrationHelpers + NonExistentMigrationError = Class.new(StandardError) BATCH_SIZE = 1_000 # Number of rows to process per job SUB_BATCH_SIZE = 100 # Number of rows to process per sub-batch BATCH_CLASS_NAME = 'PrimaryKeyBatchingStrategy' # Default batch class for batched migrations @@ -200,6 +201,12 @@ module Gitlab def ensure_batched_background_migration_is_finished(job_class_name:, table_name:, column_name:, job_arguments:, finalize: true) Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_dml_mode! + if transaction_open? + raise 'The `ensure_batched_background_migration_is_finished` cannot be run inside a transaction. ' \ + 'You can disable transactions by calling `disable_ddl_transaction!` in the body of ' \ + 'your migration class.' + end + Gitlab::Database::BackgroundMigration::BatchedMigration.reset_column_information migration = Gitlab::Database::BackgroundMigration::BatchedMigration.find_for_configuration( Gitlab::Database.gitlab_schemas_for_connection(connection), @@ -213,6 +220,10 @@ module Gitlab job_arguments: job_arguments } + if ENV['DBLAB_ENVIRONMENT'] && migration.nil? + raise NonExistentMigrationError, 'called ensure_batched_background_migration_is_finished with non-existent migration name' + end + return Gitlab::AppLogger.warn "Could not find batched background migration for the given configuration: #{configuration}" if migration.nil? return if migration.finished? diff --git a/lib/gitlab/database/migrations/constraints_helpers.rb b/lib/gitlab/database/migrations/constraints_helpers.rb index 7b849e3137a..5aafc9f1444 100644 --- a/lib/gitlab/database/migrations/constraints_helpers.rb +++ b/lib/gitlab/database/migrations/constraints_helpers.rb @@ -10,6 +10,27 @@ module Gitlab # https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS MAX_IDENTIFIER_NAME_LENGTH = 63 + def self.check_constraint_exists?(table, constraint_name, connection:) + # Constraint names are unique per table in Postgres, not per schema + # Two tables can have constraints with the same name, so we filter by + # the table name in addition to using the constraint_name + + check_sql = <<~SQL + SELECT COUNT(*) + FROM pg_catalog.pg_constraint con + INNER JOIN pg_catalog.pg_class rel + ON rel.oid = con.conrelid + INNER JOIN pg_catalog.pg_namespace nsp + ON nsp.oid = con.connamespace + WHERE con.contype = 'c' + AND con.conname = #{connection.quote(constraint_name)} + AND nsp.nspname = #{connection.quote(connection.current_schema)} + AND rel.relname = #{connection.quote(table)} + SQL + + connection.select_value(check_sql.squish) > 0 + end + # Returns the name for a check constraint # # type: @@ -29,24 +50,7 @@ module Gitlab end def check_constraint_exists?(table, constraint_name) - # Constraint names are unique per table in Postgres, not per schema - # Two tables can have constraints with the same name, so we filter by - # the table name in addition to using the constraint_name - - check_sql = <<~SQL - SELECT COUNT(*) - FROM pg_catalog.pg_constraint con - INNER JOIN pg_catalog.pg_class rel - ON rel.oid = con.conrelid - INNER JOIN pg_catalog.pg_namespace nsp - ON nsp.oid = con.connamespace - WHERE con.contype = 'c' - AND con.conname = #{connection.quote(constraint_name)} - AND nsp.nspname = #{connection.quote(current_schema)} - AND rel.relname = #{connection.quote(table)} - SQL - - connection.select_value(check_sql) > 0 + ConstraintsHelpers.check_constraint_exists?(table, constraint_name, connection: connection) end # Adds a check constraint to a table diff --git a/lib/gitlab/database/migrations/test_batched_background_runner.rb b/lib/gitlab/database/migrations/test_batched_background_runner.rb index 01fdba22c19..af853c933ba 100644 --- a/lib/gitlab/database/migrations/test_batched_background_runner.rb +++ b/lib/gitlab/database/migrations/test_batched_background_runner.rb @@ -27,7 +27,7 @@ module Gitlab table_max_value = define_batchable_model(migration.table_name, connection: connection) .maximum(migration.column_name) - largest_batch_start = table_max_value - migration.batch_size + largest_batch_start = [table_max_value - migration.batch_size, smallest_batch_start].max # variance is the portion of the batch range that we shrink between variance * 0 and variance * 1 # to pick actual batches to sample. diff --git a/lib/gitlab/database/partitioning.rb b/lib/gitlab/database/partitioning.rb index 6314aff9914..4a9e002a1a2 100644 --- a/lib/gitlab/database/partitioning.rb +++ b/lib/gitlab/database/partitioning.rb @@ -37,8 +37,9 @@ module Gitlab models_to_sync.each do |model| next if model < ::Gitlab::Database::SharedModel && !(model < TableWithoutModel) + model_connection_name = model.connection_db_config.name Gitlab::Database::EachDatabase.each_database_connection do |connection, connection_name| - if connection_name != model.connection_db_config.name + if connection_name != model_connection_name PartitionManager.new(model, connection: connection).sync_partitions end end diff --git a/lib/gitlab/database/partitioning/ci_sliding_list_strategy.rb b/lib/gitlab/database/partitioning/ci_sliding_list_strategy.rb new file mode 100644 index 00000000000..69a69091b5c --- /dev/null +++ b/lib/gitlab/database/partitioning/ci_sliding_list_strategy.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Partitioning + class CiSlidingListStrategy < SlidingListStrategy + def initial_partition + partition_for(100) + end + + def next_partition + partition_for(active_partition.value + 1) + end + + def validate_and_fix; end + + def after_adding_partitions; end + + def extra_partitions + [] + end + + private + + def ensure_partitioning_column_ignored_or_readonly!; end + + def partition_for(value) + SingleNumericListPartition.new(table_name, value, partition_name: partition_name(value)) + end + + def partition_name(value) + [ + table_name.to_s.delete_prefix('p_'), + value + ].join('_') + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning/partition_manager.rb b/lib/gitlab/database/partitioning/partition_manager.rb index 55ca9ff8645..124fae582d3 100644 --- a/lib/gitlab/database/partitioning/partition_manager.rb +++ b/lib/gitlab/database/partitioning/partition_manager.rb @@ -34,6 +34,8 @@ module Gitlab create(partitions_to_create) unless partitions_to_create.empty? detach(partitions_to_detach) unless partitions_to_detach.empty? end + rescue ArgumentError => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) rescue StandardError => e Gitlab::AppLogger.error( message: "Failed to create / detach partition(s)", diff --git a/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb index 8849191f356..7d9c12d776e 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb @@ -32,46 +32,75 @@ module Gitlab # column - The name of the column to create the foreign key on. # on_delete - The action to perform when associated data is removed, # defaults to "CASCADE". + # on_update - The action to perform when associated data is updated, + # no default value is set. # name - The name of the foreign key. + # validate - Flag that controls whether the new foreign key will be + # validated after creation and if it will be added on the parent table. + # If the flag is not set, the constraint will only be enforced for new data + # in the existing partitions. The helper will need to be called again + # with the flag set to `true` to add the foreign key on the parent table + # after validating it on all partitions. + # `validate: false` should be paired with `prepare_partitioned_async_foreign_key_validation` + # reverse_lock_order - Flag that controls whether we should attempt to acquire locks in the reverse + # order of the ALTER TABLE. This can be useful in situations where the foreign + # key creation could deadlock with another process. # - def add_concurrent_partitioned_foreign_key(source, target, column:, on_delete: :cascade, name: nil) + def add_concurrent_partitioned_foreign_key(source, target, column:, **options) assert_not_in_transaction_block(scope: ERROR_SCOPE) - partition_options = { - column: column, - on_delete: on_delete, + options.reverse_merge!({ + target_column: :id, + on_delete: :cascade, + on_update: nil, + name: nil, + validate: true, + reverse_lock_order: false, + column: column + }) - # We'll use the same FK name for all partitions and match it to - # the name used for the partitioned table to follow the convention - # used by PostgreSQL when adding FKs to new partitions - name: name.presence || concurrent_partitioned_foreign_key_name(source, column), + # We'll use the same FK name for all partitions and match it to + # the name used for the partitioned table to follow the convention + # used by PostgreSQL when adding FKs to new partitions + options[:name] ||= concurrent_partitioned_foreign_key_name(source, column) + check_options = options.slice(:column, :on_delete, :on_update, :name) + check_options[:primary_key] = options[:target_column] - # Force the FK validation to true for partitions (and the partitioned table) - validate: true - } - - if foreign_key_exists?(source, target, **partition_options) + if foreign_key_exists?(source, target, **check_options) warning_message = "Foreign key not created because it exists already " \ "(this may be due to an aborted migration or similar): " \ - "source: #{source}, target: #{target}, column: #{partition_options[:column]}, "\ - "name: #{partition_options[:name]}, on_delete: #{partition_options[:on_delete]}" + "source: #{source}, target: #{target}, column: #{options[:column]}, "\ + "name: #{options[:name]}, on_delete: #{options[:on_delete]}, "\ + "on_update: #{options[:on_update]}" Gitlab::AppLogger.warn warning_message return end - partitioned_table = find_partitioned_table(source) - - partitioned_table.postgres_partitions.order(:name).each do |partition| - add_concurrent_foreign_key(partition.identifier, target, **partition_options) + Gitlab::Database::PostgresPartitionedTable.each_partition(source) do |partition| + add_concurrent_foreign_key(partition.identifier, target, **options) end - with_lock_retries do - add_foreign_key(source, target, **partition_options) + # If we are to add the FK on the parent table now, it will trigger + # the validation on all partitions. The helper must be called one + # more time with `validate: true` after the FK is valid on all partitions. + return unless options[:validate] + + options[:allow_partitioned] = true + add_concurrent_foreign_key(source, target, **options) + end + + def validate_partitioned_foreign_key(source, column, name: nil) + assert_not_in_transaction_block(scope: ERROR_SCOPE) + + Gitlab::Database::PostgresPartitionedTable.each_partition(source) do |partition| + validate_foreign_key(partition.identifier, column, name: name) end end + private + # Returns the name for a concurrent partitioned foreign key. # # Similar to concurrent_foreign_key_name (Gitlab::Database::MigrationHelpers) diff --git a/lib/gitlab/database/postgres_foreign_key.rb b/lib/gitlab/database/postgres_foreign_key.rb index 04ef574a451..28044b42f44 100644 --- a/lib/gitlab/database/postgres_foreign_key.rb +++ b/lib/gitlab/database/postgres_foreign_key.rb @@ -38,6 +38,14 @@ module Gitlab scope :by_constrained_table_name, ->(name) { where(constrained_table_name: name) } + scope :by_constrained_table_name_or_identifier, ->(name) do + if name =~ Database::FULLY_QUALIFIED_IDENTIFIER + by_constrained_table_identifier(name) + else + by_constrained_table_name(name) + end + end + scope :not_inherited, -> { where(is_inherited: false) } scope :by_name, ->(name) { where(name: name) } diff --git a/lib/gitlab/database/postgres_partition.rb b/lib/gitlab/database/postgres_partition.rb index 36dc6818157..e63c6fc86ea 100644 --- a/lib/gitlab/database/postgres_partition.rb +++ b/lib/gitlab/database/postgres_partition.rb @@ -7,6 +7,8 @@ module Gitlab belongs_to :postgres_partitioned_table, foreign_key: 'parent_identifier', primary_key: 'identifier' + # identifier includes the partition schema. + # For example 'gitlab_partitions_static.events_03', or 'gitlab_partitions_dynamic.logs_03' scope :for_identifier, ->(identifier) do unless identifier =~ Gitlab::Database::FULLY_QUALIFIED_IDENTIFIER raise ArgumentError, "Partition name is not fully qualified with a schema: #{identifier}" @@ -19,8 +21,12 @@ module Gitlab for_identifier(identifier).first! end - scope :for_parent_table, ->(name) do - where("parent_identifier = concat(current_schema(), '.', ?)", name).order(:name) + scope :for_parent_table, ->(parent_table) do + if parent_table =~ Database::FULLY_QUALIFIED_IDENTIFIER + where(parent_identifier: parent_table).order(:name) + else + where("parent_identifier = concat(current_schema(), '.', ?)", parent_table).order(:name) + end end def self.partition_exists?(table_name) diff --git a/lib/gitlab/database/reindexing.rb b/lib/gitlab/database/reindexing.rb index 78de7161a0f..739e573b6c4 100644 --- a/lib/gitlab/database/reindexing.rb +++ b/lib/gitlab/database/reindexing.rb @@ -28,7 +28,7 @@ module Gitlab # Hack: Before we do actual reindexing work, create async indexes Gitlab::Database::AsyncIndexes.create_pending_indexes! if Feature.enabled?(:database_async_index_creation, type: :ops) Gitlab::Database::AsyncIndexes.drop_pending_indexes! - Gitlab::Database::AsyncForeignKeys.validate_pending_entries! if Feature.enabled?(:database_async_foreign_key_validation, type: :ops) + Gitlab::Database::AsyncConstraints.validate_pending_entries! if Feature.enabled?(:database_async_foreign_key_validation, type: :ops) automatic_reindexing end diff --git a/lib/gitlab/database/schema_validation/database.rb b/lib/gitlab/database/schema_validation/database.rb index dfc845f0b44..07bd02e58e1 100644 --- a/lib/gitlab/database/schema_validation/database.rb +++ b/lib/gitlab/database/schema_validation/database.rb @@ -4,6 +4,8 @@ module Gitlab module Database module SchemaValidation class Database + STATIC_PARTITIONS_SCHEMA = 'gitlab_partitions_static' + def initialize(connection) @connection = connection end @@ -12,29 +14,69 @@ module Gitlab index_map[index_name] end + def fetch_trigger_by_name(trigger_name) + trigger_map[trigger_name] + end + + def index_exists?(index_name) + index_map[index_name].present? + end + + def trigger_exists?(trigger_name) + trigger_map[trigger_name].present? + end + def indexes index_map.values end + def triggers + trigger_map.values + end + private + attr_reader :connection + + def schemas + @schemas ||= [STATIC_PARTITIONS_SCHEMA, connection.current_schema] + end + def index_map @index_map ||= fetch_indexes.transform_values! do |index_stmt| - Index.new(PgQuery.parse(index_stmt).tree.stmts.first.stmt.index_stmt) + SchemaObjects::Index.new(PgQuery.parse(index_stmt).tree.stmts.first.stmt.index_stmt) end end - attr_reader :connection + def trigger_map + @trigger_map ||= + fetch_triggers.transform_values! do |trigger_stmt| + SchemaObjects::Trigger.new(PgQuery.parse(trigger_stmt).tree.stmts.first.stmt.create_trig_stmt) + end + end def fetch_indexes sql = <<~SQL SELECT indexname, indexdef FROM pg_indexes - WHERE indexname NOT LIKE '%_pkey' AND schemaname IN ('public', 'gitlab_partitions_static'); + WHERE indexname NOT LIKE '%_pkey' AND schemaname IN ($1, $2); + SQL + + connection.select_rows(sql, nil, schemas).to_h + end + + def fetch_triggers + sql = <<~SQL + SELECT triggers.tgname, pg_get_triggerdef(triggers.oid) + FROM pg_catalog.pg_trigger triggers + INNER JOIN pg_catalog.pg_class rel ON triggers.tgrelid = rel.oid + INNER JOIN pg_catalog.pg_namespace nsp ON nsp.oid = rel.relnamespace + WHERE triggers.tgisinternal IS FALSE + AND nsp.nspname IN ($1, $2) SQL - @fetch_indexes ||= connection.exec_query(sql).rows.to_h + connection.select_rows(sql, nil, schemas).to_h end end end diff --git a/lib/gitlab/database/schema_validation/index.rb b/lib/gitlab/database/schema_validation/index.rb deleted file mode 100644 index af0d5f31f4e..00000000000 --- a/lib/gitlab/database/schema_validation/index.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - class Index - def initialize(parsed_stmt) - @parsed_stmt = parsed_stmt - end - - def name - parsed_stmt.idxname - end - - def statement - @statement ||= PgQuery.deparse_stmt(parsed_stmt) - end - - private - - attr_reader :parsed_stmt - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/indexes.rb b/lib/gitlab/database/schema_validation/indexes.rb deleted file mode 100644 index b7c3705bde9..00000000000 --- a/lib/gitlab/database/schema_validation/indexes.rb +++ /dev/null @@ -1,37 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - class Indexes - def initialize(structure_sql, database) - @structure_sql = structure_sql - @database = database - end - - def missing_indexes - structure_sql.indexes.map(&:name) - database.indexes.map(&:name) - end - - def extra_indexes - database.indexes.map(&:name) - structure_sql.indexes.map(&:name) - end - - def wrong_indexes - structure_sql.indexes.filter_map do |structure_sql_index| - database_index = database.fetch_index_by_name(structure_sql_index.name) - - next if database_index.nil? - next if database_index.statement == structure_sql_index.statement - - structure_sql_index.name - end - end - - private - - attr_reader :structure_sql, :database - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/runner.rb b/lib/gitlab/database/schema_validation/runner.rb new file mode 100644 index 00000000000..7a02c8a16d6 --- /dev/null +++ b/lib/gitlab/database/schema_validation/runner.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + class Runner + def initialize(structure_sql, database, validators: Validators::BaseValidator.all_validators) + @structure_sql = structure_sql + @database = database + @validators = validators + end + + def execute + validators.flat_map { |c| c.new(structure_sql, database).execute } + end + + private + + attr_reader :structure_sql, :database, :validators + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/schema_objects/base.rb b/lib/gitlab/database/schema_validation/schema_objects/base.rb new file mode 100644 index 00000000000..b0c8eb087dd --- /dev/null +++ b/lib/gitlab/database/schema_validation/schema_objects/base.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + module SchemaObjects + class Base + def initialize(parsed_stmt) + @parsed_stmt = parsed_stmt + end + + def name + raise NoMethodError, "subclasses of #{self.class.name} must implement #{__method__}" + end + + def statement + @statement ||= PgQuery.deparse_stmt(parsed_stmt) + end + + private + + attr_reader :parsed_stmt + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/schema_objects/index.rb b/lib/gitlab/database/schema_validation/schema_objects/index.rb new file mode 100644 index 00000000000..28d61b18266 --- /dev/null +++ b/lib/gitlab/database/schema_validation/schema_objects/index.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + module SchemaObjects + class Index < Base + def name + parsed_stmt.idxname + end + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/schema_objects/trigger.rb b/lib/gitlab/database/schema_validation/schema_objects/trigger.rb new file mode 100644 index 00000000000..508e6b27ed3 --- /dev/null +++ b/lib/gitlab/database/schema_validation/schema_objects/trigger.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + module SchemaObjects + class Trigger < Base + def name + parsed_stmt.trigname + end + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/structure_sql.rb b/lib/gitlab/database/schema_validation/structure_sql.rb index 32c69a0e5e7..cb62af8d8b8 100644 --- a/lib/gitlab/database/schema_validation/structure_sql.rb +++ b/lib/gitlab/database/schema_validation/structure_sql.rb @@ -4,29 +4,56 @@ module Gitlab module Database module SchemaValidation class StructureSql - def initialize(structure_file_path) + DEFAULT_SCHEMA = 'public' + + def initialize(structure_file_path, schema_name = DEFAULT_SCHEMA) @structure_file_path = structure_file_path + @schema_name = schema_name + end + + def index_exists?(index_name) + indexes.find { |index| index.name == index_name }.present? + end + + def trigger_exists?(trigger_name) + triggers.find { |trigger| trigger.name == trigger_name }.present? end def indexes - @indexes ||= index_statements.map do |index_statement| - index_statement.relation.schemaname = "public" if index_statement.relation.schemaname == '' + @indexes ||= map_with_default_schema(index_statements, SchemaObjects::Index) + end - Index.new(index_statement) - end + def triggers + @triggers ||= map_with_default_schema(trigger_statements, SchemaObjects::Trigger) end private - attr_reader :structure_file_path + attr_reader :structure_file_path, :schema_name def index_statements - parsed_structure_file.tree.stmts.filter_map { |s| s.stmt.index_stmt } + statements.filter_map { |s| s.stmt.index_stmt } + end + + def trigger_statements + statements.filter_map { |s| s.stmt.create_trig_stmt } + end + + def statements + @statements ||= parsed_structure_file.tree.stmts end def parsed_structure_file PgQuery.parse(File.read(structure_file_path)) end + + def map_with_default_schema(statements, validation_class) + statements.map do |statement| + statement.relation.schemaname = schema_name if statement.relation.schemaname == '' + + validation_class.new(statement) + end + end end end end diff --git a/lib/gitlab/database/schema_validation/validators/base_validator.rb b/lib/gitlab/database/schema_validation/validators/base_validator.rb new file mode 100644 index 00000000000..14995b5f378 --- /dev/null +++ b/lib/gitlab/database/schema_validation/validators/base_validator.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + module Validators + class BaseValidator + Inconsistency = Struct.new(:type, :object_name, :statement) + + def initialize(structure_sql, database) + @structure_sql = structure_sql + @database = database + end + + def self.all_validators + [ + ExtraIndexes, + ExtraTriggers, + MissingIndexes, + MissingTriggers, + DifferentDefinitionIndexes, + DifferentDefinitionTriggers + ] + end + + def execute + raise NoMethodError, "subclasses of #{self.class.name} must implement #{__method__}" + end + + private + + attr_reader :structure_sql, :database + + def build_inconsistency(validator_class, schema_object) + inconsistency_type = validator_class.name.demodulize.underscore + + Inconsistency.new(inconsistency_type, schema_object.name, schema_object.statement) + end + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/validators/different_definition_indexes.rb b/lib/gitlab/database/schema_validation/validators/different_definition_indexes.rb new file mode 100644 index 00000000000..d54b62ac1e7 --- /dev/null +++ b/lib/gitlab/database/schema_validation/validators/different_definition_indexes.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + module Validators + class DifferentDefinitionIndexes < BaseValidator + def execute + structure_sql.indexes.filter_map do |structure_sql_index| + database_index = database.fetch_index_by_name(structure_sql_index.name) + + next if database_index.nil? + next if database_index.statement == structure_sql_index.statement + + build_inconsistency(self.class, structure_sql_index) + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/validators/different_definition_triggers.rb b/lib/gitlab/database/schema_validation/validators/different_definition_triggers.rb new file mode 100644 index 00000000000..efb87a70ca8 --- /dev/null +++ b/lib/gitlab/database/schema_validation/validators/different_definition_triggers.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + module Validators + class DifferentDefinitionTriggers < BaseValidator + def execute + structure_sql.triggers.filter_map do |structure_sql_trigger| + database_trigger = database.fetch_trigger_by_name(structure_sql_trigger.name) + + next if database_trigger.nil? + next if database_trigger.statement == structure_sql_trigger.statement + + build_inconsistency(self.class, structure_sql_trigger) + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/validators/extra_indexes.rb b/lib/gitlab/database/schema_validation/validators/extra_indexes.rb new file mode 100644 index 00000000000..28384dd7cee --- /dev/null +++ b/lib/gitlab/database/schema_validation/validators/extra_indexes.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + module Validators + class ExtraIndexes < BaseValidator + def execute + database.indexes.filter_map do |index| + next if structure_sql.index_exists?(index.name) + + build_inconsistency(self.class, index) + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/validators/extra_triggers.rb b/lib/gitlab/database/schema_validation/validators/extra_triggers.rb new file mode 100644 index 00000000000..f03bb49526c --- /dev/null +++ b/lib/gitlab/database/schema_validation/validators/extra_triggers.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + module Validators + class ExtraTriggers < BaseValidator + def execute + database.triggers.filter_map do |trigger| + next if structure_sql.trigger_exists?(trigger.name) + + build_inconsistency(self.class, trigger) + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/validators/missing_indexes.rb b/lib/gitlab/database/schema_validation/validators/missing_indexes.rb new file mode 100644 index 00000000000..ac0ea0152ba --- /dev/null +++ b/lib/gitlab/database/schema_validation/validators/missing_indexes.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + module Validators + class MissingIndexes < BaseValidator + def execute + structure_sql.indexes.filter_map do |index| + next if database.index_exists?(index.name) + + build_inconsistency(self.class, index) + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/validators/missing_triggers.rb b/lib/gitlab/database/schema_validation/validators/missing_triggers.rb new file mode 100644 index 00000000000..c7137c68c1c --- /dev/null +++ b/lib/gitlab/database/schema_validation/validators/missing_triggers.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + module Validators + class MissingTriggers < BaseValidator + def execute + structure_sql.triggers.filter_map do |index| + next if database.trigger_exists?(index.name) + + build_inconsistency(self.class, index) + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/tables_locker.rb b/lib/gitlab/database/tables_locker.rb index c417ce716e8..42a2c5c02f7 100644 --- a/lib/gitlab/database/tables_locker.rb +++ b/lib/gitlab/database/tables_locker.rb @@ -16,11 +16,13 @@ module Gitlab # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/366834 next if schema_name.in? GITLAB_SCHEMAS_TO_IGNORE - lock_writes_manager(table_name, connection, database_name).unlock_writes + unlock_writes_on_table(table_name, connection, database_name) end end end + # It locks the tables on the database where they don't belong. Also it unlocks the tables + # on the database where they belong def lock_writes Gitlab::Database::EachDatabase.each_database_connection(include_shared: false) do |connection, database_name| schemas_for_connection = Gitlab::Database.gitlab_schemas_for_connection(connection) @@ -30,9 +32,9 @@ module Gitlab next if schema_name.in? GITLAB_SCHEMAS_TO_IGNORE if schemas_for_connection.include?(schema_name) - lock_writes_manager(table_name, connection, database_name).unlock_writes + unlock_writes_on_table(table_name, connection, database_name) else - lock_writes_manager(table_name, connection, database_name).lock_writes + lock_writes_on_table(table_name, connection, database_name) end end end @@ -40,6 +42,24 @@ module Gitlab private + # Unlocks the writes on the table and its partitions + def unlock_writes_on_table(table_name, connection, database_name) + lock_writes_manager(table_name, connection, database_name).unlock_writes + + table_attached_partitions(table_name, connection) do |postgres_partition| + lock_writes_manager(postgres_partition.identifier, connection, database_name).unlock_writes + end + end + + # It locks the writes on the table and its partitions + def lock_writes_on_table(table_name, connection, database_name) + lock_writes_manager(table_name, connection, database_name).lock_writes + + table_attached_partitions(table_name, connection) do |postgres_partition| + lock_writes_manager(postgres_partition.identifier, connection, database_name).lock_writes + end + end + def tables_to_lock(connection, &block) Gitlab::Database::GitlabSchema.tables_to_schema.each(&block) @@ -50,6 +70,14 @@ module Gitlab end end + def table_attached_partitions(table_name, connection, &block) + Gitlab::Database::SharedModel.using_connection(connection) do + break unless Gitlab::Database::PostgresPartitionedTable.find_by_name_in_current_schema(table_name) + + Gitlab::Database::PostgresPartitionedTable.each_partition(table_name, &block) + end + end + def lock_writes_manager(table_name, connection, database_name) Gitlab::Database::LockWritesManager.new( table_name: table_name, diff --git a/lib/gitlab/database_importers/work_items/base_type_importer.rb b/lib/gitlab/database_importers/work_items/base_type_importer.rb index 9796a5905e3..85ac816f712 100644 --- a/lib/gitlab/database_importers/work_items/base_type_importer.rb +++ b/lib/gitlab/database_importers/work_items/base_type_importer.rb @@ -18,7 +18,8 @@ module Gitlab progress: 'Progress', status: 'Status', requirement_legacy: 'Requirement legacy', - test_reports: 'Test reports' + test_reports: 'Test reports', + notifications: 'Notifications' }.freeze WIDGETS_FOR_TYPE = { @@ -32,23 +33,27 @@ module Gitlab :notes, :iteration, :weight, - :health_status + :health_status, + :notifications ], incident: [ :description, :hierarchy, - :notes + :notes, + :notifications ], test_case: [ :description, - :notes + :notes, + :notifications ], requirement: [ :description, :notes, :status, :requirement_legacy, - :test_reports + :test_reports, + :notifications ], task: [ :assignees, @@ -59,7 +64,8 @@ module Gitlab :milestone, :notes, :iteration, - :weight + :weight, + :notifications ], objective: [ :assignees, @@ -69,7 +75,8 @@ module Gitlab :milestone, :notes, :health_status, - :progress + :progress, + :notifications ], key_result: [ :assignees, @@ -79,7 +86,8 @@ module Gitlab :start_and_due_date, :notes, :health_status, - :progress + :progress, + :notifications ] }.freeze diff --git a/lib/gitlab/design_management/copy_design_collection_model_attributes.yml b/lib/gitlab/design_management/copy_design_collection_model_attributes.yml index 95f15bd6dee..fe1baeb7b67 100644 --- a/lib/gitlab/design_management/copy_design_collection_model_attributes.yml +++ b/lib/gitlab/design_management/copy_design_collection_model_attributes.yml @@ -16,6 +16,7 @@ design_attributes: - filename - relative_position + - description version_attributes: - author_id @@ -30,6 +31,8 @@ ignore_design_attributes: - issue_id - project_id - iid + - description_html + - cached_markdown_version ignore_version_attributes: - id diff --git a/lib/gitlab/doorkeeper_secret_storing/secret/pbkdf2_sha512.rb b/lib/gitlab/doorkeeper_secret_storing/secret/pbkdf2_sha512.rb index e0884557496..0624fe934f9 100644 --- a/lib/gitlab/doorkeeper_secret_storing/secret/pbkdf2_sha512.rb +++ b/lib/gitlab/doorkeeper_secret_storing/secret/pbkdf2_sha512.rb @@ -10,9 +10,7 @@ module Gitlab # additional security. SALT = '' - def self.transform_secret(plain_secret, stored_as_hash = false) - return plain_secret if Feature.disabled?(:hash_oauth_secrets) && !stored_as_hash - + def self.transform_secret(plain_secret) Devise::Pbkdf2Encryptable::Encryptors::Pbkdf2Sha512.digest(plain_secret, STRETCHES, SALT) end @@ -28,8 +26,7 @@ module Gitlab # Securely compare the given +input+ value with a +stored+ value # processed by +transform_secret+. def self.secret_matches?(input, stored) - stored_as_hash = stored.starts_with?('$pbkdf2-') - transformed_input = transform_secret(input, stored_as_hash) + transformed_input = transform_secret(input) ActiveSupport::SecurityUtils.secure_compare transformed_input, stored end end diff --git a/lib/gitlab/email/html_to_markdown_parser.rb b/lib/gitlab/email/html_to_markdown_parser.rb index 42dd012308b..5dd3725cc3e 100644 --- a/lib/gitlab/email/html_to_markdown_parser.rb +++ b/lib/gitlab/email/html_to_markdown_parser.rb @@ -5,25 +5,46 @@ require 'nokogiri' module Gitlab module Email class HtmlToMarkdownParser < Html2Text - ADDITIONAL_TAGS = %w[em strong img details].freeze - IMG_ATTRS = %w[alt src].freeze + extend Gitlab::Utils::Override + # List of tags to be converted by Markdown. + # + # All attributes are removed except for the defined ones. + # + # <tag> => [<attribute to keep>, ...] + ALLOWED_TAG_ATTRIBUTES = { + 'em' => [], + 'strong' => [], + 'details' => [], + 'img' => %w[alt src] + }.freeze + private_constant :ALLOWED_TAG_ATTRIBUTES + + # This redefinition can be removed once https://github.com/soundasleep/html2text_ruby/pull/30 + # is merged and released. def self.convert(html) html = fix_newlines(replace_entities(html)) doc = Nokogiri::HTML(html) - HtmlToMarkdownParser.new(doc).convert + new(doc).convert end + private + + override :iterate_over def iterate_over(node) - return super unless ADDITIONAL_TAGS.include?(node.name) + allowed_attributes = ALLOWED_TAG_ATTRIBUTES[node.name] + return super unless allowed_attributes - if node.name == 'img' - node.keys.each { |key| node.remove_attribute(key) unless IMG_ATTRS.include?(key) } # rubocop:disable Style/HashEachMethods - end + remove_attributes(node, allowed_attributes) Kramdown::Document.new(node.to_html, input: 'html').to_commonmark end + + def remove_attributes(node, allowed_attributes) + to_remove = (node.keys - allowed_attributes) + to_remove.each { |key| node.remove_attribute(key) } + end end end end diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index 32794a6c99d..664f0a1bb4a 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -177,7 +177,7 @@ module Gitlab def recipients_from_received_headers strong_memoize :emails_from_received_headers do - received.map { |header| header.value[RECEIVED_HEADER_REGEX, 1] }.compact + received.filter_map { |header| header.value[RECEIVED_HEADER_REGEX, 1] } end end diff --git a/lib/gitlab/etag_caching/router/graphql.rb b/lib/gitlab/etag_caching/router/graphql.rb index 7a0fb2ac269..b53164ac94c 100644 --- a/lib/gitlab/etag_caching/router/graphql.rb +++ b/lib/gitlab/etag_caching/router/graphql.rb @@ -16,7 +16,7 @@ module Gitlab [ %r(\Apipelines/sha/\w{7,40}\z), 'ci_editor', - 'pipeline_authoring' + 'pipeline_composition' ], [ %r(\Aon_demand_scan/counts/), diff --git a/lib/gitlab/exception_log_formatter.rb b/lib/gitlab/exception_log_formatter.rb index ce802b562f0..52ad67d6f8b 100644 --- a/lib/gitlab/exception_log_formatter.rb +++ b/lib/gitlab/exception_log_formatter.rb @@ -17,6 +17,10 @@ module Gitlab payload['exception.backtrace'] = Rails.backtrace_cleaner.clean(exception.backtrace) end + if exception.cause + payload['exception.cause_class'] = exception.cause.class.name + end + if sql = find_sql(exception) payload['exception.sql'] = sql end diff --git a/lib/gitlab/file_finder.rb b/lib/gitlab/file_finder.rb index 95f896a74e9..8a894901ca1 100644 --- a/lib/gitlab/file_finder.rb +++ b/lib/gitlab/file_finder.rb @@ -44,15 +44,11 @@ module Gitlab # Overridden in Gitlab::WikiFileFinder def search_paths(query) - if Feature.enabled?(:code_basic_search_files_by_regexp, project) - return [] if query.blank? || ref.blank? - - escaped_query = RE2::Regexp.escape(query) - query_regexp = Gitlab::EncodingHelper.encode_utf8_no_detect("(?i)#{escaped_query}") - repository.search_files_by_regexp(query_regexp, ref) - else - repository.search_files_by_name(query, ref) - end + return [] if query.blank? || ref.blank? + + escaped_query = RE2::Regexp.escape(query) + query_regexp = Gitlab::EncodingHelper.encode_utf8_no_detect("(?i)#{escaped_query}") + repository.search_files_by_regexp(query_regexp, ref) end end end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 8e1b51fcec5..eb204a7dd8e 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_dependency 'gitlab/encoding_helper' +require_relative 'encoding_helper' module Gitlab module Git diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 267107e04e6..3a65c7c334d 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -16,7 +16,7 @@ module Gitlab SERIALIZE_KEYS = [ :id, :message, :parent_ids, :authored_date, :author_name, :author_email, - :committed_date, :committer_name, :committer_email, :trailers + :committed_date, :committer_name, :committer_email, :trailers, :referenced_by ].freeze attr_accessor(*SERIALIZE_KEYS) @@ -414,6 +414,7 @@ module Gitlab @committer_email = commit.committer.email.dup @parent_ids = Array(commit.parent_ids) @trailers = commit.trailers.to_h { |t| [t.key, t.value] } + @referenced_by = Array(commit.referenced_by) end # Gitaly provides a UNIX timestamp in author.date.seconds, and a timezone diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index 0ffe8bee953..b4dd880ceb7 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -24,6 +24,8 @@ module Gitlab limits[:safe_max_lines] = [limits[:max_lines], defaults[:max_lines]].min limits[:safe_max_bytes] = limits[:safe_max_files] * 5.kilobytes # Average 5 KB per file limits[:max_patch_bytes] = Gitlab::Git::Diff.patch_hard_limit_bytes + limits[:max_patch_bytes_for_file_extension] = options.fetch(:max_patch_bytes_for_file_extension, {}) + limits end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index e054b6df98f..95633400aee 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -803,27 +803,17 @@ module Gitlab end end - def license(from_gitaly) + def license wrapped_gitaly_errors do response = gitaly_repository_client.find_license break nil if response.license_short_name.empty? - if from_gitaly - break ::Gitlab::Git::DeclaredLicense.new(key: response.license_short_name, - name: response.license_name, - nickname: response.license_nickname.presence, - url: response.license_url.presence, - path: response.license_path) - end - - licensee_object = Licensee::License.new(response.license_short_name) - - break nil if licensee_object.name.blank? - - licensee_object.meta.nickname = "LICENSE" if licensee_object.key == "other" - - licensee_object + ::Gitlab::Git::DeclaredLicense.new(key: response.license_short_name, + name: response.license_name, + nickname: response.license_nickname.presence, + url: response.license_url.presence, + path: response.license_path) end rescue Licensee::InvalidLicense => e Gitlab::ErrorTracking.track_exception(e) diff --git a/lib/gitlab/git/rugged_impl/tree.rb b/lib/gitlab/git/rugged_impl/tree.rb index 66cfc02130b..c7a981c7dd4 100644 --- a/lib/gitlab/git/rugged_impl/tree.rb +++ b/lib/gitlab/git/rugged_impl/tree.rb @@ -130,7 +130,6 @@ module Gitlab new( id: entry[:oid], - root_id: root_tree.oid, name: entry[:name], type: entry[:type], mode: entry[:filemode].to_s(8), diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index f0eef619e13..e437f99dab3 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -6,7 +6,7 @@ module Gitlab include Gitlab::EncodingHelper extend Gitlab::Git::WrapsGitalyErrors - attr_accessor :id, :root_id, :type, :mode, :commit_id, :submodule_url + attr_accessor :id, :type, :mode, :commit_id, :submodule_url attr_writer :name, :path, :flat_path class << self @@ -61,7 +61,7 @@ module Gitlab end def initialize(options) - %w(id root_id name path flat_path type mode commit_id).each do |key| + %w(id name path flat_path type mode commit_id).each do |key| self.send("#{key}=", options[key.to_sym]) # rubocop:disable GitlabSecurity/PublicSend end end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 4df9d800ea6..b7f2d7d3e11 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -146,7 +146,6 @@ module Gitlab message.entries.map do |gitaly_tree_entry| Gitlab::Git::Tree.new( id: gitaly_tree_entry.oid, - root_id: gitaly_tree_entry.root_oid, type: gitaly_tree_entry.type.downcase, mode: gitaly_tree_entry.mode.to_s(8), name: File.basename(gitaly_tree_entry.path), @@ -423,7 +422,8 @@ module Gitlab first_parent: !!options[:first_parent], global_options: parse_global_options!(options), disable_walk: true, # This option is deprecated. The 'walk' implementation is being removed. - trailers: options[:trailers] + trailers: options[:trailers], + include_referenced_by: options[:include_referenced_by] ) request.after = GitalyClient.timestamp(options[:after]) if options[:after] request.before = GitalyClient.timestamp(options[:before]) if options[:before] diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index ac6491e8770..525d7064dae 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -239,7 +239,7 @@ module Gitlab sort_by = 'name' if sort_by == 'name_asc' enum_value = Gitaly::FindLocalBranchesRequest::SortBy.resolve(sort_by.upcase.to_sym) - raise ArgumentError, "Invalid sort_by key `#{sort_by}`" unless enum_value + return Gitaly::FindLocalBranchesRequest::SortBy::NAME unless enum_value enum_value end diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index bcc03ca08c9..93d58710b0c 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -109,7 +109,7 @@ module Gitlab # rubocop: enable Metrics/ParameterLists def create_repository(default_branch = nil) - request = Gitaly::CreateRepositoryRequest.new(repository: @gitaly_repo, default_branch: default_branch) + request = Gitaly::CreateRepositoryRequest.new(repository: @gitaly_repo, default_branch: encode_binary(default_branch)) gitaly_client_call(@storage, :repository_service, :create_repository, request, timeout: GitalyClient.fast_timeout) end @@ -306,18 +306,18 @@ module Gitlab end def search_files_by_name(ref, query, limit: 0, offset: 0) - request = Gitaly::SearchFilesByNameRequest.new(repository: @gitaly_repo, ref: ref, query: query, limit: limit, offset: offset) + request = Gitaly::SearchFilesByNameRequest.new(repository: @gitaly_repo, ref: encode_binary(ref), query: query, limit: limit, offset: offset) gitaly_client_call(@storage, :repository_service, :search_files_by_name, request, timeout: GitalyClient.fast_timeout).flat_map(&:files) end def search_files_by_content(ref, query, options = {}) - request = Gitaly::SearchFilesByContentRequest.new(repository: @gitaly_repo, ref: ref, query: query) + request = Gitaly::SearchFilesByContentRequest.new(repository: @gitaly_repo, ref: encode_binary(ref), query: query) response = gitaly_client_call(@storage, :repository_service, :search_files_by_content, request, timeout: GitalyClient.default_timeout) search_results_from_response(response, options) end def search_files_by_regexp(ref, filter, limit: 0, offset: 0) - request = Gitaly::SearchFilesByNameRequest.new(repository: @gitaly_repo, ref: ref, query: '.', filter: filter, limit: limit, offset: offset) + request = Gitaly::SearchFilesByNameRequest.new(repository: @gitaly_repo, ref: encode_binary(ref), query: '.', filter: filter, limit: limit, offset: offset) gitaly_client_call(@storage, :repository_service, :search_files_by_name, request, timeout: GitalyClient.fast_timeout).flat_map(&:files) end diff --git a/lib/gitlab/github_import/clients/proxy.rb b/lib/gitlab/github_import/clients/proxy.rb index b12df404640..27030f5382a 100644 --- a/lib/gitlab/github_import/clients/proxy.rb +++ b/lib/gitlab/github_import/clients/proxy.rb @@ -6,6 +6,10 @@ module Gitlab class Proxy attr_reader :client + delegate :each_object, :user, :octokit, to: :client + + REPOS_COUNT_CACHE_KEY = 'github-importer/provider-repo-count/%{type}/%{user_id}' + def initialize(access_token, client_options) @client = pick_client(access_token, client_options) end @@ -13,24 +17,26 @@ module Gitlab def repos(search_text, options) return { repos: filtered(client.repos, search_text) } if use_legacy? - if use_graphql? - fetch_repos_via_graphql(search_text, options) - else - fetch_repos_via_rest(search_text, options) - end + fetch_repos_via_graphql(search_text, options) end - private + def count_repos_by(relation_type, user_id) + return if use_legacy? + + key = format(REPOS_COUNT_CACHE_KEY, type: relation_type, user_id: user_id) - def fetch_repos_via_rest(search_text, options) - { repos: client.search_repos_by_name(search_text, options)[:items] } + ::Gitlab::Cache::Import::Caching.read_integer(key, timeout: 5.minutes) || + fetch_and_cache_repos_count_via_graphql(relation_type, key) end + private + def fetch_repos_via_graphql(search_text, options) response = client.search_repos_by_name_graphql(search_text, options) { repos: response.dig(:data, :search, :nodes), - page_info: response.dig(:data, :search, :pageInfo) + page_info: response.dig(:data, :search, :pageInfo), + count: response.dig(:data, :search, :repositoryCount) } end @@ -50,8 +56,11 @@ module Gitlab Feature.disabled?(:remove_legacy_github_client) end - def use_graphql? - Feature.enabled?(:github_client_fetch_repos_via_graphql) + def fetch_and_cache_repos_count_via_graphql(relation_type, key) + response = client.count_repos_by_relation_type_graphql(relation_type: relation_type) + count = response.dig(:data, :search, :repositoryCount) + + ::Gitlab::Cache::Import::Caching.write(key, count, timeout: 5.minutes) end end end diff --git a/lib/gitlab/github_import/clients/search_repos.rb b/lib/gitlab/github_import/clients/search_repos.rb index b72e5ac7751..5e058fc0933 100644 --- a/lib/gitlab/github_import/clients/search_repos.rb +++ b/lib/gitlab/github_import/clients/search_repos.rb @@ -5,24 +5,24 @@ module Gitlab module Clients module SearchRepos def search_repos_by_name_graphql(name, options = {}) - with_retry do - octokit.post( - '/graphql', - { query: graphql_search_repos_body(name, options) }.to_json - ).to_h - end + graphql_request(graphql_search_repos_body(name, options)) + end + + def count_repos_by_relation_type_graphql(options) + graphql_request(count_by_relation_type_query(options)) end - def search_repos_by_name(name, options = {}) - search_query = search_repos_query(name, options) + private + def graphql_request(query) with_retry do - octokit.search_repositories(search_query, options).to_h + octokit.post( + '/graphql', + { query: query }.to_json + ).to_h end end - private - def graphql_search_repos_body(name, options) query = search_repos_query(name, options) query = "query: \"#{query}\"" @@ -45,7 +45,8 @@ module Gitlab endCursor hasNextPage hasPreviousPage - } + }, + repositoryCount } } TEXT @@ -64,7 +65,11 @@ module Gitlab end def organization_repos_query(search_string, options) - "#{search_string} org:#{options[:organization_login]}" + if options[:organization_login].present? + "#{search_string} org:#{options[:organization_login]}" + else + organizations_subquery + end end def collaborated_repos_query(search_string) @@ -95,6 +100,18 @@ module Gitlab .map { |org| "org:#{org[:login]}" } .join(' ') end + + def count_by_relation_type_query(options) + query = search_repos_query(nil, options) + query = "query: \"#{query}\"" + <<-TEXT + { + search(type: REPOSITORY, #{query}) { + repositoryCount + } + } + TEXT + end end end end diff --git a/lib/gitlab/github_import/importer/collaborator_importer.rb b/lib/gitlab/github_import/importer/collaborator_importer.rb new file mode 100644 index 00000000000..9a90ea5a4ed --- /dev/null +++ b/lib/gitlab/github_import/importer/collaborator_importer.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class CollaboratorImporter + attr_reader :collaborator, :project, :client, :members_finder + + # collaborator - An instance of `Gitlab::GithubImport::Representation::Collaborator` + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` + def initialize(collaborator, project, client) + @collaborator = collaborator + @project = project + @client = client + @members_finder = ::MembersFinder.new(project, project.creator) + end + + def execute + user_finder = GithubImport::UserFinder.new(project, client) + user_id = user_finder.user_id_for(collaborator) + return if user_id.nil? + + membership = existing_user_membership(user_id) + access_level = map_access_level + return if membership && membership[:access_level] >= map_access_level + + create_membership!(user_id, access_level) + end + + private + + def existing_user_membership(user_id) + members_finder.execute.find_by_user_id(user_id) + end + + def map_access_level + access_level = + case collaborator[:role_name] + when 'read' then Gitlab::Access::GUEST + when 'triage' then Gitlab::Access::REPORTER + when 'write' then Gitlab::Access::DEVELOPER + when 'maintain' then Gitlab::Access::MAINTAINER + when 'admin' then Gitlab::Access::OWNER + end + return access_level if access_level + + raise( + ::Gitlab::GithubImport::ObjectImporter::NotRetriableError, + "Unknown GitHub role: #{collaborator[:role_name]}" + ) + end + + def create_membership!(user_id, access_level) + ::ProjectMember.create!( + source: project, + access_level: access_level, + user_id: user_id, + member_namespace_id: project.project_namespace_id, + created_by_id: project.creator_id + ) + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/collaborators_importer.rb b/lib/gitlab/github_import/importer/collaborators_importer.rb new file mode 100644 index 00000000000..dd947632d01 --- /dev/null +++ b/lib/gitlab/github_import/importer/collaborators_importer.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class CollaboratorsImporter + include ParallelScheduling + + def importer_class + CollaboratorImporter + end + + def representation_class + Representation::Collaborator + end + + def sidekiq_worker_class + ImportCollaboratorWorker + end + + def object_type + :collaborator + end + + def collection_method + :collaborators + end + + def id_for_already_imported_cache(collaborator) + collaborator[:id] + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/events/cross_referenced.rb b/lib/gitlab/github_import/importer/events/cross_referenced.rb index b56ae186d3c..4fe371e5900 100644 --- a/lib/gitlab/github_import/importer/events/cross_referenced.rb +++ b/lib/gitlab/github_import/importer/events/cross_referenced.rb @@ -55,6 +55,7 @@ module Gitlab record = record_class.new(id: db_id, iid: iid) record.project = project + record.namespace = project.project_namespace if record.respond_to?(:namespace) record.readonly! record end diff --git a/lib/gitlab/github_import/importer/label_links_importer.rb b/lib/gitlab/github_import/importer/label_links_importer.rb index 52c87dda347..a20fec4b2ba 100644 --- a/lib/gitlab/github_import/importer/label_links_importer.rb +++ b/lib/gitlab/github_import/importer/label_links_importer.rb @@ -25,6 +25,8 @@ module Gitlab items = [] target_id = find_target_id + return if target_id.blank? + issue.label_names.each do |label_name| # Although unlikely it's technically possible for an issue to be # given a label that was created and assigned after we imported all diff --git a/lib/gitlab/github_import/importer/note_attachments_importer.rb b/lib/gitlab/github_import/importer/note_attachments_importer.rb index 9901c9e76f5..a84fcd253ef 100644 --- a/lib/gitlab/github_import/importer/note_attachments_importer.rb +++ b/lib/gitlab/github_import/importer/note_attachments_importer.rb @@ -19,7 +19,7 @@ module Gitlab return if attachments.blank? new_text = attachments.reduce(note_text.text) do |text, attachment| - new_url = download_attachment(attachment) + new_url = gitlab_attachment_link(attachment) text.gsub(attachment.url, new_url) end @@ -28,6 +28,28 @@ module Gitlab private + def gitlab_attachment_link(attachment) + project_import_source = project.import_source + + if attachment.part_of_project_blob?(project_import_source) + convert_project_content_link(attachment.url, project_import_source) + elsif attachment.media? || attachment.doc_belongs_to_project?(project_import_source) + download_attachment(attachment) + else # url to other GitHub project + attachment.url + end + end + + # From: https://github.com/login/test-import-attachments-source/blob/main/example.md + # To: https://gitlab.com/login/test-import-attachments-target/-/blob/main/example.md + def convert_project_content_link(attachment_url, import_source) + path_without_domain = attachment_url.gsub(::Gitlab::GithubImport::MarkdownText.github_url, '') + path_without_import_source = path_without_domain.gsub(import_source, '').delete_prefix('/') + path_with_blob_prefix = "/-#{path_without_import_source}" + + ::Gitlab::Routing.url_helpers.project_url(project) + path_with_blob_prefix + end + # in: an instance of Gitlab::GithubImport::Markdown::Attachment # out: gitlab attachment markdown url def download_attachment(attachment) diff --git a/lib/gitlab/github_import/importer/pull_requests/review_request_importer.rb b/lib/gitlab/github_import/importer/pull_requests/review_request_importer.rb index bb51d856d9b..1da99942cf6 100644 --- a/lib/gitlab/github_import/importer/pull_requests/review_request_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests/review_request_importer.rb @@ -20,7 +20,7 @@ module Gitlab attr_reader :review_request, :user_finder def build_reviewers - reviewer_ids = review_request.users.map { |user| user_finder.user_id_for(user) }.compact + reviewer_ids = review_request.users.filter_map { |user| user_finder.user_id_for(user) } reviewer_ids.map do |reviewer_id| MergeRequestReviewer.new( diff --git a/lib/gitlab/github_import/importer/repository_importer.rb b/lib/gitlab/github_import/importer/repository_importer.rb index d7fe01e90f8..2654812b64a 100644 --- a/lib/gitlab/github_import/importer/repository_importer.rb +++ b/lib/gitlab/github_import/importer/repository_importer.rb @@ -66,13 +66,10 @@ module Gitlab true rescue ::Gitlab::Git::CommandError => e - if e.message !~ /repository not exported/ - project.create_wiki + return true if e.message.include?('repository not exported') - raise e - else - true - end + project.create_wiki + raise e end def wiki_url @@ -89,10 +86,8 @@ module Gitlab client_repository[:default_branch] end - def client_repository - strong_memoize(:client_repository) do - client.repository(project.import_source) - end + strong_memoize_attr def client_repository + client.repository(project.import_source) end end end diff --git a/lib/gitlab/github_import/markdown/attachment.rb b/lib/gitlab/github_import/markdown/attachment.rb index 1c814e34a39..e270cfba619 100644 --- a/lib/gitlab/github_import/markdown/attachment.rb +++ b/lib/gitlab/github_import/markdown/attachment.rb @@ -79,6 +79,22 @@ module Gitlab @url = url end + def part_of_project_blob?(import_source) + url.start_with?( + "#{::Gitlab::GithubImport::MarkdownText.github_url}/#{import_source}/blob" + ) + end + + def doc_belongs_to_project?(import_source) + url.start_with?( + "#{::Gitlab::GithubImport::MarkdownText.github_url}/#{import_source}/files" + ) + end + + def media? + url.start_with?(::Gitlab::GithubImport::MarkdownText::GITHUB_MEDIA_CDN) + end + def inspect "<#{self.class.name}: { name: #{name}, url: #{url} }>" end diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb index 4b54a77983d..f8d8e4c1e8d 100644 --- a/lib/gitlab/github_import/parallel_scheduling.rb +++ b/lib/gitlab/github_import/parallel_scheduling.rb @@ -85,14 +85,10 @@ module Gitlab def parallel_import raise 'Batch settings must be defined for parallel import' if parallel_import_batch.blank? - if Feature.enabled?(:improved_spread_parallel_import) - improved_spread_parallel_import - else - spread_parallel_import - end + spread_parallel_import end - def improved_spread_parallel_import + def spread_parallel_import enqueued_job_counter = 0 each_object_to_import do |object| @@ -108,33 +104,6 @@ module Gitlab job_waiter end - def spread_parallel_import - waiter = JobWaiter.new - - import_arguments = [] - - each_object_to_import do |object| - repr = object_representation(object) - - import_arguments << [project.id, repr.to_hash, waiter.key] - - waiter.jobs_remaining += 1 - end - - # rubocop:disable Scalability/BulkPerformWithContext - Gitlab::ApplicationContext.with_context(project: project) do - sidekiq_worker_class.bulk_perform_in( - 1.second, - import_arguments, - batch_size: parallel_import_batch[:size], - batch_delay: parallel_import_batch[:delay] - ) - end - # rubocop:enable Scalability/BulkPerformWithContext - - waiter - end - # The method that will be called for traversing through all the objects to # import, yielding them to the supplied block. def each_object_to_import diff --git a/lib/gitlab/github_import/project_relation_type.rb b/lib/gitlab/github_import/project_relation_type.rb new file mode 100644 index 00000000000..a6e598172ee --- /dev/null +++ b/lib/gitlab/github_import/project_relation_type.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + class ProjectRelationType + CACHE_ORGS_EXPIRES_IN = 5.minutes + CACHE_USER_EXPIRES_IN = 1.hour + + def initialize(client) + @client = client + end + + def for(import_source) + namespace = import_source.split('/')[0] + if user?(namespace) + 'owned' + elsif organization?(namespace) + 'organization' + else + 'collaborated' + end + end + + private + + attr_reader :client + + def user?(namespace) + github_user_login == namespace + end + + def organization?(namespace) + github_org_logins.include? namespace + end + + def github_user_login + ::Rails.cache.fetch(cache_key('user_login'), expire_in: CACHE_USER_EXPIRES_IN) do + client.user(nil)[:login] + end + end + + def github_org_logins + ::Rails.cache.fetch(cache_key('organization_logins'), expires_in: CACHE_ORGS_EXPIRES_IN) do + logins = [] + client.each_object(:organizations) { |org| logins.push(org[:login]) } + logins + end + end + + def cache_key(subject) + ['github_import', Gitlab::CryptoHelper.sha256(client.octokit.access_token), subject].join('/') + end + end + end +end diff --git a/lib/gitlab/github_import/representation/collaborator.rb b/lib/gitlab/github_import/representation/collaborator.rb new file mode 100644 index 00000000000..55f13593f4f --- /dev/null +++ b/lib/gitlab/github_import/representation/collaborator.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + class Collaborator + include ToHash + include ExposeAttribute + + attr_reader :attributes + + expose_attribute :id, :login, :role_name + + # Builds a user from a GitHub API response. + # + # collaborator - An instance of `Hash` containing the user & role details. + def self.from_api_response(collaborator, _additional_data = {}) + new( + id: collaborator[:id], + login: collaborator[:login], + role_name: collaborator[:role_name] + ) + end + + # Builds a user using a Hash that was built from a JSON payload. + def self.from_json_hash(raw_hash) + new(Representation.symbolize_hash(raw_hash)) + end + + # attributes - A Hash containing the user details. The keys of this + # Hash (and any nested hashes) must be symbols. + def initialize(attributes) + @attributes = attributes + end + + def github_identifiers + { id: id } + end + end + end + end +end diff --git a/lib/gitlab/github_import/settings.rb b/lib/gitlab/github_import/settings.rb index 77288b9fb98..22ab99df107 100644 --- a/lib/gitlab/github_import/settings.rb +++ b/lib/gitlab/github_import/settings.rb @@ -18,9 +18,9 @@ module Gitlab TEXT }, attachments_import: { - label: 'Import Markdown attachments', + label: 'Import Markdown attachments (links)', details: <<-TEXT.split("\n").map(&:strip).join(' ') - Import Markdown attachments from repository comments, release posts, issue descriptions, + Import Markdown attachments (links) from repository comments, release posts, issue descriptions, and pull request descriptions. These can include images, text, or binary attachments. If not imported, links in Markdown to attachments break after you remove the attachments from GitHub. TEXT diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index c9766ee095a..d7d06aa5271 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -66,7 +66,6 @@ module Gitlab push_frontend_feature_flag(:new_header_search) push_frontend_feature_flag(:source_editor_toolbar) push_frontend_feature_flag(:vscode_web_ide, current_user) - push_frontend_feature_flag(:integration_slack_app_notifications) push_frontend_feature_flag(:full_path_project_search, current_user) end diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 8fe5868ca57..a1b6e937396 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -44,28 +44,28 @@ module Gitlab TRANSLATION_LEVELS = { 'bg' => 0, 'cs_CZ' => 0, - 'da_DK' => 34, - 'de' => 16, + 'da_DK' => 33, + 'de' => 15, 'en' => 100, 'eo' => 0, - 'es' => 33, + 'es' => 32, 'fil_PH' => 0, 'fr' => 99, 'gl_ES' => 0, 'id_ID' => 0, 'it' => 1, 'ja' => 31, - 'ko' => 20, - 'nb_NO' => 23, + 'ko' => 19, + 'nb_NO' => 22, 'nl_NL' => 0, 'pl_PL' => 3, - 'pt_BR' => 57, - 'ro_RO' => 91, - 'ru' => 26, - 'si_LK' => 11, + 'pt_BR' => 56, + 'ro_RO' => 89, + 'ru' => 25, + 'si_LK' => 10, 'tr_TR' => 10, - 'uk' => 55, - 'zh_CN' => 98, + 'uk' => 53, + 'zh_CN' => 96, 'zh_HK' => 1, 'zh_TW' => 98 }.freeze @@ -118,12 +118,17 @@ module Gitlab end def setup(domain:, default_locale:) + custom_pluralization setup_repositories(domain) setup_default_locale(default_locale) end private + def custom_pluralization + Gitlab::I18n::Pluralization.install_on(FastGettext) + end + def setup_repositories(domain) translation_repositories = [ (po_repository(domain, 'jh/locale') if Gitlab.jh?), diff --git a/lib/gitlab/i18n/pluralization.rb b/lib/gitlab/i18n/pluralization.rb new file mode 100644 index 00000000000..5d4a05f1fd1 --- /dev/null +++ b/lib/gitlab/i18n/pluralization.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +module Gitlab + module I18n + # Pluralization formulas per locale used by FastGettext via: + # `FastGettext.pluralisation_rule.call(count)`. + module Pluralization + # rubocop:disable all + MAP = { + "bg" => ->(n) { (n != 1) }, + "cs_CZ" => ->(n) { (n==1) ? 0 : (n>=2 && n<=4) ? 1 : 3 }, + "da_DK" => ->(n) { (n != 1) }, + "de" => ->(n) { (n != 1) }, + "en" => ->(n) { (n != 1) }, + "eo" => ->(n) { (n != 1) }, + "es" => ->(n) { (n != 1) }, + "fil_PH" => ->(n) { (n > 1) }, + "fr" => ->(n) { (n > 1) }, + "gl_ES" => ->(n) { (n != 1) }, + "id_ID" => ->(n) { 0 }, + "it" => ->(n) { (n != 1) }, + "ja" => ->(n) { 0 }, + "ko" => ->(n) { 0 }, + "nb_NO" => ->(n) { (n != 1) }, + "nl_NL" => ->(n) { (n != 1) }, + "pl_PL" => ->(n) { (n==1 ? 0 : (n%10>=2 && n%10<=4) && (n%100<12 || n%100>14) ? 1 : n!=1 && (n%10>=0 && n%10<=1) || (n%10>=5 && n%10<=9) || (n%100>=12 && n%100<=14) ? 2 : 3) }, + "pt_BR" => ->(n) { (n != 1) }, + "ro_RO" => ->(n) { (n==1 ? 0 : (n==0 || (n%100>0 && n%100<20)) ? 1 : 2) }, + "ru" => ->(n) { ((n%10==1 && n%100!=11) ? 0 : ((n%10 >= 2 && n%10 <=4 && (n%100 < 12 || n%100 > 14)) ? 1 : ((n%10 == 0 || (n%10 >= 5 && n%10 <=9)) || (n%100 >= 11 && n%100 <= 14)) ? 2 : 3)) }, + "si_LK" => ->(n) { (n != 1) }, + "tr_TR" => ->(n) { (n != 1) }, + "uk" => ->(n) { ((n%10==1 && n%100!=11) ? 0 : ((n%10 >= 2 && n%10 <=4 && (n%100 < 12 || n%100 > 14)) ? 1 : ((n%10 == 0 || (n%10 >= 5 && n%10 <=9)) || (n%100 >= 11 && n%100 <= 14)) ? 2 : 3)) }, + "zh_CN" => ->(n) { 0 }, + "zh_HK" => ->(n) { 0 }, + "zh_TW" => ->(n) { 0 } + }.freeze + # rubocop:enable + + NOT_FOUND_ERROR = lambda do |locale| + po = File.expand_path("../../../locale/#{locale}/gitlab.po", __dir__) + + forms = File.read(po)[/Plural-Forms:.*; plural=(.*?);\\n/, 1] if File.exist?(po) + suggestion = <<~TEXT if forms + Add the following line to #{__FILE__}: + + MAP = { + ... + "#{locale}" => ->(n) { #{forms} }, + ... + }.freeze + + This rule was extracted from #{po}. + TEXT + + raise ArgumentError, <<~MESSAGE + Missing pluralization rule for locale #{locale.inspect}. + + #{suggestion} + MESSAGE + end + + def self.call(count) + locale = FastGettext.locale + + MAP.fetch(locale, &NOT_FOUND_ERROR).call(count) + end + + def self.install_on(klass) + klass.extend(FastGettextClassMethods) + end + + module FastGettextClassMethods + # FastGettext allows to set the rule via + # `FastGettext.pluralisation_rule=` which is on thread-level. + # + # Because we are patching FastGettext at boot time per thread values + # won't work so we have to override the method implementation. + # + # `FastGettext.pluralisation_rule=` has now no effect. + def pluralisation_rule + Gitlab::I18n::Pluralization + end + end + end + end +end diff --git a/lib/gitlab/import/errors.rb b/lib/gitlab/import/errors.rb new file mode 100644 index 00000000000..b9e8c9135fd --- /dev/null +++ b/lib/gitlab/import/errors.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Gitlab + module Import + module Errors + # Merges all nested subrelation errors into base errors object. + # + # @example + # issue = Project.last.issues.new( + # title: 'test', + # author: User.first, + # notes: [Note.new( + # award_emoji: [AwardEmoji.new(name: 'test')] + # )]) + # + # issue.validate + # issue.errors.full_messages + # => ["Notes is invalid"] + # + # Gitlab::Import::Errors.merge_nested_errors(issue) + # issue.errors.full_messages + # => ["Notes is invalid", + # "Award emoji is invalid", + # "Awardable can't be blank", + # "Name is not a valid emoji name", + # ... + # ] + def self.merge_nested_errors(object) + object.errors.each do |error| + association = object.class.reflect_on_association(error.attribute) + + next unless association&.collection? + + records = object.public_send(error.attribute).select(&:invalid?) # rubocop: disable GitlabSecurity/PublicSend + + records.each do |record| + merge_nested_errors(record) + + object.errors.merge!(record.errors) + end + end + end + end + end +end diff --git a/lib/gitlab/import/import_failure_service.rb b/lib/gitlab/import/import_failure_service.rb index bebd64b29a9..714d9b3edd9 100644 --- a/lib/gitlab/import/import_failure_service.rb +++ b/lib/gitlab/import/import_failure_service.rb @@ -50,10 +50,10 @@ module Gitlab def execute track_exception - persist_failure - - track_metrics if metrics - import_state.mark_as_failed(exception.message) if fail_import + persist_failure.tap do + import_state.mark_as_failed(exception.message) if fail_import + track_metrics if metrics + end end private diff --git a/lib/gitlab/import/metrics.rb b/lib/gitlab/import/metrics.rb index 7a0cf1682a6..e457d9ec57c 100644 --- a/lib/gitlab/import/metrics.rb +++ b/lib/gitlab/import/metrics.rb @@ -32,6 +32,14 @@ module Gitlab return unless project.github_import? track_usage_event(:github_import_project_failure, project.id) + track_import_state('github') + end + + def track_canceled_import + return unless project.github_import? + + track_usage_event(:github_import_project_cancelled, project.id) + track_import_state('github') end def issues_counter @@ -75,7 +83,24 @@ module Gitlab def track_finish_metric return unless project.github_import? - track_usage_event(:github_import_project_success, project.id) + track_import_state('github') + + case project.beautified_import_status_name + when 'partially completed' + track_usage_event(:github_import_project_partially_completed, project.id) + when 'completed' + track_usage_event(:github_import_project_success, project.id) + end + end + + def track_import_state(type) + Gitlab::Tracking.event( + importer, + 'create', + label: "#{type}_import_project_state", + project: project, + extra: { import_type: type, state: project.beautified_import_status_name } + ) end end end diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb index 8843b4f5755..dea989931c7 100644 --- a/lib/gitlab/import_export/attributes_finder.rb +++ b/lib/gitlab/import_export/attributes_finder.rb @@ -3,7 +3,8 @@ module Gitlab module ImportExport class AttributesFinder - attr_reader :tree, :included_attributes, :excluded_attributes, :methods, :preloads, :export_reorders + attr_reader :tree, :included_attributes, :excluded_attributes, :methods, :preloads, :export_reorders, + :import_only_tree def initialize(config:) @tree = config[:tree] || {} @@ -13,13 +14,16 @@ module Gitlab @preloads = config[:preloads] || {} @export_reorders = config[:export_reorders] || {} @include_if_exportable = config[:include_if_exportable] || {} + @import_only_tree = config[:import_only_tree] || {} end def find_root(model_key) find(model_key, @tree[model_key]) end - def find_relations_tree(model_key) + def find_relations_tree(model_key, include_import_only_tree: false) + return @tree[model_key].deep_merge(@import_only_tree[model_key] || {}) if include_import_only_tree + @tree[model_key] end diff --git a/lib/gitlab/import_export/attributes_permitter.rb b/lib/gitlab/import_export/attributes_permitter.rb index 8c7a6c13246..889cab88de4 100644 --- a/lib/gitlab/import_export/attributes_permitter.rb +++ b/lib/gitlab/import_export/attributes_permitter.rb @@ -80,7 +80,7 @@ module Gitlab # Deep traverse relations tree to build a list of allowed model relations def build_associations - stack = @attributes_finder.tree.to_a + stack = @attributes_finder.tree.deep_merge(@attributes_finder.import_only_tree).to_a while stack.any? model_name, relations = stack.pop diff --git a/lib/gitlab/import_export/base/relation_object_saver.rb b/lib/gitlab/import_export/base/relation_object_saver.rb index 77b85fc9f15..986191bdb6b 100644 --- a/lib/gitlab/import_export/base/relation_object_saver.rb +++ b/lib/gitlab/import_export/base/relation_object_saver.rb @@ -17,6 +17,8 @@ module Gitlab BATCH_SIZE = 100 MIN_RECORDS_SIZE = 1 + attr_reader :invalid_subrelations + # @param relation_object [Object] Object of a project/group, e.g. an issue # @param relation_key [String] Name of the object association to group/project, e.g. :issues # @param relation_definition [Hash] Object subrelations as defined in import_export.yml @@ -43,14 +45,11 @@ module Gitlab relation_object.save! save_subrelations - ensure - log_invalid_subrelations end private - attr_reader :relation_object, :relation_key, :relation_definition, - :importable, :collection_subrelations, :invalid_subrelations + attr_reader :relation_object, :relation_key, :relation_definition, :importable, :collection_subrelations # rubocop:disable GitlabSecurity/PublicSend def save_subrelations @@ -92,30 +91,6 @@ module Gitlab end end # rubocop:enable GitlabSecurity/PublicSend - - def log_invalid_subrelations - invalid_subrelations.flatten.each do |record| - Gitlab::Import::Logger.info( - message: '[Project/Group Import] Invalid subrelation', - importable_column_name => importable.id, - relation_key: relation_key, - error_messages: record.errors.full_messages.to_sentence - ) - - ImportFailure.create( - source: 'RelationObjectSaver#save!', - relation_key: relation_key, - exception_class: 'RecordInvalid', - exception_message: record.errors.full_messages.to_sentence, - correlation_id_value: Labkit::Correlation::CorrelationId.current_or_new_id, - importable_column_name => importable.id - ) - end - end - - def importable_column_name - @column_name ||= importable.class.reflect_on_association(:import_failures).foreign_key.to_sym - end end end end diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index 64ef3dd4830..d681f39f00b 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -90,6 +90,7 @@ module Gitlab def untar_with_options(archive:, dir:, options:) execute_cmd(%W(tar -#{options} #{archive} -C #{dir})) execute_cmd(%W(chmod -R #{UNTAR_MASK} #{dir})) + remove_symlinks(dir) end # rubocop:disable Gitlab/ModuleWithInstanceVariables @@ -120,6 +121,19 @@ module Gitlab FileUtils.copy_entry(source, destination) true end + + def remove_symlinks(dir) + ignore_file_names = %w[. ..] + + # Using File::FNM_DOTMATCH to also delete symlinks starting with "." + Dir.glob("#{dir}/**/*", File::FNM_DOTMATCH) + .reject { |f| ignore_file_names.include?(File.basename(f)) } + .each do |filepath| + FileUtils.rm(filepath) if File.lstat(filepath).symlink? + end + + true + end end end end diff --git a/lib/gitlab/import_export/config.rb b/lib/gitlab/import_export/config.rb index 83c4bc47349..423e0933605 100644 --- a/lib/gitlab/import_export/config.rb +++ b/lib/gitlab/import_export/config.rb @@ -10,6 +10,7 @@ module Gitlab @ee_hash = @hash.delete(:ee) || {} @hash[:tree] = normalize_tree(@hash[:tree]) + @hash[:import_only_tree] = normalize_tree(@hash[:import_only_tree] || {}) @ee_hash[:tree] = normalize_tree(@ee_hash[:tree] || {}) end diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 1878b5b1a30..d2593289c23 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -8,7 +8,6 @@ module Gitlab ImporterError = Class.new(StandardError) MAX_RETRIES = 8 - IGNORED_FILENAMES = %w(. ..).freeze def self.import(*args, **kwargs) new(*args, **kwargs).import @@ -24,7 +23,7 @@ module Gitlab mkdir_p(@shared.export_path) mkdir_p(@shared.archive_path) - remove_symlinks + remove_symlinks(@shared.export_path) copy_archive wait_for_archived_file do @@ -36,7 +35,7 @@ module Gitlab false ensure remove_import_file - remove_symlinks + remove_symlinks(@shared.export_path) end private @@ -86,22 +85,10 @@ module Gitlab end end - def remove_symlinks - extracted_files.each do |path| - FileUtils.rm(path) if File.lstat(path).symlink? - end - - true - end - def remove_import_file FileUtils.rm_rf(@archive_file) end - def extracted_files - Dir.glob("#{@shared.export_path}/**/*", File::FNM_DOTMATCH).reject { |f| IGNORED_FILENAMES.include?(File.basename(f)) } - end - def validate_decompressed_archive_size raise ImporterError, _('Decompressed archive size validation failed.') unless size_validator.valid? end diff --git a/lib/gitlab/import_export/group/relation_tree_restorer.rb b/lib/gitlab/import_export/group/relation_tree_restorer.rb index 5a78f2fb531..624acd3bb2a 100644 --- a/lib/gitlab/import_export/group/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/group/relation_tree_restorer.rb @@ -90,13 +90,23 @@ module Gitlab def save_relation_object(relation_object, relation_key, relation_definition, relation_index) if relation_object.new_record? - Gitlab::ImportExport::Base::RelationObjectSaver.new( + saver = Gitlab::ImportExport::Base::RelationObjectSaver.new( relation_object: relation_object, relation_key: relation_key, relation_definition: relation_definition, importable: @importable - ).execute + ) + + saver.execute + + log_invalid_subrelations(saver.invalid_subrelations, relation_key) else + if relation_object.invalid? + Gitlab::Import::Errors.merge_nested_errors(relation_object) + + raise(ActiveRecord::RecordInvalid, relation_object) + end + import_failure_service.with_retry(action: 'relation_object.save!', relation_key: relation_key, relation_index: relation_index) do relation_object.save! end @@ -113,7 +123,7 @@ module Gitlab @relations ||= @reader .attributes_finder - .find_relations_tree(importable_class_sym) + .find_relations_tree(importable_class_sym, include_import_only_tree: true) .deep_stringify_keys end @@ -290,6 +300,32 @@ module Gitlab message: '[Project/Group Import] Created new object relation' ) end + + def log_invalid_subrelations(invalid_subrelations, relation_key) + invalid_subrelations.flatten.each do |record| + Gitlab::Import::Errors.merge_nested_errors(record) + + @shared.logger.info( + message: '[Project/Group Import] Invalid subrelation', + importable_column_name => @importable.id, + relation_key: relation_key, + error_messages: record.errors.full_messages.to_sentence + ) + + ::ImportFailure.create( + source: 'RelationTreeRestorer#save_relation_object', + relation_key: relation_key, + exception_class: 'ActiveRecord::RecordInvalid', + exception_message: record.errors.full_messages.to_sentence, + correlation_id_value: Labkit::Correlation::CorrelationId.current_or_new_id, + importable_column_name => @importable.id + ) + end + end + + def importable_column_name + @column_name ||= @importable.class.reflect_on_association(:import_failures).foreign_key.to_sym + end end end end diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index d97ffee8698..335096faed6 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -89,13 +89,15 @@ tree: - :milestone - :resource_state_events - :external_pull_requests + - commit_notes: + - :author + - events: + - :push_event_payload - ci_pipelines: - - notes: - - :author - - events: - - :push_event_payload - stages: - - :statuses + - :builds + - :generic_commit_statuses + - :bridges - :external_pull_request - :merge_request - :pipeline_metadata @@ -122,6 +124,21 @@ tree: group_members: - :user +# Used to support old exports that were exported before the removal/rename of the associations +# +# For example, statuses of ci_pipelines are no longer exported, and instead, statuses are +# exported as builds, generic_commit_statuses, and bridges. So in order to allow statuses +# to be still imported, it is added to the list below. +import_only_tree: + project: + - ci_pipelines: + - notes: + - :author + - events: + - :push_event_payload + - stages: + - :statuses + # Only include the following attributes for the models specified. included_attributes: user: @@ -529,7 +546,7 @@ included_attributes: - :source_sha - :target_sha external_pull_requests: *external_pull_request_definition - statuses: + statuses: &statuses_definition - :project_id - :status - :finished_at @@ -562,6 +579,9 @@ included_attributes: - :scheduled_at - :scheduling_type - :ci_stage + builds: *statuses_definition + generic_commit_statuses: *statuses_definition + bridges: *statuses_definition ci_pipelines: - :ref - :sha @@ -596,6 +616,7 @@ included_attributes: - :project_id - :created_at - :updated_at + # - :statuses # old exports use statuses instead of builds, generic_commit_statuses and bridges actions: - :event design: &design_definition @@ -896,7 +917,7 @@ excluded_attributes: merge_requests: *merge_request_excluded_definition award_emoji: - :awardable_id - statuses: + statuses: &statuses_excluded_definition - :trace - :token - :token_encrypted @@ -918,6 +939,9 @@ excluded_attributes: - :processed - :id_convert_to_bigint - :stage_id_convert_to_bigint + builds: *statuses_excluded_definition + generic_commit_statuses: *statuses_excluded_definition + bridges: *statuses_excluded_definition sentry_issue: - :issue_id push_event_payload: @@ -955,6 +979,9 @@ excluded_attributes: notes: - :noteable_id - :review_id + commit_notes: + - :noteable_id + - :review_id label_links: - :label_id - :target_id @@ -1079,6 +1106,8 @@ methods: - :squash_option notes: - :type + commit_notes: + - :type labels: - :type label: @@ -1100,8 +1129,6 @@ methods: - :type lists: - :list_type - ci_pipelines: - - :notes issues: - :state @@ -1111,10 +1138,11 @@ methods: preloads: issues: project: :route - statuses: - # TODO: We cannot preload tags, as they are not part of `GenericCommitStatus` - # tags: # needed by tag_list - project: # deprecated: needed by coverage_regex of Ci::Build + builds: + metadata: + project: + bridges: + metadata: merge_requests: source_project: :route # needed by source_branch_sha and diff_head_sha target_project: :route # needed by target_branch_sha diff --git a/lib/gitlab/import_export/project/object_builder.rb b/lib/gitlab/import_export/project/object_builder.rb index 50a67a746f8..0962ad9f028 100644 --- a/lib/gitlab/import_export/project/object_builder.rb +++ b/lib/gitlab/import_export/project/object_builder.rb @@ -64,6 +64,7 @@ module Gitlab if label? atts['type'] = 'ProjectLabel' # Always create project labels + atts.delete('group_id') elsif milestone? if atts['group_id'] # Transform new group milestones into project ones atts['iid'] = nil diff --git a/lib/gitlab/import_export/project/relation_factory.rb b/lib/gitlab/import_export/project/relation_factory.rb index 4134c428500..ab95e306abf 100644 --- a/lib/gitlab/import_export/project/relation_factory.rb +++ b/lib/gitlab/import_export/project/relation_factory.rb @@ -5,6 +5,7 @@ module Gitlab module Project class RelationFactory < Base::RelationFactory OVERRIDES = { snippets: :project_snippets, + commit_notes: 'Note', ci_pipelines: 'Ci::Pipeline', pipelines: 'Ci::Pipeline', stages: 'Ci::Stage', @@ -12,6 +13,7 @@ module Gitlab triggers: 'Ci::Trigger', pipeline_schedules: 'Ci::PipelineSchedule', builds: 'Ci::Build', + bridges: 'Ci::Bridge', runners: 'Ci::Runner', pipeline_metadata: 'Ci::PipelineMetadata', hooks: 'ProjectHook', @@ -37,7 +39,7 @@ module Gitlab committer: 'MergeRequest::DiffCommitUser', merge_request_diff_commits: 'MergeRequestDiffCommit' }.freeze - BUILD_MODELS = %i[Ci::Build commit_status].freeze + BUILD_MODELS = %i[Ci::Build Ci::Bridge commit_status generic_commit_status].freeze GROUP_REFERENCES = %w[group_id].freeze @@ -83,7 +85,7 @@ module Gitlab def setup_models case @relation_name when :merge_request_diff_files then setup_diff - when :notes then setup_note + when :notes, :Note then setup_note when :'Ci::Pipeline' then setup_pipeline when *BUILD_MODELS then setup_build when :issues then setup_issue @@ -142,9 +144,22 @@ module Gitlab def setup_pipeline @relation_hash.fetch('stages', []).each do |stage| + # old export files have statuses stage.statuses.each do |status| status.pipeline = imported_object end + + stage.builds.each do |status| + status.pipeline = imported_object + end + + stage.bridges.each do |status| + status.pipeline = imported_object + end + + stage.generic_commit_statuses.each do |status| + status.pipeline = imported_object + end end end diff --git a/lib/gitlab/instrumentation/redis_base.rb b/lib/gitlab/instrumentation/redis_base.rb index de24132a28e..00a7387afe2 100644 --- a/lib/gitlab/instrumentation/redis_base.rb +++ b/lib/gitlab/instrumentation/redis_base.rb @@ -118,6 +118,14 @@ module Gitlab @exception_counter.increment({ storage: storage_key, exception: ex.class.to_s }) end + def instance_count_cluster_redirection(ex) + # This metric is meant to give a client side view of how often are commands + # redirected to the right node, especially during resharding.. + # This metric can be used for Redis alerting and service health monitoring. + @redirection_counter ||= Gitlab::Metrics.counter(:gitlab_redis_client_redirections_total, 'Client side Redis Cluster redirection count, per Redis node, per slot') + @redirection_counter.increment(decompose_redirection_message(ex.message).merge({ storage: storage_key })) + end + def instance_observe_duration(duration) @request_latency_histogram ||= Gitlab::Metrics.histogram( :gitlab_redis_client_requests_duration_seconds, @@ -129,6 +137,10 @@ module Gitlab @request_latency_histogram.observe({ storage: storage_key }, duration) end + def log_exception(ex) + ::Gitlab::ErrorTracking.log_exception(ex, storage: storage_key) + end + private def request_count_key @@ -162,6 +174,11 @@ module Gitlab def build_key(namespace) "#{storage_key}_#{namespace}" end + + def decompose_redirection_message(err_msg) + redirection_type, _, target_node_key = err_msg.split + { redirection_type: redirection_type, target_node_key: target_node_key } + end end end end diff --git a/lib/gitlab/instrumentation/redis_interceptor.rb b/lib/gitlab/instrumentation/redis_interceptor.rb index 35dd7cbfeb8..2a86b9e4202 100644 --- a/lib/gitlab/instrumentation/redis_interceptor.rb +++ b/lib/gitlab/instrumentation/redis_interceptor.rb @@ -40,7 +40,13 @@ module Gitlab yield rescue ::Redis::BaseError => ex - instrumentation_class.instance_count_exception(ex) + if ex.message.start_with?('MOVED', 'ASK') + instrumentation_class.instance_count_cluster_redirection(ex) + else + instrumentation_class.instance_count_exception(ex) + end + + instrumentation_class.log_exception(ex) raise ex ensure duration = Gitlab::Metrics::System.monotonic_time - start diff --git a/lib/gitlab/instrumentation/zoekt.rb b/lib/gitlab/instrumentation/zoekt.rb new file mode 100644 index 00000000000..cd9b15bcee8 --- /dev/null +++ b/lib/gitlab/instrumentation/zoekt.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Gitlab + module Instrumentation + class Zoekt + ZOEKT_REQUEST_COUNT = :zoekt_request_count + ZOEKT_CALL_DURATION = :zoekt_call_duration + ZOEKT_CALL_DETAILS = :zoekt_call_details + + class << self + def get_request_count + ::Gitlab::SafeRequestStore[ZOEKT_REQUEST_COUNT] || 0 + end + + def increment_request_count + ::Gitlab::SafeRequestStore[ZOEKT_REQUEST_COUNT] ||= 0 + ::Gitlab::SafeRequestStore[ZOEKT_REQUEST_COUNT] += 1 + end + + def detail_store + ::Gitlab::SafeRequestStore[ZOEKT_CALL_DETAILS] ||= [] + end + + def query_time + query_time = ::Gitlab::SafeRequestStore[ZOEKT_CALL_DURATION] || 0 + query_time.round(::Gitlab::InstrumentationHelper::DURATION_PRECISION) + end + + def add_duration(duration) + ::Gitlab::SafeRequestStore[ZOEKT_CALL_DURATION] ||= 0 + ::Gitlab::SafeRequestStore[ZOEKT_CALL_DURATION] += duration + end + + def add_call_details(duration:, method:, path:, params: nil, body: nil) + return unless Gitlab::PerformanceBar.enabled_for_request? + + detail_store << { + method: method, + path: path, + params: params, + body: body, + duration: duration, + backtrace: ::Gitlab::BacktraceCleaner.clean_backtrace(caller) + } + end + end + end + end +end diff --git a/lib/gitlab/instrumentation_helper.rb b/lib/gitlab/instrumentation_helper.rb index 15a760fada0..1b81ec012d1 100644 --- a/lib/gitlab/instrumentation_helper.rb +++ b/lib/gitlab/instrumentation_helper.rb @@ -23,6 +23,7 @@ module Gitlab instrument_rugged(payload) instrument_redis(payload) instrument_elasticsearch(payload) + instrument_zoekt(payload) instrument_throttle(payload) instrument_active_record(payload) instrument_external_http(payload) @@ -72,6 +73,17 @@ module Gitlab payload[:elasticsearch_timed_out_count] = Gitlab::Instrumentation::ElasticsearchTransport.get_timed_out_count end + def instrument_zoekt(payload) + # Zoekt integration is only available in EE but instrumentation + # only depends on the Gem which is also available in FOSS. + zoekt_calls = Gitlab::Instrumentation::Zoekt.get_request_count + + return if zoekt_calls == 0 + + payload[:zoekt_calls] = zoekt_calls + payload[:zoekt_duration_s] = Gitlab::Instrumentation::Zoekt.query_time + end + def instrument_external_http(payload) external_http_count = Gitlab::Metrics::Subscribers::ExternalHttp.request_count diff --git a/lib/gitlab/issuable/clone/copy_resource_events_service.rb b/lib/gitlab/issuable/clone/copy_resource_events_service.rb index 448ac4c2ae0..22b0d5c4fb2 100644 --- a/lib/gitlab/issuable/clone/copy_resource_events_service.rb +++ b/lib/gitlab/issuable/clone/copy_resource_events_service.rb @@ -69,9 +69,9 @@ module Gitlab def copy_events(table_name, events_to_copy) events_to_copy.find_in_batches do |batch| - events = batch.map do |event| + events = batch.filter_map do |event| yield(event) - end.compact + end ApplicationRecord.legacy_bulk_insert(table_name, events) # rubocop:disable Gitlab/BulkInsert end diff --git a/lib/gitlab/issues/rebalancing/state.rb b/lib/gitlab/issues/rebalancing/state.rb index 36346564b39..f1f6cc55a2b 100644 --- a/lib/gitlab/issues/rebalancing/state.rb +++ b/lib/gitlab/issues/rebalancing/state.rb @@ -38,10 +38,10 @@ module Gitlab def rebalance_in_progress? is_running = case rebalanced_container_type when NAMESPACE - namespace_ids = self.class.current_rebalancing_containers.map { |string| string.split("#{NAMESPACE}/").second.to_i }.compact + namespace_ids = self.class.current_rebalancing_containers.filter_map { |string| string.split("#{NAMESPACE}/").second.to_i } namespace_ids.include?(root_namespace.id) when PROJECT - project_ids = self.class.current_rebalancing_containers.map { |string| string.split("#{PROJECT}/").second.to_i }.compact + project_ids = self.class.current_rebalancing_containers.filter_map { |string| string.split("#{PROJECT}/").second.to_i } project_ids.include?(projects.take.id) # rubocop:disable CodeReuse/ActiveRecord else false diff --git a/lib/gitlab/jira_import/issues_importer.rb b/lib/gitlab/jira_import/issues_importer.rb index 7b031c26b72..458f7c3f470 100644 --- a/lib/gitlab/jira_import/issues_importer.rb +++ b/lib/gitlab/jira_import/issues_importer.rb @@ -48,7 +48,7 @@ module Gitlab end def schedule_issue_import_workers(issues) - next_iid = Issue.with_project_iid_supply(project, &:next_value) + next_iid = Issue.with_namespace_iid_supply(project.project_namespace, &:next_value) issues.each do |jira_issue| # Technically it's possible that the same work is performed multiple @@ -71,7 +71,7 @@ module Gitlab job_waiter.jobs_remaining += 1 - next_iid = Issue.with_project_iid_supply(project, &:next_value) + next_iid = Issue.with_namespace_iid_supply(project.project_namespace, &:next_value) # Mark the issue as imported immediately so we don't end up # importing it multiple times within same import. diff --git a/lib/gitlab/jira_import/metadata_collector.rb b/lib/gitlab/jira_import/metadata_collector.rb index 4551f38ba98..090b95ac14a 100644 --- a/lib/gitlab/jira_import/metadata_collector.rb +++ b/lib/gitlab/jira_import/metadata_collector.rb @@ -45,7 +45,7 @@ module Gitlab def add_versions return if fields['fixVersions'].blank? || !fields['fixVersions'].is_a?(Array) - versions = fields['fixVersions'].map { |version| version['name'] }.compact.join(', ') + versions = fields['fixVersions'].filter_map { |version| version['name'] }.join(', ') metadata << "- Fix versions: #{versions}" end diff --git a/lib/gitlab/json_cache.rb b/lib/gitlab/json_cache.rb index d2916a01809..0087c2accc3 100644 --- a/lib/gitlab/json_cache.rb +++ b/lib/gitlab/json_cache.rb @@ -112,7 +112,7 @@ module Gitlab end def parse_entries(values, klass) - values.map { |value| parse_entry(value, klass) }.compact + values.filter_map { |value| parse_entry(value, klass) } end end end diff --git a/lib/gitlab/kas/user_access.rb b/lib/gitlab/kas/user_access.rb new file mode 100644 index 00000000000..65ae399d826 --- /dev/null +++ b/lib/gitlab/kas/user_access.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +module Gitlab + module Kas + # The name of the cookie that will be used for the KAS cookie + COOKIE_KEY = '_gitlab_kas' + DEFAULT_ENCRYPTED_COOKIE_CIPHER = 'aes-256-gcm' + + class UserAccess + class << self + def enabled? + ::Gitlab::Kas.enabled? && ::Feature.enabled?(:kas_user_access) + end + + def enabled_for?(agent) + enabled? && ::Feature.enabled?(:kas_user_access_project, agent.project) + end + + def encrypt_public_session_id(data) + encryptor.encrypt_and_sign(data.to_json, purpose: public_session_id_purpose) + end + + def decrypt_public_session_id(data) + decrypted = encryptor.decrypt_and_verify(data, purpose: public_session_id_purpose) + ::Gitlab::Json.parse(decrypted) + end + + def valid_authenticity_token?(session, masked_authenticity_token) + # rubocop:disable GitlabSecurity/PublicSend + ActionController::Base.new.send(:valid_authenticity_token?, session, masked_authenticity_token) + # rubocop:enable GitlabSecurity/PublicSend + end + + def cookie_data(public_session_id) + uri = URI(::Gitlab::Kas.tunnel_url) + + cookie = { + value: encrypt_public_session_id(public_session_id), + expires: 1.day, + httponly: true, + path: uri.path.presence || '/', + secure: Gitlab.config.gitlab.https + } + # Only set domain attribute if KAS is on a subdomain. + # When on the same domain, we can omit the attribute. + gitlab_host = Gitlab.config.gitlab.host + cookie[:domain] = gitlab_host if uri.host.end_with?(".#{gitlab_host}") + + cookie + end + + private + + def encryptor + action_dispatch_config = Gitlab::Application.config.action_dispatch + serializer = ActiveSupport::MessageEncryptor::NullSerializer + key_generator = ::Gitlab::Application.key_generator + + cipher = action_dispatch_config.encrypted_cookie_cipher || DEFAULT_ENCRYPTED_COOKIE_CIPHER + salt = action_dispatch_config.authenticated_encrypted_cookie_salt + key_len = ActiveSupport::MessageEncryptor.key_len(cipher) + secret = key_generator.generate_key(salt, key_len) + + ActiveSupport::MessageEncryptor.new(secret, cipher: cipher, serializer: serializer) + end + + def public_session_id_purpose + "kas.user_public_session_id" + end + end + end + end +end diff --git a/lib/gitlab/language_detection.rb b/lib/gitlab/language_detection.rb index b259f58350b..68deafc68b2 100644 --- a/lib/gitlab/language_detection.rb +++ b/lib/gitlab/language_detection.rb @@ -45,11 +45,11 @@ module Gitlab # Returns the ids of the programming languages that do not occur in the detection # as current repository languages def deletions - @repository_languages.map do |repo_lang| + @repository_languages.filter_map do |repo_lang| next if detection.key?(repo_lang.name) repo_lang.programming_language_id - end.compact + end end private diff --git a/lib/gitlab/legacy_github_import/client.rb b/lib/gitlab/legacy_github_import/client.rb index dd1502bbbcd..16c3bc09c4d 100644 --- a/lib/gitlab/legacy_github_import/client.rb +++ b/lib/gitlab/legacy_github_import/client.rb @@ -34,6 +34,7 @@ module Gitlab } ) end + alias_method :octokit, :api def client unless config diff --git a/lib/gitlab/legacy_github_import/importer.rb b/lib/gitlab/legacy_github_import/importer.rb index 331eab7b62a..5eeea8f9548 100644 --- a/lib/gitlab/legacy_github_import/importer.rb +++ b/lib/gitlab/legacy_github_import/importer.rb @@ -22,7 +22,7 @@ module Gitlab unless credentials raise Projects::ImportService::Error, - "Unable to find project import data credentials for project ID: #{@project.id}" + "Unable to find project import data credentials for project ID: #{@project.id}" end opts = {} @@ -55,9 +55,7 @@ module Gitlab import_comments(:issues) # Gitea doesn't have an API endpoint for pull requests comments - unless project.gitea_import? - import_comments(:pull_requests) - end + import_comments(:pull_requests) unless project.gitea_import? import_wiki @@ -67,9 +65,7 @@ module Gitlab # See: # 1) https://gitlab.com/gitlab-org/gitlab/-/issues/343448#note_985979730 # 2) https://gitlab.com/gitlab-org/gitlab/-/merge_requests/89694/diffs#dfc4a8141aa296465ea3c50b095a30292fb6ebc4_180_182 - unless project.gitea_import? - import_releases - end + import_releases unless project.gitea_import? handle_errors @@ -142,8 +138,8 @@ module Gitlab # rubocop: enable CodeReuse/ActiveRecord def import_pull_requests - fetch_resources(:pull_requests, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |pull_requests| - pull_requests.each do |raw| + fetch_resources(:pull_requests, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |prs| + prs.each do |raw| raw = raw.to_h gh_pull_request = PullRequestFormatter.new(project, raw, client) @@ -156,11 +152,13 @@ module Gitlab merge_request = gh_pull_request.create! # Gitea doesn't return PR in the Issue API endpoint, so labels must be assigned at this stage - if project.gitea_import? - apply_labels(merge_request, raw) - end + apply_labels(merge_request, raw) if project.gitea_import? rescue StandardError => e - errors << { type: :pull_request, url: Gitlab::UrlSanitizer.sanitize(gh_pull_request.url), errors: e.message } + errors << { + type: :pull_request, + url: Gitlab::UrlSanitizer.sanitize(gh_pull_request.url), + errors: e.message + } ensure clean_up_restored_branches(gh_pull_request) end @@ -196,9 +194,7 @@ module Gitlab return unless raw[:labels].count > 0 - label_ids = raw[:labels] - .map { |attrs| @labels[attrs[:name]] } - .compact + label_ids = raw[:labels].filter_map { |attrs| @labels[attrs[:name]] } issuable.update_attribute(:label_ids, label_ids) end @@ -208,10 +204,14 @@ module Gitlab resource_type = "#{issuable_type}_comments".to_sym # Two notes here: - # 1. We don't have a distinctive attribute for comments (unlike issues iid), so we fetch the last inserted note, - # compare it against every comment in the current imported page until we find match, and that's where start importing - # 2. GH returns comments for _both_ issues and PRs through issues_comments API, while pull_requests_comments returns - # only comments on diffs, so select last note not based on noteable_type but on line_code + # 1. We don't have a distinctive attribute for comments (unlike issues + # iid), so we fetch the last inserted note, compare it against every + # comment in the current imported page until we find match, and that's + # where start importing + # 2. GH returns comments for _both_ issues and PRs through + # issues_comments API, while pull_requests_comments returns only + # comments on diffs, so select last note not based on noteable_type but + # on line_code line_code_is = issuable_type == :pull_requests ? 'NOT NULL' : 'NULL' last_note = project.notes.where("line_code IS #{line_code_is}").last @@ -264,7 +264,8 @@ module Gitlab comment_attrs.with_indifferent_access == last_note_attrs end - # No matching resource in the collection, which means we got halted right on the end of the last page, so all good + # No matching resource in the collection, which means we got halted + # right on the end of the last page, so all good return unless cut_off_index # Otherwise, remove the resources we've already inserted @@ -280,9 +281,7 @@ module Gitlab # GitHub error message when the wiki repo has not been created, # this means that repo has wiki enabled, but have no pages. So, # we can skip the import. - if e.message !~ /repository not exported/ - errors << { type: :wiki, errors: e.message } - end + errors << { type: :wiki, errors: e.message } if e.message.exclude?('repository not exported') end def import_releases diff --git a/lib/gitlab/loggable.rb b/lib/gitlab/loggable.rb new file mode 100644 index 00000000000..393e3eb37b6 --- /dev/null +++ b/lib/gitlab/loggable.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Gitlab + module Loggable + ANONYMOUS = '<Anonymous>' + + def build_structured_payload(**params) + { class: self.class.name || ANONYMOUS }.merge(params).stringify_keys + end + end +end diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb index f635deabf76..b4baeba72e8 100644 --- a/lib/gitlab/metrics/requests_rack_middleware.rb +++ b/lib/gitlab/metrics/requests_rack_middleware.rb @@ -22,7 +22,7 @@ module Gitlab # reasonable default. If we initialize every category we'll end up # with an explosion in unused metric combinations, but we want the # most common ones to be always present. - FEATURE_CATEGORIES_TO_INITIALIZE = ['authentication_and_authorization', + FEATURE_CATEGORIES_TO_INITIALIZE = ['system_access', 'code_review_workflow', 'continuous_integration', 'not_owned', 'source_code_management', FEATURE_CATEGORY_DEFAULT].freeze diff --git a/lib/gitlab/metrics/subscribers/rack_attack.rb b/lib/gitlab/metrics/subscribers/rack_attack.rb index e6cf14a6c8c..2196122df01 100644 --- a/lib/gitlab/metrics/subscribers/rack_attack.rb +++ b/lib/gitlab/metrics/subscribers/rack_attack.rb @@ -22,11 +22,6 @@ module Gitlab } end - def redis(event) - self.class.payload[:rack_attack_redis_count] += 1 - self.class.payload[:rack_attack_redis_duration_s] += event.duration.to_f / 1000 - end - def safelist(event) req = event.payload[:request] Gitlab::Instrumentation::Throttle.safelist = req.env['rack.attack.matched'] diff --git a/lib/gitlab/metrics/subscribers/rails_cache.rb b/lib/gitlab/metrics/subscribers/rails_cache.rb index b12db9df66d..d2b6d0e3c14 100644 --- a/lib/gitlab/metrics/subscribers/rails_cache.rb +++ b/lib/gitlab/metrics/subscribers/rails_cache.rb @@ -9,7 +9,7 @@ module Gitlab attach_to :active_support def cache_read_multi(event) - observe(:read_multi, event.duration) + observe(:read_multi, event) return unless current_transaction @@ -20,7 +20,7 @@ module Gitlab end def cache_read(event) - observe(:read, event.duration) + observe(:read, event) return unless current_transaction return if event.payload[:super_operation] == :fetch @@ -33,15 +33,15 @@ module Gitlab end def cache_write(event) - observe(:write, event.duration) + observe(:write, event) end def cache_delete(event) - observe(:delete, event.duration) + observe(:delete, event) end def cache_exist?(event) - observe(:exists, event.duration) + observe(:exists, event) end def cache_fetch_hit(event) @@ -60,17 +60,17 @@ module Gitlab current_transaction.increment(:gitlab_transaction_cache_read_miss_count_total, 1) end - def observe(key, duration) + def observe(key, event) return unless current_transaction - labels = { operation: key } + labels = { operation: key, store: event.payload[:store].split('::').last } current_transaction.increment(:gitlab_cache_operations_total, 1, labels) do docstring 'Cache operations' label_keys labels.keys end - metric_cache_operation_duration_seconds.observe(labels, duration / 1000.0) + metric_cache_operation_duration_seconds.observe(labels, event.duration / 1000.0) end private diff --git a/lib/gitlab/multi_collection_paginator.rb b/lib/gitlab/multi_collection_paginator.rb index 87cc0a0d3d2..e122f0b9317 100644 --- a/lib/gitlab/multi_collection_paginator.rb +++ b/lib/gitlab/multi_collection_paginator.rb @@ -9,6 +9,8 @@ module Gitlab @per_page = (per_page || Kaminari.config.default_per_page).to_i @first_collection, @second_collection = collections + + raise ArgumentError, 'Page size must be at least 1' if @per_page < 1 end def paginate(page) diff --git a/lib/gitlab/nav/top_nav_menu_item.rb b/lib/gitlab/nav/top_nav_menu_item.rb index 75eb0e8a264..a83cdbe15df 100644 --- a/lib/gitlab/nav/top_nav_menu_item.rb +++ b/lib/gitlab/nav/top_nav_menu_item.rb @@ -8,7 +8,10 @@ module Gitlab # this is already :/. We could also take a hash and manually check every # entry, but it's much more maintainable to do rely on native Ruby. # rubocop: disable Metrics/ParameterLists - def self.build(id:, title:, active: false, icon: '', href: '', view: '', css_class: nil, data: nil, emoji: nil) + def self.build( + id:, title:, active: false, icon: '', href: '', view: '', + css_class: nil, data: nil, partial: nil, component: nil + ) { id: id, type: :item, @@ -19,7 +22,8 @@ module Gitlab view: view.to_s, css_class: css_class, data: data || { qa_selector: 'menu_item_link', qa_title: title }, - emoji: emoji + partial: partial, + component: component } end # rubocop: enable Metrics/ParameterLists diff --git a/lib/gitlab/no_cache_headers.rb b/lib/gitlab/no_cache_headers.rb index 2d03741480d..6afb108dcfd 100644 --- a/lib/gitlab/no_cache_headers.rb +++ b/lib/gitlab/no_cache_headers.rb @@ -4,7 +4,6 @@ module Gitlab module NoCacheHeaders DEFAULT_GITLAB_NO_CACHE_HEADERS = { 'Cache-Control' => "#{ActionDispatch::Http::Cache::Response::DEFAULT_CACHE_CONTROL}, no-store, no-cache", - 'Pragma' => 'no-cache', # HTTP 1.0 compatibility 'Expires' => 'Fri, 01 Jan 1990 00:00:00 GMT' }.freeze diff --git a/lib/gitlab/observability.rb b/lib/gitlab/observability.rb index 8dbd2f41ccb..f7f65c91339 100644 --- a/lib/gitlab/observability.rb +++ b/lib/gitlab/observability.rb @@ -2,8 +2,19 @@ module Gitlab module Observability - module_function + extend self + ACTION_TO_PERMISSION = { + explore: :read_observability, + datasources: :admin_observability, + manage: :admin_observability, + dashboards: :read_observability + }.freeze + + EMBEDDABLE_PATHS = %w[explore goto].freeze + + # Returns the GitLab Observability URL + # def observability_url return ENV['OVERRIDE_OBSERVABILITY_URL'] if ENV['OVERRIDE_OBSERVABILITY_URL'] # TODO Make observability URL configurable https://gitlab.com/gitlab-org/opstrace/opstrace-ui/-/issues/80 @@ -12,8 +23,123 @@ module Gitlab 'https://observe.gitlab.com' end - def observability_enabled?(user, group) - Gitlab::Observability.observability_url.present? && Ability.allowed?(user, :read_observability, group) + # Returns true if the Observability feature flag is enabled + # + def enabled?(group = nil) + return Feature.enabled?(:observability_group_tab, group) if group + + Feature.enabled?(:observability_group_tab) + end + + # Returns the embeddable Observability URL of a given URL + # + # - Validates the URL + # - Checks that the path is embeddable + # - Converts the gitlab.com URL to observe.gitlab.com URL + # + # e.g. + # + # from: gitlab.com/groups/GROUP_PATH/-/observability/explore?observability_path=/explore + # to observe.gitlab.com/-/GROUP_ID/explore + # + # Returns nil if the URL is not a valid Observability URL or the path is not embeddable + # + def embeddable_url(url) + uri = validate_url(url, Gitlab.config.gitlab.url) + return unless uri + + group = group_from_observability_url(url) + return unless group + + parsed_query = CGI.parse(uri.query.to_s).transform_values(&:first).symbolize_keys + observability_path = parsed_query[:observability_path] + + return build_full_url(group, observability_path, '/') if observability_path_embeddable?(observability_path) + end + + # Returns true if the user is allowed to perform an action within a group + # + def allowed_for_action?(user, group, action) + return false if action.nil? + + permission = ACTION_TO_PERMISSION.fetch(action.to_sym, :admin_observability) + allowed?(user, group, permission) + end + + # Returns true if the user has the specified permission within the group + def allowed?(user, group, permission = :admin_observability) + return false unless group && user + + observability_url.present? && Ability.allowed?(user, permission, group) + end + + # Builds the full Observability URL given a certan group and path + # + # If unsanitized_observability_path is not valid or missing, fallbacks to fallback_path + # + def build_full_url(group, unsanitized_observability_path, fallback_path) + return unless group + + # When running Observability UI in standalone mode (i.e. not backed by Observability Backend) + # the group-id is not required. !!This is only used for local dev!! + base_url = ENV['STANDALONE_OBSERVABILITY_UI'] == 'true' ? observability_url : "#{observability_url}/-/#{group.id}" + + sanitized_path = if unsanitized_observability_path && sanitize(unsanitized_observability_path) != '' + CGI.unescapeHTML(sanitize(unsanitized_observability_path)) + else + fallback_path || '/' + end + + sanitized_path.prepend('/') if sanitized_path[0] != '/' + + "#{base_url}#{sanitized_path}" + end + + private + + def validate_url(url, reference_url) + uri = URI.parse(url) + reference_uri = URI.parse(reference_url) + + return uri if uri.scheme == reference_uri.scheme && + uri.port == reference_uri.port && + uri.host.casecmp?(reference_uri.host) + rescue URI::InvalidURIError + nil + end + + def link_sanitizer + @link_sanitizer ||= Rails::Html::Sanitizer.safe_list_sanitizer.new + end + + def sanitize(input) + link_sanitizer.sanitize(input, {})&.html_safe + end + + def group_from_observability_url(url) + match = Rails.application.routes.recognize_path(url) + + return if match[:unmatched_route].present? + return if match[:group_id].blank? || match[:action].blank? || match[:controller] != "groups/observability" + + group_path = match[:group_id] + Group.find_by_full_path(group_path) + rescue ActionController::RoutingError + nil + end + + def observability_path_embeddable?(observability_path) + return false unless observability_path + + observability_path = observability_path[1..] if observability_path[0] == '/' + + parsed_observability_path = URI.parse(observability_path).path.split('/') + + base_path = parsed_observability_path[0] + + EMBEDDABLE_PATHS.include?(base_path) + rescue URI::InvalidURIError + false end end end diff --git a/lib/gitlab/optimistic_locking.rb b/lib/gitlab/optimistic_locking.rb index 9f39b5f122f..3c8ac55f70b 100644 --- a/lib/gitlab/optimistic_locking.rb +++ b/lib/gitlab/optimistic_locking.rb @@ -10,8 +10,11 @@ module Gitlab start_time = Gitlab::Metrics::System.monotonic_time retry_attempts = 0 + # prevent scope override, see https://gitlab.com/gitlab-org/gitlab/-/issues/391186 + klass = subject.is_a?(ActiveRecord::Relation) ? subject.klass : subject.class + begin - subject.transaction do + klass.transaction do yield(subject) end rescue ActiveRecord::StaleObjectError diff --git a/lib/gitlab/pages/random_domain.rb b/lib/gitlab/pages/random_domain.rb new file mode 100644 index 00000000000..8aa7611c910 --- /dev/null +++ b/lib/gitlab/pages/random_domain.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Gitlab + module Pages + class RandomDomain + PROJECT_PATH_LIMIT = 48 + SUBDOMAIN_LABEL_LIMIT = 63 + + def self.generate(project_path:, namespace_path:) + new(project_path: project_path, namespace_path: namespace_path).generate + end + + def initialize(project_path:, namespace_path:) + @project_path = project_path + @namespace_path = namespace_path + end + + # Subdomains have a limit of 63 bytes (https://www.freesoft.org/CIE/RFC/1035/9.htm) + # For this reason we're limiting each part of the unique subdomain + # + # The domain is made up of 3 parts, like: projectpath-namespacepath-randomstring + # - project path: between 1 and 48 chars + # - namespace path: when the project path has less than 48 chars, + # the namespace full path will be used to fill the value up to 48 chars + # - random hexadecimal: to ensure a random value, the domain is then filled + # with a random hexadecimal value to complete 63 chars + def generate + domain = project_path.byteslice(0, PROJECT_PATH_LIMIT) + + # if the project_path has less than PROJECT_PATH_LIMIT chars, + # fill the domain with the parent full_path up to 48 chars like: + # projectpath-namespacepath + if domain.length < PROJECT_PATH_LIMIT + namespace_size = PROJECT_PATH_LIMIT - domain.length - 1 + domain.concat('-', namespace_path.byteslice(0, namespace_size)) + end + + # Complete the domain with random hexadecimal values util it is 63 chars long + # PS.: SecureRandom.hex return an string twice the size passed as argument. + domain.concat('-', SecureRandom.hex(SUBDOMAIN_LABEL_LIMIT - domain.length - 1)) + + # Slugify ensures the format and size (63 chars) of the given string + Gitlab::Utils.slugify(domain) + end + + private + + attr_reader :project_path, :namespace_path + end + end +end diff --git a/lib/gitlab/pages/virtual_host_finder.rb b/lib/gitlab/pages/virtual_host_finder.rb new file mode 100644 index 00000000000..87fbf547770 --- /dev/null +++ b/lib/gitlab/pages/virtual_host_finder.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +module Gitlab + module Pages + class VirtualHostFinder + def initialize(host) + @host = host&.downcase + end + + def execute + return if host.blank? + + gitlab_host = ::Settings.pages.host.downcase.prepend(".") + + if host.ends_with?(gitlab_host) + name = host.delete_suffix(gitlab_host) + + by_namespace_domain(name) || + by_unique_domain(name) + else + by_custom_domain(host) + end + end + + private + + attr_reader :host + + def by_unique_domain(name) + return unless Feature.enabled?(:pages_unique_domain) + + project = Project.by_pages_enabled_unique_domain(name) + + return unless project&.pages_deployed? + + ::Pages::VirtualDomain.new(projects: [project]) + end + + def by_namespace_domain(name) + namespace = Namespace.top_most.by_path(name) + + return if namespace.blank? + + cache = if Feature.enabled?(:cache_pages_domain_api, namespace) + ::Gitlab::Pages::CacheControl.for_namespace(namespace.id) + end + + ::Pages::VirtualDomain.new( + trim_prefix: namespace.full_path, + projects: namespace.all_projects_with_pages, + cache: cache + ) + end + + def by_custom_domain(host) + domain = PagesDomain.find_by_domain_case_insensitive(host) + + return unless domain&.pages_deployed? + + cache = if Feature.enabled?(:cache_pages_domain_api, domain.project.root_namespace) + ::Gitlab::Pages::CacheControl.for_domain(domain.id) + end + + ::Pages::VirtualDomain.new( + projects: [domain.project], + domain: domain, + cache: cache + ) + end + end + end +end diff --git a/lib/gitlab/patch/node_loader.rb b/lib/gitlab/patch/node_loader.rb new file mode 100644 index 00000000000..79f4b17dd93 --- /dev/null +++ b/lib/gitlab/patch/node_loader.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +# Patch to address https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/2212#note_1287996694 +# It uses hostname instead of IP address if the former is present in `CLUSTER NODES` output. +if Gem::Version.new(Redis::VERSION) > Gem::Version.new('4.8.1') + raise 'New version of redis detected, please remove or update this patch' +end + +module Gitlab + module Patch + module NodeLoader + def self.prepended(base) + base.class_eval do + # monkey-patches https://github.com/redis/redis-rb/blob/v4.8.0/lib/redis/cluster/node_loader.rb#L23 + def self.fetch_node_info(node) + node.call(%i[cluster nodes]).split("\n").map(&:split).to_h do |arr| + [ + extract_host_identifier(arr[1]), + (arr[2].split(',') & %w[master slave]).first # rubocop:disable Naming/InclusiveLanguage + ] + end + end + + # Since `CLUSTER SLOT` uses the preferred endpoint determined by + # the `cluster-preferred-endpoint-type` config value, we will prefer hostname over IP address. + # See https://redis.io/commands/cluster-nodes/ for details on the output format. + # + # @param [String] Address info matching fhe format: <ip:port@cport[,hostname[,auxiliary_field=value]*]> + def self.extract_host_identifier(node_address) + ip_chunk, hostname, _auxiliaries = node_address.split(',') + return ip_chunk.split('@').first if hostname.blank? + + port = ip_chunk.split('@').first.split(':')[1] + "#{hostname}:#{port}" + end + end + end + end + end +end diff --git a/lib/gitlab/private_commit_email.rb b/lib/gitlab/private_commit_email.rb index 886c2c32d36..176402cadfe 100644 --- a/lib/gitlab/private_commit_email.rb +++ b/lib/gitlab/private_commit_email.rb @@ -19,7 +19,7 @@ module Gitlab end def user_ids_for_emails(emails) - emails.map { |email| user_id_for_email(email) }.compact.uniq + emails.filter_map { |email| user_id_for_email(email) }.uniq end def for_user(user) diff --git a/lib/gitlab/prometheus/queries/knative_invocation_query.rb b/lib/gitlab/prometheus/queries/knative_invocation_query.rb deleted file mode 100644 index 6438995b576..00000000000 --- a/lib/gitlab/prometheus/queries/knative_invocation_query.rb +++ /dev/null @@ -1,42 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Prometheus - module Queries - class KnativeInvocationQuery < BaseQuery - include QueryAdditionalMetrics - - def query(serverless_function_id) - PrometheusMetricsFinder - .new(identifier: :system_metrics_knative_function_invocation_count, common: true) - .execute - .first - .to_query_metric - .tap do |q| - q.queries[0][:result] = run_query(q.queries[0][:query_range], context(serverless_function_id)) - end - end - - protected - - def context(function_id) - function = ::Serverless::Function.find_by_id(function_id) - { - function_name: function.name, - kube_namespace: function.namespace - } - end - - def run_query(query, context) - query %= context - client_query_range(query, start_time: 8.hours.ago.to_f, end_time: Time.now.to_f) - end - - def self.transform_reactive_result(result) - result[:metrics] = result.delete :data - result - end - end - end - end -end diff --git a/lib/gitlab/rack_attack.rb b/lib/gitlab/rack_attack.rb index bedbe9c0bff..d999b706d6c 100644 --- a/lib/gitlab/rack_attack.rb +++ b/lib/gitlab/rack_attack.rb @@ -19,7 +19,7 @@ module Gitlab [429, { 'Content-Type' => 'text/plain' }.merge(throttled_headers), [Gitlab::Throttle.rate_limiting_response_text]] end - rack_attack.cache.store = Gitlab::RackAttack::InstrumentedCacheStore.new + rack_attack.cache.store = Gitlab::RackAttack::Store.new # Configure the throttles configure_throttles(rack_attack) diff --git a/lib/gitlab/rack_attack/instrumented_cache_store.rb b/lib/gitlab/rack_attack/instrumented_cache_store.rb deleted file mode 100644 index d8beb259fba..00000000000 --- a/lib/gitlab/rack_attack/instrumented_cache_store.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module RackAttack - # This class is a proxy for all Redis calls made by RackAttack. All - # the calls are instrumented, then redirected to the underlying - # store (in `.store). This class instruments the standard interfaces - # of ActiveRecord::Cache defined in - # https://github.com/rails/rails/blob/v6.0.3.1/activesupport/lib/active_support/cache.rb#L315 - # - # For more information, please see - # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/751 - class InstrumentedCacheStore - NOTIFICATION_CHANNEL = 'redis.rack_attack' - - delegate :silence!, :mute, to: :@upstream_store - - def initialize(upstream_store: ::Gitlab::Redis::RateLimiting.cache_store, notifier: ActiveSupport::Notifications) - @upstream_store = upstream_store - @notifier = notifier - end - - [:fetch, :read, :read_multi, :write_multi, :fetch_multi, :write, :delete, - :exist?, :delete_matched, :increment, :decrement, :cleanup, :clear].each do |interface| - define_method interface do |*args, **k_args, &block| - @notifier.instrument(NOTIFICATION_CHANNEL, operation: interface) do - @upstream_store.public_send(interface, *args, **k_args, &block) # rubocop:disable GitlabSecurity/PublicSend - end - end - end - end - end -end diff --git a/lib/gitlab/rack_attack/store.rb b/lib/gitlab/rack_attack/store.rb new file mode 100644 index 00000000000..e4a1b022c32 --- /dev/null +++ b/lib/gitlab/rack_attack/store.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module RackAttack + class Store + InvalidAmount = Class.new(StandardError) + + # The increment method gets called very often. The implementation below + # aims to minimize the number of Redis calls we make. + def increment(key, amount = 1, options = {}) + # Our code below that prevents calling EXPIRE after every INCR assumes + # we always increment by 1. This is true in Rack::Attack as of v6.6.1. + # This guard should alert us if Rack::Attack changes its behavior in a + # future version. + raise InvalidAmount unless amount == 1 + + with do |redis| + key = namespace(key) + new_value = redis.incr(key) + expires_in = options[:expires_in] + redis.expire(key, expires_in) if new_value == 1 && expires_in + new_value + end + end + + def read(key, _options = {}) + with { |redis| redis.get(namespace(key)) } + end + + def write(key, value, options = {}) + with { |redis| redis.set(namespace(key), value, ex: options[:expires_in]) } + end + + def delete(key, _options = {}) + with { |redis| redis.del(namespace(key)) } + end + + private + + def with(&block) + # rubocop: disable CodeReuse/ActiveRecord + Gitlab::Redis::RateLimiting.with(&block) + # rubocop: enable CodeReuse/ActiveRecord + rescue ::Redis::BaseConnectionError + # Following the example of + # https://github.com/rack/rack-attack/blob/v6.6.1/lib/rack/attack/store_proxy/redis_proxy.rb#L61-L65, + # do not raise an error if we cannot connect to Redis. If + # Redis::RateLimiting is unavailable it should not take the site down. + nil + end + + def namespace(key) + "#{Gitlab::Redis::Cache::CACHE_NAMESPACE}:#{key}" + end + end + end +end diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb index 647573e59fe..ba3af3e7a6f 100644 --- a/lib/gitlab/redis/cache.rb +++ b/lib/gitlab/redis/cache.rb @@ -2,16 +2,6 @@ module Gitlab module Redis - # Match signature in - # https://github.com/rails/rails/blob/v6.1.7.2/activesupport/lib/active_support/cache/redis_cache_store.rb#L59 - ERROR_HANDLER = ->(method:, returning:, exception:) do - Gitlab::ErrorTracking.log_exception( - exception, - method: method, - returning: returning.inspect - ) - end - class Cache < ::Gitlab::Redis::Wrapper CACHE_NAMESPACE = 'cache:gitlab' @@ -22,8 +12,7 @@ module Gitlab redis: pool, compress: Gitlab::Utils.to_boolean(ENV.fetch('ENABLE_REDIS_CACHE_COMPRESSION', '1')), namespace: CACHE_NAMESPACE, - expires_in: default_ttl_seconds, - error_handler: ::Gitlab::Redis::ERROR_HANDLER + expires_in: default_ttl_seconds } end diff --git a/lib/gitlab/redis/multi_store.rb b/lib/gitlab/redis/multi_store.rb index a102267d52b..9571e2f92e6 100644 --- a/lib/gitlab/redis/multi_store.rb +++ b/lib/gitlab/redis/multi_store.rb @@ -5,12 +5,6 @@ module Gitlab class MultiStore include Gitlab::Utils::StrongMemoize - class ReadFromPrimaryError < StandardError - def message - 'Value not found on the redis primary store. Read from the redis secondary store successful.' - end - end - class PipelinedDiffError < StandardError def initialize(result_primary, result_secondary) @result_primary = result_primary @@ -32,41 +26,33 @@ module Gitlab attr_reader :primary_store, :secondary_store, :instance_name - FAILED_TO_READ_ERROR_MESSAGE = 'Failed to read from the redis primary_store.' + FAILED_TO_READ_ERROR_MESSAGE = 'Failed to read from the redis default_store.' FAILED_TO_WRITE_ERROR_MESSAGE = 'Failed to write to the redis primary_store.' FAILED_TO_RUN_PIPELINE = 'Failed to execute pipeline on the redis primary_store.' SKIP_LOG_METHOD_MISSING_FOR_COMMANDS = %i[info].freeze - # For ENUMERATOR_CACHE_HIT_VALIDATOR and READ_CACHE_HIT_VALIDATOR, - # we define procs to validate cache hit. The only other acceptable value is nil, - # in the case of errors being raised. - # - # If a command has no empty response, set ->(val) { true } - # - # Ref: https://www.rubydoc.info/github/redis/redis-rb/Redis/Commands - # - READ_CACHE_HIT_VALIDATOR = { - exists: ->(val) { val != 0 }, - exists?: ->(val) { val }, - get: ->(val) { !val.nil? }, - hexists: ->(val) { val }, - hget: ->(val) { !val.nil? }, - hgetall: ->(val) { val.is_a?(Hash) && !val.empty? }, - hlen: ->(val) { val != 0 }, - hmget: ->(val) { val.is_a?(Array) && !val.compact.empty? }, - hscan_each: ->(val) { val.is_a?(Enumerator) && !val.first.nil? }, - mapped_hmget: ->(val) { val.is_a?(Hash) && !val.compact.empty? }, - mget: ->(val) { val.is_a?(Array) && !val.compact.empty? }, - scan_each: ->(val) { val.is_a?(Enumerator) && !val.first.nil? }, - scard: ->(val) { val != 0 }, - sismember: ->(val) { val }, - smembers: ->(val) { val.is_a?(Array) && !val.empty? }, - sscan: ->(val) { val != ['0', []] }, - sscan_each: ->(val) { val.is_a?(Enumerator) && !val.first.nil? }, - ttl: ->(val) { val != 0 && val != -2 }, # ttl returns -2 if the key does not exist. See https://redis.io/commands/ttl/ - zscan_each: ->(val) { val.is_a?(Enumerator) && !val.first.nil? } - }.freeze + READ_COMMANDS = %i[ + exists + exists? + get + hexists + hget + hgetall + hlen + hmget + hscan_each + mapped_hmget + mget + scan_each + scard + sismember + smembers + sscan + sscan_each + ttl + zscan_each + ].freeze WRITE_COMMANDS = %i[ del @@ -111,7 +97,7 @@ module Gitlab end # rubocop:disable GitlabSecurity/PublicSend - READ_CACHE_HIT_VALIDATOR.each_key do |name| + READ_COMMANDS.each do |name| define_method(name) do |*args, **kwargs, &block| if use_primary_and_secondary_stores? read_command(name, *args, **kwargs, &block) @@ -186,12 +172,6 @@ module Gitlab @pipelined_command_error.increment(command: command_name, instance_name: instance_name) end - def increment_read_fallback_count(command_name) - @read_fallback_counter ||= Gitlab::Metrics.counter(:gitlab_redis_multi_store_read_fallback_total, - 'Client side Redis MultiStore reading fallback') - @read_fallback_counter.increment(command: command_name, instance_name: instance_name) - end - def increment_method_missing_count(command_name) @method_missing_counter ||= Gitlab::Metrics.counter(:gitlab_redis_multi_store_method_missing_total, 'Client side Redis MultiStore method missing') @@ -247,7 +227,7 @@ module Gitlab if @instance send_command(@instance, command_name, *args, **kwargs, &block) else - read_one_with_fallback(command_name, *args, **kwargs, &block) + read_from_default(command_name, *args, **kwargs, &block) end end @@ -259,35 +239,12 @@ module Gitlab end end - def read_one_with_fallback(command_name, *args, **kwargs, &block) - begin - value = send_command(default_store, command_name, *args, **kwargs, &block) - rescue StandardError => e - log_error(e, command_name, - multi_store_error_message: FAILED_TO_READ_ERROR_MESSAGE) - end - - return value if block.nil? && cache_hit?(command_name, value) - - fallback_read(command_name, *args, **kwargs, &block) - end - - def cache_hit?(command, value) - validator = READ_CACHE_HIT_VALIDATOR[command] - return false unless validator - - !value.nil? && validator.call(value) - end - - def fallback_read(command_name, *args, **kwargs, &block) - value = send_command(fallback_store, command_name, *args, **kwargs, &block) - - if value - log_error(ReadFromPrimaryError.new, command_name) - increment_read_fallback_count(command_name) - end - - value + def read_from_default(command_name, *args, **kwargs, &block) + send_command(default_store, command_name, *args, **kwargs, &block) + rescue StandardError => e + log_error(e, command_name, + multi_store_error_message: FAILED_TO_READ_ERROR_MESSAGE) + raise end def write_both(command_name, *args, **kwargs, &block) diff --git a/lib/gitlab/redis/rate_limiting.rb b/lib/gitlab/redis/rate_limiting.rb index 12710bafbea..62ab00c2408 100644 --- a/lib/gitlab/redis/rate_limiting.rb +++ b/lib/gitlab/redis/rate_limiting.rb @@ -3,6 +3,9 @@ module Gitlab module Redis class RateLimiting < ::Gitlab::Redis::Wrapper + # We create a subclass only for the purpose of differentiating between different stores in cache metrics + RateLimitingStore = Class.new(ActiveSupport::Cache::RedisCacheStore) + class << self # The data we store on RateLimiting used to be stored on Cache. def config_fallback @@ -10,11 +13,7 @@ module Gitlab end def cache_store - @cache_store ||= ActiveSupport::Cache::RedisCacheStore.new( - redis: pool, - namespace: Cache::CACHE_NAMESPACE, - error_handler: ::Gitlab::Redis::ERROR_HANDLER - ) + @cache_store ||= RateLimitingStore.new(redis: pool, namespace: Cache::CACHE_NAMESPACE) end private diff --git a/lib/gitlab/redis/repository_cache.rb b/lib/gitlab/redis/repository_cache.rb index 6c7bc8c41d5..966c6584aa5 100644 --- a/lib/gitlab/redis/repository_cache.rb +++ b/lib/gitlab/redis/repository_cache.rb @@ -3,6 +3,9 @@ module Gitlab module Redis class RepositoryCache < ::Gitlab::Redis::Wrapper + # We create a subclass only for the purpose of differentiating between different stores in cache metrics + RepositoryCacheStore = Class.new(ActiveSupport::Cache::RedisCacheStore) + class << self # The data we store on RepositoryCache used to be stored on Cache. def config_fallback @@ -10,12 +13,11 @@ module Gitlab end def cache_store - @cache_store ||= ActiveSupport::Cache::RedisCacheStore.new( + @cache_store ||= RepositoryCacheStore.new( redis: pool, compress: Gitlab::Utils.to_boolean(ENV.fetch('ENABLE_REDIS_CACHE_COMPRESSION', '1')), namespace: Cache::CACHE_NAMESPACE, - expires_in: Cache.default_ttl_seconds, - error_handler: ::Gitlab::Redis::ERROR_HANDLER + expires_in: Cache.default_ttl_seconds ) end end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 93d23add5eb..5b235639ae8 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -5,7 +5,12 @@ module Gitlab module Packages CONAN_RECIPE_FILES = %w[conanfile.py conanmanifest.txt conan_sources.tgz conan_export.tgz].freeze CONAN_PACKAGE_FILES = %w[conaninfo.txt conanmanifest.txt conan_package.tgz].freeze + PYPI_NORMALIZED_NAME_REGEX_STRING = '[-_.]+' + + # see https://github.com/apache/maven/blob/c1dfb947b509e195c75d4275a113598cf3063c3e/maven-artifact/src/main/java/org/apache/maven/artifact/Artifact.java#L46 + MAVEN_SNAPSHOT_DYNAMIC_PARTS = /\A.{0,1000}(-\d{8}\.\d{6}-\d+).{0,1000}\z/.freeze + API_PATH_REGEX = %r{^/api/v\d+/(projects/[^/]+/|groups?/[^/]+/-/)?packages/[A-Za-z]+}.freeze def conan_package_reference_regex @@ -141,7 +146,7 @@ module Gitlab end def debian_direct_upload_filename_regex - @debian_direct_upload_filename_regex ||= %r{\A.*\.(deb|udeb)\z}o.freeze + @debian_direct_upload_filename_regex ||= %r{\A.*\.(deb|udeb|ddeb)\z}o.freeze end def helm_channel_regex @@ -265,7 +270,7 @@ module Gitlab # eg 'source/full/path' or 'destination_namespace' not 'https://example.com/destination/namespace/path' # the regex also allows for an empty string ('') to be accepted as this is allowed in # a bulk_import POST request - @bulk_import_destination_namespace_path_regex ||= %r/((\A\z)|\A([.]?)[^\W](\/?[.]?[0-9a-z][-_]*)+\z)/i + @bulk_import_destination_namespace_path_regex ||= %r/((\A\z)|\A([.]?)\w*([0-9a-z][-_]*)(\/?[.]?[0-9a-z][-_]*)+\z)/i end def bulk_import_source_full_path_regex @@ -548,11 +553,11 @@ module Gitlab end def issue - @issue ||= /(?<issue>\d+)(?<format>\+)?(?=\W|\z)/ + @issue ||= /(?<issue>\d+)(?<format>\+s{,1})?(?=\W|\z)/ end def merge_request - @merge_request ||= /(?<merge_request>\d+)(?<format>\+)?/ + @merge_request ||= /(?<merge_request>\d+)(?<format>\+s{,1})?/ end def base64_regex diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 37414f9e2b1..93befc2df57 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -120,6 +120,14 @@ module Gitlab [] end + def failed? + false + end + + def error + nil + end + private def collection_for(scope) diff --git a/lib/gitlab/seeders/ci/runner/runner_fleet_seeder.rb b/lib/gitlab/seeders/ci/runner/runner_fleet_seeder.rb index 082d267442c..9340f67f73e 100644 --- a/lib/gitlab/seeders/ci/runner/runner_fleet_seeder.rb +++ b/lib/gitlab/seeders/ci/runner/runner_fleet_seeder.rb @@ -66,14 +66,18 @@ module Gitlab plan_limits = Plan.default.actual_limits if plan_limits.ci_registered_group_runners < @runner_count - logger.error('The plan limits for group runners is set to ' \ + warn 'The plan limits for group runners is set to ' \ "#{plan_limits.ci_registered_group_runners} runners. " \ - 'You should raise the plan limits to avoid errors during runner creation') + "You should raise the plan limits to avoid errors during runner creation by running " \ + "the following command in the Rails console:\n" \ + "Plan.default.actual_limits.update!(ci_registered_group_runners: #{@runner_count})" return false elsif plan_limits.ci_registered_project_runners < @runner_count - logger.error('The plan limits for project runners is set to ' \ + warn 'The plan limits for project runners is set to ' \ "#{plan_limits.ci_registered_project_runners} runners. " \ - 'You should raise the plan limits to avoid errors during runner creation') + "You should raise the plan limits to avoid errors during runner creation by running " \ + "the following command in the Rails console:\n" \ + "Plan.default.actual_limits.update!(ci_registered_project_runners: #{@runner_count})" return false end diff --git a/lib/gitlab/serializer/ci/variables.rb b/lib/gitlab/serializer/ci/variables.rb index 9abf3a54f37..a12bda0e5a7 100644 --- a/lib/gitlab/serializer/ci/variables.rb +++ b/lib/gitlab/serializer/ci/variables.rb @@ -12,7 +12,7 @@ module Gitlab def load(string) return unless string - object = YAML.safe_load(string, [Symbol]) + object = YAML.safe_load(string, permitted_classes: [Symbol]) object.map do |variable| variable.symbolize_keys.tap do |variable| diff --git a/lib/gitlab/serverless/service.rb b/lib/gitlab/serverless/service.rb deleted file mode 100644 index c3ab2e9ddeb..00000000000 --- a/lib/gitlab/serverless/service.rb +++ /dev/null @@ -1,102 +0,0 @@ -# frozen_string_literal: true - -class Gitlab::Serverless::Service - include Gitlab::Utils::StrongMemoize - - def initialize(attributes) - @attributes = attributes - end - - def name - @attributes.dig('metadata', 'name') - end - - def namespace - @attributes.dig('metadata', 'namespace') - end - - def environment_scope - @attributes.dig('environment_scope') - end - - def environment - @attributes.dig('environment') - end - - def podcount - @attributes.dig('podcount') - end - - def created_at - strong_memoize(:created_at) do - timestamp = @attributes.dig('metadata', 'creationTimestamp') - DateTime.parse(timestamp) if timestamp - end - end - - def image - @attributes.dig( - 'spec', - 'runLatest', - 'configuration', - 'build', - 'template', - 'name') - end - - def description - knative_07_description || knative_05_06_description - end - - def cluster - @attributes.dig('cluster') - end - - def url - proxy_url || knative_06_07_url || knative_05_url - end - - private - - def proxy_url - if cluster&.serverless_domain - ::Serverless::Domain.new( - function_name: name, - serverless_domain_cluster: cluster.serverless_domain, - environment: environment - ).uri.to_s - end - end - - def knative_07_description - @attributes.dig( - 'spec', - 'template', - 'metadata', - 'annotations', - 'Description' - ) - end - - def knative_05_06_description - @attributes.dig( - 'spec', - 'runLatest', - 'configuration', - 'revisionTemplate', - 'metadata', - 'annotations', - 'Description') - end - - def knative_05_url - domain = @attributes.dig('status', 'domain') - return unless domain - - "http://#{domain}" - end - - def knative_06_07_url - @attributes.dig('status', 'url') - end -end diff --git a/lib/gitlab/slash_commands/incident_management/incident_new.rb b/lib/gitlab/slash_commands/incident_management/incident_new.rb index 722fcff151d..ce91edfd51a 100644 --- a/lib/gitlab/slash_commands/incident_management/incident_new.rb +++ b/lib/gitlab/slash_commands/incident_management/incident_new.rb @@ -8,8 +8,8 @@ module Gitlab 'incident declare' end - def self.allowed?(project, user) - Feature.enabled?(:incident_declare_slash_command, user) && can?(user, :create_incident, project) + def self.allowed?(_project, _user) + Feature.enabled?(:incident_declare_slash_command) end def self.match(text) diff --git a/lib/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb index 9dba8c99b99..b9800a4db73 100644 --- a/lib/gitlab/task_helpers.rb +++ b/lib/gitlab/task_helpers.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true require 'rainbow/ext/string' -require_dependency 'gitlab/utils/strong_memoize' +require_relative 'utils/strong_memoize' # rubocop:disable Rails/Output module Gitlab diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index 00e609511f2..1be9190e5f8 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -7,6 +7,8 @@ module Gitlab class UrlBlocker BlockedUrlError = Class.new(StandardError) + DENY_ALL_REQUESTS_EXCEPT_ALLOWED_DEFAULT = proc { deny_all_requests_except_allowed_app_setting }.freeze + class << self # Validates the given url according to the constraints specified by arguments. # @@ -17,6 +19,7 @@ module Gitlab # ascii_only - Raises error if URL has unicode characters and argument is true. # enforce_user - Raises error if URL user doesn't start with alphanumeric characters and argument is true. # enforce_sanitization - Raises error if URL includes any HTML/CSS/JS tags and argument is true. + # deny_all_requests_except_allowed - Raises error if URL is not in the allow list and argument is true. Can be Boolean or Proc. Defaults to instance app setting. # # Returns an array with [<uri>, <original-hostname>]. # rubocop:disable Metrics/ParameterLists @@ -30,6 +33,7 @@ module Gitlab ascii_only: false, enforce_user: false, enforce_sanitization: false, + deny_all_requests_except_allowed: DENY_ALL_REQUESTS_EXCEPT_ALLOWED_DEFAULT, dns_rebind_protection: true) # rubocop:enable Metrics/ParameterLists @@ -49,21 +53,28 @@ module Gitlab ascii_only: ascii_only ) - address_info = get_address_info(uri, dns_rebind_protection) - return [uri, nil] unless address_info + begin + address_info = get_address_info(uri) + rescue SocketError + return [uri, nil] unless enforce_address_info_retrievable?(uri, dns_rebind_protection, deny_all_requests_except_allowed) + + raise BlockedUrlError, 'Host cannot be resolved or invalid' + end ip_address = ip_address(address_info) - return [uri, nil] if domain_allowed?(uri) + return [uri, nil] if domain_in_allow_list?(uri) protected_uri_with_hostname = enforce_uri_hostname(ip_address, uri, dns_rebind_protection) - return protected_uri_with_hostname if ip_allowed?(ip_address, port: get_port(uri)) + return protected_uri_with_hostname if ip_in_allow_list?(ip_address, port: get_port(uri)) # Allow url from the GitLab instance itself but only for the configured hostname and ports return protected_uri_with_hostname if internal?(uri) return protected_uri_with_hostname if allow_object_storage && object_storage_endpoint?(uri) + validate_deny_all_requests_except_allowed!(deny_all_requests_except_allowed) + validate_local_request( address_info: address_info, allow_localhost: allow_localhost, @@ -115,29 +126,41 @@ module Gitlab validate_unicode_restriction(uri) if ascii_only end - def get_address_info(uri, dns_rebind_protection) + # Returns addrinfo object for the URI. + # + # @param uri [Addressable::URI] + # + # @raise [Gitlab::UrlBlocker::BlockedUrlError, ArgumentError] - BlockedUrlError raised if host is too long. + # + # @return [Array<Addrinfo>] + def get_address_info(uri) Addrinfo.getaddrinfo(uri.hostname, get_port(uri), nil, :STREAM).map do |addr| addr.ipv6_v4mapped? ? addr.ipv6_to_ipv4 : addr end - rescue SocketError - # If the dns rebinding protection is not enabled or the domain - # is allowed we avoid the dns rebinding checks - return if domain_allowed?(uri) || !dns_rebind_protection + rescue ArgumentError => error + # Addrinfo.getaddrinfo errors if the domain exceeds 1024 characters. + raise unless error.message.include?('hostname too long') + + raise BlockedUrlError, "Host is too long (maximum is 1024 characters)" + end + + def enforce_address_info_retrievable?(uri, dns_rebind_protection, deny_all_requests_except_allowed) + # Do not enforce if URI is in the allow list + return false if domain_in_allow_list?(uri) + + # Enforce if the instance should block requests + return true if deny_all_requests_except_allowed?(deny_all_requests_except_allowed) + + # Do not enforce unless DNS rebinding protection is enabled + return false unless dns_rebind_protection # In the test suite we use a lot of mocked urls that are either invalid or # don't exist. In order to avoid modifying a ton of tests and factories # we allow invalid urls unless the environment variable RSPEC_ALLOW_INVALID_URLS # is not true - return if Rails.env.test? && ENV['RSPEC_ALLOW_INVALID_URLS'] == 'true' - - # If the addr can't be resolved or the url is invalid (i.e http://1.1.1.1.1) - # we block the url - raise BlockedUrlError, "Host cannot be resolved or invalid" - rescue ArgumentError => error - # Addrinfo.getaddrinfo errors if the domain exceeds 1024 characters. - raise unless error.message.include?('hostname too long') + return false if Rails.env.test? && ENV['RSPEC_ALLOW_INVALID_URLS'] == 'true' - raise BlockedUrlError, "Host is too long (maximum is 1024 characters)" + true end def validate_local_request( @@ -260,6 +283,15 @@ module Gitlab raise BlockedUrlError, "Requests to the link local network are not allowed" end + # Raises a BlockedUrlError if the instance is configured to deny all requests. + # + # This should only be called after allow list checks have been made. + def validate_deny_all_requests_except_allowed!(should_deny) + return unless deny_all_requests_except_allowed?(should_deny) + + raise BlockedUrlError, "Requests to hosts and IP addresses not on the Allow List are denied" + end + # Raises a BlockedUrlError if any IP in `addrs_info` is the limited # broadcast address. # https://datatracker.ietf.org/doc/html/rfc919#section-7 @@ -302,6 +334,15 @@ module Gitlab end.compact.uniq end + def deny_all_requests_except_allowed?(should_deny) + should_deny.is_a?(Proc) ? should_deny.call : should_deny + end + + def deny_all_requests_except_allowed_app_setting + Gitlab::CurrentSettings.current_application_settings? && + Gitlab::CurrentSettings.deny_all_requests_except_allowed? + end + def object_storage_endpoint?(uri) enabled_object_storage_endpoints.any? do |endpoint| endpoint_uri = URI(endpoint) @@ -312,11 +353,11 @@ module Gitlab end end - def domain_allowed?(uri) + def domain_in_allow_list?(uri) Gitlab::UrlBlockers::UrlAllowlist.domain_allowed?(uri.normalized_host, port: get_port(uri)) end - def ip_allowed?(ip_address, port: nil) + def ip_in_allow_list?(ip_address, port: nil) Gitlab::UrlBlockers::UrlAllowlist.ip_allowed?(ip_address, port: port) end diff --git a/lib/gitlab/usage/metrics/aggregates/aggregate.rb b/lib/gitlab/usage/metrics/aggregates/aggregate.rb index b68e1ace658..a0a58534661 100644 --- a/lib/gitlab/usage/metrics/aggregates/aggregate.rb +++ b/lib/gitlab/usage/metrics/aggregates/aggregate.rb @@ -7,11 +7,6 @@ module Gitlab class Aggregate include Gitlab::Usage::TimeFrame - # TODO: define this missing event https://gitlab.com/gitlab-org/gitlab/-/issues/385080 - EVENTS_NOT_DEFINED_YET = %w[ - i_code_review_merge_request_widget_license_compliance_warning - ].freeze - def initialize(recorded_at) @recorded_at = recorded_at end @@ -84,7 +79,7 @@ module Gitlab return events if source != ::Gitlab::Usage::Metrics::Aggregates::REDIS_SOURCE events.select do |event| - ::Gitlab::UsageDataCounters::HLLRedisCounter.known_event?(event) || EVENTS_NOT_DEFINED_YET.include?(event) + ::Gitlab::UsageDataCounters::HLLRedisCounter.known_event?(event) end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb index 642b67a3b02..ca122ccf6f3 100644 --- a/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb @@ -23,6 +23,7 @@ module Gitlab scope = super scope = scope.where(source_type: source_type) if source_type.present? scope = scope.where(status: status) if status.present? + scope = scope.where(has_failures: failures) if failures.present? scope end @@ -34,6 +35,10 @@ module Gitlab options[:status] end + def failures + options[:has_failures].to_s + end + def allowed_source_types BulkImports::Entity.source_types.keys.map(&:to_s) end diff --git a/lib/gitlab/usage/metrics/instrumentations/gitlab_dedicated_metric.rb b/lib/gitlab/usage/metrics/instrumentations/gitlab_dedicated_metric.rb new file mode 100644 index 00000000000..b7ca5fadd5b --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/gitlab_dedicated_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class GitlabDedicatedMetric < GenericMetric + value do + Gitlab::CurrentSettings.gitlab_dedicated_instance + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/index_inconsistencies_metric.rb b/lib/gitlab/usage/metrics/instrumentations/index_inconsistencies_metric.rb new file mode 100644 index 00000000000..409027925d1 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/index_inconsistencies_metric.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class IndexInconsistenciesMetric < GenericMetric + value do + runner = Gitlab::Database::SchemaValidation::Runner.new(structure_sql, database, validators: validators) + + inconsistencies = runner.execute + + inconsistencies.map do |inconsistency| + { + object_name: inconsistency.object_name, + inconsistency_type: inconsistency.type + } + end + end + + class << self + private + + def database + database_model = Gitlab::Database.database_base_models[Gitlab::Database::MAIN_DATABASE_NAME] + Gitlab::Database::SchemaValidation::Database.new(database_model.connection) + end + + def structure_sql + stucture_sql_path = Rails.root.join('db/structure.sql') + Gitlab::Database::SchemaValidation::StructureSql.new(stucture_sql_path) + end + + def validators + [ + Gitlab::Database::SchemaValidation::Validators::MissingIndexes, + Gitlab::Database::SchemaValidation::Validators::DifferentDefinitionIndexes, + Gitlab::Database::SchemaValidation::Validators::ExtraIndexes + ] + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/installation_creation_date_metric.rb b/lib/gitlab/usage/metrics/instrumentations/installation_creation_date_metric.rb new file mode 100644 index 00000000000..c2ca62f9eba --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/installation_creation_date_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class InstallationCreationDateMetric < GenericMetric + value do + User.where(id: 1).pick(:created_at) + end + end + end + end + end +end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 53794854bd0..52b8d70c113 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -333,24 +333,10 @@ module Gitlab end def jira_usage - # Jira Cloud does not support custom domains as per https://jira.atlassian.com/browse/CLOUD-6999 - # so we can just check for subdomains of atlassian.net - jira_integration_data_hash = jira_integration_data - if jira_integration_data_hash.nil? - return { projects_jira_server_active: FALLBACK, projects_jira_cloud_active: FALLBACK } - end - - results = { - projects_jira_server_active: 0, - projects_jira_cloud_active: 0, + { projects_jira_dvcs_cloud_active: count(ProjectFeatureUsage.with_jira_dvcs_integration_enabled), projects_jira_dvcs_server_active: count(ProjectFeatureUsage.with_jira_dvcs_integration_enabled(cloud: false)) } - - results[:projects_jira_server_active] = jira_integration_data_hash[:projects_jira_server_active] - results[:projects_jira_cloud_active] = jira_integration_data_hash[:projects_jira_cloud_active] - - results end # rubocop: enable CodeReuse/ActiveRecord @@ -385,13 +371,11 @@ module Gitlab end def merge_requests_users(time_period) - counter = Gitlab::UsageDataCounters::TrackUniqueEvents - redis_usage_data do - counter.count_unique_events( - event_action: Gitlab::UsageDataCounters::TrackUniqueEvents::MERGE_REQUEST_ACTION, - date_from: time_period[:created_at].first, - date_to: time_period[:created_at].last + Gitlab::UsageDataCounters::HLLRedisCounter.unique_events( + event_names: :merge_request_action, + start_date: time_period[:created_at].first, + end_date: time_period[:created_at].last ) end end @@ -410,7 +394,7 @@ module Gitlab end.data platform = ohai_data['platform'] - platform = 'raspbian' if ohai_data['platform'] == 'debian' && /armv/.match?(ohai_data['kernel']['machine']) + platform = 'raspbian' if ohai_data['platform'] == 'debian' && ohai_data['kernel']['machine']&.include?('armv') "#{platform}-#{ohai_data['platform_version']}" end @@ -464,10 +448,7 @@ module Gitlab remote_mirrors: distinct_count(::Project.with_remote_mirrors.where(time_period), :creator_id), snippets: distinct_count(::Snippet.where(time_period), :author_id) }.tap do |h| - if time_period.present? - h[:merge_requests_users] = merge_requests_users(time_period) - h.merge!(action_monthly_active_users(time_period)) - end + h[:merge_requests_users] = merge_requests_users(time_period) if time_period.present? end end # rubocop: enable CodeReuse/ActiveRecord @@ -527,7 +508,6 @@ module Gitlab # Omitted because no user, creator or author associated: `boards`, `labels`, `milestones`, `uploads` # Omitted because too expensive: `epics_deepest_relationship_level` - # Omitted because of encrypted properties: `projects_jira_cloud_active`, `projects_jira_server_active` # rubocop: disable CodeReuse/ActiveRecord def usage_activity_by_stage_plan(time_period) time_frame = metric_time_period(time_period) @@ -582,17 +562,6 @@ module Gitlab {} end - def action_monthly_active_users(time_period) - counter = Gitlab::UsageDataCounters::EditorUniqueCounter - date_range = { date_from: time_period[:created_at].first, date_to: time_period[:created_at].last } - - { - action_monthly_active_users_web_ide_edit: redis_usage_data { counter.count_web_ide_edit_actions(**date_range) }, - action_monthly_active_users_sfe_edit: redis_usage_data { counter.count_sfe_edit_actions(**date_range) }, - action_monthly_active_users_snippet_editor_edit: redis_usage_data { counter.count_snippet_editor_edit_actions(**date_range) } - } - end - def with_metadata result = nil error = nil diff --git a/lib/gitlab/usage_data_counters/container_registry_event_counter.rb b/lib/gitlab/usage_data_counters/container_registry_event_counter.rb new file mode 100644 index 00000000000..5d54bb18443 --- /dev/null +++ b/lib/gitlab/usage_data_counters/container_registry_event_counter.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + class ContainerRegistryEventCounter < BaseCounter + KNOWN_EVENTS = %w[i_container_registry_delete_manifest].freeze + PREFIX = 'container_registry_events' + end + end +end diff --git a/lib/gitlab/usage_data_counters/editor_unique_counter.rb b/lib/gitlab/usage_data_counters/editor_unique_counter.rb index 2aebc1b8813..4e4a01ed301 100644 --- a/lib/gitlab/usage_data_counters/editor_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/editor_unique_counter.rb @@ -38,18 +38,16 @@ module Gitlab def track_unique_action(event_name, author, time, project = nil) return unless author - if Feature.enabled?(:route_hll_to_snowplow_phase2) - Gitlab::Tracking.event( - name, - 'ide_edit', - property: event_name.to_s, - project: project, - namespace: project&.namespace, - user: author, - label: 'usage_activity_by_stage_monthly.create.action_monthly_active_users_ide_edit', - context: [Gitlab::Tracking::ServicePingContext.new(data_source: :redis_hll, event: event_name).to_context] - ) - end + Gitlab::Tracking.event( + name, + 'ide_edit', + property: event_name.to_s, + project: project, + namespace: project&.namespace, + user: author, + label: 'usage_activity_by_stage_monthly.create.action_monthly_active_users_ide_edit', + context: [Gitlab::Tracking::ServicePingContext.new(data_source: :redis_hll, event: event_name).to_context] + ) Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name, values: author.id, time: time) end diff --git a/lib/gitlab/usage_data_counters/gitlab_cli_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/gitlab_cli_activity_unique_counter.rb index 8a57a0331b8..b30c4b675f9 100644 --- a/lib/gitlab/usage_data_counters/gitlab_cli_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/gitlab_cli_activity_unique_counter.rb @@ -4,7 +4,9 @@ module Gitlab module UsageDataCounters module GitLabCliActivityUniqueCounter GITLAB_CLI_API_REQUEST_ACTION = 'i_code_review_user_gitlab_cli_api_request' - GITLAB_CLI_USER_AGENT_REGEX = /GitLab\sCLI$/.freeze + + # This regex will match to user agents ending with GitLab CLI or starting with glab/v" + GITLAB_CLI_USER_AGENT_REGEX = %r{(GitLab\sCLI$|^glab/v)}.freeze class << self def track_api_request_when_trackable(user_agent:, user:) diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index b809e6c4e42..4b7ec45bcca 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -5,26 +5,17 @@ module Gitlab module HLLRedisCounter DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH = 6.weeks DEFAULT_DAILY_KEY_EXPIRY_LENGTH = 29.days - DEFAULT_REDIS_SLOT = '' + REDIS_SLOT = 'hll_counters' EventError = Class.new(StandardError) UnknownEvent = Class.new(EventError) UnknownAggregation = Class.new(EventError) AggregationMismatch = Class.new(EventError) - SlotMismatch = Class.new(EventError) - CategoryMismatch = Class.new(EventError) InvalidContext = Class.new(EventError) KNOWN_EVENTS_PATH = File.expand_path('known_events/*.yml', __dir__) ALLOWED_AGGREGATIONS = %i(daily weekly).freeze - CATEGORIES_FOR_TOTALS = %w[ - compliance - error_tracking - ide_edit - pipeline_authoring - ].freeze - # Track event on entity_id # Increment a Redis HLL counter for unique event_name and entity_id # @@ -33,10 +24,7 @@ module Gitlab # Event example: # # - name: g_compliance_dashboard # Unique event name - # redis_slot: compliance # Optional slot name, if not defined it will use name as a slot, used for totals - # category: compliance # Group events in categories # aggregation: daily # Aggregation level, keys are stored daily or weekly - # feature_flag: # The event feature flag # # Usage: # @@ -76,23 +64,11 @@ module Gitlab # context - Event context, plan level tracking. Available if set when tracking. def unique_events(event_names:, start_date:, end_date:, context: '') count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date, context: context) do |events| - raise SlotMismatch, events unless events_in_same_slot?(events) - raise CategoryMismatch, events unless events_in_same_category?(events) raise AggregationMismatch, events unless events_same_aggregation?(events) raise InvalidContext if context.present? && !context.in?(valid_context_list) end end - def categories - @categories ||= known_events.map { |event| event[:category] }.uniq - end - - # @param category [String] the category name - # @return [Array<String>] list of event names for given category - def events_for_category(category) - known_events.select { |event| event[:category] == category.to_s }.map { |event| event[:name] } - end - def known_event?(event_name) event_for(event_name).present? end @@ -103,7 +79,6 @@ module Gitlab def calculate_events_union(event_names:, start_date:, end_date:) count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date) do |events| - raise SlotMismatch, events unless events_in_same_slot?(events) raise AggregationMismatch, events unless events_same_aggregation?(events) end end @@ -117,7 +92,7 @@ module Gitlab Gitlab::ErrorTracking.track_and_raise_for_dev_exception(UnknownEvent.new("Unknown event #{event_name}")) unless event.present? return if event.blank? - return unless feature_enabled?(event) + return unless Feature.enabled?(:redis_hll_tracking, type: :ops) Gitlab::Redis::HLL.add(key: redis_key(event, time, context), value: values, expiry: expiry(event)) rescue StandardError => e @@ -145,21 +120,6 @@ module Gitlab redis_usage_data { Gitlab::Redis::HLL.count(keys: keys) } end - def feature_enabled?(event) - return true if event[:feature_flag].blank? - - Feature.enabled?(event[:feature_flag]) && Feature.enabled?(:redis_hll_tracking, type: :ops) - end - - # Allow to add totals for events that are in the same redis slot, category and have the same aggregation level - # and if there are more than 1 event - def eligible_for_totals?(events_names) - return false if events_names.size <= 1 - - events = events_for(events_names) - events_in_same_slot?(events) && events_in_same_category?(events) && events_same_aggregation?(events) - end - def keys_for_aggregation(aggregation, events:, start_date:, end_date:, context: '') if aggregation.to_sym == :daily daily_redis_keys(events: events, start_date: start_date, end_date: end_date, context: context) @@ -182,20 +142,6 @@ module Gitlab known_events.map { |event| event[:name] } end - def events_in_same_slot?(events) - # if we check one event then redis_slot is only one to check - return false if events.empty? - return true if events.size == 1 - - slot = events.first[:redis_slot] - events.all? { |event| event[:redis_slot].present? && event[:redis_slot] == slot } - end - - def events_in_same_category?(events) - category = events.first[:category] - events.all? { |event| event[:category] == category } - end - def events_same_aggregation?(events) aggregation = events.first[:aggregation] events.all? { |event| event[:aggregation] == aggregation } @@ -213,30 +159,17 @@ module Gitlab known_events.select { |event| event_names.include?(event[:name]) } end - def redis_slot(event) - event[:redis_slot] || DEFAULT_REDIS_SLOT - end - # Compose the key in order to store events daily or weekly def redis_key(event, time, context = '') raise UnknownEvent, "Unknown event #{event[:name]}" unless known_events_names.include?(event[:name].to_s) raise UnknownAggregation, "Use :daily or :weekly aggregation" unless ALLOWED_AGGREGATIONS.include?(event[:aggregation].to_sym) - key = apply_slot(event) + key = "{#{REDIS_SLOT}}_#{event[:name]}" key = apply_time_aggregation(key, time, event) key = "#{context}_#{key}" if context.present? key end - def apply_slot(event) - slot = redis_slot(event) - if slot.present? - event[:name].to_s.gsub(slot, "{#{slot}}") - else - "{#{event[:name]}}" - end - end - def apply_time_aggregation(key, time, event) if event[:aggregation].to_sym == :daily year_day = time.strftime('%G-%j') diff --git a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb index a59ea36961d..c0d1af8a43a 100644 --- a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb @@ -180,7 +180,6 @@ module Gitlab private def track_snowplow_action(event_name, author, project) - return unless Feature.enabled?(:route_hll_to_snowplow_phase2, project.namespace) return unless author Gitlab::Tracking.event( diff --git a/lib/gitlab/usage_data_counters/known_events/analytics.yml b/lib/gitlab/usage_data_counters/known_events/analytics.yml index 85524c766ca..0b30308b552 100644 --- a/lib/gitlab/usage_data_counters/known_events/analytics.yml +++ b/lib/gitlab/usage_data_counters/known_events/analytics.yml @@ -1,52 +1,26 @@ - name: users_viewing_analytics_group_devops_adoption - category: analytics - redis_slot: analytics aggregation: weekly - name: i_analytics_dev_ops_adoption - category: analytics - redis_slot: analytics aggregation: weekly - name: i_analytics_dev_ops_score - category: analytics - redis_slot: analytics aggregation: weekly - name: i_analytics_instance_statistics - category: analytics - redis_slot: analytics aggregation: weekly - name: p_analytics_pipelines - category: analytics - redis_slot: analytics aggregation: weekly - name: p_analytics_valuestream - category: analytics - redis_slot: analytics aggregation: weekly - name: p_analytics_repo - category: analytics - redis_slot: analytics aggregation: weekly - name: i_analytics_cohorts - category: analytics - redis_slot: analytics aggregation: weekly - name: p_analytics_ci_cd_pipelines - category: analytics - redis_slot: analytics aggregation: weekly - name: p_analytics_ci_cd_deployment_frequency - category: analytics - redis_slot: analytics aggregation: weekly - name: p_analytics_ci_cd_lead_time - category: analytics - redis_slot: analytics aggregation: weekly - name: p_analytics_ci_cd_time_to_restore_service - category: analytics - redis_slot: analytics aggregation: weekly - name: p_analytics_ci_cd_change_failure_rate - category: analytics - redis_slot: analytics aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml index b13e3d631c7..82c023e6e38 100644 --- a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml +++ b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml @@ -4,602 +4,304 @@ # Do not edit it manually! --- - name: p_ci_templates_terraform_base_latest - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_terraform_base - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_dotnet - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_nodejs - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_openshift - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_auto_devops - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_bash - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_rust - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_elixir - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_clojure - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_crystal - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_getting_started - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_code_quality - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_verify_load_performance_testing - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_verify_accessibility - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_verify_failfast - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_verify_browser_performance - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_verify_browser_performance_latest - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_grails - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_sast - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_dast_runner_validation - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_dast_on_demand_scan - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_secret_detection - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_license_scanning - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_coverage_fuzzing_latest - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_dast_on_demand_api_scan - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_coverage_fuzzing - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_api_fuzzing_latest - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_secure_binaries - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_dast_api - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_container_scanning - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_dast_latest - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_sast_iac - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_dependency_scanning - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_dast_api_latest - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_container_scanning_latest - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_api_fuzzing - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_dast - category: ci_templates - redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_security_api_discovery aggregation: weekly - name: p_ci_templates_security_fortify_fod_sast - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_sast_iac_latest - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_qualys_iac_security - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_ios_fastlane - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_composer - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_c - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_python - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_android_fastlane - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_android_latest - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_django - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_maven - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_liquibase - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_flutter - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_workflows_branch_pipelines - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_workflows_mergerequest_pipelines - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_laravel - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_kaniko - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_php - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_packer - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_themekit - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_terraform - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_katalon - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_mono - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_go - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_scala - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_latex - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_android - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_indeni_cloudrail - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_matlab - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_deploy_ecs - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_aws_cf_provision_and_deploy_ec2 - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_aws_deploy_ecs - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_gradle - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_chef - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_dast_default_branch_deploy - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_load_performance_testing - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_helm_2to3 - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_sast - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_secret_detection - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_license_scanning - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_code_intelligence - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_code_quality - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_deploy_ecs - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_deploy_ec2 - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_license_scanning_latest - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_deploy - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_build - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_browser_performance_testing - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_container_scanning - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_container_scanning_latest - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_dependency_scanning_latest - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_test - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_sast_latest - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_sast_iac - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_secret_detection_latest - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_dependency_scanning - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_deploy_latest - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_browser_performance_testing_latest - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_cf_provision - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_build_latest - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_sast_iac_latest - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_terraform_latest - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_swift - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_jekyll - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_harp - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_octopress - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_brunch - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_doxygen - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_hyde - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_lektor - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_jbake - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_hexo - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_middleman - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_hugo - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_pelican - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_nanoc - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_swaggerui - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_jigsaw - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_metalsmith - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_gatsby - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_html - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_dart - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_docker - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_julia - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_npm - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_dotnet_core - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_5_minute_production_app - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_ruby - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_auto_devops - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_browser_performance_testing - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_build - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_code_intelligence - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_code_quality - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_container_scanning - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_dast_default_branch_deploy - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_dependency_scanning - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_deploy - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_deploy_ec2 - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_deploy_ecs - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_helm_2to3 - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_license_scanning - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_sast - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_secret_detection - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_test - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_security_container_scanning - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_security_dast - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_security_dependency_scanning - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_security_license_scanning - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_security_sast - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_security_secret_detection - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_terraform_module_base - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_terraform_module - category: ci_templates - redis_slot: ci_templates aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/ci_users.yml b/lib/gitlab/usage_data_counters/known_events/ci_users.yml index b012d61eef5..49757c6e672 100644 --- a/lib/gitlab/usage_data_counters/known_events/ci_users.yml +++ b/lib/gitlab/usage_data_counters/known_events/ci_users.yml @@ -1,10 +1,4 @@ - name: ci_users_executing_deployment_job - category: ci_users - redis_slot: ci_users aggregation: weekly - feature_flag: - name: ci_users_executing_verify_environment_job - category: ci_users - redis_slot: ci_users aggregation: weekly - feature_flag: diff --git a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml index 3bb6655d762..db0c0653f63 100644 --- a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml @@ -1,457 +1,233 @@ --- - name: i_code_review_create_note_in_ipynb_diff - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_create_note_in_ipynb_diff_mr - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_create_note_in_ipynb_diff_commit - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_create_note_in_ipynb_diff - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_create_note_in_ipynb_diff_mr - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_create_note_in_ipynb_diff_commit - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_mr_diffs - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_single_file_diffs - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_mr_single_file_diffs - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_toggled_task_item_status - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_create_mr - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_create_mr - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_close_mr - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_reopen_mr - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_approve_mr - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_unapprove_mr - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_resolve_thread - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_unresolve_thread - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_edit_mr_title - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_edit_mr_desc - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_merge_mr - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_create_mr_comment - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_edit_mr_comment - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_remove_mr_comment - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_create_review_note - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_publish_review - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_create_multiline_mr_comment - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_edit_multiline_mr_comment - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_remove_multiline_mr_comment - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_add_suggestion - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_apply_suggestion - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_assigned - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_marked_as_draft - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_unmarked_as_draft - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_review_requested - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_approval_rule_added - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_approval_rule_deleted - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_approval_rule_edited - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_vs_code_api_request - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_jetbrains_api_request - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_gitlab_cli_api_request - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_create_mr_from_issue - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_mr_discussion_locked - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_mr_discussion_unlocked - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_time_estimate_changed - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_time_spent_changed - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_assignees_changed - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_reviewers_changed - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_milestone_changed - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_labels_changed - redis_slot: code_review - category: code_review aggregation: weekly # Diff settings events - name: i_code_review_click_diff_view_setting - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_click_single_file_mode_setting - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_click_file_browser_setting - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_click_whitespace_setting - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_diff_view_inline - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_diff_view_parallel - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_file_browser_tree_view - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_file_browser_list_view - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_diff_show_whitespace - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_diff_hide_whitespace - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_diff_single_file - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_diff_multiple_files - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_load_conflict_ui - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_resolve_conflict - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_searches_diff - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_total_suggestions_applied - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_total_suggestions_added - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_resolve_thread_in_issue - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_widget_nothing_merge_click_new_file - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_post_merge_delete_branch - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_post_merge_click_revert - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_post_merge_click_cherry_pick - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_post_merge_submit_revert_modal - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_post_merge_submit_cherry_pick_modal - redis_slot: code_review - category: code_review aggregation: weekly # MR Widget Extensions ## Test Summary - name: i_code_review_merge_request_widget_test_summary_view - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_test_summary_full_report_clicked - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_test_summary_expand - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_test_summary_expand_success - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_test_summary_expand_warning - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_test_summary_expand_failed - redis_slot: code_review - category: code_review aggregation: weekly ## Accessibility - name: i_code_review_merge_request_widget_accessibility_view - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_accessibility_full_report_clicked - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_accessibility_expand - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_accessibility_expand_success - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_accessibility_expand_warning - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_accessibility_expand_failed - redis_slot: code_review - category: code_review aggregation: weekly ## Code Quality - name: i_code_review_merge_request_widget_code_quality_view - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_code_quality_full_report_clicked - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_code_quality_expand - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_code_quality_expand_success - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_code_quality_expand_warning - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_code_quality_expand_failed - redis_slot: code_review - category: code_review aggregation: weekly ## Terraform - name: i_code_review_merge_request_widget_terraform_view - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_terraform_full_report_clicked - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_terraform_expand - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_terraform_expand_success - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_terraform_expand_warning - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_terraform_expand_failed - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_submit_review_approve - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_submit_review_comment - redis_slot: code_review - category: code_review aggregation: weekly ## License Compliance - name: i_code_review_merge_request_widget_license_compliance_view - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_license_compliance_full_report_clicked - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_license_compliance_expand - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_license_compliance_expand_success - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_license_compliance_expand_warning - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_license_compliance_expand_failed - redis_slot: code_review - category: code_review aggregation: weekly ## Security Reports - name: i_code_review_merge_request_widget_security_reports_view - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_security_reports_full_report_clicked - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_security_reports_expand - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_security_reports_expand_success - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_security_reports_expand_warning - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_security_reports_expand_failed - redis_slot: code_review - category: code_review aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index ae15530f0d0..f5973587ebb 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -1,313 +1,167 @@ --- # Compliance category - name: g_edit_by_web_ide - category: ide_edit - redis_slot: edit aggregation: daily - name: g_edit_by_sfe - category: ide_edit - redis_slot: edit aggregation: daily - name: g_edit_by_snippet_ide - category: ide_edit - redis_slot: edit aggregation: daily - name: g_edit_by_live_preview - category: ide_edit - redis_slot: edit aggregation: daily - name: i_search_total - category: search - redis_slot: search aggregation: weekly - name: wiki_action - category: source_code aggregation: daily - name: design_action - category: source_code aggregation: daily - name: project_action - category: source_code aggregation: daily - name: git_write_action - category: source_code aggregation: daily - name: merge_request_action - category: source_code aggregation: daily - name: i_source_code_code_intelligence - redis_slot: source_code - category: source_code aggregation: daily # Incident management - name: incident_management_alert_status_changed - redis_slot: incident_management - category: incident_management aggregation: weekly - name: incident_management_alert_assigned - redis_slot: incident_management - category: incident_management aggregation: weekly - name: incident_management_alert_todo - redis_slot: incident_management - category: incident_management aggregation: weekly - name: incident_management_incident_created - redis_slot: incident_management - category: incident_management aggregation: weekly - name: incident_management_incident_reopened - redis_slot: incident_management - category: incident_management aggregation: weekly - name: incident_management_incident_closed - redis_slot: incident_management - category: incident_management aggregation: weekly - name: incident_management_incident_assigned - redis_slot: incident_management - category: incident_management aggregation: weekly - name: incident_management_incident_todo - redis_slot: incident_management - category: incident_management aggregation: weekly - name: incident_management_incident_comment - redis_slot: incident_management - category: incident_management aggregation: weekly - name: incident_management_incident_zoom_meeting - redis_slot: incident_management - category: incident_management aggregation: weekly - name: incident_management_incident_relate - redis_slot: incident_management - category: incident_management aggregation: weekly - name: incident_management_incident_unrelate - redis_slot: incident_management - category: incident_management aggregation: weekly - name: incident_management_incident_change_confidential - redis_slot: incident_management - category: incident_management aggregation: weekly # Incident management timeline events - name: incident_management_timeline_event_created - redis_slot: incident_management - category: incident_management aggregation: weekly - name: incident_management_timeline_event_edited - redis_slot: incident_management - category: incident_management aggregation: weekly - name: incident_management_timeline_event_deleted - redis_slot: incident_management - category: incident_management aggregation: weekly # Incident management alerts - name: incident_management_alert_create_incident - redis_slot: incident_management - category: incident_management_alerts aggregation: weekly # Testing category - name: i_testing_test_case_parsed - category: testing - redis_slot: testing aggregation: weekly - name: i_testing_summary_widget_total - category: testing - redis_slot: testing aggregation: weekly - name: i_testing_test_report_uploaded - category: testing - redis_slot: testing aggregation: weekly - name: i_testing_coverage_report_uploaded - category: testing - redis_slot: testing aggregation: weekly # Project Management group - name: g_project_management_issue_title_changed - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_description_changed - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_assignee_changed - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_made_confidential - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_made_visible - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_created - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_closed - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_reopened - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_label_changed - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_milestone_changed - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_cross_referenced - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_moved - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_related - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_unrelated - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_marked_as_duplicate - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_locked - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_unlocked - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_designs_added - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_designs_modified - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_designs_removed - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_due_date_changed - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_design_comments_removed - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_time_estimate_changed - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_time_spent_changed - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_comment_added - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_comment_edited - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_comment_removed - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_cloned - category: issues_edit - redis_slot: project_management aggregation: daily # Runner group - name: g_runner_fleet_read_jobs_statistics - category: runner - redis_slot: runner aggregation: weekly # Secrets Management - name: i_snippets_show - category: snippets - redis_slot: snippets aggregation: weekly # Terraform - name: p_terraform_state_api_unique_users - category: terraform - redis_slot: terraform aggregation: weekly # Pipeline Authoring group - name: o_pipeline_authoring_unique_users_committing_ciconfigfile - category: pipeline_authoring - redis_slot: pipeline_authoring aggregation: weekly - name: o_pipeline_authoring_unique_users_pushing_mr_ciconfigfile - category: pipeline_authoring - redis_slot: pipeline_authoring aggregation: weekly - name: i_ci_secrets_management_id_tokens_build_created - category: ci_secrets_management - redis_slot: ci_secrets_management aggregation: weekly # Merge request widgets - name: users_expanding_secure_security_report - redis_slot: secure - category: secure aggregation: weekly - name: users_expanding_testing_code_quality_report - redis_slot: testing - category: testing aggregation: weekly - name: users_expanding_testing_accessibility_report - redis_slot: testing - category: testing aggregation: weekly - name: users_expanding_testing_license_compliance_report - redis_slot: testing - category: testing aggregation: weekly - name: users_visiting_testing_license_compliance_full_report - redis_slot: testing - category: testing aggregation: weekly - name: users_visiting_testing_manage_license_compliance - redis_slot: testing - category: testing aggregation: weekly - name: users_clicking_license_testing_visiting_external_website - redis_slot: testing - category: testing aggregation: weekly # Geo group - name: g_geo_proxied_requests - category: geo - redis_slot: geo aggregation: daily # Manage - name: unique_active_user - category: manage aggregation: weekly # Environments page - name: users_visiting_environments_pages - category: environments - redis_slot: users aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/container_registry_events.yml b/lib/gitlab/usage_data_counters/known_events/container_registry_events.yml index e8b14de1769..aa0f9965fa7 100644 --- a/lib/gitlab/usage_data_counters/known_events/container_registry_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/container_registry_events.yml @@ -1,22 +1,11 @@ --- - name: i_container_registry_push_tag_user - category: user_container_registry aggregation: weekly - redis_slot: container_registry - name: i_container_registry_delete_tag_user - category: user_container_registry aggregation: weekly - redis_slot: container_registry - name: i_container_registry_push_repository_user - category: user_container_registry aggregation: weekly - redis_slot: container_registry - name: i_container_registry_delete_repository_user - category: user_container_registry aggregation: weekly - redis_slot: container_registry - name: i_container_registry_create_repository_user - category: user_container_registry aggregation: weekly - redis_slot: container_registry -
\ No newline at end of file diff --git a/lib/gitlab/usage_data_counters/known_events/ecosystem.yml b/lib/gitlab/usage_data_counters/known_events/ecosystem.yml index 7f7c9166086..6e4a893d19a 100644 --- a/lib/gitlab/usage_data_counters/known_events/ecosystem.yml +++ b/lib/gitlab/usage_data_counters/known_events/ecosystem.yml @@ -1,46 +1,24 @@ --- # Ecosystem category - name: i_ecosystem_jira_service_close_issue - category: ecosystem - redis_slot: ecosystem aggregation: weekly - name: i_ecosystem_jira_service_cross_reference - category: ecosystem - redis_slot: ecosystem aggregation: weekly - name: i_ecosystem_slack_service_issue_notification - category: ecosystem - redis_slot: ecosystem aggregation: weekly - name: i_ecosystem_slack_service_push_notification - category: ecosystem - redis_slot: ecosystem aggregation: weekly - name: i_ecosystem_slack_service_deployment_notification - category: ecosystem - redis_slot: ecosystem aggregation: weekly - name: i_ecosystem_slack_service_wiki_page_notification - category: ecosystem - redis_slot: ecosystem aggregation: weekly - name: i_ecosystem_slack_service_merge_request_notification - category: ecosystem - redis_slot: ecosystem aggregation: weekly - name: i_ecosystem_slack_service_note_notification - category: ecosystem - redis_slot: ecosystem aggregation: weekly - name: i_ecosystem_slack_service_tag_push_notification - category: ecosystem - redis_slot: ecosystem aggregation: weekly - name: i_ecosystem_slack_service_confidential_note_notification - category: ecosystem - redis_slot: ecosystem aggregation: weekly - name: i_ecosystem_slack_service_confidential_issue_notification - category: ecosystem - redis_slot: ecosystem aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/error_tracking.yml b/lib/gitlab/usage_data_counters/known_events/error_tracking.yml index d80b711f8eb..ebfd1b274f9 100644 --- a/lib/gitlab/usage_data_counters/known_events/error_tracking.yml +++ b/lib/gitlab/usage_data_counters/known_events/error_tracking.yml @@ -1,9 +1,5 @@ --- - name: error_tracking_view_details - category: error_tracking - redis_slot: error_tracking aggregation: weekly - name: error_tracking_view_list - category: error_tracking - redis_slot: error_tracking aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/importer_events.yml b/lib/gitlab/usage_data_counters/known_events/importer_events.yml index c84d756a013..3346c0556d6 100644 --- a/lib/gitlab/usage_data_counters/known_events/importer_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/importer_events.yml @@ -1,14 +1,14 @@ --- # Importer events - name: github_import_project_start - category: importer - redis_slot: import aggregation: weekly - name: github_import_project_success - category: importer - redis_slot: import aggregation: weekly - name: github_import_project_failure - category: importer + aggregation: weekly +- name: github_import_project_cancelled + redis_slot: import + aggregation: weekly +- name: github_import_project_partially_completed redis_slot: import aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/kubernetes_agent.yml b/lib/gitlab/usage_data_counters/known_events/kubernetes_agent.yml index 966e6c584c7..b3d1c51c0e7 100644 --- a/lib/gitlab/usage_data_counters/known_events/kubernetes_agent.yml +++ b/lib/gitlab/usage_data_counters/known_events/kubernetes_agent.yml @@ -1,4 +1,2 @@ - name: agent_users_using_ci_tunnel - category: kubernetes_agent - redis_slot: agent aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/package_events.yml b/lib/gitlab/usage_data_counters/known_events/package_events.yml index ef8d02fa365..47cc7f98838 100644 --- a/lib/gitlab/usage_data_counters/known_events/package_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/package_events.yml @@ -1,89 +1,45 @@ --- - name: i_package_composer_deploy_token - category: deploy_token_packages aggregation: weekly - redis_slot: package - name: i_package_composer_user - category: user_packages aggregation: weekly - redis_slot: package - name: i_package_conan_deploy_token - category: deploy_token_packages aggregation: weekly - redis_slot: package - name: i_package_conan_user - category: user_packages aggregation: weekly - redis_slot: package - name: i_package_generic_deploy_token - category: deploy_token_packages aggregation: weekly - redis_slot: package - name: i_package_generic_user - category: user_packages aggregation: weekly - redis_slot: package - name: i_package_helm_deploy_token - category: deploy_token_packages aggregation: weekly - redis_slot: package - name: i_package_helm_user - category: user_packages aggregation: weekly - redis_slot: package - name: i_package_maven_deploy_token - category: deploy_token_packages aggregation: weekly - redis_slot: package - name: i_package_maven_user - category: user_packages aggregation: weekly - redis_slot: package - name: i_package_npm_deploy_token - category: deploy_token_packages aggregation: weekly - redis_slot: package - name: i_package_npm_user - category: user_packages aggregation: weekly - redis_slot: package - name: i_package_nuget_deploy_token - category: deploy_token_packages aggregation: weekly - redis_slot: package - name: i_package_nuget_user - category: user_packages aggregation: weekly - redis_slot: package - name: i_package_pypi_deploy_token - category: deploy_token_packages aggregation: weekly - redis_slot: package - name: i_package_pypi_user - category: user_packages aggregation: weekly - redis_slot: package - name: i_package_rubygems_deploy_token - category: deploy_token_packages aggregation: weekly - redis_slot: package - name: i_package_rubygems_user - category: user_packages aggregation: weekly - redis_slot: package - name: i_package_terraform_module_deploy_token - category: deploy_token_packages aggregation: weekly - redis_slot: package - name: i_package_terraform_module_user - category: user_packages aggregation: weekly - redis_slot: package - name: i_package_rpm_user - category: user_packages aggregation: weekly - redis_slot: package - name: i_package_rpm_deploy_token - category: deploy_token_packages aggregation: weekly - redis_slot: package diff --git a/lib/gitlab/usage_data_counters/known_events/quickactions.yml b/lib/gitlab/usage_data_counters/known_events/quickactions.yml index 69b348b9a22..7006173cc59 100644 --- a/lib/gitlab/usage_data_counters/known_events/quickactions.yml +++ b/lib/gitlab/usage_data_counters/known_events/quickactions.yml @@ -1,253 +1,127 @@ --- - name: i_quickactions_assign_multiple - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_approve - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_unapprove - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_assign_single - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_assign_self - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_assign_reviewer - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_award - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_board_move - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_clone - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_close - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_confidential - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_copy_metadata_merge_request - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_copy_metadata_issue - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_create_merge_request - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_done - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_draft - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_due - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_duplicate - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_estimate - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_label - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_lock - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_merge - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_milestone - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_move - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_promote_to_incident - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_timeline - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_ready - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_reassign - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_reassign_reviewer - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_rebase - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_relabel - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_relate - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_remove_due_date - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_remove_estimate - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_remove_milestone - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_remove_time_spent - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_remove_zoom - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_reopen - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_severity - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_shrug - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_spend_subtract - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_spend_add - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_submit_review - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_subscribe - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_tableflip - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_tag - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_target_branch - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_title - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_todo - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_unassign_specific - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_unassign_all - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_unassign_reviewer - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_unlabel_specific - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_unlabel_all - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_unlock - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_unsubscribe - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_wip - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_zoom - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_link - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_invite_email_single - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_invite_email_multiple - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_add_contacts - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_remove_contacts - category: quickactions - redis_slot: quickactions aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/work_items.yml b/lib/gitlab/usage_data_counters/known_events/work_items.yml index d088b6d7e5a..a6e5b9e1af5 100644 --- a/lib/gitlab/usage_data_counters/known_events/work_items.yml +++ b/lib/gitlab/usage_data_counters/known_events/work_items.yml @@ -1,42 +1,21 @@ --- - name: users_updating_work_item_title - category: work_items - redis_slot: users aggregation: weekly - feature_flag: track_work_items_activity - name: users_creating_work_items - category: work_items - redis_slot: users aggregation: weekly - feature_flag: track_work_items_activity - name: users_updating_work_item_dates - category: work_items - redis_slot: users aggregation: weekly - feature_flag: track_work_items_activity - name: users_updating_work_item_labels - category: work_items - redis_slot: users aggregation: weekly - feature_flag: track_work_items_activity - name: users_updating_work_item_milestone - category: work_items - redis_slot: users aggregation: weekly - feature_flag: track_work_items_activity - name: users_updating_work_item_iteration # The event tracks an EE feature. # It's added here so it can be aggregated into the CE/EE 'OR' aggregate metrics. # It will report 0 for CE instances and should not be used with 'AND' aggregators. - category: work_items - redis_slot: users aggregation: weekly - feature_flag: track_work_items_activity - name: users_updating_weight_estimate # The event tracks an EE feature. # It's added here so it can be aggregated into the CE/EE 'OR' aggregate metrics. # It will report 0 for CE instances and should not be used with 'AND' aggregators. - category: work_items - redis_slot: users aggregation: weekly - feature_flag: track_work_items_activity diff --git a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb index c8768164710..fceeacb60ca 100644 --- a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb @@ -68,8 +68,6 @@ module Gitlab track_unique_action_by_merge_request(MR_CREATE_ACTION, merge_request) project = merge_request.target_project - return unless Feature.enabled?(:route_hll_to_snowplow_phase2, project.namespace) - Gitlab::Tracking.event( name, :create, @@ -99,8 +97,6 @@ module Gitlab track_unique_action_by_user(MR_APPROVE_ACTION, user) project = merge_request.target_project - return unless Feature.enabled?(:route_hll_to_snowplow_phase2, project.namespace) - Gitlab::Tracking.event( name, :approve, diff --git a/lib/gitlab/usage_data_counters/track_unique_events.rb b/lib/gitlab/usage_data_counters/track_unique_events.rb deleted file mode 100644 index 20da9665876..00000000000 --- a/lib/gitlab/usage_data_counters/track_unique_events.rb +++ /dev/null @@ -1,81 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module UsageDataCounters - module TrackUniqueEvents - WIKI_ACTION = :wiki_action - DESIGN_ACTION = :design_action - PUSH_ACTION = :project_action - MERGE_REQUEST_ACTION = :merge_request_action - - GIT_WRITE_ACTIONS = [WIKI_ACTION, DESIGN_ACTION, PUSH_ACTION].freeze - GIT_WRITE_ACTION = :git_write_action - - ACTION_TRANSFORMATIONS = HashWithIndifferentAccess.new({ - wiki: { - created: WIKI_ACTION, - updated: WIKI_ACTION, - destroyed: WIKI_ACTION - }, - design: { - created: DESIGN_ACTION, - updated: DESIGN_ACTION, - destroyed: DESIGN_ACTION - }, - project: { - pushed: PUSH_ACTION - }, - merge_request: { - closed: MERGE_REQUEST_ACTION, - merged: MERGE_REQUEST_ACTION, - created: MERGE_REQUEST_ACTION, - commented: MERGE_REQUEST_ACTION - } - }).freeze - - class << self - def track_event(event_action:, event_target:, author_id:, time: Time.zone.now) - return unless valid_target?(event_target) - return unless valid_action?(event_action) - - transformed_target = transform_target(event_target) - transformed_action = transform_action(event_action, transformed_target) - - return unless Gitlab::UsageDataCounters::HLLRedisCounter.known_event?(transformed_action.to_s) - - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(transformed_action.to_s, values: author_id, time: time) - - track_git_write_action(author_id, transformed_action, time) - end - - def count_unique_events(event_action:, date_from:, date_to:) - Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: event_action.to_s, start_date: date_from, end_date: date_to) - end - - private - - def transform_action(event_action, event_target) - ACTION_TRANSFORMATIONS.dig(event_target, event_action) || event_action - end - - def transform_target(event_target) - Event::TARGET_TYPES.key(event_target) - end - - def valid_target?(target) - Event::TARGET_TYPES.value?(target) - end - - def valid_action?(action) - Event.actions.key?(action) - end - - def track_git_write_action(author_id, transformed_action, time) - return unless GIT_WRITE_ACTIONS.include?(transformed_action) - - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(GIT_WRITE_ACTION, values: author_id, time: time) - end - end - end - end -end diff --git a/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb index b99c9ebb24f..9de575d8567 100644 --- a/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb @@ -33,7 +33,7 @@ module Gitlab private def track_unique_action(action, author) - return unless author + return unless author && Feature.enabled?(:track_work_items_activity) Gitlab::UsageDataCounters::HLLRedisCounter.track_event(action, values: author.id) end diff --git a/lib/gitlab/usage_data_non_sql_metrics.rb b/lib/gitlab/usage_data_non_sql_metrics.rb index 79d4b45a1ce..71386a58ba7 100644 --- a/lib/gitlab/usage_data_non_sql_metrics.rb +++ b/lib/gitlab/usage_data_non_sql_metrics.rb @@ -40,13 +40,6 @@ module Gitlab def minimum_id(model, column = nil) end - - def jira_integration_data - { - projects_jira_server_active: 0, - projects_jira_cloud_active: 0 - } - end end end end diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb index 3a163e5dde9..534a08cad9a 100644 --- a/lib/gitlab/usage_data_queries.rb +++ b/lib/gitlab/usage_data_queries.rb @@ -68,13 +68,6 @@ module Gitlab end end - def jira_integration_data - { - projects_jira_server_active: 0, - projects_jira_cloud_active: 0 - } - end - def topology_usage_data { duration_s: 0, diff --git a/lib/gitlab/utils/error_message.rb b/lib/gitlab/utils/error_message.rb new file mode 100644 index 00000000000..e9c6f8a5847 --- /dev/null +++ b/lib/gitlab/utils/error_message.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module Utils + module ErrorMessage + extend self + + def to_user_facing(message) + "UF: #{message}" + end + end + end +end diff --git a/lib/gitlab/utils/override.rb b/lib/gitlab/utils/override.rb index 7f43e25e50d..1d02bcbb2d2 100644 --- a/lib/gitlab/utils/override.rb +++ b/lib/gitlab/utils/override.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true -require_dependency 'gitlab/utils' -require_dependency 'gitlab/environment' +require_relative '../utils' +require_relative '../environment' module Gitlab module Utils diff --git a/lib/gitlab/utils/uniquify.rb b/lib/gitlab/utils/uniquify.rb new file mode 100644 index 00000000000..b5908d18103 --- /dev/null +++ b/lib/gitlab/utils/uniquify.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +# Uniquify +# +# Return a version of the given 'base' string that is unique +# by appending a counter to it. Uniqueness is determined by +# repeated calls to the passed block. +# +# You can pass an initial value for the counter, if not given +# counting starts from 1. +# +# If `base` is a function/proc, we expect that calling it with a +# candidate counter returns a string to test/return. + +module Gitlab + module Utils + class Uniquify + def initialize(counter = nil) + @counter = counter + end + + def string(base) + @base = base + + increment_counter! while yield(base_string) + base_string + end + + private + + def base_string + if @base.respond_to?(:call) + @base.call(@counter) + else + "#{@base}#{@counter}" + end + end + + def increment_counter! + @counter ||= 0 + @counter += 1 + end + end + end +end diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb index fab8617bcda..4106084b301 100644 --- a/lib/gitlab/utils/usage_data.rb +++ b/lib/gitlab/utils/usage_data.rb @@ -255,33 +255,6 @@ module Gitlab end end - # rubocop: disable UsageData/LargeTable: - def jira_integration_data - with_metadata do - data = { - projects_jira_server_active: 0, - projects_jira_cloud_active: 0 - } - - # rubocop: disable CodeReuse/ActiveRecord - ::Integrations::Jira.active.includes(:jira_tracker_data).find_in_batches(batch_size: 100) do |services| - counts = services.group_by do |service| - # TODO: Simplify as part of https://gitlab.com/gitlab-org/gitlab/issues/29404 - service_url = service.data_fields&.url || (service.properties && service.properties['url']) - service_url&.include?('.atlassian.net') ? :cloud : :server - end - - data[:projects_jira_server_active] += counts[:server].size if counts[:server] - data[:projects_jira_cloud_active] += counts[:cloud].size if counts[:cloud] - end - - data - end - end - - # rubocop: enable CodeReuse/ActiveRecord - # rubocop: enable UsageData/LargeTable: - def minimum_id(model, column = nil) key = :"#{model.name.downcase.gsub('::', '_')}_minimum_id" column_to_read = column || :id diff --git a/lib/gitlab/utils/username_and_email_generator.rb b/lib/gitlab/utils/username_and_email_generator.rb new file mode 100644 index 00000000000..38c9bb7050d --- /dev/null +++ b/lib/gitlab/utils/username_and_email_generator.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require 'securerandom' + +module Gitlab + module Utils + class UsernameAndEmailGenerator + include Gitlab::Utils::StrongMemoize + + def initialize(username_prefix:, email_domain: Gitlab.config.gitlab.host) + @username_prefix = username_prefix + @email_domain = email_domain + end + + def username + uniquify.string(->(counter) { Kernel.sprintf(username_pattern, counter) }) do |suggested_username| + ::Namespace.by_path(suggested_username) || ::User.find_by_any_email(email_for(suggested_username)) + end + end + strong_memoize_attr :username + + def email + email_for(username) + end + strong_memoize_attr :email + + private + + def username_pattern + "#{@username_prefix}_#{SecureRandom.hex(16)}%s" + end + + def email_for(name) + "#{name}@#{@email_domain}" + end + + def uniquify + Gitlab::Utils::Uniquify.new + end + end + end +end diff --git a/lib/gitlab/verify/batch_verifier.rb b/lib/gitlab/verify/batch_verifier.rb index 71d106db742..bff95743dbd 100644 --- a/lib/gitlab/verify/batch_verifier.rb +++ b/lib/gitlab/verify/batch_verifier.rb @@ -34,7 +34,7 @@ module Gitlab private def run_batch_for(batch) - batch.map { |upload| verify(upload) }.compact.to_h + batch.filter_map { |upload| verify(upload) }.to_h end def verify(object) |