diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-08-18 13:50:51 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-08-18 13:50:51 +0300 |
commit | db384e6b19af03b4c3c82a5760d83a3fd79f7982 (patch) | |
tree | 34beaef37df5f47ccbcf5729d7583aae093cffa0 /lib/gitlab | |
parent | 54fd7b1bad233e3944434da91d257fa7f63c3996 (diff) |
Add latest changes from gitlab-org/gitlab@16-3-stable-eev16.3.0-rc42
Diffstat (limited to 'lib/gitlab')
285 files changed, 3671 insertions, 4979 deletions
diff --git a/lib/gitlab/action_cable/request_store_callbacks.rb b/lib/gitlab/action_cable/request_store_callbacks.rb index 14d80a7c40c..f6dda18a444 100644 --- a/lib/gitlab/action_cable/request_store_callbacks.rb +++ b/lib/gitlab/action_cable/request_store_callbacks.rb @@ -9,7 +9,7 @@ module Gitlab def self.wrapper lambda do |_, inner| - ::Gitlab::WithRequestStore.with_request_store do + ::Gitlab::SafeRequestStore.ensure_request_store do inner.call end end diff --git a/lib/gitlab/alert_management/payload.rb b/lib/gitlab/alert_management/payload.rb index de34a0f5d47..c6244124022 100644 --- a/lib/gitlab/alert_management/payload.rb +++ b/lib/gitlab/alert_management/payload.rb @@ -18,31 +18,20 @@ module Gitlab # @param monitoring_tool [String] # @param integration [AlertManagement::HttpIntegration] def parse(project, payload, monitoring_tool: nil, integration: nil) - payload_class = payload_class_for( - monitoring_tool: monitoring_tool || payload&.dig('monitoring_tool'), - payload: payload - ) + payload_class = payload_class_for(monitoring_tool: monitoring_tool || payload&.dig('monitoring_tool')) payload_class.new(project: project, payload: payload, integration: integration) end private - def payload_class_for(monitoring_tool:, payload:) + def payload_class_for(monitoring_tool:) if monitoring_tool == MONITORING_TOOLS[:prometheus] - if gitlab_managed_prometheus?(payload) - ::Gitlab::AlertManagement::Payload::ManagedPrometheus - else - ::Gitlab::AlertManagement::Payload::Prometheus - end + ::Gitlab::AlertManagement::Payload::Prometheus else ::Gitlab::AlertManagement::Payload::Generic end end - - def gitlab_managed_prometheus?(payload) - payload&.dig('labels', 'gitlab_alert_id').present? - end end end end diff --git a/lib/gitlab/alert_management/payload/managed_prometheus.rb b/lib/gitlab/alert_management/payload/managed_prometheus.rb deleted file mode 100644 index 4ed21108d3e..00000000000 --- a/lib/gitlab/alert_management/payload/managed_prometheus.rb +++ /dev/null @@ -1,46 +0,0 @@ -# frozen_string_literal: true - -# Attribute mapping for alerts via prometheus alerting integration, -# and for which payload includes gitlab-controlled attributes. -module Gitlab - module AlertManagement - module Payload - class ManagedPrometheus < ::Gitlab::AlertManagement::Payload::Prometheus - attribute :gitlab_prometheus_alert_id, - paths: %w(labels gitlab_prometheus_alert_id), - type: :integer - attribute :metric_id, - paths: %w(labels gitlab_alert_id), - type: :integer - - def gitlab_alert - strong_memoize(:gitlab_alert) do - next unless metric_id || gitlab_prometheus_alert_id - - alerts = Projects::Prometheus::AlertsFinder - .new(project: project, metric: metric_id, id: gitlab_prometheus_alert_id) - .execute - - next if alerts.blank? || alerts.size > 1 - - alerts.first - end - end - - def full_query - gitlab_alert&.full_query || super - end - - def environment - gitlab_alert&.environment || super - end - - private - - def plain_gitlab_fingerprint - [metric_id, starts_at_raw].join('/') - end - end - end - end -end diff --git a/lib/gitlab/audit/auditor.rb b/lib/gitlab/audit/auditor.rb index a59237fbb1f..a035b6face9 100644 --- a/lib/gitlab/audit/auditor.rb +++ b/lib/gitlab/audit/auditor.rb @@ -76,13 +76,11 @@ module Gitlab @authentication_event = @context.fetch(:authentication_event, false) @authentication_provider = @context[:authentication_provider] - # TODO: Remove this code once we close https://gitlab.com/gitlab-org/gitlab/-/issues/367870 return if @is_audit_event_yaml_defined - message = 'Logging audit events without an event type definition will be deprecated soon ' \ - '(https://docs.gitlab.com/ee/development/audit_event_guide/#event-type-definitions)' - - Gitlab::AppLogger.warn(message: message, event_type: @name) + raise StandardError, "Audit event type YML file is not defined for #{@name}. Please read " \ + "https://docs.gitlab.com/ee/development/audit_event_guide/" \ + "#how-to-instrument-new-audit-events for adding a new audit event" end def single_audit diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 1bb92b7fa62..cafa75d5f59 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -222,11 +222,11 @@ module Gitlab return unless valid_scoped_token?(token, all_available_scopes) - if project && token.user.project_bot? + if project && (token.user.project_bot? || token.user.service_account?) return unless can_read_project?(token.user, project) end - if token.user.can_log_in_with_non_expired_password? || token.user.project_bot? + if token.user.can_log_in_with_non_expired_password? || (token.user.project_bot? || token.user.service_account?) ::PersonalAccessTokens::LastUsedService.new(token).execute Gitlab::Auth::Result.new(token.user, nil, :personal_access_token, abilities_for_scopes(token.scopes)) @@ -238,7 +238,7 @@ module Gitlab end def bot_user_can_read_project?(user, project) - (user.project_bot? || user.security_policy_bot?) && can_read_project?(user, project) + (user.project_bot? || user.service_account? || user.security_policy_bot?) && can_read_project?(user, project) end def valid_oauth_token?(token) diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb index 966520655a5..a715f17ecd6 100644 --- a/lib/gitlab/auth/auth_finders.rb +++ b/lib/gitlab/auth/auth_finders.rb @@ -403,10 +403,14 @@ module Gitlab end def revoke_token_family(token) - return unless Feature.enabled?(:pat_reuse_detection) + return unless access_token_rotation_request? PersonalAccessTokens::RevokeTokenFamilyService.new(token).execute end + + def access_token_rotation_request? + current_request.path.match(%r{access_tokens/\d+/rotate$}) + end end end end diff --git a/lib/gitlab/auth/saml/auth_hash.rb b/lib/gitlab/auth/saml/auth_hash.rb index 592d88264e9..4a88d30fa93 100644 --- a/lib/gitlab/auth/saml/auth_hash.rb +++ b/lib/gitlab/auth/saml/auth_hash.rb @@ -8,6 +8,10 @@ module Gitlab Array.wrap(get_raw(Gitlab::Auth::Saml::Config.new(auth_hash.provider).groups)) end + def azure_group_overage_claim? + get_raw('http://schemas.microsoft.com/claims/groups.link').present? + end + def authn_context response_object = auth_hash.extra[:response_object] return if response_object.blank? diff --git a/lib/gitlab/auth/two_factor_auth_verifier.rb b/lib/gitlab/auth/two_factor_auth_verifier.rb index 5a203a1fe9c..fbdfd105ee3 100644 --- a/lib/gitlab/auth/two_factor_auth_verifier.rb +++ b/lib/gitlab/auth/two_factor_auth_verifier.rb @@ -3,10 +3,11 @@ module Gitlab module Auth class TwoFactorAuthVerifier - attr_reader :current_user + attr_reader :current_user, :request - def initialize(current_user) + def initialize(current_user, request = nil) @current_user = current_user + @request = request end def two_factor_authentication_enforced? @@ -14,6 +15,8 @@ module Gitlab end def two_factor_authentication_required? + return false if allow_2fa_bypass_for_provider + Gitlab::CurrentSettings.require_two_factor_authentication? || current_user&.require_two_factor_authentication_from_group? end @@ -35,6 +38,12 @@ module Gitlab two_factor_grace_period.hours.since(time) < Time.current end + + def allow_2fa_bypass_for_provider + return false if Feature.disabled?(:by_pass_two_factor_for_current_session) + + request.session[:provider_2FA].present? if request + end end end end diff --git a/lib/gitlab/authorized_keys.rb b/lib/gitlab/authorized_keys.rb index 3e529a0d2f3..00d9480334e 100644 --- a/lib/gitlab/authorized_keys.rb +++ b/lib/gitlab/authorized_keys.rb @@ -153,7 +153,7 @@ module Gitlab end def command(id) - unless /\A[a-z0-9-]+\z/ =~ id + unless /\A[a-z0-9-]+\z/.match?(id) raise KeyError, "Invalid ID: #{id.inspect}" end diff --git a/lib/gitlab/background_migration/backfill_default_branch_protection_namespace_setting.rb b/lib/gitlab/background_migration/backfill_default_branch_protection_namespace_setting.rb new file mode 100644 index 00000000000..8da29a61d61 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_default_branch_protection_namespace_setting.rb @@ -0,0 +1,131 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class is used to update the default_branch_protection_defaults column + # for user namespaces of the namespace_settings table. + class BackfillDefaultBranchProtectionNamespaceSetting < BatchedMigrationJob + operation_name :set_default_branch_protection_defaults + feature_category :database + + # Migration only version of `namespaces` table + class Namespace < ::ApplicationRecord + self.table_name = 'namespaces' + self.inheritance_column = :_type_disabled + + has_one :namespace_setting, + class_name: '::Gitlab::BackgroundMigration::BackfillDefaultBranchProtectionNamespaceSetting::NamespaceSetting' + end + + # Migration only version of `namespace_settings` table + class NamespaceSetting < ::ApplicationRecord + self.table_name = 'namespace_settings' + belongs_to :namespace, + class_name: '::Gitlab::BackgroundMigration::BackfillDefaultBranchProtectionNamespaceSetting::Namespace' + end + + # Migration only version of Gitlab::Access:BranchProtection application code. + class BranchProtection + attr_reader :level + + def initialize(level) + @level = level + end + + PROTECTION_NONE = 0 + PROTECTION_DEV_CAN_PUSH = 1 + PROTECTION_FULL = 2 + PROTECTION_DEV_CAN_MERGE = 3 + PROTECTION_DEV_CAN_INITIAL_PUSH = 4 + + DEVELOPER = 30 + MAINTAINER = 40 + + def to_hash + case level + when PROTECTION_NONE + self.class.protection_none + when PROTECTION_DEV_CAN_PUSH + self.class.protection_partial + when PROTECTION_FULL + self.class.protected_fully + when PROTECTION_DEV_CAN_MERGE + self.class.protected_against_developer_pushes + when PROTECTION_DEV_CAN_INITIAL_PUSH + self.class.protected_after_initial_push + end + end + + class << self + def protection_none + { + allowed_to_push: [{ 'access_level' => DEVELOPER }], + allowed_to_merge: [{ 'access_level' => DEVELOPER }], + allow_force_push: true + } + end + + def protection_partial + protection_none.merge(allow_force_push: false) + end + + def protected_fully + { + allowed_to_push: [{ 'access_level' => MAINTAINER }], + allowed_to_merge: [{ 'access_level' => MAINTAINER }], + allow_force_push: false + } + end + + def protected_against_developer_pushes + { + allowed_to_push: [{ 'access_level' => MAINTAINER }], + allowed_to_merge: [{ 'access_level' => DEVELOPER }], + allow_force_push: true + } + end + + def protected_after_initial_push + { + allowed_to_push: [{ 'access_level' => MAINTAINER }], + allowed_to_merge: [{ 'access_level' => DEVELOPER }], + allow_force_push: true, + developer_can_initial_push: true + } + end + end + end + + def perform + each_sub_batch do |sub_batch| + update_default_protection_branch_defaults(sub_batch) + end + end + + private + + def update_default_protection_branch_defaults(batch) + namespace_settings = NamespaceSetting.where(namespace_id: batch.pluck(:namespace_id)).includes(:namespace) + + values_list = namespace_settings.map do |namespace_setting| + level = namespace_setting.namespace.default_branch_protection.to_i + value = BranchProtection.new(level).to_hash.to_json + "(#{namespace_setting.namespace_id}, '#{value}'::jsonb)" + end.join(", ") + + sql = <<~SQL + WITH new_values (namespace_id, default_branch_protection_defaults) AS ( + VALUES + #{values_list} + ) + UPDATE namespace_settings + SET default_branch_protection_defaults = new_values.default_branch_protection_defaults + FROM new_values + WHERE namespace_settings.namespace_id = new_values.namespace_id; + SQL + + connection.execute(sql) + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_dismissal_reason_in_vulnerability_reads.rb b/lib/gitlab/background_migration/backfill_dismissal_reason_in_vulnerability_reads.rb new file mode 100644 index 00000000000..d7972a6a7a9 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_dismissal_reason_in_vulnerability_reads.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This batched background migration is EE-only, + # see ee/lib/ee/gitlab/background_migration/backfill_dismissal_reason_in_vulnerability_reads.rb for the actual + # migration code. + # + # This batched background migration will backfill `dismissal_reason` field in `vulnerability_reads` table for + # records with `state: 2` and `dismissal_reason: null`. + class BackfillDismissalReasonInVulnerabilityReads < BatchedMigrationJob + feature_category :vulnerability_management + + def perform; end + end + end +end + +Gitlab::BackgroundMigration::BackfillDismissalReasonInVulnerabilityReads.prepend_mod diff --git a/lib/gitlab/background_migration/backfill_missing_vulnerability_dismissal_details.rb b/lib/gitlab/background_migration/backfill_missing_vulnerability_dismissal_details.rb new file mode 100644 index 00000000000..8399f53b724 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_missing_vulnerability_dismissal_details.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Style/Documentation + class BackfillMissingVulnerabilityDismissalDetails < BatchedMigrationJob + feature_category :vulnerability_management + + def perform + # no-op. The logic is defined in EE module. + end + end + # rubocop: enable Style/Documentation + end +end + +::Gitlab::BackgroundMigration::BackfillMissingVulnerabilityDismissalDetails.prepend_mod diff --git a/lib/gitlab/background_migration/backfill_namespace_id_for_project_route.rb b/lib/gitlab/background_migration/backfill_namespace_id_for_project_route.rb index 0282531ae17..b0b7882d54d 100644 --- a/lib/gitlab/background_migration/backfill_namespace_id_for_project_route.rb +++ b/lib/gitlab/background_migration/backfill_namespace_id_for_project_route.rb @@ -54,6 +54,7 @@ module Gitlab .where(namespace_id: nil) .where(source_type: 'Project') .where.not(projects: { project_namespace_id: nil }) + .allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/421843') .select("routes.id, projects.project_namespace_id") end end diff --git a/lib/gitlab/background_migration/backfill_project_statistics_storage_size_without_pipeline_artifacts_size_job.rb b/lib/gitlab/background_migration/backfill_project_statistics_storage_size_without_pipeline_artifacts_size_job.rb new file mode 100644 index 00000000000..88d0f27282a --- /dev/null +++ b/lib/gitlab/background_migration/backfill_project_statistics_storage_size_without_pipeline_artifacts_size_job.rb @@ -0,0 +1,95 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop:disable Style/Documentation + class BackfillProjectStatisticsStorageSizeWithoutPipelineArtifactsSizeJob < Gitlab::BackgroundMigration::BatchedMigrationJob # rubocop:disable Layout/LineLength + class Project < ::ApplicationRecord + self.table_name = 'projects' + + has_one :statistics, class_name: '::Gitlab::BackgroundMigration::BackfillProjectStatisticsStorageSizeWithoutPipelineArtifactsSizeJob::ProjectStatistics' # rubocop:disable Layout/LineLength + end + + class ProjectStatistics < ::ApplicationRecord + include ::EachBatch + + self.table_name = 'project_statistics' + + belongs_to :project, class_name: '::Gitlab::BackgroundMigration::BackfillProjectStatisticsStorageSizeWithoutPipelineArtifactsSizeJob::Project' # rubocop:disable Layout/LineLength + + def update_storage_size(storage_size_components) + new_storage_size = storage_size_components.sum { |component| method(component).call } + + # Only update storage_size if storage_size needs updating + return unless storage_size != new_storage_size + + self.storage_size = new_storage_size + save! + + ::Namespaces::ScheduleAggregationWorker.perform_async(project.namespace_id) + log_with_data('Scheduled Namespaces::ScheduleAggregationWorker') + end + + def wiki_size + super.to_i + end + + def snippets_size + super.to_i + end + + private + + def log_with_data(log_line) + log_info( + log_line, + project_id: project.id, + pipeline_artifacts_size: pipeline_artifacts_size, + storage_size: storage_size, + namespace_id: project.namespace_id + ) + end + + def log_info(message, **extra) + ::Gitlab::BackgroundMigration::Logger.info( + migrator: 'BackfillProjectStatisticsStorageSizeWithoutPipelineArtifactsSizeJob', + message: message, + **extra + ) + end + end + + scope_to ->(relation) { + relation.where.not(pipeline_artifacts_size: 0) + } + operation_name :update_storage_size + feature_category :consumables_cost_management + + def perform + each_sub_batch do |sub_batch| + ProjectStatistics.merge(sub_batch).each do |statistics| + statistics.update_storage_size(storage_size_components) + end + end + end + + private + + # Overridden in EE + def storage_size_components + [ + :repository_size, + :wiki_size, + :lfs_objects_size, + :build_artifacts_size, + :packages_size, + :snippets_size, + :uploads_size + ] + end + end + # rubocop:enable Style/Documentation + end +end + +Gitlab::BackgroundMigration::BackfillProjectStatisticsStorageSizeWithoutPipelineArtifactsSizeJob.prepend_mod diff --git a/lib/gitlab/background_migration/cleanup_orphaned_routes.rb b/lib/gitlab/background_migration/cleanup_orphaned_routes.rb index 5c0ddf0ba8b..c221f8ea411 100644 --- a/lib/gitlab/background_migration/cleanup_orphaned_routes.rb +++ b/lib/gitlab/background_migration/cleanup_orphaned_routes.rb @@ -35,25 +35,31 @@ module Gitlab end def update_namespace_id(batch_column, non_orphaned_namespace_routes, sub_batch_size) - non_orphaned_namespace_routes.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| - batch_metrics.time_operation(:fix_missing_namespace_id) do - ApplicationRecord.connection.execute <<~SQL - WITH route_and_ns(route_id, namespace_id) AS #{::Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( - #{sub_batch.to_sql} - ) - UPDATE routes - SET namespace_id = route_and_ns.namespace_id - FROM route_and_ns - WHERE id = route_and_ns.route_id - SQL + Gitlab::Database.allow_cross_joins_across_databases( + url: "https://gitlab.com/gitlab-org/gitlab/-/issues/420046") do + non_orphaned_namespace_routes.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| + batch_metrics.time_operation(:fix_missing_namespace_id) do + ApplicationRecord.connection.execute <<~SQL + WITH route_and_ns(route_id, namespace_id) AS #{::Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( + #{sub_batch.to_sql} + ) + UPDATE routes + SET namespace_id = route_and_ns.namespace_id + FROM route_and_ns + WHERE id = route_and_ns.route_id + SQL + end end end end def cleanup_relations(batch_column, orphaned_namespace_routes, pause_ms, sub_batch_size) - orphaned_namespace_routes.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| - batch_metrics.time_operation(:cleanup_orphaned_routes) do - sub_batch.delete_all + Gitlab::Database.allow_cross_joins_across_databases( + url: "https://gitlab.com/gitlab-org/gitlab/-/issues/420046") do + orphaned_namespace_routes.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| + batch_metrics.time_operation(:cleanup_orphaned_routes) do + sub_batch.delete_all + end end end end diff --git a/lib/gitlab/background_migration/delete_orphaned_transferred_project_approval_rules.rb b/lib/gitlab/background_migration/delete_orphaned_transferred_project_approval_rules.rb new file mode 100644 index 00000000000..c0f87644b59 --- /dev/null +++ b/lib/gitlab/background_migration/delete_orphaned_transferred_project_approval_rules.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Background migration for deleting orphaned approval project rules for projects that were transferred + class DeleteOrphanedTransferredProjectApprovalRules < BatchedMigrationJob + feature_category :security_policy_management + + def perform; end + end + end +end + +Gitlab::BackgroundMigration::DeleteOrphanedTransferredProjectApprovalRules.prepend_mod diff --git a/lib/gitlab/background_migration/fix_allow_descendants_override_disabled_shared_runners.rb b/lib/gitlab/background_migration/fix_allow_descendants_override_disabled_shared_runners.rb new file mode 100644 index 00000000000..44bda3fe2b6 --- /dev/null +++ b/lib/gitlab/background_migration/fix_allow_descendants_override_disabled_shared_runners.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Fixes invalid combination of shared runners being enabled and + # allow_descendants_override = true + # This combination fails validation and doesn't make sense: + # we always allow descendants to disable shared runners + class FixAllowDescendantsOverrideDisabledSharedRunners < BatchedMigrationJob + feature_category :runner_fleet + operation_name :fix_allow_descendants_override_disabled_shared_runners + + def perform + each_sub_batch do |sub_batch| + sub_batch.where(shared_runners_enabled: true, + allow_descendants_override_disabled_shared_runners: true) + .update_all(allow_descendants_override_disabled_shared_runners: false) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects.rb b/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects.rb index 74f5bc3f725..e1ea0c66ad4 100644 --- a/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects.rb +++ b/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects.rb @@ -9,14 +9,18 @@ module Gitlab relation.where.not(creator_id: nil) .joins('LEFT OUTER JOIN users ON users.id = projects.creator_id') .where(users: { id: nil }) + .allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/421843') end operation_name :update_all feature_category :groups_and_projects def perform - each_sub_batch do |sub_batch| - sub_batch.update_all(creator_id: nil) + ::Gitlab::Database.allow_cross_joins_across_databases(url: + 'https://gitlab.com/gitlab-org/gitlab/-/issues/421843') do + each_sub_batch do |sub_batch| + sub_batch.update_all(creator_id: nil) + end end end end diff --git a/lib/gitlab/background_migration/populate_projects_star_count.rb b/lib/gitlab/background_migration/populate_projects_star_count.rb index 8417dc91b1b..0790bd98018 100644 --- a/lib/gitlab/background_migration/populate_projects_star_count.rb +++ b/lib/gitlab/background_migration/populate_projects_star_count.rb @@ -39,20 +39,23 @@ module Gitlab # rubocop:enable Database/RescueStatementTimeout def update_batch(sub_batch) - ApplicationRecord.connection.execute <<~SQL - WITH batched_relation AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (#{sub_batch.select(:id).to_sql}) - UPDATE projects - SET star_count = ( - SELECT COUNT(*) - FROM users_star_projects - INNER JOIN users - ON users_star_projects.user_id = users.id - WHERE users_star_projects.project_id = batched_relation.id - AND users.state = 'active' - ) - FROM batched_relation - WHERE projects.id = batched_relation.id - SQL + ::Gitlab::Database.allow_cross_joins_across_databases(url: + 'https://gitlab.com/gitlab-org/gitlab/-/issues/421843') do + ApplicationRecord.connection.execute <<~SQL + WITH batched_relation AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (#{sub_batch.select(:id).to_sql}) + UPDATE projects + SET star_count = ( + SELECT COUNT(*) + FROM users_star_projects + INNER JOIN users + ON users_star_projects.user_id = users.id + WHERE users_star_projects.project_id = batched_relation.id + AND users.state = 'active' + ) + FROM batched_relation + WHERE projects.id = batched_relation.id + SQL + end end end end diff --git a/lib/gitlab/blame.rb b/lib/gitlab/blame.rb index e210c18e3d1..ac39daf375f 100644 --- a/lib/gitlab/blame.rb +++ b/lib/gitlab/blame.rb @@ -20,13 +20,13 @@ module Gitlab current_group = nil i = first_line - 1 - blame.each do |commit, line, previous_path| + blame.each do |commit, line, previous_path, span| commit = Commit.new(commit, project) commit.lazy_author # preload author if prev_sha != commit.sha groups << current_group if current_group - current_group = { commit: commit, lines: [], previous_path: previous_path } + current_group = { commit: commit, lines: [], previous_path: previous_path, span: span, lineno: i + 1 } end current_group[:lines] << (highlight ? highlighted_lines[i].html_safe : line) diff --git a/lib/gitlab/checks/branch_check.rb b/lib/gitlab/checks/branch_check.rb index aa89c2711f9..b675eca826a 100644 --- a/lib/gitlab/checks/branch_check.rb +++ b/lib/gitlab/checks/branch_check.rb @@ -43,7 +43,7 @@ module Gitlab def prohibited_branch_checks return if deletion? - if branch_name =~ %r{\A\h{40}(-/|/|\z)} + if %r{\A#{Gitlab::Git::Commit::RAW_FULL_SHA_PATTERN}(-/|/|\z)}o.match?(branch_name) raise GitAccess::ForbiddenError, ERROR_MESSAGES[:prohibited_hex_branch_name] end diff --git a/lib/gitlab/checks/diff_check.rb b/lib/gitlab/checks/diff_check.rb index bce4f969284..15b38188f13 100644 --- a/lib/gitlab/checks/diff_check.rb +++ b/lib/gitlab/checks/diff_check.rb @@ -18,10 +18,7 @@ module Gitlab return unless should_run_validations? return if commits.empty? - paths = project.repository.find_changed_paths( - commits.map(&:sha), merge_commit_diff_mode: :all_parents - ) - + paths = project.repository.find_changed_paths(treeish_objects, merge_commit_diff_mode: :all_parents) paths.each do |path| validate_path(path) end @@ -31,6 +28,29 @@ module Gitlab private + def treeish_objects + objects = commits + + return objects unless project.repository.empty? && + Feature.enabled?(:verify_push_rules_for_first_commit, project) + + # It's a special case for the push to the empty repository + # + # Git doesn't display a diff of the initial commit of the repository + # if we just provide a commit sha. + # + # To fix that we can use TreeRequest to check the difference + # between empty tree sha and the tree sha of the initial commit + # + # `commits` are sorted in reverse order, the initial commit is the last one. + init_commit = objects.last + + diff_tree = Gitlab::Git::DiffTree.from_commit(init_commit) + return [diff_tree] + objects if diff_tree + + objects + end + def validate_lfs_file_locks? strong_memoize(:validate_lfs_file_locks) do project.lfs_enabled? && project.any_lfs_file_locks? diff --git a/lib/gitlab/checks/file_size_check/allow_existing_oversized_blobs.rb b/lib/gitlab/checks/file_size_check/allow_existing_oversized_blobs.rb deleted file mode 100644 index 78f1716274e..00000000000 --- a/lib/gitlab/checks/file_size_check/allow_existing_oversized_blobs.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Checks - module FileSizeCheck - class AllowExistingOversizedBlobs - def initialize(project:, changes:, file_size_limit_megabytes:) - @project = project - @changes = changes - @oldrevs = changes.pluck(:oldrev).compact # rubocop:disable CodeReuse/ActiveRecord just plucking from an array - @file_size_limit_megabytes = file_size_limit_megabytes - end - - def find(timeout: nil) - oversize_blobs = any_oversize_blobs.find(timeout: timeout) - - return oversize_blobs unless oldrevs.present? - - revs_paths = oldrevs.product(oversize_blobs.map(&:path)) - existing_blobs = project.repository.blobs_at(revs_paths, blob_size_limit: 1) - map_existing_path_to_size = existing_blobs.group_by(&:path).transform_values { |blobs| blobs.map(&:size).max } - - # return blobs that are going to be over the limit that were previously within the limit - oversize_blobs.select { |blob| map_existing_path_to_size.fetch(blob.path, 0) <= file_size_limit_megabytes } - end - - private - - attr_reader :project, :changes, :newrevs, :oldrevs, :file_size_limit_megabytes - - def any_oversize_blobs - AnyOversizedBlobs.new(project: project, changes: changes, - file_size_limit_megabytes: file_size_limit_megabytes) - end - end - end - end -end diff --git a/lib/gitlab/checks/file_size_check/hook_environment_aware_any_oversized_blobs.rb b/lib/gitlab/checks/file_size_check/hook_environment_aware_any_oversized_blobs.rb new file mode 100644 index 00000000000..952def83658 --- /dev/null +++ b/lib/gitlab/checks/file_size_check/hook_environment_aware_any_oversized_blobs.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Gitlab + module Checks + module FileSizeCheck + class HookEnvironmentAwareAnyOversizedBlobs + def initialize(project:, changes:, file_size_limit_megabytes:) + @project = project + @repository = project.repository + @changes = changes + @file_size_limit_megabytes = file_size_limit_megabytes + end + + def find(timeout: nil) + if ignore_alternate_directories? + blobs = repository.list_all_blobs(bytes_limit: 0, dynamic_timeout: timeout, + ignore_alternate_object_directories: true).to_a + + blobs.select! do |blob| + ::Gitlab::Utils.bytes_to_megabytes(blob.size) > file_size_limit_megabytes + end + filter_existing(blobs) + else + any_oversize_blobs.find(timeout: timeout) + end + end + + private + + attr_reader :project, :repository, :changes, :file_size_limit_megabytes + + def filter_existing(blobs) + gitaly_repo = repository.gitaly_repository.dup.tap { |repo| repo.git_object_directory = "" } + + map_blob_id_to_existence = repository.gitaly_commit_client.object_existence_map(blobs.map(&:id), + gitaly_repo: gitaly_repo) + + blobs.reject { |blob| map_blob_id_to_existence[blob.id].present? } + end + + def ignore_alternate_directories? + git_env = ::Gitlab::Git::HookEnv.all(repository.gl_repository) + + git_env['GIT_OBJECT_DIRECTORY_RELATIVE'].present? + end + + def any_oversize_blobs + AnyOversizedBlobs.new(project: project, changes: changes, + file_size_limit_megabytes: file_size_limit_megabytes) + end + end + end + end +end diff --git a/lib/gitlab/checks/global_file_size_check.rb b/lib/gitlab/checks/global_file_size_check.rb index 418d2d32b57..62facf52239 100644 --- a/lib/gitlab/checks/global_file_size_check.rb +++ b/lib/gitlab/checks/global_file_size_check.rb @@ -3,7 +3,6 @@ module Gitlab module Checks class GlobalFileSizeCheck < BaseBulkChecker - MAX_FILE_SIZE_MB = 100 LOG_MESSAGE = 'Checking for blobs over the file size limit' def validate! @@ -11,19 +10,38 @@ module Gitlab Gitlab::AppJsonLogger.info(LOG_MESSAGE) logger.log_timed(LOG_MESSAGE) do - Gitlab::Checks::FileSizeCheck::AllowExistingOversizedBlobs.new( + oversized_blobs = Gitlab::Checks::FileSizeCheck::HookEnvironmentAwareAnyOversizedBlobs.new( project: project, changes: changes, - file_size_limit_megabytes: MAX_FILE_SIZE_MB + file_size_limit_megabytes: file_size_limit ).find - # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/393535 - # - set limit per plan tier - # - raise an error if large blobs are found + if oversized_blobs.present? + Gitlab::AppJsonLogger.info( + message: 'Found blob over global limit', + blob_sizes: oversized_blobs.map(&:size) + ) + + if enforce_global_file_size_limit? + raise ::Gitlab::GitAccess::ForbiddenError, + "Changes include a file that is larger than the allowed size of #{file_size_limit} MiB. " \ + "Use Git LFS to manage this file.)" + end + end end true end + + private + + def file_size_limit + project.actual_limits.file_size_limit_mb + end + + def enforce_global_file_size_limit? + Feature.enabled?(:enforce_global_file_size_limit, project) + end end end end diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index 88d624503df..d0ab4916c90 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -70,8 +70,8 @@ module Gitlab # abort if we don't get any data. next unless path && meta next unless path.valid_encoding? && meta.valid_encoding? - next unless path =~ match_pattern - next if path =~ INVALID_PATH_PATTERN + next unless match_pattern.match?(path) + next if INVALID_PATH_PATTERN.match?(path) entries[path] = Gitlab::Json.parse(meta, symbolize_names: true) rescue JSON::ParserError, Encoding::CompatibilityError @@ -90,7 +90,7 @@ module Gitlab raise ParserError, 'Artifacts metadata file empty!' end - unless version_string =~ VERSION_PATTERN + unless VERSION_PATTERN.match?(version_string) raise ParserError, 'Invalid version!' end diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb index 355fffbf9c6..23db15d58b6 100644 --- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -102,7 +102,7 @@ module Gitlab def total_size descendant_pattern = /^#{Regexp.escape(@path.to_s)}/ entries.sum do |path, entry| - (entry[:size] if path =~ descendant_pattern).to_i + (entry[:size] if descendant_pattern.match?(path)).to_i end end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/gitlab/ci/config/entry/include/rules.rb b/lib/gitlab/ci/config/entry/include/rules.rb index 8eaf9e35aaf..71418e6752d 100644 --- a/lib/gitlab/ci/config/entry/include/rules.rb +++ b/lib/gitlab/ci/config/entry/include/rules.rb @@ -5,6 +5,12 @@ module Gitlab class Config module Entry class Include + ## + # Include rules are validated separately from all other entries. This + # is because included files are expanded before `@root.compose!` runs + # in Ci::Config. As such, this class is directly instantiated and + # composed in lib/gitlab/ci/config/external/rules.rb. + # class Rules < ::Gitlab::Config::Entry::ComposableArray include ::Gitlab::Config::Entry::Validatable @@ -13,8 +19,9 @@ module Gitlab validates :config, type: Array end + # Remove this method when FF `ci_refactor_external_rules` is removed def value - @config + Feature.enabled?(:ci_refactor_external_rules) ? super : @config end def composable_class diff --git a/lib/gitlab/ci/config/entry/include/rules/rule.rb b/lib/gitlab/ci/config/entry/include/rules/rule.rb index 9cdbd8cd037..1a68e95913c 100644 --- a/lib/gitlab/ci/config/entry/include/rules/rule.rb +++ b/lib/gitlab/ci/config/entry/include/rules/rule.rb @@ -14,9 +14,6 @@ module Gitlab attributes :if, :exists, :when - # Include rules are validated before Entry validations. This is because - # the include files are expanded before `compose!` runs in Ci::Config. - # The actual validation logic is in lib/gitlab/ci/config/external/rules.rb. validations do validates :config, presence: true validates :config, type: { with: Hash } @@ -24,8 +21,14 @@ module Gitlab with_options allow_nil: true do validates :if, expression: true + validates :exists, array_of_strings_or_string: true, allow_blank: true + validates :when, allowed_values: { in: ALLOWED_WHEN } end end + + def value + Feature.enabled?(:ci_refactor_external_rules) ? config.compact : super + end end end end diff --git a/lib/gitlab/ci/config/entry/need.rb b/lib/gitlab/ci/config/entry/need.rb index f1b67635c08..cf727134f32 100644 --- a/lib/gitlab/ci/config/entry/need.rb +++ b/lib/gitlab/ci/config/entry/need.rb @@ -42,11 +42,15 @@ module Gitlab end class JobHash < ::Gitlab::Config::Entry::Node - include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable + include ::Gitlab::Config::Entry::Configurable + + ALLOWED_KEYS = %i[job artifacts optional parallel].freeze + attributes :job, :artifacts, :optional, :parallel - ALLOWED_KEYS = %i[job artifacts optional].freeze - attributes :job, :artifacts, :optional + entry :parallel, Entry::Product::Parallel, + description: 'Parallel needs configuration for this job', + inherit: true validations do validates :config, presence: true @@ -61,9 +65,15 @@ module Gitlab end def value - { name: job, + result = { + name: job, artifacts: artifacts || artifacts.nil?, - optional: !!optional } + optional: !!optional + } + + result[:parallel] = parallel_value if has_parallel? + + result end end diff --git a/lib/gitlab/ci/config/entry/needs.rb b/lib/gitlab/ci/config/entry/needs.rb index 11b202ddde9..f0bfad80e6f 100644 --- a/lib/gitlab/ci/config/entry/needs.rb +++ b/lib/gitlab/ci/config/entry/needs.rb @@ -21,6 +21,10 @@ module Gitlab if config.is_a?(Hash) && config.empty? errors.add(:config, 'can not be an empty Hash') end + + if number_parallel_build? + errors.add(:config, 'cannot use "parallel: <number>".') + end end validate on: :composed do @@ -47,6 +51,14 @@ module Gitlab end end + def number_parallel_build? + if config.is_a?(Array) + config.any? { |need_values| need_values.is_a?(Hash) && need_values[:parallel].is_a?(Numeric) } + elsif config.is_a?(Hash) + config[:parallel].is_a?(Numeric) + end + end + def composable_class Entry::Need end diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb index 6408f412e6f..3c180674f2a 100644 --- a/lib/gitlab/ci/config/entry/reports.rb +++ b/lib/gitlab/ci/config/entry/reports.rb @@ -17,7 +17,7 @@ module Gitlab dast performance browser_performance load_performance license_scanning metrics lsif dotenv terraform accessibility coverage_fuzzing api_fuzzing cluster_image_scanning - requirements requirements_v2 coverage_report cyclonedx].freeze + requirements requirements_v2 coverage_report cyclonedx annotations].freeze attributes ALLOWED_KEYS @@ -50,6 +50,7 @@ module Gitlab validates :requirements, array_of_strings_or_string: true validates :requirements_v2, array_of_strings_or_string: true validates :cyclonedx, array_of_strings_or_string: true + validates :annotations, array_of_strings_or_string: true end end diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb index b8e012ec851..c57391d355c 100644 --- a/lib/gitlab/ci/config/external/context.rb +++ b/lib/gitlab/ci/config/external/context.rb @@ -14,7 +14,9 @@ module Gitlab include ::Gitlab::Utils::StrongMemoize attr_reader :project, :sha, :user, :parent_pipeline, :variables, :pipeline_config - attr_reader :expandset, :execution_deadline, :logger, :max_includes + attr_reader :expandset, :execution_deadline, :logger, :max_includes, :max_total_yaml_size_bytes + + attr_accessor :total_file_size_in_bytes delegate :instrument, to: :logger @@ -32,6 +34,9 @@ module Gitlab @execution_deadline = 0 @logger = logger || Gitlab::Ci::Pipeline::Logger.new(project: project) @max_includes = Gitlab::CurrentSettings.current_application_settings.ci_max_includes + @max_total_yaml_size_bytes = + Gitlab::CurrentSettings.current_application_settings.ci_max_total_yaml_size_bytes + @total_file_size_in_bytes = 0 yield self if block_given? end @@ -59,6 +64,7 @@ module Gitlab ctx.execution_deadline = execution_deadline ctx.logger = logger ctx.max_includes = max_includes + ctx.max_total_yaml_size_bytes = max_total_yaml_size_bytes end end @@ -100,7 +106,7 @@ module Gitlab protected - attr_writer :expandset, :execution_deadline, :logger, :max_includes + attr_writer :expandset, :execution_deadline, :logger, :max_includes, :max_total_yaml_size_bytes private diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb index 8bcb2a389d2..efba81c7420 100644 --- a/lib/gitlab/ci/config/external/file/base.rb +++ b/lib/gitlab/ci/config/external/file/base.rb @@ -92,6 +92,11 @@ module Gitlab def load_and_validate_expanded_hash! return errors.push("`#{masked_location}`: #{content_result.error}") unless content_result.valid? + if content_result.interpolated? && context.user.present? + ::Gitlab::UsageDataCounters::HLLRedisCounter + .track_event('ci_interpolation_users', values: context.user.id) + end + context.logger.instrument(:config_file_expand_content_includes) do expanded_content_hash # calling the method expands then memoizes the result end @@ -109,7 +114,7 @@ module Gitlab def content_result context.logger.instrument(:config_file_fetch_content_hash) do - ::Gitlab::Ci::Config::Yaml::Loader.new(content, inputs: content_inputs, current_user: context.user).load + ::Gitlab::Ci::Config::Yaml::Loader.new(content, inputs: content_inputs).load end end strong_memoize_attr :content_result diff --git a/lib/gitlab/ci/config/external/file/component.rb b/lib/gitlab/ci/config/external/file/component.rb index 9679d78a1aa..15cc0783b86 100644 --- a/lib/gitlab/ci/config/external/file/component.rb +++ b/lib/gitlab/ci/config/external/file/component.rb @@ -15,10 +15,6 @@ module Gitlab super end - def matching? - super && ::Feature.enabled?(:ci_include_components, context.project&.root_namespace) - end - def content return unless component_result.success? diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index 61b4d1ada10..cff7954235f 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -10,6 +10,7 @@ module Gitlab Error = Class.new(StandardError) AmbigiousSpecificationError = Class.new(Error) TooManyIncludesError = Class.new(Error) + TooMuchDataInPipelineTreeError = Class.new(Error) def initialize(values, context) @locations = Array.wrap(values.fetch(:include, [])).compact diff --git a/lib/gitlab/ci/config/external/mapper/matcher.rb b/lib/gitlab/ci/config/external/mapper/matcher.rb index 5072d0971cf..f9d5b0ebd01 100644 --- a/lib/gitlab/ci/config/external/mapper/matcher.rb +++ b/lib/gitlab/ci/config/external/mapper/matcher.rb @@ -40,19 +40,14 @@ module Gitlab strong_memoize_attr :file_subkeys def file_classes - classes = [ + [ External::File::Local, External::File::Project, External::File::Remote, External::File::Template, - External::File::Artifact + External::File::Artifact, + External::File::Component ] - - if Feature.enabled?(:ci_include_components, context.project&.root_namespace) - classes << External::File::Component - end - - classes end strong_memoize_attr :file_classes end diff --git a/lib/gitlab/ci/config/external/mapper/verifier.rb b/lib/gitlab/ci/config/external/mapper/verifier.rb index 95975e4661b..580cae8a207 100644 --- a/lib/gitlab/ci/config/external/mapper/verifier.rb +++ b/lib/gitlab/ci/config/external/mapper/verifier.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'objspace' + module Gitlab module Ci class Config @@ -37,6 +39,13 @@ module Gitlab file.validate_content! if file.valid? file.load_and_validate_expanded_hash! if file.valid? + + next unless Feature.enabled?(:introduce_ci_max_total_yaml_size_bytes, context.project) && file.valid? + + # We are checking the file.content.to_s because that is returning the actual content of the file, + # whereas file.content would return the BatchLoader. + context.total_file_size_in_bytes += ObjectSpace.memsize_of(file.content.to_s) + verify_max_total_pipeline_size! end end # rubocop: enable Metrics/CyclomaticComplexity @@ -50,6 +59,12 @@ module Gitlab def verify_execution_time! context.check_execution_time! end + + def verify_max_total_pipeline_size! + return if context.total_file_size_in_bytes <= context.max_total_yaml_size_bytes + + raise Mapper::TooMuchDataInPipelineTreeError, "Total size of combined CI/CD configuration is too big" + end end end end diff --git a/lib/gitlab/ci/config/external/rules.rb b/lib/gitlab/ci/config/external/rules.rb index 59e666b8bb5..0e6209460e0 100644 --- a/lib/gitlab/ci/config/external/rules.rb +++ b/lib/gitlab/ci/config/external/rules.rb @@ -5,15 +5,29 @@ module Gitlab class Config module External class Rules + # Remove these two constants when FF `ci_refactor_external_rules` is removed ALLOWED_KEYS = Entry::Include::Rules::Rule::ALLOWED_KEYS ALLOWED_WHEN = Entry::Include::Rules::Rule::ALLOWED_WHEN InvalidIncludeRulesError = Class.new(Mapper::Error) def initialize(rule_hashes) - validate(rule_hashes) + if Feature.enabled?(:ci_refactor_external_rules) + return unless rule_hashes - @rule_list = Build::Rules::Rule.fabricate_list(rule_hashes) + # We must compose the include rules entry here because included + # files are expanded before `@root.compose!` runs in Ci::Config. + rules_entry = Entry::Include::Rules.new(rule_hashes) + rules_entry.compose! + + raise InvalidIncludeRulesError, "include:#{rules_entry.errors.first}" unless rules_entry.valid? + + @rule_list = Build::Rules::Rule.fabricate_list(rules_entry.value) + else + validate(rule_hashes) + + @rule_list = Build::Rules::Rule.fabricate_list(rule_hashes) + end end def evaluate(context) @@ -32,6 +46,7 @@ module Gitlab @rule_list.find { |rule| rule.matches?(nil, context) } end + # Remove this method when FF `ci_refactor_external_rules` is removed def validate(rule_hashes) return unless rule_hashes.is_a?(Array) @@ -42,6 +57,7 @@ module Gitlab end end + # Remove this method when FF `ci_refactor_external_rules` is removed def valid_when?(rule_hash) rule_hash[:when].nil? || rule_hash[:when].in?(ALLOWED_WHEN) end diff --git a/lib/gitlab/ci/config/header/input.rb b/lib/gitlab/ci/config/header/input.rb index 7f0edaaac4c..76a89a3080e 100644 --- a/lib/gitlab/ci/config/header/input.rb +++ b/lib/gitlab/ci/config/header/input.rb @@ -11,12 +11,13 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable - attributes :default, prefix: :input + attributes :default, :type, prefix: :input validations do - validates :config, type: Hash, allowed_keys: [:default] + validates :config, type: Hash, allowed_keys: [:default, :type] validates :key, alphanumeric: true validates :input_default, alphanumeric: true, allow_nil: true + validates :input_type, allow_nil: true, allowed_values: Interpolation::Inputs.input_types end end end diff --git a/lib/gitlab/ci/config/interpolation/access.rb b/lib/gitlab/ci/config/interpolation/access.rb new file mode 100644 index 00000000000..4c1cd32d28d --- /dev/null +++ b/lib/gitlab/ci/config/interpolation/access.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Interpolation + class Access + attr_reader :content, :errors + + MAX_ACCESS_OBJECTS = 5 + MAX_ACCESS_BYTESIZE = 1024 + + def initialize(access, ctx) + @content = access + @ctx = ctx + @errors = [] + + if objects.count <= 1 # rubocop:disable Style/IfUnlessModifier + @errors.push('invalid interpolation access pattern') + end + + if access.bytesize > MAX_ACCESS_BYTESIZE # rubocop:disable Style/IfUnlessModifier + @errors.push('maximum interpolation expression size exceeded') + end + + evaluate! if valid? + end + + def valid? + errors.none? + end + + def objects + @objects ||= @content.split('.', MAX_ACCESS_OBJECTS) + end + + def value + raise ArgumentError, 'access path invalid' unless valid? + + @value + end + + private + + def evaluate! + raise ArgumentError, 'access path invalid' unless valid? + + @value ||= objects.inject(@ctx) do |memo, value| + key = value.to_sym + + break @errors.push("unknown interpolation key: `#{key}`") unless memo.key?(key) + + memo.fetch(key) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/interpolation/block.rb b/lib/gitlab/ci/config/interpolation/block.rb new file mode 100644 index 00000000000..cf8420f924e --- /dev/null +++ b/lib/gitlab/ci/config/interpolation/block.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Interpolation + ## + # This class represents an interpolation block. The format supported is: + # $[[ <access> | <function1> | <function2> | ... <functionN> ]] + # + # <access> specifies the value to retrieve (e.g. `inputs.key`). + # <function> can be optionally provided with or without arguments to + # manipulate the access value. Functions are evaluated in the order + # they are presented. + class Block + PREFIX = '$[[' + PATTERN = /(?<block>\$\[\[\s*(?<data>.*?)\s*\]\])/ + MAX_FUNCTIONS = 3 + + attr_reader :block, :data, :ctx, :errors + + def initialize(block, data, ctx) + @block = block + @data = data + @ctx = ctx + @errors = [] + @value = nil + + evaluate! + end + + def valid? + errors.none? + end + + def content + data + end + + def value + raise ArgumentError, 'block invalid' unless valid? + + @value + end + + def self.match(data) + return data unless data.is_a?(String) && data.include?(PREFIX) + + data.gsub(PATTERN) do + yield ::Regexp.last_match(1), ::Regexp.last_match(2) + end + end + + private + + # We expect the block data to be a string with one or more entities delimited by pipes: + # <access> | <function1> | <function2> | ... <functionN> + def evaluate! + data_access, *functions = data.split('|').map(&:strip) + access = Interpolation::Access.new(data_access, ctx) + + return @errors.concat(access.errors) unless access.valid? + return @errors.push('too many functions in interpolation block') if functions.count > MAX_FUNCTIONS + + result = Interpolation::FunctionsStack.new(functions).evaluate(access.value) + + if result.success? + @value = result.value + else + @errors.concat(result.errors) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/interpolation/config.rb b/lib/gitlab/ci/config/interpolation/config.rb new file mode 100644 index 00000000000..064d451b0b2 --- /dev/null +++ b/lib/gitlab/ci/config/interpolation/config.rb @@ -0,0 +1,129 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Interpolation + ## + # Interpolation::Config represents a configuration artifact that we want to perform interpolation on. + # + class Config + include Gitlab::Utils::StrongMemoize + ## + # Total number of hash nodes traversed. + # For example, loading a YAML below would result in a hash having 12 nodes + # instead of 9, because hash values are being counted before we recursively traverse them. + # + # test: + # spec: + # env: $[[ inputs.env ]] + # + # $[[ inputs.key ]]: + # name: $[[ inputs.key ]] + # script: my-value + # + # According to our benchmarks performed when developing this code, the worst-case scenario of processing + # a hash with 500_000 nodes takes around 1 second and consumes around 225 megabytes of memory. + # + # The typical scenario, using just a few interpolations, + # takes 250ms and consumes around 20 megabytes of memory. + # + # Given the above the 500_000 nodes should be an upper limit, provided that the are additional safeguard + # present in other parts of the code + # (example: maximum number of interpolation blocks found). Typical size of a + # YAML configuration with 500k nodes might be around 10 megabytes, which is an order of magnitude higher than + # the 1MB limit for loading YAML on GitLab.com + # + MAX_NODES = 500_000 + MAX_NODE_SIZE = 1024 * 1024 # 1MB + + TooManyNodesError = Class.new(StandardError) + NodeTooLargeError = Class.new(StandardError) + + Visitor = Class.new do + def initialize + @visited = 0 + end + + def visit! + @visited += 1 + + raise Config::TooManyNodesError if @visited > Config::MAX_NODES + end + end + + attr_reader :errors + + def initialize(hash) + @config = hash + @errors = [] + end + + def to_h + @config + end + + ## + # The replace! method will yield a block and replace each of the hash config nodes with + # the return value of the block. + # + # It returns `nil` if there were errors found during the process. + # + def replace!(&block) + recursive_replace(@config, Visitor.new, &block) + rescue TooManyNodesError + @errors.push('config too large') + nil + rescue NodeTooLargeError + @errors.push('config node too large') + nil + end + strong_memoize_attr :replace! + + def self.fabricate(config) + case config + when Hash + new(config) + when Interpolation::Config + config + else + raise ArgumentError, 'unknown interpolation config' + end + end + + private + + def recursive_replace(config, visitor, &block) + visitor.visit! + + case config + when Hash + {}.tap do |new_hash| + config.each_pair do |key, value| + new_key = recursive_replace(key, visitor, &block) + new_value = recursive_replace(value, visitor, &block) + + if new_key != key + new_hash[new_key] = new_value + else + new_hash[key] = new_value + end + end + end + when Array + config.map { |value| recursive_replace(value, visitor, &block) } + when Symbol + recursive_replace(config.to_s, visitor, &block) + when String + raise NodeTooLargeError if config.bytesize > MAX_NODE_SIZE + + yield config + else + config + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/interpolation/context.rb b/lib/gitlab/ci/config/interpolation/context.rb new file mode 100644 index 00000000000..f5e7db03291 --- /dev/null +++ b/lib/gitlab/ci/config/interpolation/context.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Interpolation + ## + # Interpolation::Context is a class that represents the data that can be used + # when performing string interpolation on a CI configuration. + # + class Context + ContextTooComplexError = Class.new(StandardError) + NotSymbolizedContextError = Class.new(StandardError) + + MAX_DEPTH = 3 + + def initialize(hash) + @context = hash + + raise ContextTooComplexError if depth > MAX_DEPTH + end + + def valid? + errors.none? + end + + ## + # This method is here because `Context` will be responsible for validating specs, inputs and defaults. + # + def errors + [] + end + + def depth + deep_depth(@context) + end + + def fetch(field) + @context.fetch(field) + end + + def key?(name) + @context.key?(name) + end + + def to_h + @context.to_h + end + + private + + def deep_depth(context, depth = 0) + values = context.values.map do |value| + if value.is_a?(Hash) + deep_depth(value, depth + 1) + else + depth + 1 + end + end + + values.max.to_i + end + + def self.fabricate(context) + case context + when Hash + new(context) + when Interpolation::Context + context + else + raise ArgumentError, 'unknown interpolation context' + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/interpolation/functions/base.rb b/lib/gitlab/ci/config/interpolation/functions/base.rb new file mode 100644 index 00000000000..b9ce8cdc5bc --- /dev/null +++ b/lib/gitlab/ci/config/interpolation/functions/base.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Interpolation + module Functions + class Base + attr_reader :errors + + def self.function_expression_pattern + raise NotImplementedError + end + + def self.name + raise NotImplementedError + end + + def self.matches?(function_expression) + function_expression_pattern.match?(function_expression) + end + + def initialize(function_expression) + @errors = [] + @function_args = parse_args(function_expression) + end + + def valid? + errors.empty? + end + + def execute(_input_value) + raise NotImplementedError + end + + private + + attr_reader :function_args + + def error(message) + errors << "error in `#{self.class.name}` function: #{message}" + end + + def parse_args(function_expression) + self.class.function_expression_pattern.match(function_expression) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/interpolation/functions/truncate.rb b/lib/gitlab/ci/config/interpolation/functions/truncate.rb new file mode 100644 index 00000000000..3771756ba48 --- /dev/null +++ b/lib/gitlab/ci/config/interpolation/functions/truncate.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Interpolation + module Functions + class Truncate < Base + def self.function_expression_pattern + /^#{name}\(\s*(?<offset>\d+)\s*,\s*(?<length>\d+)\s*\)?$/ + end + + def self.name + 'truncate' + end + + def execute(input_value) + if input_value.is_a?(String) + input_value[offset, length].to_s + else + error('invalid input type: truncate can only be used with string inputs') + nil + end + end + + private + + def offset + function_args[:offset].to_i + end + + def length + function_args[:length].to_i + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/interpolation/functions_stack.rb b/lib/gitlab/ci/config/interpolation/functions_stack.rb new file mode 100644 index 00000000000..951d1121d4f --- /dev/null +++ b/lib/gitlab/ci/config/interpolation/functions_stack.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Interpolation + ## + # This class matches the given function string with a predefined + # function and then applies it to the input value. + # + class FunctionsStack + Output = Struct.new(:value, :errors) do + def success? + errors.empty? + end + end + + FUNCTIONS = [ + Functions::Truncate + ].freeze + + attr_reader :errors + + def initialize(function_expressions) + @errors = [] + @functions = build_stack(function_expressions) + end + + def valid? + errors.none? + end + + def evaluate(input_value) + return Output.new(nil, errors) unless valid? + + functions.reduce(Output.new(input_value, [])) do |output, function| + break output unless output.success? + + output_value = function.execute(output.value) + + if function.valid? + Output.new(output_value, []) + else + Output.new(nil, function.errors) + end + end + end + + private + + attr_reader :functions + + def build_stack(function_expressions) + function_expressions.map do |function_expression| + matching_function = FUNCTIONS.find { |function| function.matches?(function_expression) } + + if matching_function.present? + matching_function.new(function_expression) + else + message = "no function matching `#{function_expression}`: " \ + 'check that the function name, arguments, and types are correct' + + errors << message + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/interpolation/inputs.rb b/lib/gitlab/ci/config/interpolation/inputs.rb new file mode 100644 index 00000000000..0fd3238d503 --- /dev/null +++ b/lib/gitlab/ci/config/interpolation/inputs.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Interpolation + # Interpolation inputs provided by the user. + class Inputs + UnknownInputTypeError = Class.new(StandardError) + + TYPES = [ + BooleanInput, + NumberInput, + StringInput + ].freeze + + def self.input_types + TYPES.map(&:type_name) + end + + def initialize(specs, args) + @specs = specs.to_h + @args = args.to_h + @inputs = [] + @errors = [] + + validate! + fabricate! + end + + def errors + @errors + @inputs.flat_map(&:errors) + end + + def valid? + errors.none? + end + + def to_hash + @inputs.inject({}) do |hash, input| + hash.merge(input.to_hash) + end + end + + private + + def validate! + unknown_inputs = @args.keys - @specs.keys + return if unknown_inputs.empty? + + @errors.push("unknown input arguments: #{unknown_inputs.join(', ')}") + end + + def fabricate! + @specs.each do |input_name, spec| + input_type = TYPES.find { |klass| klass.matches?(spec) } + + unless input_type + @errors.push( + "unknown input specification for `#{input_name}` (valid types: #{valid_type_names.join(', ')})") + next + end + + @inputs.push(input_type.new( + name: input_name, + spec: spec, + value: @args[input_name])) + end + end + + def valid_type_names + TYPES.map(&:type_name) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/interpolation/inputs/base_input.rb b/lib/gitlab/ci/config/interpolation/inputs/base_input.rb new file mode 100644 index 00000000000..5648c4d31ea --- /dev/null +++ b/lib/gitlab/ci/config/interpolation/inputs/base_input.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Interpolation + class Inputs + ## + # This is a common abstraction for all input types + class BaseInput + ArgumentNotValidError = Class.new(StandardError) + + # Checks whether the class matches the type in the specification + def self.matches?(spec) + raise NotImplementedError + end + + # Human readable type used in error messages + def self.type_name + raise NotImplementedError + end + + # Checks whether the provided value is of the given type + def valid_value?(value) + raise NotImplementedError + end + + attr_reader :errors, :name, :spec, :value + + def initialize(name:, spec:, value:) + @name = name + @errors = [] + + # Treat minimal spec definition (nil) as a valid hash: + # spec: + # inputs: + # website: + @spec = spec || {} # specification from input definition + @value = value # actual value provided by the user + + validate! + end + + def to_hash + raise ArgumentNotValidError unless valid? + + { name => actual_value } + end + + def valid? + @errors.none? + end + + private + + def validate! + return error('required value has not been provided') if required_input? && value.nil? + + # validate default value + if !required_input? && !valid_value?(default) + return error("default value is not a #{self.class.type_name}") + end + + # validate provided value + error("provided value is not a #{self.class.type_name}") unless valid_value?(actual_value) + end + + def error(message) + @errors.push("`#{name}` input: #{message}") + end + + def actual_value + # nil check is to support boolean values. + value.nil? ? default : value + end + + # An input specification without a default value is required. + # For example: + # ```yaml + # spec: + # inputs: + # website: + # ``` + def required_input? + !spec.key?(:default) + end + + def default + spec[:default] + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/interpolation/inputs/boolean_input.rb b/lib/gitlab/ci/config/interpolation/inputs/boolean_input.rb new file mode 100644 index 00000000000..0293c01a5a8 --- /dev/null +++ b/lib/gitlab/ci/config/interpolation/inputs/boolean_input.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Interpolation + class Inputs + class BooleanInput < BaseInput + def self.matches?(spec) + spec.is_a?(Hash) && spec[:type] == type_name + end + + def self.type_name + 'boolean' + end + + def valid_value?(value) + [true, false].include?(value) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/interpolation/inputs/number_input.rb b/lib/gitlab/ci/config/interpolation/inputs/number_input.rb new file mode 100644 index 00000000000..314315d2b6d --- /dev/null +++ b/lib/gitlab/ci/config/interpolation/inputs/number_input.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Interpolation + class Inputs + class NumberInput < BaseInput + def self.matches?(spec) + spec.is_a?(Hash) && spec[:type] == type_name + end + + def self.type_name + 'number' + end + + def valid_value?(value) + value.is_a?(Numeric) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/interpolation/inputs/string_input.rb b/lib/gitlab/ci/config/interpolation/inputs/string_input.rb new file mode 100644 index 00000000000..39870582d0c --- /dev/null +++ b/lib/gitlab/ci/config/interpolation/inputs/string_input.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Interpolation + class Inputs + class StringInput < BaseInput + def self.matches?(spec) + # The input spec can be `nil` when using a minimal specification + # and also when `type` is not specified. + # + # ```yaml + # spec: + # inputs: + # foo: + # ``` + spec.nil? || (spec.is_a?(Hash) && [nil, type_name].include?(spec[:type])) + end + + def self.type_name + 'string' + end + + def valid_value?(value) + value.nil? || value.is_a?(String) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/yaml/interpolator.rb b/lib/gitlab/ci/config/interpolation/interpolator.rb index 2909c2ac798..58965890184 100644 --- a/lib/gitlab/ci/config/yaml/interpolator.rb +++ b/lib/gitlab/ci/config/interpolation/interpolator.rb @@ -3,19 +3,18 @@ module Gitlab module Ci class Config - module Yaml + module Interpolation ## - # Config::Yaml::Interpolator performs CI config file interpolation, and surfaces all possible interpolation - # errors. It is designed to provide an external file's validation context too. + # Performs CI config file interpolation, and surfaces all possible interpolation errors. # class Interpolator - attr_reader :config, :args, :current_user, :errors + attr_reader :config, :args, :errors - def initialize(config, args, current_user: nil) + def initialize(config, args) @config = config @args = args.to_h - @current_user = current_user @errors = [] + @interpolated = false end def valid? @@ -38,6 +37,7 @@ module Gitlab def interpolate! return @errors.push(config.error) unless config.valid? + return @errors.push('unknown input arguments') if inputs_without_header? return @result ||= config.content unless config.has_header? return @errors.concat(header.errors) unless header.valid? @@ -45,18 +45,23 @@ module Gitlab return @errors.concat(context.errors) unless context.valid? return @errors.concat(template.errors) unless template.valid? - if current_user.present? - ::Gitlab::UsageDataCounters::HLLRedisCounter - .track_event('ci_interpolation_users', values: current_user.id) - end + @interpolated = true @result ||= template.interpolated.to_h.deep_symbolize_keys end + def interpolated? + @interpolated + end + private + def inputs_without_header? + args.any? && !config.has_header? + end + def header - @entry ||= Ci::Config::Header::Root.new(config.header).tap do |header| + @entry ||= Header::Root.new(config.header).tap do |header| header.key = 'header' header.compose! @@ -72,16 +77,15 @@ module Gitlab end def inputs - @inputs ||= Ci::Input::Inputs.new(spec, args) + @inputs ||= Inputs.new(spec, args) end def context - @context ||= Ci::Interpolation::Context.new({ inputs: inputs.to_hash }) + @context ||= Context.new({ inputs: inputs.to_hash }) end def template - @template ||= ::Gitlab::Ci::Interpolation::Template - .new(content, context) + @template ||= Template.new(content, context) end end end diff --git a/lib/gitlab/ci/config/interpolation/template.rb b/lib/gitlab/ci/config/interpolation/template.rb new file mode 100644 index 00000000000..ece2a4756aa --- /dev/null +++ b/lib/gitlab/ci/config/interpolation/template.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Interpolation + class Template + include Gitlab::Utils::StrongMemoize + + attr_reader :blocks, :ctx + + TooManyBlocksError = Class.new(StandardError) + InvalidBlockError = Class.new(StandardError) + + MAX_BLOCKS = 10_000 + + def initialize(config, ctx) + @config = Interpolation::Config.fabricate(config) + @ctx = Interpolation::Context.fabricate(ctx) + @errors = [] + @blocks = {} + + interpolate! if valid? + end + + def valid? + errors.none? + end + + def errors + @errors + @config.errors + @ctx.errors + @blocks.values.flat_map(&:errors) + end + + def size + @blocks.size + end + + def interpolated + @result if valid? + end + + private + + def interpolate! + @result = @config.replace! do |data| + Interpolation::Block.match(data) do |block, data| + evaluate_block(block, data) + end + end + rescue TooManyBlocksError + @errors.push('too many interpolation blocks') + rescue InvalidBlockError + @errors.push('interpolation interrupted by errors') + end + strong_memoize_attr :interpolate! + + def evaluate_block(block, data) + block = (@blocks[block] ||= Interpolation::Block.new(block, data, ctx)) + + raise TooManyBlocksError if @blocks.count > MAX_BLOCKS + raise InvalidBlockError unless block.valid? + + block.value + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/normalizer.rb b/lib/gitlab/ci/config/normalizer.rb index 22fcd84c968..a5b692b26d6 100644 --- a/lib/gitlab/ci/config/normalizer.rb +++ b/lib/gitlab/ci/config/normalizer.rb @@ -44,6 +44,10 @@ module Gitlab job_need_name = job_need[:name].to_sym if all_jobs = parallelized_jobs[job_need_name] + if job_need.key?(:parallel) + all_jobs = parallelize_job_config(job_need_name, job_need.delete(:parallel)) + end + all_jobs.map { |job| job_need.merge(name: job.name) } else job_need diff --git a/lib/gitlab/ci/config/yaml.rb b/lib/gitlab/ci/config/yaml.rb index e3010ac3fdb..51221304430 100644 --- a/lib/gitlab/ci/config/yaml.rb +++ b/lib/gitlab/ci/config/yaml.rb @@ -7,8 +7,8 @@ module Gitlab LoadError = Class.new(StandardError) class << self - def load!(content, current_user: nil) - Loader.new(content, current_user: current_user).load.then do |result| + def load!(content) + Loader.new(content).load.then do |result| raise result.error_class, result.error if !result.valid? && result.error_class.present? raise LoadError, result.error unless result.valid? diff --git a/lib/gitlab/ci/config/yaml/loader.rb b/lib/gitlab/ci/config/yaml/loader.rb index fb24a2874e4..5d56061a8bb 100644 --- a/lib/gitlab/ci/config/yaml/loader.rb +++ b/lib/gitlab/ci/config/yaml/loader.rb @@ -10,9 +10,8 @@ module Gitlab AVAILABLE_TAGS = [Config::Yaml::Tags::Reference].freeze MAX_DOCUMENTS = 2 - def initialize(content, inputs: {}, current_user: nil) + def initialize(content, inputs: {}) @content = content - @current_user = current_user @inputs = inputs end @@ -21,21 +20,21 @@ module Gitlab return yaml_result unless yaml_result.valid? - interpolator = Yaml::Interpolator.new(yaml_result, inputs, current_user: current_user) + interpolator = Interpolation::Interpolator.new(yaml_result, inputs) interpolator.interpolate! if interpolator.valid? # This Result contains only the interpolated config and does not have a header - Yaml::Result.new(config: interpolator.to_hash, error: nil) + Yaml::Result.new(config: interpolator.to_hash, error: nil, interpolated: interpolator.interpolated?) else - Yaml::Result.new(error: interpolator.error_message) + Yaml::Result.new(error: interpolator.error_message, interpolated: interpolator.interpolated?) end end private - attr_reader :content, :current_user, :inputs + attr_reader :content, :inputs def load_uninterpolated_yaml Yaml::Result.new(config: load_yaml!, error: nil) diff --git a/lib/gitlab/ci/config/yaml/result.rb b/lib/gitlab/ci/config/yaml/result.rb index 6b20eeae203..a68cfde6653 100644 --- a/lib/gitlab/ci/config/yaml/result.rb +++ b/lib/gitlab/ci/config/yaml/result.rb @@ -7,16 +7,21 @@ module Gitlab class Result attr_reader :error, :error_class - def initialize(config: nil, error: nil, error_class: nil) + def initialize(config: nil, error: nil, error_class: nil, interpolated: false) @config = Array.wrap(config) @error = error @error_class = error_class + @interpolated = interpolated end def valid? error.nil? end + def interpolated? + !!@interpolated + end + def has_header? return false unless @config.first.is_a?(Hash) diff --git a/lib/gitlab/ci/input/arguments/base.rb b/lib/gitlab/ci/input/arguments/base.rb deleted file mode 100644 index a46037c40ce..00000000000 --- a/lib/gitlab/ci/input/arguments/base.rb +++ /dev/null @@ -1,62 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Input - module Arguments - ## - # Input::Arguments::Base is a common abstraction for input arguments: - # - required - # - optional - # - with a default value - # - class Base - attr_reader :key, :value, :spec, :errors - - ArgumentNotValidError = Class.new(StandardError) - - def initialize(key, spec, value) - @key = key # hash key / argument name - @value = value # user-provided value - @spec = spec # configured specification - @errors = [] - - unless value.is_a?(String) || value.nil? # rubocop:disable Style/IfUnlessModifier - @errors.push("unsupported value in input argument `#{key}`") - end - - validate! - end - - def valid? - @errors.none? - end - - def validate! - raise NotImplementedError - end - - def to_value - raise NotImplementedError - end - - def to_hash - raise ArgumentNotValidError unless valid? - - @output ||= { key => to_value } - end - - def self.matches?(spec) - raise NotImplementedError - end - - private - - def error(message) - @errors.push("`#{@key}` input: #{message}") - end - end - end - end - end -end diff --git a/lib/gitlab/ci/input/arguments/default.rb b/lib/gitlab/ci/input/arguments/default.rb deleted file mode 100644 index c6762b04870..00000000000 --- a/lib/gitlab/ci/input/arguments/default.rb +++ /dev/null @@ -1,48 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Input - module Arguments - ## - # Input::Arguments::Default class represents user-provided input argument that has a default value. - # - class Default < Input::Arguments::Base - def validate! - return error('argument specification invalid') unless spec.key?(:default) - - error('invalid default value') unless default.is_a?(String) || default.nil? - end - - ## - # User-provided value needs to be specified, but it may be an empty string: - # - # ```yaml - # inputs: - # env: - # default: development - # - # with: - # env: "" - # ``` - # - # The configuration above will result in `env` being an empty string. - # - def to_value - value.nil? ? default : value - end - - def default - spec[:default] - end - - def self.matches?(spec) - return false unless spec.is_a?(Hash) - - spec.count == 1 && spec.each_key.first == :default - end - end - end - end - end -end diff --git a/lib/gitlab/ci/input/arguments/options.rb b/lib/gitlab/ci/input/arguments/options.rb deleted file mode 100644 index 855dab129be..00000000000 --- a/lib/gitlab/ci/input/arguments/options.rb +++ /dev/null @@ -1,55 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Input - module Arguments - ## - # Input::Arguments::Options class represents user-provided input argument that is an enum, and is only valid - # when the value provided is listed as an acceptable one. - # - class Options < Input::Arguments::Base - ## - # An empty value is valid if it is allowlisted: - # - # ```yaml - # inputs: - # run: - # - "" - # - tests - # - # with: - # run: "" - # ``` - # - # The configuration above will return an empty value. - # - def validate! - return error('argument specification invalid') unless options.is_a?(Array) - return error('options argument empty') if options.empty? - - if !value.nil? - error("argument value #{value} not allowlisted") unless options.include?(value) - else - error('argument not provided') - end - end - - def to_value - value - end - - def options - spec[:options] - end - - def self.matches?(spec) - return false unless spec.is_a?(Hash) - - spec.count == 1 && spec.each_key.first == :options - end - end - end - end - end -end diff --git a/lib/gitlab/ci/input/arguments/required.rb b/lib/gitlab/ci/input/arguments/required.rb deleted file mode 100644 index 2e39f548731..00000000000 --- a/lib/gitlab/ci/input/arguments/required.rb +++ /dev/null @@ -1,55 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Input - module Arguments - ## - # Input::Arguments::Required class represents user-provided required input argument. - # - class Required < Input::Arguments::Base - ## - # The value has to be defined, but it may be empty. - # - def validate! - error('required value has not been provided') if value.nil? - end - - def to_value - value - end - - ## - # Required arguments do not have nested configuration. It has to be defined a null value. - # - # ```yaml - # spec: - # inputs: - # website: - # ``` - # - # An empty string value, that has no specification is also considered as a "required" input, however we should - # never see that being used, because it will be rejected by Ci::Config::Header validation. - # - # ```yaml - # spec: - # inputs: - # website: "" - # ``` - # - # An empty hash value is also considered to be a required argument: - # - # ```yaml - # spec: - # inputs: - # website: {} - # ``` - # - def self.matches?(spec) - spec.blank? - end - end - end - end - end -end diff --git a/lib/gitlab/ci/input/arguments/unknown.rb b/lib/gitlab/ci/input/arguments/unknown.rb deleted file mode 100644 index 5873e6e66a6..00000000000 --- a/lib/gitlab/ci/input/arguments/unknown.rb +++ /dev/null @@ -1,31 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Input - module Arguments - ## - # Input::Arguments::Unknown object gets fabricated when we can't match an input argument entry with any known - # specification. It is matched as the last one, and always returns an error. - # - class Unknown < Input::Arguments::Base - def validate! - if spec.is_a?(Hash) && spec.count == 1 - error("unrecognized input argument specification: `#{spec.each_key.first}`") - else - error('unrecognized input argument definition') - end - end - - def to_value - raise ArgumentError, 'unknown argument value' - end - - def self.matches?(*) - true - end - end - end - end - end -end diff --git a/lib/gitlab/ci/input/inputs.rb b/lib/gitlab/ci/input/inputs.rb deleted file mode 100644 index 1b544e63e7d..00000000000 --- a/lib/gitlab/ci/input/inputs.rb +++ /dev/null @@ -1,73 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Input - ## - # Inputs::Input class represents user-provided inputs, configured using `with:` keyword. - # - # Input arguments are only valid with an associated component's inputs specification from component's header. - # - class Inputs - UnknownSpecArgumentError = Class.new(StandardError) - - ARGUMENTS = [ - Input::Arguments::Required, # Input argument is required - Input::Arguments::Default, # Input argument has a default value - Input::Arguments::Options, # Input argument that needs to be allowlisted - Input::Arguments::Unknown # Input argument has not been recognized - ].freeze - - def initialize(spec, args) - @spec = spec.to_h - @args = args.to_h - @inputs = [] - @errors = [] - - validate! - fabricate! - end - - def errors - @errors + @inputs.flat_map(&:errors) - end - - def valid? - errors.none? - end - - def unknown - @args.keys - @spec.keys - end - - def count - @inputs.count - end - - def to_hash - @inputs.inject({}) do |hash, argument| - raise ArgumentError unless argument.valid? - - hash.merge(argument.to_hash) - end - end - - private - - def validate! - @errors.push("unknown input arguments: #{unknown.inspect}") if unknown.any? - end - - def fabricate! - @spec.each do |key, spec| - argument = ARGUMENTS.find { |klass| klass.matches?(spec) } - - raise UnknownSpecArgumentError if argument.nil? - - @inputs.push(argument.new(key, spec, @args[key])) - end - end - end - end - end -end diff --git a/lib/gitlab/ci/interpolation/access.rb b/lib/gitlab/ci/interpolation/access.rb deleted file mode 100644 index f9bbd3e118d..00000000000 --- a/lib/gitlab/ci/interpolation/access.rb +++ /dev/null @@ -1,60 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Interpolation - class Access - attr_reader :content, :errors - - MAX_ACCESS_OBJECTS = 5 - MAX_ACCESS_BYTESIZE = 1024 - - def initialize(access, ctx) - @content = access - @ctx = ctx - @errors = [] - - if objects.count <= 1 # rubocop:disable Style/IfUnlessModifier - @errors.push('invalid interpolation access pattern') - end - - if access.bytesize > MAX_ACCESS_BYTESIZE # rubocop:disable Style/IfUnlessModifier - @errors.push('maximum interpolation expression size exceeded') - end - - evaluate! if valid? - end - - def valid? - errors.none? - end - - def objects - @objects ||= @content.split('.', MAX_ACCESS_OBJECTS) - end - - def value - raise ArgumentError, 'access path invalid' unless valid? - - @value - end - - private - - def evaluate! - raise ArgumentError, 'access path invalid' unless valid? - - @value ||= objects.inject(@ctx) do |memo, value| - key = value.to_sym - - break @errors.push("unknown interpolation key: `#{key}`") unless memo.key?(key) - - memo.fetch(key) - end - rescue KeyError => e - @errors.push(e) - end - end - end - end -end diff --git a/lib/gitlab/ci/interpolation/block.rb b/lib/gitlab/ci/interpolation/block.rb deleted file mode 100644 index 389cbf378a2..00000000000 --- a/lib/gitlab/ci/interpolation/block.rb +++ /dev/null @@ -1,48 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Interpolation - class Block - PREFIX = '$[[' - PATTERN = /(?<block>\$\[\[\s*(?<access>.*?)\s*\]\])/.freeze - - attr_reader :block, :data, :ctx - - def initialize(block, data, ctx) - @block = block - @ctx = ctx - @data = data - - @access = Interpolation::Access.new(@data, ctx) - end - - def valid? - errors.none? - end - - def errors - @access.errors - end - - def content - @access.content - end - - def value - raise ArgumentError, 'block invalid' unless valid? - - @access.value - end - - def self.match(data) - return data unless data.is_a?(String) && data.include?(PREFIX) - - data.gsub(PATTERN) do - yield ::Regexp.last_match(1), ::Regexp.last_match(2) - end - end - end - end - end -end diff --git a/lib/gitlab/ci/interpolation/config.rb b/lib/gitlab/ci/interpolation/config.rb deleted file mode 100644 index 32f58521139..00000000000 --- a/lib/gitlab/ci/interpolation/config.rb +++ /dev/null @@ -1,124 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Interpolation - ## - # Interpolation::Config represents a configuration artifact that we want to perform interpolation on. - # - class Config - include Gitlab::Utils::StrongMemoize - ## - # Total number of hash nodes traversed. For example, loading a YAML below would result in a hash having 12 nodes - # instead of 9, because hash values are being counted before we recursively traverse them. - # - # test: - # spec: - # env: $[[ inputs.env ]] - # - # $[[ inputs.key ]]: - # name: $[[ inputs.key ]] - # script: my-value - # - # According to our benchmarks performed when developing this code, the worst-case scenario of processing - # a hash with 500_000 nodes takes around 1 second and consumes around 225 megabytes of memory. - # - # The typical scenario, using just a few interpolations takes 250ms and consumes around 20 megabytes of memory. - # - # Given the above the 500_000 nodes should be an upper limit, provided that the are additional safeguard - # present in other parts of the code (example: maximum number of interpolation blocks found). Typical size of a - # YAML configuration with 500k nodes might be around 10 megabytes, which is an order of magnitude higher than - # the 1MB limit for loading YAML on GitLab.com - # - MAX_NODES = 500_000 - MAX_NODE_SIZE = 1024 * 1024 # 1MB - - TooManyNodesError = Class.new(StandardError) - NodeTooLargeError = Class.new(StandardError) - - Visitor = Class.new do - def initialize - @visited = 0 - end - - def visit! - @visited += 1 - - raise Config::TooManyNodesError if @visited > Config::MAX_NODES - end - end - - attr_reader :errors - - def initialize(hash) - @config = hash - @errors = [] - end - - def to_h - @config - end - - ## - # The replace! method will yield a block and replace a each of the hash config nodes with a return value of the - # block. - # - # It returns `nil` if there were errors found during the process. - # - def replace!(&block) - recursive_replace(@config, Visitor.new, &block) - rescue TooManyNodesError - @errors.push('config too large') - nil - rescue NodeTooLargeError - @errors.push('config node too large') - nil - end - strong_memoize_attr :replace! - - def self.fabricate(config) - case config - when Hash - new(config) - when Interpolation::Config - config - else - raise ArgumentError, 'unknown interpolation config' - end - end - - private - - def recursive_replace(config, visitor, &block) - visitor.visit! - - case config - when Hash - {}.tap do |new_hash| - config.each_pair do |key, value| - new_key = recursive_replace(key, visitor, &block) - new_value = recursive_replace(value, visitor, &block) - - if new_key != key - new_hash[new_key] = new_value - else - new_hash[key] = new_value - end - end - end - when Array - config.map { |value| recursive_replace(value, visitor, &block) } - when Symbol - recursive_replace(config.to_s, visitor, &block) - when String - raise NodeTooLargeError if config.bytesize > MAX_NODE_SIZE - - yield config - else - config - end - end - end - end - end -end diff --git a/lib/gitlab/ci/interpolation/context.rb b/lib/gitlab/ci/interpolation/context.rb deleted file mode 100644 index 69c1fbb792c..00000000000 --- a/lib/gitlab/ci/interpolation/context.rb +++ /dev/null @@ -1,76 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Interpolation - ## - # Interpolation::Context is a class that represents the data that can be used when performing string interpolation - # on a CI configuration. - # - class Context - ContextTooComplexError = Class.new(StandardError) - NotSymbolizedContextError = Class.new(StandardError) - - MAX_DEPTH = 3 - - def initialize(hash) - @context = hash - - raise ContextTooComplexError if depth > MAX_DEPTH - end - - def valid? - errors.none? - end - - ## - # This method is here because `Context` will be responsible for validating specs, inputs and defaults. - # - def errors - [] - end - - def depth - deep_depth(@context) - end - - def fetch(field) - @context.fetch(field) - end - - def key?(name) - @context.key?(name) - end - - def to_h - @context.to_h - end - - private - - def deep_depth(context, depth = 0) - values = context.values.map do |value| - if value.is_a?(Hash) - deep_depth(value, depth + 1) - else - depth + 1 - end - end - - values.max.to_i - end - - def self.fabricate(context) - case context - when Hash - new(context) - when Interpolation::Context - context - else - raise ArgumentError, 'unknown interpolation context' - end - end - end - end - end -end diff --git a/lib/gitlab/ci/interpolation/template.rb b/lib/gitlab/ci/interpolation/template.rb deleted file mode 100644 index 0211279f266..00000000000 --- a/lib/gitlab/ci/interpolation/template.rb +++ /dev/null @@ -1,67 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Interpolation - class Template - include Gitlab::Utils::StrongMemoize - - attr_reader :blocks, :ctx - - TooManyBlocksError = Class.new(StandardError) - InvalidBlockError = Class.new(StandardError) - - MAX_BLOCKS = 10_000 - - def initialize(config, ctx) - @config = Interpolation::Config.fabricate(config) - @ctx = Interpolation::Context.fabricate(ctx) - @errors = [] - @blocks = {} - - interpolate! if valid? - end - - def valid? - errors.none? - end - - def errors - @errors + @config.errors + @ctx.errors + @blocks.values.flat_map(&:errors) - end - - def size - @blocks.size - end - - def interpolated - @result if valid? - end - - private - - def interpolate! - @result = @config.replace! do |data| - Interpolation::Block.match(data) do |block, data| - evaluate_block(block, data) - end - end - rescue TooManyBlocksError - @errors.push('too many interpolation blocks') - rescue InvalidBlockError - @errors.push('interpolation interrupted by errors') - end - strong_memoize_attr :interpolate! - - def evaluate_block(block, data) - block = (@blocks[block] ||= Interpolation::Block.new(block, data, ctx)) - - raise TooManyBlocksError if @blocks.count > MAX_BLOCKS - raise InvalidBlockError unless block.valid? - - block.value - end - end - end - end -end diff --git a/lib/gitlab/ci/jwt_v2.rb b/lib/gitlab/ci/jwt_v2.rb index 6ce662bdead..8c730a9548f 100644 --- a/lib/gitlab/ci/jwt_v2.rb +++ b/lib/gitlab/ci/jwt_v2.rb @@ -44,26 +44,14 @@ module Gitlab end def custom_claims - additional_claims = { + mapper = ClaimMapper.new(project_config, pipeline) + + super.merge({ runner_id: runner&.id, runner_environment: runner_environment, - sha: pipeline.sha - } - - if project_config&.source == :repository_source - additional_claims[:ci_config_ref_uri] = ci_config_ref_uri - additional_claims[:ci_config_sha] = pipeline.sha - end - - super.merge(additional_claims) - end - - def ci_config_ref_uri - "#{project_config&.url}@#{pipeline.source_ref_path}" - rescue StandardError => e - # We don't want endpoints relying on this code to fail if there's an error here. - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, pipeline_id: pipeline.id) - nil + sha: pipeline.sha, + project_visibility: Gitlab::VisibilityLevel.string_level(project.visibility_level) + }).merge(mapper.to_h) end def project_config diff --git a/lib/gitlab/ci/jwt_v2/claim_mapper.rb b/lib/gitlab/ci/jwt_v2/claim_mapper.rb new file mode 100644 index 00000000000..8e7f024d2d6 --- /dev/null +++ b/lib/gitlab/ci/jwt_v2/claim_mapper.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class JwtV2 + class ClaimMapper + MAPPER_FOR_CONFIG_SOURCE = { + repository_source: ClaimMapper::Repository + }.freeze + + def initialize(project_config, pipeline) + return unless project_config + + mapper_class = MAPPER_FOR_CONFIG_SOURCE[project_config.source] + @mapper = mapper_class&.new(project_config, pipeline) + end + + def to_h + mapper&.to_h || {} + end + + private + + attr_reader :mapper + end + end + end +end diff --git a/lib/gitlab/ci/jwt_v2/claim_mapper/repository.rb b/lib/gitlab/ci/jwt_v2/claim_mapper/repository.rb new file mode 100644 index 00000000000..1949d44b8b5 --- /dev/null +++ b/lib/gitlab/ci/jwt_v2/claim_mapper/repository.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class JwtV2 + class ClaimMapper + class Repository + def initialize(project_config, pipeline) + @project_config = project_config + @pipeline = pipeline + end + + def to_h + { + ci_config_ref_uri: ci_config_ref_uri, + ci_config_sha: pipeline.sha + } + end + + private + + attr_reader :project_config, :pipeline + + def ci_config_ref_uri + "#{project_config.url}@#{pipeline.source_ref_path}" + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/ensure_environments.rb b/lib/gitlab/ci/pipeline/chain/ensure_environments.rb index ebea6a538ef..3ff9aff68ac 100644 --- a/lib/gitlab/ci/pipeline/chain/ensure_environments.rb +++ b/lib/gitlab/ci/pipeline/chain/ensure_environments.rb @@ -16,7 +16,7 @@ module Gitlab private def ensure_environment(build) - ::Environments::CreateForBuildService.new.execute(build) + ::Environments::CreateForJobService.new.execute(build) end end end diff --git a/lib/gitlab/ci/project_config/remote.rb b/lib/gitlab/ci/project_config/remote.rb index 19cbf8e9c1e..23ed510e378 100644 --- a/lib/gitlab/ci/project_config/remote.rb +++ b/lib/gitlab/ci/project_config/remote.rb @@ -6,7 +6,7 @@ module Gitlab class Remote < Source def content strong_memoize(:content) do - next unless ci_config_path =~ URI::DEFAULT_PARSER.make_regexp(%w[http https]) + next unless URI::DEFAULT_PARSER.make_regexp(%w[http https]).match?(ci_config_path) YAML.dump('include' => [{ 'remote' => ci_config_path }]) end diff --git a/lib/gitlab/ci/project_config/repository.rb b/lib/gitlab/ci/project_config/repository.rb index a08cf27b74c..93e13f57943 100644 --- a/lib/gitlab/ci/project_config/repository.rb +++ b/lib/gitlab/ci/project_config/repository.rb @@ -33,7 +33,7 @@ module Gitlab return unless project return unless sha - project.repository.gitlab_ci_yml_for(sha, ci_config_path).present? + project.repository.blob_at(sha, ci_config_path).present? rescue GRPC::NotFound, GRPC::Internal nil end diff --git a/lib/gitlab/ci/queue/metrics.rb b/lib/gitlab/ci/queue/metrics.rb index 5cee73238ca..a18542288c9 100644 --- a/lib/gitlab/ci/queue/metrics.rb +++ b/lib/gitlab/ci/queue/metrics.rb @@ -74,7 +74,7 @@ module Gitlab end def observe_queue_depth(queue, size) - return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics) + return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, type: :ops) if !Rails.env.production? && !QUEUE_DEPTH_HISTOGRAMS.include?(queue) raise ArgumentError, "unknown queue depth label: #{queue}" @@ -84,7 +84,7 @@ module Gitlab end def observe_queue_size(size_proc, runner_type) - return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics) + return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, type: :ops) size = size_proc.call.to_f self.class.queue_size_total.observe({ runner_type: runner_type }, size) @@ -96,7 +96,7 @@ module Gitlab result = yield - return result unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics) + return result unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, type: :ops) seconds = ::Gitlab::Metrics::System.monotonic_time - start_time @@ -121,7 +121,7 @@ module Gitlab end def self.observe_active_runners(runners_proc) - return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics) + return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, type: :ops) queue_active_runners_total.observe({}, runners_proc.call.to_f) end @@ -133,7 +133,7 @@ module Gitlab def self.failed_attempt_counter strong_memoize(:failed_attempt_counter) do name = :job_register_attempts_failed_total - comment = 'Counts the times a runner tries to register a job' + comment = 'Counts the times a runner fails to register a job' Gitlab::Metrics.counter(name, comment) end diff --git a/lib/gitlab/ci/reports/sbom/component.rb b/lib/gitlab/ci/reports/sbom/component.rb index 08307580987..51fd6af7bc4 100644 --- a/lib/gitlab/ci/reports/sbom/component.rb +++ b/lib/gitlab/ci/reports/sbom/component.rb @@ -5,23 +5,36 @@ module Gitlab module Reports module Sbom class Component - attr_reader :component_type, :name, :version + include Gitlab::Utils::StrongMemoize + + attr_reader :component_type, :version def initialize(type:, name:, purl:, version:) @component_type = type @name = name - @purl = purl + @raw_purl = purl @version = version end + def <=>(other) + sort_by_attributes(self) <=> sort_by_attributes(other) + end + def ingestible? supported_component_type? && supported_purl_type? end def purl - return unless @purl + return unless @raw_purl + + ::Sbom::PackageUrl.parse(@raw_purl) + end + strong_memoize_attr :purl + + def name + return @name unless purl - ::Sbom::PackageUrl.parse(@purl) + [purl.namespace, purl.name].compact.join('/') end private @@ -37,6 +50,23 @@ module Gitlab # however, if the purl type is provided, it _must be valid_ ::Enums::Sbom.purl_types.include?(purl.type.to_sym) end + + def sort_by_attributes(component) + [ + component.name, + purl_type_int(component), + component_type_int(component), + component.version.to_s + ] + end + + def component_type_int(component) + ::Enums::Sbom::COMPONENT_TYPES.fetch(component.component_type.to_sym) + end + + def purl_type_int(component) + ::Enums::Sbom::PURL_TYPES.fetch(component.purl&.type&.to_sym, 0) + end end end end diff --git a/lib/gitlab/ci/status/bridge/factory.rb b/lib/gitlab/ci/status/bridge/factory.rb index fbe2e2282a0..20773237547 100644 --- a/lib/gitlab/ci/status/bridge/factory.rb +++ b/lib/gitlab/ci/status/bridge/factory.rb @@ -9,6 +9,7 @@ module Gitlab [[Status::Bridge::Retryable], [Status::Bridge::Failed], [Status::Bridge::Manual], + [Status::Bridge::WaitingForApproval], [Status::Bridge::WaitingForResource], [Status::Bridge::Play], [Status::Bridge::Action], diff --git a/lib/gitlab/ci/status/bridge/waiting_for_approval.rb b/lib/gitlab/ci/status/bridge/waiting_for_approval.rb new file mode 100644 index 00000000000..2d3165f41ed --- /dev/null +++ b/lib/gitlab/ci/status/bridge/waiting_for_approval.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Status + module Bridge + class WaitingForApproval < Status::Extended + ## Extended in EE + def self.matches?(_bridge, _user) + false + end + end + end + end + end +end + +Gitlab::Ci::Status::Bridge::WaitingForApproval.prepend_mod_with('Gitlab::Ci::Status::Bridge::WaitingForApproval') diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index 4f12f0cd3b8..2a07530c00d 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -16,7 +16,6 @@ # Test jobs may be disabled by setting environment variables: # * test: TEST_DISABLED # * code_quality: CODE_QUALITY_DISABLED -# * license_management: LICENSE_MANAGEMENT_DISABLED # * browser_performance: BROWSER_PERFORMANCE_DISABLED # * load_performance: LOAD_PERFORMANCE_DISABLED # * sast: SAST_DISABLED @@ -178,7 +177,6 @@ include: - template: Security/DAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml - template: Jobs/Container-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml - template: Jobs/Dependency-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml - - template: Jobs/License-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml - template: Jobs/SAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml - template: Jobs/Secret-Detection.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index 46d0b92b243..c1aedbe1111 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.37.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.38.1' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml index 46d0b92b243..c1aedbe1111 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.37.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.38.1' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 7c9aa82b1ae..f9440bfe904 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -10,6 +10,7 @@ code_quality: DOCKER_TLS_CERTDIR: "" CODE_QUALITY_IMAGE_TAG: "0.96.0" CODE_QUALITY_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/gitlab-org/ci-cd/codequality:$CODE_QUALITY_IMAGE_TAG" + DOCKER_SOCKET_PATH: /var/run/docker.sock needs: [] script: - export SOURCE_CODE=$PWD @@ -46,9 +47,10 @@ code_quality: CODECLIMATE_PREFIX \ CODECLIMATE_REGISTRY_USERNAME \ CODECLIMATE_REGISTRY_PASSWORD \ + DOCKER_SOCKET_PATH \ ) \ --volume "$PWD":/code \ - --volume /var/run/docker.sock:/var/run/docker.sock \ + --volume "$DOCKER_SOCKET_PATH":/var/run/docker.sock \ "$CODE_QUALITY_IMAGE" /code artifacts: reports: diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index b1e498a9d09..7b2fb49b65e 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.51.0' + DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.53.0' .dast-auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 5a7e69b62d9..1e482ccca82 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.51.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.53.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index dac559db8d5..6eac691b293 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.51.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.53.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml index b1c81e9ed5b..58846d31e2f 100644 --- a/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml @@ -14,7 +14,7 @@ variables: SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" LICENSE_MANAGEMENT_SETUP_CMD: '' # If needed, specify a command to setup your environment with a custom package manager. - LICENSE_MANAGEMENT_VERSION: 4 + LICENSE_MANAGEMENT_VERSION: 'removed' license_scanning: stage: test diff --git a/lib/gitlab/ci/templates/Security/BAS.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/BAS.latest.gitlab-ci.yml index b626a7ca770..8b3f4a619d0 100644 --- a/lib/gitlab/ci/templates/Security/BAS.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/BAS.latest.gitlab-ci.yml @@ -18,11 +18,8 @@ # # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/breach_and_attack_simulation/index.html#extend-dynamic-application-security-testing-dast -# Include the DAST.latest template if $DAST_VERSION is null because this means a DAST template has not been included already. include: - template: Security/DAST.latest.gitlab-ci.yml - rules: - - if: $DAST_VERSION == null variables: BAS_CALLBACK_IMAGE_TAG: "latest" diff --git a/lib/gitlab/ci/variables/builder/pipeline.rb b/lib/gitlab/ci/variables/builder/pipeline.rb index c3b0cb856ba..3e66290ecde 100644 --- a/lib/gitlab/ci/variables/builder/pipeline.rb +++ b/lib/gitlab/ci/variables/builder/pipeline.rb @@ -16,6 +16,7 @@ module Gitlab variables.append(key: 'CI_PIPELINE_IID', value: pipeline.iid.to_s) variables.append(key: 'CI_PIPELINE_SOURCE', value: pipeline.source.to_s) variables.append(key: 'CI_PIPELINE_CREATED_AT', value: pipeline.created_at&.iso8601) + variables.append(key: 'CI_PIPELINE_NAME', value: pipeline.name) variables.concat(predefined_commit_variables) if pipeline.sha.present? variables.concat(predefined_commit_tag_variables) if pipeline.tag? diff --git a/lib/gitlab/ci/variables/downstream/expandable_variable_generator.rb b/lib/gitlab/ci/variables/downstream/expandable_variable_generator.rb index 6690e9f1c1f..bb7a6e7ab59 100644 --- a/lib/gitlab/ci/variables/downstream/expandable_variable_generator.rb +++ b/lib/gitlab/ci/variables/downstream/expandable_variable_generator.rb @@ -6,9 +6,40 @@ module Gitlab module Downstream class ExpandableVariableGenerator < Base def for(item) - expanded_value = ::ExpandVariables.expand(item.value, context.all_bridge_variables) + expanded_var = expanded_var_for(item) + file_vars = file_var_dependencies_for(item) - [{ key: item.key, value: expanded_value }] + [expanded_var].concat(file_vars) + end + + private + + def expanded_var_for(item) + { + key: item.key, + value: ::ExpandVariables.expand( + item.value, + context.all_bridge_variables, + expand_file_refs: context.expand_file_refs + ) + } + end + + def file_var_dependencies_for(item) + return [] if context.expand_file_refs + return [] unless item.depends_on + + item.depends_on.filter_map do |dependency| + dependency_variable = context.all_bridge_variables[dependency] + + if dependency_variable&.file? + { + key: dependency_variable.key, + value: dependency_variable.value, + variable_type: :file + } + end + end end end end diff --git a/lib/gitlab/ci/variables/downstream/generator.rb b/lib/gitlab/ci/variables/downstream/generator.rb index 93c995cc918..350d29958cf 100644 --- a/lib/gitlab/ci/variables/downstream/generator.rb +++ b/lib/gitlab/ci/variables/downstream/generator.rb @@ -7,12 +7,12 @@ module Gitlab class Generator include Gitlab::Utils::StrongMemoize - Context = Struct.new(:all_bridge_variables, keyword_init: true) + Context = Struct.new(:all_bridge_variables, :expand_file_refs, keyword_init: true) def initialize(bridge) @bridge = bridge - context = Context.new(all_bridge_variables: bridge.variables) + context = Context.new(all_bridge_variables: bridge.variables, expand_file_refs: bridge.expand_file_refs?) @raw_variable_generator = RawVariableGenerator.new(context) @expandable_variable_generator = ExpandableVariableGenerator.new(context) diff --git a/lib/gitlab/config/entry/validators.rb b/lib/gitlab/config/entry/validators.rb index 28cfb6d8fee..87b7cab3f6d 100644 --- a/lib/gitlab/config/entry/validators.rb +++ b/lib/gitlab/config/entry/validators.rb @@ -40,6 +40,17 @@ module Gitlab end end + class OnlyOneOfKeysValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + present_keys = value.try(:keys).to_a + + unless options[:in].one? { |key| present_keys.include?(key) } + record.errors.add(attribute, "must use exactly one of these keys: " + + options[:in].join(', ')) + end + end + end + class MutuallyExclusiveKeysValidator < ActiveModel::EachValidator def validate_each(record, attribute, value) mutually_exclusive_keys = value.try(:keys).to_a & options[:in] diff --git a/lib/gitlab/content_security_policy/config_loader.rb b/lib/gitlab/content_security_policy/config_loader.rb index 669c447c09b..9fb3c7d362f 100644 --- a/lib/gitlab/content_security_policy/config_loader.rb +++ b/lib/gitlab/content_security_policy/config_loader.rb @@ -3,186 +3,228 @@ module Gitlab module ContentSecurityPolicy class ConfigLoader - DIRECTIVES = %w(base_uri child_src connect_src default_src font_src - form_action frame_ancestors frame_src img_src manifest_src - media_src object_src report_uri script_src style_src worker_src).freeze - + DIRECTIVES = %w[ + base_uri child_src connect_src default_src font_src form_action + frame_ancestors frame_src img_src manifest_src media_src object_src + report_uri script_src style_src worker_src + ].freeze DEFAULT_FALLBACK_VALUE = '<default_value>' + HTTP_PORTS = [80, 443].freeze - def self.default_enabled - Rails.env.development? || Rails.env.test? - end + class << self + def default_enabled + Rails.env.development? || Rails.env.test? + end - def self.default_directives - directives = { - 'default_src' => "'self'", - 'base_uri' => "'self'", - 'connect_src' => ContentSecurityPolicy::Directives.connect_src, - 'font_src' => "'self'", - 'form_action' => "'self' https: http:", - 'frame_ancestors' => "'self'", - 'frame_src' => ContentSecurityPolicy::Directives.frame_src, - 'img_src' => "'self' data: blob: http: https:", - 'manifest_src' => "'self'", - 'media_src' => "'self' data: blob: http: https:", - 'script_src' => ContentSecurityPolicy::Directives.script_src, - 'style_src' => ContentSecurityPolicy::Directives.style_src, - 'worker_src' => "#{Gitlab::Utils.append_path(Gitlab.config.gitlab.url, 'assets/')} blob: data:", - 'object_src' => "'none'", - 'report_uri' => nil - } + def default_directives + directives = default_directives_defaults + + allow_development_tooling(directives) + allow_websocket_connections(directives) + allow_lfs(directives) + allow_cdn(directives) + allow_zuora(directives) + allow_sentry(directives) + allow_framed_gitlab_paths(directives) + allow_customersdot(directives) + allow_review_apps(directives) + csp_level_3_backport(directives) + + directives + end + + def default_directives_defaults + { + 'default_src' => "'self'", + 'base_uri' => "'self'", + 'connect_src' => ContentSecurityPolicy::Directives.connect_src, + 'font_src' => "'self'", + 'form_action' => "'self' https: http:", + 'frame_ancestors' => "'self'", + 'frame_src' => ContentSecurityPolicy::Directives.frame_src, + 'img_src' => "'self' data: blob: http: https:", + 'manifest_src' => "'self'", + 'media_src' => "'self' data: blob: http: https:", + 'script_src' => ContentSecurityPolicy::Directives.script_src, + 'style_src' => ContentSecurityPolicy::Directives.style_src, + 'worker_src' => "#{Gitlab::Utils.append_path(Gitlab.config.gitlab.url, 'assets/')} blob: data:", + 'object_src' => "'none'", + 'report_uri' => nil + } + end # connect_src with 'self' includes https/wss variations of the origin, # however, safari hasn't covered this yet and we need to explicitly add # support for websocket origins until Safari catches up with the specs - if Rails.env.development? + def allow_development_tooling(directives) + return unless Rails.env.development? + allow_webpack_dev_server(directives) allow_letter_opener(directives) allow_snowplow_micro(directives) if Gitlab::Tracking.snowplow_micro_enabled? end - allow_websocket_connections(directives) - allow_cdn(directives, Settings.gitlab.cdn_host) if Settings.gitlab.cdn_host.present? - allow_zuora(directives) if Gitlab.com? - # Support for Sentry setup via configuration files will be removed in 16.0 - # in favor of Gitlab::CurrentSettings. - allow_legacy_sentry(directives) if Gitlab.config.sentry&.enabled && Gitlab.config.sentry&.clientside_dsn - allow_sentry(directives) if Gitlab::CurrentSettings.try(:sentry_enabled) && Gitlab::CurrentSettings.try(:sentry_clientside_dsn) - allow_framed_gitlab_paths(directives) - allow_customersdot(directives) if ENV['CUSTOMER_PORTAL_URL'].present? - allow_review_apps(directives) if ENV['REVIEW_APPS_ENABLED'] - - # The follow section contains workarounds to patch Safari's lack of support for CSP Level 3 - # See https://gitlab.com/gitlab-org/gitlab/-/issues/343579 - # frame-src was deprecated in CSP level 2 in favor of child-src - # CSP level 3 "undeprecated" frame-src and browsers fall back on child-src if it's missing - # However Safari seems to read child-src first so we'll just keep both equal - append_to_directive(directives, 'child_src', directives['frame_src']) - - # Safari also doesn't support worker-src and only checks child-src - # So for compatibility until it catches up to other browsers we need to - # append worker-src's content to child-src - append_to_directive(directives, 'child_src', directives['worker_src']) - - directives - end + def allow_webpack_dev_server(directives) + secure = Settings.webpack.dev_server['https'] + host_and_port = "#{Settings.webpack.dev_server['host']}:#{Settings.webpack.dev_server['port']}" + http_url = "#{secure ? 'https' : 'http'}://#{host_and_port}" + ws_url = "#{secure ? 'wss' : 'ws'}://#{host_and_port}" - def initialize(csp_directives) - # Using <default_value> falls back to the default values. - directives = csp_directives.reject { |_, value| value == DEFAULT_FALLBACK_VALUE } - @merged_csp_directives = - HashWithIndifferentAccess.new(directives) - .reverse_merge(::Gitlab::ContentSecurityPolicy::ConfigLoader.default_directives) - end + append_to_directive(directives, 'connect_src', "#{http_url} #{ws_url}") + end - def load(policy) - DIRECTIVES.each do |directive| - arguments = arguments_for(directive) + def allow_letter_opener(directives) + url = Gitlab::Utils.append_path(Gitlab.config.gitlab.url, '/rails/letter_opener/') + append_to_directive(directives, 'frame_src', url) + end - next unless arguments.present? + def allow_snowplow_micro(directives) + url = URI.join(Gitlab::Tracking::Destinations::SnowplowMicro.new.uri, '/').to_s + append_to_directive(directives, 'connect_src', url) + end - policy.public_send(directive, *arguments) # rubocop:disable GitlabSecurity/PublicSend + def allow_lfs(directives) + return unless Gitlab.config.lfs.enabled && LfsObjectUploader.direct_download_enabled? + + lfs_url = build_lfs_url + return unless lfs_url.present? + + append_to_directive(directives, 'connect_src', lfs_url) end - end - private + def allow_websocket_connections(directives) + host = Gitlab.config.gitlab.host + port = Gitlab.config.gitlab.port + secure = Gitlab.config.gitlab.https + protocol = secure ? 'wss' : 'ws' - def arguments_for(directive) - # In order to disable a directive, the user can explicitly - # set a falsy value like nil, false or empty string - arguments = @merged_csp_directives[directive] - return unless arguments.present? && arguments.is_a?(String) + ws_url = "#{protocol}://#{host}" + ws_url = "#{ws_url}:#{port}" unless HTTP_PORTS.include?(port) - arguments.strip.split(' ').map(&:strip) - end + append_to_directive(directives, 'connect_src', ws_url) + end - def self.allow_websocket_connections(directives) - http_ports = [80, 443] - host = Gitlab.config.gitlab.host - port = Gitlab.config.gitlab.port - secure = Gitlab.config.gitlab.https - protocol = secure ? 'wss' : 'ws' + def allow_cdn(directives) + cdn_host = Settings.gitlab.cdn_host.presence + return unless cdn_host + + append_to_directive(directives, 'script_src', cdn_host) + append_to_directive(directives, 'style_src', cdn_host) + append_to_directive(directives, 'font_src', cdn_host) + append_to_directive(directives, 'worker_src', cdn_host) + append_to_directive(directives, 'frame_src', cdn_host) + end - ws_url = "#{protocol}://#{host}" + def allow_zuora(directives) + return unless Gitlab.com? - unless http_ports.include?(port) - ws_url = "#{ws_url}:#{port}" + append_to_directive(directives, 'frame_src', zuora_host) end - append_to_directive(directives, 'connect_src', ws_url) - end + def allow_sentry(directives) + allow_legacy_sentry(directives) if legacy_sentry_configured? + return unless sentry_client_side_dsn_enabled? - def self.allow_webpack_dev_server(directives) - secure = Settings.webpack.dev_server['https'] - host_and_port = "#{Settings.webpack.dev_server['host']}:#{Settings.webpack.dev_server['port']}" - http_url = "#{secure ? 'https' : 'http'}://#{host_and_port}" - ws_url = "#{secure ? 'wss' : 'ws'}://#{host_and_port}" + sentry_uri = URI(Gitlab::CurrentSettings.sentry_clientside_dsn) - append_to_directive(directives, 'connect_src', "#{http_url} #{ws_url}") - end + append_to_directive(directives, 'connect_src', "#{sentry_uri.scheme}://#{sentry_uri.host}") + end - def self.allow_cdn(directives, cdn_host) - append_to_directive(directives, 'script_src', cdn_host) - append_to_directive(directives, 'style_src', cdn_host) - append_to_directive(directives, 'font_src', cdn_host) - append_to_directive(directives, 'worker_src', cdn_host) - append_to_directive(directives, 'frame_src', cdn_host) - end + def allow_legacy_sentry(directives) + # Support for Sentry setup via configuration files will be removed in 16.0 + # in favor of Gitlab::CurrentSettings. + sentry_uri = URI(Gitlab.config.sentry.clientside_dsn) - def self.zuora_host - "https://*.zuora.com/apps/PublicHostedPageLite.do" - end + append_to_directive(directives, 'connect_src', "#{sentry_uri.scheme}://#{sentry_uri.host}") + end - def self.allow_zuora(directives) - append_to_directive(directives, 'frame_src', zuora_host) - end + def legacy_sentry_configured? + Gitlab.config.sentry&.enabled && Gitlab.config.sentry&.clientside_dsn + end - def self.append_to_directive(directives, directive, text) - directives[directive] = "#{directives[directive]} #{text}".strip - end + def sentry_client_side_dsn_enabled? + Gitlab::CurrentSettings.try(:sentry_enabled) && Gitlab::CurrentSettings.try(:sentry_clientside_dsn) + end - def self.allow_customersdot(directives) - customersdot_host = ENV['CUSTOMER_PORTAL_URL'] + # Using 'self' in the CSP introduces several CSP bypass opportunities + # for this reason we list the URLs where GitLab frames itself instead + def allow_framed_gitlab_paths(directives) + ['/admin/', '/assets/', '/-/speedscope/index.html', '/-/sandbox/'].map do |path| + append_to_directive(directives, 'frame_src', Gitlab::Utils.append_path(Gitlab.config.gitlab.url, path)) + end + end - append_to_directive(directives, 'frame_src', customersdot_host) - end + def allow_customersdot(directives) + customersdot_host = ENV['CUSTOMER_PORTAL_URL'].presence + return unless customersdot_host - def self.allow_legacy_sentry(directives) - # Support for Sentry setup via configuration files will be removed in 16.0 - # in favor of Gitlab::CurrentSettings. - sentry_dsn = Gitlab.config.sentry.clientside_dsn - sentry_uri = URI(sentry_dsn) + append_to_directive(directives, 'frame_src', customersdot_host) + end - append_to_directive(directives, 'connect_src', "#{sentry_uri.scheme}://#{sentry_uri.host}") - end + def allow_review_apps(directives) + return unless ENV['REVIEW_APPS_ENABLED'].presence - def self.allow_sentry(directives) - sentry_dsn = Gitlab::CurrentSettings.sentry_clientside_dsn - sentry_uri = URI(sentry_dsn) + # Allow-listed to allow POSTs to https://gitlab.com/api/v4/projects/278964/merge_requests/:merge_request_iid/visual_review_discussions + append_to_directive(directives, 'connect_src', 'https://gitlab.com/api/v4/projects/278964/merge_requests/') + end - append_to_directive(directives, 'connect_src', "#{sentry_uri.scheme}://#{sentry_uri.host}") - end + # The follow contains workarounds to patch Safari's lack of support for CSP Level 3 + def csp_level_3_backport(directives) + # See https://gitlab.com/gitlab-org/gitlab/-/issues/343579 + # frame-src was deprecated in CSP level 2 in favor of child-src + # CSP level 3 "undeprecated" frame-src and browsers fall back on child-src if it's missing + # However Safari seems to read child-src first so we'll just keep both equal + append_to_directive(directives, 'child_src', directives['frame_src']) + + # Safari also doesn't support worker-src and only checks child-src + # So for compatibility until it catches up to other browsers we need to + # append worker-src's content to child-src + append_to_directive(directives, 'child_src', directives['worker_src']) + end + + def append_to_directive(directives, directive, text) + directives[directive] = "#{directives[directive]} #{text}".strip + end + + def zuora_host + "https://*.zuora.com/apps/PublicHostedPageLite.do" + end - def self.allow_letter_opener(directives) - append_to_directive(directives, 'frame_src', Gitlab::Utils.append_path(Gitlab.config.gitlab.url, '/rails/letter_opener/')) + def build_lfs_url + uploader = LfsObjectUploader.new(nil) + fog = CarrierWave::Storage::Fog.new(uploader) + fog_file = CarrierWave::Storage::Fog::File.new(uploader, fog, nil) + fog_file.public_url || fog_file.url + end end - def self.allow_snowplow_micro(directives) - url = URI.join(Gitlab::Tracking::Destinations::SnowplowMicro.new.uri, '/').to_s - append_to_directive(directives, 'connect_src', url) + def initialize(csp_directives) + # Using <default_value> falls back to the default values. + @merged_csp_directives = csp_directives + .reject { |_, value| value == DEFAULT_FALLBACK_VALUE } + .with_indifferent_access + .reverse_merge(ConfigLoader.default_directives) end - # Using 'self' in the CSP introduces several CSP bypass opportunities - # for this reason we list the URLs where GitLab frames itself instead - def self.allow_framed_gitlab_paths(directives) - ['/admin/', '/assets/', '/-/speedscope/index.html', '/-/sandbox/'].map do |path| - append_to_directive(directives, 'frame_src', Gitlab::Utils.append_path(Gitlab.config.gitlab.url, path)) + def load(policy) + DIRECTIVES.each do |directive| + arguments = arguments_for(directive) + + next unless arguments.present? + + policy.public_send(directive, *arguments) # rubocop:disable GitlabSecurity/PublicSend end end - def self.allow_review_apps(directives) - # Allow-listed to allow POSTs to https://gitlab.com/api/v4/projects/278964/merge_requests/:merge_request_iid/visual_review_discussions - append_to_directive(directives, 'connect_src', 'https://gitlab.com/api/v4/projects/278964/merge_requests/') + private + + def arguments_for(directive) + # In order to disable a directive, the user can explicitly + # set a falsy value like nil, false or empty string + arguments = @merged_csp_directives[directive] + return unless arguments.is_a?(String) + + arguments.split(' ') end end end diff --git a/lib/gitlab/data_builder/build.rb b/lib/gitlab/data_builder/build.rb index e1e9e4720bb..7abad66cf13 100644 --- a/lib/gitlab/data_builder/build.rb +++ b/lib/gitlab/data_builder/build.rb @@ -12,7 +12,7 @@ module Gitlab author_url = build_author_url(build.commit, commit) - { + attrs = { object_kind: 'build', ref: build.ref, @@ -68,8 +68,13 @@ module Gitlab visibility_level: project.visibility_level }, + project: project.hook_attrs(backward: false), + environment: build_environment(build) } + + attrs[:source_pipeline] = source_pipeline_attrs(commit.source_pipeline) if commit.source_pipeline.present? + attrs end private @@ -100,6 +105,20 @@ module Gitlab action: build.environment_action } end + + def source_pipeline_attrs(source_pipeline) + project = source_pipeline.source_project + + { + project: { + id: project.id, + web_url: project.web_url, + path_with_namespace: project.full_path + }, + job_id: source_pipeline.source_job_id, + pipeline_id: source_pipeline.source_pipeline_id + } + end end end end diff --git a/lib/gitlab/data_builder/issuable.rb b/lib/gitlab/data_builder/issuable.rb index 9a7b4d0e2aa..57a69c7c0c1 100644 --- a/lib/gitlab/data_builder/issuable.rb +++ b/lib/gitlab/data_builder/issuable.rb @@ -16,12 +16,12 @@ module Gitlab object_kind: object_kind, event_type: event_type, user: user.hook_attrs, - project: issuable.project.hook_attrs, + project: issuable.project&.hook_attrs, object_attributes: issuable_builder.new(issuable).build, labels: issuable.labels_hook_attrs, changes: final_changes(changes.slice(*safe_keys)), # DEPRECATED - repository: issuable.project.hook_attrs.slice(:name, :url, :description, :homepage) + repository: issuable.project&.hook_attrs&.slice(:name, :url, :description, :homepage) } hook_data[:assignees] = issuable.assignees.map(&:hook_attrs) if issuable.assignees.any? diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index fd83f27ef31..89065c11c4f 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -328,14 +328,19 @@ module Gitlab db_config&.name || 'unknown' end - # Currently the database configuration can only be shared with `main:` - # If the `database_tasks: false` is being used - # This is to be refined: https://gitlab.com/gitlab-org/gitlab/-/issues/356580 + # If the `database_tasks: false` is being used, + # return the expected fallback database for this database configuration def self.db_config_share_with(db_config) - if db_config.database_tasks? - nil # no sharing + # no sharing + return if db_config.database_tasks? + + database_connection_info = all_database_connections[db_config.name] + + if database_connection_info + database_connection_info.fallback_database&.to_s else - 'main' # share with `main:` + # legacy behaviour + 'main' end end diff --git a/lib/gitlab/database/batch_counter.rb b/lib/gitlab/database/batch_counter.rb index abb62140503..7abcef5c93b 100644 --- a/lib/gitlab/database/batch_counter.rb +++ b/lib/gitlab/database/batch_counter.rb @@ -16,17 +16,18 @@ module Gitlab DEFAULT_DISTINCT_BATCH_SIZE = 10_000 DEFAULT_BATCH_SIZE = 100_000 - def initialize(relation, column: nil, operation: :count, operation_args: nil) + def initialize(relation, column: nil, operation: :count, operation_args: nil, max_allowed_loops: nil) @relation = relation @column = column || relation.primary_key @operation = operation @operation_args = operation_args + @max_allowed_loops = max_allowed_loops || MAX_ALLOWED_LOOPS end def unwanted_configuration?(finish, batch_size, start) (@operation == :count && batch_size <= MIN_REQUIRED_BATCH_SIZE) || (@operation == :sum && batch_size < DEFAULT_SUM_BATCH_SIZE) || - (finish - start) / batch_size >= MAX_ALLOWED_LOOPS || + (finish - start) / batch_size >= @max_allowed_loops || start >= finish end @@ -139,6 +140,7 @@ module Gitlab relation: @relation.table_name, operation: @operation, operation_args: @operation_args, + max_allowed_loops: @max_allowed_loops, start: batch_start, mode: mode, query: query, diff --git a/lib/gitlab/database/bump_sequences.rb b/lib/gitlab/database/bump_sequences.rb new file mode 100644 index 00000000000..f1b725b2de9 --- /dev/null +++ b/lib/gitlab/database/bump_sequences.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class BumpSequences + SEQUENCE_NAME_MATCHER = /nextval\('([a-z_]+)'::regclass\)/ + + # gitlab_schema: can be 'gitlab_main', 'gitlab_ci', 'gitlab_main_cell', 'gitlab_shared' + # increase_by: positive number, to increase the sequence by + # base_model: is to choose which connection to use to query the tables + def initialize(gitlab_schema, increase_by, base_model = ApplicationRecord) + @base_model = base_model + @gitlab_schema = gitlab_schema + @increase_by = increase_by + end + + def execute + sequences_by_gitlab_schema(base_model, gitlab_schema).each do |sequence_name| + increment_sequence_by(base_model.connection, sequence_name, increase_by) + end + end + + private + + attr_reader :base_model, :gitlab_schema, :increase_by + + def sequences_by_gitlab_schema(base_model, gitlab_schema) + tables = Gitlab::Database::GitlabSchema.tables_to_schema.select do |_table_name, schema_name| + schema_name == gitlab_schema + end.keys + + sequences = [] + + tables.each do |table| + model = Class.new(base_model) do + self.table_name = table + end + + model.columns.each do |column| + match_result = column.default_function&.match(SEQUENCE_NAME_MATCHER) + next unless match_result + + sequences << match_result[1] + end + end + + sequences + end + + # This method is going to increase the sequence next_value by: + # - increment_by + 1 if the sequence has the attribute is_called = True (which is the common case) + # - increment_by if the sequence has the attribute is_called = False (for example, a newly created sequence) + # It uses ALTER SEQUENCE as a safety mechanism to avoid that no concurrent insertions + # will cause conflicts on the sequence. + # This is because ALTER SEQUENCE blocks concurrent nextval, currval, lastval, and setval calls. + def increment_sequence_by(connection, sequence_name, increment_by) + connection.transaction do + # The first call is to make sure that the sequence's is_called value is set to `true` + # This guarantees that the next call to `nextval` will increase the sequence by `increment_by` + connection.select_value("SELECT nextval($1)", nil, [sequence_name]) + connection.execute("ALTER SEQUENCE #{sequence_name} INCREMENT BY #{increment_by}") + connection.select_value("select nextval($1)", nil, [sequence_name]) + connection.execute("ALTER SEQUENCE #{sequence_name} INCREMENT BY 1") + end + end + end + end +end diff --git a/lib/gitlab/database/ci_builds_partitioning.rb b/lib/gitlab/database/ci_builds_partitioning.rb deleted file mode 100644 index 9f8b19f2d23..00000000000 --- a/lib/gitlab/database/ci_builds_partitioning.rb +++ /dev/null @@ -1,224 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - class CiBuildsPartitioning - include AsyncDdlExclusiveLeaseGuard - - ATTEMPTS = 5 - LOCK_TIMEOUT = 10.seconds - LEASE_TIMEOUT = 30.minutes - - FK_NAME = :fk_e20479742e_p - TEMP_FK_NAME = :temp_fk_e20479742e_p - NEXT_PARTITION_ID = 101 - BUILDS_PARTITION_NAME = 'gitlab_partitions_dynamic.ci_builds_101' - ANNOTATION_PARTITION_NAME = 'gitlab_partitions_dynamic.ci_job_annotations_101' - RUNNER_MACHINE_PARTITION_NAME = 'gitlab_partitions_dynamic.ci_runner_machine_builds_101' - - def initialize(logger: Gitlab::AppLogger) - @connection = ::Ci::ApplicationRecord.connection - @timing_configuration = Array.new(ATTEMPTS) { [LOCK_TIMEOUT, 3.minutes] } - @logger = logger - end - - def execute - return unless can_execute? - - try_obtain_lease do - swap_foreign_keys - create_new_ci_builds_partition - create_new_job_annotations_partition - create_new_runner_machine_partition - end - - rescue StandardError => e - log_info("Failed to execute: #{e.message}") - end - - private - - attr_reader :connection, :timing_configuration, :logger - - delegate :quote_table_name, :quote_column_name, to: :connection - - def swap_foreign_keys - if new_foreign_key_exists? - log_info('Foreign key already renamed, nothing to do') - - return - end - - with_lock_retries do - connection.execute drop_old_foreign_key_sql - - rename_constraint :p_ci_builds_metadata, TEMP_FK_NAME, FK_NAME - - each_partition do |partition| - rename_constraint partition.identifier, TEMP_FK_NAME, FK_NAME - end - end - - log_info('Foreign key successfully renamed') - end - - def create_new_ci_builds_partition - if connection.table_exists?(BUILDS_PARTITION_NAME) - log_info('p_ci_builds partition exists, nothing to do') - return - end - - with_lock_retries do - connection.execute new_ci_builds_partition_sql - end - - log_info('Partition for p_ci_builds successfully created') - end - - def create_new_job_annotations_partition - if connection.table_exists?(ANNOTATION_PARTITION_NAME) - log_info('p_ci_job_annotations partition exists, nothing to do') - return - end - - with_lock_retries do - connection.execute new_job_annotations_partition_sql - end - - log_info('Partition for p_ci_job_annotations successfully created') - end - - def create_new_runner_machine_partition - if connection.table_exists?(RUNNER_MACHINE_PARTITION_NAME) - log_info('p_ci_runner_machine_builds partition exists, nothing to do') - return - end - - with_lock_retries do - connection.execute new_runner_machine_partition_sql - end - - log_info('Partition for p_ci_runner_machine_builds successfully created') - end - - def can_execute? - return false if process_disabled? - return false unless Gitlab.com? - - if vacuum_running? - log_info('Autovacuum detected') - - return false - end - - true - end - - def process_disabled? - ::Feature.disabled?(:complete_p_ci_builds_partitioning) - end - - def new_foreign_key_exists? - Gitlab::Database::SharedModel.using_connection(connection) do - Gitlab::Database::PostgresForeignKey - .by_constrained_table_name_or_identifier(:p_ci_builds_metadata) - .by_referenced_table_name(:p_ci_builds) - .by_name(FK_NAME) - .exists? - end - end - - def vacuum_running? - Gitlab::Database::SharedModel.using_connection(connection) do - Gitlab::Database::PostgresAutovacuumActivity - .wraparound_prevention - .for_tables(%i[ci_builds ci_builds_metadata]) - .any? - end - end - - def drop_old_foreign_key_sql - <<~SQL.squish - SET LOCAL statement_timeout TO '11s'; - - LOCK TABLE ci_builds, p_ci_builds_metadata IN ACCESS EXCLUSIVE MODE; - - ALTER TABLE p_ci_builds_metadata DROP CONSTRAINT #{FK_NAME}; - SQL - end - - def rename_constraint(table_name, old_name, new_name) - connection.execute <<~SQL - ALTER TABLE #{quote_table_name(table_name)} - RENAME CONSTRAINT #{quote_column_name(old_name)} TO #{quote_column_name(new_name)} - SQL - end - - def new_ci_builds_partition_sql - <<~SQL - SET LOCAL statement_timeout TO '11s'; - - LOCK ci_pipelines, ci_stages IN SHARE ROW EXCLUSIVE MODE; - LOCK TABLE ONLY p_ci_builds IN ACCESS EXCLUSIVE MODE; - - CREATE TABLE IF NOT EXISTS #{BUILDS_PARTITION_NAME} - PARTITION OF p_ci_builds - FOR VALUES IN (#{NEXT_PARTITION_ID}); - SQL - end - - def new_job_annotations_partition_sql - <<~SQL - SET LOCAL statement_timeout TO '11s'; - - LOCK TABLE p_ci_builds IN SHARE ROW EXCLUSIVE MODE; - LOCK TABLE ONLY p_ci_job_annotations IN ACCESS EXCLUSIVE MODE; - - CREATE TABLE IF NOT EXISTS #{ANNOTATION_PARTITION_NAME} - PARTITION OF p_ci_job_annotations - FOR VALUES IN (#{NEXT_PARTITION_ID}); - SQL - end - - def new_runner_machine_partition_sql - <<~SQL - SET LOCAL statement_timeout TO '11s'; - - LOCK TABLE p_ci_builds IN SHARE ROW EXCLUSIVE MODE; - LOCK TABLE ONLY p_ci_runner_machine_builds IN ACCESS EXCLUSIVE MODE; - - CREATE TABLE IF NOT EXISTS #{RUNNER_MACHINE_PARTITION_NAME} - PARTITION OF p_ci_runner_machine_builds - FOR VALUES IN (#{NEXT_PARTITION_ID}); - SQL - end - - def with_lock_retries(&block) - Gitlab::Database::WithLockRetries.new( - timing_configuration: timing_configuration, - connection: connection, - logger: logger, - klass: self.class - ).run(raise_on_exhaustion: true, &block) - end - - def each_partition(&block) - Gitlab::Database::SharedModel.using_connection(connection) do - Gitlab::Database::PostgresPartitionedTable.each_partition(:p_ci_builds_metadata, &block) - end - end - - def log_info(message) - logger.info(message: message, class: self.class.to_s) - end - - def connection_db_config - ::Ci::ApplicationRecord.connection_db_config - end - - def lease_timeout - LEASE_TIMEOUT - end - end - end -end diff --git a/lib/gitlab/database/health_status.rb b/lib/gitlab/database/health_status.rb index 69bb8a70afd..b8dba802b56 100644 --- a/lib/gitlab/database/health_status.rb +++ b/lib/gitlab/database/health_status.rb @@ -6,7 +6,8 @@ module Gitlab DEFAULT_INIDICATORS = [ Indicators::AutovacuumActiveOnTable, Indicators::WriteAheadLog, - Indicators::PatroniApdex + Indicators::PatroniApdex, + Indicators::WalRate ].freeze class << self diff --git a/lib/gitlab/database/health_status/indicators/patroni_apdex.rb b/lib/gitlab/database/health_status/indicators/patroni_apdex.rb index 680c86cf7b2..5631b6db6ff 100644 --- a/lib/gitlab/database/health_status/indicators/patroni_apdex.rb +++ b/lib/gitlab/database/health_status/indicators/patroni_apdex.rb @@ -4,82 +4,19 @@ module Gitlab module Database module HealthStatus module Indicators - class PatroniApdex - include Gitlab::Utils::StrongMemoize - - def initialize(context) - @gitlab_schema = context.gitlab_schema.to_sym - end - - def evaluate - return Signals::NotAvailable.new(self.class, reason: 'indicator disabled') unless enabled? - - connection_error_message = fetch_connection_error_message - return unknown_signal(connection_error_message) if connection_error_message.present? - - apdex_sli = fetch_sli(apdex_sli_query) - return unknown_signal('Patroni service apdex can not be calculated') unless apdex_sli.present? - - if apdex_sli.to_f > apdex_slo.to_f - Signals::Normal.new(self.class, reason: 'Patroni service apdex is above SLO') - else - Signals::Stop.new(self.class, reason: 'Patroni service apdex is below SLO') - end - end - + class PatroniApdex < PrometheusAlertIndicator private - attr_reader :gitlab_schema - def enabled? Feature.enabled?(:batched_migrations_health_status_patroni_apdex, type: :ops) end - def unknown_signal(reason) - Signals::Unknown.new(self.class, reason: reason) - end - - def fetch_connection_error_message - return 'Patroni Apdex Settings not configured' unless database_apdex_settings.present? - return 'Prometheus client is not ready' unless client.ready? - return 'Apdex SLI query is not configured' unless apdex_sli_query - return 'Apdex SLO is not configured' unless apdex_slo - end - - def client - @client ||= Gitlab::PrometheusClient.new( - database_apdex_settings[:prometheus_api_url], - allow_local_requests: true, - verify: true - ) - end - - def database_apdex_settings - @database_apdex_settings ||= Gitlab::CurrentSettings.database_apdex_settings&.with_indifferent_access + def sli_query_key + :apdex_sli_query end - def apdex_sli_query - { - gitlab_main: database_apdex_settings[:apdex_sli_query][:main], - gitlab_ci: database_apdex_settings[:apdex_sli_query][:ci] - }.fetch(gitlab_schema) - end - strong_memoize_attr :apdex_sli_query - - def apdex_slo - { - gitlab_main: database_apdex_settings[:apdex_slo][:main], - gitlab_ci: database_apdex_settings[:apdex_slo][:ci] - }.fetch(gitlab_schema) - end - strong_memoize_attr :apdex_slo - - def fetch_sli(query) - response = client.query(query) - metric = response&.first || {} - value = metric.fetch('value', []) - - Array.wrap(value).second + def slo_key + :apdex_slo end end end diff --git a/lib/gitlab/database/health_status/indicators/prometheus_alert_indicator.rb b/lib/gitlab/database/health_status/indicators/prometheus_alert_indicator.rb new file mode 100644 index 00000000000..3d630d21d4c --- /dev/null +++ b/lib/gitlab/database/health_status/indicators/prometheus_alert_indicator.rb @@ -0,0 +1,113 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module HealthStatus + module Indicators + class PrometheusAlertIndicator + include Gitlab::Utils::StrongMemoize + + ALERT_CONDITIONS = { + above: :above, + below: :below + }.freeze + + def initialize(context) + @gitlab_schema = context.gitlab_schema.to_sym + end + + def evaluate + return Signals::NotAvailable.new(self.class, reason: 'indicator disabled') unless enabled? + + connection_error_message = fetch_connection_error_message + return unknown_signal(connection_error_message) if connection_error_message.present? + + sli = fetch_sli(sli_query) + return unknown_signal("#{indicator_name} can not be calculated") unless sli.present? + + if alert_condition == ALERT_CONDITIONS[:above] ? sli.to_f > slo.to_f : sli.to_f < slo.to_f + Signals::Normal.new(self.class, reason: "#{indicator_name} SLI condition met") + else + Signals::Stop.new(self.class, reason: "#{indicator_name} SLI condition not met") + end + end + + private + + attr_reader :gitlab_schema + + def indicator_name + self.class.name.demodulize + end + + # By default SLIs are expected to be above SLOs, but there can be cases + # where we want it to be below SLO (eg: WAL rate). For such indicators + # the sub-class should override this default alert_condition. + def alert_condition + ALERT_CONDITIONS[:above] + end + + def enabled? + raise NotImplementedError, "prometheus alert based indicators must implement #{__method__}" + end + + def slo_key + raise NotImplementedError, "prometheus alert based indicators must implement #{__method__}" + end + + def sli_key + raise NotImplementedError, "prometheus alert based indicators must implement #{__method__}" + end + + def fetch_connection_error_message + return 'Prometheus Settings not configured' unless prometheus_alert_db_indicators_settings.present? + return 'Prometheus client is not ready' unless client.ready? + return "#{indicator_name} SLI query is not configured" unless sli_query + return "#{indicator_name} SLO is not configured" unless slo + end + + def prometheus_alert_db_indicators_settings + @prometheus_alert_db_indicators_settings ||= Gitlab::CurrentSettings + .prometheus_alert_db_indicators_settings&.with_indifferent_access + end + + def client + @client ||= Gitlab::PrometheusClient.new( + prometheus_alert_db_indicators_settings[:prometheus_api_url], + allow_local_requests: true, + verify: true + ) + end + + def sli_query + { + gitlab_main: prometheus_alert_db_indicators_settings[sli_query_key][:main], + gitlab_ci: prometheus_alert_db_indicators_settings[sli_query_key][:ci] + }.fetch(gitlab_schema) + end + strong_memoize_attr :sli_query + + def slo + { + gitlab_main: prometheus_alert_db_indicators_settings[slo_key][:main], + gitlab_ci: prometheus_alert_db_indicators_settings[slo_key][:ci] + }.fetch(gitlab_schema) + end + strong_memoize_attr :slo + + def fetch_sli(query) + response = client.query(query) + metric = response&.first || {} + value = metric.fetch('value', []) + + Array.wrap(value).second + end + + def unknown_signal(reason) + Signals::Unknown.new(self.class, reason: reason) + end + end + end + end + end +end diff --git a/lib/gitlab/database/health_status/indicators/wal_rate.rb b/lib/gitlab/database/health_status/indicators/wal_rate.rb new file mode 100644 index 00000000000..de31b5899eb --- /dev/null +++ b/lib/gitlab/database/health_status/indicators/wal_rate.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module HealthStatus + module Indicators + class WalRate < PrometheusAlertIndicator + private + + def enabled? + Feature.enabled?(:db_health_check_wal_rate, type: :ops) + end + + def sli_query_key + :wal_rate_sli_query + end + + def slo_key + :wal_rate_slo + end + + def alert_condition + ALERT_CONDITIONS[:below] + end + end + end + end + end +end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 256c524e989..60cec12b4b5 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -1191,6 +1191,19 @@ into similar problems in the future (e.g. when new tables are created). .present? end + # While it is safe to call `change_column_default` on a column without + # default it would still require access exclusive lock on the table + # and for tables with high autovacuum(wraparound prevention) it will + # fail if their executions overlap. + # + def remove_column_default(table_name, column_name) + column = connection.columns(table_name).find { |col| col.name == column_name.to_s } + + if column.default || column.default_function + change_column_default(table_name, column_name, to: nil) + end + end + private def multiple_columns(columns, separator: ', ') diff --git a/lib/gitlab/database/migration_helpers/convert_to_bigint.rb b/lib/gitlab/database/migration_helpers/convert_to_bigint.rb index 63928d7dc09..11f1e62e8b9 100644 --- a/lib/gitlab/database/migration_helpers/convert_to_bigint.rb +++ b/lib/gitlab/database/migration_helpers/convert_to_bigint.rb @@ -15,6 +15,20 @@ module Gitlab Gitlab.com? && !Gitlab.jh? end + + def temp_column_removed?(table_name, column_name) + !column_exists?(table_name.to_s, convert_to_bigint_column(column_name)) + end + + def columns_swapped?(table_name, column_name) + table_columns = columns(table_name.to_s) + temp_column_name = convert_to_bigint_column(column_name) + + column = table_columns.find { |c| c.name == column_name.to_s } + temp_column = table_columns.find { |c| c.name == temp_column_name } + + column.sql_type == 'bigint' && temp_column.sql_type == 'integer' + end end end end diff --git a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb index cb2a98b553f..efb1957d5e7 100644 --- a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb @@ -242,7 +242,7 @@ module Gitlab "\n\n" \ "For more information, check the documentation" \ "\n\n" \ - "\thttps://docs.gitlab.com/ee/user/admin_area/monitoring/background_migrations.html#database-migrations-failing-because-of-batched-background-migration-not-finished" + "\thttps://docs.gitlab.com/ee/update/background_migrations.html#database-migrations-failing-because-of-batched-background-migration-not-finished" end end end diff --git a/lib/gitlab/database/migrations/squasher.rb b/lib/gitlab/database/migrations/squasher.rb new file mode 100644 index 00000000000..98fdf873aa5 --- /dev/null +++ b/lib/gitlab/database/migrations/squasher.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +require 'set' + +module Gitlab + module Database + module Migrations + class Squasher + RSPEC_FILENAME_REGEXP = /\A([0-9]+_)?([_a-z0-9]*)\.rb\z/ + + def initialize(git_output) + @migration_data = migration_files_from_git(git_output).filter_map do |mf| + basename = Pathname(mf).basename.to_s + file_name_match = ActiveRecord::Migration::MigrationFilenameRegexp.match(basename) + slug = file_name_match[2] + unless slug == 'init_schema' + { + path: mf, + basename: basename, + timestamp: file_name_match[1], + slug: slug + } + end + end + end + + def files_to_delete + @migration_data.pluck(:path) + schema_migrations + find_migration_specs + end + + private + + def schema_migrations + @migration_data.map { |m| "db/schema_migrations/#{m[:timestamp]}" } + end + + def find_migration_specs + @file_slugs = Set.new @migration_data.pluck(:slug) + (migration_specs + ee_migration_specs).select { |f| file_has_slug?(f) } + end + + def migration_files_from_git(body) + body.chomp + .split("\n") + .select { |fn| fn.end_with?('.rb') } + end + + def match_file_slug(filename) + m = RSPEC_FILENAME_REGEXP.match(filename) + return if m.nil? + + m[2].sub(/_spec$/, '') + end + + def file_has_slug?(filename) + spec_slug = match_file_slug(Pathname(filename).basename.to_s) + return false if spec_slug.nil? + + @file_slugs.include?(spec_slug) + end + + def migration_specs + Dir.glob(Rails.root.join('spec/migrations/*.rb')) + end + + def ee_migration_specs + Dir.glob(Rails.root.join('ee/spec/migrations/*.rb')) + end + end + end + end +end diff --git a/lib/gitlab/database/postgres_constraint.rb b/lib/gitlab/database/postgres_constraint.rb index fa3870cb9c7..9c49251664b 100644 --- a/lib/gitlab/database/postgres_constraint.rb +++ b/lib/gitlab/database/postgres_constraint.rb @@ -17,7 +17,7 @@ module Gitlab scope :valid, -> { where(constraint_valid: true) } scope :by_table_identifier, ->(identifier) do - unless identifier =~ Gitlab::Database::FULLY_QUALIFIED_IDENTIFIER + unless Gitlab::Database::FULLY_QUALIFIED_IDENTIFIER.match?(identifier) raise ArgumentError, "Table name is not fully qualified with a schema: #{identifier}" end diff --git a/lib/gitlab/database/postgres_foreign_key.rb b/lib/gitlab/database/postgres_foreign_key.rb index bb3e1d45f15..9fb3098efe0 100644 --- a/lib/gitlab/database/postgres_foreign_key.rb +++ b/lib/gitlab/database/postgres_foreign_key.rb @@ -21,7 +21,7 @@ module Gitlab enum on_update_action: ACTION_TYPES, _prefix: :on_update scope :by_referenced_table_identifier, ->(identifier) do - unless identifier =~ Database::FULLY_QUALIFIED_IDENTIFIER + unless Database::FULLY_QUALIFIED_IDENTIFIER.match?(identifier) raise ArgumentError, "Referenced table name is not fully qualified with a schema: #{identifier}" end @@ -31,7 +31,7 @@ module Gitlab scope :by_referenced_table_name, ->(name) { where(referenced_table_name: name) } scope :by_constrained_table_identifier, ->(identifier) do - unless identifier =~ Database::FULLY_QUALIFIED_IDENTIFIER + unless Database::FULLY_QUALIFIED_IDENTIFIER.match?(identifier) raise ArgumentError, "Constrained table name is not fully qualified with a schema: #{identifier}" end @@ -41,7 +41,7 @@ module Gitlab scope :by_constrained_table_name, ->(name) { where(constrained_table_name: name) } scope :by_constrained_table_name_or_identifier, ->(name) do - if name =~ Database::FULLY_QUALIFIED_IDENTIFIER + if Database::FULLY_QUALIFIED_IDENTIFIER.match?(name) by_constrained_table_identifier(name) else by_constrained_table_name(name) diff --git a/lib/gitlab/database/postgres_index.rb b/lib/gitlab/database/postgres_index.rb index 50009cadf5d..1c775482e7e 100644 --- a/lib/gitlab/database/postgres_index.rb +++ b/lib/gitlab/database/postgres_index.rb @@ -14,7 +14,7 @@ module Gitlab has_many :queued_reindexing_actions, class_name: 'Gitlab::Database::Reindexing::QueuedAction', foreign_key: :index_identifier scope :by_identifier, ->(identifier) do - unless identifier =~ Gitlab::Database::FULLY_QUALIFIED_IDENTIFIER + unless Gitlab::Database::FULLY_QUALIFIED_IDENTIFIER.match?(identifier) raise ArgumentError, "Index name is not fully qualified with a schema: #{identifier}" end diff --git a/lib/gitlab/database/postgres_partition.rb b/lib/gitlab/database/postgres_partition.rb index e63c6fc86ea..f79b8b5e32c 100644 --- a/lib/gitlab/database/postgres_partition.rb +++ b/lib/gitlab/database/postgres_partition.rb @@ -10,7 +10,7 @@ module Gitlab # identifier includes the partition schema. # For example 'gitlab_partitions_static.events_03', or 'gitlab_partitions_dynamic.logs_03' scope :for_identifier, ->(identifier) do - unless identifier =~ Gitlab::Database::FULLY_QUALIFIED_IDENTIFIER + unless Gitlab::Database::FULLY_QUALIFIED_IDENTIFIER.match?(identifier) raise ArgumentError, "Partition name is not fully qualified with a schema: #{identifier}" end @@ -22,7 +22,7 @@ module Gitlab end scope :for_parent_table, ->(parent_table) do - if parent_table =~ Database::FULLY_QUALIFIED_IDENTIFIER + if Database::FULLY_QUALIFIED_IDENTIFIER.match?(parent_table) where(parent_identifier: parent_table).order(:name) else where("parent_identifier = concat(current_schema(), '.', ?)", parent_table).order(:name) diff --git a/lib/gitlab/database/postgres_partitioned_table.rb b/lib/gitlab/database/postgres_partitioned_table.rb index fead7379e43..76e2cd48f80 100644 --- a/lib/gitlab/database/postgres_partitioned_table.rb +++ b/lib/gitlab/database/postgres_partitioned_table.rb @@ -10,7 +10,7 @@ module Gitlab has_many :postgres_partitions, foreign_key: 'parent_identifier', primary_key: 'identifier' scope :by_identifier, ->(identifier) do - unless identifier =~ Gitlab::Database::FULLY_QUALIFIED_IDENTIFIER + unless Gitlab::Database::FULLY_QUALIFIED_IDENTIFIER.match?(identifier) raise ArgumentError, "Table name is not fully qualified with a schema: #{identifier}" end diff --git a/lib/gitlab/database/postgresql_adapter/force_disconnectable_mixin.rb b/lib/gitlab/database/postgresql_adapter/force_disconnectable_mixin.rb index 71d2554844e..21392283ccf 100644 --- a/lib/gitlab/database/postgresql_adapter/force_disconnectable_mixin.rb +++ b/lib/gitlab/database/postgresql_adapter/force_disconnectable_mixin.rb @@ -11,6 +11,8 @@ module Gitlab end def force_disconnect_if_old! + return if Rails.env.test? && transaction_open? + if force_disconnect_timer.expired? disconnect! reset_force_disconnect_timer! diff --git a/lib/gitlab/database/query_analyzers/query_recorder.rb b/lib/gitlab/database/query_analyzers/query_recorder.rb deleted file mode 100644 index 63b4fbb8c1d..00000000000 --- a/lib/gitlab/database/query_analyzers/query_recorder.rb +++ /dev/null @@ -1,53 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module QueryAnalyzers - class QueryRecorder < Base - LOG_PATH = 'query_recorder/' - LIST_PARAMETER_REGEX = %r{\$\d+(?:\s*,\s*\$\d+)+}.freeze - SINGLE_PARAMETER_REGEX = %r{\$\d+}.freeze - - class << self - def enabled? - # Only enable QueryRecorder in CI on database MRs or default branch - ENV['CI_MERGE_REQUEST_LABELS']&.include?('database') || - (ENV['CI_COMMIT_REF_NAME'].present? && ENV['CI_COMMIT_REF_NAME'] == ENV['CI_DEFAULT_BRANCH']) - end - - def analyze(parsed) - payload = { - normalized: normalize_query(parsed.sql) - } - - log_query(payload) - end - - def log_file - Rails.root.join(LOG_PATH, "#{ENV.fetch('CI_JOB_NAME_SLUG', 'rspec')}.ndjson") - end - - private - - def log_query(payload) - log_dir = Rails.root.join(LOG_PATH) - - # Create log directory if it does not exist since it is only created - # ahead of time by certain CI jobs - FileUtils.mkdir_p(log_dir) unless Dir.exist?(log_dir) - - log_line = "#{Gitlab::Json.dump(payload)}\n" - - File.write(log_file, log_line, mode: 'a') - end - - def normalize_query(query) - query - .gsub(LIST_PARAMETER_REGEX, '?,?,?') # Replace list parameters with ?,?,? - .gsub(SINGLE_PARAMETER_REGEX, '?') # Replace single parameters with ? - end - end - end - end - end -end diff --git a/lib/gitlab/database/reindexing.rb b/lib/gitlab/database/reindexing.rb index 9c860ebc6aa..4a1b0be848e 100644 --- a/lib/gitlab/database/reindexing.rb +++ b/lib/gitlab/database/reindexing.rb @@ -59,7 +59,6 @@ module Gitlab # most bloated indexes for reindexing. def self.perform_with_heuristic(candidate_indexes = Gitlab::Database::PostgresIndex.reindexing_support, maximum_records: DEFAULT_INDEXES_PER_INVOCATION) IndexSelection.new(candidate_indexes).take(maximum_records).each do |index| - Gitlab::Database::CiBuildsPartitioning.new.execute Coordinator.new(index).perform end end diff --git a/lib/gitlab/database/reindexing/reindex_concurrently.rb b/lib/gitlab/database/reindexing/reindex_concurrently.rb index 60fa4deda39..aa445082aa9 100644 --- a/lib/gitlab/database/reindexing/reindex_concurrently.rb +++ b/lib/gitlab/database/reindexing/reindex_concurrently.rb @@ -20,7 +20,7 @@ module Gitlab def perform raise ReindexError, 'indexes serving an exclusion constraint are currently not supported' if index.exclusion? - raise ReindexError, 'index is a left-over temporary index from a previous reindexing run' if index.name =~ /#{TEMPORARY_INDEX_PATTERN}/o + raise ReindexError, 'index is a left-over temporary index from a previous reindexing run' if /#{TEMPORARY_INDEX_PATTERN}/o.match?(index.name) # Expression indexes require additional statistics in `pg_statistic`: # select * from pg_statistic where starelid = (select oid from pg_class where relname = 'some_index'); diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb index 562e651cabc..72ae2849911 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb @@ -23,6 +23,7 @@ module Gitlab with_paths = MigrationClasses::Route.arel_table[:path] .matches_any(path_patterns) namespaces.joins(:route).where(with_paths) + .allow_cross_joins_across_databases(url: "https://gitlab.com/gitlab-org/gitlab/-/issues/420046") end def rename_namespace(namespace) @@ -45,6 +46,7 @@ module Gitlab reverts_for_type('namespace') do |path_before_rename, current_path| matches_path = MigrationClasses::Route.arel_table[:path].matches(current_path) namespace = MigrationClasses::Namespace.joins(:route) + .allow_cross_joins_across_databases(url: "https://gitlab.com/gitlab-org/gitlab/-/issues/420046") .find_by(matches_path)&.becomes(MigrationClasses::Namespace) # rubocop: disable Cop/AvoidBecomes if namespace diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb index 5dbf30bad4e..155e35b64f4 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb @@ -37,6 +37,8 @@ module Gitlab reverts_for_type('project') do |path_before_rename, current_path| matches_path = MigrationClasses::Route.arel_table[:path].matches(current_path) project = MigrationClasses::Project.joins(:route) + .allow_cross_joins_across_databases(url: + 'https://gitlab.com/gitlab-org/gitlab/-/issues/421843') .find_by(matches_path) if project @@ -67,6 +69,7 @@ module Gitlab .matches_any(path_patterns) @projects_for_paths = MigrationClasses::Project.joins(:route).where(with_paths) + .allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/421843') end end end diff --git a/lib/gitlab/database/schema_validation/schema_inconsistency.rb b/lib/gitlab/database/schema_validation/schema_inconsistency.rb deleted file mode 100644 index 9f39db5b4c0..00000000000 --- a/lib/gitlab/database/schema_validation/schema_inconsistency.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - class SchemaInconsistency < ApplicationRecord - self.table_name = :schema_inconsistencies - - belongs_to :issue - - validates :object_name, :valitador_name, :table_name, :diff, presence: true - - scope :with_open_issues, -> { joins(:issue).where('issue.state_id': Issue.available_states[:opened]) } - end - end - end -end diff --git a/lib/gitlab/database/tables_sorted_by_foreign_keys.rb b/lib/gitlab/database/tables_sorted_by_foreign_keys.rb index b2a7f5442e9..9593cb45947 100644 --- a/lib/gitlab/database/tables_sorted_by_foreign_keys.rb +++ b/lib/gitlab/database/tables_sorted_by_foreign_keys.rb @@ -34,7 +34,7 @@ module Gitlab def all_foreign_keys @all_foreign_keys ||= @tables.each_with_object(Hash.new { |h, k| h[k] = [] }) do |table, hash| foreign_keys_for(table).each do |fk| - hash[fk.to_table] << table + hash[fk.referenced_table_name] << table end end end @@ -45,12 +45,10 @@ module Gitlab # # See spec/lib/gitlab/database/tables_sorted_by_foreign_keys_spec.rb # for an example - name = ActiveRecord::ConnectionAdapters::PostgreSQL::Utils.extract_schema_qualified_name(table) + table = ActiveRecord::ConnectionAdapters::PostgreSQL::Utils.extract_schema_qualified_name(table) - if name.schema == ::Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA.to_s - @connection.foreign_keys(name.identifier) - else - @connection.foreign_keys(table) + Gitlab::Database::SharedModel.using_connection(@connection) do + Gitlab::Database::PostgresForeignKey.by_constrained_table_name_or_identifier(table.identifier).load end end end diff --git a/lib/gitlab/database_importers/work_items/base_type_importer.rb b/lib/gitlab/database_importers/work_items/base_type_importer.rb index 8f8f44e8392..990fd53a370 100644 --- a/lib/gitlab/database_importers/work_items/base_type_importer.rb +++ b/lib/gitlab/database_importers/work_items/base_type_importer.rb @@ -21,7 +21,8 @@ module Gitlab test_reports: 'Test reports', notifications: 'Notifications', current_user_todos: "Current user todos", - award_emoji: 'Award emoji' + award_emoji: 'Award emoji', + linked_items: 'Linked items' }.freeze WIDGETS_FOR_TYPE = { @@ -38,7 +39,8 @@ module Gitlab :health_status, :notifications, :current_user_todos, - :award_emoji + :award_emoji, + :linked_items ], incident: [ :assignees, @@ -47,14 +49,16 @@ module Gitlab :notes, :notifications, :current_user_todos, - :award_emoji + :award_emoji, + :linked_items ], test_case: [ :description, :notes, :notifications, :current_user_todos, - :award_emoji + :award_emoji, + :linked_items ], requirement: [ :description, @@ -64,7 +68,8 @@ module Gitlab :test_reports, :notifications, :current_user_todos, - :award_emoji + :award_emoji, + :linked_items ], task: [ :assignees, @@ -78,7 +83,8 @@ module Gitlab :weight, :notifications, :current_user_todos, - :award_emoji + :award_emoji, + :linked_items ], objective: [ :assignees, @@ -91,7 +97,8 @@ module Gitlab :progress, :notifications, :current_user_todos, - :award_emoji + :award_emoji, + :linked_items ], key_result: [ :assignees, @@ -104,6 +111,35 @@ module Gitlab :progress, :notifications, :current_user_todos, + :award_emoji, + :linked_items + ], + epic: [ + :assignees, + :description, + :hierarchy, + :labels, + :notes, + :start_and_due_date, + :health_status, + :status, + :notifications, + :award_emoji, + :linked_items + ], + ticket: [ + :assignees, + :labels, + :description, + :hierarchy, + :start_and_due_date, + :milestone, + :notes, + :iteration, + :weight, + :health_status, + :notifications, + :current_user_todos, :award_emoji ] }.freeze diff --git a/lib/gitlab/database_importers/work_items/hierarchy_restrictions_importer.rb b/lib/gitlab/database_importers/work_items/hierarchy_restrictions_importer.rb index 1181c259a5c..4e7a4ec748b 100644 --- a/lib/gitlab/database_importers/work_items/hierarchy_restrictions_importer.rb +++ b/lib/gitlab/database_importers/work_items/hierarchy_restrictions_importer.rb @@ -10,12 +10,17 @@ module Gitlab issue = find_or_create_type(::WorkItems::Type::TYPE_NAMES[:issue]) task = find_or_create_type(::WorkItems::Type::TYPE_NAMES[:task]) incident = find_or_create_type(::WorkItems::Type::TYPE_NAMES[:incident]) + epic = find_or_create_type(::WorkItems::Type::TYPE_NAMES[:epic]) + ticket = find_or_create_type(::WorkItems::Type::TYPE_NAMES[:ticket]) restrictions = [ { parent_type_id: objective.id, child_type_id: objective.id, maximum_depth: 9 }, { parent_type_id: objective.id, child_type_id: key_result.id, maximum_depth: 1 }, { parent_type_id: issue.id, child_type_id: task.id, maximum_depth: 1 }, - { parent_type_id: incident.id, child_type_id: task.id, maximum_depth: 1 } + { parent_type_id: incident.id, child_type_id: task.id, maximum_depth: 1 }, + { parent_type_id: epic.id, child_type_id: epic.id, maximum_depth: 9 }, + { parent_type_id: epic.id, child_type_id: issue.id, maximum_depth: 1 }, + { parent_type_id: ticket.id, child_type_id: task.id, maximum_depth: 1 } ] ::WorkItems::HierarchyRestriction.upsert_all( diff --git a/lib/gitlab/dependency_linker/base_linker.rb b/lib/gitlab/dependency_linker/base_linker.rb index 203cee1fd5e..aba6e0f033a 100644 --- a/lib/gitlab/dependency_linker/base_linker.rb +++ b/lib/gitlab/dependency_linker/base_linker.rb @@ -31,7 +31,7 @@ module Gitlab end def external_url(name, external_ref) - return if external_ref =~ GIT_INVALID_URL_REGEX + return if GIT_INVALID_URL_REGEX.match?(external_ref) case external_ref when /\A#{URL_REGEX}\z/o diff --git a/lib/gitlab/dependency_linker/cargo_toml_linker.rb b/lib/gitlab/dependency_linker/cargo_toml_linker.rb index cba4319ce83..7d442429d61 100644 --- a/lib/gitlab/dependency_linker/cargo_toml_linker.rb +++ b/lib/gitlab/dependency_linker/cargo_toml_linker.rb @@ -33,8 +33,16 @@ module Gitlab def link_toml(key, value, type, &url_proc) if value.is_a? String link_regex(/^(?<name>#{key})\s*=\s*"#{value}"/, &url_proc) - else - link_regex(/^\[#{type}\.(?<name>#{key})]/, &url_proc) + elsif value.is_a? Hash + # Don't link when using a custom registry + return if value['registry'] + + # Don't link unless a crates.io version is provided + # See https://doc.rust-lang.org/cargo/reference/specifying-dependencies.html#multiple-locations + return unless value['version'] + + link_regex(/^(?<name>#{key})\s*=\s*\{/, &url_proc) + link_regex(/^\[#{type}\.(?<name>#{key})\]/, &url_proc) end end diff --git a/lib/gitlab/dependency_linker/composer_json_linker.rb b/lib/gitlab/dependency_linker/composer_json_linker.rb index 965ed8bb95e..762d7f3e73e 100644 --- a/lib/gitlab/dependency_linker/composer_json_linker.rb +++ b/lib/gitlab/dependency_linker/composer_json_linker.rb @@ -13,7 +13,7 @@ module Gitlab end def package_url(name) - "https://packagist.org/packages/#{name}" if name =~ /\A#{REPO_REGEX}\z/o + "https://packagist.org/packages/#{name}" if /\A#{REPO_REGEX}\z/o.match?(name) end end end diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb index a506bc3aaa2..ad901dc958b 100644 --- a/lib/gitlab/diff/parser.rb +++ b/lib/gitlab/diff/parser.rb @@ -25,7 +25,7 @@ module Gitlab full_line = line.delete("\n") - if line =~ /^@@ -/ + if /^@@ -/.match?(line) type = "match" diff_hunk = Gitlab::WordDiff::Segments::DiffHunk.new(line) diff --git a/lib/gitlab/discussions_diff/highlight_cache.rb b/lib/gitlab/discussions_diff/highlight_cache.rb index 31b214a4af9..18ff7c28e17 100644 --- a/lib/gitlab/discussions_diff/highlight_cache.rb +++ b/lib/gitlab/discussions_diff/highlight_cache.rb @@ -42,10 +42,8 @@ module Gitlab with_redis do |redis| Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do if Gitlab::Redis::ClusterUtil.cluster?(redis) - redis.with_readonly_pipeline do - Gitlab::Redis::CrossSlot::Pipeline.new(redis).pipelined do |pipeline| - keys.each { |key| pipeline.get(key) } - end + Gitlab::Redis::CrossSlot::Pipeline.new(redis).pipelined do |pipeline| + keys.each { |key| pipeline.get(key) } end else redis.mget(keys) diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb index e7462b711f1..ecacd02996d 100644 --- a/lib/gitlab/email/reply_parser.rb +++ b/lib/gitlab/email/reply_parser.rb @@ -54,7 +54,7 @@ module Gitlab return "" unless decoded # Certain trigger phrases that means we didn't parse correctly - if decoded =~ %r{(Content\-Type\:|multipart/alternative|text/plain)} + if %r{(Content\-Type\:|multipart/alternative|text/plain)}.match?(decoded) return "" end diff --git a/lib/gitlab/etag_caching/router/graphql.rb b/lib/gitlab/etag_caching/router/graphql.rb index 92b21a0859d..7946e768e00 100644 --- a/lib/gitlab/etag_caching/router/graphql.rb +++ b/lib/gitlab/etag_caching/router/graphql.rb @@ -14,7 +14,7 @@ module Gitlab 'continuous_integration' ], [ - %r(\Apipelines/sha/\w{7,40}\z), + %r(\Apipelines/sha/\w{#{Gitlab::Git::Commit::MIN_SHA_LENGTH},#{Gitlab::Git::Commit::MAX_SHA_LENGTH}}\z)o, 'ci_editor', 'pipeline_composition' ], diff --git a/lib/gitlab/etag_caching/store.rb b/lib/gitlab/etag_caching/store.rb index bc97c88ce85..f36a7a0603c 100644 --- a/lib/gitlab/etag_caching/store.rb +++ b/lib/gitlab/etag_caching/store.rb @@ -9,15 +9,15 @@ module Gitlab SHARED_STATE_NAMESPACE = 'etag:' def get(key) - Gitlab::Redis::SharedState.with { |redis| redis.get(redis_shared_state_key(key)) } + with_redis { |redis| redis.get(redis_shared_state_key(key)) } end def touch(*keys, only_if_missing: false) etags = keys.map { generate_etag } Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - Gitlab::Redis::SharedState.with do |redis| - redis.pipelined do |pipeline| + with_redis do |redis| + Gitlab::Redis::CrossSlot::Pipeline.new(redis).pipelined do |pipeline| keys.each_with_index do |key, i| pipeline.set(redis_shared_state_key(key), etags[i], ex: EXPIRY_TIME, nx: only_if_missing) end @@ -30,6 +30,12 @@ module Gitlab private + def with_redis(&blk) + # We use multistore as n interweaving double-write will result in n-1 subsequent requests + # becoming a cache-miss, however, 2 interweaving .touch will lead to 1 cache miss anyway. + Gitlab::Redis::EtagCache.with(&blk) # rubocop:disable CodeReuse/ActiveRecord + end + def generate_etag SecureRandom.hex end diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index 0b18a337707..8679f17eb9b 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -12,6 +12,8 @@ module Gitlab # ExclusiveLease. # class ExclusiveLease + include Gitlab::Utils::StrongMemoize + PREFIX = 'gitlab:exclusive_lease' NoKey = Class.new(ArgumentError) @@ -31,7 +33,7 @@ module Gitlab EOS def self.get_uuid(key) - Gitlab::Redis::SharedState.with do |redis| + with_read_redis do |redis| redis.get(redis_shared_state_key(key)) || false end end @@ -61,7 +63,7 @@ module Gitlab def self.cancel(key, uuid) return unless key.present? - Gitlab::Redis::SharedState.with do |redis| + with_write_redis do |redis| redis.eval(LUA_CANCEL_SCRIPT, keys: [ensure_prefixed_key(key)], argv: [uuid]) end end @@ -84,6 +86,21 @@ module Gitlab redis.del(key) end end + + Gitlab::Redis::ClusterSharedState.with do |redis| + redis.scan_each(match: redis_shared_state_key(scope)).each do |key| + redis.del(key) + end + end + end + + def self.use_cluster_shared_state? + Gitlab::SafeRequestStore[:use_cluster_shared_state] ||= + Feature.enabled?(:use_cluster_shared_state_for_exclusive_lease) + end + + def self.use_double_lock? + Gitlab::SafeRequestStore[:use_double_lock] ||= Feature.enabled?(:enable_exclusive_lease_double_lock_rw) end def initialize(key, uuid: nil, timeout:) @@ -95,10 +112,23 @@ module Gitlab # Try to obtain the lease. Return lease UUID on success, # false if the lease is already taken. def try_obtain + return try_obtain_with_new_lock if self.class.use_cluster_shared_state? + # Performing a single SET is atomic - Gitlab::Redis::SharedState.with do |redis| - redis.set(@redis_shared_state_key, @uuid, nx: true, ex: @timeout) && @uuid - end + obtained = set_lease(Gitlab::Redis::SharedState) && @uuid + + # traffic to new store is minimal since only the first lock holder can run SETNX in ClusterSharedState + return false unless obtained + return obtained unless self.class.use_double_lock? + return obtained if same_store # 2nd setnx will surely fail if store are the same + + second_lock_obtained = set_lease(Gitlab::Redis::ClusterSharedState) && @uuid + + # cancel is safe since it deletes key only if value matches uuid + # i.e. it will not delete the held lock on ClusterSharedState + cancel unless second_lock_obtained + + second_lock_obtained end # This lease is waiting to obtain @@ -109,7 +139,7 @@ module Gitlab # Try to renew an existing lease. Return lease UUID on success, # false if the lease is taken by a different UUID or inexistent. def renew - Gitlab::Redis::SharedState.with do |redis| + self.class.with_write_redis do |redis| result = redis.eval(LUA_RENEW_SCRIPT, keys: [@redis_shared_state_key], argv: [@uuid, @timeout]) result == @uuid end @@ -117,7 +147,7 @@ module Gitlab # Returns true if the key for this lease is set. def exists? - Gitlab::Redis::SharedState.with do |redis| + self.class.with_read_redis do |redis| redis.exists?(@redis_shared_state_key) # rubocop:disable CodeReuse/ActiveRecord end end @@ -126,17 +156,66 @@ module Gitlab # # This method will return `nil` if no TTL could be obtained. def ttl - Gitlab::Redis::SharedState.with do |redis| + self.class.with_read_redis do |redis| ttl = redis.ttl(@redis_shared_state_key) ttl if ttl > 0 end end + # rubocop:disable CodeReuse/ActiveRecord + def self.with_write_redis(&blk) + if use_cluster_shared_state? + result = Gitlab::Redis::ClusterSharedState.with(&blk) + Gitlab::Redis::SharedState.with(&blk) + + result + elsif use_double_lock? + result = Gitlab::Redis::SharedState.with(&blk) + Gitlab::Redis::ClusterSharedState.with(&blk) + + result + else + Gitlab::Redis::SharedState.with(&blk) + end + end + + def self.with_read_redis(&blk) + if use_cluster_shared_state? + Gitlab::Redis::ClusterSharedState.with(&blk) + elsif use_double_lock? + Gitlab::Redis::SharedState.with(&blk) || Gitlab::Redis::ClusterSharedState.with(&blk) + else + Gitlab::Redis::SharedState.with(&blk) + end + end + # rubocop:enable CodeReuse/ActiveRecord + # Gives up this lease, allowing it to be obtained by others. def cancel self.class.cancel(@redis_shared_state_key, @uuid) end + + private + + def set_lease(redis_class) + redis_class.with do |redis| + redis.set(@redis_shared_state_key, @uuid, nx: true, ex: @timeout) + end + end + + def try_obtain_with_new_lock + # checks shared-state to avoid 2 versions of the application acquiring 1 lock + # wait for held lock to expire or yielded in case any process on old version is running + return false if Gitlab::Redis::SharedState.with { |c| c.exists?(@redis_shared_state_key) } # rubocop:disable CodeReuse/ActiveRecord + + set_lease(Gitlab::Redis::ClusterSharedState) && @uuid + end + + def same_store + Gitlab::Redis::ClusterSharedState.with(&:id) == Gitlab::Redis::SharedState.with(&:id) # rubocop:disable CodeReuse/ActiveRecord + end + strong_memoize_attr :same_store end end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index ef5c242e68a..4e574d6de74 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -8,7 +8,7 @@ module Gitlab # https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012 EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904' BLANK_SHA = ('0' * 40).freeze - COMMIT_ID = /\A[0-9a-f]{40}\z/.freeze + COMMIT_ID = /\A#{Gitlab::Git::Commit::RAW_FULL_SHA_PATTERN}\z/.freeze TAG_REF_PREFIX = "refs/tags/" BRANCH_REF_PREFIX = "refs/heads/" diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb index 30977adaea1..21d2eaec041 100644 --- a/lib/gitlab/git/blame.rb +++ b/lib/gitlab/git/blame.rb @@ -18,7 +18,7 @@ module Gitlab def each @blames.each do |blame| - yield(blame.commit, blame.line, blame.previous_path) + yield(blame.commit, blame.line, blame.previous_path, blame.span) end end @@ -49,12 +49,12 @@ module Gitlab output.split("\n").each do |line| if line[0, 1] == "\t" lines << line[1, line.size] - elsif m = /^(\w{40}) (\d+) (\d+)/.match(line) + elsif m = /^(\w{40}) (\d+) (\d+)\s?(\d+)?/.match(line) # Removed these instantiations for performance but keeping them for reference: - # commit_id, old_lineno, lineno = m[1], m[2].to_i, m[3].to_i + # commit_id, old_lineno, lineno, span = m[1], m[2].to_i, m[3].to_i, m[4].to_i commit_id = m[1] commits[commit_id] = nil unless commits.key?(commit_id) - info[m[3].to_i] = [commit_id, m[2].to_i] + info[m[3].to_i] = [commit_id, m[2].to_i, m[4].to_i] # Assumption: the first line returned by git blame is lowest-numbered # This is true unless we start passing it `--incremental`. @@ -72,13 +72,14 @@ module Gitlab end # get it together - info.sort.each do |lineno, (commit_id, old_lineno)| + info.sort.each do |lineno, (commit_id, old_lineno, span)| final << BlameLine.new( lineno, old_lineno, commits[commit_id], lines[lineno - start_line], - previous_paths[commit_id] + previous_paths[commit_id], + span ) end @@ -87,14 +88,15 @@ module Gitlab end class BlameLine - attr_accessor :lineno, :oldlineno, :commit, :line, :previous_path + attr_accessor :lineno, :oldlineno, :commit, :line, :previous_path, :span - def initialize(lineno, oldlineno, commit, line, previous_path) + def initialize(lineno, oldlineno, commit, line, previous_path, span) @lineno = lineno @oldlineno = oldlineno @commit = commit @line = line @previous_path = previous_path + @span = span end end end diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index c0601c7795c..571dde6fcfc 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -12,7 +12,20 @@ module Gitlab attr_accessor :raw_commit, :head MAX_COMMIT_MESSAGE_DISPLAY_SIZE = 10.megabytes + + SHA1_LENGTH = 40 + SHA256_LENGTH = 64 + MIN_SHA_LENGTH = 7 + MAX_SHA_LENGTH = SHA256_LENGTH + + RAW_SHA_PATTERN = "\\h{#{MIN_SHA_LENGTH},#{MAX_SHA_LENGTH}}".freeze + SHA_PATTERN = /#{RAW_SHA_PATTERN}/ + # Match a full SHA. Note that because this expression is not anchored it will match any SHA that is at + # least SHA1_LENGTH long. + RAW_FULL_SHA_PATTERN = "\\h{#{SHA1_LENGTH}}(?:\\h{#{SHA256_LENGTH - SHA1_LENGTH}})?".freeze + FULL_SHA_PATTERN = /#{RAW_FULL_SHA_PATTERN}/ + SERIALIZE_KEYS = [ :id, :message, :parent_ids, :authored_date, :author_name, :author_email, @@ -226,6 +239,12 @@ module Gitlab id.to_s[0..length] end + def tree_id + return unless raw_commit + + raw_commit.tree_id + end + def safe_message @safe_message ||= message end diff --git a/lib/gitlab/git/diff_tree.rb b/lib/gitlab/git/diff_tree.rb new file mode 100644 index 00000000000..df48baeb1c3 --- /dev/null +++ b/lib/gitlab/git/diff_tree.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module Git + # Represents a tree-ish object for git diff-tree command + # See: https://git-scm.com/docs/git-diff-tree + class DiffTree + attr_reader :left_tree_id, :right_tree_id + + def initialize(left_tree_id, right_tree_id) + @left_tree_id = left_tree_id + @right_tree_id = right_tree_id + end + + def self.from_commit(commit) + return unless commit.tree_id + + parent_tree_id = + if commit.parent_ids.blank? + Gitlab::Git::EMPTY_TREE_ID + else + parent_id = commit.parent_ids.first + commit.repository.commit(parent_id).tree_id + end + + new(parent_tree_id, commit.tree_id) + end + end + end +end diff --git a/lib/gitlab/git/gitmodules_parser.rb b/lib/gitlab/git/gitmodules_parser.rb index 92940c352d3..a5a84d79720 100644 --- a/lib/gitlab/git/gitmodules_parser.rb +++ b/lib/gitlab/git/gitmodules_parser.rb @@ -50,7 +50,7 @@ module Gitlab @content.split("\n").each_with_object(iterator) do |text, iterator| text.chomp! - next if text =~ /^\s*#/ + next if /^\s*#/.match?(text) if text =~ /\A\[submodule "(?<name>[^"]+)"\]\z/ iterator.start_section($~[:name]) diff --git a/lib/gitlab/git/object_pool.rb b/lib/gitlab/git/object_pool.rb index d0577d7a4ff..4b46fa5f82a 100644 --- a/lib/gitlab/git/object_pool.rb +++ b/lib/gitlab/git/object_pool.rb @@ -12,6 +12,17 @@ module Gitlab attr_reader :storage, :relative_path, :source_repository, :gl_project_path + def self.init_from_gitaly(gitaly_object_pool, source_repository) + repository = gitaly_object_pool.repository + + new( + repository.storage_name, + repository.relative_path, + source_repository, + repository.gl_project_path + ) + end + def initialize(storage, relative_path, source_repository, gl_project_path) @storage = storage @relative_path = relative_path diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 71be986882c..d27f721bb2c 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -47,6 +47,8 @@ module Gitlab attr_reader :storage, :gl_repository, :gl_project_path, :container + delegate :list_all_blobs, to: :gitaly_blob_client + # This remote name has to be stable for all types of repositories that # can join an object pool. If it's structure ever changes, a migration # has to be performed on the object pools to update the remote names. @@ -338,16 +340,28 @@ module Gitlab # Return repo size in megabytes def size if Feature.enabled?(:use_repository_info_for_repository_size) - bytes = gitaly_repository_client.repository_info.size - - (bytes.to_f / 1024 / 1024).round(2) + repository_info_size_megabytes else kilobytes = gitaly_repository_client.repository_size - (kilobytes.to_f / 1024).round(2) end end + # Return repository recent objects size in mebibytes + # + # This differs from the #size method in that it does not include the size of: + # - stale objects + # - cruft packs of unreachable objects + # + # see: https://gitlab.com/gitlab-org/gitaly/-/blob/257ee33ca268d48c8f99dcbfeaaf7d8b19e07f06/internal/gitaly/service/repository/repository_info.go#L41-62 + def recent_objects_size + wrapped_gitaly_errors do + recent_size_in_bytes = gitaly_repository_client.repository_info.objects.recent_size + + Gitlab::Utils.bytes_to_megabytes(recent_size_in_bytes) + end + end + # Return git object directory size in bytes def object_directory_size gitaly_repository_client.get_object_directory_size.to_f * 1024 @@ -525,15 +539,13 @@ module Gitlab empty_diff_stats end - def find_changed_paths(commits, merge_commit_diff_mode: nil) - processed_commits = commits.reject { |ref| ref.blank? || Gitlab::Git.blank_ref?(ref) } + def find_changed_paths(treeish_objects, merge_commit_diff_mode: nil) + processed_objects = treeish_objects.compact - return [] if processed_commits.empty? + return [] if processed_objects.empty? wrapped_gitaly_errors do - gitaly_commit_client.find_changed_paths( - processed_commits, merge_commit_diff_mode: merge_commit_diff_mode - ) + gitaly_commit_client.find_changed_paths(processed_objects, merge_commit_diff_mode: merge_commit_diff_mode) end rescue CommandError, TypeError, NoRepository [] @@ -741,6 +753,16 @@ module Gitlab raise DeleteBranchError, e end + def async_delete_refs(*refs) + raise "async_delete_refs only supports project repositories" unless container.is_a?(Project) + + records = refs.map do |ref| + BatchedGitRefUpdates::Deletion.new(project_id: container.id, ref: ref, created_at: Time.current, updated_at: Time.current) + end + + BatchedGitRefUpdates::Deletion.bulk_insert!(records) + end + def delete_refs(...) wrapped_gitaly_errors do gitaly_delete_refs(...) @@ -956,6 +978,18 @@ module Gitlab end end + def rebase_to_ref(user, source_sha:, target_ref:, first_parent_ref:, expected_old_oid: "") + wrapped_gitaly_errors do + gitaly_operation_client.user_rebase_to_ref( + user, + source_sha: source_sha, + target_ref: target_ref, + first_parent_ref: first_parent_ref, + expected_old_oid: expected_old_oid + ) + end + end + def squash(user, start_sha:, end_sha:, author:, message:) wrapped_gitaly_errors do gitaly_operation_client.user_squash(user, start_sha, end_sha, author, message) @@ -1164,8 +1198,26 @@ module Gitlab end end + def get_patch_id(old_revision, new_revision) + wrapped_gitaly_errors do + gitaly_commit_client.get_patch_id(old_revision, new_revision) + end + end + + def object_pool + wrapped_gitaly_errors do + gitaly_repository_client.object_pool.object_pool + end + end + private + def repository_info_size_megabytes + bytes = gitaly_repository_client.repository_info.size + + Gitlab::Utils.bytes_to_megabytes(bytes).round(2) + end + def empty_diff_stats Gitlab::Git::DiffStatsCollection.new([]) end diff --git a/lib/gitlab/git/rugged_impl/tree.rb b/lib/gitlab/git/rugged_impl/tree.rb index c7a981c7dd4..bc3ff01e1e2 100644 --- a/lib/gitlab/git/rugged_impl/tree.rb +++ b/lib/gitlab/git/rugged_impl/tree.rb @@ -16,7 +16,7 @@ module Gitlab TREE_SORT_ORDER = { tree: 0, blob: 1, commit: 2 }.freeze override :tree_entries - def tree_entries(repository, sha, path, recursive, skip_flat_paths, pagination_params = nil) + def tree_entries(repository, sha, path, recursive, skip_flat_paths, rescue_not_found, pagination_params = nil) if use_rugged?(repository, :rugged_tree_entries) entries = execute_rugged_call( :tree_entries_with_flat_path_from_rugged, repository, sha, path, recursive, skip_flat_paths) diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index 140dc791135..0895c0b8a22 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -15,22 +15,27 @@ module Gitlab # Uses rugged for raw objects # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/320 - def where(repository, sha, path = nil, recursive = false, skip_flat_paths = true, pagination_params = nil) + def where( + repository, sha, path = nil, recursive = false, skip_flat_paths = true, rescue_not_found = true, + pagination_params = nil) path = nil if path == '' || path == '/' - tree_entries(repository, sha, path, recursive, skip_flat_paths, pagination_params) + tree_entries(repository, sha, path, recursive, skip_flat_paths, rescue_not_found, pagination_params) end - def tree_entries(repository, sha, path, recursive, skip_flat_paths, pagination_params = nil) + def tree_entries(repository, sha, path, recursive, skip_flat_paths, rescue_not_found, pagination_params = nil) wrapped_gitaly_errors do repository.gitaly_commit_client.tree_entries( repository, sha, path, recursive, skip_flat_paths, pagination_params) end # Incorrect revision or path could lead to index error. - # We silently handle such errors by returning an empty set of entries and cursor. - rescue Gitlab::Git::Index::IndexError - [[], nil] + # We silently handle such errors by returning an empty set of entries and cursor + # unless the parameter rescue_not_found is set to false. + rescue Gitlab::Git::Index::IndexError => e + return [[], nil] if rescue_not_found + + raise e end private diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 9d19695363a..a28952ab7bc 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -513,7 +513,7 @@ module Gitlab end def check_size_against_limit(size) - if size_checker.changes_will_exceed_size_limit?(size) + if size_checker.changes_will_exceed_size_limit?(size, project) raise ForbiddenError, size_checker.error_message.new_changes_error end end diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb index 6d87c3329d7..cd7b9a3c095 100644 --- a/lib/gitlab/gitaly_client/blob_service.rb +++ b/lib/gitlab/gitaly_client/blob_service.rb @@ -22,6 +22,32 @@ module Gitlab consume_blob_response(response) end + def list_all_blobs(limit: nil, bytes_limit: 0, dynamic_timeout: nil, ignore_alternate_object_directories: false) + repository = @gitaly_repo + + if ignore_alternate_object_directories + repository = @gitaly_repo.dup.tap do |g_repo| + g_repo.git_alternate_object_directories = Google::Protobuf::RepeatedField.new(:string) + end + end + + request = Gitaly::ListAllBlobsRequest.new( + repository: repository, + limit: limit, + bytes_limit: bytes_limit + ) + + timeout = + if dynamic_timeout + [dynamic_timeout, GitalyClient.medium_timeout].min + else + GitalyClient.medium_timeout + end + + response = Gitlab::GitalyClient.call(repository.storage_name, :blob_service, :list_all_blobs, request, timeout: timeout) + GitalyClient::BlobsStitcher.new(GitalyClient::ListBlobsAdapter.new(response)) + end + def list_blobs(revisions, limit: 0, bytes_limit: 0, with_paths: false, dynamic_timeout: nil) request = Gitaly::ListBlobsRequest.new( repository: @gitaly_repo, diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index c10f780665c..1ef5b0f96c2 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -274,8 +274,10 @@ module Gitlab # else # defaults to :include_merges behavior # ['foo_bar.rb', 'bar_baz.rb'], # - def find_changed_paths(commits, merge_commit_diff_mode: nil) - request = find_changed_paths_request(commits, merge_commit_diff_mode) + def find_changed_paths(objects, merge_commit_diff_mode: nil) + request = find_changed_paths_request(objects, merge_commit_diff_mode) + + return [] if request.nil? response = gitaly_client_call(@repository.storage, :diff_service, :find_changed_paths, request, timeout: GitalyClient.medium_timeout) response.flat_map do |msg| @@ -587,6 +589,15 @@ module Gitlab Hash[commit_refs] end + def get_patch_id(old_revision, new_revision) + request = Gitaly::GetPatchIDRequest + .new(repository: @gitaly_repo, old_revision: old_revision, new_revision: new_revision) + + response = gitaly_client_call(@repository.storage, :diff_service, :get_patch_id, request, timeout: GitalyClient.medium_timeout) + + response.patch_id + end + private def parse_global_options!(options) @@ -646,16 +657,27 @@ module Gitlab response.commit end - def find_changed_paths_request(commits, merge_commit_diff_mode) + def find_changed_paths_request(objects, merge_commit_diff_mode) diff_mode = MERGE_COMMIT_DIFF_MODES[merge_commit_diff_mode] if Feature.enabled?(:merge_commit_diff_modes) - commit_requests = commits.map do |commit| - Gitaly::FindChangedPathsRequest::Request.new( - commit_request: Gitaly::FindChangedPathsRequest::Request::CommitRequest.new(commit_revision: commit) - ) + requests = objects.filter_map do |object| + case object + when Gitlab::Git::DiffTree + Gitaly::FindChangedPathsRequest::Request.new( + tree_request: Gitaly::FindChangedPathsRequest::Request::TreeRequest.new(left_tree_revision: object.left_tree_id, right_tree_revision: object.right_tree_id) + ) + when Commit, Gitlab::Git::Commit + next if object.sha.blank? || Gitlab::Git.blank_ref?(object.sha) + + Gitaly::FindChangedPathsRequest::Request.new( + commit_request: Gitaly::FindChangedPathsRequest::Request::CommitRequest.new(commit_revision: object.sha) + ) + end end - Gitaly::FindChangedPathsRequest.new(repository: @gitaly_repo, requests: commit_requests, merge_commit_diff_mode: diff_mode) + return if requests.blank? + + Gitaly::FindChangedPathsRequest.new(repository: @gitaly_repo, requests: requests, merge_commit_diff_mode: diff_mode) end def path_error_message(path_error) diff --git a/lib/gitlab/gitaly_client/conflicts_service.rb b/lib/gitlab/gitaly_client/conflicts_service.rb index 38f648ccc31..ffe65307c80 100644 --- a/lib/gitlab/gitaly_client/conflicts_service.rb +++ b/lib/gitlab/gitaly_client/conflicts_service.rb @@ -17,19 +17,21 @@ module Gitlab self.repository_actor = repository end - def list_conflict_files(allow_tree_conflicts: false) + def list_conflict_files(allow_tree_conflicts: false, skip_content: false) request = Gitaly::ListConflictFilesRequest.new( repository: @gitaly_repo, our_commit_oid: @our_commit_oid, their_commit_oid: @their_commit_oid, - allow_tree_conflicts: allow_tree_conflicts + allow_tree_conflicts: allow_tree_conflicts, + skip_content: skip_content ) response = gitaly_client_call(@repository.storage, :conflicts_service, :list_conflict_files, request, timeout: GitalyClient.long_timeout) GitalyClient::ConflictFilesStitcher.new(response, @gitaly_repo) end def conflicts? - list_conflict_files.any? + skip_content = Feature.enabled?(:skip_conflict_files_in_gitaly, type: :experiment) + list_conflict_files(skip_content: skip_content).any? rescue GRPC::FailedPrecondition, GRPC::Unknown # The server raises FailedPrecondition when it encounters # ConflictSideMissing, which means a conflict exists but its `theirs` or diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 67e135bb530..fe76543548b 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -135,7 +135,7 @@ module Gitlab end end - def user_merge_to_ref(user, source_sha:, branch:, target_ref:, message:, first_parent_ref:) + def user_merge_to_ref(user, source_sha:, branch:, target_ref:, message:, first_parent_ref:, expected_old_oid: "") request = Gitaly::UserMergeToRefRequest.new( repository: @gitaly_repo, source_sha: source_sha, @@ -144,6 +144,7 @@ module Gitlab user: Gitlab::Git::User.from_gitlab(user).to_gitaly, message: encode_binary(message), first_parent_ref: encode_binary(first_parent_ref), + expected_old_oid: expected_old_oid, timestamp: Google::Protobuf::Timestamp.new(seconds: Time.now.utc.to_i) ) @@ -344,6 +345,23 @@ module Gitlab request_enum.close end + def user_rebase_to_ref(user, source_sha:, target_ref:, first_parent_ref:, expected_old_oid: "") + request = Gitaly::UserRebaseToRefRequest.new( + user: Gitlab::Git::User.from_gitlab(user).to_gitaly, + repository: @gitaly_repo, + source_sha: source_sha, + target_ref: encode_binary(target_ref), + first_parent_ref: encode_binary(first_parent_ref), + expected_old_oid: expected_old_oid, + timestamp: Google::Protobuf::Timestamp.new(seconds: Time.now.utc.to_i) + ) + + response = gitaly_client_call(@repository.storage, :operation_service, + :user_rebase_to_ref, request, timeout: GitalyClient.long_timeout) + + response.commit_id + end + def user_squash(user, start_sha, end_sha, author, message, time = Time.now.utc) request = Gitaly::UserSquashRequest.new( repository: @gitaly_repo, diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index b5b7d94b4d0..b2d5f9c7e13 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -363,6 +363,18 @@ module Gitlab ) end + def object_pool + request = Gitaly::GetObjectPoolRequest.new(repository: @gitaly_repo) + + gitaly_client_call( + @storage, + :object_pool_service, + :get_object_pool, + request, + timeout: GitalyClient.medium_timeout + ) + end + private def search_results_from_response(gitaly_response, options = {}) diff --git a/lib/gitlab/github_import.rb b/lib/gitlab/github_import.rb index 24e77363e1b..3b19b9d16d2 100644 --- a/lib/gitlab/github_import.rb +++ b/lib/gitlab/github_import.rb @@ -16,7 +16,7 @@ module Gitlab } if token_pool - ClientPool.new(token_pool: token_pool, **options) + ClientPool.new(token_pool: token_pool.append(token_to_use), **options) else Client.new(token_to_use, **options) end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index ff171c24549..27c4ec2f7be 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -51,6 +51,7 @@ module Gitlab gon.dot_com = Gitlab.com? gon.uf_error_prefix = ::Gitlab::Utils::ErrorMessage::UF_ERROR_PREFIX gon.pat_prefix = Gitlab::CurrentSettings.current_application_settings.personal_access_token_prefix + gon.use_new_navigation = NavHelper.show_super_sidebar?(current_user) gon.diagramsnet_url = Gitlab::CurrentSettings.diagramsnet_url if Gitlab::CurrentSettings.diagramsnet_enabled @@ -61,7 +62,6 @@ module Gitlab gon.current_user_fullname = current_user.name gon.current_user_avatar_url = current_user.avatar_url gon.time_display_relative = current_user.time_display_relative - gon.use_new_navigation = NavHelper.show_super_sidebar?(current_user) end # Initialize gon.features with any flags that should be @@ -76,6 +76,7 @@ module Gitlab push_frontend_feature_flag(:remove_monitor_metrics) push_frontend_feature_flag(:gitlab_duo, current_user) push_frontend_feature_flag(:custom_emoji) + push_frontend_feature_flag(:super_sidebar_flyout_menus, current_user) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/graphql/loaders/full_path_model_loader.rb b/lib/gitlab/graphql/loaders/full_path_model_loader.rb index 2ea3fa71d5e..a99b8c81930 100644 --- a/lib/gitlab/graphql/loaders/full_path_model_loader.rb +++ b/lib/gitlab/graphql/loaders/full_path_model_loader.rb @@ -20,7 +20,10 @@ module Gitlab # this logic cannot be placed in the NamespaceResolver due to N+1 scope = scope.without_project_namespaces if scope == Namespace # `with_route` avoids an N+1 calculating full_path - scope.where_full_path_in(full_paths).with_route.each do |model_instance| + scope = scope.where_full_path_in(full_paths).with_route + .allow_cross_joins_across_databases(url: "https://gitlab.com/gitlab-org/gitlab/-/issues/420046") + + scope.each do |model_instance| loader.call(model_instance.full_path.downcase, model_instance) end end diff --git a/lib/gitlab/hook_data/issue_builder.rb b/lib/gitlab/hook_data/issue_builder.rb index bd0603c5e5b..aacc3f8b821 100644 --- a/lib/gitlab/hook_data/issue_builder.rb +++ b/lib/gitlab/hook_data/issue_builder.rb @@ -11,6 +11,7 @@ module Gitlab time_change severity escalation_status + customer_relations_contacts ].freeze end @@ -56,7 +57,8 @@ module Gitlab assignee_id: issue.assignee_ids.first, # This key is deprecated labels: issue.labels_hook_attrs, state: issue.state, - severity: issue.severity + severity: issue.severity, + customer_relations_contacts: issue.customer_relations_contacts.map(&:hook_attrs) } if issue.supports_escalation? && issue.escalation_status diff --git a/lib/gitlab/hook_data/merge_request_builder.rb b/lib/gitlab/hook_data/merge_request_builder.rb index a6ca8323a20..22af06ba09b 100644 --- a/lib/gitlab/hook_data/merge_request_builder.rb +++ b/lib/gitlab/hook_data/merge_request_builder.rb @@ -10,6 +10,7 @@ module Gitlab blocking_discussions_resolved created_at description + draft head_pipeline_id id iid @@ -55,6 +56,7 @@ module Gitlab target: merge_request.target_project.hook_attrs, last_commit: merge_request.diff_head_commit&.hook_attrs, work_in_progress: merge_request.draft?, + draft: merge_request.draft?, total_time_spent: merge_request.total_time_spent, time_change: merge_request.time_change, human_total_time_spent: merge_request.human_total_time_spent, diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index fabc02af70a..6b154c7033f 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -44,8 +44,8 @@ module Gitlab TRANSLATION_LEVELS = { 'bg' => 0, 'cs_CZ' => 0, - 'da_DK' => 30, - 'de' => 99, + 'da_DK' => 29, + 'de' => 96, 'en' => 100, 'eo' => 0, 'es' => 29, @@ -55,19 +55,19 @@ module Gitlab 'id_ID' => 0, 'it' => 1, 'ja' => 99, - 'ko' => 17, - 'nb_NO' => 22, + 'ko' => 20, + 'nb_NO' => 21, 'nl_NL' => 0, 'pl_PL' => 3, - 'pt_BR' => 57, - 'ro_RO' => 80, - 'ru' => 23, - 'si_LK' => 10, + 'pt_BR' => 55, + 'ro_RO' => 78, + 'ru' => 22, + 'si_LK' => 9, 'tr_TR' => 9, - 'uk' => 53, + 'uk' => 52, 'zh_CN' => 98, 'zh_HK' => 1, - 'zh_TW' => 100 + 'zh_TW' => 99 }.freeze private_constant :TRANSLATION_LEVELS diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index e2f365fcbf8..924ca4e83ea 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -56,7 +56,7 @@ module Gitlab private - def download_or_copy_upload(uploader, upload_path, size_limit: nil) + def download_or_copy_upload(uploader, upload_path, size_limit: 0) if uploader.upload.local? copy_files(uploader.path, upload_path) else @@ -64,7 +64,7 @@ module Gitlab end end - def download(url, upload_path, size_limit: nil) + def download(url, upload_path, size_limit: 0) File.open(upload_path, 'wb') do |file| current_size = 0 @@ -74,7 +74,7 @@ module Gitlab elsif fragment.code == 200 current_size += fragment.bytesize - raise FileOversizedError if size_limit.present? && current_size > size_limit + raise FileOversizedError if size_limit > 0 && current_size > size_limit file.write(fragment) else diff --git a/lib/gitlab/import_export/decompressed_archive_size_validator.rb b/lib/gitlab/import_export/decompressed_archive_size_validator.rb index 2e39f3f38c2..3609df89958 100644 --- a/lib/gitlab/import_export/decompressed_archive_size_validator.rb +++ b/lib/gitlab/import_export/decompressed_archive_size_validator.rb @@ -5,7 +5,6 @@ module Gitlab class DecompressedArchiveSizeValidator include Gitlab::Utils::StrongMemoize - DEFAULT_MAX_BYTES = 10.gigabytes.freeze TIMEOUT_LIMIT = 210.seconds ServiceError = Class.new(StandardError) @@ -22,7 +21,7 @@ module Gitlab end def self.max_bytes - DEFAULT_MAX_BYTES + Gitlab::CurrentSettings.current_application_settings.max_decompressed_archive_size.megabytes end private @@ -52,7 +51,7 @@ module Gitlab if status.success? result = stdout.readline - if result.to_i > @max_bytes + if @max_bytes > 0 && result.to_i > @max_bytes valid_archive = false log_error('Decompressed archive size limit reached') diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 37c83e88ef2..7fb7a9f30a0 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -74,17 +74,21 @@ module Gitlab download( import_export_upload.remote_import_url, @archive_file, - size_limit: ::Import::GitlabProjects::RemoteFileValidator::FILE_SIZE_LIMIT + size_limit: file_size_limit ) else download_or_copy_upload( import_export_upload.import_file, @archive_file, - size_limit: ::Import::GitlabProjects::RemoteFileValidator::FILE_SIZE_LIMIT + size_limit: file_size_limit ) end end + def file_size_limit + Gitlab::CurrentSettings.current_application_settings.max_import_remote_file_size.megabytes + end + def remove_import_file FileUtils.rm_rf(@archive_file) end diff --git a/lib/gitlab/import_export/project/base_task.rb b/lib/gitlab/import_export/project/base_task.rb index 356e261e251..3cbe3cb7153 100644 --- a/lib/gitlab/import_export/project/base_task.rb +++ b/lib/gitlab/import_export/project/base_task.rb @@ -4,8 +4,6 @@ module Gitlab module ImportExport module Project class BaseTask - include Gitlab::WithRequestStore - def initialize(opts, logger: Logger.new($stdout)) @project_path = opts.fetch(:project_path) @file_path = opts.fetch(:file_path) diff --git a/lib/gitlab/import_export/project/export_task.rb b/lib/gitlab/import_export/project/export_task.rb index 5e105b4653d..3cd0d3f4c2b 100644 --- a/lib/gitlab/import_export/project/export_task.rb +++ b/lib/gitlab/import_export/project/export_task.rb @@ -35,7 +35,7 @@ module Gitlab end def with_export - with_request_store do + ::Gitlab::SafeRequestStore.ensure_request_store do # We are disabling ObjectStorage for `export` # since when direct upload is enabled, remote storage will be used # and Gitlab::ImportExport::AfterExportStrategies::MoveFileStrategy will fail to copy exported archive diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 5986c5de441..850c89c1fb1 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -984,9 +984,11 @@ excluded_attributes: notes: - :noteable_id - :review_id + - :namespace_id commit_notes: - :noteable_id - :review_id + - :namespace_id label_links: - :label_id - :target_id diff --git a/lib/gitlab/import_export/project/import_task.rb b/lib/gitlab/import_export/project/import_task.rb index 4ea47a5624a..47cdb630ada 100644 --- a/lib/gitlab/import_export/project/import_task.rb +++ b/lib/gitlab/import_export/project/import_task.rb @@ -25,7 +25,7 @@ module Gitlab # to general Sidekiq clusters/nodes. def with_isolated_sidekiq_job Sidekiq::Testing.fake! do - with_request_store do + ::Gitlab::SafeRequestStore.ensure_request_store do # If you are attempting to import a large project into a development environment, # you may see Gitaly throw an error about too many calls or invocations. # This is due to a n+1 calls limit being set for development setups (not enforced in production) diff --git a/lib/gitlab/internal_events/event_definitions.rb b/lib/gitlab/internal_events/event_definitions.rb index e1c9faa12de..f3c8092bcb0 100644 --- a/lib/gitlab/internal_events/event_definitions.rb +++ b/lib/gitlab/internal_events/event_definitions.rb @@ -8,10 +8,6 @@ module Gitlab class << self VALID_UNIQUE_VALUES = %w[user.id project.id namespace.id].freeze - def clear_events - @events = nil - end - def load_configurations @events = load_metric_definitions nil diff --git a/lib/gitlab/jwt_authenticatable.rb b/lib/gitlab/jwt_authenticatable.rb index d7a341b3ba2..b8282163cbc 100644 --- a/lib/gitlab/jwt_authenticatable.rb +++ b/lib/gitlab/jwt_authenticatable.rb @@ -13,10 +13,12 @@ module Gitlab module ClassMethods include Gitlab::Utils::StrongMemoize - def decode_jwt(encoded_message, jwt_secret = secret, algorithm: 'HS256', issuer: nil, iat_after: nil) + def decode_jwt( + encoded_message, jwt_secret = secret, algorithm: 'HS256', issuer: nil, iat_after: nil, audience: nil) options = { algorithm: algorithm } options = options.merge(iss: issuer, verify_iss: true) if issuer.present? options = options.merge(verify_iat: true) if iat_after.present? + options = options.merge(aud: audience, verify_aud: true) if audience.present? decoded_message = JWT.decode(encoded_message, jwt_secret, true, options) payload = decoded_message[0] diff --git a/lib/gitlab/kas.rb b/lib/gitlab/kas.rb index a1e290a54e6..255d8802c1c 100644 --- a/lib/gitlab/kas.rb +++ b/lib/gitlab/kas.rb @@ -5,13 +5,14 @@ module Gitlab INTERNAL_API_REQUEST_HEADER = 'Gitlab-Kas-Api-Request' VERSION_FILE = 'GITLAB_KAS_VERSION' JWT_ISSUER = 'gitlab-kas' + JWT_AUDIENCE = 'gitlab' K8S_PROXY_PATH = 'k8s-proxy' include JwtAuthenticatable class << self def verify_api_request(request_headers) - decode_jwt(request_headers[INTERNAL_API_REQUEST_HEADER], issuer: JWT_ISSUER) + decode_jwt(request_headers[INTERNAL_API_REQUEST_HEADER], issuer: JWT_ISSUER, audience: JWT_AUDIENCE) rescue JWT::DecodeError nil end @@ -54,6 +55,13 @@ module Gitlab uri.to_s end + def tunnel_ws_url + return tunnel_url if ws? + return tunnel_url.sub('https', 'wss') if ssl? + + tunnel_url.sub('http', 'ws') + end + # Return GitLab KAS internal_url # # @return [String] internal_url @@ -67,6 +75,16 @@ module Gitlab def enabled? !!Gitlab.config['gitlab_kas']&.fetch('enabled', false) end + + private + + def ssl? + URI(tunnel_url).scheme === 'https' + end + + def ws? + URI(tunnel_url).scheme.start_with?('ws') + end end end end diff --git a/lib/gitlab/markdown_cache/redis/store.rb b/lib/gitlab/markdown_cache/redis/store.rb index 52260623c55..f742cb82b8d 100644 --- a/lib/gitlab/markdown_cache/redis/store.rb +++ b/lib/gitlab/markdown_cache/redis/store.rb @@ -11,11 +11,9 @@ module Gitlab Gitlab::Redis::Cache.with do |r| Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - r.with_readonly_pipeline do - Gitlab::Redis::CrossSlot::Pipeline.new(r).pipelined do |pipeline| - subjects.each do |subject| - results[subject.cache_key] = new(subject).read(pipeline) - end + Gitlab::Redis::CrossSlot::Pipeline.new(r).pipelined do |pipeline| + subjects.each do |subject| + results[subject.cache_key] = new(subject).read(pipeline) end end end diff --git a/lib/gitlab/merge_requests/message_generator.rb b/lib/gitlab/merge_requests/message_generator.rb index 523e0e665dc..5ca26fdae86 100644 --- a/lib/gitlab/merge_requests/message_generator.rb +++ b/lib/gitlab/merge_requests/message_generator.rb @@ -52,6 +52,7 @@ module Gitlab 'description' => ->(merge_request, _, _) { merge_request.description }, 'reference' => ->(merge_request, _, _) { merge_request.to_reference(full: true) }, 'local_reference' => ->(merge_request, _, _) { merge_request.to_reference(full: false) }, + 'source_project_id' => ->(merge_request, _, _) { merge_request.source_project.id.to_s }, 'first_commit' => -> (merge_request, _, _) { return unless merge_request.persisted? || merge_request.compare_commits.present? diff --git a/lib/gitlab/metrics/dashboard/defaults.rb b/lib/gitlab/metrics/dashboard/defaults.rb deleted file mode 100644 index 6a5f98a18c8..00000000000 --- a/lib/gitlab/metrics/dashboard/defaults.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -# Central point for managing default attributes from within -# the metrics dashboard module. -module Gitlab - module Metrics - module Dashboard - module Defaults - DEFAULT_PANEL_TYPE = 'area-chart' - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/finder.rb b/lib/gitlab/metrics/dashboard/finder.rb deleted file mode 100644 index 12f7c347b2d..00000000000 --- a/lib/gitlab/metrics/dashboard/finder.rb +++ /dev/null @@ -1,95 +0,0 @@ -# frozen_string_literal: true - -# Returns DB-supplmented dashboard info for determining -# the layout of UI. Intended entry-point for the Metrics::Dashboard -# module. -module Gitlab - module Metrics - module Dashboard - class Finder - PREDEFINED_DASHBOARD_LIST = [ - ::Metrics::Dashboard::PodDashboardService, - ::Metrics::Dashboard::SystemDashboardService - ].freeze - - class << self - # Returns a formatted dashboard packed with DB info. - # @param project [Project] - # @param user [User] - # @param environment [Environment] - # @param options [Hash<Symbol,Any>] - # @param options - embedded [Boolean] Determines whether the - # dashboard is to be rendered as part of an - # issue or location other than the primary - # metrics dashboard UI. Returns only the - # Memory/CPU charts of the system dash. - # @param options - dashboard_path [String] Path at which the - # dashboard can be found. Nil values will - # default to the system dashboard. - # @param options - group [String, Group] Title of the group - # to which a panel might belong. Used by - # embedded dashboards. If cluster dashboard, - # refers to the Group corresponding to the cluster. - # @param options - title [String] Title of the panel. - # Used by embedded dashboards. - # @param options - y_label [String] Y-Axis label of - # a panel. Used by embedded dashboards. - # @param options - cluster [Cluster]. Used by - # embedded and un-embedded dashboards. - # @param options - cluster_type [Symbol] The level of - # cluster, one of [:admin, :project, :group]. Used by - # embedded and un-embedded dashboards. - # @param options - grafana_url [String] URL pointing - # to a grafana dashboard panel - # @param options - prometheus_alert_id [Integer] ID of - # a PrometheusAlert. For dashboard embeds. - # @return [Hash] - def find(project, user, options = {}) - service_for(options) - .new(project, user, options) - .get_dashboard - end - - # Returns a dashboard without any supplemental info. - # Returns only full, yml-defined dashboards. - # @return [Hash] - def find_raw(project, dashboard_path: nil) - service_for(dashboard_path: dashboard_path) - .new(project, nil, dashboard_path: dashboard_path) - .raw_dashboard - end - - # Summary of all known dashboards. - # @return [Array<Hash>] ex) [{ path: String, - # display_name: String, - # default: Boolean }] - def find_all_paths(project) - dashboards = user_facing_dashboard_services.flat_map do |service| - service.all_dashboard_paths(project) - end - - Gitlab::Utils.stable_sort_by(dashboards) { |dashboard| dashboard[:display_name].downcase } - end - - private - - def user_facing_dashboard_services - PREDEFINED_DASHBOARD_LIST + [project_service] - end - - def system_service - ::Metrics::Dashboard::SystemDashboardService - end - - def project_service - ::Metrics::Dashboard::CustomDashboardService - end - - def service_for(options) - Gitlab::Metrics::Dashboard::ServiceSelector.call(options) - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/importer.rb b/lib/gitlab/metrics/dashboard/importer.rb deleted file mode 100644 index ca835650648..00000000000 --- a/lib/gitlab/metrics/dashboard/importer.rb +++ /dev/null @@ -1,41 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - class Importer - def initialize(dashboard_path, project) - @dashboard_path = dashboard_path.to_s - @project = project - end - - def execute - return false unless Dashboard::Validator.validate(dashboard_hash, project: project, dashboard_path: dashboard_path) - - Dashboard::Importers::PrometheusMetrics.new(dashboard_hash, project: project, dashboard_path: dashboard_path).execute - rescue Gitlab::Config::Loader::FormatError - false - end - - def execute! - Dashboard::Validator.validate!(dashboard_hash, project: project, dashboard_path: dashboard_path) - - Dashboard::Importers::PrometheusMetrics.new(dashboard_hash, project: project, dashboard_path: dashboard_path).execute! - end - - private - - attr_accessor :dashboard_path, :project - - def dashboard_hash - @dashboard_hash ||= begin - raw_dashboard = Dashboard::RepoDashboardFinder.read_dashboard(project, dashboard_path) - return unless raw_dashboard.present? - - ::Gitlab::Config::Loader::Yaml.new(raw_dashboard).load_raw! - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb b/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb deleted file mode 100644 index 531e4079632..00000000000 --- a/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb +++ /dev/null @@ -1,76 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Importers - class PrometheusMetrics - ALLOWED_ATTRIBUTES = %i(title query y_label unit legend group dashboard_path).freeze - - # Takes a JSON schema validated dashboard hash and - # imports metrics to database - def initialize(dashboard_hash, project:, dashboard_path:) - @dashboard_hash = dashboard_hash - @project = project - @dashboard_path = dashboard_path - @affected_environment_ids = [] - end - - def execute - import - rescue ActiveRecord::RecordInvalid, Dashboard::Transformers::Errors::BaseError - false - end - - def execute! - import - end - - private - - attr_reader :dashboard_hash, :project, :dashboard_path - - def import - delete_stale_metrics - create_or_update_metrics - end - - # rubocop: disable CodeReuse/ActiveRecord - def create_or_update_metrics - # TODO: use upsert and worker for callbacks? - - affected_metric_ids = [] - prometheus_metrics_attributes.each do |attributes| - prometheus_metric = PrometheusMetric.find_or_initialize_by(attributes.slice(:dashboard_path, :identifier, :project)) - prometheus_metric.update!(attributes.slice(*ALLOWED_ATTRIBUTES)) - - affected_metric_ids << prometheus_metric.id - end - end - # rubocop: enable CodeReuse/ActiveRecord - - def delete_stale_metrics - identifiers_from_yml = prometheus_metrics_attributes.map { |metric_attributes| metric_attributes[:identifier] } - - stale_metrics = PrometheusMetric.for_project(project) - .for_dashboard_path(dashboard_path) - .for_group(Enums::PrometheusMetric.groups[:custom]) - .not_identifier(identifiers_from_yml) - - return unless stale_metrics.exists? - - stale_metrics.each_batch { |batch| batch.delete_all } - end - - def prometheus_metrics_attributes - @prometheus_metrics_attributes ||= Dashboard::Transformers::Yml::V1::PrometheusMetrics.new( - dashboard_hash, - project: project, - dashboard_path: dashboard_path - ).execute - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/service_selector.rb b/lib/gitlab/metrics/dashboard/service_selector.rb deleted file mode 100644 index 67bf4ce7e9a..00000000000 --- a/lib/gitlab/metrics/dashboard/service_selector.rb +++ /dev/null @@ -1,54 +0,0 @@ -# frozen_string_literal: true - -# Responsible for determining which dashboard service should -# be used to fetch or generate a dashboard hash. -# The services can be considered in two categories - embeds -# and dashboards. Embed hashes are identical to dashboard hashes except -# that they contain a subset of panels. -module Gitlab - module Metrics - module Dashboard - class ServiceSelector - class << self - include Gitlab::Utils::StrongMemoize - - SERVICES = [ - ::Metrics::Dashboard::ClusterMetricsEmbedService, - ::Metrics::Dashboard::ClusterDashboardService, - ::Metrics::Dashboard::GitlabAlertEmbedService, - ::Metrics::Dashboard::CustomMetricEmbedService, - ::Metrics::Dashboard::GrafanaMetricEmbedService, - ::Metrics::Dashboard::TransientEmbedService, - ::Metrics::Dashboard::DynamicEmbedService, - ::Metrics::Dashboard::DefaultEmbedService, - ::Metrics::Dashboard::SystemDashboardService, - ::Metrics::Dashboard::PodDashboardService, - ::Metrics::Dashboard::CustomDashboardService - ].freeze - - # Returns a class which inherits from the BaseService - # class that can be used to obtain a dashboard for - # the provided params. - # @return [Metrics::Dashboard::BaseService] - def call(params) - service = services.find do |service_class| - service_class.valid_params?(params) - end - - service || default_service - end - - private - - def services - SERVICES - end - - def default_service - ::Metrics::Dashboard::SystemDashboardService - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/stages/base_stage.rb b/lib/gitlab/metrics/dashboard/stages/base_stage.rb index c2a8a88108f..b869a633030 100644 --- a/lib/gitlab/metrics/dashboard/stages/base_stage.rb +++ b/lib/gitlab/metrics/dashboard/stages/base_stage.rb @@ -5,8 +5,6 @@ module Gitlab module Dashboard module Stages class BaseStage - include Gitlab::Metrics::Dashboard::Defaults - attr_reader :project, :dashboard, :params def initialize(project, dashboard, params) diff --git a/lib/gitlab/metrics/dashboard/stages/cluster_endpoint_inserter.rb b/lib/gitlab/metrics/dashboard/stages/cluster_endpoint_inserter.rb deleted file mode 100644 index 56a82d1df46..00000000000 --- a/lib/gitlab/metrics/dashboard/stages/cluster_endpoint_inserter.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Stages - class ClusterEndpointInserter < BaseStage - def transform! - verify_params - end - - private - - def error!(message) - raise Errors::DashboardProcessingError, message - end - - def query_type(metric) - metric[:query] ? :query : :query_range - end - - def query_for_metric(metric) - query = metric[query_type(metric)] - - raise Errors::MissingQueryError, 'Each "metric" must define one of :query or :query_range' unless query - - query - end - - def verify_params - raise Errors::DashboardProcessingError, _('Cluster is required for Stages::ClusterEndpointInserter') unless params[:cluster] - raise Errors::DashboardProcessingError, _('Cluster type must be specified for Stages::ClusterEndpointInserter') unless params[:cluster_type] - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/stages/common_metrics_inserter.rb b/lib/gitlab/metrics/dashboard/stages/common_metrics_inserter.rb deleted file mode 100644 index 62479ed6de4..00000000000 --- a/lib/gitlab/metrics/dashboard/stages/common_metrics_inserter.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Stages - class CommonMetricsInserter < BaseStage - # For each metric in the dashboard config, attempts to - # find a corresponding database record. If found, - # includes the record's id in the dashboard config. - def transform! - common_metrics = ::PrometheusMetricsFinder.new(common: true).execute - - for_metrics do |metric| - metric_record = common_metrics.find { |m| m.identifier == metric[:id] } - metric[:metric_id] = metric_record.id if metric_record - end - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/stages/custom_dashboard_metrics_inserter.rb b/lib/gitlab/metrics/dashboard/stages/custom_dashboard_metrics_inserter.rb deleted file mode 100644 index 5ed4466f440..00000000000 --- a/lib/gitlab/metrics/dashboard/stages/custom_dashboard_metrics_inserter.rb +++ /dev/null @@ -1,24 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Stages - # Acts on metrics which have been ingested from source controlled dashboards - class CustomDashboardMetricsInserter < BaseStage - # For each metric in the dashboard config, attempts to - # find a corresponding database record. If found, includes - # the record's id in the dashboard config. - def transform! - database_metrics = ::PrometheusMetricsFinder.new(common: false, group: :custom, project: project).execute - - for_metrics do |metric| - metric_record = database_metrics.find { |m| m.identifier == metric[:id] } - metric[:metric_id] = metric_record.id if metric_record - end - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/stages/custom_metrics_details_inserter.rb b/lib/gitlab/metrics/dashboard/stages/custom_metrics_details_inserter.rb deleted file mode 100644 index 06cfa5cc58e..00000000000 --- a/lib/gitlab/metrics/dashboard/stages/custom_metrics_details_inserter.rb +++ /dev/null @@ -1,40 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Stages - class CustomMetricsDetailsInserter < BaseStage - def transform! - dashboard[:panel_groups].each do |panel_group| - next unless panel_group - - has_custom_metrics = custom_group_titles.include?(panel_group[:group]) - panel_group[:has_custom_metrics] = has_custom_metrics - - panel_group[:panels].each do |panel| - next unless panel - - panel[:metrics].each do |metric| - next unless metric - - metric[:edit_path] = has_custom_metrics ? edit_path(metric) : nil - end - end - end - end - - private - - def custom_group_titles - @custom_group_titles ||= Enums::PrometheusMetric.custom_group_details.values.map { |group_details| group_details[:group_title] } - end - - def edit_path(metric) - Gitlab::Routing.url_helpers.edit_project_prometheus_metric_path(project, metric[:metric_id]) - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/stages/custom_metrics_inserter.rb b/lib/gitlab/metrics/dashboard/stages/custom_metrics_inserter.rb deleted file mode 100644 index 3b49eb1c837..00000000000 --- a/lib/gitlab/metrics/dashboard/stages/custom_metrics_inserter.rb +++ /dev/null @@ -1,109 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Stages - class CustomMetricsInserter < BaseStage - # Inserts project-specific metrics into the dashboard - # config. If there are no project-specific metrics, - # this will have no effect. - def transform! - custom_metrics = PrometheusMetricsFinder.new(project: project, ordered: true).execute - custom_metrics = Gitlab::Utils.stable_sort_by(custom_metrics) { |metric| -metric.priority } - - custom_metrics.each do |project_metric| - group = find_or_create_panel_group(dashboard[:panel_groups], project_metric) - panel = find_or_create_panel(group[:panels], project_metric) - find_or_create_metric(panel[:metrics], project_metric) - end - end - - private - - # Looks for a panel_group corresponding to the - # provided metric object. If unavailable, inserts one. - # @param panel_groups [Array<Hash>] - # @param metric [PrometheusMetric] - def find_or_create_panel_group(panel_groups, metric) - panel_group = find_panel_group(panel_groups, metric) - return panel_group if panel_group - - panel_group = new_panel_group(metric) - panel_groups << panel_group - - panel_group - end - - # Looks for a panel corresponding to the provided - # metric object. If unavailable, inserts one. - # @param panels [Array<Hash>] - # @param metric [PrometheusMetric] - def find_or_create_panel(panels, metric) - panel = find_panel(panels, metric) - return panel if panel - - panel = new_panel(metric) - panels << panel - - panel - end - - # Looks for a metric corresponding to the provided - # metric object. If unavailable, inserts one. - # @param metrics [Array<Hash>] - # @param metric [PrometheusMetric] - def find_or_create_metric(metrics, metric) - target_metric = find_metric(metrics, metric) - return target_metric if target_metric - - target_metric = new_metric(metric) - metrics << target_metric - - target_metric - end - - def find_panel_group(panel_groups, metric) - return unless panel_groups - - panel_groups.find { |group| group[:group] == metric.group_title } - end - - def find_panel(panels, metric) - return unless panels - - panel_identifiers = [DEFAULT_PANEL_TYPE, metric.title, metric.y_label] - panels.find { |panel| panel.values_at(:type, :title, :y_label) == panel_identifiers } - end - - def find_metric(metrics, metric) - return unless metrics - return unless metric.identifier - - metrics.find { |m| m[:id] == metric.identifier } - end - - def new_panel_group(metric) - { - group: metric.group_title, - panels: [] - } - end - - def new_panel(metric) - { - type: DEFAULT_PANEL_TYPE, - title: metric.title, - y_label: metric.y_label, - metrics: [] - } - end - - def new_metric(metric) - metric.to_metric_hash - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb b/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb deleted file mode 100644 index 03370ae7370..00000000000 --- a/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb +++ /dev/null @@ -1,151 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Stages - class GrafanaFormatter < BaseStage - include Gitlab::Utils::StrongMemoize - - CHART_TYPE = 'area-chart' - PROXY_PATH = 'api/v1/query_range' - - # Reformats the specified panel in the Gitlab - # dashboard-yml format - def transform! - validate_input! - - new_dashboard = formatted_dashboard - - dashboard.clear - dashboard.merge!(new_dashboard) - end - - private - - def validate_input! - ::Grafana::Validator.new( - grafana_dashboard, - datasource, - panel, - query_params - ).validate! - rescue ::Grafana::Validator::Error => e - raise ::Gitlab::Metrics::Dashboard::Errors::DashboardProcessingError, e.message - end - - def formatted_dashboard - { panel_groups: [{ panels: [formatted_panel] }] } - end - - def formatted_panel - { - title: panel[:title], - type: CHART_TYPE, - y_label: '', # Grafana panels do not include a Y-Axis label - metrics: panel[:targets].map.with_index do |target, idx| - formatted_metric(target, idx) - end - } - end - - def formatted_metric(metric, idx) - { - id: "#{metric[:legendFormat]}_#{idx}", - query_range: format_query(metric), - label: replace_variables(metric[:legendFormat]) - }.compact - end - - # Panel specified by the url from the Grafana dashboard - def panel - strong_memoize(:panel) do - grafana_dashboard[:dashboard][:panels].find do |panel| - query_params[:panelId] ? matching_panel?(panel) : valid_panel?(panel) - end - end - end - - # Determines whether a given panel is the one - # specified by the linked grafana url - def matching_panel?(panel) - panel[:id].to_s == query_params[:panelId] - end - - # Determines whether any given panel has the potenial - # to return valid results from grafana/prometheus - def valid_panel?(panel) - ::Grafana::Validator - .new(grafana_dashboard, datasource, panel, query_params) - .valid? - end - - # Grafana url query parameters. Includes information - # on which panel to select and time range. - def query_params - strong_memoize(:query_params) do - Gitlab::Metrics::Dashboard::Url.parse_query(grafana_url) - end - end - - # Reformats query for compatibility with prometheus api. - def format_query(metric) - expression = remove_new_lines(metric[:expr]) - expression = replace_variables(expression) - replace_global_variables(expression, metric) - end - - # Accomodates instance-defined Grafana variables. - # These are variables defined by users, and values - # must be provided in the query parameters. - def replace_variables(expression) - return expression unless grafana_dashboard[:dashboard][:templating] - - grafana_dashboard[:dashboard][:templating][:list] - .sort_by { |variable| variable[:name].length } - .each do |variable| - variable_value = query_params[:"var-#{variable[:name]}"] - - expression = expression.gsub("$#{variable[:name]}", variable_value) - expression = expression.gsub("[[#{variable[:name]}]]", variable_value) - expression = expression.gsub("{{#{variable[:name]}}}", variable_value) - end - - expression - end - - # Replaces Grafana global built-in variables with values. - # Only $__interval and $__from and $__to are supported. - # - # See https://grafana.com/docs/reference/templating/#global-built-in-variables - def replace_global_variables(expression, metric) - expression = expression.gsub('$__interval', metric[:interval]) if metric[:interval] - expression = expression.gsub('$__from', query_params[:from]) - expression.gsub('$__to', query_params[:to]) - end - - # Removes new lines from expression. - def remove_new_lines(expression) - expression.gsub(/\R+/, '') - end - - # Grafana datasource object corresponding to the - # specified dashboard - def datasource - params[:datasource] - end - - # The specified Grafana dashboard - def grafana_dashboard - params[:grafana_dashboard] - end - - # The URL specifying which Grafana panel to embed - def grafana_url - params[:grafana_url] - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/stages/metric_endpoint_inserter.rb b/lib/gitlab/metrics/dashboard/stages/metric_endpoint_inserter.rb deleted file mode 100644 index d885d978524..00000000000 --- a/lib/gitlab/metrics/dashboard/stages/metric_endpoint_inserter.rb +++ /dev/null @@ -1,56 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Stages - class MetricEndpointInserter < BaseStage - def transform! - raise Errors::DashboardProcessingError, _('Environment is required for Stages::MetricEndpointInserter') unless params[:environment] - - for_metrics do |metric| - metric[:prometheus_endpoint_path] = endpoint_for_metric(metric) - end - end - - private - - def endpoint_for_metric(metric) - if params[:sample_metrics] - Gitlab::Routing.url_helpers.sample_metrics_project_environment_path( - project, - params[:environment], - identifier: metric[:id] - ) - else - Gitlab::Routing.url_helpers.prometheus_api_project_environment_path( - project, - params[:environment], - proxy_path: query_type(metric), - query: query_for_metric(metric) - ) - end - end - - def query_type(metric) - if metric[:query] - ::Prometheus::ProxyService::PROMETHEUS_QUERY_API.to_sym - else - ::Prometheus::ProxyService::PROMETHEUS_QUERY_RANGE_API.to_sym - end - end - - def query_for_metric(metric) - query = metric[query_type(metric)] - - raise Errors::MissingQueryError, 'Each "metric" must define one of :query or :query_range' unless query - - # We need to remove any newlines since our UrlBlocker does not allow - # multiline URLs. - query.to_s.squish - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/stages/panel_ids_inserter.rb b/lib/gitlab/metrics/dashboard/stages/panel_ids_inserter.rb deleted file mode 100644 index 239b5161256..00000000000 --- a/lib/gitlab/metrics/dashboard/stages/panel_ids_inserter.rb +++ /dev/null @@ -1,61 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Stages - class PanelIdsInserter < BaseStage - # For each panel within given dashboard inserts panel_id unique in scope of the dashboard - def transform! - missing_panel_groups! unless dashboard[:panel_groups] - - for_panels_group_with_panels do |panel_group, panel| - id = generate_panel_id(panel_group, panel) - remove_panel_ids! && break if duplicated_panel_id?(id) - - insert_panel_id(id, panel) - end - rescue ActiveModel::UnknownAttributeError => error - remove_panel_ids! - Gitlab::ErrorTracking.log_exception(error) - end - - private - - def generate_panel_id(group, panel) - ::PerformanceMonitoring::PrometheusPanel.new(panel.with_indifferent_access).id(group[:group]) - end - - def insert_panel_id(id, panel) - track_inserted_panel_ids(id, panel) - panel[:id] = id - end - - def track_inserted_panel_ids(id, panel) - panel_ids[id] = panel - end - - def duplicated_panel_id?(id) - panel_ids.key?(id) - end - - def remove_panel_ids! - panel_ids.each_value { |panel| panel.delete(:id) } - end - - def panel_ids - @_panel_ids ||= {} - end - - def for_panels_group_with_panels - for_panel_groups do |panel_group| - for_panels_in(panel_group) do |panel| - yield panel_group, panel - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/stages/track_panel_type.rb b/lib/gitlab/metrics/dashboard/stages/track_panel_type.rb deleted file mode 100644 index 71da779d16c..00000000000 --- a/lib/gitlab/metrics/dashboard/stages/track_panel_type.rb +++ /dev/null @@ -1,27 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Stages - class TrackPanelType < BaseStage - def transform! - for_panel_groups do |panel_group| - for_panels_in(panel_group) do |panel| - track_panel_type(panel) - end - end - end - - private - - def track_panel_type(panel) - panel_type = panel[:type] - - Gitlab::Tracking.event('MetricsDashboard::Chart', 'chart_rendered', label: panel_type) - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/stages/variable_endpoint_inserter.rb b/lib/gitlab/metrics/dashboard/stages/variable_endpoint_inserter.rb deleted file mode 100644 index b3ce0b79675..00000000000 --- a/lib/gitlab/metrics/dashboard/stages/variable_endpoint_inserter.rb +++ /dev/null @@ -1,34 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Stages - class VariableEndpointInserter < BaseStage - VARIABLE_TYPE_METRIC_LABEL_VALUES = 'metric_label_values' - - def transform! - raise Errors::DashboardProcessingError, _('Environment is required for Stages::VariableEndpointInserter') unless params[:environment] - - for_variables do |variable_name, variable| - if variable.is_a?(Hash) && variable[:type] == VARIABLE_TYPE_METRIC_LABEL_VALUES - variable[:options][:prometheus_endpoint_path] = endpoint_for_variable(variable.dig(:options, :series_selector)) - end - end - end - - private - - def endpoint_for_variable(series_selector) - Gitlab::Routing.url_helpers.prometheus_api_project_environment_path( - project, - params[:environment], - proxy_path: ::Prometheus::ProxyService::PROMETHEUS_SERIES_API, - match: Array(series_selector) - ) - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/transformers/yml/v1/prometheus_metrics.rb b/lib/gitlab/metrics/dashboard/transformers/yml/v1/prometheus_metrics.rb deleted file mode 100644 index 3650ddf698a..00000000000 --- a/lib/gitlab/metrics/dashboard/transformers/yml/v1/prometheus_metrics.rb +++ /dev/null @@ -1,54 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Transformers - module Yml - module V1 - # Takes a JSON schema validated dashboard hash and - # maps it to PrometheusMetric model attributes - class PrometheusMetrics - def initialize(dashboard_hash, project: nil, dashboard_path: nil) - @dashboard_hash = dashboard_hash.with_indifferent_access - @project = project - @dashboard_path = dashboard_path - - @dashboard_hash.default_proc = -> (h, k) { raise Transformers::Errors::MissingAttribute, k.to_s } - end - - def execute - prometheus_metrics = [] - - dashboard_hash[:panel_groups].each do |panel_group| - panel_group[:panels].each do |panel| - panel[:metrics].each do |metric| - prometheus_metrics << { - project: project, - title: panel[:title], - y_label: panel[:y_label], - query: metric[:query_range] || metric[:query], - unit: metric[:unit], - legend: metric[:label], - identifier: metric[:id], - group: Enums::PrometheusMetric.groups[:custom], - common: false, - dashboard_path: dashboard_path - }.compact - end - end - end - - prometheus_metrics - end - - private - - attr_reader :dashboard_hash, :project, :dashboard_path - end - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/validator.rb b/lib/gitlab/metrics/dashboard/validator.rb deleted file mode 100644 index 57b4b5c068d..00000000000 --- a/lib/gitlab/metrics/dashboard/validator.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Validator - DASHBOARD_SCHEMA_PATH = Rails.root.join(*%w[lib gitlab metrics dashboard validator schemas dashboard.json]).freeze - - class << self - def validate(content, schema_path = DASHBOARD_SCHEMA_PATH, dashboard_path: nil, project: nil) - errors(content, schema_path, dashboard_path: dashboard_path, project: project).empty? - end - - def validate!(content, schema_path = DASHBOARD_SCHEMA_PATH, dashboard_path: nil, project: nil) - errors = errors(content, schema_path, dashboard_path: dashboard_path, project: project) - errors.empty? || raise(errors.first) - end - - private - - def errors(content, schema_path = DASHBOARD_SCHEMA_PATH, dashboard_path: nil, project: nil) - Validator::Client - .new(content, schema_path, dashboard_path: dashboard_path, project: project) - .execute - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/validator/client.rb b/lib/gitlab/metrics/dashboard/validator/client.rb deleted file mode 100644 index 29f1274a097..00000000000 --- a/lib/gitlab/metrics/dashboard/validator/client.rb +++ /dev/null @@ -1,56 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Validator - class Client - # @param content [Hash] Representing a raw, unprocessed - # dashboard object - # @param schema_path [String] Representing path to dashboard schema file - # @param dashboard_path[String] Representing path to dashboard content file - # @param project [Project] Project to validate dashboard against - def initialize(content, schema_path, dashboard_path: nil, project: nil) - @content = content - @schema_path = schema_path - @dashboard_path = dashboard_path - @project = project - end - - def execute - errors = validate_against_schema - errors += post_schema_validator.validate - - errors.compact - end - - private - - attr_reader :content, :schema_path, :project, :dashboard_path - - def custom_formats - @custom_formats ||= CustomFormats.new - end - - def post_schema_validator - PostSchemaValidator.new( - project: project, - metric_ids: custom_formats.metric_ids_cache, - dashboard_path: dashboard_path - ) - end - - def schemer - @schemer ||= ::JSONSchemer.schema(Pathname.new(schema_path), formats: custom_formats.format_handlers) - end - - def validate_against_schema - schemer.validate(content).map do |error| - ::Gitlab::Metrics::Dashboard::Validator::Errors::SchemaValidationError.new(error) - end - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/validator/custom_formats.rb b/lib/gitlab/metrics/dashboard/validator/custom_formats.rb deleted file mode 100644 index 485e80ad1b7..00000000000 --- a/lib/gitlab/metrics/dashboard/validator/custom_formats.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Validator - class CustomFormats - def format_handlers - # Key is custom JSON Schema format name. Value is a proc that takes data and schema and handles - # validations. - @format_handlers ||= { - "add_to_metric_id_cache" => ->(data, schema) { metric_ids_cache << data } - } - end - - def metric_ids_cache - @metric_ids_cache ||= [] - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/validator/errors.rb b/lib/gitlab/metrics/dashboard/validator/errors.rb deleted file mode 100644 index 0f6e687d291..00000000000 --- a/lib/gitlab/metrics/dashboard/validator/errors.rb +++ /dev/null @@ -1,60 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Validator - module Errors - InvalidDashboardError = Class.new(StandardError) - - class SchemaValidationError < InvalidDashboardError - def initialize(error = {}) - super(error_message(error)) - end - - private - - def error_message(error) - if error.is_a?(Hash) && error.present? - pretty(error) - else - "Dashboard failed schema validation" - end - end - - # based on https://github.com/davishmcclurg/json_schemer/blob/master/lib/json_schemer/errors.rb - # with addition ability to translate error messages - def pretty(error) - data, data_pointer, type, schema = error.values_at('data', 'data_pointer', 'type', 'schema') - location = data_pointer.empty? ? 'root' : data_pointer - - case type - when 'required' - keys = error.fetch('details').fetch('missing_keys').join(', ') - _("%{location} is missing required keys: %{keys}") % { location: location, keys: keys } - when 'null', 'string', 'boolean', 'integer', 'number', 'array', 'object' - _("'%{data}' at %{location} is not of type: %{type}") % { data: data, location: location, type: type } - when 'pattern' - _("'%{data}' at %{location} does not match pattern: %{pattern}") % { data: data, location: location, pattern: schema.fetch('pattern') } - when 'format' - _("'%{data}' at %{location} does not match format: %{format}") % { data: data, location: location, format: schema.fetch('format') } - when 'const' - _("'%{data}' at %{location} is not: %{const}") % { data: data, location: location, const: schema.fetch('const').inspect } - when 'enum' - _("'%{data}' at %{location} is not one of: %{enum}") % { data: data, location: location, enum: schema.fetch('enum') } - else - _("'%{data}' at %{location} is invalid: error_type=%{type}") % { data: data, location: location, type: type } - end - end - end - - class DuplicateMetricIds < InvalidDashboardError - def initialize - super(_("metric_id must be unique across a project")) - end - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/validator/post_schema_validator.rb b/lib/gitlab/metrics/dashboard/validator/post_schema_validator.rb deleted file mode 100644 index 73bfc5a6294..00000000000 --- a/lib/gitlab/metrics/dashboard/validator/post_schema_validator.rb +++ /dev/null @@ -1,52 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Validator - class PostSchemaValidator - def initialize(metric_ids:, project: nil, dashboard_path: nil) - @metric_ids = metric_ids - @project = project - @dashboard_path = dashboard_path - end - - def validate - errors = [] - errors << uniq_metric_ids - errors.compact - end - - private - - attr_reader :project, :metric_ids, :dashboard_path - - def uniq_metric_ids - return Validator::Errors::DuplicateMetricIds.new if metric_ids.uniq! - - uniq_metric_ids_across_project if project.present? || dashboard_path.present? - end - - # rubocop: disable CodeReuse/ActiveRecord - def uniq_metric_ids_across_project - return ArgumentError.new(_('Both project and dashboard_path are required')) unless - dashboard_path.present? && project.present? - - # If PrometheusMetric identifier is not unique across project and dashboard_path, - # we need to error because we don't know if the user is trying to create a new metric - # or update an existing one. - identifier_on_other_dashboard = PrometheusMetric.where( - project: project, - identifier: metric_ids - ).where.not( - dashboard_path: dashboard_path - ).exists? - - Validator::Errors::DuplicateMetricIds.new if identifier_on_other_dashboard - end - # rubocop: enable CodeReuse/ActiveRecord - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/validator/schemas/axis.json b/lib/gitlab/metrics/dashboard/validator/schemas/axis.json deleted file mode 100644 index 54334022426..00000000000 --- a/lib/gitlab/metrics/dashboard/validator/schemas/axis.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "type": "object", - "properties": { - "name": { "type": "string" }, - "format": { - "type": "string", - "default": "engineering" - }, - "precision": { - "type": "number", - "default": 2 - } - } -} diff --git a/lib/gitlab/metrics/dashboard/validator/schemas/dashboard.json b/lib/gitlab/metrics/dashboard/validator/schemas/dashboard.json deleted file mode 100644 index 313f03be7dc..00000000000 --- a/lib/gitlab/metrics/dashboard/validator/schemas/dashboard.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "type": "object", - "required": ["dashboard", "panel_groups"], - "properties": { - "dashboard": { "type": "string" }, - "panel_groups": { - "type": "array", - "items": { "$ref": "./panel_group.json" } - }, - "templating": { - "$ref": "./templating.json" - }, - "links": { - "type": "array", - "items": { "$ref": "./link.json" } - } - } -} diff --git a/lib/gitlab/metrics/dashboard/validator/schemas/link.json b/lib/gitlab/metrics/dashboard/validator/schemas/link.json deleted file mode 100644 index 4ea7b5dd324..00000000000 --- a/lib/gitlab/metrics/dashboard/validator/schemas/link.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "type": "object", - "required": ["url"], - "properties": { - "url": { "type": "string" }, - "title": { "type": "string" }, - "type": { - "type": "string", - "enum": ["grafana"] - } - } -} diff --git a/lib/gitlab/metrics/dashboard/validator/schemas/metric.json b/lib/gitlab/metrics/dashboard/validator/schemas/metric.json deleted file mode 100644 index 13831b77e3e..00000000000 --- a/lib/gitlab/metrics/dashboard/validator/schemas/metric.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "type": "object", - "required": ["unit"], - "oneOf": [{ "required": ["query"] }, { "required": ["query_range"] }], - "properties": { - "id": { - "type": "string", - "format": "add_to_metric_id_cache" - }, - "unit": { "type": "string" }, - "label": { "type": "string" }, - "query": { "type": ["string", "number"] }, - "query_range": { "type": ["string", "number"] }, - "step": { "type": "number" } - } -} diff --git a/lib/gitlab/metrics/dashboard/validator/schemas/panel.json b/lib/gitlab/metrics/dashboard/validator/schemas/panel.json deleted file mode 100644 index 2ae9608036e..00000000000 --- a/lib/gitlab/metrics/dashboard/validator/schemas/panel.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "type": "object", - "required": ["title", "metrics"], - "properties": { - "type": { - "type": "string", - "enum": ["area-chart", "line-chart", "anomaly-chart", "bar", "column", "stacked-column", "single-stat", "heatmap", "gauge"], - "default": "area-chart" - }, - "title": { "type": "string" }, - "y_label": { "type": "string" }, - "y_axis": { "$ref": "./axis.json" }, - "max_value": { "type": "number" }, - "weight": { "type": "number" }, - "metrics": { - "type": "array", - "items": { "$ref": "./metric.json" } - }, - "links": { - "type": "array", - "items": { "$ref": "./link.json" } - } - } -} diff --git a/lib/gitlab/metrics/dashboard/validator/schemas/panel_group.json b/lib/gitlab/metrics/dashboard/validator/schemas/panel_group.json deleted file mode 100644 index 1306fc475db..00000000000 --- a/lib/gitlab/metrics/dashboard/validator/schemas/panel_group.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "type": "object", - "required": ["group", "panels"], - "properties": { - "group": { "type": "string" }, - "priority": { "type": "number" }, - "panels": { - "type": "array", - "items": { "$ref": "./panel.json" } - } - } -} diff --git a/lib/gitlab/metrics/dashboard/validator/schemas/templating.json b/lib/gitlab/metrics/dashboard/validator/schemas/templating.json deleted file mode 100644 index 6f8664c89af..00000000000 --- a/lib/gitlab/metrics/dashboard/validator/schemas/templating.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "type": "object", - "required": ["variables"], - "properties": { - "variables": { "type": "object" } - } -} diff --git a/lib/gitlab/metrics/global_search_slis.rb b/lib/gitlab/metrics/global_search_slis.rb index c361d755a12..530bebd72ab 100644 --- a/lib/gitlab/metrics/global_search_slis.rb +++ b/lib/gitlab/metrics/global_search_slis.rb @@ -11,6 +11,7 @@ module Gitlab BASIC_CODE_TARGET_S = 27.538 ADVANCED_CONTENT_TARGET_S = 2.452 ADVANCED_CODE_TARGET_S = 15.52 + ZOEKT_TARGET_S = 15.52 def initialize_slis! Gitlab::Metrics::Sli::Apdex.initialize_sli(:global_search, possible_labels) @@ -42,6 +43,8 @@ module Gitlab ADVANCED_CONTENT_TARGET_S elsif search_type == 'advanced' && code_search?(search_scope) ADVANCED_CODE_TARGET_S + elsif search_type == 'zoekt' && code_search?(search_scope) + ZOEKT_TARGET_S end end diff --git a/lib/gitlab/metrics/samplers/threads_sampler.rb b/lib/gitlab/metrics/samplers/threads_sampler.rb index a460594fb59..1357e0a5d9b 100644 --- a/lib/gitlab/metrics/samplers/threads_sampler.rb +++ b/lib/gitlab/metrics/samplers/threads_sampler.rb @@ -54,7 +54,7 @@ module Gitlab if thread_name.presence.nil? 'unnamed' - elsif thread_name =~ /puma threadpool \d+/ + elsif /puma threadpool \d+/.match?(thread_name) # These are the puma workers processing requests 'puma threadpool' elsif use_thread_name?(thread_name) diff --git a/lib/gitlab/middleware/sidekiq_web_static.rb b/lib/gitlab/middleware/sidekiq_web_static.rb index 61b5fb9e0c6..c5d2ecbe00e 100644 --- a/lib/gitlab/middleware/sidekiq_web_static.rb +++ b/lib/gitlab/middleware/sidekiq_web_static.rb @@ -15,7 +15,7 @@ module Gitlab end def call(env) - env.delete('HTTP_X_SENDFILE_TYPE') if env['PATH_INFO'] =~ SIDEKIQ_REGEX + env.delete('HTTP_X_SENDFILE_TYPE') if SIDEKIQ_REGEX.match?(env['PATH_INFO']) @app.call(env) end diff --git a/lib/gitlab/middleware/static.rb b/lib/gitlab/middleware/static.rb index 972fed2134c..324d929a93d 100644 --- a/lib/gitlab/middleware/static.rb +++ b/lib/gitlab/middleware/static.rb @@ -6,7 +6,7 @@ module Gitlab UPLOADS_REGEX = %r{\A/uploads(/|\z)}.freeze def call(env) - return @app.call(env) if env['PATH_INFO'] =~ UPLOADS_REGEX + return @app.call(env) if UPLOADS_REGEX.match?(env['PATH_INFO']) super end diff --git a/lib/gitlab/null_request_store.rb b/lib/gitlab/null_request_store.rb deleted file mode 100644 index 4642dcf9e91..00000000000 --- a/lib/gitlab/null_request_store.rb +++ /dev/null @@ -1,41 +0,0 @@ -# frozen_string_literal: true - -# Used by Gitlab::SafeRequestStore -module Gitlab - # The methods `begin!`, `clear!`, and `end!` are not defined because they - # should only be called directly on `RequestStore`. - class NullRequestStore - def store - {} - end - - def active? - end - - def read(key) - end - - def [](key) - end - - def write(key, value) - value - end - - def []=(key, value) - value - end - - def exist?(key) - false - end - - def fetch(key, &block) - yield - end - - def delete(key, &block) - yield(key) if block - end - end -end diff --git a/lib/gitlab/pages/url_builder.rb b/lib/gitlab/pages/url_builder.rb index 215154b7248..5a28a5ffd23 100644 --- a/lib/gitlab/pages/url_builder.rb +++ b/lib/gitlab/pages/url_builder.rb @@ -82,8 +82,7 @@ module Gitlab end def unique_domain_enabled? - Feature.enabled?(:pages_unique_domain, project) && - project.project_setting.pages_unique_domain_enabled? + project.project_setting.pages_unique_domain_enabled? end def config diff --git a/lib/gitlab/pages/virtual_host_finder.rb b/lib/gitlab/pages/virtual_host_finder.rb index d5e2159fb52..88ee0e44c00 100644 --- a/lib/gitlab/pages/virtual_host_finder.rb +++ b/lib/gitlab/pages/virtual_host_finder.rb @@ -28,7 +28,6 @@ module Gitlab def by_unique_domain(name) project = Project.by_pages_enabled_unique_domain(name) - return unless Feature.enabled?(:pages_unique_domain, project) return unless project&.pages_deployed? ::Pages::VirtualDomain.new(projects: [project]) diff --git a/lib/gitlab/patch/command_loader.rb b/lib/gitlab/patch/command_loader.rb new file mode 100644 index 00000000000..357b6270b0d --- /dev/null +++ b/lib/gitlab/patch/command_loader.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Patch + module CommandLoader + extend ActiveSupport::Concern + + class_methods do + # Shuffle the node list to spread out initial connection creation amongst all nodes + # + # The input is a Redis::Cluster::Node instance which is an Enumerable. + # `super` receives an Array of Redis::Client instead of a Redis::Cluster::Node + def load(nodes) + super(nodes.to_a.shuffle) + end + end + end + end +end diff --git a/lib/gitlab/patch/node_loader.rb b/lib/gitlab/patch/node_loader.rb index 79f4b17dd93..85237abc137 100644 --- a/lib/gitlab/patch/node_loader.rb +++ b/lib/gitlab/patch/node_loader.rb @@ -9,6 +9,18 @@ end module Gitlab module Patch module NodeLoader + extend ActiveSupport::Concern + + class_methods do + # Shuffle the node list to spread out initial connection creation amongst all nodes + # + # The input is a Redis::Cluster::Node instance which is an Enumerable. + # `super` receives an Array of Redis::Client instead of a Redis::Cluster::Node + def load_flags(nodes) + super(nodes.to_a.shuffle) + end + end + def self.prepended(base) base.class_eval do # monkey-patches https://github.com/redis/redis-rb/blob/v4.8.0/lib/redis/cluster/node_loader.rb#L23 diff --git a/lib/gitlab/patch/redis_cache_store.rb b/lib/gitlab/patch/redis_cache_store.rb index 041cb2d44bd..ea6e1f11bc9 100644 --- a/lib/gitlab/patch/redis_cache_store.rb +++ b/lib/gitlab/patch/redis_cache_store.rb @@ -44,11 +44,7 @@ module Gitlab values = failsafe(:patched_read_multi_mget, returning: {}) do redis.with do |c| - if c.is_a?(Gitlab::Redis::MultiStore) - c.with_readonly_pipeline { pipeline_mget(c, keys) } - else - pipeline_mget(c, keys) - end + pipeline_mget(c, keys) end end diff --git a/lib/gitlab/patch/sidekiq_poller.rb b/lib/gitlab/patch/sidekiq_poller.rb index d4264cec1ab..4daee902a8e 100644 --- a/lib/gitlab/patch/sidekiq_poller.rb +++ b/lib/gitlab/patch/sidekiq_poller.rb @@ -5,7 +5,7 @@ module Gitlab module SidekiqPoller def enqueue Rails.application.reloader.wrap do - ::Gitlab::WithRequestStore.with_request_store do + ::Gitlab::SafeRequestStore.ensure_request_store do super ensure ::Gitlab::Database::LoadBalancing.release_hosts diff --git a/lib/gitlab/patch/slot_loader.rb b/lib/gitlab/patch/slot_loader.rb new file mode 100644 index 00000000000..e302d844078 --- /dev/null +++ b/lib/gitlab/patch/slot_loader.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Patch + module SlotLoader + extend ActiveSupport::Concern + + class_methods do + # Shuffle the node list to spread out initial connection creation amongst all nodes + # + # The input is a Redis::Cluster::Node instance which is an Enumerable. + # `super` receives an Array of Redis::Client instead of a Redis::Cluster::Node + def load(nodes) + super(nodes.to_a.shuffle) + end + end + end + end +end diff --git a/lib/gitlab/path_traversal.rb b/lib/gitlab/path_traversal.rb index 1123ff73136..d42b5fde615 100644 --- a/lib/gitlab/path_traversal.rb +++ b/lib/gitlab/path_traversal.rb @@ -14,7 +14,7 @@ module Gitlab # Ensure that the relative path will not traverse outside the base directory # We url decode the path to avoid passing invalid paths forward in url encoded format. # Also see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24223#note_284122580 - # It also checks for ALT_SEPARATOR aka '\' (forward slash) + # It also checks for backslash '\', which is sometimes a File::ALT_SEPARATOR. def check_path_traversal!(path) return unless path diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb index 5af06e82c55..1a6feff915f 100644 --- a/lib/gitlab/profiler.rb +++ b/lib/gitlab/profiler.rb @@ -2,8 +2,6 @@ module Gitlab module Profiler - extend WithRequestStore - FILTERED_STRING = '[FILTERED]' IGNORE_BACKTRACES = %w[ @@ -62,7 +60,7 @@ module Gitlab logger = create_custom_logger(logger, private_token: private_token) - result = with_request_store do + result = ::Gitlab::SafeRequestStore.ensure_request_store do # Make an initial call for an asset path in development mode to avoid # sprockets dominating the profiler output. ActionController::Base.helpers.asset_path('katex.css') if Rails.env.development? diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 13718e63b25..b4297cc695b 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -50,6 +50,7 @@ module Gitlab if @project.is_a?(Array) team_members_for_projects = User.joins(:project_authorizations).where(project_authorizations: { project_id: @project }) + .allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/422045') results = results.where(id: team_members_for_projects) else results = results.where(id: @project.team.members) diff --git a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb index e01be4e0604..c94deea0dfb 100644 --- a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb +++ b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb @@ -145,10 +145,18 @@ module Gitlab desc { _('Set time estimate') } explanation do |time_estimate| - formatted_time_estimate = format_time_estimate(time_estimate) - _("Sets time estimate to %{time_estimate}.") % { time_estimate: formatted_time_estimate } if formatted_time_estimate + next unless time_estimate + + if time_estimate == 0 + _('Removes time estimate.') + elsif time_estimate > 0 + formatted_time_estimate = format_time_estimate(time_estimate) + _("Sets time estimate to %{time_estimate}.") % { time_estimate: formatted_time_estimate } if formatted_time_estimate + end end execution_message do |time_estimate| + next _('Removed time estimate.') if time_estimate == 0 + formatted_time_estimate = format_time_estimate(time_estimate) _("Set time estimate to %{time_estimate}.") % { time_estimate: formatted_time_estimate } if formatted_time_estimate end @@ -159,12 +167,10 @@ module Gitlab current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) end parse_params do |raw_duration| - Gitlab::TimeTrackingFormatter.parse(raw_duration) + Gitlab::TimeTrackingFormatter.parse(raw_duration, keep_zero: true) end command :estimate, :estimate_time do |time_estimate| - if time_estimate - @updates[:time_estimate] = time_estimate - end + @updates[:time_estimate] = time_estimate end desc { _('Add or subtract spent time') } diff --git a/lib/gitlab/quick_actions/relate_actions.rb b/lib/gitlab/quick_actions/relate_actions.rb index 058c1e7e9bf..294ddd985de 100644 --- a/lib/gitlab/quick_actions/relate_actions.rb +++ b/lib/gitlab/quick_actions/relate_actions.rb @@ -36,7 +36,7 @@ module Gitlab extract_references(issue_param, :issue).first end command :unlink do |issue| - link = IssueLink.for_issues(quick_action_target, issue).first + link = IssueLink.for_items(quick_action_target, issue).first if link call_link_service(IssueLinks::DestroyService.new(link, current_user)) diff --git a/lib/gitlab/quick_actions/work_item_actions.rb b/lib/gitlab/quick_actions/work_item_actions.rb index a5c3c6a56be..0a96d502862 100644 --- a/lib/gitlab/quick_actions/work_item_actions.rb +++ b/lib/gitlab/quick_actions/work_item_actions.rb @@ -54,18 +54,9 @@ module Gitlab def validate_promote_to(type) return error_msg(:not_found, action: 'promote') unless type && supports_promote_to?(type.name) + return if current_user.can?(:"create_#{type.base_type}", quick_action_target) - unless current_user.can?(:"create_#{type.base_type}", quick_action_target) - return error_msg(:forbidden, action: 'promote') - end - - validate_hierarchy - end - - def validate_hierarchy - return unless current_type.task? && quick_action_target.parent_link - - error_msg(:hierarchy, action: 'promote') + error_msg(:forbidden, action: 'promote') end def current_type @@ -88,8 +79,7 @@ module Gitlab message = { not_found: 'Provided type is not supported', same_type: 'Types are the same', - forbidden: 'You have insufficient permissions', - hierarchy: 'A task cannot be promoted when a parent issue is present' + forbidden: 'You have insufficient permissions' }.freeze format(_("Failed to %{action} this work item: %{reason}."), { action: action, reason: message[reason] }) diff --git a/lib/gitlab/rack_attack.rb b/lib/gitlab/rack_attack.rb index d999b706d6c..829b305d1ee 100644 --- a/lib/gitlab/rack_attack.rb +++ b/lib/gitlab/rack_attack.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # When adding new user-configurable throttles, remember to update the documentation -# in doc/user/admin_area/settings/user_and_ip_rate_limits.md +# in doc/administration/settings/user_and_ip_rate_limits.md # # Integration specs for throttling can be found in: # spec/requests/rack_attack_global_spec.rb diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb index 06bce7649bf..e760a576253 100644 --- a/lib/gitlab/redis.rb +++ b/lib/gitlab/redis.rb @@ -9,7 +9,7 @@ module Gitlab # config/initializers/7_redis.rb, instrumented, and used in health- & readiness checks. ALL_CLASSES = [ Gitlab::Redis::Cache, - Gitlab::Redis::ClusterCache, + Gitlab::Redis::ClusterSharedState, Gitlab::Redis::DbLoadBalancing, Gitlab::Redis::FeatureFlag, Gitlab::Redis::Queues, diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb index 60944268f91..d63905cd896 100644 --- a/lib/gitlab/redis/cache.rb +++ b/lib/gitlab/redis/cache.rb @@ -8,9 +8,14 @@ module Gitlab class << self # Full list of options: # https://api.rubyonrails.org/classes/ActiveSupport/Cache/RedisCacheStore.html#method-c-new + # pool argument event not documented in the link above is handled by RedisCacheStore see: + # https://github.com/rails/rails/blob/593893c901f87b4ed205751f72df41519b4d2da3/activesupport/lib/active_support/cache/redis_cache_store.rb#L165 + # and + # https://github.com/rails/rails/blob/ad790cb2f6bc724a89e4266b505b3c57d5089dae/activesupport/lib/active_support/cache.rb#L206 def active_support_config { redis: pool, + pool: false, compress: Gitlab::Utils.to_boolean(ENV.fetch('ENABLE_REDIS_CACHE_COMPRESSION', '1')), namespace: CACHE_NAMESPACE, expires_in: default_ttl_seconds @@ -20,20 +25,6 @@ module Gitlab def default_ttl_seconds ENV.fetch('GITLAB_RAILS_CACHE_DEFAULT_TTL_SECONDS', 8.hours).to_i end - - # Exposes redis for Peek adapter. To be removed after ClusterCache migration. - def multistore_redis - redis - end - - private - - def redis - primary_store = ::Redis.new(Gitlab::Redis::ClusterCache.params) - secondary_store = ::Redis.new(params) - - MultiStore.new(primary_store, secondary_store, store_name) - end end end end diff --git a/lib/gitlab/redis/cluster_cache.rb b/lib/gitlab/redis/cluster_shared_state.rb index 15a87739c6d..678566a0c9c 100644 --- a/lib/gitlab/redis/cluster_cache.rb +++ b/lib/gitlab/redis/cluster_shared_state.rb @@ -2,10 +2,10 @@ module Gitlab module Redis - class ClusterCache < ::Gitlab::Redis::Wrapper + class ClusterSharedState < ::Gitlab::Redis::Wrapper class << self def config_fallback - Cache + SharedState end end end diff --git a/lib/gitlab/redis/etag_cache.rb b/lib/gitlab/redis/etag_cache.rb new file mode 100644 index 00000000000..6aafdc8e518 --- /dev/null +++ b/lib/gitlab/redis/etag_cache.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module Redis + class EtagCache < ::Gitlab::Redis::Wrapper + class << self + def store_name + 'Cache' + end + + private + + def redis + primary_store = ::Redis.new(Gitlab::Redis::Cache.params) + secondary_store = ::Redis.new(Gitlab::Redis::SharedState.params) + + MultiStore.new(primary_store, secondary_store, name.demodulize) + end + end + end + end +end diff --git a/lib/gitlab/redis/feature_flag.rb b/lib/gitlab/redis/feature_flag.rb index 441ff669035..395805792d7 100644 --- a/lib/gitlab/redis/feature_flag.rb +++ b/lib/gitlab/redis/feature_flag.rb @@ -14,6 +14,7 @@ module Gitlab def cache_store @cache_store ||= FeatureFlagStore.new( redis: pool, + pool: false, compress: Gitlab::Utils.to_boolean(ENV.fetch('ENABLE_REDIS_CACHE_COMPRESSION', '1')), namespace: Cache::CACHE_NAMESPACE, expires_in: 1.hour diff --git a/lib/gitlab/redis/repository_cache.rb b/lib/gitlab/redis/repository_cache.rb index 966c6584aa5..6d0c35a6829 100644 --- a/lib/gitlab/redis/repository_cache.rb +++ b/lib/gitlab/redis/repository_cache.rb @@ -15,6 +15,7 @@ module Gitlab def cache_store @cache_store ||= RepositoryCacheStore.new( redis: pool, + pool: false, compress: Gitlab::Utils.to_boolean(ENV.fetch('ENABLE_REDIS_CACHE_COMPRESSION', '1')), namespace: Cache::CACHE_NAMESPACE, expires_in: Cache.default_ttl_seconds diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 4e666dbaf77..40facabff23 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -2,308 +2,10 @@ module Gitlab module Regex - module Packages - CONAN_RECIPE_FILES = %w[conanfile.py conanmanifest.txt conan_sources.tgz conan_export.tgz].freeze - CONAN_PACKAGE_FILES = %w[conaninfo.txt conanmanifest.txt conan_package.tgz].freeze - - PYPI_NORMALIZED_NAME_REGEX_STRING = '[-_.]+' - - # see https://github.com/apache/maven/blob/c1dfb947b509e195c75d4275a113598cf3063c3e/maven-artifact/src/main/java/org/apache/maven/artifact/Artifact.java#L46 - MAVEN_SNAPSHOT_DYNAMIC_PARTS = /\A.{0,1000}(-\d{8}\.\d{6}-\d+).{0,1000}\z/.freeze - - API_PATH_REGEX = %r{^/api/v\d+/(projects/[^/]+/|groups?/[^/]+/-/)?packages/[A-Za-z]+}.freeze - - def conan_package_reference_regex - @conan_package_reference_regex ||= %r{\A[A-Za-z0-9]+\z}.freeze - end - - def conan_revision_regex - @conan_revision_regex ||= %r{\A0\z}.freeze - end - - def conan_recipe_user_channel_regex - %r{\A(_|#{conan_name_regex})\z}.freeze - end - - def conan_recipe_component_regex - # https://docs.conan.io/en/latest/reference/conanfile/attributes.html#name - @conan_recipe_component_regex ||= %r{\A#{conan_name_regex}\z}.freeze - end - - def composer_package_version_regex - # see https://github.com/composer/semver/blob/31f3ea725711245195f62e54ffa402d8ef2fdba9/src/VersionParser.php#L215 - @composer_package_version_regex ||= %r{\Av?((\d++)(\.(?:\d++|[xX*]))?(\.(?:\d++|[xX*]))?(\.(?:\d++|[xX*]))?)?\z}.freeze - end - - def composer_dev_version_regex - @composer_dev_version_regex ||= %r{(^dev-)|(-dev$)}.freeze - end - - def package_name_regex - @package_name_regex ||= - %r{ - \A\@? - (?> # atomic group to prevent backtracking - (([\w\-\.\+]*)\/)*([\w\-\.]+) - ) - @? - (?> # atomic group to prevent backtracking - (([\w\-\.\+]*)\/)*([\w\-\.]*) - ) - \z - }x.freeze - end - - def maven_file_name_regex - @maven_file_name_regex ||= %r{\A[A-Za-z0-9\.\_\-\+]+\z}.freeze - end - - def maven_path_regex - @maven_path_regex ||= %r{\A\@?(([\w\-\.]*)/)*([\w\-\.\+]*)\z}.freeze - end - - def maven_app_name_regex - @maven_app_name_regex ||= /\A[\w\-\.]+\z/.freeze - end - - def maven_version_regex - @maven_version_regex ||= /\A(?!.*\.\.)[\w+.-]+\z/.freeze - end - - def maven_app_group_regex - maven_app_name_regex - end - - def npm_package_name_regex - @npm_package_name_regex ||= %r{\A(?:@(#{Gitlab::PathRegex::NAMESPACE_FORMAT_REGEX})/)?[-+\.\_a-zA-Z0-9]+\z}o - end - - def npm_package_name_regex_message - 'should be a valid NPM package name: https://github.com/npm/validate-npm-package-name#naming-rules.' - end - - def nuget_package_name_regex - @nuget_package_name_regex ||= %r{\A[-+\.\_a-zA-Z0-9]+\z}.freeze - end - - def nuget_version_regex - @nuget_version_regex ||= / - \A#{_semver_major_regex} - \.#{_semver_minor_regex} - (\.#{_semver_patch_regex})? - (\.\d*)? - #{_semver_prerelease_build_regex}\z - /x.freeze - end - - def terraform_module_package_name_regex - @terraform_module_package_name_regex ||= %r{\A[-a-z0-9]+\/[-a-z0-9]+\z}.freeze - end - - def pypi_version_regex - # See the official regex: https://github.com/pypa/packaging/blob/16.7/packaging/version.py#L159 - - @pypi_version_regex ||= %r{ - \A(?: - v? - (?:([0-9]+)!)? (?# epoch) - ([0-9]+(?:\.[0-9]+)*) (?# release segment) - ([-_\.]?((a|b|c|rc|alpha|beta|pre|preview))[-_\.]?([0-9]+)?)? (?# pre-release) - ((?:-([0-9]+))|(?:[-_\.]?(post|rev|r)[-_\.]?([0-9]+)?))? (?# post release) - ([-_\.]?(dev)[-_\.]?([0-9]+)?)? (?# dev release) - (?:\+([a-z0-9]+(?:[-_\.][a-z0-9]+)*))? (?# local version) - )\z}xi.freeze - end - - def debian_package_name_regex - # See official parser - # https://git.dpkg.org/cgit/dpkg/dpkg.git/tree/lib/dpkg/parsehelp.c?id=9e0c88ec09475f4d1addde9cdba1ad7849720356#n122 - # @debian_package_name_regex ||= %r{\A[a-z0-9][-+\._a-z0-9]*\z}i.freeze - # But we prefer a more strict version from Lintian - # https://salsa.debian.org/lintian/lintian/-/blob/5080c0068ffc4a9ddee92022a91d0c2ff53e56d1/lib/Lintian/Util.pm#L116 - @debian_package_name_regex ||= %r{\A[a-z0-9][-+\.a-z0-9]+\z}.freeze - end - - def debian_version_regex - # See official parser: https://git.dpkg.org/cgit/dpkg/dpkg.git/tree/lib/dpkg/parsehelp.c?id=9e0c88ec09475f4d1addde9cdba1ad7849720356#n205 - @debian_version_regex ||= %r{ - \A(?: - (?:([0-9]{1,9}):)? (?# epoch) - ([0-9][0-9a-z\.+~]*) (?# version) - (-[0-9a-z\.+~]+){0,14} (?# -revision) - (?<!-) - )\z}xi.freeze - end - - def debian_architecture_regex - # See official parser: https://git.dpkg.org/cgit/dpkg/dpkg.git/tree/lib/dpkg/arch.c?id=9e0c88ec09475f4d1addde9cdba1ad7849720356#n43 - # But we limit to lower case - @debian_architecture_regex ||= %r{\A#{::Packages::Debian::ARCHITECTURE_REGEX}\z}o.freeze - end - - def debian_distribution_regex - @debian_distribution_regex ||= %r{\A#{::Packages::Debian::DISTRIBUTION_REGEX}\z}io.freeze - end - - def debian_component_regex - @debian_component_regex ||= %r{\A#{::Packages::Debian::COMPONENT_REGEX}\z}o.freeze - end - - def debian_direct_upload_filename_regex - @debian_direct_upload_filename_regex ||= %r{\A.*\.(deb|udeb|ddeb)\z}o.freeze - end - - def helm_channel_regex - @helm_channel_regex ||= %r{\A([a-zA-Z0-9](\.|-|_)?){1,255}(?<!\.|-|_)\z}.freeze - end - - def helm_package_regex - @helm_package_regex ||= %r{#{helm_channel_regex}}.freeze - end - - def helm_version_regex - # identical to semver_regex, with optional preceding 'v' - @helm_version_regex ||= Regexp.new("\\Av?#{::Gitlab::Regex.unbounded_semver_regex.source}\\z", ::Gitlab::Regex.unbounded_semver_regex.options) - end - - def unbounded_semver_regex - # See the official regex: https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string - - # The order of the alternatives in <prerelease> are intentionally - # reordered to be greedy. Without this change, the unbounded regex would - # only partially match "v0.0.0-20201230123456-abcdefabcdef". - @unbounded_semver_regex ||= / - #{_semver_major_minor_patch_regex}#{_semver_prerelease_build_regex} - /x.freeze - end - - def semver_regex - @semver_regex ||= Regexp.new("\\A#{::Gitlab::Regex.unbounded_semver_regex.source}\\z", ::Gitlab::Regex.unbounded_semver_regex.options).freeze - end - - def semver_regex_message - 'should follow SemVer: https://semver.org' - end - - # These partial semver regexes are intended for use in composing other - # regexes rather than being used alone. - def _semver_major_minor_patch_regex - @_semver_major_minor_patch_regex ||= / - #{_semver_major_regex}\.#{_semver_minor_regex}\.#{_semver_patch_regex} - /x.freeze - end - - def _semver_major_regex - @_semver_major_regex ||= / - (?<major>0|[1-9]\d*) - /x.freeze - end - - def _semver_minor_regex - @_semver_minor_regex ||= / - (?<minor>0|[1-9]\d*) - /x.freeze - end - - def _semver_patch_regex - @_semver_patch_regex ||= / - (?<patch>0|[1-9]\d*) - /x.freeze - end - - def _semver_prerelease_build_regex - @_semver_prerelease_build_regex ||= / - (?:-(?<prerelease>(?:\d*[a-zA-Z-][0-9a-zA-Z-]*|[1-9]\d*|0)(?:\.(?:\d*[a-zA-Z-][0-9a-zA-Z-]*|[1-9]\d*|0))*))? - (?:\+(?<build>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))? - /x.freeze - end - - def prefixed_semver_regex - # identical to semver_regex, except starting with 'v' - @prefixed_semver_regex ||= Regexp.new("\\Av#{::Gitlab::Regex.unbounded_semver_regex.source}\\z", ::Gitlab::Regex.unbounded_semver_regex.options) - end - - def go_package_regex - # A Go package name looks like a URL but is not; it: - # - Must not have a scheme, such as http:// or https:// - # - Must not have a port number, such as :8080 or :8443 - - @go_package_regex ||= %r{ - \b (?# word boundary) - (?<domain> - [0-9a-z](?:(?:-|[0-9a-z]){0,61}[0-9a-z])? (?# first domain) - (?:\.[0-9a-z](?:(?:-|[0-9a-z]){0,61}[0-9a-z])?)* (?# inner domains) - \.[a-z]{2,} (?# top-level domain) - ) - (?<path>/(?: - [-/$_.+!*'(),0-9a-z] (?# plain URL character) - | %[0-9a-f]{2})* (?# URL encoded character) - )? (?# path) - \b (?# word boundary) - }ix.freeze - end - - def generic_package_version_regex - maven_version_regex - end - - def generic_package_name_regex - maven_file_name_regex - end - - def generic_package_file_name_regex - generic_package_name_regex - end - - def sha256_regex - @sha256_regex ||= /\A[0-9a-f]{64}\z/i.freeze - end - - private - - def conan_name_regex - @conan_name_regex ||= %r{[a-zA-Z0-9_][a-zA-Z0-9_\+\.-]{1,49}}.freeze - end - end - - module BulkImports - def bulk_import_destination_namespace_path_regex - # This regexp validates the string conforms to rules for a destination_namespace path: - # i.e does not start with a non-alphanumeric character, - # contains only alphanumeric characters, forward slashes, periods, and underscores, - # does not end with a period or forward slash, and has a relative path structure - # with no http protocol chars or leading or trailing forward slashes - # eg 'source/full/path' or 'destination_namespace' not 'https://example.com/destination/namespace/path' - # the regex also allows for an empty string ('') to be accepted as this is allowed in - # a bulk_import POST request - @bulk_import_destination_namespace_path_regex ||= %r/((\A\z)|(\A[0-9a-z]*(-_.)?[0-9a-z])(\/?[0-9a-z]*[-_.]?[0-9a-z])+\z)/i - end - - def bulk_import_source_full_path_regex - # This regexp validates the string conforms to rules for a source_full_path path: - # i.e does not start with a non-alphanumeric character except for periods or underscores, - # contains only alphanumeric characters, forward slashes, periods, and underscores, - # does not end with a period or forward slash, and has a relative path structure - # with no http protocol chars or leading or trailing forward slashes - # eg 'source/full/path' or 'destination_namespace' not 'https://example.com/source/full/path' - @bulk_import_source_full_path_regex ||= %r/\A([.]?)[^\W](\/?([-_.+]*)*[0-9a-z][-_]*)+\z/i - end - - def bulk_import_source_full_path_regex_message - bulk_import_destination_namespace_path_regex_message - end - - def bulk_import_destination_namespace_path_regex_message - "must have a relative path structure " \ - "with no HTTP protocol characters, or leading or trailing forward slashes. " \ - "Path segments must not start or end with a special character, " \ - "and must not contain consecutive special characters." - end - end - extend self - extend Packages extend BulkImports + extend MergeRequests + extend Packages def group_path_regex # This regexp validates the string conforms to rules for a group slug: @@ -328,7 +30,7 @@ module Gitlab end def project_name_regex_message - "can contain only letters, digits, emojis, '_', '.', '+', dashes, or spaces. " \ + "can contain only letters, digits, emoji, '_', '.', '+', dashes, or spaces. " \ "It must start with a letter, digit, emoji, or '_'." end @@ -350,7 +52,7 @@ module Gitlab end def group_name_regex_message - "can contain only letters, digits, emojis, '_', '.', dash, space, parenthesis. " \ + "can contain only letters, digits, emoji, '_', '.', dash, space, parenthesis. " \ "It must start with letter, digit, emoji or '_'." end @@ -594,10 +296,6 @@ module Gitlab @utc_date_regex ||= /\A[0-9]{4}-[0-9]{2}-[0-9]{2}\z/.freeze end - def merge_request_draft - /\A(?i)(\[draft\]|\(draft\)|draft:)/ - end - def issue @issue ||= /(?<issue>\d+)(?<format>\+s{,1})?(?=\W|\z)/ end @@ -606,10 +304,6 @@ module Gitlab @work_item ||= /(?<work_item>\d+)(?<format>\+s{,1})?(?=\W|\z)/ end - def merge_request - @merge_request ||= /(?<merge_request>\d+)(?<format>\+s{,1})?/ - end - def base64_regex @base64_regex ||= %r{(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?}.freeze end @@ -618,11 +312,6 @@ module Gitlab /\A[a-z]([-_a-z0-9]*[a-z0-9])?\z/ end - def feature_flag_regex_message - "can contain only lowercase letters, digits, '_' and '-'. " \ - "Must start with a letter, and cannot end with '-' or '_'" - end - # One or more `part`s, separated by separator def sep_by_1(separator, part) %r(#{part} (#{separator} #{part})*)x @@ -632,10 +321,6 @@ module Gitlab @x509_subject_key_identifier_regex ||= /\A(?:\h{2}:)*\h{2}\z/.freeze end - def ml_model_version_regex - maven_version_regex - end - def ml_model_name_regex package_name_regex end diff --git a/lib/gitlab/regex/bulk_imports.rb b/lib/gitlab/regex/bulk_imports.rb new file mode 100644 index 00000000000..e9ec24b831f --- /dev/null +++ b/lib/gitlab/regex/bulk_imports.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module Regex + module BulkImports + def bulk_import_destination_namespace_path_regex + # This regexp validates the string conforms to rules for a destination_namespace path: + # i.e does not start with a non-alphanumeric character, + # contains only alphanumeric characters, forward slashes, periods, and underscores, + # does not end with a period or forward slash, and has a relative path structure + # with no http protocol chars or leading or trailing forward slashes + # eg 'source/full/path' or 'destination_namespace' not 'https://example.com/destination/namespace/path' + # the regex also allows for an empty string ('') to be accepted as this is allowed in + # a bulk_import POST request + @bulk_import_destination_namespace_path_regex ||= %r/((\A\z)|(\A[0-9a-z]*(-_.)?[0-9a-z])(\/?[0-9a-z]*[-_.]?[0-9a-z])+\z)/i + end + + def bulk_import_source_full_path_regex + # This regexp validates the string conforms to rules for a source_full_path path: + # i.e does not start with a non-alphanumeric character except for periods or underscores, + # contains only alphanumeric characters, forward slashes, periods, and underscores, + # does not end with a period or forward slash, and has a relative path structure + # with no http protocol chars or leading or trailing forward slashes + # eg 'source/full/path' or 'destination_namespace' not 'https://example.com/source/full/path' + @bulk_import_source_full_path_regex ||= %r/\A([.]?)[^\W](\/?([-_.+]*)*[0-9a-z][-_]*)+\z/i + end + + def bulk_import_source_full_path_regex_message + bulk_import_destination_namespace_path_regex_message + end + + def bulk_import_destination_namespace_path_regex_message + "must have a relative path structure " \ + "with no HTTP protocol characters, or leading or trailing forward slashes. " \ + "Path segments must not start or end with a special character, " \ + "and must not contain consecutive special characters." + end + end + end +end diff --git a/lib/gitlab/regex/merge_requests.rb b/lib/gitlab/regex/merge_requests.rb new file mode 100644 index 00000000000..0fb47e4fbfe --- /dev/null +++ b/lib/gitlab/regex/merge_requests.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Regex + module MergeRequests + def merge_request + @merge_request ||= /(?<merge_request>\d+)(?<format>\+s{,1})?/ + end + + def merge_request_draft + /\A(?i)(\[draft\]|\(draft\)|draft:)/ + end + + def git_diff_prefix + /\A@@( -\d+,\d+ \+\d+(,\d+)? )@@/ + end + end + end +end diff --git a/lib/gitlab/regex/packages.rb b/lib/gitlab/regex/packages.rb new file mode 100644 index 00000000000..107f2070801 --- /dev/null +++ b/lib/gitlab/regex/packages.rb @@ -0,0 +1,273 @@ +# frozen_string_literal: true + +module Gitlab + module Regex + module Packages + CONAN_RECIPE_FILES = %w[conanfile.py conanmanifest.txt conan_sources.tgz conan_export.tgz].freeze + CONAN_PACKAGE_FILES = %w[conaninfo.txt conanmanifest.txt conan_package.tgz].freeze + + PYPI_NORMALIZED_NAME_REGEX_STRING = '[-_.]+' + + # see https://github.com/apache/maven/blob/c1dfb947b509e195c75d4275a113598cf3063c3e/maven-artifact/src/main/java/org/apache/maven/artifact/Artifact.java#L46 + MAVEN_SNAPSHOT_DYNAMIC_PARTS = /\A.{0,1000}(-\d{8}\.\d{6}-\d+).{0,1000}\z/.freeze + + API_PATH_REGEX = %r{^/api/v\d+/(projects/[^/]+/|groups?/[^/]+/-/)?packages/[A-Za-z]+}.freeze + + def conan_package_reference_regex + @conan_package_reference_regex ||= %r{\A[A-Za-z0-9]+\z}.freeze + end + + def conan_revision_regex + @conan_revision_regex ||= %r{\A0\z}.freeze + end + + def conan_recipe_user_channel_regex + %r{\A(_|#{conan_name_regex})\z}.freeze + end + + def conan_recipe_component_regex + # https://docs.conan.io/en/latest/reference/conanfile/attributes.html#name + @conan_recipe_component_regex ||= %r{\A#{conan_name_regex}\z}.freeze + end + + def composer_package_version_regex + # see https://github.com/composer/semver/blob/31f3ea725711245195f62e54ffa402d8ef2fdba9/src/VersionParser.php#L215 + @composer_package_version_regex ||= %r{\Av?((\d++)(\.(?:\d++|[xX*]))?(\.(?:\d++|[xX*]))?(\.(?:\d++|[xX*]))?)?\z}.freeze + end + + def composer_dev_version_regex + @composer_dev_version_regex ||= %r{(^dev-)|(-dev$)}.freeze + end + + def package_name_regex + @package_name_regex ||= + %r{ + \A\@? + (?> # atomic group to prevent backtracking + (([\w\-\.\+]*)\/)*([\w\-\.]+) + ) + @? + (?> # atomic group to prevent backtracking + (([\w\-\.\+]*)\/)*([\w\-\.]*) + ) + \z + }x.freeze + end + + def maven_file_name_regex + @maven_file_name_regex ||= %r{\A[A-Za-z0-9\.\_\-\+]+\z}.freeze + end + + def maven_path_regex + @maven_path_regex ||= %r{\A\@?(([\w\-\.]*)/)*([\w\-\.\+]*)\z}.freeze + end + + def maven_app_name_regex + @maven_app_name_regex ||= /\A[\w\-\.]+\z/.freeze + end + + def maven_version_regex + @maven_version_regex ||= /\A(?!.*\.\.)[\w+.-]+\z/.freeze + end + + def maven_app_group_regex + maven_app_name_regex + end + + def npm_package_name_regex + @npm_package_name_regex ||= %r{\A(?:@(#{Gitlab::PathRegex::NAMESPACE_FORMAT_REGEX})/)?[-+\.\_a-zA-Z0-9]+\z}o + end + + def npm_package_name_regex_message + 'should be a valid NPM package name: https://github.com/npm/validate-npm-package-name#naming-rules.' + end + + def nuget_package_name_regex + @nuget_package_name_regex ||= %r{\A[-+\.\_a-zA-Z0-9]+\z}.freeze + end + + def nuget_version_regex + @nuget_version_regex ||= / + \A#{_semver_major_regex} + \.#{_semver_minor_regex} + (\.#{_semver_patch_regex})? + (\.\d*)? + #{_semver_prerelease_build_regex}\z + /x.freeze + end + + def terraform_module_package_name_regex + @terraform_module_package_name_regex ||= %r{\A[-a-z0-9]+\/[-a-z0-9]+\z}.freeze + end + + def pypi_version_regex + # See the official regex: https://github.com/pypa/packaging/blob/16.7/packaging/version.py#L159 + + @pypi_version_regex ||= %r{ + \A(?: + v? + (?:([0-9]+)!)? (?# epoch) + ([0-9]+(?:\.[0-9]+)*) (?# release segment) + ([-_\.]?((a|b|c|rc|alpha|beta|pre|preview))[-_\.]?([0-9]+)?)? (?# pre-release) + ((?:-([0-9]+))|(?:[-_\.]?(post|rev|r)[-_\.]?([0-9]+)?))? (?# post release) + ([-_\.]?(dev)[-_\.]?([0-9]+)?)? (?# dev release) + (?:\+([a-z0-9]+(?:[-_\.][a-z0-9]+)*))? (?# local version) + )\z}xi.freeze + end + + def debian_package_name_regex + # See official parser + # https://git.dpkg.org/cgit/dpkg/dpkg.git/tree/lib/dpkg/parsehelp.c?id=9e0c88ec09475f4d1addde9cdba1ad7849720356#n122 + # @debian_package_name_regex ||= %r{\A[a-z0-9][-+\._a-z0-9]*\z}i.freeze + # But we prefer a more strict version from Lintian + # https://salsa.debian.org/lintian/lintian/-/blob/5080c0068ffc4a9ddee92022a91d0c2ff53e56d1/lib/Lintian/Util.pm#L116 + @debian_package_name_regex ||= %r{\A[a-z0-9][-+\.a-z0-9]+\z}.freeze + end + + def debian_version_regex + # See official parser: https://git.dpkg.org/cgit/dpkg/dpkg.git/tree/lib/dpkg/parsehelp.c?id=9e0c88ec09475f4d1addde9cdba1ad7849720356#n205 + @debian_version_regex ||= %r{ + \A(?: + (?:([0-9]{1,9}):)? (?# epoch) + ([0-9][0-9a-z\.+~]*) (?# version) + (-[0-9a-z\.+~]+){0,14} (?# -revision) + (?<!-) + )\z}xi.freeze + end + + def debian_architecture_regex + # See official parser: https://git.dpkg.org/cgit/dpkg/dpkg.git/tree/lib/dpkg/arch.c?id=9e0c88ec09475f4d1addde9cdba1ad7849720356#n43 + # But we limit to lower case + @debian_architecture_regex ||= %r{\A#{::Packages::Debian::ARCHITECTURE_REGEX}\z}o.freeze + end + + def debian_distribution_regex + @debian_distribution_regex ||= %r{\A#{::Packages::Debian::DISTRIBUTION_REGEX}\z}io.freeze + end + + def debian_component_regex + @debian_component_regex ||= %r{\A#{::Packages::Debian::COMPONENT_REGEX}\z}o.freeze + end + + def debian_direct_upload_filename_regex + @debian_direct_upload_filename_regex ||= %r{\A.*\.(deb|udeb|ddeb)\z}o.freeze + end + + def helm_channel_regex + @helm_channel_regex ||= %r{\A([a-zA-Z0-9](\.|-|_)?){1,255}(?<!\.|-|_)\z}.freeze + end + + def helm_package_regex + @helm_package_regex ||= %r{#{helm_channel_regex}}.freeze + end + + def helm_version_regex + # identical to semver_regex, with optional preceding 'v' + @helm_version_regex ||= Regexp.new("\\Av?#{::Gitlab::Regex.unbounded_semver_regex.source}\\z", ::Gitlab::Regex.unbounded_semver_regex.options) + end + + def unbounded_semver_regex + # See the official regex: https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string + + # The order of the alternatives in <prerelease> are intentionally + # reordered to be greedy. Without this change, the unbounded regex would + # only partially match "v0.0.0-20201230123456-abcdefabcdef". + @unbounded_semver_regex ||= / + #{_semver_major_minor_patch_regex}#{_semver_prerelease_build_regex} + /x.freeze + end + + def semver_regex + @semver_regex ||= Regexp.new("\\A#{::Gitlab::Regex.unbounded_semver_regex.source}\\z", ::Gitlab::Regex.unbounded_semver_regex.options).freeze + end + + def semver_regex_message + 'should follow SemVer: https://semver.org' + end + + # These partial semver regexes are intended for use in composing other + # regexes rather than being used alone. + def _semver_major_minor_patch_regex + @_semver_major_minor_patch_regex ||= / + #{_semver_major_regex}\.#{_semver_minor_regex}\.#{_semver_patch_regex} + /x.freeze + end + + def _semver_major_regex + @_semver_major_regex ||= / + (?<major>0|[1-9]\d*) + /x.freeze + end + + def _semver_minor_regex + @_semver_minor_regex ||= / + (?<minor>0|[1-9]\d*) + /x.freeze + end + + def _semver_patch_regex + @_semver_patch_regex ||= / + (?<patch>0|[1-9]\d*) + /x.freeze + end + + def _semver_prerelease_build_regex + @_semver_prerelease_build_regex ||= / + (?:-(?<prerelease>(?:\d*[a-zA-Z-][0-9a-zA-Z-]*|[1-9]\d*|0)(?:\.(?:\d*[a-zA-Z-][0-9a-zA-Z-]*|[1-9]\d*|0))*))? + (?:\+(?<build>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))? + /x.freeze + end + + def prefixed_semver_regex + # identical to semver_regex, except starting with 'v' + @prefixed_semver_regex ||= Regexp.new("\\Av#{::Gitlab::Regex.unbounded_semver_regex.source}\\z", ::Gitlab::Regex.unbounded_semver_regex.options) + end + + def go_package_regex + # A Go package name looks like a URL but is not; it: + # - Must not have a scheme, such as http:// or https:// + # - Must not have a port number, such as :8080 or :8443 + + @go_package_regex ||= %r{ + \b (?# word boundary) + (?<domain> + [0-9a-z](?:(?:-|[0-9a-z]){0,61}[0-9a-z])? (?# first domain) + (?:\.[0-9a-z](?:(?:-|[0-9a-z]){0,61}[0-9a-z])?)* (?# inner domains) + \.[a-z]{2,} (?# top-level domain) + ) + (?<path>/(?: + [-/$_.+!*'(),0-9a-z] (?# plain URL character) + | %[0-9a-f]{2})* (?# URL encoded character) + )? (?# path) + \b (?# word boundary) + }ix.freeze + end + + def generic_package_version_regex + maven_version_regex + end + + def generic_package_name_regex + maven_file_name_regex + end + + def generic_package_file_name_regex + generic_package_name_regex + end + + def sha256_regex + @sha256_regex ||= /\A[0-9a-f]{64}\z/i.freeze + end + + def slack_link_regex + @slack_link_regex ||= /<(.*[|].*)>/i.freeze + end + + private + + def conan_name_regex + @conan_name_regex ||= %r{[a-zA-Z0-9_][a-zA-Z0-9_\+\.-]{1,49}}.freeze + end + end + end +end diff --git a/lib/gitlab/repository_size_checker.rb b/lib/gitlab/repository_size_checker.rb index 2afc5e8d668..6ccdf1f4354 100644 --- a/lib/gitlab/repository_size_checker.rb +++ b/lib/gitlab/repository_size_checker.rb @@ -29,7 +29,7 @@ module Gitlab end # @param change_size [int] in bytes - def changes_will_exceed_size_limit?(change_size) + def changes_will_exceed_size_limit?(change_size, _project) return false unless enabled? above_size_limit? || exceeded_size(change_size) > 0 diff --git a/lib/gitlab/repository_size_error_message.rb b/lib/gitlab/repository_size_error_message.rb index e7d527dd4ce..fc700c3c62e 100644 --- a/lib/gitlab/repository_size_error_message.rb +++ b/lib/gitlab/repository_size_error_message.rb @@ -7,8 +7,7 @@ module Gitlab delegate :current_size, :limit, :exceeded_size, :additional_repo_storage_available?, to: :@checker # @param checker [RepositorySizeChecker] - def initialize(checker, message_params = {}) - @message_params = message_params + def initialize(checker) @checker = checker end @@ -20,14 +19,6 @@ module Gitlab "This merge request cannot be merged, #{base_message}" end - def push_warning - _("##### WARNING ##### You have used %{usage_percentage} of the storage quota for %{namespace_name} " \ - "(%{current_size} of %{size_limit}). If %{namespace_name} exceeds the storage quota, " \ - "all projects in the namespace will be locked and actions will be restricted. " \ - "To manage storage, or purchase additional storage, see %{manage_storage_url}. " \ - "To learn more about restricted actions, see %{restricted_actions_url}") % push_message_params - end - def push_error(change_size = 0) "Your push has been rejected, #{base_message(change_size)}. #{more_info_message}" end @@ -50,19 +41,6 @@ module Gitlab private - attr_reader :message_params - - def push_message_params - { - namespace_name: message_params[:namespace_name], - manage_storage_url: help_page_url('user/usage_quotas', 'manage-your-storage-usage'), - restricted_actions_url: help_page_url('user/read_only_namespaces', 'restricted-actions'), - current_size: formatted(current_size), - size_limit: formatted(limit), - usage_percentage: usage_percentage - } - end - def base_message(change_size = 0) "because this repository has exceeded its size limit of #{formatted(limit)} by #{formatted(exceeded_size(change_size))}" end @@ -70,13 +48,5 @@ module Gitlab def formatted(number) number_to_human_size(number, delimiter: ',', precision: 2) end - - def usage_percentage - number_to_percentage(@checker.usage_ratio * 100, precision: 0) - end - - def help_page_url(path, anchor = nil) - ::Gitlab::Routing.url_helpers.help_page_url(path, anchor: anchor) - end end end diff --git a/lib/gitlab/safe_request_store.rb b/lib/gitlab/safe_request_store.rb deleted file mode 100644 index 203d7d10532..00000000000 --- a/lib/gitlab/safe_request_store.rb +++ /dev/null @@ -1,46 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module SafeRequestStore - NULL_STORE = Gitlab::NullRequestStore.new - - class << self - # These methods should always run directly against RequestStore - delegate :clear!, :begin!, :end!, :active?, to: :RequestStore - - # These methods will run against NullRequestStore if RequestStore is disabled - delegate :read, :[], :write, :[]=, :exist?, :fetch, :delete, to: :store - end - - def self.store - if RequestStore.active? - RequestStore - else - NULL_STORE - end - end - - # Access to the backing storage of the request store. This returns an object - # with `[]` and `[]=` methods that does not discard values. - # - # This can be useful if storage is needed for a delimited purpose, and the - # forgetful nature of the null store is undesirable. - def self.storage - store.store - end - - # This method accept an options hash to be compatible with - # ActiveSupport::Cache::Store#write method. The options are - # not passed to the underlying cache implementation because - # RequestStore#write accepts only a key, and value params. - def self.write(key, value, options = nil) - store.write(key, value) - end - - def self.delete_if(&block) - return unless RequestStore.active? - - storage.delete_if { |k, v| yield(k) } - end - end -end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 4fedc450f9b..0e419d0162c 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -3,7 +3,7 @@ module Gitlab class SearchResults COUNT_LIMIT = 100 - COUNT_LIMIT_MESSAGE = "#{COUNT_LIMIT - 1}+" + COUNT_LIMIT_MESSAGE = "#{COUNT_LIMIT - 1}+".freeze DEFAULT_PAGE = 1 DEFAULT_PER_PAGE = 20 @@ -24,7 +24,14 @@ module Gitlab # query attr_reader :default_project_filter - def initialize(current_user, query, limit_projects = nil, order_by: nil, sort: nil, default_project_filter: false, filters: {}) + def initialize( + current_user, + query, + limit_projects = nil, + order_by: nil, + sort: nil, + default_project_filter: false, + filters: {}) @current_user = current_user @query = query @limit_projects = limit_projects || Project.all @@ -111,12 +118,12 @@ module Gitlab end # highlighting is only performed by Elasticsearch backed results - def highlight_map(scope) + def highlight_map(_scope) {} end # aggregations are only performed by Elasticsearch backed results - def aggregations(scope) + def aggregations(_scope) [] end @@ -152,13 +159,11 @@ module Gitlab sort_by = ::Gitlab::Search::SortOptions.sort_and_direction(order_by, sort) # Reset sort to default if the chosen one is not supported by scope - sort_by = nil if SCOPE_ONLY_SORT[sort_by] && !SCOPE_ONLY_SORT[sort_by].include?(scope) + sort_by = nil if SCOPE_ONLY_SORT[sort_by] && SCOPE_ONLY_SORT[sort_by].exclude?(scope) case sort_by when :created_at_asc results.reorder('created_at ASC') - when :created_at_desc - results.reorder('created_at DESC') when :updated_at_asc results.reorder('updated_at ASC') when :updated_at_desc @@ -168,6 +173,7 @@ module Gitlab when :popularity_desc results.reorder('upvotes_count DESC') else + # :created_at_desc is default results.reorder('created_at DESC') end end @@ -175,7 +181,11 @@ module Gitlab def projects scope = limit_projects - scope = scope.non_archived if Feature.enabled?(:search_projects_hide_archived) && !filters[:include_archived] + + if Feature.enabled?(:search_projects_hide_archived, current_user) && !filters[:include_archived] + scope = scope.non_archived + end + scope.search(query) end @@ -184,6 +194,7 @@ module Gitlab unless default_project_filter issues = issues.in_projects(project_ids_relation) + .allow_cross_joins_across_databases(url: "https://gitlab.com/gitlab-org/gitlab/-/issues/420046") end apply_sort(issues, scope: 'issues') @@ -203,7 +214,13 @@ module Gitlab merge_requests = MergeRequestsFinder.new(current_user, issuable_params).execute unless default_project_filter - merge_requests = merge_requests.of_projects(project_ids_relation) + project_ids = project_ids_relation + + if Feature.enabled?(:search_merge_requests_hide_archived_projects, current_user) && !filters[:include_archived] + project_ids = project_ids.non_archived + end + + merge_requests = merge_requests.of_projects(project_ids) end apply_sort(merge_requests, scope: 'merge_requests') @@ -251,9 +268,7 @@ module Gitlab params[:state] = filters[:state] if filters.key?(:state) - if [true, false].include?(filters[:confidential]) - params[:confidential] = filters[:confidential] - end + params[:confidential] = filters[:confidential] if [true, false].include?(filters[:confidential]) end end diff --git a/lib/gitlab/sidekiq_logging/logs_jobs.rb b/lib/gitlab/sidekiq_logging/logs_jobs.rb index 3e6e6e05e95..edf2cffa0c9 100644 --- a/lib/gitlab/sidekiq_logging/logs_jobs.rb +++ b/lib/gitlab/sidekiq_logging/logs_jobs.rb @@ -12,7 +12,8 @@ module Gitlab # Error information from the previous try is in the payload for # displaying in the Sidekiq UI, but is very confusing in logs! job = job.except( - 'exception.backtrace', 'exception.class', 'exception.message', 'exception.sql' + 'exception.backtrace', 'exception.class', 'exception.message', 'exception.sql', + 'error_message', 'error_class', 'error_backtrace', 'failed_at' ) job['class'] = job.delete('wrapped') if job['wrapped'].present? diff --git a/lib/gitlab/sidekiq_logging/pause_control_logger.rb b/lib/gitlab/sidekiq_logging/pause_control_logger.rb new file mode 100644 index 00000000000..d48b2b12f9d --- /dev/null +++ b/lib/gitlab/sidekiq_logging/pause_control_logger.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqLogging + class PauseControlLogger + include Singleton + include LogsJobs + + def paused_log(job, strategy:) + payload = parse_job(job) + payload['job_status'] = 'paused' + payload['message'] = "#{base_message(payload)}: paused: #{strategy}" + payload['pause_control.strategy'] = strategy + + Sidekiq.logger.info payload + end + + def resumed_log(worker_name, args) + job = { + 'class' => worker_name, + 'args' => args + } + payload = parse_job(job) + payload['job_status'] = 'resumed' + payload['message'] = "#{base_message(payload)}: resumed" + + Sidekiq.logger.info payload + end + end + end +end diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb index 56762c0fb4b..c65d9c5ddd5 100644 --- a/lib/gitlab/sidekiq_logging/structured_logger.rb +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -100,6 +100,8 @@ module Gitlab unless job_urgency.empty? payload['urgency'] = job_urgency payload['target_duration_s'] = Gitlab::Metrics::SidekiqSlis.execution_duration_for_urgency(job_urgency) + payload['target_scheduling_latency_s'] = + Gitlab::Metrics::SidekiqSlis.queueing_duration_for_urgency(job_urgency) end payload diff --git a/lib/gitlab/sidekiq_middleware.rb b/lib/gitlab/sidekiq_middleware.rb index 614cd11421e..e1c155a4848 100644 --- a/lib/gitlab/sidekiq_middleware.rb +++ b/lib/gitlab/sidekiq_middleware.rb @@ -36,6 +36,7 @@ module Gitlab chain.add ::Gitlab::SidekiqVersioning::Middleware chain.add ::Gitlab::SidekiqStatus::ServerMiddleware chain.add ::Gitlab::SidekiqMiddleware::WorkerContext::Server + chain.add ::Gitlab::SidekiqMiddleware::PauseControl::Server # DuplicateJobs::Server should be placed at the bottom, but before the SidekiqServerMiddleware, # so we can compare the latest WAL location against replica chain.add ::Gitlab::SidekiqMiddleware::DuplicateJobs::Server @@ -54,6 +55,7 @@ module Gitlab # Sidekiq Client Middleware should be placed before DuplicateJobs::Client middleware, # so we can store WAL location before we deduplicate the job. chain.add ::Gitlab::Database::LoadBalancing::SidekiqClientMiddleware + chain.add ::Gitlab::SidekiqMiddleware::PauseControl::Client chain.add ::Gitlab::SidekiqMiddleware::DuplicateJobs::Client chain.add ::Gitlab::SidekiqStatus::ClientMiddleware chain.add ::Gitlab::SidekiqMiddleware::AdminMode::Client diff --git a/lib/gitlab/sidekiq_middleware/pause_control.rb b/lib/gitlab/sidekiq_middleware/pause_control.rb new file mode 100644 index 00000000000..2f0fd0cc799 --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/pause_control.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + module PauseControl + DEFAULT_STRATEGY = :none + + UnknownStrategyError = Class.new(StandardError) + + STRATEGIES = { + zoekt: ::Gitlab::SidekiqMiddleware::PauseControl::Strategies::Zoekt, + none: ::Gitlab::SidekiqMiddleware::PauseControl::Strategies::None + }.freeze + + def self.for(name) + STRATEGIES.fetch(name, STRATEGIES[DEFAULT_STRATEGY]) + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/pause_control/client.rb b/lib/gitlab/sidekiq_middleware/pause_control/client.rb new file mode 100644 index 00000000000..406a956e9ff --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/pause_control/client.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + module PauseControl + class Client + def call(worker_class, job, _queue, _redis_pool, &block) + ::Gitlab::SidekiqMiddleware::PauseControl::StrategyHandler.new(worker_class, job).schedule(&block) + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/pause_control/pause_control_service.rb b/lib/gitlab/sidekiq_middleware/pause_control/pause_control_service.rb new file mode 100644 index 00000000000..73f42beaf9e --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/pause_control/pause_control_service.rb @@ -0,0 +1,118 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + module PauseControl + class PauseControlService + # Class for managing queues for paused workers + # When a worker is paused all jobs are saved in a separate sorted sets in redis + LIMIT = 1000 + PROJECT_CONTEXT_KEY = "#{Gitlab::ApplicationContext::LOG_KEY}.project".freeze + + def initialize(worker_name) + @worker_name = worker_name + + worker_name = @worker_name.underscore + @redis_set_key = "sidekiq:pause_control:paused_jobs:zset:{#{worker_name}}" + @redis_score_key = "sidekiq:pause_control:paused_jobs:score:{#{worker_name}}" + end + + class << self + def add_to_waiting_queue!(worker_name, args, context) + new(worker_name).add_to_waiting_queue!(args, context) + end + + def has_jobs_in_waiting_queue?(worker_name) + new(worker_name).has_jobs_in_waiting_queue? + end + + def resume_processing!(worker_name) + new(worker_name).resume_processing! + end + + def queue_size(worker_name) + new(worker_name).queue_size + end + end + + def add_to_waiting_queue!(args, context) + with_redis do |redis| + redis.zadd(redis_set_key, generate_unique_score(redis), serialize(args, context)) + end + end + + def queue_size + with_redis { |redis| redis.zcard(redis_set_key) } + end + + def has_jobs_in_waiting_queue? + with_redis { |redis| redis.exists?(redis_set_key) } # rubocop:disable CodeReuse/ActiveRecord + end + + def resume_processing!(iterations: 1) + with_redis do |redis| + iterations.times do + jobs_with_scores = next_batch_from_waiting_queue(redis) + break if jobs_with_scores.empty? + + parsed_jobs = jobs_with_scores.map { |j, _| deserialize(j) } + + parsed_jobs.each { |j| send_to_processing_queue(j) } + + remove_jobs_from_waiting_queue(redis, jobs_with_scores) + end + + size = queue_size + redis.del(redis_score_key, redis_set_key) if size == 0 + + size + end + end + + private + + attr_reader :worker_name, :redis_set_key, :redis_score_key + + def with_redis(&blk) + Gitlab::Redis::SharedState.with(&blk) # rubocop:disable CodeReuse/ActiveRecord + end + + def serialize(args, context) + { + args: args, + # Only include part of the context that would not prevent deduplication + context: context.slice(PROJECT_CONTEXT_KEY) + }.to_json + end + + def deserialize(json) + Gitlab::Json.parse(json) + end + + def send_to_processing_queue(job) + Gitlab::ApplicationContext.with_raw_context(job['context']) do + args = job['args'] + + Gitlab::SidekiqLogging::PauseControlLogger.instance.resumed_log(worker_name, args) + + worker_name.safe_constantize&.perform_async(*args) + end + end + + def generate_unique_score(redis) + redis.incr(redis_score_key) + end + + def next_batch_from_waiting_queue(redis) + redis.zrangebyscore(redis_set_key, '-inf', '+inf', limit: [0, LIMIT], with_scores: true) + end + + def remove_jobs_from_waiting_queue(redis, jobs_with_scores) + first_score = jobs_with_scores.first.last + last_score = jobs_with_scores.last.last + redis.zremrangebyscore(redis_set_key, first_score, last_score) + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/pause_control/server.rb b/lib/gitlab/sidekiq_middleware/pause_control/server.rb new file mode 100644 index 00000000000..cfa02b3ec3a --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/pause_control/server.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + module PauseControl + class Server + def call(worker_class, job, _queue, &block) + ::Gitlab::SidekiqMiddleware::PauseControl::StrategyHandler.new(worker_class, job).perform(&block) + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/pause_control/strategies/base.rb b/lib/gitlab/sidekiq_middleware/pause_control/strategies/base.rb new file mode 100644 index 00000000000..d92cbccc94e --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/pause_control/strategies/base.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + module PauseControl + module Strategies + class Base + extend ::Gitlab::Utils::Override + + def self.should_pause? + new.should_pause? + end + + def schedule(job) + if should_pause? + pause_job!(job) + + return + end + + yield + end + + def perform(job) + if should_pause? + pause_job!(job) + + return + end + + yield + end + + def should_pause? + # All children must implement this method + # return false when the jobs shouldn't be paused and true when it should + # A cron job PauseControl::ResumeWorker will execute this method to check if jobs should remain paused + raise NotImplementedError + end + + private + + def pause_job!(job) + Gitlab::SidekiqLogging::PauseControlLogger.instance.paused_log(job, strategy: strategy_name) + + Gitlab::SidekiqMiddleware::PauseControl::PauseControlService.add_to_waiting_queue!( + job['class'], + job['args'], + current_context + ) + end + + def strategy_name + Gitlab::SidekiqMiddleware::PauseControl::STRATEGIES.key(self.class) + end + + def current_context + Gitlab::ApplicationContext.current + end + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/pause_control/strategies/none.rb b/lib/gitlab/sidekiq_middleware/pause_control/strategies/none.rb new file mode 100644 index 00000000000..c31f0a9918e --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/pause_control/strategies/none.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + module PauseControl + module Strategies + # This strategy will never pause a job + class None < Base + override :should_pause? + def should_pause? + false + end + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/pause_control/strategies/zoekt.rb b/lib/gitlab/sidekiq_middleware/pause_control/strategies/zoekt.rb new file mode 100644 index 00000000000..23cba5553e2 --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/pause_control/strategies/zoekt.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + module PauseControl + module Strategies + class Zoekt < Base + override :should_pause? + def should_pause? + ::Feature.enabled?(:zoekt_pause_indexing, type: :ops) + end + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/pause_control/strategy_handler.rb b/lib/gitlab/sidekiq_middleware/pause_control/strategy_handler.rb new file mode 100644 index 00000000000..93c668052b0 --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/pause_control/strategy_handler.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + module PauseControl + class StrategyHandler + def initialize(worker_class, job) + @worker_class = worker_class + @job = job + end + + # This will continue the middleware chain if the job should be scheduled + # It will return false if the job needs to be cancelled + def schedule(&block) + PauseControl.for(strategy).new.schedule(job, &block) + end + + # This will continue the server middleware chain if the job should be + # executed. + # It will return false if the job should not be executed. + def perform(&block) + PauseControl.for(strategy).new.perform(job, &block) + end + + private + + attr_reader :job, :worker_class + + def strategy + Gitlab::SidekiqMiddleware::PauseControl::WorkersMap.strategy_for(worker: worker_class) + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/pause_control/workers_map.rb b/lib/gitlab/sidekiq_middleware/pause_control/workers_map.rb new file mode 100644 index 00000000000..dc6aff92f50 --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/pause_control/workers_map.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + module PauseControl + class WorkersMap + class << self + attr_reader :workers + + def set_strategy_for(strategy:, worker:) + raise ArgumentError, "Unknown strategy: #{strategy}" unless PauseControl::STRATEGIES.key?(strategy) + + @workers ||= Hash.new { |h, k| h[k] = [] } + @workers[strategy].push(worker) + end + + def strategy_for(worker:) + return unless @workers + + @workers.find { |_, v| v.include?(worker) }&.first + end + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/request_store_middleware.rb b/lib/gitlab/sidekiq_middleware/request_store_middleware.rb index f6142bd6ca5..32314bbe0f8 100644 --- a/lib/gitlab/sidekiq_middleware/request_store_middleware.rb +++ b/lib/gitlab/sidekiq_middleware/request_store_middleware.rb @@ -3,10 +3,8 @@ module Gitlab module SidekiqMiddleware class RequestStoreMiddleware - include Gitlab::WithRequestStore - def call(worker, job, queue) - with_request_store do + ::Gitlab::SafeRequestStore.ensure_request_store do yield end end diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb index 058c23178f8..a8b3683e09f 100644 --- a/lib/gitlab/sidekiq_middleware/server_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb @@ -11,28 +11,25 @@ module Gitlab # most of the durations for cpu, gitaly, db and elasticsearch SIDEKIQ_LATENCY_BUCKETS = [0.1, 0.5, 1, 2.5].freeze - # These are the buckets we currently use for alerting, we will likely - # replace these histograms with Application SLIs - # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1313 + # These buckets are only available on self-managed. + # We have replaced with Application SLIs on GitLab.com. + # https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/700 SIDEKIQ_JOB_DURATION_BUCKETS = [10, 300].freeze SIDEKIQ_QUEUE_DURATION_BUCKETS = [10, 60].freeze # These labels from Gitlab::SidekiqMiddleware::MetricsHelper are included in SLI metrics - SIDEKIQ_SLI_LABELS = [:worker, :feature_category, :urgency, :external_dependencies].freeze + SIDEKIQ_SLI_LABELS = [:worker, :feature_category, :urgency, :external_dependencies, :queue].freeze class << self include ::Gitlab::SidekiqMiddleware::MetricsHelper def metrics - { + metrics = { sidekiq_jobs_cpu_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_cpu_seconds, 'Seconds this Sidekiq job spent on the CPU', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_completion_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete Sidekiq job', {}, SIDEKIQ_JOB_DURATION_BUCKETS), sidekiq_jobs_db_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_db_seconds, 'Seconds of database time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), sidekiq_jobs_gitaly_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_gitaly_seconds, 'Seconds of Gitaly time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_queue_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_queue_duration_seconds, 'Duration in seconds that a Sidekiq job was queued before being executed', {}, SIDEKIQ_QUEUE_DURATION_BUCKETS), sidekiq_redis_requests_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_redis_requests_duration_seconds, 'Duration in seconds that a Sidekiq job spent requests a Redis server', {}, Gitlab::Instrumentation::Redis::QUERY_TIME_BUCKETS), sidekiq_elasticsearch_requests_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_elasticsearch_requests_duration_seconds, 'Duration in seconds that a Sidekiq job spent in requests to an Elasticsearch server', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_failed_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_failed_total, 'Sidekiq jobs failed'), sidekiq_jobs_retried_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_retried_total, 'Sidekiq jobs retried'), sidekiq_jobs_interrupted_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_interrupted_total, 'Sidekiq jobs interrupted'), sidekiq_redis_requests_total: ::Gitlab::Metrics.counter(:sidekiq_redis_requests_total, 'Redis requests during a Sidekiq job execution'), @@ -41,6 +38,17 @@ module Gitlab sidekiq_concurrency: ::Gitlab::Metrics.gauge(:sidekiq_concurrency, 'Maximum number of Sidekiq jobs', {}, :all), sidekiq_mem_total_bytes: ::Gitlab::Metrics.gauge(:sidekiq_mem_total_bytes, 'Number of bytes allocated for both objects consuming an object slot and objects that required a malloc', {}, :all) } + + if Feature.enabled?(:emit_sidekiq_histogram_metrics, type: :ops) + metrics[:sidekiq_jobs_completion_seconds] = ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete Sidekiq job', {}, SIDEKIQ_JOB_DURATION_BUCKETS) + metrics[:sidekiq_jobs_queue_duration_seconds] = ::Gitlab::Metrics.histogram(:sidekiq_jobs_queue_duration_seconds, 'Duration in seconds that a Sidekiq job was queued before being executed', {}, SIDEKIQ_QUEUE_DURATION_BUCKETS) + metrics[:sidekiq_jobs_failed_total] = ::Gitlab::Metrics.counter(:sidekiq_jobs_failed_total, 'Sidekiq jobs failed') + else + # The sum metric is still used in GitLab.com for dashboards + metrics[:sidekiq_jobs_completion_seconds_sum] = ::Gitlab::Metrics.counter(:sidekiq_jobs_completion_seconds_sum, 'Total of seconds to complete Sidekiq job') + end + + metrics end def initialize_process_metrics @@ -59,13 +67,15 @@ module Gitlab base_labels = create_labels(worker_class, queue, {}) possible_sli_labels << base_labels.slice(*SIDEKIQ_SLI_LABELS) + next unless Feature.enabled?(:emit_sidekiq_histogram_metrics, type: :ops) + %w[done fail].each do |status| metrics[:sidekiq_jobs_completion_seconds].get(base_labels.merge(job_status: status)) end end - Gitlab::Metrics::SidekiqSlis.initialize_execution_slis!(possible_sli_labels) if ::Feature.enabled?(:sidekiq_execution_application_slis) - Gitlab::Metrics::SidekiqSlis.initialize_queueing_slis!(possible_sli_labels) if ::Feature.enabled?(:sidekiq_queueing_application_slis) + Gitlab::Metrics::SidekiqSlis.initialize_execution_slis!(possible_sli_labels) + Gitlab::Metrics::SidekiqSlis.initialize_queueing_slis!(possible_sli_labels) end end @@ -92,7 +102,8 @@ module Gitlab def instrument(job, labels) queue_duration = ::Gitlab::InstrumentationHelper.queue_duration_for_job(job) - @metrics[:sidekiq_jobs_queue_duration_seconds].observe(labels, queue_duration) if queue_duration + @metrics[:sidekiq_jobs_queue_duration_seconds]&.observe(labels, queue_duration) if queue_duration + @metrics[:sidekiq_running_jobs].increment(labels, 1) if job['retry_count'].present? @@ -119,13 +130,21 @@ module Gitlab # sidekiq_running_jobs, sidekiq_jobs_failed_total should not include the job_status label @metrics[:sidekiq_running_jobs].increment(labels, -1) - @metrics[:sidekiq_jobs_failed_total].increment(labels, 1) unless job_succeeded + + if Feature.enabled?(:emit_sidekiq_histogram_metrics, type: :ops) + @metrics[:sidekiq_jobs_failed_total].increment(labels, 1) unless job_succeeded + else + # we don't need job_status label here + @metrics[:sidekiq_jobs_completion_seconds_sum].increment(labels, monotonic_time) + end # job_status: done, fail match the job_status attribute in structured logging labels[:job_status] = job_succeeded ? "done" : "fail" instrumentation = job[:instrumentation] || {} @metrics[:sidekiq_jobs_cpu_seconds].observe(labels, job_thread_cputime) - @metrics[:sidekiq_jobs_completion_seconds].observe(labels, monotonic_time) + + @metrics[:sidekiq_jobs_completion_seconds]&.observe(labels, monotonic_time) + @metrics[:sidekiq_jobs_db_seconds].observe(labels, ActiveRecord::LogSubscriber.runtime / 1000) @metrics[:sidekiq_jobs_gitaly_seconds].observe(labels, get_gitaly_time(instrumentation)) @metrics[:sidekiq_redis_requests_total].increment(labels, get_redis_calls(instrumentation)) @@ -143,16 +162,10 @@ module Gitlab @metrics[:sidekiq_load_balancing_count].increment(labels.merge(load_balancing_labels), 1) end - if ::Feature.enabled?(:sidekiq_execution_application_slis) - sli_labels = labels.slice(*SIDEKIQ_SLI_LABELS) - Gitlab::Metrics::SidekiqSlis.record_execution_apdex(sli_labels, monotonic_time) if job_succeeded - Gitlab::Metrics::SidekiqSlis.record_execution_error(sli_labels, !job_succeeded) - end - - if ::Feature.enabled?(:sidekiq_queueing_application_slis) - sli_labels = labels.slice(*SIDEKIQ_SLI_LABELS) - Gitlab::Metrics::SidekiqSlis.record_queueing_apdex(sli_labels, queue_duration) if queue_duration - end + sli_labels = labels.slice(*SIDEKIQ_SLI_LABELS) + Gitlab::Metrics::SidekiqSlis.record_execution_apdex(sli_labels, monotonic_time) if job_succeeded + Gitlab::Metrics::SidekiqSlis.record_execution_error(sli_labels, !job_succeeded) + Gitlab::Metrics::SidekiqSlis.record_queueing_apdex(sli_labels, queue_duration) if queue_duration end end diff --git a/lib/gitlab/sidekiq_middleware/skip_jobs.rb b/lib/gitlab/sidekiq_middleware/skip_jobs.rb index 8932607df52..6cc394aa5f4 100644 --- a/lib/gitlab/sidekiq_middleware/skip_jobs.rb +++ b/lib/gitlab/sidekiq_middleware/skip_jobs.rb @@ -84,8 +84,8 @@ module Gitlab health_context = Gitlab::Database::HealthStatus::Context.new( DatabaseHealthStatusChecker.new(job['jid'], worker_class.name), job_base_model.connection, - health_check_attrs[:gitlab_schema], - health_check_attrs[:tables] + health_check_attrs[:tables], + health_check_attrs[:gitlab_schema] ) Gitlab::Database::HealthStatus.evaluate(health_context).any?(&:stop?) diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index ba50a42cd37..31256101bd2 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -258,7 +258,7 @@ module Gitlab def multiline_blocked?(parsed_url) url = parsed_url.to_s - return true if url =~ /\n|\r/ + return true if /\n|\r/.match?(url) # Google Cloud Storage uses a multi-line, encoded Signature query string return false if %w(http https).include?(parsed_url.scheme&.downcase) @@ -282,7 +282,7 @@ module Gitlab def validate_user(value) return if value.blank? - return if value =~ /\A\p{Alnum}/ + return if /\A\p{Alnum}/.match?(value) raise BlockedUrlError, "Username needs to start with an alphanumeric character" end @@ -290,7 +290,7 @@ module Gitlab def validate_hostname(value) return if value.blank? return if IPAddress.valid?(value) - return if value =~ /\A\p{Alnum}/ + return if /\A\p{Alnum}/.match?(value) raise BlockedUrlError, "Hostname or IP address invalid" end diff --git a/lib/gitlab/usage/metric.rb b/lib/gitlab/usage/metric.rb index 30f2efc8638..d7e983d126a 100644 --- a/lib/gitlab/usage/metric.rb +++ b/lib/gitlab/usage/metric.rb @@ -25,10 +25,6 @@ module Gitlab with_availability(proc { instrumentation_object.instrumentation }) end - def with_suggested_name - with_availability(proc { instrumentation_object.suggested_name }) - end - private def with_availability(value_proc) diff --git a/lib/gitlab/usage/metrics/instrumentations/aggregated_metric.rb b/lib/gitlab/usage/metrics/instrumentations/aggregated_metric.rb index 66be7a7b64e..de8726f71b5 100644 --- a/lib/gitlab/usage/metrics/instrumentations/aggregated_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/aggregated_metric.rb @@ -40,10 +40,6 @@ module Gitlab end end - def suggested_name - Gitlab::Usage::Metrics::NameSuggestion.for(:alt) - end - private attr_accessor :source, :aggregate diff --git a/lib/gitlab/usage/metrics/instrumentations/batched_background_migration_failed_jobs_metric.rb b/lib/gitlab/usage/metrics/instrumentations/batched_background_migration_failed_jobs_metric.rb new file mode 100644 index 00000000000..f5d963cf522 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/batched_background_migration_failed_jobs_metric.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class BatchedBackgroundMigrationFailedJobsMetric < DatabaseMetric + relation do + Gitlab::Database::BackgroundMigration::BatchedMigration + .joins(:batched_jobs) + .where(batched_jobs: { status: '2' }) + .group(%w[table_name job_class_name]) + .order(%w[table_name job_class_name]) + .select(['table_name', 'job_class_name', 'COUNT(batched_jobs) AS number_of_failed_jobs']) + end + + timestamp_column(:created_at) + + operation :count + + def value + relation.map do |batched_migration| + { + job_class_name: batched_migration.job_class_name, + table_name: batched_migration.table_name, + number_of_failed_jobs: batched_migration.number_of_failed_jobs + } + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_imported_projects_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_imported_projects_metric.rb index 05e29f2d885..f3e81766b4c 100644 --- a/lib/gitlab/usage/metrics/instrumentations/count_imported_projects_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/count_imported_projects_metric.rb @@ -44,7 +44,7 @@ module Gitlab private def relation - super.imported_from(import_type) # rubocop: disable CodeReuse/ActiveRecord + super.imported_from(import_type) end def import_type diff --git a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb index f731057309e..2af7c208fce 100644 --- a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb @@ -87,14 +87,6 @@ module Gitlab to_sql end - def suggested_name - Gitlab::Usage::Metrics::NameSuggestion.for( - self.class.metric_operation, - relation: relation, - column: self.class.column - ) - end - private def start diff --git a/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb b/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb index d57dd7eac20..774f65da3bf 100644 --- a/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb @@ -37,10 +37,6 @@ module Gitlab self.class.metric_value.call(...) end end - - def suggested_name - Gitlab::Usage::Metrics::NameSuggestion.for(:alt) - end end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/numbers_metric.rb b/lib/gitlab/usage/metrics/instrumentations/numbers_metric.rb index 3b20e6ad100..67fcd226a0a 100644 --- a/lib/gitlab/usage/metrics/instrumentations/numbers_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/numbers_metric.rb @@ -44,10 +44,6 @@ module Gitlab method(self.class.metric_operation).call(*data) end - def suggested_name - Gitlab::Usage::Metrics::NameSuggestion.for(:alt) - end - private def data diff --git a/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb b/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb index 17009f7638e..d3bbb3ee02c 100644 --- a/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb @@ -30,10 +30,6 @@ module Gitlab end end - def suggested_name - Gitlab::Usage::Metrics::NameSuggestion.for(:redis) - end - private def time_constraints diff --git a/lib/gitlab/usage/metrics/instrumentations/redis_metric.rb b/lib/gitlab/usage/metrics/instrumentations/redis_metric.rb index ae3326fa845..ca5e5b706c4 100644 --- a/lib/gitlab/usage/metrics/instrumentations/redis_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/redis_metric.rb @@ -48,10 +48,6 @@ module Gitlab end end - def suggested_name - Gitlab::Usage::Metrics::NameSuggestion.for(:redis) - end - private def redis_key diff --git a/lib/gitlab/usage/metrics/instrumentations/work_items_activity_aggregated_metric.rb b/lib/gitlab/usage/metrics/instrumentations/work_items_activity_aggregated_metric.rb deleted file mode 100644 index d045265495a..00000000000 --- a/lib/gitlab/usage/metrics/instrumentations/work_items_activity_aggregated_metric.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Usage - module Metrics - module Instrumentations - class WorkItemsActivityAggregatedMetric < AggregatedMetric - available? { Feature.enabled?(:track_work_items_activity) } - end - end - end - end -end diff --git a/lib/gitlab/usage/metrics/name_suggestion.rb b/lib/gitlab/usage/metrics/name_suggestion.rb deleted file mode 100644 index 44723b6f3d4..00000000000 --- a/lib/gitlab/usage/metrics/name_suggestion.rb +++ /dev/null @@ -1,216 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Usage - module Metrics - class NameSuggestion - FREE_TEXT_METRIC_NAME = "<please fill metric name>" - REDIS_EVENT_METRIC_NAME = "<please fill metric name, suggested format is: {subject}_{verb}{ing|ed}_{object} eg: users_creating_epics or merge_requests_viewed_in_single_file_mode>" - CONSTRAINTS_PROMPT_TEMPLATE = "<adjective describing: '%{constraints}'>" - EMPTY_CONSTRAINT = "()" - - class << self - def for(operation, relation: nil, column: nil) - case operation - when :count - name_suggestion(column: column, relation: relation, prefix: 'count') - when :distinct_count - name_suggestion(column: column, relation: relation, prefix: 'count_distinct', distinct: :distinct) - when :estimate_batch_distinct_count - name_suggestion(column: column, relation: relation, prefix: 'estimate_distinct_count') - when :sum - name_suggestion(column: column, relation: relation, prefix: 'sum') - when :average - name_suggestion(column: column, relation: relation, prefix: 'average') - when :redis - REDIS_EVENT_METRIC_NAME - when :alt - FREE_TEXT_METRIC_NAME - else - raise ArgumentError, "#{operation} operation not supported" - end - end - - private - - def name_suggestion(relation:, column: nil, prefix: nil, distinct: nil) - # rubocop: disable CodeReuse/ActiveRecord - relation = relation.unscope(where: :created_at) - # rubocop: enable CodeReuse/ActiveRecord - - parts = [prefix] - arel_column = arelize_column(relation, column) - - # nil as column indicates that the counting would use fallback value of primary key. - # Because counting primary key from relation is the conceptual equal to counting all - # records from given relation, in order to keep name suggestion more condensed - # primary key column is skipped. - # eg: SELECT COUNT(id) FROM issues would translate as count_issues and not - # as count_id_from_issues since it does not add more information to the name suggestion - if arel_column != Arel::Table.new(relation.table_name)[relation.primary_key] - parts << arel_column.name - parts << 'from' - end - - arel = arel_query(relation: relation, column: arel_column, distinct: distinct) - where_constraints = parse_where_constraints(relation: relation, arel: arel) - having_constraints = parse_having_constraints(relation: relation, arel: arel) - - # In some cases due to performance reasons metrics are instrumented with joined relations - # where relation listed in FROM statement is not the one that includes counted attribute - # in such situations to make name suggestion more intuitive source should be inferred based - # on the relation that provide counted attribute - # EG: SELECT COUNT(deployments.environment_id) FROM clusters - # JOIN deployments ON deployments.cluster_id = cluster.id - # should be translated into: - # count_environment_id_from_deployments_with_clusters - # instead of - # count_environment_id_from_clusters_with_deployments - actual_source = parse_source(relation, arel_column) - - append_constraints_prompt(actual_source, [where_constraints], [having_constraints], parts) - - parts << actual_source - parts += process_joined_relations(actual_source, arel, relation, where_constraints) - parts.compact.join('_').delete('"') - end - - def append_constraints_prompt(target, where_constraints, having_constraints, parts) - where_constraints.select! do |constraint| - constraint.include?(target) - end - having_constraints.delete(EMPTY_CONSTRAINT) - applicable_constraints = where_constraints + having_constraints - return unless applicable_constraints.any? - - parts << CONSTRAINTS_PROMPT_TEMPLATE % { constraints: applicable_constraints.join(' AND ') } - end - - def parse_where_constraints(relation:, arel:) - connection = relation.connection - ::Gitlab::Usage::Metrics::NamesSuggestions::RelationParsers::WhereConstraints - .new(connection) - .accept(arel, collector(connection)) - .value - end - - def parse_having_constraints(relation:, arel:) - connection = relation.connection - ::Gitlab::Usage::Metrics::NamesSuggestions::RelationParsers::HavingConstraints - .new(connection) - .accept(arel, collector(connection)) - .value - end - - # TODO: joins with `USING` keyword - def process_joined_relations(actual_source, arel, relation, where_constraints) - joins = parse_joins(connection: relation.connection, arel: arel) - return [] unless joins.any? - - sources = [relation.table_name, *joins.map { |join| join[:source] }] - joins = extract_joins_targets(joins, sources) - - relations = if actual_source != relation.table_name - build_relations_tree(joins + [{ source: relation.table_name }], actual_source) - else - # in case where counter attribute comes from joined relations, the relations - # diagram has to be built bottom up, thus source and target are reverted - build_relations_tree(joins + [{ source: relation.table_name }], actual_source, source_key: :target, target_key: :source) - end - - collect_join_parts(relations: relations[actual_source], joins: joins, wheres: where_constraints) - end - - def parse_joins(connection:, arel:) - ::Gitlab::Usage::Metrics::NamesSuggestions::RelationParsers::Joins - .new(connection) - .accept(arel) - end - - def extract_joins_targets(joins, sources) - joins.map do |join| - source_regex = /(#{join[:source]})\.(\w+_)*id/i - - tables_except_src = (sources - [join[:source]]).join('|') - target_regex = /(?<target>#{tables_except_src})\.(\w+_)*id/i - - join_cond_regex = /(#{source_regex}\s+=\s+#{target_regex})|(#{target_regex}\s+=\s+#{source_regex})/i - matched = join_cond_regex.match(join[:constraints]) - - if matched - join[:target] = matched[:target] - join[:constraints].gsub!(/#{join_cond_regex}(\s+(and|or))*/i, '') - end - - join - end - end - - def build_relations_tree(joins, parent, source_key: :source, target_key: :target) - return [] if joins.blank? - - tree = {} - tree[parent] = [] - - joins.each do |join| - if join[source_key] == parent - tree[parent] << build_relations_tree(joins - [join], join[target_key], source_key: source_key, target_key: target_key) - end - end - tree - end - - def collect_join_parts(relations:, joins:, wheres:, parts: [], conjunctions: %w[with having including].cycle) - conjunction = conjunctions.next - relations.each do |subtree| - subtree.each do |parent, children| - parts << "<#{conjunction}>" - join_constraints = joins.find { |join| join[:source] == parent }&.dig(:constraints) - append_constraints_prompt(parent, [wheres, join_constraints].compact, [], parts) - parts << parent - collect_join_parts(relations: children, joins: joins, wheres: wheres, parts: parts, conjunctions: conjunctions) - end - end - parts - end - - def arelize_column(relation, column) - case column - when Arel::Attribute - column - when NilClass - Arel::Table.new(relation.table_name)[relation.primary_key] - when String - if column.include?('.') - table, col = column.split('.') - Arel::Table.new(table)[col] - else - Arel::Table.new(relation.table_name)[column] - end - when Symbol - arelize_column(relation, column.to_s) - end - end - - def parse_source(relation, column) - column.relation.name || relation.table_name - end - - def collector(connection) - Arel::Collectors::SubstituteBinds.new(connection, Arel::Collectors::SQLString.new) - end - - def arel_query(relation:, column: nil, distinct: nil) - column ||= relation.primary_key - - if column.is_a?(Arel::Attribute) - relation.select(column.count(distinct)).arel - else - relation.select(relation.all.table[column].count(distinct)).arel - end - end - end - end - end - end -end diff --git a/lib/gitlab/usage/metrics/names_suggestions/generator.rb b/lib/gitlab/usage/metrics/names_suggestions/generator.rb deleted file mode 100644 index 626bd3d4ad4..00000000000 --- a/lib/gitlab/usage/metrics/names_suggestions/generator.rb +++ /dev/null @@ -1,57 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Usage - module Metrics - module NamesSuggestions - class Generator < ::Gitlab::UsageData - class << self - def generate(key_path) - data.deep_stringify_keys.dig(*key_path.split('.')) - end - - def add_metric(metric, time_frame: 'none', options: {}) - metric_class = "Gitlab::Usage::Metrics::Instrumentations::#{metric}".constantize - - metric_class.new(time_frame: time_frame, options: options).suggested_name - end - - private - - def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) - Gitlab::Usage::Metrics::NameSuggestion.for(:count, column: column, relation: relation) - end - - def distinct_count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) - Gitlab::Usage::Metrics::NameSuggestion.for(:distinct_count, column: column, relation: relation) - end - - def redis_usage_counter - Gitlab::Usage::Metrics::NameSuggestion.for(:redis) - end - - def alt_usage_data(*) - Gitlab::Usage::Metrics::NameSuggestion.for(:alt) - end - - def redis_usage_data_totals(counter) - counter.fallback_totals.transform_values { |_| Gitlab::Usage::Metrics::NameSuggestion.for(:redis) } - end - - def sum(relation, column, *rest) - Gitlab::Usage::Metrics::NameSuggestion.for(:sum, column: column, relation: relation) - end - - def estimate_batch_distinct_count(relation, column = nil, *rest) - Gitlab::Usage::Metrics::NameSuggestion.for(:estimate_batch_distinct_count, column: column, relation: relation) - end - - def add(*args) - "add_#{args.join('_and_')}" - end - end - end - end - end - end -end diff --git a/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/having_constraints.rb b/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/having_constraints.rb deleted file mode 100644 index 8dd3b1ff5c6..00000000000 --- a/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/having_constraints.rb +++ /dev/null @@ -1,31 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Usage - module Metrics - module NamesSuggestions - module RelationParsers - class HavingConstraints < ::Arel::Visitors::PostgreSQL - # rubocop:disable Naming/MethodName - def visit_Arel_Nodes_SelectCore(object, collector) - collect_nodes_for(object.havings, collector, "") || collector - end - # rubocop:enable Naming/MethodName - - def quote(value) - value.to_s - end - - def quote_table_name(name) - name.to_s - end - - def quote_column_name(name) - name.to_s - end - end - end - end - end - end -end diff --git a/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/joins.rb b/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/joins.rb deleted file mode 100644 index d52e4903f3c..00000000000 --- a/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/joins.rb +++ /dev/null @@ -1,74 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Usage - module Metrics - module NamesSuggestions - module RelationParsers - class Joins < ::Arel::Visitors::PostgreSQL - def accept(object) - object.source.right.map do |join| - visit(join, collector) - end - end - - private - - # rubocop:disable Naming/MethodName - def visit_Arel_Nodes_StringJoin(object, collector) - result = visit(object.left, collector) - source, constraints = result.value.split('ON') - { - source: source.split('JOIN').last&.strip, - constraints: constraints&.strip - }.compact - end - - def visit_Arel_Nodes_FullOuterJoin(object, _) - parse_join(object) - end - - def visit_Arel_Nodes_OuterJoin(object, _) - parse_join(object) - end - - def visit_Arel_Nodes_RightOuterJoin(object, _) - parse_join(object) - end - - def visit_Arel_Nodes_InnerJoin(object, _) - { - source: visit(object.left, collector).value, - constraints: object.right ? visit(object.right.expr, collector).value : nil - }.compact - end - # rubocop:enable Naming/MethodName - - def parse_join(object) - { - source: visit(object.left, collector).value, - constraints: visit(object.right.expr, collector).value - } - end - - def quote(value) - "#{value}" - end - - def quote_table_name(name) - "#{name}" - end - - def quote_column_name(name) - "#{name}" - end - - def collector - Arel::Collectors::SubstituteBinds.new(@connection, Arel::Collectors::SQLString.new) - end - end - end - end - end - end -end diff --git a/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/where_constraints.rb b/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/where_constraints.rb deleted file mode 100644 index 9f829067214..00000000000 --- a/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/where_constraints.rb +++ /dev/null @@ -1,31 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Usage - module Metrics - module NamesSuggestions - module RelationParsers - class WhereConstraints < ::Arel::Visitors::PostgreSQL - # rubocop:disable Naming/MethodName - def visit_Arel_Nodes_SelectCore(object, collector) - collect_nodes_for(object.wheres, collector, "") || collector - end - # rubocop:enable Naming/MethodName - - def quote(value) - value.to_s - end - - def quote_table_name(name) - name.to_s - end - - def quote_column_name(name) - name.to_s - end - end - end - end - end - end -end diff --git a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb index 97091ff975b..21cc9368f44 100644 --- a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb @@ -3,7 +3,6 @@ module Gitlab::UsageDataCounters class CiTemplateUniqueCounter PREFIX = 'ci_templates' - KNOWN_EVENTS_FILE_PATH = File.expand_path('known_events/ci_templates.yml', __dir__) class << self def track_unique_project_event(project:, template:, config_source:, user:) diff --git a/lib/gitlab/usage_data_counters/editor_unique_counter.rb b/lib/gitlab/usage_data_counters/editor_unique_counter.rb index 4e4a01ed301..7955c19b7e6 100644 --- a/lib/gitlab/usage_data_counters/editor_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/editor_unique_counter.rb @@ -9,24 +9,24 @@ module Gitlab EDIT_CATEGORY = 'ide_edit' class << self - def track_web_ide_edit_action(author:, time: Time.zone.now, project:) - track_unique_action(EDIT_BY_WEB_IDE, author, time, project) + def track_web_ide_edit_action(author:, project:) + track_internal_event(EDIT_BY_WEB_IDE, author, project) end def count_web_ide_edit_actions(date_from:, date_to:) count_unique(EDIT_BY_WEB_IDE, date_from, date_to) end - def track_sfe_edit_action(author:, time: Time.zone.now, project:) - track_unique_action(EDIT_BY_SFE, author, time, project) + def track_sfe_edit_action(author:, project:) + track_internal_event(EDIT_BY_SFE, author, project) end def count_sfe_edit_actions(date_from:, date_to:) count_unique(EDIT_BY_SFE, date_from, date_to) end - def track_snippet_editor_edit_action(author:, time: Time.zone.now, project:) - track_unique_action(EDIT_BY_SNIPPET_EDITOR, author, time, project) + def track_snippet_editor_edit_action(author:, project:) + track_internal_event(EDIT_BY_SNIPPET_EDITOR, author, project) end def count_snippet_editor_edit_actions(date_from:, date_to:) @@ -35,21 +35,15 @@ module Gitlab private - def track_unique_action(event_name, author, time, project = nil) + def track_internal_event(event_name, author, project = nil) return unless author - Gitlab::Tracking.event( - name, - 'ide_edit', - property: event_name.to_s, - project: project, - namespace: project&.namespace, + Gitlab::InternalEvents.track_event( + event_name, user: author, - label: 'usage_activity_by_stage_monthly.create.action_monthly_active_users_ide_edit', - context: [Gitlab::Tracking::ServicePingContext.new(data_source: :redis_hll, event: event_name).to_context] + project: project, + namespace: project&.namespace ) - - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name, values: author.id, time: time) end def count_unique(actions, date_from, date_to) diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index e71061c4522..53594a27867 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -8,9 +8,6 @@ module Gitlab EventError = Class.new(StandardError) UnknownEvent = Class.new(EventError) - InvalidContext = Class.new(EventError) - - KNOWN_EVENTS_PATH = File.expand_path('known_events/*.yml', __dir__) # Track event on entity_id # Increment a Redis HLL counter for unique event_name and entity_id @@ -31,29 +28,13 @@ module Gitlab track(values, event_name, time: time) end - # Track unique events - # - # event_name - The event name. - # values - One or multiple values counted. - # context - Event context, plan level tracking. - # time - Time of the action, set to Time.current. - def track_event_in_context(event_name, values:, context:, time: Time.zone.now) - return if context.blank? - return unless context.in?(valid_context_list) - - track(values, event_name, context: context, time: time) - end - # Count unique events for a given time range. # # event_names - The list of the events to count. # start_date - The start date of the time range. # end_date - The end date of the time range. - # context - Event context, plan level tracking. Available if set when tracking. - def unique_events(event_names:, start_date:, end_date:, context: '') - count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date, context: context) do - raise InvalidContext if context.present? && !context.in?(valid_context_list) - end + def unique_events(event_names:, start_date:, end_date:) + count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date) end def known_event?(event_name) @@ -61,7 +42,7 @@ module Gitlab end def known_events - @known_events ||= load_events(KNOWN_EVENTS_PATH) + @known_events ||= load_events end def calculate_events_union(event_names:, start_date:, end_date:) @@ -70,7 +51,7 @@ module Gitlab private - def track(values, event_name, context: '', time: Time.zone.now) + def track(values, event_name, time: Time.zone.now) return unless ::ServicePing::ServicePingSettings.enabled? event = event_for(event_name) @@ -79,7 +60,7 @@ module Gitlab return if event.blank? return unless Feature.enabled?(:redis_hll_tracking, type: :ops) - Gitlab::Redis::HLL.add(key: redis_key(event, time, context), value: values, expiry: KEY_EXPIRY_LENGTH) + Gitlab::Redis::HLL.add(key: redis_key(event, time), value: values, expiry: KEY_EXPIRY_LENGTH) rescue StandardError => e # Ignore any exceptions unless is dev or test env @@ -87,48 +68,33 @@ module Gitlab Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) end - # The array of valid context on which we allow tracking - def valid_context_list - Plan.all_plans - end - - def count_unique_events(event_names:, start_date:, end_date:, context: '') + def count_unique_events(event_names:, start_date:, end_date:) events = events_for(Array(event_names).map(&:to_s)) - yield events if block_given? - - keys = keys_for_aggregation(events: events, start_date: start_date, end_date: end_date, context: context) + keys = keys_for_aggregation(events: events, start_date: start_date, end_date: end_date) return FALLBACK unless keys.any? redis_usage_data { Gitlab::Redis::HLL.count(keys: keys) } end - def keys_for_aggregation(events:, start_date:, end_date:, context: '') + def keys_for_aggregation(events:, start_date:, end_date:) end_date = end_date.end_of_week - 1.week (start_date.to_date..end_date.to_date).map do |date| - events.map { |event| redis_key(event, date, context) } + events.map { |event| redis_key(event, date) } end.flatten.uniq end - def load_events(wildcard) - if Feature.enabled?(:use_metric_definitions_for_events_list) - events = Gitlab::Usage::MetricDefinition.not_removed.values.map do |d| - d.attributes[:options] && d.attributes[:options][:events] - end.flatten.compact.uniq - - events.map do |e| - { name: e }.with_indifferent_access - end - else - Dir[wildcard].each_with_object([]) do |path, events| - events.push(*load_yaml_from_path(path)) - end - end - end + def load_events + events = Gitlab::Usage::MetricDefinition.all.map do |d| + next unless d.available? + + d.attributes[:options] && d.attributes[:options][:events] + end.flatten.compact.uniq - def load_yaml_from_path(path) - YAML.safe_load(File.read(path))&.map(&:with_indifferent_access) + events.map do |e| + { name: e }.with_indifferent_access + end end def known_events_names @@ -144,20 +110,15 @@ module Gitlab end # Compose the key in order to store events daily or weekly - def redis_key(event, time, context = '') + def redis_key(event, time) raise UnknownEvent, "Unknown event #{event[:name]}" unless known_events_names.include?(event[:name].to_s) key = "{#{REDIS_SLOT}}_#{event[:name]}" year_week = time.strftime('%G-%V') - key = "#{key}-#{year_week}" - - key = "#{context}_#{key}" if context.present? - key + "#{key}-#{year_week}" end end end end end - -Gitlab::UsageDataCounters::HLLRedisCounter.prepend_mod_with('Gitlab::UsageDataCounters::HLLRedisCounter') diff --git a/lib/gitlab/usage_data_counters/ipynb_diff_activity_counter.rb b/lib/gitlab/usage_data_counters/ipynb_diff_activity_counter.rb index a34ae909c82..05eb88468c7 100644 --- a/lib/gitlab/usage_data_counters/ipynb_diff_activity_counter.rb +++ b/lib/gitlab/usage_data_counters/ipynb_diff_activity_counter.rb @@ -1,6 +1,5 @@ # frozen_string_literal: true -# noinspection RubyConstantNamingConvention module Gitlab module UsageDataCounters module IpynbDiffActivityCounter diff --git a/lib/gitlab/usage_data_counters/known_events/analytics.yml b/lib/gitlab/usage_data_counters/known_events/analytics.yml deleted file mode 100644 index 0b30308b552..00000000000 --- a/lib/gitlab/usage_data_counters/known_events/analytics.yml +++ /dev/null @@ -1,26 +0,0 @@ -- name: users_viewing_analytics_group_devops_adoption - aggregation: weekly -- name: i_analytics_dev_ops_adoption - aggregation: weekly -- name: i_analytics_dev_ops_score - aggregation: weekly -- name: i_analytics_instance_statistics - aggregation: weekly -- name: p_analytics_pipelines - aggregation: weekly -- name: p_analytics_valuestream - aggregation: weekly -- name: p_analytics_repo - aggregation: weekly -- name: i_analytics_cohorts - aggregation: weekly -- name: p_analytics_ci_cd_pipelines - aggregation: weekly -- name: p_analytics_ci_cd_deployment_frequency - aggregation: weekly -- name: p_analytics_ci_cd_lead_time - aggregation: weekly -- name: p_analytics_ci_cd_time_to_restore_service - aggregation: weekly -- name: p_analytics_ci_cd_change_failure_rate - aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml deleted file mode 100644 index c3e1c34151b..00000000000 --- a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml +++ /dev/null @@ -1,311 +0,0 @@ -# This file is generated automatically by -# bin/rake gitlab:usage_data:generate_ci_template_events -# -# Do not edit it manually! ---- -- name: p_ci_templates_terraform_base_latest - aggregation: weekly -- name: p_ci_templates_terraform_base - aggregation: weekly -- name: p_ci_templates_dotnet - aggregation: weekly -- name: p_ci_templates_nodejs - aggregation: weekly -- name: p_ci_templates_openshift - aggregation: weekly -- name: p_ci_templates_auto_devops - aggregation: weekly -- name: p_ci_templates_bash - aggregation: weekly -- name: p_ci_templates_rust - aggregation: weekly -- name: p_ci_templates_elixir - aggregation: weekly -- name: p_ci_templates_clojure - aggregation: weekly -- name: p_ci_templates_crystal - aggregation: weekly -- name: p_ci_templates_getting_started - aggregation: weekly -- name: p_ci_templates_code_quality - aggregation: weekly -- name: p_ci_templates_verify_load_performance_testing - aggregation: weekly -- name: p_ci_templates_verify_accessibility - aggregation: weekly -- name: p_ci_templates_verify_failfast - aggregation: weekly -- name: p_ci_templates_verify_browser_performance - aggregation: weekly -- name: p_ci_templates_verify_browser_performance_latest - aggregation: weekly -- name: p_ci_templates_grails - aggregation: weekly -- name: p_ci_templates_security_sast - aggregation: weekly -- name: p_ci_templates_security_dast_runner_validation - aggregation: weekly -- name: p_ci_templates_security_dast_on_demand_scan - aggregation: weekly -- name: p_ci_templates_security_secret_detection - aggregation: weekly -- name: p_ci_templates_security_license_scanning - aggregation: weekly -- name: p_ci_templates_security_coverage_fuzzing_latest - aggregation: weekly -- name: p_ci_templates_security_dast_on_demand_api_scan - aggregation: weekly -- name: p_ci_templates_security_coverage_fuzzing - aggregation: weekly -- name: p_ci_templates_security_api_fuzzing_latest - aggregation: weekly -- name: p_ci_templates_security_secure_binaries - aggregation: weekly -- name: p_ci_templates_security_dast_api - aggregation: weekly -- name: p_ci_templates_security_container_scanning - aggregation: weekly -- name: p_ci_templates_security_dast_latest - aggregation: weekly -- name: p_ci_templates_security_sast_iac - aggregation: weekly -- name: p_ci_templates_security_dependency_scanning - aggregation: weekly -- name: p_ci_templates_security_dast_api_latest - aggregation: weekly -- name: p_ci_templates_security_container_scanning_latest - aggregation: weekly -- name: p_ci_templates_security_api_fuzzing - aggregation: weekly -- name: p_ci_templates_security_dast - aggregation: weekly -- name: p_ci_templates_security_api_discovery - aggregation: weekly -- name: p_ci_templates_security_fortify_fod_sast - aggregation: weekly -- name: p_ci_templates_security_sast_iac_latest - aggregation: weekly -- name: p_ci_templates_security_bas_latest - aggregation: weekly -- name: p_ci_templates_qualys_iac_security - aggregation: weekly -- name: p_ci_templates_ios_fastlane - aggregation: weekly -- name: p_ci_templates_composer - aggregation: weekly -- name: p_ci_templates_c - aggregation: weekly -- name: p_ci_templates_python - aggregation: weekly -- name: p_ci_templates_android_fastlane - aggregation: weekly -- name: p_ci_templates_android_latest - aggregation: weekly -- name: p_ci_templates_django - aggregation: weekly -- name: p_ci_templates_maven - aggregation: weekly -- name: p_ci_templates_liquibase - aggregation: weekly -- name: p_ci_templates_flutter - aggregation: weekly -- name: p_ci_templates_workflows_branch_pipelines - aggregation: weekly -- name: p_ci_templates_workflows_mergerequest_pipelines - aggregation: weekly -- name: p_ci_templates_laravel - aggregation: weekly -- name: p_ci_templates_kaniko - aggregation: weekly -- name: p_ci_templates_php - aggregation: weekly -- name: p_ci_templates_packer - aggregation: weekly -- name: p_ci_templates_themekit - aggregation: weekly -- name: p_ci_templates_terraform - aggregation: weekly -- name: p_ci_templates_katalon - aggregation: weekly -- name: p_ci_templates_mono - aggregation: weekly -- name: p_ci_templates_go - aggregation: weekly -- name: p_ci_templates_scala - aggregation: weekly -- name: p_ci_templates_latex - aggregation: weekly -- name: p_ci_templates_android - aggregation: weekly -- name: p_ci_templates_indeni_cloudrail - aggregation: weekly -- name: p_ci_templates_matlab - aggregation: weekly -- name: p_ci_templates_deploy_ecs - aggregation: weekly -- name: p_ci_templates_aws_cf_provision_and_deploy_ec2 - aggregation: weekly -- name: p_ci_templates_aws_deploy_ecs - aggregation: weekly -- name: p_ci_templates_gradle - aggregation: weekly -- name: p_ci_templates_chef - aggregation: weekly -- name: p_ci_templates_jobs_dast_default_branch_deploy - aggregation: weekly -- name: p_ci_templates_jobs_load_performance_testing - aggregation: weekly -- name: p_ci_templates_jobs_helm_2to3 - aggregation: weekly -- name: p_ci_templates_jobs_sast - aggregation: weekly -- name: p_ci_templates_jobs_secret_detection - aggregation: weekly -- name: p_ci_templates_jobs_license_scanning - aggregation: weekly -- name: p_ci_templates_jobs_code_intelligence - aggregation: weekly -- name: p_ci_templates_jobs_code_quality - aggregation: weekly -- name: p_ci_templates_jobs_deploy_ecs - aggregation: weekly -- name: p_ci_templates_jobs_deploy_ec2 - aggregation: weekly -- name: p_ci_templates_jobs_license_scanning_latest - aggregation: weekly -- name: p_ci_templates_jobs_deploy - aggregation: weekly -- name: p_ci_templates_jobs_build - aggregation: weekly -- name: p_ci_templates_jobs_browser_performance_testing - aggregation: weekly -- name: p_ci_templates_jobs_container_scanning - aggregation: weekly -- name: p_ci_templates_jobs_container_scanning_latest - aggregation: weekly -- name: p_ci_templates_jobs_dependency_scanning_latest - aggregation: weekly -- name: p_ci_templates_jobs_test - aggregation: weekly -- name: p_ci_templates_jobs_sast_latest - aggregation: weekly -- name: p_ci_templates_jobs_sast_iac - aggregation: weekly -- name: p_ci_templates_jobs_secret_detection_latest - aggregation: weekly -- name: p_ci_templates_jobs_dependency_scanning - aggregation: weekly -- name: p_ci_templates_jobs_deploy_latest - aggregation: weekly -- name: p_ci_templates_jobs_browser_performance_testing_latest - aggregation: weekly -- name: p_ci_templates_jobs_cf_provision - aggregation: weekly -- name: p_ci_templates_jobs_build_latest - aggregation: weekly -- name: p_ci_templates_jobs_sast_iac_latest - aggregation: weekly -- name: p_ci_templates_terraform_latest - aggregation: weekly -- name: p_ci_templates_swift - aggregation: weekly -- name: p_ci_templates_pages_jekyll - aggregation: weekly -- name: p_ci_templates_pages_harp - aggregation: weekly -- name: p_ci_templates_pages_octopress - aggregation: weekly -- name: p_ci_templates_pages_brunch - aggregation: weekly -- name: p_ci_templates_pages_doxygen - aggregation: weekly -- name: p_ci_templates_pages_hyde - aggregation: weekly -- name: p_ci_templates_pages_lektor - aggregation: weekly -- name: p_ci_templates_pages_jbake - aggregation: weekly -- name: p_ci_templates_pages_hexo - aggregation: weekly -- name: p_ci_templates_pages_middleman - aggregation: weekly -- name: p_ci_templates_pages_hugo - aggregation: weekly -- name: p_ci_templates_pages_pelican - aggregation: weekly -- name: p_ci_templates_pages_nanoc - aggregation: weekly -- name: p_ci_templates_pages_swaggerui - aggregation: weekly -- name: p_ci_templates_pages_jigsaw - aggregation: weekly -- name: p_ci_templates_pages_metalsmith - aggregation: weekly -- name: p_ci_templates_pages_gatsby - aggregation: weekly -- name: p_ci_templates_pages_html - aggregation: weekly -- name: p_ci_templates_dart - aggregation: weekly -- name: p_ci_templates_docker - aggregation: weekly -- name: p_ci_templates_julia - aggregation: weekly -- name: p_ci_templates_npm - aggregation: weekly -- name: p_ci_templates_dotnet_core - aggregation: weekly -- name: p_ci_templates_5_minute_production_app - aggregation: weekly -- name: p_ci_templates_ruby - aggregation: weekly -- name: p_ci_templates_implicit_auto_devops - aggregation: weekly -- name: p_ci_templates_implicit_jobs_browser_performance_testing - aggregation: weekly -- name: p_ci_templates_implicit_jobs_build - aggregation: weekly -- name: p_ci_templates_implicit_jobs_code_intelligence - aggregation: weekly -- name: p_ci_templates_implicit_jobs_code_quality - aggregation: weekly -- name: p_ci_templates_implicit_jobs_container_scanning - aggregation: weekly -- name: p_ci_templates_implicit_jobs_dast_default_branch_deploy - aggregation: weekly -- name: p_ci_templates_implicit_jobs_dependency_scanning - aggregation: weekly -- name: p_ci_templates_implicit_jobs_deploy - aggregation: weekly -- name: p_ci_templates_implicit_jobs_deploy_ec2 - aggregation: weekly -- name: p_ci_templates_implicit_jobs_deploy_ecs - aggregation: weekly -- name: p_ci_templates_implicit_jobs_helm_2to3 - aggregation: weekly -- name: p_ci_templates_implicit_jobs_license_scanning - aggregation: weekly -- name: p_ci_templates_implicit_jobs_sast - aggregation: weekly -- name: p_ci_templates_implicit_jobs_secret_detection - aggregation: weekly -- name: p_ci_templates_implicit_jobs_test - aggregation: weekly -- name: p_ci_templates_implicit_security_container_scanning - aggregation: weekly -- name: p_ci_templates_implicit_security_dast - aggregation: weekly -- name: p_ci_templates_implicit_security_dependency_scanning - aggregation: weekly -- name: p_ci_templates_implicit_security_license_scanning - aggregation: weekly -- name: p_ci_templates_implicit_security_sast - aggregation: weekly -- name: p_ci_templates_implicit_security_secret_detection - aggregation: weekly -- name: p_ci_templates_terraform_module_base - aggregation: weekly -- name: p_ci_templates_terraform_module - aggregation: weekly -- name: p_ci_templates_pages_zola - aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/ci_users.yml b/lib/gitlab/usage_data_counters/known_events/ci_users.yml deleted file mode 100644 index 49757c6e672..00000000000 --- a/lib/gitlab/usage_data_counters/known_events/ci_users.yml +++ /dev/null @@ -1,4 +0,0 @@ -- name: ci_users_executing_deployment_job - aggregation: weekly -- name: ci_users_executing_verify_environment_job - aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml deleted file mode 100644 index bd8c79f4801..00000000000 --- a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml +++ /dev/null @@ -1,235 +0,0 @@ ---- -- name: i_code_review_create_note_in_ipynb_diff - aggregation: weekly -- name: i_code_review_create_note_in_ipynb_diff_mr - aggregation: weekly -- name: i_code_review_create_note_in_ipynb_diff_commit - aggregation: weekly -- name: i_code_review_user_create_note_in_ipynb_diff - aggregation: weekly -- name: i_code_review_user_create_note_in_ipynb_diff_mr - aggregation: weekly -- name: i_code_review_user_create_note_in_ipynb_diff_commit - aggregation: weekly -- name: i_code_review_mr_diffs - aggregation: weekly -- name: i_code_review_user_single_file_diffs - aggregation: weekly -- name: i_code_review_mr_single_file_diffs - aggregation: weekly -- name: i_code_review_user_toggled_task_item_status - aggregation: weekly -- name: i_code_review_create_mr - aggregation: weekly -- name: i_code_review_user_create_mr - aggregation: weekly -- name: i_code_review_user_close_mr - aggregation: weekly -- name: i_code_review_user_reopen_mr - aggregation: weekly -- name: i_code_review_user_approve_mr - aggregation: weekly -- name: i_code_review_user_unapprove_mr - aggregation: weekly -- name: i_code_review_user_resolve_thread - aggregation: weekly -- name: i_code_review_user_unresolve_thread - aggregation: weekly -- name: i_code_review_edit_mr_title - aggregation: weekly -- name: i_code_review_edit_mr_desc - aggregation: weekly -- name: i_code_review_user_merge_mr - aggregation: weekly -- name: i_code_review_user_create_mr_comment - aggregation: weekly -- name: i_code_review_user_edit_mr_comment - aggregation: weekly -- name: i_code_review_user_remove_mr_comment - aggregation: weekly -- name: i_code_review_user_create_review_note - aggregation: weekly -- name: i_code_review_user_publish_review - aggregation: weekly -- name: i_code_review_user_create_multiline_mr_comment - aggregation: weekly -- name: i_code_review_user_edit_multiline_mr_comment - aggregation: weekly -- name: i_code_review_user_remove_multiline_mr_comment - aggregation: weekly -- name: i_code_review_user_add_suggestion - aggregation: weekly -- name: i_code_review_user_apply_suggestion - aggregation: weekly -- name: i_code_review_user_assigned - aggregation: weekly -- name: i_code_review_user_marked_as_draft - aggregation: weekly -- name: i_code_review_user_unmarked_as_draft - aggregation: weekly -- name: i_code_review_user_review_requested - aggregation: weekly -- name: i_code_review_user_approval_rule_added - aggregation: weekly -- name: i_code_review_user_approval_rule_deleted - aggregation: weekly -- name: i_code_review_user_approval_rule_edited - aggregation: weekly -- name: i_code_review_user_vs_code_api_request - aggregation: weekly -- name: i_code_review_user_jetbrains_api_request - aggregation: weekly -- name: i_editor_extensions_user_jetbrains_bundled_api_request - aggregation: weekly -- name: i_code_review_user_gitlab_cli_api_request - aggregation: weekly -- name: i_code_review_user_create_mr_from_issue - aggregation: weekly -- name: i_code_review_user_mr_discussion_locked - aggregation: weekly -- name: i_code_review_user_mr_discussion_unlocked - aggregation: weekly -- name: i_code_review_user_time_estimate_changed - aggregation: weekly -- name: i_code_review_user_time_spent_changed - aggregation: weekly -- name: i_code_review_user_assignees_changed - aggregation: weekly -- name: i_code_review_user_reviewers_changed - aggregation: weekly -- name: i_code_review_user_milestone_changed - aggregation: weekly -- name: i_code_review_user_labels_changed - aggregation: weekly -# Diff settings events -- name: i_code_review_click_diff_view_setting - aggregation: weekly -- name: i_code_review_click_single_file_mode_setting - aggregation: weekly -- name: i_code_review_click_file_browser_setting - aggregation: weekly -- name: i_code_review_click_whitespace_setting - aggregation: weekly -- name: i_code_review_diff_view_inline - aggregation: weekly -- name: i_code_review_diff_view_parallel - aggregation: weekly -- name: i_code_review_file_browser_tree_view - aggregation: weekly -- name: i_code_review_file_browser_list_view - aggregation: weekly -- name: i_code_review_diff_show_whitespace - aggregation: weekly -- name: i_code_review_diff_hide_whitespace - aggregation: weekly -- name: i_code_review_diff_single_file - aggregation: weekly -- name: i_code_review_diff_multiple_files - aggregation: weekly -- name: i_code_review_user_load_conflict_ui - aggregation: weekly -- name: i_code_review_user_resolve_conflict - aggregation: weekly -- name: i_code_review_user_searches_diff - aggregation: weekly -- name: i_code_review_total_suggestions_applied - aggregation: weekly -- name: i_code_review_total_suggestions_added - aggregation: weekly -- name: i_code_review_user_resolve_thread_in_issue - aggregation: weekly -- name: i_code_review_widget_nothing_merge_click_new_file - aggregation: weekly -- name: i_code_review_post_merge_delete_branch - aggregation: weekly -- name: i_code_review_post_merge_click_revert - aggregation: weekly -- name: i_code_review_post_merge_click_cherry_pick - aggregation: weekly -- name: i_code_review_post_merge_submit_revert_modal - aggregation: weekly -- name: i_code_review_post_merge_submit_cherry_pick_modal - aggregation: weekly -# MR Widget Extensions -## Test Summary -- name: i_code_review_merge_request_widget_test_summary_view - aggregation: weekly -- name: i_code_review_merge_request_widget_test_summary_full_report_clicked - aggregation: weekly -- name: i_code_review_merge_request_widget_test_summary_expand - aggregation: weekly -- name: i_code_review_merge_request_widget_test_summary_expand_success - aggregation: weekly -- name: i_code_review_merge_request_widget_test_summary_expand_warning - aggregation: weekly -- name: i_code_review_merge_request_widget_test_summary_expand_failed - aggregation: weekly -## Accessibility -- name: i_code_review_merge_request_widget_accessibility_view - aggregation: weekly -- name: i_code_review_merge_request_widget_accessibility_full_report_clicked - aggregation: weekly -- name: i_code_review_merge_request_widget_accessibility_expand - aggregation: weekly -- name: i_code_review_merge_request_widget_accessibility_expand_success - aggregation: weekly -- name: i_code_review_merge_request_widget_accessibility_expand_warning - aggregation: weekly -- name: i_code_review_merge_request_widget_accessibility_expand_failed - aggregation: weekly -## Code Quality -- name: i_code_review_merge_request_widget_code_quality_view - aggregation: weekly -- name: i_code_review_merge_request_widget_code_quality_full_report_clicked - aggregation: weekly -- name: i_code_review_merge_request_widget_code_quality_expand - aggregation: weekly -- name: i_code_review_merge_request_widget_code_quality_expand_success - aggregation: weekly -- name: i_code_review_merge_request_widget_code_quality_expand_warning - aggregation: weekly -- name: i_code_review_merge_request_widget_code_quality_expand_failed - aggregation: weekly -## Terraform -- name: i_code_review_merge_request_widget_terraform_view - aggregation: weekly -- name: i_code_review_merge_request_widget_terraform_full_report_clicked - aggregation: weekly -- name: i_code_review_merge_request_widget_terraform_expand - aggregation: weekly -- name: i_code_review_merge_request_widget_terraform_expand_success - aggregation: weekly -- name: i_code_review_merge_request_widget_terraform_expand_warning - aggregation: weekly -- name: i_code_review_merge_request_widget_terraform_expand_failed - aggregation: weekly -- name: i_code_review_submit_review_approve - aggregation: weekly -- name: i_code_review_submit_review_comment - aggregation: weekly -## License Compliance -- name: i_code_review_merge_request_widget_license_compliance_view - aggregation: weekly -- name: i_code_review_merge_request_widget_license_compliance_full_report_clicked - aggregation: weekly -- name: i_code_review_merge_request_widget_license_compliance_expand - aggregation: weekly -- name: i_code_review_merge_request_widget_license_compliance_expand_success - aggregation: weekly -- name: i_code_review_merge_request_widget_license_compliance_expand_warning - aggregation: weekly -- name: i_code_review_merge_request_widget_license_compliance_expand_failed - aggregation: weekly -## Security Reports -- name: i_code_review_merge_request_widget_security_reports_view - aggregation: weekly -- name: i_code_review_merge_request_widget_security_reports_full_report_clicked - aggregation: weekly -- name: i_code_review_merge_request_widget_security_reports_expand - aggregation: weekly -- name: i_code_review_merge_request_widget_security_reports_expand_success - aggregation: weekly -- name: i_code_review_merge_request_widget_security_reports_expand_warning - aggregation: weekly -- name: i_code_review_merge_request_widget_security_reports_expand_failed - aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml deleted file mode 100644 index 0583d85c3cc..00000000000 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ /dev/null @@ -1,167 +0,0 @@ ---- -# Compliance category -- name: g_edit_by_web_ide - aggregation: daily -- name: g_edit_by_sfe - aggregation: daily -- name: g_edit_by_snippet_ide - aggregation: daily -- name: g_edit_by_live_preview - aggregation: daily -- name: i_search_total - aggregation: weekly -- name: wiki_action - aggregation: daily -- name: design_action - aggregation: daily -- name: project_action - aggregation: daily -- name: git_write_action - aggregation: daily -- name: merge_request_action - aggregation: daily -- name: i_source_code_code_intelligence - aggregation: daily -# Incident management -- name: incident_management_alert_status_changed - aggregation: weekly -- name: incident_management_alert_assigned - aggregation: weekly -- name: incident_management_alert_todo - aggregation: weekly -- name: incident_management_incident_created - aggregation: weekly -- name: incident_management_incident_reopened - aggregation: weekly -- name: incident_management_incident_closed - aggregation: weekly -- name: incident_management_incident_assigned - aggregation: weekly -- name: incident_management_incident_todo - aggregation: weekly -- name: incident_management_incident_comment - aggregation: weekly -- name: incident_management_incident_zoom_meeting - aggregation: weekly -- name: incident_management_incident_relate - aggregation: weekly -- name: incident_management_incident_unrelate - aggregation: weekly -- name: incident_management_incident_change_confidential - aggregation: weekly -# Incident management timeline events -- name: incident_management_timeline_event_created - aggregation: weekly -- name: incident_management_timeline_event_edited - aggregation: weekly -- name: incident_management_timeline_event_deleted - aggregation: weekly -# Incident management alerts -- name: incident_management_alert_create_incident - aggregation: weekly -# Testing category -- name: i_testing_test_case_parsed - aggregation: weekly -- name: i_testing_test_report_uploaded - aggregation: weekly -- name: i_testing_coverage_report_uploaded - aggregation: weekly -# Project Management group -- name: g_project_management_issue_title_changed - aggregation: daily -- name: g_project_management_issue_description_changed - aggregation: daily -- name: g_project_management_issue_assignee_changed - aggregation: daily -- name: g_project_management_issue_made_confidential - aggregation: daily -- name: g_project_management_issue_made_visible - aggregation: daily -- name: g_project_management_issue_created - aggregation: daily -- name: g_project_management_issue_closed - aggregation: daily -- name: g_project_management_issue_reopened - aggregation: daily -- name: g_project_management_issue_label_changed - aggregation: daily -- name: g_project_management_issue_milestone_changed - aggregation: daily -- name: g_project_management_issue_cross_referenced - aggregation: daily -- name: g_project_management_issue_moved - aggregation: daily -- name: g_project_management_issue_related - aggregation: daily -- name: g_project_management_issue_unrelated - aggregation: daily -- name: g_project_management_issue_marked_as_duplicate - aggregation: daily -- name: g_project_management_issue_locked - aggregation: daily -- name: g_project_management_issue_unlocked - aggregation: daily -- name: g_project_management_issue_designs_added - aggregation: daily -- name: g_project_management_issue_designs_modified - aggregation: daily -- name: g_project_management_issue_designs_removed - aggregation: daily -- name: g_project_management_issue_due_date_changed - aggregation: daily -- name: g_project_management_issue_design_comments_removed - aggregation: daily -- name: g_project_management_issue_time_estimate_changed - aggregation: daily -- name: g_project_management_issue_time_spent_changed - aggregation: daily -- name: g_project_management_issue_comment_added - aggregation: daily -- name: g_project_management_issue_comment_edited - aggregation: daily -- name: g_project_management_issue_comment_removed - aggregation: daily -- name: g_project_management_issue_cloned - aggregation: daily -# Runner group -- name: g_runner_fleet_read_jobs_statistics - aggregation: weekly -# Secrets Management -- name: i_snippets_show - aggregation: weekly -# Terraform -- name: p_terraform_state_api_unique_users - aggregation: weekly -# Pipeline Authoring group -- name: ci_interpolation_users - aggregation: weekly -- name: o_pipeline_authoring_unique_users_committing_ciconfigfile - aggregation: weekly -- name: o_pipeline_authoring_unique_users_pushing_mr_ciconfigfile - aggregation: weekly -- name: i_ci_secrets_management_id_tokens_build_created - aggregation: weekly -# Merge request widgets -- name: users_expanding_secure_security_report - aggregation: weekly -- name: users_expanding_testing_code_quality_report - aggregation: weekly -- name: users_expanding_testing_accessibility_report - aggregation: weekly -- name: users_expanding_testing_license_compliance_report - aggregation: weekly -- name: users_visiting_testing_license_compliance_full_report - aggregation: weekly -- name: users_visiting_testing_manage_license_compliance - aggregation: weekly -- name: users_clicking_license_testing_visiting_external_website - aggregation: weekly -# Geo group -- name: g_geo_proxied_requests - aggregation: daily -# Manage -- name: unique_active_user - aggregation: weekly -# Environments page -- name: users_visiting_environments_pages - aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/container_registry_events.yml b/lib/gitlab/usage_data_counters/known_events/container_registry_events.yml deleted file mode 100644 index aa0f9965fa7..00000000000 --- a/lib/gitlab/usage_data_counters/known_events/container_registry_events.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -- name: i_container_registry_push_tag_user - aggregation: weekly -- name: i_container_registry_delete_tag_user - aggregation: weekly -- name: i_container_registry_push_repository_user - aggregation: weekly -- name: i_container_registry_delete_repository_user - aggregation: weekly -- name: i_container_registry_create_repository_user - aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/ecosystem.yml b/lib/gitlab/usage_data_counters/known_events/ecosystem.yml deleted file mode 100644 index 6e4a893d19a..00000000000 --- a/lib/gitlab/usage_data_counters/known_events/ecosystem.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- -# Ecosystem category -- name: i_ecosystem_jira_service_close_issue - aggregation: weekly -- name: i_ecosystem_jira_service_cross_reference - aggregation: weekly -- name: i_ecosystem_slack_service_issue_notification - aggregation: weekly -- name: i_ecosystem_slack_service_push_notification - aggregation: weekly -- name: i_ecosystem_slack_service_deployment_notification - aggregation: weekly -- name: i_ecosystem_slack_service_wiki_page_notification - aggregation: weekly -- name: i_ecosystem_slack_service_merge_request_notification - aggregation: weekly -- name: i_ecosystem_slack_service_note_notification - aggregation: weekly -- name: i_ecosystem_slack_service_tag_push_notification - aggregation: weekly -- name: i_ecosystem_slack_service_confidential_note_notification - aggregation: weekly -- name: i_ecosystem_slack_service_confidential_issue_notification - aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/error_tracking.yml b/lib/gitlab/usage_data_counters/known_events/error_tracking.yml deleted file mode 100644 index ebfd1b274f9..00000000000 --- a/lib/gitlab/usage_data_counters/known_events/error_tracking.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -- name: error_tracking_view_details - aggregation: weekly -- name: error_tracking_view_list - aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/importer_events.yml b/lib/gitlab/usage_data_counters/known_events/importer_events.yml deleted file mode 100644 index abbd83a012b..00000000000 --- a/lib/gitlab/usage_data_counters/known_events/importer_events.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -# Importer events -- name: github_import_project_start - aggregation: weekly -- name: github_import_project_success - aggregation: weekly -- name: github_import_project_failure - aggregation: weekly -- name: github_import_project_cancelled - aggregation: weekly -- name: github_import_project_partially_completed - aggregation: weekly - diff --git a/lib/gitlab/usage_data_counters/known_events/integrations.yml b/lib/gitlab/usage_data_counters/known_events/integrations.yml deleted file mode 100644 index 4a83581e9f0..00000000000 --- a/lib/gitlab/usage_data_counters/known_events/integrations.yml +++ /dev/null @@ -1,18 +0,0 @@ -- name: i_integrations_gitlab_for_slack_app_issue_notification - aggregation: weekly -- name: i_integrations_gitlab_for_slack_app_push_notification - aggregation: weekly -- name: i_integrations_gitlab_for_slack_app_deployment_notification - aggregation: weekly -- name: i_integrations_gitlab_for_slack_app_wiki_page_notification - aggregation: weekly -- name: i_integrations_gitlab_for_slack_app_merge_request_notification - aggregation: weekly -- name: i_integrations_gitlab_for_slack_app_note_notification - aggregation: weekly -- name: i_integrations_gitlab_for_slack_app_tag_push_notification - aggregation: weekly -- name: i_integrations_gitlab_for_slack_app_confidential_note_notification - aggregation: weekly -- name: i_integrations_gitlab_for_slack_app_confidential_issue_notification - aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/kubernetes_agent.yml b/lib/gitlab/usage_data_counters/known_events/kubernetes_agent.yml deleted file mode 100644 index fe779a9a25f..00000000000 --- a/lib/gitlab/usage_data_counters/known_events/kubernetes_agent.yml +++ /dev/null @@ -1,22 +0,0 @@ -- name: agent_users_using_ci_tunnel - aggregation: weekly -- name: k8s_api_proxy_requests_unique_users_via_ci_access - aggregation: weekly -- name: k8s_api_proxy_requests_unique_users_via_ci_access - aggregation: monthly -- name: k8s_api_proxy_requests_unique_agents_via_ci_access - aggregation: weekly -- name: k8s_api_proxy_requests_unique_agents_via_ci_access - aggregation: monthly -- name: k8s_api_proxy_requests_unique_users_via_user_access - aggregation: weekly -- name: k8s_api_proxy_requests_unique_users_via_user_access - aggregation: monthly -- name: k8s_api_proxy_requests_unique_agents_via_user_access - aggregation: weekly -- name: k8s_api_proxy_requests_unique_agents_via_user_access - aggregation: monthly -- name: flux_git_push_notified_unique_projects - aggregation: weekly -- name: flux_git_push_notified_unique_projects - aggregation: monthly diff --git a/lib/gitlab/usage_data_counters/known_events/package_events.yml b/lib/gitlab/usage_data_counters/known_events/package_events.yml deleted file mode 100644 index fa99798cde0..00000000000 --- a/lib/gitlab/usage_data_counters/known_events/package_events.yml +++ /dev/null @@ -1,49 +0,0 @@ ---- -- name: i_package_composer_deploy_token - aggregation: weekly -- name: i_package_composer_user - aggregation: weekly -- name: i_package_conan_deploy_token - aggregation: weekly -- name: i_package_conan_user - aggregation: weekly -- name: i_package_debian_deploy_token - aggregation: weekly -- name: i_package_debian_user - aggregation: weekly -- name: i_package_generic_deploy_token - aggregation: weekly -- name: i_package_generic_user - aggregation: weekly -- name: i_package_helm_deploy_token - aggregation: weekly -- name: i_package_helm_user - aggregation: weekly -- name: i_package_maven_deploy_token - aggregation: weekly -- name: i_package_maven_user - aggregation: weekly -- name: i_package_npm_deploy_token - aggregation: weekly -- name: i_package_npm_user - aggregation: weekly -- name: i_package_nuget_deploy_token - aggregation: weekly -- name: i_package_nuget_user - aggregation: weekly -- name: i_package_pypi_deploy_token - aggregation: weekly -- name: i_package_pypi_user - aggregation: weekly -- name: i_package_rubygems_deploy_token - aggregation: weekly -- name: i_package_rubygems_user - aggregation: weekly -- name: i_package_terraform_module_deploy_token - aggregation: weekly -- name: i_package_terraform_module_user - aggregation: weekly -- name: i_package_rpm_user - aggregation: weekly -- name: i_package_rpm_deploy_token - aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/product_analytics.yml b/lib/gitlab/usage_data_counters/known_events/product_analytics.yml deleted file mode 100644 index c43bf9040dd..00000000000 --- a/lib/gitlab/usage_data_counters/known_events/product_analytics.yml +++ /dev/null @@ -1,8 +0,0 @@ -- name: project_created_analytics_dashboard - aggregation: weekly -- name: project_initialized_product_analytics - aggregation: weekly -- name: user_created_analytics_dashboard - aggregation: weekly -- name: user_visited_dashboard - aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/quickactions.yml b/lib/gitlab/usage_data_counters/known_events/quickactions.yml deleted file mode 100644 index 69f92ac5c0a..00000000000 --- a/lib/gitlab/usage_data_counters/known_events/quickactions.yml +++ /dev/null @@ -1,139 +0,0 @@ ---- -- name: i_quickactions_assign_multiple - aggregation: weekly -- name: i_quickactions_approve - aggregation: weekly -- name: i_quickactions_unapprove - aggregation: weekly -- name: i_quickactions_assign_single - aggregation: weekly -- name: i_quickactions_assign_self - aggregation: weekly -- name: i_quickactions_assign_reviewer - aggregation: weekly -- name: i_quickactions_award - aggregation: weekly -- name: i_quickactions_board_move - aggregation: weekly -- name: i_quickactions_clone - aggregation: weekly -- name: i_quickactions_close - aggregation: weekly -- name: i_quickactions_confidential - aggregation: weekly -- name: i_quickactions_copy_metadata_merge_request - aggregation: weekly -- name: i_quickactions_copy_metadata_issue - aggregation: weekly -- name: i_quickactions_create_merge_request - aggregation: weekly -- name: i_quickactions_done - aggregation: weekly -- name: i_quickactions_draft - aggregation: weekly -- name: i_quickactions_due - aggregation: weekly -- name: i_quickactions_duplicate - aggregation: weekly -- name: i_quickactions_estimate - aggregation: weekly -- name: i_quickactions_label - aggregation: weekly -- name: i_quickactions_lock - aggregation: weekly -- name: i_quickactions_merge - aggregation: weekly -- name: i_quickactions_milestone - aggregation: weekly -- name: i_quickactions_move - aggregation: weekly -- name: i_quickactions_promote_to_incident - aggregation: weekly -- name: i_quickactions_timeline - aggregation: weekly -- name: i_quickactions_ready - aggregation: weekly -- name: i_quickactions_reassign - aggregation: weekly -- name: i_quickactions_reassign_reviewer - aggregation: weekly -- name: i_quickactions_rebase - aggregation: weekly -- name: i_quickactions_relabel - aggregation: weekly -- name: i_quickactions_relate - aggregation: weekly -- name: i_quickactions_remove_due_date - aggregation: weekly -- name: i_quickactions_remove_estimate - aggregation: weekly -- name: i_quickactions_remove_milestone - aggregation: weekly -- name: i_quickactions_remove_time_spent - aggregation: weekly -- name: i_quickactions_remove_zoom - aggregation: weekly -- name: i_quickactions_reopen - aggregation: weekly -- name: i_quickactions_severity - aggregation: weekly -- name: i_quickactions_shrug - aggregation: weekly -- name: i_quickactions_spend_subtract - aggregation: weekly -- name: i_quickactions_spend_add - aggregation: weekly -- name: i_quickactions_submit_review - aggregation: weekly -- name: i_quickactions_subscribe - aggregation: weekly -- name: i_quickactions_summarize_diff - aggregation: weekly -- name: i_quickactions_tableflip - aggregation: weekly -- name: i_quickactions_tag - aggregation: weekly -- name: i_quickactions_target_branch - aggregation: weekly -- name: i_quickactions_title - aggregation: weekly -- name: i_quickactions_todo - aggregation: weekly -- name: i_quickactions_unassign_specific - aggregation: weekly -- name: i_quickactions_unassign_all - aggregation: weekly -- name: i_quickactions_unassign_reviewer - aggregation: weekly -- name: i_quickactions_unlabel_specific - aggregation: weekly -- name: i_quickactions_unlabel_all - aggregation: weekly -- name: i_quickactions_unlock - aggregation: weekly -- name: i_quickactions_unsubscribe - aggregation: weekly -- name: i_quickactions_wip - aggregation: weekly -- name: i_quickactions_zoom - aggregation: weekly -- name: i_quickactions_link - aggregation: weekly -- name: i_quickactions_invite_email_single - aggregation: weekly -- name: i_quickactions_invite_email_multiple - aggregation: weekly -- name: i_quickactions_add_contacts - aggregation: weekly -- name: i_quickactions_remove_contacts - aggregation: weekly -- name: i_quickactions_type - aggregation: weekly -- name: i_quickactions_blocked_by - aggregation: weekly -- name: i_quickactions_blocks - aggregation: weekly -- name: i_quickactions_unlink - aggregation: weekly -- name: i_quickactions_promote_to - aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/work_items.yml b/lib/gitlab/usage_data_counters/known_events/work_items.yml deleted file mode 100644 index a6e5b9e1af5..00000000000 --- a/lib/gitlab/usage_data_counters/known_events/work_items.yml +++ /dev/null @@ -1,21 +0,0 @@ ---- -- name: users_updating_work_item_title - aggregation: weekly -- name: users_creating_work_items - aggregation: weekly -- name: users_updating_work_item_dates - aggregation: weekly -- name: users_updating_work_item_labels - aggregation: weekly -- name: users_updating_work_item_milestone - aggregation: weekly -- name: users_updating_work_item_iteration - # The event tracks an EE feature. - # It's added here so it can be aggregated into the CE/EE 'OR' aggregate metrics. - # It will report 0 for CE instances and should not be used with 'AND' aggregators. - aggregation: weekly -- name: users_updating_weight_estimate - # The event tracks an EE feature. - # It's added here so it can be aggregated into the CE/EE 'OR' aggregate metrics. - # It will report 0 for CE instances and should not be used with 'AND' aggregators. - aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/workspaces.yml b/lib/gitlab/usage_data_counters/known_events/workspaces.yml deleted file mode 100644 index 8a96524b167..00000000000 --- a/lib/gitlab/usage_data_counters/known_events/workspaces.yml +++ /dev/null @@ -1,5 +0,0 @@ -- name: users_updating_workspaces - aggregation: weekly - -- name: users_creating_workspaces - aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/neovim_plugin_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/neovim_plugin_activity_unique_counter.rb new file mode 100644 index 00000000000..7cf89a96e5d --- /dev/null +++ b/lib/gitlab/usage_data_counters/neovim_plugin_activity_unique_counter.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + module NeovimPluginActivityUniqueCounter + NEOVIM_PLUGIN_API_REQUEST_ACTION = 'i_editor_extensions_user_neovim_plugin_api_request' + NEOVIM_PLUGIN_USER_AGENT_REGEX = /gitlab.vim/ + + class << self + def track_api_request_when_trackable(user_agent:, user:) + user_agent&.match?(NEOVIM_PLUGIN_USER_AGENT_REGEX) && + track_unique_action_by_user(NEOVIM_PLUGIN_API_REQUEST_ACTION, user) + end + + private + + def track_unique_action_by_user(action, user) + return unless user + + track_unique_action(action, user.id) + end + + def track_unique_action(action, value) + Gitlab::UsageDataCounters::HLLRedisCounter.track_usage_event(action, value) + end + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/visual_studio_extension_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/visual_studio_extension_activity_unique_counter.rb new file mode 100644 index 00000000000..771e466f5f9 --- /dev/null +++ b/lib/gitlab/usage_data_counters/visual_studio_extension_activity_unique_counter.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + module VisualStudioExtensionActivityUniqueCounter + VISUAL_STUDIO_EXTENSION_API_REQUEST_ACTION = 'i_editor_extensions_user_visual_studio_api_request' + VISUAL_STUDIO_EXTENSION_USER_AGENT_REGEX = /gl-visual-studio-extension/ + + class << self + def track_api_request_when_trackable(user_agent:, user:) + user_agent&.match?(VISUAL_STUDIO_EXTENSION_USER_AGENT_REGEX) && + track_unique_action_by_user(VISUAL_STUDIO_EXTENSION_API_REQUEST_ACTION, user) + end + + private + + def track_unique_action_by_user(action, user) + return unless user + + track_unique_action(action, user.id) + end + + def track_unique_action(action, value) + Gitlab::UsageDataCounters::HLLRedisCounter.track_usage_event(action, value) + end + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb index 9de575d8567..b99c9ebb24f 100644 --- a/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb @@ -33,7 +33,7 @@ module Gitlab private def track_unique_action(action, author) - return unless author && Feature.enabled?(:track_work_items_activity) + return unless author Gitlab::UsageDataCounters::HLLRedisCounter.track_event(action, values: author.id) end diff --git a/lib/gitlab/utils/markdown.rb b/lib/gitlab/utils/markdown.rb index c95398a15df..c041953e7c8 100644 --- a/lib/gitlab/utils/markdown.rb +++ b/lib/gitlab/utils/markdown.rb @@ -4,7 +4,7 @@ module Gitlab module Utils module Markdown PUNCTUATION_REGEXP = /[^\p{Word}\- ]/u.freeze - PRODUCT_SUFFIX = /\s*\**\((core|starter|premium|ultimate|free|bronze|silver|gold)(\s+(only|self|saas))?\)\**/.freeze + PRODUCT_SUFFIX = /\s*\**\((core|starter|premium|ultimate|free|bronze|silver|gold)(\s+(all|only|self|saas))?\)\**/.freeze def string_to_anchor(string) string diff --git a/lib/gitlab/web_hooks/logger.rb b/lib/gitlab/web_hooks/logger.rb new file mode 100644 index 00000000000..010e40a3dab --- /dev/null +++ b/lib/gitlab/web_hooks/logger.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Gitlab + module WebHooks + class Logger < ::Gitlab::JsonLogger + def self.file_name_noext + 'web_hooks' + end + end + end +end diff --git a/lib/gitlab/with_request_store.rb b/lib/gitlab/with_request_store.rb deleted file mode 100644 index d13cd9a72f7..00000000000 --- a/lib/gitlab/with_request_store.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module WithRequestStore - def with_request_store(&block) - # Skip enabling the request store if it was already active. Whatever - # instantiated the request store first is responsible for clearing it - return yield if RequestStore.active? - - enabling_request_store(&block) - end - - private - - def enabling_request_store - RequestStore.begin! - yield - ensure - RequestStore.end! - RequestStore.clear! - end - - extend self - end -end diff --git a/lib/gitlab/x509/signature.rb b/lib/gitlab/x509/signature.rb index d101a6d2522..649e5379927 100644 --- a/lib/gitlab/x509/signature.rb +++ b/lib/gitlab/x509/signature.rb @@ -42,11 +42,13 @@ module Gitlab !verified_signature || signed_by_user.nil? - if signed_by_user.verified_emails.include?(@email.downcase) && certificate_email.casecmp?(@email) - :verified - else - :unverified + if signed_by_user.verified_emails.include?(@email.downcase) + return :verified if certificate_emails.find do |ce| + ce.casecmp?(@email) + end end + + :unverified end private @@ -173,18 +175,13 @@ module Gitlab end def certificate_email - email = nil + certificate_emails.first + end - get_certificate_extension('subjectAltName').split(',').each do |item| - if item.strip.start_with?("email") - email = item.split('email:')[1] - break - end + def certificate_emails + get_certificate_extension('subjectAltName').split(',').each.with_object([]) do |item, emails| + emails << item.split('email:')[1] if item.strip.start_with?("email") end - - return if email.nil? - - email end def x509_issuer @@ -206,6 +203,7 @@ module Gitlab subject_key_identifier: certificate_subject_key_identifier, subject: certificate_subject, email: certificate_email, + emails: certificate_emails, serial_number: cert.serial.to_i, x509_issuer_id: x509_issuer.id } |