From edaa33dee2ff2f7ea3fac488d41558eb5f86d68c Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Thu, 20 Jan 2022 09:16:11 +0000 Subject: Add latest changes from gitlab-org/gitlab@14-7-stable-ee --- lib/gitlab/anonymous_session.rb | 8 +- lib/gitlab/application_rate_limiter.rb | 10 +- .../syntax_highlighter/html_pipeline_adapter.rb | 6 +- lib/gitlab/auth.rb | 12 +- lib/gitlab/auth/auth_finders.rb | 2 +- lib/gitlab/auth/ldap/config.rb | 3 +- lib/gitlab/auth/o_auth/user.rb | 4 +- .../backfill_ci_namespace_mirrors.rb | 77 +++++++ .../backfill_ci_project_mirrors.rb | 52 +++++ .../backfill_incident_issue_escalation_statuses.rb | 32 +++ lib/gitlab/background_migration/base_job.rb | 23 +++ .../cleanup_concurrent_rename.rb | 14 -- .../cleanup_concurrent_schema_change.rb | 56 ----- .../cleanup_concurrent_type_change.rb | 14 -- lib/gitlab/background_migration/copy_column.rb | 41 ---- .../encrypt_static_object_token.rb | 70 +++++++ ...lity_occurrences_with_hashes_as_raw_metadata.rb | 124 +++++++++++ lib/gitlab/background_migration/job_coordinator.rb | 14 +- .../migrate_legacy_artifacts.rb | 130 ------------ .../populate_test_reports_issue_id.rb | 14 ++ ...recalculate_vulnerabilities_occurrences_uuid.rb | 148 +++++++++++-- .../remove_duplicate_services.rb | 58 ------ lib/gitlab/checks/changes_access.rb | 35 +++- lib/gitlab/ci/build/policy/refs.rb | 5 +- lib/gitlab/ci/build/status/reason.rb | 37 ++++ lib/gitlab/ci/config.rb | 2 +- lib/gitlab/ci/config/entry/root.rb | 8 +- lib/gitlab/ci/jwt_v2.rb | 17 ++ lib/gitlab/ci/pipeline/chain/create.rb | 48 ++--- lib/gitlab/ci/pipeline/chain/create_deployments.rb | 15 +- lib/gitlab/ci/pipeline/chain/seed.rb | 9 +- lib/gitlab/ci/pipeline/logger.rb | 36 +++- lib/gitlab/ci/pipeline/seed/build.rb | 16 +- lib/gitlab/ci/pipeline/seed/context.rb | 11 +- lib/gitlab/ci/queue/metrics.rb | 37 +++- lib/gitlab/ci/status/build/factory.rb | 3 +- lib/gitlab/ci/status/build/waiting_for_approval.rb | 24 +++ lib/gitlab/ci/tags/bulk_insert.rb | 20 +- .../ci/templates/Jobs/Code-Quality.gitlab-ci.yml | 2 +- .../templates/Jobs/Secret-Detection.gitlab-ci.yml | 16 +- lib/gitlab/ci/templates/Ruby.gitlab-ci.yml | 2 +- .../Security/Coverage-Fuzzing.gitlab-ci.yml | 1 + .../Security/DAST-On-Demand-API-Scan.gitlab-ci.yml | 27 +++ .../ci/templates/Terraform.latest.gitlab-ci.yml | 2 + lib/gitlab/ci/trace/remote_checksum.rb | 1 - lib/gitlab/ci/trace/stream.rb | 5 +- lib/gitlab/ci/variables/builder.rb | 64 ++++++ lib/gitlab/ci/yaml_processor.rb | 8 + lib/gitlab/color_schemes.rb | 28 +-- lib/gitlab/config/entry/configurable.rb | 3 +- lib/gitlab/config/entry/factory.rb | 5 + lib/gitlab/config/entry/node.rb | 20 +- .../content_security_policy/config_loader.rb | 2 +- lib/gitlab/data_builder/archive_trace.rb | 19 ++ lib/gitlab/data_builder/deployment.rb | 3 +- .../database/background_migration/batched_job.rb | 18 +- .../background_migration/batched_migration.rb | 2 +- lib/gitlab/database/background_migration_job.rb | 2 +- lib/gitlab/database/batch_counter.rb | 29 +-- lib/gitlab/database/gitlab_loose_foreign_keys.yml | 93 ++++++++- lib/gitlab/database/gitlab_schemas.yml | 12 +- lib/gitlab/database/grant.rb | 2 +- lib/gitlab/database/load_balancing/setup.rb | 4 +- .../database/loose_index_scan_distinct_count.rb | 102 --------- lib/gitlab/database/migration_helpers.rb | 180 ---------------- .../migrations/background_migration_helpers.rb | 70 +++---- .../database/partitioning/partition_manager.rb | 4 + .../database/partitioning/sliding_list_strategy.rb | 28 ++- .../backfill_partitioned_table.rb | 20 +- .../table_management_helpers.rb | 3 +- lib/gitlab/database/reflection.rb | 29 +++ lib/gitlab/database/reindexing.rb | 15 +- lib/gitlab/database/reindexing/coordinator.rb | 19 ++ .../work_items/base_type_importer.rb | 4 +- lib/gitlab/email.rb | 1 + lib/gitlab/email/failure_handler.rb | 46 +++++ .../error_tracking/processor/sidekiq_processor.rb | 2 + lib/gitlab/event_store.rb | 42 ++++ lib/gitlab/event_store/event.rb | 54 +++++ lib/gitlab/event_store/store.rb | 54 +++++ lib/gitlab/event_store/subscriber.rb | 36 ++++ lib/gitlab/event_store/subscription.rb | 37 ++++ lib/gitlab/exceptions_app.rb | 43 ++++ lib/gitlab/experimentation.rb | 8 - lib/gitlab/gon_helper.rb | 1 + lib/gitlab/gpg/commit.rb | 2 +- lib/gitlab/http.rb | 3 +- lib/gitlab/i18n.rb | 22 +- lib/gitlab/import/set_async_jid.rb | 2 +- lib/gitlab/import_export/base/relation_factory.rb | 2 +- .../import_export/group/relation_tree_restorer.rb | 10 +- lib/gitlab/import_export/project/import_export.yml | 2 + lib/gitlab/jwt_authenticatable.rb | 36 ++-- lib/gitlab/kas.rb | 2 +- lib/gitlab/lfs/client.rb | 81 ++++++-- lib/gitlab/logger.rb | 6 +- lib/gitlab/mail_room.rb | 16 +- lib/gitlab/mail_room/authenticator.rb | 50 +++++ .../merge_requests/commit_message_generator.rb | 72 ++++--- lib/gitlab/metrics/exporter/base_exporter.rb | 45 ++-- .../metrics/exporter/gc_request_middleware.rb | 19 ++ .../metrics/exporter/health_checks_middleware.rb | 35 ++++ lib/gitlab/metrics/exporter/metrics_middleware.rb | 41 ++++ lib/gitlab/metrics/exporter/sidekiq_exporter.rb | 11 +- lib/gitlab/metrics/exporter/web_exporter.rb | 8 +- .../metrics/samplers/action_cable_sampler.rb | 4 +- lib/gitlab/metrics/samplers/base_sampler.rb | 14 +- lib/gitlab/metrics/samplers/ruby_sampler.rb | 4 +- lib/gitlab/middleware/multipart.rb | 4 +- .../middleware/webhook_recursion_detection.rb | 19 ++ lib/gitlab/pages.rb | 2 +- .../pagination/keyset/column_order_definition.rb | 25 ++- .../keyset/in_operator_optimization/column_data.rb | 19 +- .../order_by_column_data.rb | 37 ++++ .../in_operator_optimization/order_by_columns.rb | 6 +- .../in_operator_optimization/query_builder.rb | 4 +- .../strategies/order_values_loader_strategy.rb | 15 +- .../strategies/record_loader_strategy.rb | 16 ++ .../pagination/keyset/sql_type_missing_error.rb | 19 ++ lib/gitlab/password.rb | 14 ++ lib/gitlab/redis/multi_store.rb | 229 --------------------- lib/gitlab/redis/sessions.rb | 36 +--- lib/gitlab/redis/sessions_store_helper.rb | 27 --- lib/gitlab/regex.rb | 26 ++- lib/gitlab/repository_archive_rate_limiter.rb | 2 +- lib/gitlab/search/params.rb | 11 +- lib/gitlab/sherlock.rb | 21 -- lib/gitlab/sherlock/collection.rb | 51 ----- lib/gitlab/sherlock/file_sample.rb | 33 --- lib/gitlab/sherlock/line_profiler.rb | 100 --------- lib/gitlab/sherlock/line_sample.rb | 38 ---- lib/gitlab/sherlock/location.rb | 28 --- lib/gitlab/sherlock/middleware.rb | 43 ---- lib/gitlab/sherlock/query.rb | 112 ---------- lib/gitlab/sherlock/transaction.rb | 140 ------------- lib/gitlab/sidekiq_logging/json_formatter.rb | 1 + lib/gitlab/sidekiq_logging/structured_logger.rb | 2 + lib/gitlab/sidekiq_middleware/monitor.rb | 2 + lib/gitlab/sidekiq_status.rb | 22 +- lib/gitlab/sidekiq_status/client_middleware.rb | 4 +- lib/gitlab/sourcegraph.rb | 7 +- lib/gitlab/ssh_public_key.rb | 26 ++- lib/gitlab/themes.rb | 40 ++-- lib/gitlab/tracking/standard_context.rb | 5 +- lib/gitlab/untrusted_regexp/ruby_syntax.rb | 16 +- lib/gitlab/usage_data.rb | 33 +-- .../counter_events/package_events.yml | 7 - .../usage_data_counters/hll_redis_counter.rb | 2 +- .../known_events/ci_templates.yml | 8 + .../usage_data_counters/known_events/common.yml | 10 + .../known_events/package_events.yml | 32 --- .../known_events/quickactions.yml | 8 + lib/gitlab/utils/sanitize_node_link.rb | 6 + lib/gitlab/utils/usage_data.rb | 12 +- lib/gitlab/web_hooks.rb | 7 + lib/gitlab/web_hooks/recursion_detection.rb | 94 +++++++++ lib/gitlab/web_hooks/recursion_detection/uuid.rb | 46 +++++ lib/gitlab/workhorse.rb | 6 +- 158 files changed, 2349 insertions(+), 2014 deletions(-) create mode 100644 lib/gitlab/background_migration/backfill_ci_namespace_mirrors.rb create mode 100644 lib/gitlab/background_migration/backfill_ci_project_mirrors.rb create mode 100644 lib/gitlab/background_migration/backfill_incident_issue_escalation_statuses.rb create mode 100644 lib/gitlab/background_migration/base_job.rb delete mode 100644 lib/gitlab/background_migration/cleanup_concurrent_rename.rb delete mode 100644 lib/gitlab/background_migration/cleanup_concurrent_schema_change.rb delete mode 100644 lib/gitlab/background_migration/cleanup_concurrent_type_change.rb delete mode 100644 lib/gitlab/background_migration/copy_column.rb create mode 100644 lib/gitlab/background_migration/encrypt_static_object_token.rb create mode 100644 lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb delete mode 100644 lib/gitlab/background_migration/migrate_legacy_artifacts.rb create mode 100644 lib/gitlab/background_migration/populate_test_reports_issue_id.rb delete mode 100644 lib/gitlab/background_migration/remove_duplicate_services.rb create mode 100644 lib/gitlab/ci/build/status/reason.rb create mode 100644 lib/gitlab/ci/jwt_v2.rb create mode 100644 lib/gitlab/ci/status/build/waiting_for_approval.rb create mode 100644 lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml create mode 100644 lib/gitlab/data_builder/archive_trace.rb delete mode 100644 lib/gitlab/database/loose_index_scan_distinct_count.rb create mode 100644 lib/gitlab/email/failure_handler.rb create mode 100644 lib/gitlab/event_store.rb create mode 100644 lib/gitlab/event_store/event.rb create mode 100644 lib/gitlab/event_store/store.rb create mode 100644 lib/gitlab/event_store/subscriber.rb create mode 100644 lib/gitlab/event_store/subscription.rb create mode 100644 lib/gitlab/exceptions_app.rb create mode 100644 lib/gitlab/mail_room/authenticator.rb create mode 100644 lib/gitlab/metrics/exporter/gc_request_middleware.rb create mode 100644 lib/gitlab/metrics/exporter/health_checks_middleware.rb create mode 100644 lib/gitlab/metrics/exporter/metrics_middleware.rb create mode 100644 lib/gitlab/middleware/webhook_recursion_detection.rb create mode 100644 lib/gitlab/pagination/keyset/in_operator_optimization/order_by_column_data.rb create mode 100644 lib/gitlab/pagination/keyset/sql_type_missing_error.rb create mode 100644 lib/gitlab/password.rb delete mode 100644 lib/gitlab/redis/multi_store.rb delete mode 100644 lib/gitlab/redis/sessions_store_helper.rb delete mode 100644 lib/gitlab/sherlock.rb delete mode 100644 lib/gitlab/sherlock/collection.rb delete mode 100644 lib/gitlab/sherlock/file_sample.rb delete mode 100644 lib/gitlab/sherlock/line_profiler.rb delete mode 100644 lib/gitlab/sherlock/line_sample.rb delete mode 100644 lib/gitlab/sherlock/location.rb delete mode 100644 lib/gitlab/sherlock/middleware.rb delete mode 100644 lib/gitlab/sherlock/query.rb delete mode 100644 lib/gitlab/sherlock/transaction.rb create mode 100644 lib/gitlab/web_hooks.rb create mode 100644 lib/gitlab/web_hooks/recursion_detection.rb create mode 100644 lib/gitlab/web_hooks/recursion_detection/uuid.rb (limited to 'lib/gitlab') diff --git a/lib/gitlab/anonymous_session.rb b/lib/gitlab/anonymous_session.rb index e58240e16b4..6904945a755 100644 --- a/lib/gitlab/anonymous_session.rb +++ b/lib/gitlab/anonymous_session.rb @@ -2,14 +2,12 @@ module Gitlab class AnonymousSession - include ::Gitlab::Redis::SessionsStoreHelper - def initialize(remote_ip) @remote_ip = remote_ip end def count_session_ip - redis_store_class.with do |redis| + Gitlab::Redis::Sessions.with do |redis| redis.pipelined do |pipeline| pipeline.incr(session_lookup_name) pipeline.expire(session_lookup_name, 24.hours) @@ -18,13 +16,13 @@ module Gitlab end def session_count - redis_store_class.with do |redis| + Gitlab::Redis::Sessions.with do |redis| redis.get(session_lookup_name).to_i end end def cleanup_session_per_ip_count - redis_store_class.with do |redis| + Gitlab::Redis::Sessions.with do |redis| redis.del(session_lookup_name) end end diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index fb90ad9e275..12f1b15f820 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -49,9 +49,15 @@ module Gitlab group_testing_hook: { threshold: 5, interval: 1.minute }, profile_add_new_email: { threshold: 5, interval: 1.minute }, web_hook_calls: { interval: 1.minute }, + users_get_by_id: { threshold: 10, interval: 1.minute }, + username_exists: { threshold: 20, interval: 1.minute }, + user_sign_up: { threshold: 20, interval: 1.minute }, profile_resend_email_confirmation: { threshold: 5, interval: 1.minute }, + profile_update_username: { threshold: 10, interval: 1.minute }, update_environment_canary_ingress: { threshold: 1, interval: 1.minute }, - auto_rollback_deployment: { threshold: 1, interval: 3.minutes } + auto_rollback_deployment: { threshold: 1, interval: 3.minutes }, + user_email_lookup: { threshold: -> { application_settings.user_email_lookup_limit }, interval: 1.minute }, + gitlab_shell_operation: { threshold: 600, interval: 1.minute } }.freeze end @@ -59,7 +65,7 @@ module Gitlab # be throttled. # # @param key [Symbol] Key attribute registered in `.rate_limits` - # @param scope [Array] Array of ActiveRecord models to scope throttling to a specific request (e.g. per user per project) + # @param scope [Array] Array of ActiveRecord models, Strings or Symbols to scope throttling to a specific request (e.g. per user per project) # @param threshold [Integer] Optional threshold value to override default one registered in `.rate_limits` # @param users_allowlist [Array] Optional list of usernames to exclude from the limit. This param will only be functional if Scope includes a current user. # @param peek [Boolean] Optional. When true the key will not be incremented but the current throttled state will be returned. diff --git a/lib/gitlab/asciidoc/syntax_highlighter/html_pipeline_adapter.rb b/lib/gitlab/asciidoc/syntax_highlighter/html_pipeline_adapter.rb index 3ada3f947ee..b2e1c9e2379 100644 --- a/lib/gitlab/asciidoc/syntax_highlighter/html_pipeline_adapter.rb +++ b/lib/gitlab/asciidoc/syntax_highlighter/html_pipeline_adapter.rb @@ -7,11 +7,7 @@ module Gitlab register_for 'gitlab-html-pipeline' def format(node, lang, opts) - if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) - %(
#{node.content}
) - else - %(
#{node.content}
) - end + %(
#{node.content}
) end end end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 3e982168339..38bc50a2cb8 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -84,7 +84,7 @@ module Gitlab Gitlab::Auth::UniqueIpsLimiter.limit_user! do user = User.by_login(login) - break if user && !can_user_login_with_non_expired_password?(user) + break if user && !user.can_log_in_with_non_expired_password? authenticators = [] @@ -187,7 +187,7 @@ module Gitlab if valid_oauth_token?(token) user = User.id_in(token.resource_owner_id).first - return unless user && can_user_login_with_non_expired_password?(user) + return unless user && user.can_log_in_with_non_expired_password? Gitlab::Auth::Result.new(user, nil, :oauth, abilities_for_scopes(token.scopes)) end @@ -210,7 +210,7 @@ module Gitlab return unless token_bot_in_project?(token.user, project) || token_bot_in_group?(token.user, project) end - if can_user_login_with_non_expired_password?(token.user) || token.user.project_bot? + if token.user.can_log_in_with_non_expired_password? || token.user.project_bot? Gitlab::Auth::Result.new(token.user, nil, :personal_access_token, abilities_for_scopes(token.scopes)) end end @@ -309,7 +309,7 @@ module Gitlab return unless build.project.builds_enabled? if build.user - return unless can_user_login_with_non_expired_password?(build.user) || (build.user.project_bot? && build.project.bots&.include?(build.user)) + return unless build.user.can_log_in_with_non_expired_password? || (build.user.project_bot? && build.project.bots&.include?(build.user)) # If user is assigned to build, use restricted credentials of user Gitlab::Auth::Result.new(build.user, build.project, :build, build_authentication_abilities) @@ -406,10 +406,6 @@ module Gitlab user.increment_failed_attempts! end - - def can_user_login_with_non_expired_password?(user) - user.can?(:log_in) && !user.password_expired_if_applicable? - end end end end diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb index 9c33a5fc872..ecda96af403 100644 --- a/lib/gitlab/auth/auth_finders.rb +++ b/lib/gitlab/auth/auth_finders.rb @@ -165,7 +165,7 @@ module Gitlab authorization_token, _options = token_and_options(current_request) - ::Clusters::AgentToken.find_by_token(authorization_token) + ::Clusters::AgentToken.active.find_by_token(authorization_token) end def find_runner_from_token diff --git a/lib/gitlab/auth/ldap/config.rb b/lib/gitlab/auth/ldap/config.rb index 7bfe776fed0..82c6411c712 100644 --- a/lib/gitlab/auth/ldap/config.rb +++ b/lib/gitlab/auth/ldap/config.rb @@ -206,7 +206,8 @@ module Gitlab def base_options { host: options['host'], - port: options['port'] + port: options['port'], + hosts: options['hosts'] } end diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index feb5fea4c85..9f142727ebb 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -230,8 +230,8 @@ module Gitlab name: name.strip.presence || valid_username, username: valid_username, email: email, - password: auth_hash.password, - password_confirmation: auth_hash.password, + password: Gitlab::Password.test_default(21), + password_confirmation: Gitlab::Password.test_default(21), password_automatically_set: true } end diff --git a/lib/gitlab/background_migration/backfill_ci_namespace_mirrors.rb b/lib/gitlab/background_migration/backfill_ci_namespace_mirrors.rb new file mode 100644 index 00000000000..2247747ba08 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_ci_namespace_mirrors.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job to create ci_namespace_mirrors entries in batches + class BackfillCiNamespaceMirrors + class Namespace < ActiveRecord::Base # rubocop:disable Style/Documentation + include ::EachBatch + + self.table_name = 'namespaces' + self.inheritance_column = nil + + scope :base_query, -> do + select(:id, :parent_id) + end + end + + PAUSE_SECONDS = 0.1 + SUB_BATCH_SIZE = 500 + + def perform(start_id, end_id) + batch_query = Namespace.base_query.where(id: start_id..end_id) + batch_query.each_batch(of: SUB_BATCH_SIZE) do |sub_batch| + first, last = sub_batch.pluck(Arel.sql('MIN(id), MAX(id)')).first + ranged_query = Namespace.unscoped.base_query.where(id: first..last) + + update_sql = <<~SQL + INSERT INTO ci_namespace_mirrors (namespace_id, traversal_ids) + #{insert_values(ranged_query)} + ON CONFLICT (namespace_id) DO NOTHING + SQL + # We do nothing on conflict because we consider they were already filled. + + Namespace.connection.execute(update_sql) + + sleep PAUSE_SECONDS + end + + mark_job_as_succeeded(start_id, end_id) + end + + private + + def insert_values(batch) + calculated_traversal_ids( + batch.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336433') + ) + end + + # Copied from lib/gitlab/background_migration/backfill_namespace_traversal_ids_children.rb + def calculated_traversal_ids(batch) + <<~SQL + WITH RECURSIVE cte(source_id, namespace_id, parent_id, height) AS ( + ( + SELECT batch.id, batch.id, batch.parent_id, 1 + FROM (#{batch.to_sql}) AS batch + ) + UNION ALL + ( + SELECT cte.source_id, n.id, n.parent_id, cte.height+1 + FROM namespaces n, cte + WHERE n.id = cte.parent_id + ) + ) + SELECT flat_hierarchy.source_id as namespace_id, + array_agg(flat_hierarchy.namespace_id ORDER BY flat_hierarchy.height DESC) as traversal_ids + FROM (SELECT * FROM cte FOR UPDATE) flat_hierarchy + GROUP BY flat_hierarchy.source_id + SQL + end + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded('BackfillCiNamespaceMirrors', arguments) + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_ci_project_mirrors.rb b/lib/gitlab/background_migration/backfill_ci_project_mirrors.rb new file mode 100644 index 00000000000..ff6ab9928b0 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_ci_project_mirrors.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job to create ci_project_mirrors entries in batches + class BackfillCiProjectMirrors + class Project < ActiveRecord::Base # rubocop:disable Style/Documentation + include ::EachBatch + + self.table_name = 'projects' + + scope :base_query, -> do + select(:id, :namespace_id) + end + end + + PAUSE_SECONDS = 0.1 + SUB_BATCH_SIZE = 500 + + def perform(start_id, end_id) + batch_query = Project.base_query.where(id: start_id..end_id) + batch_query.each_batch(of: SUB_BATCH_SIZE) do |sub_batch| + first, last = sub_batch.pluck(Arel.sql('MIN(id), MAX(id)')).first + ranged_query = Project.unscoped.base_query.where(id: first..last) + + update_sql = <<~SQL + INSERT INTO ci_project_mirrors (project_id, namespace_id) + #{insert_values(ranged_query)} + ON CONFLICT (project_id) DO NOTHING + SQL + # We do nothing on conflict because we consider they were already filled. + + Project.connection.execute(update_sql) + + sleep PAUSE_SECONDS + end + + mark_job_as_succeeded(start_id, end_id) + end + + private + + def insert_values(batch) + batch.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336433').to_sql + end + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded('BackfillCiProjectMirrors', arguments) + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_incident_issue_escalation_statuses.rb b/lib/gitlab/background_migration/backfill_incident_issue_escalation_statuses.rb new file mode 100644 index 00000000000..2d46ff6b933 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_incident_issue_escalation_statuses.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # BackfillIncidentIssueEscalationStatuses adds + # IncidentManagement::IssuableEscalationStatus records for existing Incident issues. + # They will be added with no policy, and escalations_started_at as nil. + class BackfillIncidentIssueEscalationStatuses + def perform(start_id, stop_id) + ActiveRecord::Base.connection.execute <<~SQL + INSERT INTO incident_management_issuable_escalation_statuses (issue_id, created_at, updated_at) + SELECT issues.id, current_timestamp, current_timestamp + FROM issues + WHERE issues.issue_type = 1 + AND issues.id BETWEEN #{start_id} AND #{stop_id} + ON CONFLICT (issue_id) DO NOTHING; + SQL + + mark_job_as_succeeded(start_id, stop_id) + end + + private + + def mark_job_as_succeeded(*arguments) + ::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + self.class.name.demodulize, + arguments + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/base_job.rb b/lib/gitlab/background_migration/base_job.rb new file mode 100644 index 00000000000..e21e7e0e4a3 --- /dev/null +++ b/lib/gitlab/background_migration/base_job.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Simple base class for background migration job classes which are executed through the sidekiq queue. + # + # Any job class that inherits from the base class will have connection to the tracking database set on + # initialization. + class BaseJob + def initialize(connection:) + @connection = connection + end + + def perform(*arguments) + raise NotImplementedError, "subclasses of #{self.class.name} must implement #{__method__}" + end + + private + + attr_reader :connection + end + end +end diff --git a/lib/gitlab/background_migration/cleanup_concurrent_rename.rb b/lib/gitlab/background_migration/cleanup_concurrent_rename.rb deleted file mode 100644 index d3f366f3480..00000000000 --- a/lib/gitlab/background_migration/cleanup_concurrent_rename.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Background migration for cleaning up a concurrent column rename. - class CleanupConcurrentRename < CleanupConcurrentSchemaChange - RESCHEDULE_DELAY = 10.minutes - - def cleanup_concurrent_schema_change(table, old_column, new_column) - cleanup_concurrent_column_rename(table, old_column, new_column) - end - end - end -end diff --git a/lib/gitlab/background_migration/cleanup_concurrent_schema_change.rb b/lib/gitlab/background_migration/cleanup_concurrent_schema_change.rb deleted file mode 100644 index 91b50c1a493..00000000000 --- a/lib/gitlab/background_migration/cleanup_concurrent_schema_change.rb +++ /dev/null @@ -1,56 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Base class for background migration for rename/type changes. - class CleanupConcurrentSchemaChange - include Database::MigrationHelpers - - # table - The name of the table the migration is performed for. - # old_column - The name of the old (to drop) column. - # new_column - The name of the new column. - def perform(table, old_column, new_column) - return unless column_exists?(table, new_column) && column_exists?(table, old_column) - - rows_to_migrate = define_model_for(table) - .where(new_column => nil) - .where - .not(old_column => nil) - - if rows_to_migrate.any? - BackgroundMigrationWorker.perform_in( - RESCHEDULE_DELAY, - self.class.name, - [table, old_column, new_column] - ) - else - cleanup_concurrent_schema_change(table, old_column, new_column) - end - end - - def cleanup_concurrent_schema_change(_table, _old_column, _new_column) - raise NotImplementedError - end - - # These methods are necessary so we can re-use the migration helpers in - # this class. - def connection - ActiveRecord::Base.connection - end - - def method_missing(name, *args, &block) - connection.__send__(name, *args, &block) # rubocop: disable GitlabSecurity/PublicSend - end - - def respond_to_missing?(*args) - connection.respond_to?(*args) || super - end - - def define_model_for(table) - Class.new(ActiveRecord::Base) do - self.table_name = table - end - end - end - end -end diff --git a/lib/gitlab/background_migration/cleanup_concurrent_type_change.rb b/lib/gitlab/background_migration/cleanup_concurrent_type_change.rb deleted file mode 100644 index 48411095dbb..00000000000 --- a/lib/gitlab/background_migration/cleanup_concurrent_type_change.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Background migration for cleaning up a concurrent column type changeb. - class CleanupConcurrentTypeChange < CleanupConcurrentSchemaChange - RESCHEDULE_DELAY = 10.minutes - - def cleanup_concurrent_schema_change(table, old_column, new_column) - cleanup_concurrent_column_type_change(table, old_column) - end - end - end -end diff --git a/lib/gitlab/background_migration/copy_column.rb b/lib/gitlab/background_migration/copy_column.rb deleted file mode 100644 index ef70f37d5eb..00000000000 --- a/lib/gitlab/background_migration/copy_column.rb +++ /dev/null @@ -1,41 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # CopyColumn is a simple (reusable) background migration that can be used to - # update the value of a column based on the value of another column in the - # same table. - # - # For this background migration to work the table that is migrated _has_ to - # have an `id` column as the primary key. - class CopyColumn - # table - The name of the table that contains the columns. - # copy_from - The column containing the data to copy. - # copy_to - The column to copy the data to. - # start_id - The start ID of the range of rows to update. - # end_id - The end ID of the range of rows to update. - def perform(table, copy_from, copy_to, start_id, end_id) - return unless connection.column_exists?(table, copy_to) - - quoted_table = connection.quote_table_name(table) - quoted_copy_from = connection.quote_column_name(copy_from) - quoted_copy_to = connection.quote_column_name(copy_to) - - # We're using raw SQL here since this job may be frequently executed. As - # a result dynamically defining models would lead to many unnecessary - # schema information queries. - connection.execute <<-SQL.strip_heredoc - UPDATE #{quoted_table} - SET #{quoted_copy_to} = #{quoted_copy_from} - WHERE id BETWEEN #{start_id} AND #{end_id} - AND #{quoted_copy_from} IS NOT NULL - AND #{quoted_copy_to} IS NULL - SQL - end - - def connection - ActiveRecord::Base.connection - end - end - end -end diff --git a/lib/gitlab/background_migration/encrypt_static_object_token.rb b/lib/gitlab/background_migration/encrypt_static_object_token.rb new file mode 100644 index 00000000000..80931353e2f --- /dev/null +++ b/lib/gitlab/background_migration/encrypt_static_object_token.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Populates "static_object_token_encrypted" field with encrypted versions + # of values from "static_object_token" field + class EncryptStaticObjectToken + # rubocop:disable Style/Documentation + class User < ActiveRecord::Base + include ::EachBatch + self.table_name = 'users' + scope :with_static_object_token, -> { where.not(static_object_token: nil) } + scope :without_static_object_token_encrypted, -> { where(static_object_token_encrypted: nil) } + end + # rubocop:enable Style/Documentation + + BATCH_SIZE = 100 + + def perform(start_id, end_id) + ranged_query = User + .where(id: start_id..end_id) + .with_static_object_token + .without_static_object_token_encrypted + + ranged_query.each_batch(of: BATCH_SIZE) do |sub_batch| + first, last = sub_batch.pluck(Arel.sql('min(id), max(id)')).first + + batch_query = User.unscoped + .where(id: first..last) + .with_static_object_token + .without_static_object_token_encrypted + + user_tokens = batch_query.pluck(:id, :static_object_token) + + user_encrypted_tokens = user_tokens.map do |(id, plaintext_token)| + next if plaintext_token.blank? + + [id, Gitlab::CryptoHelper.aes256_gcm_encrypt(plaintext_token)] + end + + encrypted_tokens_sql = user_encrypted_tokens.compact.map { |(id, token)| "(#{id}, '#{token}')" }.join(',') + + if user_encrypted_tokens.present? + User.connection.execute(<<~SQL) + WITH cte(cte_id, cte_token) AS #{::Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( + SELECT * + FROM (VALUES #{encrypted_tokens_sql}) AS t (id, token) + ) + UPDATE #{User.table_name} + SET static_object_token_encrypted = cte_token + FROM cte + WHERE cte_id = id + SQL + end + + mark_job_as_succeeded(start_id, end_id) + end + end + + private + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + self.class.name.demodulize, + arguments + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb b/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb new file mode 100644 index 00000000000..2b049ea2d2f --- /dev/null +++ b/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb @@ -0,0 +1,124 @@ +# frozen_string_literal: true + +require 'parser/ruby27' + +module Gitlab + module BackgroundMigration + # This migration fixes raw_metadata entries which have incorrectly been passed a Ruby Hash instead of JSON data. + class FixVulnerabilityOccurrencesWithHashesAsRawMetadata + CLUSTER_IMAGE_SCANNING_REPORT_TYPE = 7 + GENERIC_REPORT_TYPE = 99 + + # Type error is used to handle unexpected types when parsing stringified hashes. + class TypeError < ::StandardError + attr_reader :message, :type + + def initialize(message, type) + @message = message + @type = type + end + end + + # Migration model namespace isolated from application code. + class Finding < ActiveRecord::Base + include EachBatch + + self.table_name = 'vulnerability_occurrences' + + scope :by_api_report_types, -> { where(report_type: [CLUSTER_IMAGE_SCANNING_REPORT_TYPE, GENERIC_REPORT_TYPE]) } + end + + def perform(start_id, end_id) + Finding.by_api_report_types.where(id: start_id..end_id).each do |finding| + next if valid_json?(finding.raw_metadata) + + metadata = hash_from_s(finding.raw_metadata) + + finding.update(raw_metadata: metadata.to_json) if metadata + end + mark_job_as_succeeded(start_id, end_id) + end + + def hash_from_s(str_hash) + ast = Parser::Ruby27.parse(str_hash) + + unless ast.type == :hash + ::Gitlab::AppLogger.error(message: "expected raw_metadata to be a hash", type: ast.type) + return + end + + parse_hash(ast) + rescue Parser::SyntaxError => e + ::Gitlab::AppLogger.error(message: "error parsing raw_metadata", error: e.message) + nil + rescue TypeError => e + ::Gitlab::AppLogger.error(message: "error parsing raw_metadata", error: e.message, type: e.type) + nil + end + + private + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + 'FixVulnerabilityOccurrencesWithHashesAsRawMetadata', + arguments + ) + end + + def valid_json?(metadata) + Oj.load(metadata) + true + rescue Oj::ParseError, Encoding::UndefinedConversionError + false + end + + def parse_hash(hash) + out = {} + hash.children.each do |node| + unless node.type == :pair + raise TypeError.new("expected child of hash to be a `pair`", node.type) + end + + key, value = node.children + + key = parse_key(key) + value = parse_value(value) + + out[key] = value + end + + out + end + + def parse_key(key) + case key.type + when :sym, :str, :int + key.children.first + else + raise TypeError.new("expected key to be either symbol, string, or integer", key.type) + end + end + + def parse_value(value) + case value.type + when :sym, :str, :int + value.children.first + # rubocop:disable Lint/BooleanSymbol + when :true + true + when :false + false + # rubocop:enable Lint/BooleanSymbol + when :nil + nil + when :array + value.children.map { |c| parse_value(c) } + when :hash + parse_hash(value) + else + raise TypeError.new("value of a pair was an unexpected type", value.type) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/job_coordinator.rb b/lib/gitlab/background_migration/job_coordinator.rb index cfbe7167677..5dc77f935e3 100644 --- a/lib/gitlab/background_migration/job_coordinator.rb +++ b/lib/gitlab/background_migration/job_coordinator.rb @@ -36,6 +36,8 @@ module Gitlab attr_reader :worker_class + delegate :minimum_interval, :perform_in, to: :worker_class + def queue @queue ||= worker_class.sidekiq_options['queue'] end @@ -79,7 +81,7 @@ module Gitlab def perform(class_name, arguments) with_shared_connection do - migration_class_for(class_name).new.perform(*arguments) + migration_instance_for(class_name).perform(*arguments) end end @@ -113,6 +115,16 @@ module Gitlab enqueued_job?([retry_set], migration_class) end + def migration_instance_for(class_name) + migration_class = migration_class_for(class_name) + + if migration_class < Gitlab::BackgroundMigration::BaseJob + migration_class.new(connection: connection) + else + migration_class.new + end + end + def migration_class_for(class_name) Gitlab::BackgroundMigration.const_get(class_name, false) end diff --git a/lib/gitlab/background_migration/migrate_legacy_artifacts.rb b/lib/gitlab/background_migration/migrate_legacy_artifacts.rb deleted file mode 100644 index 23d99274232..00000000000 --- a/lib/gitlab/background_migration/migrate_legacy_artifacts.rb +++ /dev/null @@ -1,130 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Metrics/ClassLength - -module Gitlab - module BackgroundMigration - ## - # The class to migrate job artifacts from `ci_builds` to `ci_job_artifacts` - class MigrateLegacyArtifacts - FILE_LOCAL_STORE = 1 # equal to ObjectStorage::Store::LOCAL - ARCHIVE_FILE_TYPE = 1 # equal to Ci::JobArtifact.file_types['archive'] - METADATA_FILE_TYPE = 2 # equal to Ci::JobArtifact.file_types['metadata'] - LEGACY_PATH_FILE_LOCATION = 1 # equal to Ci::JobArtifact.file_location['legacy_path'] - - def perform(start_id, stop_id) - ActiveRecord::Base.transaction do - insert_archives(start_id, stop_id) - insert_metadatas(start_id, stop_id) - delete_legacy_artifacts(start_id, stop_id) - end - end - - private - - def insert_archives(start_id, stop_id) - ActiveRecord::Base.connection.execute <<~SQL - INSERT INTO - ci_job_artifacts ( - project_id, - job_id, - expire_at, - file_location, - created_at, - updated_at, - file, - size, - file_store, - file_type - ) - SELECT - project_id, - id, - artifacts_expire_at #{add_missing_db_timezone}, - #{LEGACY_PATH_FILE_LOCATION}, - created_at #{add_missing_db_timezone}, - created_at #{add_missing_db_timezone}, - artifacts_file, - artifacts_size, - COALESCE(artifacts_file_store, #{FILE_LOCAL_STORE}), - #{ARCHIVE_FILE_TYPE} - FROM - ci_builds - WHERE - id BETWEEN #{start_id.to_i} AND #{stop_id.to_i} - AND artifacts_file <> '' - AND NOT EXISTS ( - SELECT - 1 - FROM - ci_job_artifacts - WHERE - ci_builds.id = ci_job_artifacts.job_id - AND ci_job_artifacts.file_type = #{ARCHIVE_FILE_TYPE}) - SQL - end - - def insert_metadatas(start_id, stop_id) - ActiveRecord::Base.connection.execute <<~SQL - INSERT INTO - ci_job_artifacts ( - project_id, - job_id, - expire_at, - file_location, - created_at, - updated_at, - file, - size, - file_store, - file_type - ) - SELECT - project_id, - id, - artifacts_expire_at #{add_missing_db_timezone}, - #{LEGACY_PATH_FILE_LOCATION}, - created_at #{add_missing_db_timezone}, - created_at #{add_missing_db_timezone}, - artifacts_metadata, - NULL, - COALESCE(artifacts_metadata_store, #{FILE_LOCAL_STORE}), - #{METADATA_FILE_TYPE} - FROM - ci_builds - WHERE - id BETWEEN #{start_id.to_i} AND #{stop_id.to_i} - AND artifacts_file <> '' - AND artifacts_metadata <> '' - AND NOT EXISTS ( - SELECT - 1 - FROM - ci_job_artifacts - WHERE - ci_builds.id = ci_job_artifacts.job_id - AND ci_job_artifacts.file_type = #{METADATA_FILE_TYPE}) - SQL - end - - def delete_legacy_artifacts(start_id, stop_id) - ActiveRecord::Base.connection.execute <<~SQL - UPDATE - ci_builds - SET - artifacts_file = NULL, - artifacts_file_store = NULL, - artifacts_size = NULL, - artifacts_metadata = NULL, - artifacts_metadata_store = NULL - WHERE - id BETWEEN #{start_id.to_i} AND #{stop_id.to_i} - AND artifacts_file <> '' - SQL - end - - def add_missing_db_timezone - 'at time zone \'UTC\'' - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_test_reports_issue_id.rb b/lib/gitlab/background_migration/populate_test_reports_issue_id.rb new file mode 100644 index 00000000000..301efd0c943 --- /dev/null +++ b/lib/gitlab/background_migration/populate_test_reports_issue_id.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true +# rubocop: disable Style/Documentation + +module Gitlab + module BackgroundMigration + class PopulateTestReportsIssueId + def perform(start_id, stop_id) + # NO OP + end + end + end +end + +Gitlab::BackgroundMigration::PopulateTestReportsIssueId.prepend_mod diff --git a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb index 84ff7423254..c1b8de1f6aa 100644 --- a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb +++ b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # rubocop: disable Style/Documentation -class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid +class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid # rubocop:disable Metrics/ClassLength # rubocop: disable Gitlab/NamespacedClass class VulnerabilitiesIdentifier < ActiveRecord::Base self.table_name = "vulnerability_identifiers" @@ -9,10 +9,14 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid end class VulnerabilitiesFinding < ActiveRecord::Base + include EachBatch include ShaAttribute self.table_name = "vulnerability_occurrences" + + has_many :signatures, foreign_key: 'finding_id', class_name: 'VulnerabilityFindingSignature', inverse_of: :finding belongs_to :primary_identifier, class_name: 'VulnerabilitiesIdentifier', inverse_of: :primary_findings, foreign_key: 'primary_identifier_id' + REPORT_TYPES = { sast: 0, dependency_scanning: 1, @@ -20,7 +24,9 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid dast: 3, secret_detection: 4, coverage_fuzzing: 5, - api_fuzzing: 6 + api_fuzzing: 6, + cluster_image_scanning: 7, + generic: 99 }.with_indifferent_access.freeze enum report_type: REPORT_TYPES @@ -28,6 +34,25 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid sha_attribute :location_fingerprint end + class VulnerabilityFindingSignature < ActiveRecord::Base + include ShaAttribute + + self.table_name = 'vulnerability_finding_signatures' + belongs_to :finding, foreign_key: 'finding_id', inverse_of: :signatures, class_name: 'VulnerabilitiesFinding' + + sha_attribute :signature_sha + end + + class VulnerabilitiesFindingPipeline < ActiveRecord::Base + include EachBatch + self.table_name = "vulnerability_occurrence_pipelines" + end + + class Vulnerability < ActiveRecord::Base + include EachBatch + self.table_name = "vulnerabilities" + end + class CalculateFindingUUID FINDING_NAMESPACES_IDS = { development: "a143e9e2-41b3-47bc-9a19-081d089229f4", @@ -52,35 +77,122 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid end # rubocop: enable Gitlab/NamespacedClass + # rubocop: disable Metrics/AbcSize,Metrics/MethodLength,Metrics/BlockLength def perform(start_id, end_id) - findings = VulnerabilitiesFinding - .joins(:primary_identifier) - .select(:id, :report_type, :fingerprint, :location_fingerprint, :project_id) - .where(id: start_id..end_id) - - mappings = findings.each_with_object({}) do |finding, hash| - hash[finding] = { uuid: calculate_uuid_v5_for_finding(finding) } + unless Feature.enabled?(:migrate_vulnerability_finding_uuids, default_enabled: true) + return log_info('Migration is disabled by the feature flag', start_id: start_id, end_id: end_id) end - ::Gitlab::Database::BulkUpdate.execute(%i[uuid], mappings) + log_info('Migration started', start_id: start_id, end_id: end_id) - logger.info(message: 'RecalculateVulnerabilitiesOccurrencesUuid Migration: recalculation is done for:', - finding_ids: mappings.keys.pluck(:id)) + VulnerabilitiesFinding + .joins(:primary_identifier) + .includes(:signatures) + .select(:id, :report_type, :primary_identifier_id, :fingerprint, :location_fingerprint, :project_id, :created_at, :vulnerability_id, :uuid) + .where(id: start_id..end_id) + .each_batch(of: 50) do |relation| + duplicates = find_duplicates(relation) + remove_findings(ids: duplicates) if duplicates.present? + + to_update = relation.reject { |finding| duplicates.include?(finding.id) } + + begin + known_uuids = Set.new + to_be_deleted = [] + + mappings = to_update.each_with_object({}) do |finding, hash| + uuid = calculate_uuid_v5_for_finding(finding) + + if known_uuids.add?(uuid) + hash[finding] = { uuid: uuid } + else + to_be_deleted << finding.id + end + end + + # It is technically still possible to have duplicate uuids + # if the data integrity is broken somehow and the primary identifiers of + # the findings are pointing to different projects with the same fingerprint values. + if to_be_deleted.present? + log_info('Conflicting UUIDs found within the batch', finding_ids: to_be_deleted) + + remove_findings(ids: to_be_deleted) + end + + ::Gitlab::Database::BulkUpdate.execute(%i[uuid], mappings) if mappings.present? + + log_info('Recalculation is done', finding_ids: mappings.keys.pluck(:id)) + rescue ActiveRecord::RecordNotUnique => error + log_info('RecordNotUnique error received') + + match_data = /\(uuid\)=\((?\S{36})\)/.match(error.message) + + # This exception returns the **correct** UUIDv5 which probably comes from a later record + # and it's the one we can drop in the easiest way before retrying the UPDATE query + if match_data + uuid = match_data[:uuid] + log_info('Conflicting UUID found', uuid: uuid) + + id = VulnerabilitiesFinding.find_by(uuid: uuid)&.id + remove_findings(ids: id) if id + retry + else + log_error('Couldnt find conflicting uuid') + + Gitlab::ErrorTracking.track_and_raise_exception(error) + end + end + end mark_job_as_succeeded(start_id, end_id) rescue StandardError => error - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) + log_error('An exception happened') + + Gitlab::ErrorTracking.track_and_raise_exception(error) end + # rubocop: disable Metrics/AbcSize,Metrics/MethodLength,Metrics/BlockLength private + def find_duplicates(relation) + to_exclude = [] + relation.flat_map do |record| + # Assuming we're scanning id 31 and the duplicate is id 40 + # first we'd process 31 and add 40 to the list of ids to remove + # then we would process record 40 and add 31 to the list of removals + # so we would drop both records + to_exclude << record.id + + VulnerabilitiesFinding.where( + report_type: record.report_type, + location_fingerprint: record.location_fingerprint, + primary_identifier_id: record.primary_identifier_id, + project_id: record.project_id + ).where.not(id: to_exclude).pluck(:id) + end + end + + def remove_findings(ids:) + ids = Array(ids) + log_info('Removing Findings and associated records', ids: ids) + + vulnerability_ids = VulnerabilitiesFinding.where(id: ids).pluck(:vulnerability_id).uniq.compact + + VulnerabilitiesFindingPipeline.where(occurrence_id: ids).each_batch { |batch| batch.delete_all } + Vulnerability.where(id: vulnerability_ids).each_batch { |batch| batch.delete_all } + VulnerabilitiesFinding.where(id: ids).delete_all + end + def calculate_uuid_v5_for_finding(vulnerability_finding) return unless vulnerability_finding + signatures = vulnerability_finding.signatures.sort_by { |signature| signature.algorithm_type_before_type_cast } + location_fingerprint = signatures.last&.signature_sha || vulnerability_finding.location_fingerprint + uuid_v5_name_components = { report_type: vulnerability_finding.report_type, primary_identifier_fingerprint: vulnerability_finding.fingerprint, - location_fingerprint: vulnerability_finding.location_fingerprint, + location_fingerprint: location_fingerprint, project_id: vulnerability_finding.project_id } @@ -89,6 +201,14 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid CalculateFindingUUID.call(name) end + def log_info(message, **extra) + logger.info(migrator: 'RecalculateVulnerabilitiesOccurrencesUuid', message: message, **extra) + end + + def log_error(message, **extra) + logger.error(migrator: 'RecalculateVulnerabilitiesOccurrencesUuid', message: message, **extra) + end + def logger @logger ||= Gitlab::BackgroundMigration::Logger.build end diff --git a/lib/gitlab/background_migration/remove_duplicate_services.rb b/lib/gitlab/background_migration/remove_duplicate_services.rb deleted file mode 100644 index 59fb9143a72..00000000000 --- a/lib/gitlab/background_migration/remove_duplicate_services.rb +++ /dev/null @@ -1,58 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Remove duplicated service records with the same project and type. - # These were created in the past for unknown reasons, and should be blocked - # now by the uniqueness validation in the Service model. - class RemoveDuplicateServices - # See app/models/service - class Service < ActiveRecord::Base - include EachBatch - - self.table_name = 'services' - self.inheritance_column = :_type_disabled - - scope :project_ids_with_duplicates, -> do - select(:project_id) - .distinct - .where.not(project_id: nil) - .group(:project_id, :type) - .having('count(*) > 1') - end - - scope :types_with_duplicates, -> (project_ids) do - select(:project_id, :type) - .where(project_id: project_ids) - .group(:project_id, :type) - .having('count(*) > 1') - end - end - - def perform(*project_ids) - types_with_duplicates = Service.types_with_duplicates(project_ids).pluck(:project_id, :type) - - types_with_duplicates.each do |project_id, type| - remove_duplicates(project_id, type) - end - end - - private - - def remove_duplicates(project_id, type) - scope = Service.where(project_id: project_id, type: type) - - # Build a subquery to determine which service record is actually in use, - # by querying for it without specifying an order. - # - # This should match the record returned by `Project#find_service`, - # and the `has_one` service associations on `Project`. - correct_service = scope.select(:id).limit(1) - - # Delete all other services with the same `project_id` and `type` - duplicate_services = scope.where.not(id: correct_service) - duplicate_services.delete_all - end - end - end -end diff --git a/lib/gitlab/checks/changes_access.rb b/lib/gitlab/checks/changes_access.rb index 3ce2e50c548..84c01cf4baf 100644 --- a/lib/gitlab/checks/changes_access.rb +++ b/lib/gitlab/checks/changes_access.rb @@ -33,18 +33,33 @@ module Gitlab # changes. This set may also contain commits which are not referenced by # any of the new revisions. def commits + allow_quarantine = true + newrevs = @changes.map do |change| + oldrev = change[:oldrev] newrev = change[:newrev] - newrev unless newrev.blank? || Gitlab::Git.blank_ref?(newrev) + + next if blank_rev?(newrev) + + # In case any of the old revisions is blank, then we cannot reliably + # detect which commits are new for a given change when enumerating + # objects via the object quarantine directory given that the client + # may have pushed too many commits, and we don't know when to + # terminate the walk. We thus fall back to using `git rev-list --not + # --all`, which is a lot less efficient but at least can only ever + # returns commits which really are new. + allow_quarantine = false if allow_quarantine && blank_rev?(oldrev) + + newrev end.compact return [] if newrevs.empty? - @commits ||= project.repository.new_commits(newrevs, allow_quarantine: true) + @commits ||= project.repository.new_commits(newrevs, allow_quarantine: allow_quarantine) end # All commits which have been newly introduced via the given revision. - def commits_for(newrev) + def commits_for(oldrev, newrev) commits_by_id = commits.index_by(&:id) result = [] @@ -65,9 +80,11 @@ module Gitlab # Only add the parent ID to the pending set if we actually know its # commit to guards us against readding an ID which we have already - # queued up before. + # queued up before. Furthermore, we stop walking as soon as we hit + # `oldrev` such that we do not include any commits in our checks + # which have been "over-pushed" by the client. commit.parent_ids.each do |parent_id| - pending.add(parent_id) if commits_by_id.has_key?(parent_id) + pending.add(parent_id) if commits_by_id.has_key?(parent_id) && parent_id != oldrev end result << commit @@ -80,10 +97,10 @@ module Gitlab @single_changes_accesses ||= changes.map do |change| commits = - if change[:newrev].blank? || Gitlab::Git.blank_ref?(change[:newrev]) + if blank_rev?(change[:newrev]) [] else - Gitlab::Lazy.new { commits_for(change[:newrev]) } + Gitlab::Lazy.new { commits_for(change[:oldrev], change[:newrev]) } end Checks::SingleChangeAccess.new( @@ -109,6 +126,10 @@ module Gitlab def bulk_access_checks! Gitlab::Checks::LfsCheck.new(self).validate! end + + def blank_rev?(rev) + rev.blank? || Gitlab::Git.blank_ref?(rev) + end end end end diff --git a/lib/gitlab/ci/build/policy/refs.rb b/lib/gitlab/ci/build/policy/refs.rb index afe0ccb361e..7ade9ca5085 100644 --- a/lib/gitlab/ci/build/policy/refs.rb +++ b/lib/gitlab/ci/build/policy/refs.rb @@ -35,7 +35,10 @@ module Gitlab # patterns can be matched only when branch or tag is used # the pattern matching does not work for merge requests pipelines if pipeline.branch? || pipeline.tag? - if regexp = Gitlab::UntrustedRegexp::RubySyntax.fabricate(pattern, fallback: true) + regexp = Gitlab::UntrustedRegexp::RubySyntax + .fabricate(pattern, fallback: true, project: pipeline.project) + + if regexp regexp.match?(pipeline.ref) else pattern == pipeline.ref diff --git a/lib/gitlab/ci/build/status/reason.rb b/lib/gitlab/ci/build/status/reason.rb new file mode 100644 index 00000000000..82e07faef63 --- /dev/null +++ b/lib/gitlab/ci/build/status/reason.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + module Status + class Reason + attr_reader :build, :failure_reason, :exit_code + + def initialize(build, failure_reason, exit_code = nil) + @build = build + @failure_reason = failure_reason + @exit_code = exit_code + end + + def failure_reason_enum + ::CommitStatus.failure_reasons[failure_reason] + end + + def force_allow_failure? + return false if exit_code.nil? + + !build.allow_failure? && build.allowed_to_fail_with_code?(exit_code) + end + + def self.fabricate(build, reason) + if reason.is_a?(self) + new(build, reason.failure_reason, reason.exit_code) + else + new(build, reason) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 42b487fdf81..4c98941e032 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -36,7 +36,7 @@ module Gitlab end @root = self.logger.instrument(:config_compose) do - Entry::Root.new(@config).tap(&:compose!) + Entry::Root.new(@config, project: project, user: user).tap(&:compose!) end rescue *rescue_errors => e raise Config::ConfigError, e.message diff --git a/lib/gitlab/ci/config/entry/root.rb b/lib/gitlab/ci/config/entry/root.rb index e6290ef2479..41a3c87037b 100644 --- a/lib/gitlab/ci/config/entry/root.rb +++ b/lib/gitlab/ci/config/entry/root.rb @@ -59,7 +59,8 @@ module Gitlab entry :types, Entry::Stages, description: 'Deprecated: stages for this pipeline.', - reserved: true + reserved: true, + deprecation: { deprecated: '9.0', warning: '14.8', removed: '15.0', documentation: 'https://docs.gitlab.com/ee/ci/yaml/#deprecated-keywords' } entry :cache, Entry::Caches, description: 'Configure caching between build jobs.', @@ -122,8 +123,9 @@ module Gitlab # Deprecated `:types` key workaround - if types are defined and # stages are not defined we use types definition as stages. # - if types_defined? && !stages_defined? - @entries[:stages] = @entries[:types] + if types_defined? + @entries[:stages] = @entries[:types] unless stages_defined? + log_and_warn_deprecated_entry(@entries[:types]) end @entries.delete(:types) diff --git a/lib/gitlab/ci/jwt_v2.rb b/lib/gitlab/ci/jwt_v2.rb new file mode 100644 index 00000000000..278353220e4 --- /dev/null +++ b/lib/gitlab/ci/jwt_v2.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class JwtV2 < Jwt + private + + def reserved_claims + super.merge( + iss: Settings.gitlab.base_url, + sub: "project_path:#{project.full_path}:ref_type:#{ref_type}:ref:#{source_ref}", + aud: Settings.gitlab.base_url + ) + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index 15b0ff3c04d..54b54bd0514 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -9,13 +9,13 @@ module Gitlab include Gitlab::Utils::StrongMemoize def perform! - logger.instrument(:pipeline_save) do + logger.instrument_with_sql(:pipeline_save) do BulkInsertableAssociations.with_bulk_insert do - tags = extract_tag_list_by_status - - pipeline.transaction do - pipeline.save! - CommitStatus.bulk_insert_tags!(statuses, tags) if bulk_insert_tags? + with_bulk_insert_tags do + pipeline.transaction do + pipeline.save! + CommitStatus.bulk_insert_tags!(statuses) if bulk_insert_tags? + end end end end @@ -29,32 +29,26 @@ module Gitlab private - def statuses - strong_memoize(:statuses) do - pipeline.stages.flat_map(&:statuses) + def bulk_insert_tags? + strong_memoize(:bulk_insert_tags) do + ::Feature.enabled?(:ci_bulk_insert_tags, project, default_enabled: :yaml) end end - # We call `job.tag_list=` to assign tags to the jobs from the - # Chain::Seed step which uses the `@tag_list` instance variable to - # store them on the record. We remove them here because we want to - # bulk insert them, otherwise they would be inserted and assigned one - # by one with callbacks. We must use `remove_instance_variable` - # because having the instance variable defined would still run the callbacks - def extract_tag_list_by_status - return {} unless bulk_insert_tags? - - statuses.each.with_object({}) do |job, acc| - tag_list = job.clear_memoization(:tag_list) - next unless tag_list - - acc[job.name] = tag_list - end + def with_bulk_insert_tags + previous = Thread.current['ci_bulk_insert_tags'] + Thread.current['ci_bulk_insert_tags'] = bulk_insert_tags? + yield + ensure + Thread.current['ci_bulk_insert_tags'] = previous end - def bulk_insert_tags? - strong_memoize(:bulk_insert_tags) do - ::Feature.enabled?(:ci_bulk_insert_tags, project, default_enabled: :yaml) + def statuses + strong_memoize(:statuses) do + pipeline + .stages + .flat_map(&:statuses) + .select { |status| status.respond_to?(:tag_list) } end end end diff --git a/lib/gitlab/ci/pipeline/chain/create_deployments.rb b/lib/gitlab/ci/pipeline/chain/create_deployments.rb index b92aa89d62d..b913ba3c87d 100644 --- a/lib/gitlab/ci/pipeline/chain/create_deployments.rb +++ b/lib/gitlab/ci/pipeline/chain/create_deployments.rb @@ -5,8 +5,6 @@ module Gitlab module Pipeline module Chain class CreateDeployments < Chain::Base - DeploymentCreationError = Class.new(StandardError) - def perform! return unless pipeline.create_deployment_in_separate_transaction? @@ -24,18 +22,7 @@ module Gitlab end def create_deployment(build) - return unless build.instance_of?(::Ci::Build) && build.persisted_environment.present? - - deployment = ::Gitlab::Ci::Pipeline::Seed::Deployment - .new(build, build.persisted_environment).to_resource - - return unless deployment - - deployment.deployable = build - deployment.save! - rescue ActiveRecord::RecordInvalid => e - Gitlab::ErrorTracking.track_and_raise_for_dev_exception( - DeploymentCreationError.new(e.message), build_id: build.id) + ::Deployments::CreateForBuildService.new.execute(build) end end end diff --git a/lib/gitlab/ci/pipeline/chain/seed.rb b/lib/gitlab/ci/pipeline/chain/seed.rb index 356eeb76908..feae123f216 100644 --- a/lib/gitlab/ci/pipeline/chain/seed.rb +++ b/lib/gitlab/ci/pipeline/chain/seed.rb @@ -53,13 +53,18 @@ module Gitlab end def context - Gitlab::Ci::Pipeline::Seed::Context.new(pipeline, root_variables: root_variables) + Gitlab::Ci::Pipeline::Seed::Context.new( + pipeline, + root_variables: root_variables, + logger: logger + ) end def root_variables logger.instrument(:pipeline_seed_merge_variables) do ::Gitlab::Ci::Variables::Helpers.merge_variables( - @command.yaml_processor_result.root_variables, @command.workflow_rules_result.variables + @command.yaml_processor_result.root_variables, + @command.workflow_rules_result.variables ) end end diff --git a/lib/gitlab/ci/pipeline/logger.rb b/lib/gitlab/ci/pipeline/logger.rb index 97f7dddd09a..fbba12c11a9 100644 --- a/lib/gitlab/ci/pipeline/logger.rb +++ b/lib/gitlab/ci/pipeline/logger.rb @@ -37,6 +37,16 @@ module Gitlab result end + def instrument_with_sql(operation, &block) + op_start_db_counters = current_db_counter_payload + + result = instrument(operation, &block) + + observe_sql_counters(operation, op_start_db_counters, current_db_counter_payload) + + result + end + def observe(operation, value) return unless enabled? @@ -50,11 +60,20 @@ module Gitlab class: self.class.name.to_s, pipeline_creation_caller: caller, project_id: project.id, - pipeline_id: pipeline.id, pipeline_persisted: pipeline.persisted?, pipeline_source: pipeline.source, pipeline_creation_service_duration_s: age - }.stringify_keys.merge(observations_hash) + } + + if pipeline.persisted? + attributes[:pipeline_builds_tags_count] = pipeline.tags_count + attributes[:pipeline_builds_distinct_tags_count] = pipeline.distinct_tags_count + attributes[:pipeline_id] = pipeline.id + end + + attributes.compact! + attributes.stringify_keys! + attributes.merge!(observations_hash) destination.info(attributes) end @@ -97,6 +116,19 @@ module Gitlab def observations @observations ||= Hash.new { |hash, key| hash[key] = [] } end + + def observe_sql_counters(operation, start_db_counters, end_db_counters) + end_db_counters.each do |key, value| + result = value - start_db_counters.fetch(key, 0) + next if result == 0 + + observe("#{operation}_#{key}", result) + end + end + + def current_db_counter_payload + ::Gitlab::Metrics::Subscribers::ActiveRecord.db_counter_payload + end end end end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 762292f0fa3..5a0ad695741 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -41,12 +41,14 @@ module Gitlab def included? strong_memoize(:inclusion) do - if @using_rules - rules_result.pass? - elsif @using_only || @using_except - all_of_only? && none_of_except? - else - true + logger.instrument(:pipeline_seed_build_inclusion) do + if @using_rules + rules_result.pass? + elsif @using_only || @using_except + all_of_only? && none_of_except? + else + true + end end end end @@ -122,6 +124,8 @@ module Gitlab private + delegate :logger, to: :@context + def all_of_only? @only.all? { |spec| spec.satisfied_by?(@pipeline, evaluate_context) } end diff --git a/lib/gitlab/ci/pipeline/seed/context.rb b/lib/gitlab/ci/pipeline/seed/context.rb index 6194a78f682..c0b8ebeb833 100644 --- a/lib/gitlab/ci/pipeline/seed/context.rb +++ b/lib/gitlab/ci/pipeline/seed/context.rb @@ -5,11 +5,18 @@ module Gitlab module Pipeline module Seed class Context - attr_reader :pipeline, :root_variables + attr_reader :pipeline, :root_variables, :logger - def initialize(pipeline, root_variables: []) + def initialize(pipeline, root_variables: [], logger: nil) @pipeline = pipeline @root_variables = root_variables + @logger = logger || build_logger + end + + private + + def build_logger + ::Gitlab::Ci::Pipeline::Logger.new(project: pipeline.project) end end end diff --git a/lib/gitlab/ci/queue/metrics.rb b/lib/gitlab/ci/queue/metrics.rb index 7f45d626922..54fb1d19ea8 100644 --- a/lib/gitlab/ci/queue/metrics.rb +++ b/lib/gitlab/ci/queue/metrics.rb @@ -69,17 +69,6 @@ module Gitlab self.class.attempt_counter.increment end - # rubocop: disable CodeReuse/ActiveRecord - def jobs_running_for_project(job) - return '+Inf' unless runner.instance_type? - - # excluding currently started job - running_jobs_count = job.project.builds.running.where(runner: ::Ci::Runner.instance_type) - .limit(JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET + 1).count - 1 - running_jobs_count < JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET ? running_jobs_count : "#{JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET}+" - end - # rubocop: enable CodeReuse/ActiveRecord - def increment_queue_operation(operation) self.class.increment_queue_operation(operation) end @@ -242,6 +231,32 @@ module Gitlab Gitlab::Metrics.histogram(name, comment, labels, buckets) end end + + private + + # rubocop: disable CodeReuse/ActiveRecord + def jobs_running_for_project(job) + return '+Inf' unless runner.instance_type? + + # excluding currently started job + running_jobs_count = running_jobs_relation(job) + .limit(JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET + 1).count - 1 + + if running_jobs_count < JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET + running_jobs_count + else + "#{JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET}+" + end + end + + def running_jobs_relation(job) + if ::Feature.enabled?(:ci_pending_builds_maintain_denormalized_data, default_enabled: :yaml) + ::Ci::RunningBuild.instance_type.where(project_id: job.project_id) + else + job.project.builds.running.where(runner: ::Ci::Runner.instance_type) + end + end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/ci/status/build/factory.rb b/lib/gitlab/ci/status/build/factory.rb index 7e5afbad806..a4434e2c144 100644 --- a/lib/gitlab/ci/status/build/factory.rb +++ b/lib/gitlab/ci/status/build/factory.rb @@ -14,7 +14,8 @@ module Gitlab Status::Build::WaitingForResource, Status::Build::Preparing, Status::Build::Pending, - Status::Build::Skipped], + Status::Build::Skipped, + Status::Build::WaitingForApproval], [Status::Build::Cancelable, Status::Build::Retryable], [Status::Build::FailedUnmetPrerequisites, diff --git a/lib/gitlab/ci/status/build/waiting_for_approval.rb b/lib/gitlab/ci/status/build/waiting_for_approval.rb new file mode 100644 index 00000000000..59869a947a9 --- /dev/null +++ b/lib/gitlab/ci/status/build/waiting_for_approval.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Status + module Build + class WaitingForApproval < Status::Extended + def illustration + { + image: 'illustrations/manual_action.svg', + size: 'svg-394', + title: 'Waiting for approval', + content: "This job deploys to the protected environment \"#{subject.deployment&.environment&.name}\" which requires approvals. Use the Deployments API to approve or reject the deployment." + } + end + + def self.matches?(build, user) + build.waiting_for_deployment_approval? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/tags/bulk_insert.rb b/lib/gitlab/ci/tags/bulk_insert.rb index a299df7e2d9..29f3731a9b4 100644 --- a/lib/gitlab/ci/tags/bulk_insert.rb +++ b/lib/gitlab/ci/tags/bulk_insert.rb @@ -4,12 +4,13 @@ module Gitlab module Ci module Tags class BulkInsert + include Gitlab::Utils::StrongMemoize + TAGGINGS_BATCH_SIZE = 1000 TAGS_BATCH_SIZE = 500 - def initialize(statuses, tag_list_by_status) + def initialize(statuses) @statuses = statuses - @tag_list_by_status = tag_list_by_status end def insert! @@ -20,7 +21,18 @@ module Gitlab private - attr_reader :statuses, :tag_list_by_status + attr_reader :statuses + + def tag_list_by_status + strong_memoize(:tag_list_by_status) do + statuses.each.with_object({}) do |status, acc| + tag_list = status.tag_list + next unless tag_list + + acc[status] = tag_list + end + end + end def persist_build_tags! all_tags = tag_list_by_status.values.flatten.uniq.reject(&:blank?) @@ -54,7 +66,7 @@ module Gitlab def build_taggings_attributes(tag_records_by_name) taggings = statuses.flat_map do |status| - tag_list = tag_list_by_status[status.name] + tag_list = tag_list_by_status[status] next unless tag_list tags = tag_records_by_name.values_at(*tag_list) diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 00b771f1e5c..6942631a97f 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -7,7 +7,7 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.24-gitlab.1" + CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.26" needs: [] script: - export SOURCE_CODE=$PWD diff --git a/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml index 18f0f20203d..42487cc0c67 100644 --- a/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml @@ -14,6 +14,8 @@ variables: image: "$SECURE_ANALYZERS_PREFIX/secrets:$SECRETS_ANALYZER_VERSION" services: [] allow_failure: true + variables: + GIT_DEPTH: "50" # `rules` must be overridden explicitly by each child job # see https://gitlab.com/gitlab-org/gitlab/-/issues/218444 artifacts: @@ -29,8 +31,16 @@ secret_detection: script: - if [ -n "$CI_COMMIT_TAG" ]; then echo "Skipping Secret Detection for tags. No code changes have occurred."; exit 0; fi - if [ "$CI_COMMIT_BRANCH" = "$CI_DEFAULT_BRANCH" ]; then echo "Running Secret Detection on default branch."; /analyzer run; exit 0; fi - - git fetch origin $CI_DEFAULT_BRANCH $CI_COMMIT_REF_NAME - - git log --left-right --cherry-pick --pretty=format:"%H" refs/remotes/origin/$CI_DEFAULT_BRANCH...refs/remotes/origin/$CI_COMMIT_REF_NAME > "$CI_COMMIT_SHA"_commit_list.txt - - export SECRET_DETECTION_COMMITS_FILE="$CI_COMMIT_SHA"_commit_list.txt + - | + git fetch origin $CI_DEFAULT_BRANCH $CI_COMMIT_REF_NAME + git log --left-right --cherry-pick --pretty=format:"%H" refs/remotes/origin/${CI_DEFAULT_BRANCH}..refs/remotes/origin/${CI_COMMIT_REF_NAME} >${CI_COMMIT_SHA}_commit_list.txt + if [[ $(wc -l <${CI_COMMIT_SHA}_commit_list.txt) -eq "0" ]]; then + # if git log produces 0 or 1 commits we should scan $CI_COMMIT_SHA only + export SECRET_DETECTION_COMMITS=$CI_COMMIT_SHA + else + # +1 because busybox wc only counts \n and there is no trailing \n + echo "scanning $(($(wc -l <${CI_COMMIT_SHA}_commit_list.txt) + 1)) commits" + export SECRET_DETECTION_COMMITS_FILE=${CI_COMMIT_SHA}_commit_list.txt + fi - /analyzer run - rm "$CI_COMMIT_SHA"_commit_list.txt diff --git a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml index 0c8b98dc1cf..1660a9250e3 100644 --- a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml @@ -29,7 +29,7 @@ before_script: - ruby -v # Print out ruby version for debugging # Uncomment next line if your rails app needs a JS runtime: # - apt-get update -q && apt-get install nodejs -yqq - - bundle config set path 'vendor' # Install dependencies into ./vendor/ruby + - bundle config set path 'vendor' # Install dependencies into ./vendor/ruby - bundle install -j $(nproc) # Optional - Delete if not using `rubocop` diff --git a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml index 7243f240eed..f7f016b5e57 100644 --- a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml @@ -33,6 +33,7 @@ coverage_fuzzing_unlicensed: before_script: - export COVFUZZ_JOB_TOKEN=$CI_JOB_TOKEN - export COVFUZZ_PRIVATE_TOKEN=$CI_PRIVATE_TOKEN + - export COVFUZZ_PROJECT_PATH=$CI_PROJECT_PATH - export COVFUZZ_PROJECT_ID=$CI_PROJECT_ID - if [ -x "$(command -v apt-get)" ] ; then apt-get update && apt-get install -y wget; fi - wget -O gitlab-cov-fuzz "${COVFUZZ_URL_PREFIX}"/"${COVFUZZ_VERSION}"/binaries/gitlab-cov-fuzz_Linux_x86_64 diff --git a/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml new file mode 100644 index 00000000000..6888e955467 --- /dev/null +++ b/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml @@ -0,0 +1,27 @@ +stages: + - build + - test + - deploy + - dast + +variables: + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + DAST_API_VERSION: "1" + DAST_API_IMAGE: $SECURE_ANALYZERS_PREFIX/api-fuzzing:$DAST_API_VERSION + +dast: + stage: dast + image: $DAST_API_IMAGE + variables: + GIT_STRATEGY: none + allow_failure: true + script: + - /peach/analyzer-dast-api + artifacts: + when: always + paths: + - gl-assets + - gl-dast-api-report.json + - gl-*.log + reports: + dast: gl-dast-api-report.json diff --git a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml index e554742735c..12c987a8d37 100644 --- a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml @@ -5,9 +5,11 @@ include: - template: Terraform/Base.latest.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml + - template: Jobs/SAST-IaC.latest.gitlab-ci.yml stages: - validate + - test - build - deploy diff --git a/lib/gitlab/ci/trace/remote_checksum.rb b/lib/gitlab/ci/trace/remote_checksum.rb index d57f3888ec0..7f43d91e6d7 100644 --- a/lib/gitlab/ci/trace/remote_checksum.rb +++ b/lib/gitlab/ci/trace/remote_checksum.rb @@ -26,7 +26,6 @@ module Gitlab delegate :aws?, :google?, to: :object_store_config, prefix: :provider def fetch_md5_checksum - return unless Feature.enabled?(:ci_archived_build_trace_checksum, trace_artifact.project, default_enabled: :yaml) return unless object_store_config.enabled? return if trace_artifact.local_store? diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index 2d31049a0c9..dd435ba05b7 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -11,10 +11,6 @@ module Gitlab delegate :close, :tell, :seek, :size, :url, :truncate, to: :stream, allow_nil: true - delegate :valid?, to: :stream, allow_nil: true - - alias_method :present?, :valid? - def initialize(metrics = Trace::Metrics.new) @stream = yield @stream&.binmode @@ -24,6 +20,7 @@ module Gitlab def valid? self.stream.present? end + alias_method :present?, :valid? def file? self.path.present? diff --git a/lib/gitlab/ci/variables/builder.rb b/lib/gitlab/ci/variables/builder.rb index 3e2c2c7fc1a..4c777527ebc 100644 --- a/lib/gitlab/ci/variables/builder.rb +++ b/lib/gitlab/ci/variables/builder.rb @@ -13,12 +13,76 @@ module Gitlab def scoped_variables(job, environment:, dependencies:) Gitlab::Ci::Variables::Collection.new.tap do |variables| variables.concat(predefined_variables(job)) + + next variables unless pipeline.use_variables_builder_definitions? + + variables.concat(project.predefined_variables) + variables.concat(pipeline.predefined_variables) + variables.concat(job.runner.predefined_variables) if job.runnable? && job.runner + variables.concat(kubernetes_variables(job)) + variables.concat(deployment_variables(environment: environment, job: job)) + variables.concat(job.yaml_variables) + variables.concat(user_variables(job.user)) + variables.concat(job.dependency_variables) if dependencies + variables.concat(secret_instance_variables(ref: job.git_ref)) + variables.concat(secret_group_variables(environment: environment, ref: job.git_ref)) + variables.concat(secret_project_variables(environment: environment, ref: job.git_ref)) + variables.concat(job.trigger_request.user_variables) if job.trigger_request + variables.concat(pipeline.variables) + variables.concat(pipeline.pipeline_schedule.job_variables) if pipeline.pipeline_schedule + end + end + + def kubernetes_variables(job) + ::Gitlab::Ci::Variables::Collection.new.tap do |collection| + # Should get merged with the cluster kubeconfig in deployment_variables, see + # https://gitlab.com/gitlab-org/gitlab/-/issues/335089 + template = ::Ci::GenerateKubeconfigService.new(job).execute + + if template.valid? + collection.append(key: 'KUBECONFIG', value: template.to_yaml, public: false, file: true) + end end end + def deployment_variables(environment:, job:) + return [] unless environment + + project.deployment_variables( + environment: environment, + kubernetes_namespace: job.expanded_kubernetes_namespace + ) + end + + def user_variables(user) + Gitlab::Ci::Variables::Collection.new.tap do |variables| + break variables if user.blank? + + variables.append(key: 'GITLAB_USER_ID', value: user.id.to_s) + variables.append(key: 'GITLAB_USER_EMAIL', value: user.email) + variables.append(key: 'GITLAB_USER_LOGIN', value: user.username) + variables.append(key: 'GITLAB_USER_NAME', value: user.name) + end + end + + def secret_instance_variables(ref:) + project.ci_instance_variables_for(ref: ref) + end + + def secret_group_variables(environment:, ref:) + return [] unless project.group + + project.group.ci_variables_for(ref, project, environment: environment) + end + + def secret_project_variables(environment:, ref:) + project.ci_variables_for(ref: ref, environment: environment) + end + private attr_reader :pipeline + delegate :project, to: :pipeline def predefined_variables(job) Gitlab::Ci::Variables::Collection.new.tap do |variables| diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 296b0cfded2..553508c8638 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -86,11 +86,19 @@ module Gitlab def validate_job_needs!(name, job) return unless needs = job.dig(:needs, :job) + validate_duplicate_needs!(name, needs) + needs.each do |need| validate_job_dependency!(name, need[:name], 'need') end end + def validate_duplicate_needs!(name, needs) + unless needs.uniq == needs + error!("#{name} has duplicate entries in the needs section.") + end + end + def validate_job_dependency!(name, dependency, dependency_type = 'dependency') unless @jobs[dependency.to_sym] error!("#{name} job: undefined #{dependency_type}: #{dependency}") diff --git a/lib/gitlab/color_schemes.rb b/lib/gitlab/color_schemes.rb index 620b4a8aee6..3884f5f0428 100644 --- a/lib/gitlab/color_schemes.rb +++ b/lib/gitlab/color_schemes.rb @@ -7,21 +7,23 @@ module Gitlab # Struct class representing a single Scheme Scheme = Struct.new(:id, :name, :css_class) - SCHEMES = [ - Scheme.new(1, 'White', 'white'), - Scheme.new(2, 'Dark', 'dark'), - Scheme.new(3, 'Solarized Light', 'solarized-light'), - Scheme.new(4, 'Solarized Dark', 'solarized-dark'), - Scheme.new(5, 'Monokai', 'monokai'), - Scheme.new(6, 'None', 'none') - ].freeze + def self.available_schemes + [ + Scheme.new(1, s_('SynthaxHighlightingTheme|Light'), 'white'), + Scheme.new(2, s_('SynthaxHighlightingTheme|Dark'), 'dark'), + Scheme.new(3, s_('SynthaxHighlightingTheme|Solarized Light'), 'solarized-light'), + Scheme.new(4, s_('SynthaxHighlightingTheme|Solarized Dark'), 'solarized-dark'), + Scheme.new(5, s_('SynthaxHighlightingTheme|Monokai'), 'monokai'), + Scheme.new(6, s_('SynthaxHighlightingTheme|None'), 'none') + ] + end # Convenience method to get a space-separated String of all the color scheme # classes that might be applied to a code block. # # Returns a String def self.body_classes - SCHEMES.collect(&:css_class).uniq.join(' ') + available_schemes.collect(&:css_class).uniq.join(' ') end # Get a Scheme by its ID @@ -32,12 +34,12 @@ module Gitlab # # Returns a Scheme def self.by_id(id) - SCHEMES.detect { |s| s.id == id } || default + available_schemes.detect { |s| s.id == id } || default end # Returns the number of defined Schemes def self.count - SCHEMES.size + available_schemes.size end # Get the default Scheme @@ -51,7 +53,7 @@ module Gitlab # # Yields the Scheme object def self.each(&block) - SCHEMES.each(&block) + available_schemes.each(&block) end # Get the Scheme for the specified user, or the default @@ -68,7 +70,7 @@ module Gitlab end def self.valid_ids - SCHEMES.map(&:id) + available_schemes.map(&:id) end end end diff --git a/lib/gitlab/config/entry/configurable.rb b/lib/gitlab/config/entry/configurable.rb index 6bf77ebaa5b..aa6c724c2a3 100644 --- a/lib/gitlab/config/entry/configurable.rb +++ b/lib/gitlab/config/entry/configurable.rb @@ -76,7 +76,7 @@ module Gitlab private # rubocop: disable CodeReuse/ActiveRecord - def entry(key, entry, description: nil, default: nil, inherit: nil, reserved: nil, metadata: {}) + def entry(key, entry, description: nil, default: nil, inherit: nil, reserved: nil, deprecation: nil, metadata: {}) entry_name = key.to_sym raise ArgumentError, "Entry '#{key}' already defined in '#{name}'" if @nodes.to_h[entry_name] @@ -85,6 +85,7 @@ module Gitlab .with(default: default) .with(inherit: inherit) .with(reserved: reserved) + .with(deprecation: deprecation) .metadata(metadata) @nodes ||= {} diff --git a/lib/gitlab/config/entry/factory.rb b/lib/gitlab/config/entry/factory.rb index f76c98f7cbf..61f2071b62f 100644 --- a/lib/gitlab/config/entry/factory.rb +++ b/lib/gitlab/config/entry/factory.rb @@ -32,6 +32,10 @@ module Gitlab self end + def deprecation + @attributes[:deprecation] + end + def description @attributes[:description] end @@ -84,6 +88,7 @@ module Gitlab node.parent = @attributes[:parent] node.default = @attributes[:default] node.description = @attributes[:description] + node.deprecation = @attributes[:deprecation] end end end diff --git a/lib/gitlab/config/entry/node.rb b/lib/gitlab/config/entry/node.rb index 32912cb1046..6ce7046262b 100644 --- a/lib/gitlab/config/entry/node.rb +++ b/lib/gitlab/config/entry/node.rb @@ -10,7 +10,7 @@ module Gitlab InvalidError = Class.new(StandardError) attr_reader :config, :metadata - attr_accessor :key, :parent, :default, :description + attr_accessor :key, :parent, :default, :description, :deprecation def initialize(config, **metadata) @config = config @@ -128,6 +128,24 @@ module Gitlab private attr_reader :entries + + def log_and_warn_deprecated_entry(entry) + user = metadata[:user] + project = metadata[:project] + + if project && user + Gitlab::AppJsonLogger.info(event: 'ci_used_deprecated_keyword', + entry: entry.key.to_s, + user_id: user.id, + project_id: project.id) + end + + deprecation = entry.deprecation + add_warning( + "`#{entry.key}` is deprecated in " \ + "#{deprecation[:deprecated]} and will be removed in #{deprecation[:removed]}." + ) + end end end end diff --git a/lib/gitlab/content_security_policy/config_loader.rb b/lib/gitlab/content_security_policy/config_loader.rb index 87bc2ace204..78ba0916808 100644 --- a/lib/gitlab/content_security_policy/config_loader.rb +++ b/lib/gitlab/content_security_policy/config_loader.rb @@ -147,7 +147,7 @@ module Gitlab # Using 'self' in the CSP introduces several CSP bypass opportunities # for this reason we list the URLs where GitLab frames itself instead def self.allow_framed_gitlab_paths(directives) - ['/admin/', '/assets/', '/-/speedscope/index.html'].map do |path| + ['/admin/', '/assets/', '/-/speedscope/index.html', '/-/sandbox/mermaid'].map do |path| append_to_directive(directives, 'frame_src', Gitlab::Utils.append_path(Gitlab.config.gitlab.url, path)) end end diff --git a/lib/gitlab/data_builder/archive_trace.rb b/lib/gitlab/data_builder/archive_trace.rb new file mode 100644 index 00000000000..f6dd6130104 --- /dev/null +++ b/lib/gitlab/data_builder/archive_trace.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module DataBuilder + module ArchiveTrace + extend self + + def build(job) + { + object_kind: 'archive_trace', + trace_url: job.job_artifacts_trace.file.url, + build_id: job.id, + pipeline_id: job.pipeline_id, + project: job.project.hook_attrs + } + end + end + end +end diff --git a/lib/gitlab/data_builder/deployment.rb b/lib/gitlab/data_builder/deployment.rb index 267c2d32ca9..a4508bc93c5 100644 --- a/lib/gitlab/data_builder/deployment.rb +++ b/lib/gitlab/data_builder/deployment.rb @@ -25,7 +25,8 @@ module Gitlab user: deployment.deployed_by.hook_attrs, user_url: Gitlab::UrlBuilder.build(deployment.deployed_by), commit_url: Gitlab::UrlBuilder.build(deployment.commit), - commit_title: deployment.commit.title + commit_title: deployment.commit.title, + ref: deployment.ref } end end diff --git a/lib/gitlab/database/background_migration/batched_job.rb b/lib/gitlab/database/background_migration/batched_job.rb index 503172dd750..290fa51692a 100644 --- a/lib/gitlab/database/background_migration/batched_job.rb +++ b/lib/gitlab/database/background_migration/batched_job.rb @@ -12,17 +12,6 @@ module Gitlab MAX_ATTEMPTS = 3 STUCK_JOBS_TIMEOUT = 1.hour.freeze - belongs_to :batched_migration, foreign_key: :batched_background_migration_id - - scope :active, -> { where(status: [:pending, :running]) } - scope :stuck, -> { active.where('updated_at <= ?', STUCK_JOBS_TIMEOUT.ago) } - scope :retriable, -> { - failed_jobs = where(status: :failed).where('attempts < ?', MAX_ATTEMPTS) - - from_union([failed_jobs, self.stuck]) - } - scope :except_succeeded, -> { where(status: self.statuses.except(:succeeded).values) } - enum status: { pending: 0, running: 1, @@ -30,7 +19,14 @@ module Gitlab succeeded: 3 } + belongs_to :batched_migration, foreign_key: :batched_background_migration_id + + scope :active, -> { where(status: [:pending, :running]) } + scope :stuck, -> { active.where('updated_at <= ?', STUCK_JOBS_TIMEOUT.ago) } + scope :retriable, -> { from_union([failed.where('attempts < ?', MAX_ATTEMPTS), self.stuck]) } + scope :except_succeeded, -> { where(status: self.statuses.except(:succeeded).values) } scope :successful_in_execution_order, -> { where.not(finished_at: nil).succeeded.order(:finished_at) } + scope :with_preloads, -> { preload(:batched_migration) } delegate :job_class, :table_name, :column_name, :job_arguments, to: :batched_migration, prefix: :migration diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb index 2844cbe4a74..2f066039874 100644 --- a/lib/gitlab/database/background_migration/batched_migration.rb +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -113,7 +113,7 @@ module Gitlab end def smoothed_time_efficiency(number_of_jobs: 10, alpha: 0.2) - jobs = batched_jobs.successful_in_execution_order.reverse_order.limit(number_of_jobs) + jobs = batched_jobs.successful_in_execution_order.reverse_order.limit(number_of_jobs).with_preloads return if jobs.size < number_of_jobs diff --git a/lib/gitlab/database/background_migration_job.rb b/lib/gitlab/database/background_migration_job.rb index c046571a111..c0e3016fd3d 100644 --- a/lib/gitlab/database/background_migration_job.rb +++ b/lib/gitlab/database/background_migration_job.rb @@ -2,7 +2,7 @@ module Gitlab module Database - class BackgroundMigrationJob < ActiveRecord::Base # rubocop:disable Rails/ApplicationRecord + class BackgroundMigrationJob < SharedModel include EachBatch include BulkInsertSafe diff --git a/lib/gitlab/database/batch_counter.rb b/lib/gitlab/database/batch_counter.rb index 6c0ce9e481a..417511618e4 100644 --- a/lib/gitlab/database/batch_counter.rb +++ b/lib/gitlab/database/batch_counter.rb @@ -52,12 +52,7 @@ module Gitlab batch_end = [batch_start + batch_size, finish].min batch_relation = build_relation_batch(batch_start, batch_end, mode) - op_args = @operation_args - if @operation == :count && @operation_args.blank? && use_loose_index_scan_for_distinct_values?(mode) - op_args = [Gitlab::Database::LooseIndexScanDistinctCount::COLUMN_ALIAS] - end - - results = merge_results(results, batch_relation.send(@operation, *op_args)) # rubocop:disable GitlabSecurity/PublicSend + results = merge_results(results, batch_relation.send(@operation, *@operation_args)) # rubocop:disable GitlabSecurity/PublicSend batch_start = batch_end rescue ActiveRecord::QueryCanceled => error # retry with a safe batch size & warmer cache @@ -67,18 +62,6 @@ module Gitlab log_canceled_batch_fetch(batch_start, mode, batch_relation.to_sql, error) return FALLBACK end - rescue Gitlab::Database::LooseIndexScanDistinctCount::ColumnConfigurationError => error - Gitlab::AppJsonLogger - .error( - event: 'batch_count', - relation: @relation.table_name, - operation: @operation, - operation_args: @operation_args, - mode: mode, - message: "LooseIndexScanDistinctCount column error: #{error.message}" - ) - - return FALLBACK end sleep(SLEEP_TIME_IN_SECONDS) @@ -104,11 +87,7 @@ module Gitlab private def build_relation_batch(start, finish, mode) - if use_loose_index_scan_for_distinct_values?(mode) - Gitlab::Database::LooseIndexScanDistinctCount.new(@relation, @column).build_query(from: start, to: finish) - else - @relation.select(@column).public_send(mode).where(between_condition(start, finish)) # rubocop:disable GitlabSecurity/PublicSend - end + @relation.select(@column).public_send(mode).where(between_condition(start, finish)) # rubocop:disable GitlabSecurity/PublicSend end def batch_size_for_mode_and_operation(mode, operation) @@ -151,10 +130,6 @@ module Gitlab ) end - def use_loose_index_scan_for_distinct_values?(mode) - Feature.enabled?(:loose_index_scan_for_distinct_values) && not_group_by_query? && mode == :distinct - end - def not_group_by_query? !@relation.is_a?(ActiveRecord::Relation) || @relation.group_values.blank? end diff --git a/lib/gitlab/database/gitlab_loose_foreign_keys.yml b/lib/gitlab/database/gitlab_loose_foreign_keys.yml index 0343c054f23..d694165574d 100644 --- a/lib/gitlab/database/gitlab_loose_foreign_keys.yml +++ b/lib/gitlab/database/gitlab_loose_foreign_keys.yml @@ -1,3 +1,12 @@ +--- +dast_site_profiles_pipelines: + - table: ci_pipelines + column: ci_pipeline_id + on_delete: async_delete +vulnerability_feedback: + - table: ci_pipelines + column: pipeline_id + on_delete: async_nullify ci_pipeline_chat_data: - table: chat_names column: chat_name_id @@ -6,7 +15,7 @@ dast_scanner_profiles_builds: - table: ci_builds column: ci_build_id on_delete: async_delete -dast_scanner_profiles_builds: +dast_site_profiles_builds: - table: ci_builds column: ci_build_id on_delete: async_delete @@ -18,10 +27,48 @@ clusters_applications_runners: - table: ci_runners column: runner_id on_delete: async_nullify +ci_job_token_project_scope_links: + - table: users + column: added_by_id + on_delete: async_nullify +ci_daily_build_group_report_results: + - table: namespaces + column: group_id + on_delete: async_delete + - table: projects + column: project_id + on_delete: async_delete +ci_freeze_periods: + - table: projects + column: project_id + on_delete: async_delete +ci_pending_builds: + - table: namespaces + column: namespace_id + on_delete: async_delete + - table: projects + column: project_id + on_delete: async_delete +ci_resource_groups: + - table: projects + column: project_id + on_delete: async_delete +ci_runner_namespaces: + - table: namespaces + column: namespace_id + on_delete: async_delete +ci_running_builds: + - table: projects + column: project_id + on_delete: async_delete ci_namespace_mirrors: - table: namespaces column: namespace_id on_delete: async_delete +ci_build_report_results: + - table: projects + column: project_id + on_delete: async_delete ci_builds: - table: users column: user_id @@ -43,6 +90,22 @@ ci_project_mirrors: - table: namespaces column: namespace_id on_delete: async_delete +ci_unit_tests: + - table: projects + column: project_id + on_delete: async_delete +merge_requests: + - table: ci_pipelines + column: head_pipeline_id + on_delete: async_nullify +vulnerability_statistics: + - table: ci_pipelines + column: latest_pipeline_id + on_delete: async_nullify +vulnerability_occurrence_pipelines: + - table: ci_pipelines + column: pipeline_id + on_delete: async_delete packages_build_infos: - table: ci_pipelines column: pipeline_id @@ -67,3 +130,31 @@ project_pages_metadata: - table: ci_job_artifacts column: artifacts_archive_id on_delete: async_nullify +ci_pipeline_schedules: + - table: users + column: owner_id + on_delete: async_nullify +ci_group_variables: + - table: namespaces + column: group_id + on_delete: async_delete +ci_minutes_additional_packs: + - table: namespaces + column: namespace_id + on_delete: async_delete +requirements_management_test_reports: + - table: ci_builds + column: build_id + on_delete: async_nullify +security_scans: + - table: ci_builds + column: build_id + on_delete: async_delete +ci_secure_files: + - table: projects + column: project_id + on_delete: async_delete +ci_pipeline_artifacts: + - table: projects + column: project_id + on_delete: async_delete diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml index 24c2d634780..fb5d8cfa32f 100644 --- a/lib/gitlab/database/gitlab_schemas.yml +++ b/lib/gitlab/database/gitlab_schemas.yml @@ -107,6 +107,7 @@ ci_runner_projects: :gitlab_ci ci_runners: :gitlab_ci ci_running_builds: :gitlab_ci ci_sources_pipelines: :gitlab_ci +ci_secure_files: :gitlab_ci ci_sources_projects: :gitlab_ci ci_stages: :gitlab_ci ci_subscriptions_projects: :gitlab_ci @@ -200,7 +201,7 @@ experiment_subjects: :gitlab_main experiment_users: :gitlab_main external_approval_rules: :gitlab_main external_approval_rules_protected_branches: :gitlab_main -external_pull_requests: :gitlab_main +external_pull_requests: :gitlab_ci external_status_checks: :gitlab_main external_status_checks_protected_branches: :gitlab_main feature_gates: :gitlab_main @@ -231,6 +232,7 @@ gpg_key_subkeys: :gitlab_main gpg_signatures: :gitlab_main grafana_integrations: :gitlab_main group_custom_attributes: :gitlab_main +group_crm_settings: :gitlab_main group_deletion_schedules: :gitlab_main group_deploy_keys: :gitlab_main group_deploy_keys_groups: :gitlab_main @@ -460,6 +462,8 @@ security_findings: :gitlab_main security_orchestration_policy_configurations: :gitlab_main security_orchestration_policy_rule_schedules: :gitlab_main security_scans: :gitlab_main +security_training_providers: :gitlab_main +security_trainings: :gitlab_main self_managed_prometheus_alert_events: :gitlab_main sent_notifications: :gitlab_main sentry_issues: :gitlab_main @@ -521,13 +525,7 @@ vulnerabilities: :gitlab_main vulnerability_exports: :gitlab_main vulnerability_external_issue_links: :gitlab_main vulnerability_feedback: :gitlab_main -vulnerability_finding_evidence_assets: :gitlab_main -vulnerability_finding_evidence_headers: :gitlab_main -vulnerability_finding_evidence_requests: :gitlab_main -vulnerability_finding_evidence_responses: :gitlab_main vulnerability_finding_evidences: :gitlab_main -vulnerability_finding_evidence_sources: :gitlab_main -vulnerability_finding_evidence_supporting_messages: :gitlab_main vulnerability_finding_links: :gitlab_main vulnerability_finding_signatures: :gitlab_main vulnerability_findings_remediations: :gitlab_main diff --git a/lib/gitlab/database/grant.rb b/lib/gitlab/database/grant.rb index c8a30c68bc6..0093848ee6f 100644 --- a/lib/gitlab/database/grant.rb +++ b/lib/gitlab/database/grant.rb @@ -10,7 +10,7 @@ module Gitlab # We _must not_ use quote_table_name as this will produce double # quotes on PostgreSQL and for "has_table_privilege" we need single # quotes. - connection = ActiveRecord::Base.connection # rubocop: disable Database/MultipleDatabases + connection = ApplicationRecord.connection quoted_table = connection.quote(table) begin diff --git a/lib/gitlab/database/load_balancing/setup.rb b/lib/gitlab/database/load_balancing/setup.rb index ef38f42f50b..126c8bb2aa6 100644 --- a/lib/gitlab/database/load_balancing/setup.rb +++ b/lib/gitlab/database/load_balancing/setup.rb @@ -104,11 +104,9 @@ module Gitlab end end - # rubocop:disable Database/MultipleDatabases def connection - use_model_load_balancing? ? super : ActiveRecord::Base.connection + use_model_load_balancing? ? super : ApplicationRecord.connection end - # rubocop:enable Database/MultipleDatabases end end end diff --git a/lib/gitlab/database/loose_index_scan_distinct_count.rb b/lib/gitlab/database/loose_index_scan_distinct_count.rb deleted file mode 100644 index 26be07f91c4..00000000000 --- a/lib/gitlab/database/loose_index_scan_distinct_count.rb +++ /dev/null @@ -1,102 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - # This class builds efficient batched distinct query by using loose index scan. - # Consider the following example: - # > Issue.distinct(:project_id).where(project_id: (1...100)).count - # - # Note: there is an index on project_id - # - # This query will read each element in the index matching the project_id filter. - # If for a project_id has 100_000 issues, all 100_000 elements will be read. - # - # A loose index scan will only read one entry from the index for each project_id to reduce the number of disk reads. - # - # Usage: - # - # Gitlab::Database::LooseIndexScanDisctinctCount.new(Issue, :project_id).count(from: 1, to: 100) - # - # The query will return the number of distinct projects_ids between 1 and 100 - # - # Getting the Arel query: - # - # Gitlab::Database::LooseIndexScanDisctinctCount.new(Issue, :project_id).build_query(from: 1, to: 100) - class LooseIndexScanDistinctCount - COLUMN_ALIAS = 'distinct_count_column' - - ColumnConfigurationError = Class.new(StandardError) - - def initialize(scope, column) - if scope.is_a?(ActiveRecord::Relation) - @scope = scope - @model = scope.model - else - @scope = scope.where({}) - @model = scope - end - - @column = transform_column(column) - end - - def count(from:, to:) - build_query(from: from, to: to).count(COLUMN_ALIAS) - end - - def build_query(from:, to:) # rubocop:disable Metrics/AbcSize - cte = Gitlab::SQL::RecursiveCTE.new(:counter_cte, union_args: { remove_order: false }) - table = model.arel_table - - cte << @scope - .dup - .select(column.as(COLUMN_ALIAS)) - .where(column.gteq(from)) - .where(column.lt(to)) - .order(column) - .limit(1) - - inner_query = @scope - .dup - .where(column.gt(cte.table[COLUMN_ALIAS])) - .where(column.lt(to)) - .select(column.as(COLUMN_ALIAS)) - .order(column) - .limit(1) - - cte << cte.table - .project(Arel::Nodes::Grouping.new(Arel.sql(inner_query.to_sql)).as(COLUMN_ALIAS)) - .where(cte.table[COLUMN_ALIAS].lt(to)) - - model - .with - .recursive(cte.to_arel) - .from(cte.alias_to(table)) - .unscope(where: :source_type) - .unscope(where: model.inheritance_column) # Remove STI query, not needed here - end - - private - - attr_reader :column, :model - - # Transforms the column so it can be used in Arel expressions - # - # 'table.column' => 'table.column' - # 'column' => 'table_name.column' - # :column => 'table_name.column' - # Arel::Attributes::Attribute => name of the column - def transform_column(column) - if column.is_a?(String) || column.is_a?(Symbol) - column_as_string = column.to_s - column_as_string = "#{model.table_name}.#{column_as_string}" unless column_as_string.include?('.') - - Arel.sql(column_as_string) - elsif column.is_a?(Arel::Attributes::Attribute) - column - else - raise ColumnConfigurationError, "Cannot transform the column: #{column.inspect}, please provide the column name as string" - end - end - end - end -end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 4245dd80714..aa5ac1e3486 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -778,186 +778,6 @@ module Gitlab install_rename_triggers(table, old, new) end - # Changes the column type of a table using a background migration. - # - # Because this method uses a background migration it's more suitable for - # large tables. For small tables it's better to use - # `change_column_type_concurrently` since it can complete its work in a - # much shorter amount of time and doesn't rely on Sidekiq. - # - # Example usage: - # - # class Issue < ActiveRecord::Base - # self.table_name = 'issues' - # - # include EachBatch - # - # def self.to_migrate - # where('closed_at IS NOT NULL') - # end - # end - # - # change_column_type_using_background_migration( - # Issue.to_migrate, - # :closed_at, - # :datetime_with_timezone - # ) - # - # Reverting a migration like this is done exactly the same way, just with - # a different type to migrate to (e.g. `:datetime` in the above example). - # - # relation - An ActiveRecord relation to use for scheduling jobs and - # figuring out what table we're modifying. This relation _must_ - # have the EachBatch module included. - # - # column - The name of the column for which the type will be changed. - # - # new_type - The new type of the column. - # - # batch_size - The number of rows to schedule in a single background - # migration. - # - # interval - The time interval between every background migration. - def change_column_type_using_background_migration( - relation, - column, - new_type, - batch_size: 10_000, - interval: 10.minutes - ) - - unless relation.model < EachBatch - raise TypeError, 'The relation must include the EachBatch module' - end - - temp_column = "#{column}_for_type_change" - table = relation.table_name - max_index = 0 - - add_column(table, temp_column, new_type) - install_rename_triggers(table, column, temp_column) - - # Schedule the jobs that will copy the data from the old column to the - # new one. Rows with NULL values in our source column are skipped since - # the target column is already NULL at this point. - relation.where.not(column => nil).each_batch(of: batch_size) do |batch, index| - start_id, end_id = batch.pluck('MIN(id), MAX(id)').first - max_index = index - - migrate_in( - index * interval, - 'CopyColumn', - [table, column, temp_column, start_id, end_id] - ) - end - - # Schedule the renaming of the column to happen (initially) 1 hour after - # the last batch finished. - migrate_in( - (max_index * interval) + 1.hour, - 'CleanupConcurrentTypeChange', - [table, column, temp_column] - ) - - if perform_background_migration_inline? - # To ensure the schema is up to date immediately we perform the - # migration inline in dev / test environments. - Gitlab::BackgroundMigration.steal('CopyColumn') - Gitlab::BackgroundMigration.steal('CleanupConcurrentTypeChange') - end - end - - # Renames a column using a background migration. - # - # Because this method uses a background migration it's more suitable for - # large tables. For small tables it's better to use - # `rename_column_concurrently` since it can complete its work in a much - # shorter amount of time and doesn't rely on Sidekiq. - # - # Example usage: - # - # rename_column_using_background_migration( - # :users, - # :feed_token, - # :rss_token - # ) - # - # table - The name of the database table containing the column. - # - # old - The old column name. - # - # new - The new column name. - # - # type - The type of the new column. If no type is given the old column's - # type is used. - # - # batch_size - The number of rows to schedule in a single background - # migration. - # - # interval - The time interval between every background migration. - def rename_column_using_background_migration( - table, - old_column, - new_column, - type: nil, - batch_size: 10_000, - interval: 10.minutes - ) - - check_trigger_permissions!(table) - - old_col = column_for(table, old_column) - new_type = type || old_col.type - max_index = 0 - - add_column(table, new_column, new_type, - limit: old_col.limit, - precision: old_col.precision, - scale: old_col.scale) - - # We set the default value _after_ adding the column so we don't end up - # updating any existing data with the default value. This isn't - # necessary since we copy over old values further down. - change_column_default(table, new_column, old_col.default) if old_col.default - - install_rename_triggers(table, old_column, new_column) - - model = Class.new(ActiveRecord::Base) do - self.table_name = table - - include ::EachBatch - end - - # Schedule the jobs that will copy the data from the old column to the - # new one. Rows with NULL values in our source column are skipped since - # the target column is already NULL at this point. - model.where.not(old_column => nil).each_batch(of: batch_size) do |batch, index| - start_id, end_id = batch.pluck('MIN(id), MAX(id)').first - max_index = index - - migrate_in( - index * interval, - 'CopyColumn', - [table, old_column, new_column, start_id, end_id] - ) - end - - # Schedule the renaming of the column to happen (initially) 1 hour after - # the last batch finished. - migrate_in( - (max_index * interval) + 1.hour, - 'CleanupConcurrentRename', - [table, old_column, new_column] - ) - - if perform_background_migration_inline? - # To ensure the schema is up to date immediately we perform the - # migration inline in dev / test environments. - Gitlab::BackgroundMigration.steal('CopyColumn') - Gitlab::BackgroundMigration.steal('CleanupConcurrentRename') - end - end - def convert_to_bigint_column(column) "#{column}_convert_to_bigint" end diff --git a/lib/gitlab/database/migrations/background_migration_helpers.rb b/lib/gitlab/database/migrations/background_migration_helpers.rb index 8c33c41ce77..4f1b490cc8f 100644 --- a/lib/gitlab/database/migrations/background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/background_migration_helpers.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true - module Gitlab module Database module Migrations @@ -45,11 +44,11 @@ module Gitlab raise "#{model_class} does not have an ID column of #{primary_column_name} to use for batch ranges" unless model_class.column_names.include?(primary_column_name.to_s) raise "#{primary_column_name} is not an integer column" unless model_class.columns_hash[primary_column_name.to_s].type == :integer + job_coordinator = coordinator_for_tracking_database + # To not overload the worker too much we enforce a minimum interval both # when scheduling and performing jobs. - if delay_interval < BackgroundMigrationWorker.minimum_interval - delay_interval = BackgroundMigrationWorker.minimum_interval - end + delay_interval = [delay_interval, job_coordinator.minimum_interval].max final_delay = 0 batch_counter = 0 @@ -60,14 +59,14 @@ module Gitlab start_id, end_id = relation.pluck(min, max).first - # `BackgroundMigrationWorker.bulk_perform_in` schedules all jobs for + # `SingleDatabaseWorker.bulk_perform_in` schedules all jobs for # the same time, which is not helpful in most cases where we wish to # spread the work over time. final_delay = initial_delay + delay_interval * index full_job_arguments = [start_id, end_id] + other_job_arguments track_in_database(job_class_name, full_job_arguments) if track_jobs - migrate_in(final_delay, job_class_name, full_job_arguments) + migrate_in(final_delay, job_class_name, full_job_arguments, coordinator: job_coordinator) batch_counter += 1 end @@ -91,9 +90,11 @@ module Gitlab # delay_interval - The duration between each job's scheduled time # batch_size - The maximum number of jobs to fetch to memory from the database. def requeue_background_migration_jobs_by_range_at_intervals(job_class_name, delay_interval, batch_size: BATCH_SIZE, initial_delay: 0) + job_coordinator = coordinator_for_tracking_database + # To not overload the worker too much we enforce a minimum interval both # when scheduling and performing jobs. - delay_interval = [delay_interval, BackgroundMigrationWorker.minimum_interval].max + delay_interval = [delay_interval, job_coordinator.minimum_interval].max final_delay = 0 job_counter = 0 @@ -103,7 +104,7 @@ module Gitlab job_batch.each do |job| final_delay = initial_delay + delay_interval * job_counter - migrate_in(final_delay, job_class_name, job.arguments) + migrate_in(final_delay, job_class_name, job.arguments, coordinator: job_coordinator) job_counter += 1 end @@ -132,56 +133,33 @@ module Gitlab # This method does not garauntee that all jobs completed successfully. # It can only be used if the previous background migration used the queue_background_migration_jobs_by_range_at_intervals helper. def finalize_background_migration(class_name, delete_tracking_jobs: ['succeeded']) + job_coordinator = coordinator_for_tracking_database + # Empty the sidekiq queue. - Gitlab::BackgroundMigration.steal(class_name) + job_coordinator.steal(class_name) # Process pending tracked jobs. jobs = Gitlab::Database::BackgroundMigrationJob.pending.for_migration_class(class_name) + jobs.find_each do |job| - BackgroundMigrationWorker.new.perform(job.class_name, job.arguments) + job_coordinator.perform(job.class_name, job.arguments) end # Empty the sidekiq queue. - Gitlab::BackgroundMigration.steal(class_name) + job_coordinator.steal(class_name) # Delete job tracking rows. delete_job_tracking(class_name, status: delete_tracking_jobs) if delete_tracking_jobs end - def perform_background_migration_inline? - Rails.env.test? || Rails.env.development? - end - - def migrate_async(*args) - with_migration_context do - BackgroundMigrationWorker.perform_async(*args) - end - end - - def migrate_in(*args) - with_migration_context do - BackgroundMigrationWorker.perform_in(*args) - end - end - - def bulk_migrate_in(*args) + def migrate_in(*args, coordinator: coordinator_for_tracking_database) with_migration_context do - BackgroundMigrationWorker.bulk_perform_in(*args) + coordinator.perform_in(*args) end end - def bulk_migrate_async(*args) - with_migration_context do - BackgroundMigrationWorker.bulk_perform_async(*args) - end - end - - def with_migration_context(&block) - Gitlab::ApplicationContext.with_context(caller_id: self.class.to_s, &block) - end - def delete_queued_jobs(class_name) - Gitlab::BackgroundMigration.steal(class_name) do |job| + coordinator_for_tracking_database.steal(class_name) do |job| job.delete false @@ -196,9 +174,21 @@ module Gitlab private + def with_migration_context(&block) + Gitlab::ApplicationContext.with_context(caller_id: self.class.to_s, &block) + end + def track_in_database(class_name, arguments) Gitlab::Database::BackgroundMigrationJob.create!(class_name: class_name, arguments: arguments) end + + def coordinator_for_tracking_database + Gitlab::BackgroundMigration.coordinator_for_database(tracking_database) + end + + def tracking_database + Gitlab::BackgroundMigration::DEFAULT_TRACKING_DATABASE + end end end end diff --git a/lib/gitlab/database/partitioning/partition_manager.rb b/lib/gitlab/database/partitioning/partition_manager.rb index aa824dfbd2f..ba6fa0cf278 100644 --- a/lib/gitlab/database/partitioning/partition_manager.rb +++ b/lib/gitlab/database/partitioning/partition_manager.rb @@ -64,6 +64,10 @@ module Gitlab # with_lock_retries starts a requires_new transaction most of the time, but not on the last iteration with_lock_retries do connection.transaction(requires_new: false) do # so we open a transaction here if not already in progress + # Partitions might not get created (IF NOT EXISTS) so explicit locking will not happen. + # This LOCK TABLE ensures to have exclusive lock as the first step. + connection.execute "LOCK TABLE #{connection.quote_table_name(model.table_name)} IN ACCESS EXCLUSIVE MODE" + partitions.each do |partition| connection.execute partition.to_sql diff --git a/lib/gitlab/database/partitioning/sliding_list_strategy.rb b/lib/gitlab/database/partitioning/sliding_list_strategy.rb index 21b86b43ae7..e9865fb91d6 100644 --- a/lib/gitlab/database/partitioning/sliding_list_strategy.rb +++ b/lib/gitlab/database/partitioning/sliding_list_strategy.rb @@ -44,7 +44,18 @@ module Gitlab def extra_partitions possibly_extra = current_partitions[0...-1] # Never consider the most recent partition - possibly_extra.take_while { |p| detach_partition_if.call(p.value) } + extra = possibly_extra.take_while { |p| detach_partition_if.call(p.value) } + + default_value = current_default_value + if extra.any? { |p| p.value == default_value } + Gitlab::AppLogger.error(message: "Inconsistent partition detected: partition with value #{current_default_value} should not be deleted because it's used as the default value.", + partition_number: current_default_value, + table_name: model.table_name) + + extra = extra.reject { |p| p.value == default_value } + end + + extra end def after_adding_partitions @@ -64,6 +75,21 @@ module Gitlab private + def current_default_value + column_name = model.connection.quote(partitioning_key) + table_name = model.connection.quote(model.table_name) + + value = model.connection.select_value <<~SQL + SELECT columns.column_default AS default_value + FROM information_schema.columns columns + WHERE columns.column_name = #{column_name} AND columns.table_name = #{table_name} + SQL + + raise "No default value found for the #{partitioning_key} column within #{model.name}" if value.nil? + + Integer(value) + end + def ensure_partitioning_column_ignored! unless model.ignored_columns.include?(partitioning_key.to_s) raise "Add #{partitioning_key} to #{model.name}.ignored_columns to use it with SlidingListStrategy" diff --git a/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb b/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb index 17a42d997e6..f551fa06cad 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb @@ -4,7 +4,7 @@ module Gitlab module Database module PartitioningMigrationHelpers # Class that will generically copy data from a given table into its corresponding partitioned table - class BackfillPartitionedTable + class BackfillPartitionedTable < ::Gitlab::BackgroundMigration::BaseJob include ::Gitlab::Database::DynamicModelHelpers SUB_BATCH_SIZE = 2_500 @@ -21,7 +21,7 @@ module Gitlab return end - bulk_copy = BulkCopy.new(source_table, partitioned_table, source_column) + bulk_copy = BulkCopy.new(source_table, partitioned_table, source_column, connection: connection) parent_batch_relation = relation_scoped_to_range(source_table, source_column, start_id, stop_id) parent_batch_relation.each_batch(of: SUB_BATCH_SIZE) do |sub_batch| @@ -36,10 +36,6 @@ module Gitlab private - def connection - ActiveRecord::Base.connection - end - def transaction_open? connection.transaction_open? end @@ -53,7 +49,8 @@ module Gitlab end def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) - define_batchable_model(source_table).where(source_key_column => start_id..stop_id) + define_batchable_model(source_table) + .where(source_key_column => start_id..stop_id) end def mark_jobs_as_succeeded(*arguments) @@ -64,12 +61,13 @@ module Gitlab class BulkCopy DELIMITER = ', ' - attr_reader :source_table, :destination_table, :source_column + attr_reader :source_table, :destination_table, :source_column, :connection - def initialize(source_table, destination_table, source_column) + def initialize(source_table, destination_table, source_column, connection:) @source_table = source_table @destination_table = destination_table @source_column = source_column + @connection = connection end def copy_between(start_id, stop_id) @@ -85,10 +83,6 @@ module Gitlab private - def connection - @connection ||= ActiveRecord::Base.connection - end - def column_listing @column_listing ||= connection.columns(source_table).map(&:name).join(DELIMITER) end diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb index c382d2f0715..984c708aa48 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb @@ -406,7 +406,8 @@ module Gitlab end def copy_missed_records(source_table_name, partitioned_table_name, source_column) - backfill_table = BackfillPartitionedTable.new + backfill_table = BackfillPartitionedTable.new(connection: connection) + relation = ::Gitlab::Database::BackgroundMigrationJob.pending .for_partitioning_migration(MIGRATION_CLASS_NAME, source_table_name) diff --git a/lib/gitlab/database/reflection.rb b/lib/gitlab/database/reflection.rb index 48a4de28541..3ea7277571f 100644 --- a/lib/gitlab/database/reflection.rb +++ b/lib/gitlab/database/reflection.rb @@ -105,6 +105,35 @@ module Gitlab row['system_identifier'] end + def flavor + { + # Based on https://aws.amazon.com/premiumsupport/knowledge-center/aurora-version-number/ + 'Amazon Aurora PostgreSQL' => { statement: 'SELECT AURORA_VERSION()', error: /PG::UndefinedFunction/ }, + # Based on https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_PostgreSQL.html#PostgreSQL.Concepts.General.FeatureSupport.Extensions, + # this is also available for both Aurora and RDS, so we need to check for the former first. + 'PostgreSQL on Amazon RDS' => { statement: 'SHOW rds.extensions', error: /PG::UndefinedObject/ }, + # Based on https://cloud.google.com/sql/docs/postgres/flags#postgres-c this should be specific + # to Cloud SQL for PostgreSQL + 'Cloud SQL for PostgreSQL' => { statement: 'SHOW cloudsql.iam_authentication', error: /PG::UndefinedObject/ }, + # Based on + # - https://docs.microsoft.com/en-us/azure/postgresql/flexible-server/concepts-extensions + # - https://docs.microsoft.com/en-us/azure/postgresql/concepts-extensions + # this should be available only for Azure Database for PostgreSQL - Flexible Server. + 'Azure Database for PostgreSQL - Flexible Server' => { statement: 'SHOW azure.extensions', error: /PG::UndefinedObject/ }, + # Based on + # - https://docs.microsoft.com/en-us/azure/postgresql/flexible-server/concepts-servers + # - https://docs.microsoft.com/en-us/azure/postgresql/concepts-servers#managing-your-server + # this database is present on both Flexible and Single server, so we should check the former first. + 'Azure Database for PostgreSQL - Single Server' => { statement: "SELECT datname FROM pg_database WHERE datname = 'azure_maintenance'" } + }.each do |flavor, conditions| + return flavor if connection.execute(conditions[:statement]).to_a.present? + rescue ActiveRecord::StatementInvalid => e + raise if conditions[:error] && !e.message.match?(conditions[:error]) + end + + nil + end + private def connection diff --git a/lib/gitlab/database/reindexing.rb b/lib/gitlab/database/reindexing.rb index 6ffe14249f0..91c3fcc7d72 100644 --- a/lib/gitlab/database/reindexing.rb +++ b/lib/gitlab/database/reindexing.rb @@ -76,20 +76,7 @@ module Gitlab def self.cleanup_leftovers! PostgresIndex.reindexing_leftovers.each do |index| - Gitlab::AppLogger.info("Removing index #{index.identifier} which is a leftover, temporary index from previous reindexing activity") - - retries = Gitlab::Database::WithLockRetriesOutsideTransaction.new( - connection: index.connection, - timing_configuration: REMOVE_INDEX_RETRY_CONFIG, - klass: self.class, - logger: Gitlab::AppLogger - ) - - retries.run(raise_on_exhaustion: false) do - index.connection.tap do |conn| - conn.execute("DROP INDEX CONCURRENTLY IF EXISTS #{conn.quote_table_name(index.schema)}.#{conn.quote_table_name(index.name)}") - end - end + Coordinator.new(index).drop end end end diff --git a/lib/gitlab/database/reindexing/coordinator.rb b/lib/gitlab/database/reindexing/coordinator.rb index 3e4a83aa2e7..b4f7da999df 100644 --- a/lib/gitlab/database/reindexing/coordinator.rb +++ b/lib/gitlab/database/reindexing/coordinator.rb @@ -31,6 +31,25 @@ module Gitlab end end + def drop + try_obtain_lease do + Gitlab::AppLogger.info("Removing index #{index.identifier} which is a leftover, temporary index from previous reindexing activity") + + retries = Gitlab::Database::WithLockRetriesOutsideTransaction.new( + connection: index.connection, + timing_configuration: REMOVE_INDEX_RETRY_CONFIG, + klass: self.class, + logger: Gitlab::AppLogger + ) + + retries.run(raise_on_exhaustion: false) do + index.connection.tap do |conn| + conn.execute("DROP INDEX CONCURRENTLY IF EXISTS #{conn.quote_table_name(index.schema)}.#{conn.quote_table_name(index.name)}") + end + end + end + end + private def with_notifications(action) diff --git a/lib/gitlab/database_importers/work_items/base_type_importer.rb b/lib/gitlab/database_importers/work_items/base_type_importer.rb index c5acdb41de5..2d9700cb2bc 100644 --- a/lib/gitlab/database_importers/work_items/base_type_importer.rb +++ b/lib/gitlab/database_importers/work_items/base_type_importer.rb @@ -5,8 +5,8 @@ module Gitlab module WorkItems module BaseTypeImporter def self.import - WorkItem::Type::BASE_TYPES.each do |type, attributes| - WorkItem::Type.create!(base_type: type, **attributes.slice(:name, :icon_name)) + ::WorkItems::Type::BASE_TYPES.each do |type, attributes| + ::WorkItems::Type.create!(base_type: type, **attributes.slice(:name, :icon_name)) end end end diff --git a/lib/gitlab/email.rb b/lib/gitlab/email.rb index 5f935880764..2e8f076c5d8 100644 --- a/lib/gitlab/email.rb +++ b/lib/gitlab/email.rb @@ -18,5 +18,6 @@ module Gitlab InvalidMergeRequestError = Class.new(InvalidRecordError) UnknownIncomingEmail = Class.new(ProcessingError) InvalidAttachment = Class.new(ProcessingError) + EmailTooLarge = Class.new(ProcessingError) end end diff --git a/lib/gitlab/email/failure_handler.rb b/lib/gitlab/email/failure_handler.rb new file mode 100644 index 00000000000..1079a9c2bb6 --- /dev/null +++ b/lib/gitlab/email/failure_handler.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Email + module FailureHandler + def self.handle(receiver, error) + can_retry = false + reason = + case error + when Gitlab::Email::UnknownIncomingEmail + s_("EmailError|We couldn't figure out what the email is for. Please create your issue or comment through the web interface.") + when Gitlab::Email::SentNotificationNotFoundError + s_("EmailError|We couldn't figure out what the email is in reply to. Please create your comment through the web interface.") + when Gitlab::Email::ProjectNotFound + s_("EmailError|We couldn't find the project. Please check if there's any typo.") + when Gitlab::Email::EmptyEmailError + can_retry = true + s_("EmailError|It appears that the email is blank. Make sure your reply is at the top of the email, we can't process inline replies.") + when Gitlab::Email::UserNotFoundError + s_("EmailError|We couldn't figure out what user corresponds to the email. Please create your comment through the web interface.") + when Gitlab::Email::UserBlockedError + s_("EmailError|Your account has been blocked. If you believe this is in error, contact a staff member.") + when Gitlab::Email::UserNotAuthorizedError + s_("EmailError|You are not allowed to perform this action. If you believe this is in error, contact a staff member.") + when Gitlab::Email::NoteableNotFoundError + s_("EmailError|The thread you are replying to no longer exists, perhaps it was deleted? If you believe this is in error, contact a staff member.") + when Gitlab::Email::InvalidAttachment + error.message + when Gitlab::Email::InvalidRecordError + can_retry = true + error.message + when Gitlab::Email::EmailTooLarge + s_("EmailError|We couldn't process your email because it is too large. Please create your issue or comment through the web interface.") + end + + if reason + receiver.mail.body = nil + + EmailRejectionMailer.rejection(reason, receiver.mail.encoded, can_retry).deliver_later + end + + reason + end + end + end +end diff --git a/lib/gitlab/error_tracking/processor/sidekiq_processor.rb b/lib/gitlab/error_tracking/processor/sidekiq_processor.rb index 0d2f673d73c..cc8cfd827f1 100644 --- a/lib/gitlab/error_tracking/processor/sidekiq_processor.rb +++ b/lib/gitlab/error_tracking/processor/sidekiq_processor.rb @@ -53,6 +53,8 @@ module Gitlab # 'args' in :job => from default error handler job_holder = sidekiq.key?('args') ? sidekiq : sidekiq[:job] + return event unless job_holder + if job_holder['args'] job_holder['args'] = filter_arguments(job_holder['args'], job_holder['class']).to_a end diff --git a/lib/gitlab/event_store.rb b/lib/gitlab/event_store.rb new file mode 100644 index 00000000000..3d7b6b27eb0 --- /dev/null +++ b/lib/gitlab/event_store.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +# Gitlab::EventStore is a simple pub-sub mechanism that lets you publish +# domain events and use Sidekiq workers as event handlers. +# +# It can be used to decouple domains from different bounded contexts +# by publishing domain events and let any interested parties subscribe +# to them. +# +module Gitlab + module EventStore + Error = Class.new(StandardError) + InvalidEvent = Class.new(Error) + InvalidSubscriber = Class.new(Error) + + def self.publish(event) + instance.publish(event) + end + + def self.instance + @instance ||= configure! + end + + # Define all event subscriptions using: + # + # store.subscribe(DomainA::SomeWorker, to: DomainB::SomeEvent) + # + # It is possible to subscribe to a subset of events matching a condition: + # + # store.subscribe(DomainA::SomeWorker, to: DomainB::SomeEvent), if: ->(event) { event.data == :some_value } + # + def self.configure! + Store.new do |store| + ### + # Add subscriptions here: + + store.subscribe ::MergeRequests::UpdateHeadPipelineWorker, to: ::Ci::PipelineCreatedEvent + end + end + private_class_method :configure! + end +end diff --git a/lib/gitlab/event_store/event.rb b/lib/gitlab/event_store/event.rb new file mode 100644 index 00000000000..ee0c329b8e8 --- /dev/null +++ b/lib/gitlab/event_store/event.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +# An Event object represents a domain event that occurred in a bounded context. +# By publishing events we notify other bounded contexts about something +# that happened, so that they can react to it. +# +# Define new event classes under `app/events//` with a name +# representing something that happened in the past: +# +# class Projects::ProjectCreatedEvent < Gitlab::EventStore::Event +# def schema +# { +# 'type' => 'object', +# 'properties' => { +# 'project_id' => { 'type' => 'integer' } +# } +# } +# end +# end +# +# To publish it: +# +# Gitlab::EventStore.publish( +# Projects::ProjectCreatedEvent.new(data: { project_id: project.id }) +# ) +# +module Gitlab + module EventStore + class Event + attr_reader :data + + def initialize(data:) + validate_schema!(data) + @data = data + end + + def schema + raise NotImplementedError, 'must specify schema to validate the event' + end + + private + + def validate_schema!(data) + unless data.is_a?(Hash) + raise Gitlab::EventStore::InvalidEvent, "Event data must be a Hash" + end + + unless JSONSchemer.schema(schema).valid?(data.deep_stringify_keys) + raise Gitlab::EventStore::InvalidEvent, "Data for event #{self.class} does not match the defined schema: #{schema}" + end + end + end + end +end diff --git a/lib/gitlab/event_store/store.rb b/lib/gitlab/event_store/store.rb new file mode 100644 index 00000000000..ecf3cd7e562 --- /dev/null +++ b/lib/gitlab/event_store/store.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Gitlab + module EventStore + class Store + attr_reader :subscriptions + + def initialize + @subscriptions = Hash.new { |h, k| h[k] = [] } + + yield(self) if block_given? + + # freeze the subscriptions as safety measure to avoid further + # subcriptions after initialization. + lock! + end + + def subscribe(worker, to:, if: nil) + condition = binding.local_variable_get('if') + + Array(to).each do |event| + validate_subscription!(worker, event) + subscriptions[event] << Gitlab::EventStore::Subscription.new(worker, condition) + end + end + + def publish(event) + unless event.is_a?(Event) + raise InvalidEvent, "Event being published is not an instance of Gitlab::EventStore::Event: got #{event.inspect}" + end + + subscriptions[event.class].each do |subscription| + subscription.consume_event(event) + end + end + + private + + def lock! + @subscriptions.freeze + end + + def validate_subscription!(subscriber, event_class) + unless event_class < Event + raise InvalidEvent, "Event being subscribed to is not a subclass of Gitlab::EventStore::Event: got #{event_class}" + end + + unless subscriber.respond_to?(:perform_async) + raise InvalidSubscriber, "Subscriber is not an ApplicationWorker: got #{subscriber}" + end + end + end + end +end diff --git a/lib/gitlab/event_store/subscriber.rb b/lib/gitlab/event_store/subscriber.rb new file mode 100644 index 00000000000..cf326d1f9e4 --- /dev/null +++ b/lib/gitlab/event_store/subscriber.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +# This module should be included in order to turn an ApplicationWorker +# into a Subscriber. +# This module overrides the `perform` method and provides a better and +# safer interface for handling events via `handle_event` method. +# +# @example: +# class SomeEventSubscriber +# include ApplicationWorker +# include Gitlab::EventStore::Subscriber +# +# def handle_event(event) +# # ... +# end +# end + +module Gitlab + module EventStore + module Subscriber + def perform(event_type, data) + raise InvalidEvent, event_type unless self.class.const_defined?(event_type) + + event = event_type.constantize.new( + data: data.with_indifferent_access + ) + + handle_event(event) + end + + def handle_event(event) + raise NotImplementedError, 'you must implement this methods in order to handle events' + end + end + end +end diff --git a/lib/gitlab/event_store/subscription.rb b/lib/gitlab/event_store/subscription.rb new file mode 100644 index 00000000000..e5c92ab969f --- /dev/null +++ b/lib/gitlab/event_store/subscription.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module EventStore + class Subscription + attr_reader :worker, :condition + + def initialize(worker, condition) + @worker = worker + @condition = condition + end + + def consume_event(event) + return unless condition_met?(event) + + worker.perform_async(event.class.name, event.data) + # TODO: Log dispatching of events to subscriber + + # We rescue and track any exceptions here because we don't want to + # impact other subscribers if one is faulty. + # The method `condition_met?`, since it can run a block, it might encounter + # a bug. By raising an exception here we could interrupt the publishing + # process, preventing other subscribers from consuming the event. + rescue StandardError => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, event_class: event.class.name, event_data: event.data) + end + + private + + def condition_met?(event) + return true unless condition + + condition.call(event) + end + end + end +end diff --git a/lib/gitlab/exceptions_app.rb b/lib/gitlab/exceptions_app.rb new file mode 100644 index 00000000000..de07b788fb9 --- /dev/null +++ b/lib/gitlab/exceptions_app.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +require_relative 'utils/override' + +module Gitlab + class ExceptionsApp < ActionDispatch::PublicExceptions + extend ::Gitlab::Utils::Override + + REQUEST_ID_PLACEHOLDER = '' + REQUEST_ID_PARAGRAPH = '

Request ID: %s

' + + override :call + def call(env) + status, headers, body = super + + if html_rendered? && body.first&.include?(REQUEST_ID_PLACEHOLDER) + body = [insert_request_id(env, body.first)] + headers['X-GitLab-Custom-Error'] = '1' + end + + [status, headers, body] + end + + private + + override :render_html + def render_html(status) + @html_rendered = true + + super + end + + def html_rendered? + !!@html_rendered + end + + def insert_request_id(env, body) + request_id = ERB::Util.html_escape(ActionDispatch::Request.new(env).request_id) + + body.gsub(REQUEST_ID_PLACEHOLDER, REQUEST_ID_PARAGRAPH % [request_id]) + end + end +end diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index 4cc653bec43..7edda290204 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -30,14 +30,6 @@ module Gitlab module Experimentation EXPERIMENTS = { - remove_known_trial_form_fields_welcoming: { - tracking_category: 'Growth::Conversion::Experiment::RemoveKnownTrialFormFieldsWelcoming', - rollout_strategy: :user - }, - remove_known_trial_form_fields_noneditable: { - tracking_category: 'Growth::Conversion::Experiment::RemoveKnownTrialFormFieldsNoneditable', - rollout_strategy: :user - } }.freeze class << self diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index bb3ba1129fc..ac3b4de0988 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -57,6 +57,7 @@ module Gitlab push_frontend_feature_flag(:improved_emoji_picker, default_enabled: :yaml) push_frontend_feature_flag(:new_header_search, default_enabled: :yaml) push_frontend_feature_flag(:bootstrap_confirmation_modals, default_enabled: :yaml) + push_frontend_feature_flag(:sandboxed_mermaid, default_enabled: :yaml) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb index 59882e8d4f8..ab7de14b07a 100644 --- a/lib/gitlab/gpg/commit.rb +++ b/lib/gitlab/gpg/commit.rb @@ -102,7 +102,7 @@ module Gitlab end def verification_status(gpg_key) - return :multiple_signatures if multiple_signatures? && Feature.enabled?(:multiple_gpg_signatures, @commit.project, default_enabled: :yaml) + return :multiple_signatures if multiple_signatures? return :unknown_key unless gpg_key return :unverified_key unless gpg_key.verified? return :unverified unless verified_signature&.valid? diff --git a/lib/gitlab/http.rb b/lib/gitlab/http.rb index 1b860001ac0..d0918fc39bc 100644 --- a/lib/gitlab/http.rb +++ b/lib/gitlab/http.rb @@ -49,11 +49,12 @@ module Gitlab return httparty_perform_request(http_method, path, options_with_timeouts, &block) end - start_time = Gitlab::Metrics::System.monotonic_time + start_time = nil read_total_timeout = options.fetch(:timeout, DEFAULT_READ_TOTAL_TIMEOUT) tracked_timeout_error = false httparty_perform_request(http_method, path, options_with_timeouts) do |fragment| + start_time ||= Gitlab::Metrics::System.monotonic_time elapsed = Gitlab::Metrics::System.monotonic_time - start_time if elapsed > read_total_timeout diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 12203cab8c8..f056381b86a 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -43,27 +43,27 @@ module Gitlab TRANSLATION_LEVELS = { 'bg' => 0, 'cs_CZ' => 0, - 'da_DK' => 51, + 'da_DK' => 49, 'de' => 15, 'en' => 100, 'eo' => 0, - 'es' => 39, + 'es' => 38, 'fil_PH' => 0, - 'fr' => 12, + 'fr' => 11, 'gl_ES' => 0, 'id_ID' => 0, 'it' => 2, - 'ja' => 35, - 'ko' => 11, - 'nb_NO' => 33, + 'ja' => 36, + 'ko' => 12, + 'nb_NO' => 32, 'nl_NL' => 0, 'pl_PL' => 5, - 'pt_BR' => 49, - 'ro_RO' => 23, - 'ru' => 25, - 'tr_TR' => 15, + 'pt_BR' => 50, + 'ro_RO' => 22, + 'ru' => 26, + 'tr_TR' => 14, 'uk' => 45, - 'zh_CN' => 95, + 'zh_CN' => 98, 'zh_HK' => 2, 'zh_TW' => 3 }.freeze diff --git a/lib/gitlab/import/set_async_jid.rb b/lib/gitlab/import/set_async_jid.rb index 054fcdb433f..527d84477fe 100644 --- a/lib/gitlab/import/set_async_jid.rb +++ b/lib/gitlab/import/set_async_jid.rb @@ -13,7 +13,7 @@ module Gitlab def self.set_jid(import_state) jid = generate_jid(import_state) - Gitlab::SidekiqStatus.set(jid, Gitlab::Import::StuckImportJob::IMPORT_JOBS_EXPIRATION, value: 2) + Gitlab::SidekiqStatus.set(jid, Gitlab::Import::StuckImportJob::IMPORT_JOBS_EXPIRATION) import_state.update_column(:jid, jid) end diff --git a/lib/gitlab/import_export/base/relation_factory.rb b/lib/gitlab/import_export/base/relation_factory.rb index 6749ef4e276..8a8c74c302d 100644 --- a/lib/gitlab/import_export/base/relation_factory.rb +++ b/lib/gitlab/import_export/base/relation_factory.rb @@ -196,7 +196,7 @@ module Gitlab end def use_attributes_permitter? - Feature.enabled?(:permitted_attributes_for_import_export, default_enabled: :yaml) + true end def existing_or_new_object diff --git a/lib/gitlab/import_export/group/relation_tree_restorer.rb b/lib/gitlab/import_export/group/relation_tree_restorer.rb index cbc8ee9e18b..c2cbd2fdf47 100644 --- a/lib/gitlab/import_export/group/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/group/relation_tree_restorer.rb @@ -118,7 +118,7 @@ module Gitlab end def filter_attributes(params) - if use_attributes_permitter? && attributes_permitter.permitted_attributes_defined?(importable_class_sym) + if attributes_permitter.permitted_attributes_defined?(importable_class_sym) attributes_permitter.permit(importable_class_sym, params) else Gitlab::ImportExport::AttributeCleaner.clean( @@ -132,10 +132,6 @@ module Gitlab @attributes_permitter ||= Gitlab::ImportExport::AttributesPermitter.new end - def use_attributes_permitter? - Feature.enabled?(:permitted_attributes_for_import_export, default_enabled: :yaml) - end - def present_override_params # we filter out the empty strings from the overrides # keeping the default values configured @@ -264,14 +260,12 @@ module Gitlab @relation_reader.sort_ci_pipelines_by_id end - # Enable logging of each top-level relation creation when Importing - # into a Group if feature flag is enabled + # Enable logging of each top-level relation creation when Importing into a Group def log_relation_creation(importable, relation_key, relation_object) root_ancestor_group = importable.try(:root_ancestor) return unless root_ancestor_group return unless root_ancestor_group.instance_of?(::Group) - return unless Feature.enabled?(:log_import_export_relation_creation, root_ancestor_group) @shared.logger.info( importable_type: importable.class.to_s, diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index ef146359da9..059f6bd42e3 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -120,6 +120,7 @@ included_attributes: - :name ci_cd_settings: - :group_runners_enabled + - :runner_token_expiration_interval metrics_setting: - :dashboard_timezone - :external_dashboard_url @@ -904,6 +905,7 @@ excluded_attributes: - :release_id project_members: - :source_id + - :member_namespace_id - :invite_email_success - :state group_members: diff --git a/lib/gitlab/jwt_authenticatable.rb b/lib/gitlab/jwt_authenticatable.rb index 1270a148e8d..08d9f69497e 100644 --- a/lib/gitlab/jwt_authenticatable.rb +++ b/lib/gitlab/jwt_authenticatable.rb @@ -13,26 +13,38 @@ module Gitlab module ClassMethods include Gitlab::Utils::StrongMemoize - def decode_jwt_for_issuer(issuer, encoded_message) - JWT.decode( - encoded_message, - secret, - true, - { iss: issuer, verify_iss: true, algorithm: 'HS256' } - ) + def decode_jwt(encoded_message, jwt_secret = secret, issuer: nil, iat_after: nil) + options = { algorithm: 'HS256' } + options = options.merge(iss: issuer, verify_iss: true) if issuer.present? + options = options.merge(verify_iat: true) if iat_after.present? + + decoded_message = JWT.decode(encoded_message, jwt_secret, true, options) + payload = decoded_message[0] + if iat_after.present? + raise JWT::DecodeError, "JWT iat claim is missing" if payload['iat'].blank? + + iat = payload['iat'].to_i + raise JWT::ExpiredSignature, 'Token has expired' if iat < iat_after.to_i + end + + decoded_message end def secret strong_memoize(:secret) do - Base64.strict_decode64(File.read(secret_path).chomp).tap do |bytes| - raise "#{secret_path} does not contain #{SECRET_LENGTH} bytes" if bytes.length != SECRET_LENGTH - end + read_secret(secret_path) + end + end + + def read_secret(path) + Base64.strict_decode64(File.read(path).chomp).tap do |bytes| + raise "#{path} does not contain #{SECRET_LENGTH} bytes" if bytes.length != SECRET_LENGTH end end - def write_secret + def write_secret(path = secret_path) bytes = SecureRandom.random_bytes(SECRET_LENGTH) - File.open(secret_path, 'w:BINARY', 0600) do |f| + File.open(path, 'w:BINARY', 0600) do |f| f.chmod(0600) # If the file already existed, the '0600' passed to 'open' above was a no-op. f.write(Base64.strict_encode64(bytes)) end diff --git a/lib/gitlab/kas.rb b/lib/gitlab/kas.rb index 408b3afc128..ed7787ffc49 100644 --- a/lib/gitlab/kas.rb +++ b/lib/gitlab/kas.rb @@ -11,7 +11,7 @@ module Gitlab class << self def verify_api_request(request_headers) - decode_jwt_for_issuer(JWT_ISSUER, request_headers[INTERNAL_API_REQUEST_HEADER]) + decode_jwt(request_headers[INTERNAL_API_REQUEST_HEADER], issuer: JWT_ISSUER) rescue JWT::DecodeError nil end diff --git a/lib/gitlab/lfs/client.rb b/lib/gitlab/lfs/client.rb index a05e8107cad..10df9262cca 100644 --- a/lib/gitlab/lfs/client.rb +++ b/lib/gitlab/lfs/client.rb @@ -36,7 +36,7 @@ module Gitlab headers: build_request_headers ) - raise BatchSubmitError unless rsp.success? + raise BatchSubmitError.new(http_response: rsp) unless rsp.success? # HTTParty provides rsp.parsed_response, but it only kicks in for the # application/json content type in the response, which we can't rely on @@ -53,19 +53,13 @@ module Gitlab params = { body_stream: file, - headers: { - 'Content-Length' => object.size.to_s, - 'Content-Type' => 'application/octet-stream', - 'User-Agent' => GIT_LFS_USER_AGENT - }.merge(upload_action['header'] || {}) + headers: upload_headers(object, upload_action) } - authenticated = true if params[:headers].key?('Authorization') - params[:basic_auth] = basic_auth unless authenticated - - rsp = Gitlab::HTTP.put(upload_action['href'], params) + url = set_basic_auth_and_extract_lfs_url!(params, upload_action['href']) + rsp = Gitlab::HTTP.put(url, params) - raise ObjectUploadError unless rsp.success? + raise ObjectUploadError.new(http_response: rsp) unless rsp.success? ensure file&.close end @@ -76,20 +70,51 @@ module Gitlab headers: build_request_headers(verify_action['header']) } - authenticated = true if params[:headers].key?('Authorization') - params[:basic_auth] = basic_auth unless authenticated - - rsp = Gitlab::HTTP.post(verify_action['href'], params) + url = set_basic_auth_and_extract_lfs_url!(params, verify_action['href']) + rsp = Gitlab::HTTP.post(url, params) - raise ObjectVerifyError unless rsp.success? + raise ObjectVerifyError.new(http_response: rsp) unless rsp.success? end private + def set_basic_auth_and_extract_lfs_url!(params, raw_url) + authenticated = true if params[:headers].key?('Authorization') + params[:basic_auth] = basic_auth unless authenticated + strip_userinfo = authenticated || params[:basic_auth].present? + lfs_url(raw_url, strip_userinfo) + end + def build_request_headers(extra_headers = nil) DEFAULT_HEADERS.merge(extra_headers || {}) end + def upload_headers(object, upload_action) + # This uses the httprb library to handle case-insensitive HTTP headers + headers = ::HTTP::Headers.new + headers.merge!(upload_action['header']) + transfer_encodings = Array(headers['Transfer-Encoding']&.split(',')).map(&:strip) + + headers['Content-Length'] = object.size.to_s unless transfer_encodings.include?('chunked') + headers['Content-Type'] = 'application/octet-stream' + headers['User-Agent'] = GIT_LFS_USER_AGENT + + headers.to_h + end + + def lfs_url(raw_url, strip_userinfo) + # HTTParty will give precedence to the username/password + # specified in the URL. This causes problems with Azure DevOps, + # which includes a username in the URL. Stripping the userinfo + # from the URL allows the provided HTTP Basic Authentication + # credentials to be used. + if strip_userinfo + Gitlab::UrlSanitizer.new(raw_url).sanitized_url + else + raw_url + end + end + attr_reader :credentials def batch_url @@ -105,9 +130,21 @@ module Gitlab { username: credentials[:user], password: credentials[:password] } end - class BatchSubmitError < StandardError + class HttpError < StandardError + def initialize(http_response:) + super + + @http_response = http_response + end + + def http_error + "HTTP status #{@http_response.code}" + end + end + + class BatchSubmitError < HttpError def message - "Failed to submit batch" + "Failed to submit batch: #{http_error}" end end @@ -122,15 +159,15 @@ module Gitlab end end - class ObjectUploadError < StandardError + class ObjectUploadError < HttpError def message - "Failed to upload object" + "Failed to upload object: #{http_error}" end end - class ObjectVerifyError < StandardError + class ObjectVerifyError < HttpError def message - "Failed to verify object" + "Failed to verify object: #{http_error}" end end end diff --git a/lib/gitlab/logger.rb b/lib/gitlab/logger.rb index 89a4e36a232..53ad0d9cb4d 100644 --- a/lib/gitlab/logger.rb +++ b/lib/gitlab/logger.rb @@ -33,7 +33,11 @@ module Gitlab def self.build Gitlab::SafeRequestStore[self.cache_key] ||= - new(self.full_log_path, level: ::Logger::DEBUG) + new(self.full_log_path, level: log_level) + end + + def self.log_level(fallback: ::Logger::DEBUG) + ENV.fetch('GITLAB_LOG_LEVEL', fallback) end def self.full_log_path diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb index 75d27ed8cc1..e93a297cee4 100644 --- a/lib/gitlab/mail_room.rb +++ b/lib/gitlab/mail_room.rb @@ -25,7 +25,7 @@ module Gitlab # Email specific configuration which is merged with configuration # fetched from YML config file. - ADDRESS_SPECIFIC_CONFIG = { + MAILBOX_SPECIFIC_CONFIGS = { incoming_email: { queue: 'email_receiver', worker: 'EmailReceiverWorker' @@ -38,7 +38,15 @@ module Gitlab class << self def enabled_configs - @enabled_configs ||= configs.select { |config| enabled?(config) } + @enabled_configs ||= configs.select { |_key, config| enabled?(config) } + end + + def enabled_mailbox_types + enabled_configs.keys.map(&:to_s) + end + + def worker_for(mailbox_type) + MAILBOX_SPECIFIC_CONFIGS.try(:[], mailbox_type.to_sym).try(:[], :worker).try(:safe_constantize) end private @@ -48,7 +56,7 @@ module Gitlab end def configs - ADDRESS_SPECIFIC_CONFIG.keys.map { |key| fetch_config(key) } + MAILBOX_SPECIFIC_CONFIGS.to_h { |key, _value| [key, fetch_config(key)] } end def fetch_config(config_key) @@ -63,7 +71,7 @@ module Gitlab def merged_configs(config_key) yml_config = load_yaml.fetch(config_key, {}) - specific_config = ADDRESS_SPECIFIC_CONFIG.fetch(config_key, {}) + specific_config = MAILBOX_SPECIFIC_CONFIGS.fetch(config_key, {}) DEFAULT_CONFIG.merge(specific_config, yml_config) do |_key, oldval, newval| newval.nil? ? oldval : newval end diff --git a/lib/gitlab/mail_room/authenticator.rb b/lib/gitlab/mail_room/authenticator.rb new file mode 100644 index 00000000000..26ebdca8beb --- /dev/null +++ b/lib/gitlab/mail_room/authenticator.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + module MailRoom + class Authenticator + include JwtAuthenticatable + + SecretConfigurationError = Class.new(StandardError) + INTERNAL_API_REQUEST_HEADER = 'Gitlab-Mailroom-Api-Request' + INTERNAL_API_REQUEST_JWT_ISSUER = 'gitlab-mailroom' + + # Only allow token generated within the last 5 minutes + EXPIRATION = 5.minutes + + class << self + def verify_api_request(request_headers, mailbox_type) + mailbox_type = mailbox_type.to_sym + return false if enabled_configs[mailbox_type].blank? + + decode_jwt( + request_headers[INTERNAL_API_REQUEST_HEADER], + secret(mailbox_type), + issuer: INTERNAL_API_REQUEST_JWT_ISSUER, iat_after: Time.current - EXPIRATION + ) + rescue JWT::DecodeError => e + ::Gitlab::AppLogger.warn("Fail to decode MailRoom JWT token: #{e.message}") if Rails.env.development? + + false + end + + def secret(mailbox_type) + strong_memoize("jwt_secret_#{mailbox_type}".to_sym) do + secret_path = enabled_configs[mailbox_type][:secret_file] + raise SecretConfigurationError, "#{mailbox_type}'s secret_file configuration is missing" if secret_path.blank? + + begin + read_secret(secret_path) + rescue StandardError => e + raise SecretConfigurationError, "Fail to read #{mailbox_type}'s secret: #{e.message}" + end + end + end + + def enabled_configs + Gitlab::MailRoom.enabled_configs + end + end + end + end +end diff --git a/lib/gitlab/merge_requests/commit_message_generator.rb b/lib/gitlab/merge_requests/commit_message_generator.rb index 0e9ec6f5cb3..0515c17fe5d 100644 --- a/lib/gitlab/merge_requests/commit_message_generator.rb +++ b/lib/gitlab/merge_requests/commit_message_generator.rb @@ -2,8 +2,9 @@ module Gitlab module MergeRequests class CommitMessageGenerator - def initialize(merge_request:) + def initialize(merge_request:, current_user:) @merge_request = merge_request + @current_user = @merge_request.metrics&.merged_by || @merge_request.merge_user || current_user end def merge_message @@ -15,57 +16,66 @@ module Gitlab def squash_message return unless @merge_request.target_project.squash_commit_template.present? - replace_placeholders(@merge_request.target_project.squash_commit_template) + replace_placeholders(@merge_request.target_project.squash_commit_template, squash: true) end private attr_reader :merge_request + attr_reader :current_user PLACEHOLDERS = { - 'source_branch' => ->(merge_request) { merge_request.source_branch.to_s }, - 'target_branch' => ->(merge_request) { merge_request.target_branch.to_s }, - 'title' => ->(merge_request) { merge_request.title }, - 'issues' => ->(merge_request) do - return "" if merge_request.visible_closing_issues_for.blank? + 'source_branch' => ->(merge_request, _, _) { merge_request.source_branch.to_s }, + 'target_branch' => ->(merge_request, _, _) { merge_request.target_branch.to_s }, + 'title' => ->(merge_request, _, _) { merge_request.title }, + 'issues' => ->(merge_request, _, _) do + return if merge_request.visible_closing_issues_for.blank? closes_issues_references = merge_request.visible_closing_issues_for.map do |issue| issue.to_reference(merge_request.target_project) end "Closes #{closes_issues_references.to_sentence}" end, - 'description' => ->(merge_request) { merge_request.description.presence || '' }, - 'reference' => ->(merge_request) { merge_request.to_reference(full: true) }, - 'first_commit' => -> (merge_request) { merge_request.first_commit&.safe_message&.strip.presence || '' }, - 'first_multiline_commit' => -> (merge_request) { merge_request.first_multiline_commit&.safe_message&.strip.presence || merge_request.title } + 'description' => ->(merge_request, _, _) { merge_request.description }, + 'reference' => ->(merge_request, _, _) { merge_request.to_reference(full: true) }, + 'first_commit' => -> (merge_request, _, _) { merge_request.first_commit&.safe_message&.strip }, + 'first_multiline_commit' => -> (merge_request, _, _) { merge_request.first_multiline_commit&.safe_message&.strip.presence || merge_request.title }, + 'url' => ->(merge_request, _, _) { Gitlab::UrlBuilder.build(merge_request) }, + 'approved_by' => ->(merge_request, _, _) { merge_request.approved_by_users.map { |user| "Approved-by: #{user.name} <#{user.commit_email_or_default}>" }.join("\n") }, + 'merged_by' => ->(_, user, _) { "#{user&.name} <#{user&.commit_email_or_default}>" }, + 'co_authored_by' => ->(merge_request, merged_by, squash) do + commit_author = squash ? merge_request.author : merged_by + merge_request.recent_commits + .to_h { |commit| [commit.author_email, commit.author_name] } + .except(commit_author&.commit_email_or_default) + .map { |author_email, author_name| "Co-authored-by: #{author_name} <#{author_email}>" } + .join("\n") + end }.freeze - PLACEHOLDERS_REGEX = Regexp.union(PLACEHOLDERS.keys.map do |key| - Regexp.new(Regexp.escape(key)) - end).freeze - - BLANK_PLACEHOLDERS_REGEXES = (PLACEHOLDERS.map do |key, value| - [key, Regexp.new("[\n\r]+%{#{Regexp.escape(key)}}$")] - end).to_h.freeze + PLACEHOLDERS_COMBINED_REGEX = /%{(#{Regexp.union(PLACEHOLDERS.keys)})}/.freeze - def replace_placeholders(message) - # convert CRLF to LF + def replace_placeholders(message, squash: false) + # Convert CRLF to LF. message = message.delete("\r") - # Remove placeholders that correspond to empty values and are the last word in the line - # along with all whitespace characters preceding them. - # This allows us to recreate previous default merge commit message behaviour - we skipped new line character - # before empty description and before closed issues when none were present. - PLACEHOLDERS.each do |key, value| - unless value.call(merge_request).present? - message = message.gsub(BLANK_PLACEHOLDERS_REGEXES[key], '') - end + used_variables = message.scan(PLACEHOLDERS_COMBINED_REGEX).map { |value| value[0] }.uniq + values = used_variables.to_h do |variable_name| + ["%{#{variable_name}}", PLACEHOLDERS[variable_name].call(merge_request, current_user, squash)] end + names_of_empty_variables = values.filter_map { |name, value| name if value.blank? } - Gitlab::StringPlaceholderReplacer - .replace_string_placeholders(message, PLACEHOLDERS_REGEX) do |key| - PLACEHOLDERS[key].call(merge_request) + # Remove lines that contain empty variable placeholder and nothing else. + if names_of_empty_variables.present? + # If there is blank line or EOF after it, remove blank line before it as well. + message = message.gsub(/\n\n#{Regexp.union(names_of_empty_variables)}(\n\n|\Z)/, '\1') + # Otherwise, remove only the line it is in. + message = message.gsub(/^#{Regexp.union(names_of_empty_variables)}\n/, '') end + # Substitute all variables with their values. + message = message.gsub(Regexp.union(values.keys), values) if values.present? + + message end end end diff --git a/lib/gitlab/metrics/exporter/base_exporter.rb b/lib/gitlab/metrics/exporter/base_exporter.rb index 47c862c0232..190d3d3fd2f 100644 --- a/lib/gitlab/metrics/exporter/base_exporter.rb +++ b/lib/gitlab/metrics/exporter/base_exporter.rb @@ -11,44 +11,41 @@ module Gitlab attr_accessor :readiness_checks - def initialize(settings, **options) + def initialize(settings, log_enabled:, log_file:, gc_requests: false, **options) super(**options) @settings = settings + @gc_requests = gc_requests + + # log_enabled does not exist for all exporters + log_sink = log_enabled ? File.join(Rails.root, 'log', log_file) : File::NULL + @logger = WEBrick::Log.new(log_sink) + @logger.time_format = "[%Y-%m-%dT%H:%M:%S.%L%z]" end def enabled? settings.enabled end - def log_filename - raise NotImplementedError - end - private - attr_reader :settings + attr_reader :settings, :logger def start_working - logger = WEBrick::Log.new(log_filename) - logger.time_format = "[%Y-%m-%dT%H:%M:%S.%L%z]" - access_log = [ [logger, WEBrick::AccessLog::COMBINED_LOG_FORMAT] ] @server = ::WEBrick::HTTPServer.new( Port: settings.port, BindAddress: settings.address, - Logger: logger, AccessLog: access_log) - server.mount_proc '/readiness' do |req, res| - render_probe(readiness_probe, req, res) - end - server.mount_proc '/liveness' do |req, res| - render_probe(liveness_probe, req, res) - end + Logger: logger, AccessLog: access_log + ) server.mount '/', Rack::Handler::WEBrick, rack_app true + rescue StandardError => e + logger.error(e) + false end def run_thread @@ -72,8 +69,16 @@ module Gitlab end def rack_app + readiness = readiness_probe + liveness = liveness_probe + pid = thread_name + gc_requests = @gc_requests + Rack::Builder.app do use Rack::Deflater + use Gitlab::Metrics::Exporter::MetricsMiddleware, pid + use Gitlab::Metrics::Exporter::HealthChecksMiddleware, readiness, liveness + use Gitlab::Metrics::Exporter::GcRequestMiddleware if gc_requests use ::Prometheus::Client::Rack::Exporter if ::Gitlab::Metrics.metrics_folder_present? run -> (env) { [404, {}, ['']] } end @@ -86,14 +91,6 @@ module Gitlab def liveness_probe ::Gitlab::HealthChecks::Probes::Collection.new end - - def render_probe(probe, req, res) - result = probe.execute - - res.status = result.http_status - res.content_type = 'application/json; charset=utf-8' - res.body = result.json.to_json - end end end end diff --git a/lib/gitlab/metrics/exporter/gc_request_middleware.rb b/lib/gitlab/metrics/exporter/gc_request_middleware.rb new file mode 100644 index 00000000000..3806b0e2bd1 --- /dev/null +++ b/lib/gitlab/metrics/exporter/gc_request_middleware.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Exporter + class GcRequestMiddleware + def initialize(app) + @app = app + end + + def call(env) + @app.call(env).tap do + GC.start + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/exporter/health_checks_middleware.rb b/lib/gitlab/metrics/exporter/health_checks_middleware.rb new file mode 100644 index 00000000000..c43b8004b72 --- /dev/null +++ b/lib/gitlab/metrics/exporter/health_checks_middleware.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Exporter + class HealthChecksMiddleware + def initialize(app, readiness_probe, liveness_probe) + @app = app + @readiness_probe = readiness_probe + @liveness_probe = liveness_probe + end + + def call(env) + case env['PATH_INFO'] + when '/readiness' then render_probe(@readiness_probe) + when '/liveness' then render_probe(@liveness_probe) + else @app.call(env) + end + end + + private + + def render_probe(probe) + result = probe.execute + + [ + result.http_status, + { 'Content-Type' => 'application/json; charset=utf-8' }, + [result.json.to_json] + ] + end + end + end + end +end diff --git a/lib/gitlab/metrics/exporter/metrics_middleware.rb b/lib/gitlab/metrics/exporter/metrics_middleware.rb new file mode 100644 index 00000000000..e17f1c13cf0 --- /dev/null +++ b/lib/gitlab/metrics/exporter/metrics_middleware.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Exporter + class MetricsMiddleware + def initialize(app, pid) + @app = app + default_labels = { + pid: pid + } + @requests_total = Gitlab::Metrics.counter( + :exporter_http_requests_total, 'Total number of HTTP requests', default_labels + ) + @request_durations = Gitlab::Metrics.histogram( + :exporter_http_request_duration_seconds, + 'HTTP request duration histogram (seconds)', + default_labels, + [0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10] + ) + end + + def call(env) + start = Gitlab::Metrics::System.monotonic_time + @app.call(env).tap do |response| + duration = Gitlab::Metrics::System.monotonic_time - start + + labels = { + method: env['REQUEST_METHOD'].downcase, + path: env['PATH_INFO'].to_s, + code: response.first.to_s + } + + @requests_total.increment(labels) + @request_durations.observe(labels, duration) + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/exporter/sidekiq_exporter.rb b/lib/gitlab/metrics/exporter/sidekiq_exporter.rb index eea71fda6a0..afecf6546f8 100644 --- a/lib/gitlab/metrics/exporter/sidekiq_exporter.rb +++ b/lib/gitlab/metrics/exporter/sidekiq_exporter.rb @@ -4,12 +4,11 @@ module Gitlab module Metrics module Exporter class SidekiqExporter < BaseExporter - def log_filename - if settings['log_enabled'] - File.join(Rails.root, 'log', 'sidekiq_exporter.log') - else - File::NULL - end + def initialize(settings, **options) + super(settings, + log_enabled: settings['log_enabled'], + log_file: 'sidekiq_exporter.log', + **options) end end end diff --git a/lib/gitlab/metrics/exporter/web_exporter.rb b/lib/gitlab/metrics/exporter/web_exporter.rb index d41484aaaa7..c05ad8ccf42 100644 --- a/lib/gitlab/metrics/exporter/web_exporter.rb +++ b/lib/gitlab/metrics/exporter/web_exporter.rb @@ -26,8 +26,8 @@ module Gitlab attr_reader :running # This exporter is always run on master process - def initialize - super(Settings.monitoring.web_exporter) + def initialize(**options) + super(Settings.monitoring.web_exporter, log_enabled: true, log_file: 'web_exporter.log', **options) # DEPRECATED: # these `readiness_checks` are deprecated @@ -39,10 +39,6 @@ module Gitlab ] end - def log_filename - File.join(Rails.root, 'log', 'web_exporter.log') - end - def mark_as_not_running! @running = false end diff --git a/lib/gitlab/metrics/samplers/action_cable_sampler.rb b/lib/gitlab/metrics/samplers/action_cable_sampler.rb index adce3030d0d..1f50371cae9 100644 --- a/lib/gitlab/metrics/samplers/action_cable_sampler.rb +++ b/lib/gitlab/metrics/samplers/action_cable_sampler.rb @@ -6,8 +6,8 @@ module Gitlab class ActionCableSampler < BaseSampler DEFAULT_SAMPLING_INTERVAL_SECONDS = 5 - def initialize(interval = nil, action_cable: ::ActionCable.server) - super(interval) + def initialize(action_cable: ::ActionCable.server, **options) + super(**options) @action_cable = action_cable end diff --git a/lib/gitlab/metrics/samplers/base_sampler.rb b/lib/gitlab/metrics/samplers/base_sampler.rb index 52d80c3c27e..b2a9de21145 100644 --- a/lib/gitlab/metrics/samplers/base_sampler.rb +++ b/lib/gitlab/metrics/samplers/base_sampler.rb @@ -9,7 +9,10 @@ module Gitlab attr_reader :interval # interval - The sampling interval in seconds. - def initialize(interval = nil) + # warmup - When true, takes a single sample eagerly before entering the sampling loop. + # This can be useful to ensure that all metrics files exist after `start` returns, + # since prometheus-client-mmap creates them lazily upon first access. + def initialize(interval: nil, logger: Logger.new($stdout), warmup: false, **options) interval ||= ENV[interval_env_key]&.to_i interval ||= self.class::DEFAULT_SAMPLING_INTERVAL_SECONDS interval_half = interval.to_f / 2 @@ -17,13 +20,16 @@ module Gitlab @interval = interval @interval_steps = (-interval_half..interval_half).step(0.1).to_a - super() + @logger = logger + @warmup = warmup + + super(**options) end def safe_sample sample rescue StandardError => e - ::Gitlab::AppLogger.warn("#{self.class}: #{e}, stopping") + @logger.warn("#{self.class}: #{e}, stopping") stop end @@ -63,6 +69,8 @@ module Gitlab def start_working @running = true + safe_sample if @warmup + true end diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index b1c5e9800da..d71ee671b8d 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -7,12 +7,12 @@ module Gitlab DEFAULT_SAMPLING_INTERVAL_SECONDS = 60 GC_REPORT_BUCKETS = [0.01, 0.05, 0.1, 0.2, 0.3, 0.5, 1].freeze - def initialize(*) + def initialize(...) GC::Profiler.clear metrics[:process_start_time_seconds].set(labels, Time.now.to_i) - super + super(...) end def metrics diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb index a047015e54f..0b7b5e23b75 100644 --- a/lib/gitlab/middleware/multipart.rb +++ b/lib/gitlab/middleware/multipart.rb @@ -116,7 +116,7 @@ module Gitlab jwt_token = params[param_key] raise "Empty JWT param: #{param_key}" if jwt_token.blank? - payload = Gitlab::Workhorse.decode_jwt(jwt_token).first + payload = Gitlab::Workhorse.decode_jwt_with_issuer(jwt_token).first raise "Invalid JWT payload: not a Hash" unless payload.is_a?(Hash) upload_params = payload.fetch(JWT_PARAM_FIXED_KEY, {}) @@ -172,7 +172,7 @@ module Gitlab encoded_message = env.delete(RACK_ENV_KEY) return @app.call(env) if encoded_message.blank? - message = ::Gitlab::Workhorse.decode_jwt(encoded_message)[0] + message = ::Gitlab::Workhorse.decode_jwt_with_issuer(encoded_message)[0] ::Gitlab::Middleware::Multipart::Handler.new(env, message).with_open_files do @app.call(env) diff --git a/lib/gitlab/middleware/webhook_recursion_detection.rb b/lib/gitlab/middleware/webhook_recursion_detection.rb new file mode 100644 index 00000000000..2677445852c --- /dev/null +++ b/lib/gitlab/middleware/webhook_recursion_detection.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Middleware + class WebhookRecursionDetection + def initialize(app) + @app = app + end + + def call(env) + headers = ActionDispatch::Request.new(env).headers + + ::Gitlab::WebHooks::RecursionDetection.set_from_headers(headers) + + @app.call(env) + end + end + end +end diff --git a/lib/gitlab/pages.rb b/lib/gitlab/pages.rb index 98e87e9e915..4e0e5102bec 100644 --- a/lib/gitlab/pages.rb +++ b/lib/gitlab/pages.rb @@ -10,7 +10,7 @@ module Gitlab class << self def verify_api_request(request_headers) - decode_jwt_for_issuer('gitlab-pages', request_headers[INTERNAL_API_REQUEST_HEADER]) + decode_jwt(request_headers[INTERNAL_API_REQUEST_HEADER], issuer: 'gitlab-pages') rescue JWT::DecodeError false end diff --git a/lib/gitlab/pagination/keyset/column_order_definition.rb b/lib/gitlab/pagination/keyset/column_order_definition.rb index 2b968c4253f..302e7b406b1 100644 --- a/lib/gitlab/pagination/keyset/column_order_definition.rb +++ b/lib/gitlab/pagination/keyset/column_order_definition.rb @@ -114,6 +114,20 @@ module Gitlab # - When the order is a calculated expression or the column is in another table (JOIN-ed) # # If the add_to_projections is true, the query builder will automatically add the column to the SELECT values + # + # **sql_type** + # + # The SQL type of the column or SQL expression. This is an optional field which is only required when using the + # column with the InOperatorOptimization class. + # + # Example: When the order expression is a calculated SQL expression. + # + # { + # attribute_name: 'id_times_count', + # order_expression: Arel.sql('(id * count)').asc, + # sql_type: 'integer' # the SQL type here must match with the type of the produced data by the order_expression. Putting 'text' here would be incorrect. + # } + # class ColumnOrderDefinition REVERSED_ORDER_DIRECTIONS = { asc: :desc, desc: :asc }.freeze REVERSED_NULL_POSITIONS = { nulls_first: :nulls_last, nulls_last: :nulls_first }.freeze @@ -122,7 +136,8 @@ module Gitlab attr_reader :attribute_name, :column_expression, :order_expression, :add_to_projections, :order_direction - def initialize(attribute_name:, order_expression:, column_expression: nil, reversed_order_expression: nil, nullable: :not_nullable, distinct: true, order_direction: nil, add_to_projections: false) + # rubocop: disable Metrics/ParameterLists + def initialize(attribute_name:, order_expression:, column_expression: nil, reversed_order_expression: nil, nullable: :not_nullable, distinct: true, order_direction: nil, sql_type: nil, add_to_projections: false) @attribute_name = attribute_name @order_expression = order_expression @column_expression = column_expression || calculate_column_expression(order_expression) @@ -130,8 +145,10 @@ module Gitlab @reversed_order_expression = reversed_order_expression || calculate_reversed_order(order_expression) @nullable = parse_nullable(nullable, distinct) @order_direction = parse_order_direction(order_expression, order_direction) + @sql_type = sql_type @add_to_projections = add_to_projections end + # rubocop: enable Metrics/ParameterLists def reverse self.class.new( @@ -185,6 +202,12 @@ module Gitlab sql_string end + def sql_type + raise Gitlab::Pagination::Keyset::SqlTypeMissingError.for_column(self) if @sql_type.nil? + + @sql_type + end + private attr_reader :reversed_order_expression, :nullable, :distinct diff --git a/lib/gitlab/pagination/keyset/in_operator_optimization/column_data.rb b/lib/gitlab/pagination/keyset/in_operator_optimization/column_data.rb index 3f620f74eca..93b28661bb0 100644 --- a/lib/gitlab/pagination/keyset/in_operator_optimization/column_data.rb +++ b/lib/gitlab/pagination/keyset/in_operator_optimization/column_data.rb @@ -4,23 +4,35 @@ module Gitlab module Pagination module Keyset module InOperatorOptimization + # This class is used for wrapping an Arel column with + # convenient helper methods in order to make the query + # building for the InOperatorOptimization a bit cleaner. class ColumnData attr_reader :original_column_name, :as, :arel_table - def initialize(original_column_name, as, arel_table) - @original_column_name = original_column_name.to_s + # column - name of the DB column + # as - custom alias for the column + # arel_table - relation where the column is located + def initialize(column, as, arel_table) + @original_column_name = column @as = as.to_s @arel_table = arel_table end + # Generates: `issues.name AS my_alias` def projection arel_column.as(as) end + # Generates: issues.name` def arel_column arel_table[original_column_name] end + # overridden in OrderByColumnData class + alias_method :column_expression, :arel_column + + # Generates: `issues.my_alias` def arel_column_as arel_table[as] end @@ -29,8 +41,9 @@ module Gitlab "#{arel_table.name}_#{original_column_name}_array" end + # Generates: SELECT ARRAY_AGG(...) AS issues_name_array def array_aggregated_column - Arel::Nodes::NamedFunction.new('ARRAY_AGG', [arel_column]).as(array_aggregated_column_name) + Arel::Nodes::NamedFunction.new('ARRAY_AGG', [column_expression]).as(array_aggregated_column_name) end end end diff --git a/lib/gitlab/pagination/keyset/in_operator_optimization/order_by_column_data.rb b/lib/gitlab/pagination/keyset/in_operator_optimization/order_by_column_data.rb new file mode 100644 index 00000000000..9cb1ba1542d --- /dev/null +++ b/lib/gitlab/pagination/keyset/in_operator_optimization/order_by_column_data.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module Pagination + module Keyset + module InOperatorOptimization + class OrderByColumnData < ColumnData + extend ::Gitlab::Utils::Override + + attr_reader :column + + # column - a ColumnOrderDefinition object + # as - custom alias for the column + # arel_table - relation where the column is located + def initialize(column, as, arel_table) + super(column.attribute_name.to_s, as, arel_table) + @column = column + end + + override :arel_column + def arel_column + column.column_expression + end + + override :column_expression + def column_expression + arel_table[original_column_name] + end + + def column_for_projection + column.column_expression.as(original_column_name) + end + end + end + end + end +end diff --git a/lib/gitlab/pagination/keyset/in_operator_optimization/order_by_columns.rb b/lib/gitlab/pagination/keyset/in_operator_optimization/order_by_columns.rb index d8c69a74e6b..d6513114d08 100644 --- a/lib/gitlab/pagination/keyset/in_operator_optimization/order_by_columns.rb +++ b/lib/gitlab/pagination/keyset/in_operator_optimization/order_by_columns.rb @@ -9,16 +9,16 @@ module Gitlab # This class exposes collection methods for the order by columns # - # Example: by modelling the `issues.created_at ASC, issues.id ASC` ORDER BY + # Example: by modeling the `issues.created_at ASC, issues.id ASC` ORDER BY # SQL clause, this class will receive two ColumnOrderDefinition objects def initialize(columns, arel_table) @columns = columns.map do |column| - ColumnData.new(column.attribute_name, "order_by_columns_#{column.attribute_name}", arel_table) + OrderByColumnData.new(column, "order_by_columns_#{column.attribute_name}", arel_table) end end def arel_columns - columns.map(&:arel_column) + columns.map(&:column_for_projection) end def array_aggregated_columns diff --git a/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder.rb b/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder.rb index 53faf8469f2..065a3a0cf20 100644 --- a/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder.rb +++ b/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder.rb @@ -120,7 +120,7 @@ module Gitlab .from(array_cte) .join(Arel.sql("LEFT JOIN LATERAL (#{initial_keyset_query.to_sql}) #{table_name} ON TRUE")) - order_by_columns.each { |column| q.where(column.arel_column.not_eq(nil)) } + order_by_columns.each { |column| q.where(column.column_expression.not_eq(nil)) } q.as('array_scope_lateral_query') end @@ -231,7 +231,7 @@ module Gitlab order .apply_cursor_conditions(keyset_scope, cursor_values, use_union_optimization: true) - .reselect(*order_by_columns.arel_columns) + .reselect(*order_by_columns.map(&:column_for_projection)) .limit(1) end diff --git a/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/order_values_loader_strategy.rb b/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/order_values_loader_strategy.rb index fc2b56048f6..932aa0c2d28 100644 --- a/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/order_values_loader_strategy.rb +++ b/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/order_values_loader_strategy.rb @@ -12,11 +12,7 @@ module Gitlab end def initializer_columns - order_by_columns.map do |column| - column_name = column.original_column_name.to_s - type = model.columns_hash[column_name].sql_type - "NULL::#{type} AS #{column_name}" - end + order_by_columns.map { |column_data| null_with_type_cast(column_data) } end def columns @@ -30,6 +26,15 @@ module Gitlab private attr_reader :model, :order_by_columns + + def null_with_type_cast(column_data) + column_name = column_data.original_column_name.to_s + active_record_column = model.columns_hash[column_name] + + type = active_record_column ? active_record_column.sql_type : column_data.column.sql_type + + "NULL::#{type} AS #{column_name}" + end end end end diff --git a/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy.rb b/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy.rb index b12c33d6e51..51f38c1da58 100644 --- a/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy.rb +++ b/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy.rb @@ -9,6 +9,8 @@ module Gitlab RECORDS_COLUMN = 'records' def initialize(finder_query, model, order_by_columns) + verify_order_by_attributes_on_model!(model, order_by_columns) + @finder_query = finder_query @order_by_columns = order_by_columns @table_name = model.table_name @@ -34,6 +36,20 @@ module Gitlab private attr_reader :finder_query, :order_by_columns, :table_name + + def verify_order_by_attributes_on_model!(model, order_by_columns) + order_by_columns.map(&:column).each do |column| + unless model.columns_hash[column.attribute_name.to_s] + text = <<~TEXT + The "RecordLoaderStrategy" does not support the following ORDER BY column because + it's not available on the \"#{model.table_name}\" table: #{column.attribute_name} + + Omit the "finder_query" parameter to use the "OrderValuesLoaderStrategy". + TEXT + raise text + end + end + end end end end diff --git a/lib/gitlab/pagination/keyset/sql_type_missing_error.rb b/lib/gitlab/pagination/keyset/sql_type_missing_error.rb new file mode 100644 index 00000000000..0525ae13e9c --- /dev/null +++ b/lib/gitlab/pagination/keyset/sql_type_missing_error.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true +module Gitlab + module Pagination + module Keyset + class SqlTypeMissingError < StandardError + def self.for_column(column) + message = <<~TEXT + The "sql_type" attribute is not set for the following column definition: + #{column.attribute_name} + + See the ColumnOrderDefinition class for more context. + TEXT + + new(message) + end + end + end + end +end diff --git a/lib/gitlab/password.rb b/lib/gitlab/password.rb new file mode 100644 index 00000000000..00aef8754d6 --- /dev/null +++ b/lib/gitlab/password.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +# This module is used to return fake strong password for tests + +module Gitlab + module Password + DEFAULT_LENGTH = 12 + TEST_DEFAULT = "123qweQWE!@#" + "0" * (User.password_length.max - DEFAULT_LENGTH) + def self.test_default(length = 12) + password_length = [[User.password_length.min, length].max, User.password_length.max].min + TEST_DEFAULT[...password_length] + end + end +end diff --git a/lib/gitlab/redis/multi_store.rb b/lib/gitlab/redis/multi_store.rb deleted file mode 100644 index 500b62bf0e8..00000000000 --- a/lib/gitlab/redis/multi_store.rb +++ /dev/null @@ -1,229 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Redis - class MultiStore - include Gitlab::Utils::StrongMemoize - - class ReadFromPrimaryError < StandardError - def message - 'Value not found on the redis primary store. Read from the redis secondary store successful.' - end - end - class MethodMissingError < StandardError - def message - 'Method missing. Falling back to execute method on the redis secondary store.' - end - end - - attr_reader :primary_store, :secondary_store, :instance_name - - FAILED_TO_READ_ERROR_MESSAGE = 'Failed to read from the redis primary_store.' - FAILED_TO_WRITE_ERROR_MESSAGE = 'Failed to write to the redis primary_store.' - - SKIP_LOG_METHOD_MISSING_FOR_COMMANDS = %i(info).freeze - - READ_COMMANDS = %i( - get - mget - smembers - scard - ).freeze - - WRITE_COMMANDS = %i( - set - setnx - setex - sadd - srem - del - pipelined - flushdb - ).freeze - - def initialize(primary_store, secondary_store, instance_name) - @primary_store = primary_store - @secondary_store = secondary_store - @instance_name = instance_name - - validate_stores! - end - # rubocop:disable GitlabSecurity/PublicSend - READ_COMMANDS.each do |name| - define_method(name) do |*args, &block| - if use_primary_and_secondary_stores? - read_command(name, *args, &block) - else - default_store.send(name, *args, &block) - end - end - end - - WRITE_COMMANDS.each do |name| - define_method(name) do |*args, &block| - if use_primary_and_secondary_stores? - write_command(name, *args, &block) - else - default_store.send(name, *args, &block) - end - end - end - - def method_missing(...) - return @instance.send(...) if @instance - - log_method_missing(...) - - default_store.send(...) - end - # rubocop:enable GitlabSecurity/PublicSend - - def respond_to_missing?(command_name, include_private = false) - true - end - - # This is needed because of Redis::Rack::Connection is requiring Redis::Store - # https://github.com/redis-store/redis-rack/blob/a833086ba494083b6a384a1a4e58b36573a9165d/lib/redis/rack/connection.rb#L15 - # Done similarly in https://github.com/lsegal/yard/blob/main/lib/yard/templates/template.rb#L122 - def is_a?(klass) - return true if klass == default_store.class - - super(klass) - end - alias_method :kind_of?, :is_a? - - def to_s - use_primary_and_secondary_stores? ? primary_store.to_s : default_store.to_s - end - - def use_primary_and_secondary_stores? - feature_flags_available? && - Feature.enabled?("use_primary_and_secondary_stores_for_#{instance_name.underscore}", default_enabled: :yaml) && - !same_redis_store? - end - - def use_primary_store_as_default? - feature_flags_available? && - Feature.enabled?("use_primary_store_as_default_for_#{instance_name.underscore}", default_enabled: :yaml) && - !same_redis_store? - end - - private - - def default_store - use_primary_store_as_default? ? primary_store : secondary_store - end - - def log_method_missing(command_name, *_args) - return if SKIP_LOG_METHOD_MISSING_FOR_COMMANDS.include?(command_name) - - log_error(MethodMissingError.new, command_name) - increment_method_missing_count(command_name) - end - - def read_command(command_name, *args, &block) - if @instance - send_command(@instance, command_name, *args, &block) - else - read_one_with_fallback(command_name, *args, &block) - end - end - - def write_command(command_name, *args, &block) - if @instance - send_command(@instance, command_name, *args, &block) - else - write_both(command_name, *args, &block) - end - end - - def read_one_with_fallback(command_name, *args, &block) - begin - value = send_command(primary_store, command_name, *args, &block) - rescue StandardError => e - log_error(e, command_name, - multi_store_error_message: FAILED_TO_READ_ERROR_MESSAGE) - end - - value ||= fallback_read(command_name, *args, &block) - - value - end - - def fallback_read(command_name, *args, &block) - value = send_command(secondary_store, command_name, *args, &block) - - if value - log_error(ReadFromPrimaryError.new, command_name) - increment_read_fallback_count(command_name) - end - - value - end - - def write_both(command_name, *args, &block) - begin - send_command(primary_store, command_name, *args, &block) - rescue StandardError => e - log_error(e, command_name, - multi_store_error_message: FAILED_TO_WRITE_ERROR_MESSAGE) - end - - send_command(secondary_store, command_name, *args, &block) - end - - def same_redis_store? - strong_memoize(:same_redis_store) do - # " - primary_store.inspect == secondary_store.inspect - end - end - - # rubocop:disable GitlabSecurity/PublicSend - def send_command(redis_instance, command_name, *args, &block) - if block_given? - # Make sure that block is wrapped and executed only on the redis instance that is executing the block - redis_instance.send(command_name, *args) do |*params| - with_instance(redis_instance, *params, &block) - end - else - redis_instance.send(command_name, *args) - end - end - # rubocop:enable GitlabSecurity/PublicSend - - def with_instance(instance, *params) - @instance = instance - - yield(*params) - ensure - @instance = nil - end - - def increment_read_fallback_count(command_name) - @read_fallback_counter ||= Gitlab::Metrics.counter(:gitlab_redis_multi_store_read_fallback_total, 'Client side Redis MultiStore reading fallback') - @read_fallback_counter.increment(command: command_name, instance_name: instance_name) - end - - def increment_method_missing_count(command_name) - @method_missing_counter ||= Gitlab::Metrics.counter(:gitlab_redis_multi_store_method_missing_total, 'Client side Redis MultiStore method missing') - @method_missing_counter.increment(command: command_name, instance_name: instance_name) - end - - def validate_stores! - raise ArgumentError, 'primary_store is required' unless primary_store - raise ArgumentError, 'secondary_store is required' unless secondary_store - raise ArgumentError, 'instance_name is required' unless instance_name - raise ArgumentError, 'invalid primary_store' unless primary_store.is_a?(::Redis) - raise ArgumentError, 'invalid secondary_store' unless secondary_store.is_a?(::Redis) - end - - def log_error(exception, command_name, extra = {}) - Gitlab::ErrorTracking.log_exception( - exception, - command_name: command_name, - extra: extra.merge(instance_name: instance_name)) - end - end - end -end diff --git a/lib/gitlab/redis/sessions.rb b/lib/gitlab/redis/sessions.rb index c547828d907..ddcfdf6e798 100644 --- a/lib/gitlab/redis/sessions.rb +++ b/lib/gitlab/redis/sessions.rb @@ -9,39 +9,9 @@ module Gitlab IP_SESSIONS_LOOKUP_NAMESPACE = 'session:lookup:ip:gitlab2' OTP_SESSIONS_NAMESPACE = 'session:otp' - class << self - # The data we store on Sessions used to be stored on SharedState. - def config_fallback - SharedState - end - - private - - def redis - # Don't use multistore if redis.sessions configuration is not provided - return super if config_fallback? - - primary_store = ::Redis.new(params) - secondary_store = ::Redis.new(config_fallback.params) - - MultiStore.new(primary_store, secondary_store, store_name) - end - end - - def store(extras = {}) - # Don't use multistore if redis.sessions configuration is not provided - return super if self.class.config_fallback? - - primary_store = create_redis_store(redis_store_options, extras) - secondary_store = create_redis_store(self.class.config_fallback.params, extras) - - MultiStore.new(primary_store, secondary_store, self.class.store_name) - end - - private - - def create_redis_store(options, extras) - ::Redis::Store.new(options.merge(extras)) + # The data we store on Sessions used to be stored on SharedState. + def self.config_fallback + SharedState end end end diff --git a/lib/gitlab/redis/sessions_store_helper.rb b/lib/gitlab/redis/sessions_store_helper.rb deleted file mode 100644 index c80442847f1..00000000000 --- a/lib/gitlab/redis/sessions_store_helper.rb +++ /dev/null @@ -1,27 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Redis - module SessionsStoreHelper - extend ActiveSupport::Concern - - module StoreMethods - def redis_store_class - use_redis_session_store? ? Gitlab::Redis::Sessions : Gitlab::Redis::SharedState - end - - private - - def use_redis_session_store? - Gitlab::Utils.to_boolean(ENV['GITLAB_USE_REDIS_SESSIONS_STORE'], default: true) - end - end - - include StoreMethods - - included do - extend StoreMethods - end - end - end -end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 8e139ae0709..b07b9c79858 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -79,7 +79,11 @@ module Gitlab def nuget_version_regex @nuget_version_regex ||= / - \A#{_semver_major_minor_patch_regex}(\.\d*)?#{_semver_prerelease_build_regex}\z + \A#{_semver_major_regex} + \.#{_semver_minor_regex} + (\.#{_semver_patch_regex})? + (\.\d*)? + #{_semver_prerelease_build_regex}\z /x.freeze end @@ -167,9 +171,25 @@ module Gitlab # regexes rather than being used alone. def _semver_major_minor_patch_regex @_semver_major_minor_patch_regex ||= / + #{_semver_major_regex}\.#{_semver_minor_regex}\.#{_semver_patch_regex} + /x.freeze + end + + def _semver_major_regex + @_semver_major_regex ||= / (?0|[1-9]\d*) - \.(?0|[1-9]\d*) - \.(?0|[1-9]\d*) + /x.freeze + end + + def _semver_minor_regex + @_semver_minor_regex ||= / + (?0|[1-9]\d*) + /x.freeze + end + + def _semver_patch_regex + @_semver_patch_regex ||= / + (?0|[1-9]\d*) /x.freeze end diff --git a/lib/gitlab/repository_archive_rate_limiter.rb b/lib/gitlab/repository_archive_rate_limiter.rb index 31a3dc34bf6..d395b1aba7f 100644 --- a/lib/gitlab/repository_archive_rate_limiter.rb +++ b/lib/gitlab/repository_archive_rate_limiter.rb @@ -3,7 +3,7 @@ module Gitlab module RepositoryArchiveRateLimiter def check_archive_rate_limit!(current_user, project, &block) - return unless Feature.enabled?(:archive_rate_limit) + return unless Feature.enabled?(:archive_rate_limit, default_enabled: :yaml) threshold = current_user ? nil : 100 diff --git a/lib/gitlab/search/params.rb b/lib/gitlab/search/params.rb index e6a1305a82a..1ae14e5e618 100644 --- a/lib/gitlab/search/params.rb +++ b/lib/gitlab/search/params.rb @@ -7,6 +7,7 @@ module Gitlab SEARCH_CHAR_LIMIT = 4096 SEARCH_TERM_LIMIT = 64 + MIN_TERM_LENGTH = 3 # Generic validation validates :query_string, length: { maximum: SEARCH_CHAR_LIMIT } @@ -53,6 +54,10 @@ module Gitlab errors[:query_string].none? { |msg| msg.include? SEARCH_TERM_LIMIT.to_s } end + def email_lookup? + search_terms.any? { |term| term =~ URI::MailTo::EMAIL_REGEXP } + end + def validate if detect_abuse? abuse_detection.validate @@ -75,8 +80,12 @@ module Gitlab @detect_abuse end + def search_terms + @search_terms ||= query_string.split.select { |word| word.length >= MIN_TERM_LENGTH } + end + def not_too_many_terms - if query_string.split.count { |word| word.length >= 3 } > SEARCH_TERM_LIMIT + if search_terms.count > SEARCH_TERM_LIMIT errors.add :query_string, "has too many search terms (maximum is #{SEARCH_TERM_LIMIT})" end end diff --git a/lib/gitlab/sherlock.rb b/lib/gitlab/sherlock.rb deleted file mode 100644 index a1471c9de47..00000000000 --- a/lib/gitlab/sherlock.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -require 'securerandom' - -module Gitlab - module Sherlock - @collection = Collection.new - - class << self - attr_reader :collection - end - - def self.enabled? - Rails.env.development? && !!ENV['ENABLE_SHERLOCK'] - end - - def self.enable_line_profiler? - RUBY_ENGINE == 'ruby' - end - end -end diff --git a/lib/gitlab/sherlock/collection.rb b/lib/gitlab/sherlock/collection.rb deleted file mode 100644 index ce3a376cf75..00000000000 --- a/lib/gitlab/sherlock/collection.rb +++ /dev/null @@ -1,51 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Sherlock - # A collection of transactions recorded by Sherlock. - # - # Method calls for this class are synchronized using a mutex to allow - # sharing of a single Collection instance between threads (e.g. when using - # Puma as a webserver). - class Collection - include Enumerable - - def initialize - @transactions = [] - @mutex = Mutex.new - end - - def add(transaction) - synchronize { @transactions << transaction } - end - - alias_method :<<, :add - - def each(&block) - synchronize { @transactions.each(&block) } - end - - def clear - synchronize { @transactions.clear } - end - - def empty? - synchronize { @transactions.empty? } - end - - def find_transaction(id) - find { |trans| trans.id == id } - end - - def newest_first - sort { |a, b| b.finished_at <=> a.finished_at } - end - - private - - def synchronize(&block) - @mutex.synchronize(&block) - end - end - end -end diff --git a/lib/gitlab/sherlock/file_sample.rb b/lib/gitlab/sherlock/file_sample.rb deleted file mode 100644 index 5d10d8c4877..00000000000 --- a/lib/gitlab/sherlock/file_sample.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Sherlock - class FileSample - attr_reader :id, :file, :line_samples, :events, :duration - - # file - The full path to the file this sample belongs to. - # line_samples - An array of LineSample objects. - # duration - The total execution time in milliseconds. - # events - The total amount of events. - def initialize(file, line_samples, duration, events) - @id = SecureRandom.uuid - @file = file - @line_samples = line_samples - @duration = duration - @events = events - end - - def relative_path - @relative_path ||= @file.gsub(%r{^#{Rails.root}/?}, '') - end - - def to_param - @id - end - - def source - @source ||= File.read(@file) - end - end - end -end diff --git a/lib/gitlab/sherlock/line_profiler.rb b/lib/gitlab/sherlock/line_profiler.rb deleted file mode 100644 index aa25eb5a571..00000000000 --- a/lib/gitlab/sherlock/line_profiler.rb +++ /dev/null @@ -1,100 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Sherlock - # Class for profiling code on a per line basis. - # - # The LineProfiler class can be used to profile code on per line basis - # without littering your code with Ruby implementation specific profiling - # methods. - # - # This profiler only includes samples taking longer than a given threshold - # and those that occur in the actual application (e.g. files from Gems are - # ignored). - class LineProfiler - # The minimum amount of time that has to be spent in a file for it to be - # included in a list of samples. - MINIMUM_DURATION = 10.0 - - # Profiles the given block. - # - # Example: - # - # profiler = LineProfiler.new - # - # retval, samples = profiler.profile do - # "cats are amazing" - # end - # - # retval # => "cats are amazing" - # samples # => [#, ...] - # - # Returns an Array containing the block's return value and an Array of - # FileSample objects. - def profile(&block) - if mri? - profile_mri(&block) - else - raise NotImplementedError, - 'Line profiling is not supported on this platform' - end - end - - # Profiles the given block using rblineprof (MRI only). - def profile_mri - require 'rblineprof' - - retval = nil - samples = lineprof(/^#{Rails.root}/) { retval = yield } - - file_samples = aggregate_rblineprof(samples) - - [retval, file_samples] - end - - # Returns an Array of file samples based on the output of rblineprof. - # - # lineprof_stats - A Hash containing rblineprof statistics on a per file - # basis. - # - # Returns an Array of FileSample objects. - def aggregate_rblineprof(lineprof_stats) - samples = [] - - lineprof_stats.each do |(file, stats)| - source_lines = File.read(file).each_line.to_a - line_samples = [] - - total_duration = microsec_to_millisec(stats[0][0]) - total_events = stats[0][2] - - next if total_duration <= MINIMUM_DURATION - - stats[1..].each_with_index do |data, index| - next unless source_lines[index] - - duration = microsec_to_millisec(data[0]) - events = data[2] - - line_samples << LineSample.new(duration, events) - end - - samples << FileSample - .new(file, line_samples, total_duration, total_events) - end - - samples - end - - private - - def microsec_to_millisec(microsec) - microsec / 1000.0 - end - - def mri? - RUBY_ENGINE == 'ruby' - end - end - end -end diff --git a/lib/gitlab/sherlock/line_sample.rb b/lib/gitlab/sherlock/line_sample.rb deleted file mode 100644 index c92fa9ea1ff..00000000000 --- a/lib/gitlab/sherlock/line_sample.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Sherlock - class LineSample - attr_reader :duration, :events - - # duration - The execution time in milliseconds. - # events - The amount of events. - def initialize(duration, events) - @duration = duration - @events = events - end - - # Returns the sample duration percentage relative to the given duration. - # - # Example: - # - # sample.duration # => 150 - # sample.percentage_of(1500) # => 10.0 - # - # total_duration - The total duration to compare with. - # - # Returns a float - def percentage_of(total_duration) - (duration.to_f / total_duration) * 100.0 - end - - # Returns true if the current sample takes up the majority of the given - # duration. - # - # total_duration - The total duration to compare with. - def majority_of?(total_duration) - percentage_of(total_duration) >= 30 - end - end - end -end diff --git a/lib/gitlab/sherlock/location.rb b/lib/gitlab/sherlock/location.rb deleted file mode 100644 index 4bba60f3490..00000000000 --- a/lib/gitlab/sherlock/location.rb +++ /dev/null @@ -1,28 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Sherlock - class Location - attr_reader :path, :line - - SHERLOCK_DIR = File.dirname(__FILE__) - - # Creates a new Location from a `Thread::Backtrace::Location`. - def self.from_ruby_location(location) - new(location.path, location.lineno) - end - - # path - The full path of the frame as a String. - # line - The line number of the frame as a Fixnum. - def initialize(path, line) - @path = path - @line = line - end - - # Returns true if the current frame originated from the application. - def application? - @path.start_with?(Rails.root.to_s) && !path.start_with?(SHERLOCK_DIR) - end - end - end -end diff --git a/lib/gitlab/sherlock/middleware.rb b/lib/gitlab/sherlock/middleware.rb deleted file mode 100644 index f7b08d58e49..00000000000 --- a/lib/gitlab/sherlock/middleware.rb +++ /dev/null @@ -1,43 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Sherlock - # Rack middleware used for tracking request metrics. - class Middleware - CONTENT_TYPES = %r{text/html|application/json}i.freeze - - IGNORE_PATHS = %r{^/sherlock}.freeze - - def initialize(app) - @app = app - end - - # env - A Hash containing Rack environment details. - def call(env) - if instrument?(env) - call_with_instrumentation(env) - else - @app.call(env) - end - end - - def call_with_instrumentation(env) - trans = transaction_from_env(env) - retval = trans.run { @app.call(env) } - - Sherlock.collection.add(trans) - - retval - end - - def instrument?(env) - !!(env['HTTP_ACCEPT'] =~ CONTENT_TYPES && - env['REQUEST_URI'] !~ IGNORE_PATHS) - end - - def transaction_from_env(env) - Transaction.new(env['REQUEST_METHOD'], env['REQUEST_URI']) - end - end - end -end diff --git a/lib/gitlab/sherlock/query.rb b/lib/gitlab/sherlock/query.rb deleted file mode 100644 index 6f1d2ad23c1..00000000000 --- a/lib/gitlab/sherlock/query.rb +++ /dev/null @@ -1,112 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Sherlock - class Query - attr_reader :id, :query, :started_at, :finished_at, :backtrace - - # SQL identifiers that should be prefixed with newlines. - PREFIX_NEWLINE = %r{ - \s+(FROM - |(LEFT|RIGHT)?INNER\s+JOIN - |(LEFT|RIGHT)?OUTER\s+JOIN - |WHERE - |AND - |GROUP\s+BY - |ORDER\s+BY - |LIMIT - |OFFSET)\s+}ix.freeze # Vim indent breaks when this is on a newline :< - - # Creates a new Query using a String and a separate Array of bindings. - # - # query - A String containing a SQL query, optionally with numeric - # placeholders (`$1`, `$2`, etc). - # - # bindings - An Array of ActiveRecord columns and their values. - # started_at - The start time of the query as a Time-like object. - # finished_at - The completion time of the query as a Time-like object. - # - # Returns a new Query object. - def self.new_with_bindings(query, bindings, started_at, finished_at) - bindings.each_with_index do |(_, value), index| - quoted_value = ActiveRecord::Base.connection.quote(value) - - query = query.gsub("$#{index + 1}", quoted_value) - end - - new(query, started_at, finished_at) - end - - # query - The SQL query as a String (without placeholders). - # started_at - The start time of the query as a Time-like object. - # finished_at - The completion time of the query as a Time-like object. - def initialize(query, started_at, finished_at) - @id = SecureRandom.uuid - @query = query - @started_at = started_at - @finished_at = finished_at - @backtrace = caller_locations.map do |loc| - Location.from_ruby_location(loc) - end - - unless @query.end_with?(';') - @query = "#{@query};" - end - end - - # Returns the query duration in milliseconds. - def duration - @duration ||= (@finished_at - @started_at) * 1000.0 - end - - def to_param - @id - end - - # Returns a human readable version of the query. - def formatted_query - @formatted_query ||= format_sql(@query) - end - - # Returns the last application frame of the backtrace. - def last_application_frame - @last_application_frame ||= @backtrace.find(&:application?) - end - - # Returns an Array of application frames (excluding Gems and the likes). - def application_backtrace - @application_backtrace ||= @backtrace.select(&:application?) - end - - # Returns the query plan as a String. - def explain - unless @explain - ActiveRecord::Base.connection.transaction do - @explain = raw_explain(@query).values.flatten.join("\n") - - # Roll back any queries that mutate data so we don't mess up - # anything when running explain on an INSERT, UPDATE, DELETE, etc. - raise ActiveRecord::Rollback - end - end - - @explain - end - - private - - def raw_explain(query) - explain = "EXPLAIN ANALYZE #{query};" - - ActiveRecord::Base.connection.execute(explain) - end - - def format_sql(query) - query.each_line - .map { |line| line.strip } - .join("\n") - .gsub(PREFIX_NEWLINE) { "\n#{Regexp.last_match(1)} " } - end - end - end -end diff --git a/lib/gitlab/sherlock/transaction.rb b/lib/gitlab/sherlock/transaction.rb deleted file mode 100644 index d04624977dc..00000000000 --- a/lib/gitlab/sherlock/transaction.rb +++ /dev/null @@ -1,140 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Sherlock - class Transaction - attr_reader :id, :type, :path, :queries, :file_samples, :started_at, - :finished_at, :view_counts - - # type - The type of transaction (e.g. "GET", "POST", etc) - # path - The path of the transaction (e.g. the HTTP request path) - def initialize(type, path) - @id = SecureRandom.uuid - @type = type - @path = path - @queries = [] - @file_samples = [] - @started_at = nil - @finished_at = nil - @thread = Thread.current - @view_counts = Hash.new(0) - end - - # Runs the transaction and returns the block's return value. - def run - @started_at = Time.now - - retval = with_subscriptions do - profile_lines { yield } - end - - @finished_at = Time.now - - retval - end - - # Returns the duration in seconds. - def duration - @duration ||= started_at && finished_at ? finished_at - started_at : 0 - end - - # Returns the total query duration in seconds. - def query_duration - @query_duration ||= @queries.map { |q| q.duration }.inject(:+) / 1000.0 - end - - def to_param - @id - end - - # Returns the queries sorted in descending order by their durations. - def sorted_queries - @queries.sort { |a, b| b.duration <=> a.duration } - end - - # Returns the file samples sorted in descending order by their durations. - def sorted_file_samples - @file_samples.sort { |a, b| b.duration <=> a.duration } - end - - # Finds a query by the given ID. - # - # id - The query ID as a String. - # - # Returns a Query object if one could be found, nil otherwise. - def find_query(id) - @queries.find { |query| query.id == id } - end - - # Finds a file sample by the given ID. - # - # id - The query ID as a String. - # - # Returns a FileSample object if one could be found, nil otherwise. - def find_file_sample(id) - @file_samples.find { |sample| sample.id == id } - end - - def profile_lines - retval = nil - - if Sherlock.enable_line_profiler? - retval, @file_samples = LineProfiler.new.profile { yield } - else - retval = yield - end - - retval - end - - def subscribe_to_active_record - ActiveSupport::Notifications.subscribe('sql.active_record') do |_, start, finish, _, data| - next unless same_thread? - - unless data.fetch(:cached, data[:name] == 'CACHE') - track_query(data[:sql].strip, data[:binds], start, finish) - end - end - end - - def subscribe_to_action_view - regex = /render_(template|partial)\.action_view/ - - ActiveSupport::Notifications.subscribe(regex) do |_, start, finish, _, data| - next unless same_thread? - - track_view(data[:identifier]) - end - end - - private - - def track_query(query, bindings, start, finish) - @queries << Query.new_with_bindings(query, bindings, start, finish) - end - - def track_view(path) - @view_counts[path] += 1 - end - - def with_subscriptions - ar_subscriber = subscribe_to_active_record - av_subscriber = subscribe_to_action_view - - retval = yield - - ActiveSupport::Notifications.unsubscribe(ar_subscriber) - ActiveSupport::Notifications.unsubscribe(av_subscriber) - - retval - end - - # In case somebody uses a multi-threaded server locally (e.g. Puma) we - # _only_ want to track notifications that originate from the transaction - # thread. - def same_thread? - Thread.current == @thread - end - end - end -end diff --git a/lib/gitlab/sidekiq_logging/json_formatter.rb b/lib/gitlab/sidekiq_logging/json_formatter.rb index a6281bbdf26..dd50fef8c3d 100644 --- a/lib/gitlab/sidekiq_logging/json_formatter.rb +++ b/lib/gitlab/sidekiq_logging/json_formatter.rb @@ -2,6 +2,7 @@ # This is needed for sidekiq-cluster require 'json' +require 'sidekiq/job_retry' module Gitlab module SidekiqLogging diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb index 3438bc0f3ef..a9bfcce2e0a 100644 --- a/lib/gitlab/sidekiq_logging/structured_logger.rb +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -2,6 +2,8 @@ require 'active_record' require 'active_record/log_subscriber' +require 'sidekiq/job_logger' +require 'sidekiq/job_retry' module Gitlab module SidekiqLogging diff --git a/lib/gitlab/sidekiq_middleware/monitor.rb b/lib/gitlab/sidekiq_middleware/monitor.rb index ed825dbfd60..d38fed3b768 100644 --- a/lib/gitlab/sidekiq_middleware/monitor.rb +++ b/lib/gitlab/sidekiq_middleware/monitor.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'sidekiq/job_retry' + module Gitlab module SidekiqMiddleware class Monitor diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb index 120d18f63f2..66417b3697e 100644 --- a/lib/gitlab/sidekiq_status.rb +++ b/lib/gitlab/sidekiq_status.rb @@ -29,16 +29,15 @@ module Gitlab # for most jobs. DEFAULT_EXPIRATION = 30.minutes.to_i - DEFAULT_VALUE = 1 - DEFAULT_VALUE_MESSAGE = 'Keys using the default value for SidekiqStatus detected' - # Starts tracking of the given job. # # jid - The Sidekiq job ID # expire - The expiration time of the Redis key. - def self.set(jid, expire = DEFAULT_EXPIRATION, value: DEFAULT_VALUE) + def self.set(jid, expire = DEFAULT_EXPIRATION) + return unless expire + Sidekiq.redis do |redis| - redis.set(key_for(jid), value, ex: expire) + redis.set(key_for(jid), 1, ex: expire) end end @@ -94,17 +93,10 @@ module Gitlab return [] if job_ids.empty? keys = job_ids.map { |jid| key_for(jid) } - results = Sidekiq.redis { |redis| redis.mget(*keys) } - - if Feature.enabled?(:log_implicit_sidekiq_status_calls, default_enabled: :yaml) - to_log = keys.zip(results).select do |_key, result| - result == DEFAULT_VALUE.to_s - end.map(&:first) - - Sidekiq.logger.info(message: DEFAULT_VALUE_MESSAGE, keys: to_log) if to_log.any? - end - results.map { |result| !result.nil? } + Sidekiq + .redis { |redis| redis.mget(*keys) } + .map { |result| !result.nil? } end # Returns the JIDs that are completed diff --git a/lib/gitlab/sidekiq_status/client_middleware.rb b/lib/gitlab/sidekiq_status/client_middleware.rb index cee7270f2fb..8471696dcea 100644 --- a/lib/gitlab/sidekiq_status/client_middleware.rb +++ b/lib/gitlab/sidekiq_status/client_middleware.rb @@ -4,10 +4,8 @@ module Gitlab module SidekiqStatus class ClientMiddleware def call(_, job, _, _) - status_expiration = job['status_expiration'] || Gitlab::SidekiqStatus::DEFAULT_EXPIRATION - value = job['status_expiration'] ? 2 : Gitlab::SidekiqStatus::DEFAULT_VALUE + Gitlab::SidekiqStatus.set(job['jid'], job['status_expiration']) - Gitlab::SidekiqStatus.set(job['jid'], status_expiration, value: value) yield end end diff --git a/lib/gitlab/sourcegraph.rb b/lib/gitlab/sourcegraph.rb index 7ef6ab32bd4..892c4468107 100644 --- a/lib/gitlab/sourcegraph.rb +++ b/lib/gitlab/sourcegraph.rb @@ -8,13 +8,14 @@ module Gitlab end def feature_available? - # The sourcegraph_bundle feature could be conditionally applied, so check if `!off?` - !feature.off? + # The sourcegraph feature could be conditionally applied, so check if `!off?` + # We also can't just check !off? because the ActiveRecord might not exist yet + self.feature_enabled? || !feature.off? end def feature_enabled?(actor = nil) # Some CI jobs grep for Feature.enabled? in our codebase, so it is important this reference stays around. - Feature.enabled?(:sourcegraph, actor) + Feature.enabled?(:sourcegraph, actor, default_enabled: :yaml) end private diff --git a/lib/gitlab/ssh_public_key.rb b/lib/gitlab/ssh_public_key.rb index 6df54852d02..314cc5e2db6 100644 --- a/lib/gitlab/ssh_public_key.rb +++ b/lib/gitlab/ssh_public_key.rb @@ -2,13 +2,15 @@ module Gitlab class SSHPublicKey - Technology = Struct.new(:name, :key_class, :supported_sizes) + Technology = Struct.new(:name, :key_class, :supported_sizes, :supported_algorithms) + # See https://man.openbsd.org/sshd#AUTHORIZED_KEYS_FILE_FORMAT for the list of + # supported algorithms. TECHNOLOGIES = [ - Technology.new(:rsa, OpenSSL::PKey::RSA, [1024, 2048, 3072, 4096]), - Technology.new(:dsa, OpenSSL::PKey::DSA, [1024, 2048, 3072]), - Technology.new(:ecdsa, OpenSSL::PKey::EC, [256, 384, 521]), - Technology.new(:ed25519, Net::SSH::Authentication::ED25519::PubKey, [256]) + Technology.new(:rsa, OpenSSL::PKey::RSA, [1024, 2048, 3072, 4096], %w(ssh-rsa)), + Technology.new(:dsa, OpenSSL::PKey::DSA, [1024, 2048, 3072], %w(ssh-dss)), + Technology.new(:ecdsa, OpenSSL::PKey::EC, [256, 384, 521], %w(ecdsa-sha2-nistp256 ecdsa-sha2-nistp384 ecdsa-sha2-nistp521)), + Technology.new(:ed25519, Net::SSH::Authentication::ED25519::PubKey, [256], %w(ssh-ed25519)) ].freeze def self.technology(name) @@ -19,8 +21,20 @@ module Gitlab TECHNOLOGIES.find { |tech| key.is_a?(tech.key_class) } end + def self.supported_types + TECHNOLOGIES.map(&:name) + end + def self.supported_sizes(name) - technology(name)&.supported_sizes + technology(name).supported_sizes + end + + def self.supported_algorithms + TECHNOLOGIES.flat_map { |tech| tech.supported_algorithms } + end + + def self.supported_algorithms_for_name(name) + technology(name).supported_algorithms end def self.sanitize(key_content) diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb index ac1522b8a6c..228da9ee370 100644 --- a/lib/gitlab/themes.rb +++ b/lib/gitlab/themes.rb @@ -13,26 +13,28 @@ module Gitlab Theme = Struct.new(:id, :name, :css_class, :css_filename, :primary_color) # All available Themes - THEMES = [ - Theme.new(1, 'Indigo', 'ui-indigo', 'theme_indigo', '#292961'), - Theme.new(6, 'Light Indigo', 'ui-light-indigo', 'theme_light_indigo', '#4b4ba3'), - Theme.new(4, 'Blue', 'ui-blue', 'theme_blue', '#1a3652'), - Theme.new(7, 'Light Blue', 'ui-light-blue', 'theme_light_blue', '#2261a1'), - Theme.new(5, 'Green', 'ui-green', 'theme_green', '#0d4524'), - Theme.new(8, 'Light Green', 'ui-light-green', 'theme_light_green', '#156b39'), - Theme.new(9, 'Red', 'ui-red', 'theme_red', '#691a16'), - Theme.new(10, 'Light Red', 'ui-light-red', 'theme_light_red', '#a62e21'), - Theme.new(2, 'Dark', 'ui-dark', 'theme_dark', '#303030'), - Theme.new(3, 'Light', 'ui-light', 'theme_light', '#666'), - Theme.new(11, 'Dark Mode (alpha)', 'gl-dark', nil, '#303030') - ].freeze + def available_themes + [ + Theme.new(1, s_('NavigationTheme|Indigo'), 'ui-indigo', 'theme_indigo', '#292961'), + Theme.new(6, s_('NavigationTheme|Light Indigo'), 'ui-light-indigo', 'theme_light_indigo', '#4b4ba3'), + Theme.new(4, s_('NavigationTheme|Blue'), 'ui-blue', 'theme_blue', '#1a3652'), + Theme.new(7, s_('NavigationTheme|Light Blue'), 'ui-light-blue', 'theme_light_blue', '#2261a1'), + Theme.new(5, s_('NavigationTheme|Green'), 'ui-green', 'theme_green', '#0d4524'), + Theme.new(8, s_('NavigationTheme|Light Green'), 'ui-light-green', 'theme_light_green', '#156b39'), + Theme.new(9, s_('NavigationTheme|Red'), 'ui-red', 'theme_red', '#691a16'), + Theme.new(10, s_('NavigationTheme|Light Red'), 'ui-light-red', 'theme_light_red', '#a62e21'), + Theme.new(2, s_('NavigationTheme|Dark'), 'ui-dark', 'theme_dark', '#303030'), + Theme.new(3, s_('NavigationTheme|Light'), 'ui-light', 'theme_light', '#666'), + Theme.new(11, s_('NavigationTheme|Dark Mode (alpha)'), 'gl-dark', nil, '#303030') + ] + end # Convenience method to get a space-separated String of all the theme # classes that might be applied to the `body` element # # Returns a String def body_classes - THEMES.collect(&:css_class).uniq.join(' ') + available_themes.collect(&:css_class).uniq.join(' ') end # Get a Theme by its ID @@ -43,12 +45,12 @@ module Gitlab # # Returns a Theme def by_id(id) - THEMES.detect { |t| t.id == id } || default + available_themes.detect { |t| t.id == id } || default end # Returns the number of defined Themes def count - THEMES.size + available_themes.size end # Get the default Theme @@ -62,7 +64,7 @@ module Gitlab # # Yields the Theme object def each(&block) - THEMES.each(&block) + available_themes.each(&block) end # Get the Theme for the specified user, or the default @@ -79,7 +81,7 @@ module Gitlab end def self.valid_ids - THEMES.map(&:id) + available_themes.map(&:id) end private @@ -87,7 +89,7 @@ module Gitlab def default_id @default_id ||= begin id = Gitlab.config.gitlab.default_theme.to_i - theme_ids = THEMES.map(&:id) + theme_ids = available_themes.map(&:id) theme_ids.include?(id) ? id : APPLICATION_DEFAULT end diff --git a/lib/gitlab/tracking/standard_context.rb b/lib/gitlab/tracking/standard_context.rb index 837390b91fb..542dc476526 100644 --- a/lib/gitlab/tracking/standard_context.rb +++ b/lib/gitlab/tracking/standard_context.rb @@ -3,7 +3,7 @@ module Gitlab module Tracking class StandardContext - GITLAB_STANDARD_SCHEMA_URL = 'iglu:com.gitlab/gitlab_standard/jsonschema/1-0-7' + GITLAB_STANDARD_SCHEMA_URL = 'iglu:com.gitlab/gitlab_standard/jsonschema/1-0-8' GITLAB_RAILS_SOURCE = 'gitlab-rails' def initialize(namespace: nil, project: nil, user: nil, **extra) @@ -46,7 +46,8 @@ module Gitlab extra: extra, user_id: user&.id, namespace_id: namespace&.id, - project_id: project_id + project_id: project_id, + context_generated_at: Time.current } end diff --git a/lib/gitlab/untrusted_regexp/ruby_syntax.rb b/lib/gitlab/untrusted_regexp/ruby_syntax.rb index 6adf119aa75..010214cf295 100644 --- a/lib/gitlab/untrusted_regexp/ruby_syntax.rb +++ b/lib/gitlab/untrusted_regexp/ruby_syntax.rb @@ -20,13 +20,13 @@ module Gitlab !!self.fabricate(pattern, fallback: fallback) end - def self.fabricate(pattern, fallback: false) - self.fabricate!(pattern, fallback: fallback) + def self.fabricate(pattern, fallback: false, project: nil) + self.fabricate!(pattern, fallback: fallback, project: project) rescue RegexpError nil end - def self.fabricate!(pattern, fallback: false) + def self.fabricate!(pattern, fallback: false, project: nil) raise RegexpError, 'Pattern is not string!' unless pattern.is_a?(String) matches = pattern.match(PATTERN) @@ -38,6 +38,16 @@ module Gitlab raise unless fallback && Feature.enabled?(:allow_unsafe_ruby_regexp, default_enabled: false) + if Feature.enabled?(:ci_unsafe_regexp_logger, type: :ops, default_enabled: :yaml) + Gitlab::AppJsonLogger.info( + class: self.class.name, + regexp: pattern.to_s, + fabricated: 'unsafe ruby regexp', + project_id: project&.id, + project_path: project&.full_path + ) + end + create_ruby_regexp(matches[:regexp], matches[:flags]) end end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 917c273d3f6..adf920d8b52 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -304,7 +304,8 @@ module Gitlab # rubocop: disable UsageData/LargeTable adapter: alt_usage_data { ApplicationRecord.database.adapter_name }, version: alt_usage_data { ApplicationRecord.database.version }, - pg_system_id: alt_usage_data { ApplicationRecord.database.system_id } + pg_system_id: alt_usage_data { ApplicationRecord.database.system_id }, + flavor: alt_usage_data { ApplicationRecord.database.flavor } # rubocop: enable UsageData/LargeTable }, mail: { @@ -521,11 +522,7 @@ module Gitlab projects_with_disable_overriding_approvers_per_merge_request: count(::Project.where(time_period.merge(disable_overriding_approvers_per_merge_request: true))), projects_without_disable_overriding_approvers_per_merge_request: count(::Project.where(time_period.merge(disable_overriding_approvers_per_merge_request: [false, nil]))), remote_mirrors: distinct_count(::Project.with_remote_mirrors.where(time_period), :creator_id), - snippets: distinct_count(::Snippet.where(time_period), :author_id), - suggestions: distinct_count(::Note.with_suggestions.where(time_period), - :author_id, - start: minimum_id(::User), - finish: maximum_id(::User)) + snippets: distinct_count(::Snippet.where(time_period), :author_id) }.tap do |h| if time_period.present? h[:merge_requests_users] = merge_requests_users(time_period) @@ -552,33 +549,11 @@ module Gitlab user_auth_by_provider: distinct_count_user_auth_by_provider(time_period), unique_users_all_imports: unique_users_all_imports(time_period), bulk_imports: { - gitlab: DEPRECATED_VALUE, gitlab_v1: count(::BulkImport.where(**time_period, source_type: :gitlab)) }, project_imports: project_imports(time_period), issue_imports: issue_imports(time_period), - group_imports: group_imports(time_period), - - # Deprecated data to be removed - projects_imported: { - total: DEPRECATED_VALUE, - gitlab_project: DEPRECATED_VALUE, - gitlab: DEPRECATED_VALUE, - github: DEPRECATED_VALUE, - bitbucket: DEPRECATED_VALUE, - bitbucket_server: DEPRECATED_VALUE, - gitea: DEPRECATED_VALUE, - git: DEPRECATED_VALUE, - manifest: DEPRECATED_VALUE - }, - issues_imported: { - jira: DEPRECATED_VALUE, - fogbugz: DEPRECATED_VALUE, - phabricator: DEPRECATED_VALUE, - csv: DEPRECATED_VALUE - }, - groups_imported: DEPRECATED_VALUE - # End of deprecated keys + group_imports: group_imports(time_period) } end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/gitlab/usage_data_counters/counter_events/package_events.yml b/lib/gitlab/usage_data_counters/counter_events/package_events.yml index 21d637e7152..a64d0ff7e24 100644 --- a/lib/gitlab/usage_data_counters/counter_events/package_events.yml +++ b/lib/gitlab/usage_data_counters/counter_events/package_events.yml @@ -5,12 +5,8 @@ - i_package_conan_delete_package - i_package_conan_pull_package - i_package_conan_push_package -- i_package_container_delete_package -- i_package_container_pull_package -- i_package_container_push_package - i_package_debian_delete_package - i_package_debian_pull_package -- i_package_debian_push_package - i_package_delete_package - i_package_delete_package_by_deploy_token - i_package_delete_package_by_guest @@ -56,9 +52,6 @@ - i_package_rubygems_delete_package - i_package_rubygems_pull_package - i_package_rubygems_push_package -- i_package_tag_delete_package -- i_package_tag_pull_package -- i_package_tag_push_package - i_package_terraform_module_delete_package - i_package_terraform_module_pull_package - i_package_terraform_module_push_package diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index 8fc8bb5d344..c6e9db6a314 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -104,7 +104,7 @@ module Gitlab events_names = events_for_category(category) event_results = events_names.each_with_object({}) do |event, hash| - hash["#{event}_weekly"] = unique_events(**weekly_time_range.merge(event_names: [event])) + hash["#{event}_weekly"] = unique_events(**weekly_time_range.merge(event_names: [event])) unless event == "i_package_composer_deploy_token" hash["#{event}_monthly"] = unique_events(**monthly_time_range.merge(event_names: [event])) end diff --git a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml index d90960b344c..55ed9a42512 100644 --- a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml +++ b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml @@ -103,6 +103,10 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_security_dast_on_demand_api_scan + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_security_coverage_fuzzing category: ci_templates redis_slot: ci_templates @@ -539,6 +543,10 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_implicit_security_dast_on_demand_api_scan + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_implicit_security_coverage_fuzzing category: ci_templates redis_slot: ci_templates diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index bb98a0b262a..fc610f1e2d6 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -368,8 +368,18 @@ redis_slot: testing category: testing aggregation: weekly +- name: users_clicking_license_testing_visiting_external_website + redis_slot: testing + category: testing + aggregation: weekly # Container Security - Network Policies - name: clusters_using_network_policies_ui redis_slot: network_policies category: network_policies aggregation: weekly +# Geo group +- name: g_geo_proxied_requests + category: geo + redis_slot: geo + aggregation: daily + feature_flag: track_geo_proxy_events diff --git a/lib/gitlab/usage_data_counters/known_events/package_events.yml b/lib/gitlab/usage_data_counters/known_events/package_events.yml index e5031599dd0..debdbd8614f 100644 --- a/lib/gitlab/usage_data_counters/known_events/package_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/package_events.yml @@ -15,22 +15,6 @@ category: user_packages aggregation: weekly redis_slot: package -- name: i_package_container_deploy_token - category: deploy_token_packages - aggregation: weekly - redis_slot: package -- name: i_package_container_user - category: user_packages - aggregation: weekly - redis_slot: package -- name: i_package_debian_deploy_token - category: deploy_token_packages - aggregation: weekly - redis_slot: package -- name: i_package_debian_user - category: user_packages - aggregation: weekly - redis_slot: package - name: i_package_generic_deploy_token category: deploy_token_packages aggregation: weekly @@ -39,14 +23,6 @@ category: user_packages aggregation: weekly redis_slot: package -- name: i_package_golang_deploy_token - category: deploy_token_packages - aggregation: weekly - redis_slot: package -- name: i_package_golang_user - category: user_packages - aggregation: weekly - redis_slot: package - name: i_package_helm_deploy_token category: deploy_token_packages aggregation: weekly @@ -95,14 +71,6 @@ category: user_packages aggregation: weekly redis_slot: package -- name: i_package_tag_deploy_token - category: deploy_token_packages - aggregation: weekly - redis_slot: package -- name: i_package_tag_user - category: user_packages - aggregation: weekly - redis_slot: package - name: i_package_terraform_module_deploy_token category: deploy_token_packages aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/quickactions.yml b/lib/gitlab/usage_data_counters/known_events/quickactions.yml index d831ac02dd1..44f6b42d584 100644 --- a/lib/gitlab/usage_data_counters/known_events/quickactions.yml +++ b/lib/gitlab/usage_data_counters/known_events/quickactions.yml @@ -39,6 +39,10 @@ category: quickactions redis_slot: quickactions aggregation: weekly +- name: i_quickactions_clear_health_status + category: quickactions + redis_slot: quickactions + aggregation: weekly - name: i_quickactions_clone category: quickactions redis_slot: quickactions @@ -263,6 +267,10 @@ category: quickactions redis_slot: quickactions aggregation: weekly +- name: i_quickactions_health_status + category: quickactions + redis_slot: quickactions + aggregation: weekly - name: i_quickactions_wip category: quickactions redis_slot: quickactions diff --git a/lib/gitlab/utils/sanitize_node_link.rb b/lib/gitlab/utils/sanitize_node_link.rb index ab5d18e9c8a..b0dfa087fcf 100644 --- a/lib/gitlab/utils/sanitize_node_link.rb +++ b/lib/gitlab/utils/sanitize_node_link.rb @@ -8,6 +8,12 @@ module Gitlab UNSAFE_PROTOCOLS = %w(data javascript vbscript).freeze ATTRS_TO_SANITIZE = %w(href src data-src data-canonical-src).freeze + # sanitize 6.0 requires only a context argument. Do not add any default + # arguments to this method. + def sanitize_unsafe_links(env) + remove_unsafe_links(env) + end + def remove_unsafe_links(env, remove_invalid_links: true) node = env[:node] diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb index e347168f419..6c182f98dd0 100644 --- a/lib/gitlab/utils/usage_data.rb +++ b/lib/gitlab/utils/usage_data.rb @@ -169,7 +169,8 @@ module Gitlab return -1 if args.any?(&:negative?) args.sum - rescue StandardError + rescue StandardError => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) FALLBACK end @@ -179,7 +180,8 @@ module Gitlab else value end - rescue StandardError + rescue StandardError => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) fallback end @@ -295,13 +297,15 @@ module Gitlab def redis_usage_counter yield - rescue ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent, Gitlab::UsageDataCounters::HLLRedisCounter::EventError + rescue ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent, Gitlab::UsageDataCounters::HLLRedisCounter::EventError => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) FALLBACK end def redis_usage_data_totals(counter) counter.totals - rescue ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent + rescue ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) counter.fallback_totals end end diff --git a/lib/gitlab/web_hooks.rb b/lib/gitlab/web_hooks.rb new file mode 100644 index 00000000000..349c7a020cc --- /dev/null +++ b/lib/gitlab/web_hooks.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +module Gitlab + module WebHooks + GITLAB_EVENT_HEADER = 'X-Gitlab-Event' + end +end diff --git a/lib/gitlab/web_hooks/recursion_detection.rb b/lib/gitlab/web_hooks/recursion_detection.rb new file mode 100644 index 00000000000..1b5350d4a4e --- /dev/null +++ b/lib/gitlab/web_hooks/recursion_detection.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +# This module detects and blocks recursive webhook requests. +# +# Recursion can happen when a webhook has been configured to make a call +# to its own GitLab instance (i.e., its API), and during the execution of +# the call the webhook is triggered again to create an infinite loop of +# being triggered. +# +# Additionally the module blocks a webhook once the number of requests to +# the instance made by a series of webhooks triggering other webhooks reaches +# a limit. +# +# Blocking recursive webhooks allows GitLab to continue to support workflows +# that use webhooks to call the API non-recursively, or do not go on to +# trigger an unreasonable number of other webhooks. +module Gitlab + module WebHooks + module RecursionDetection + COUNT_LIMIT = 100 + TOUCH_CACHE_TTL = 30.minutes + + class << self + def set_from_headers(headers) + uuid = headers[UUID::HEADER] + + return unless uuid + + set_request_uuid(uuid) + end + + def set_request_uuid(uuid) + UUID.instance.request_uuid = uuid + end + + # Before a webhook is executed, `.register!` should be called. + # Adds the webhook ID to a cache (see `#cache_key_for_hook` for + # details of the cache). + def register!(hook) + cache_key = cache_key_for_hook(hook) + + ::Gitlab::Redis::SharedState.with do |redis| + redis.multi do + redis.sadd(cache_key, hook.id) + redis.expire(cache_key, TOUCH_CACHE_TTL) + end + end + end + + # Returns true if the webhook ID is present in the cache, or if the + # number of IDs in the cache exceeds the limit (see + # `#cache_key_for_hook` for details of the cache). + def block?(hook) + # If a request UUID has not been set then we know the request was not + # made by a webhook, and no recursion is possible. + return false unless UUID.instance.request_uuid + + cache_key = cache_key_for_hook(hook) + + ::Gitlab::Redis::SharedState.with do |redis| + redis.sismember(cache_key, hook.id) || + redis.scard(cache_key) >= COUNT_LIMIT + end + end + + def header(hook) + UUID.instance.header(hook) + end + + def to_log(hook) + { + uuid: UUID.instance.uuid_for_hook(hook), + ids: ::Gitlab::Redis::SharedState.with { |redis| redis.smembers(cache_key_for_hook(hook)).map(&:to_i) } + } + end + + private + + # Returns a cache key scoped to a UUID. + # + # The particular UUID will be either: + # + # - A UUID that was recycled from the request headers if the request was made by a webhook. + # - a new UUID initialized for the webhook. + # + # This means that cycles of webhooks that are triggered from other webhooks + # will share the same cache, and other webhooks will use a new cache. + def cache_key_for_hook(hook) + [:webhook_recursion_detection, UUID.instance.uuid_for_hook(hook)].join(':') + end + end + end + end +end diff --git a/lib/gitlab/web_hooks/recursion_detection/uuid.rb b/lib/gitlab/web_hooks/recursion_detection/uuid.rb new file mode 100644 index 00000000000..9c52399818d --- /dev/null +++ b/lib/gitlab/web_hooks/recursion_detection/uuid.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module WebHooks + module RecursionDetection + class UUID + HEADER = "#{::Gitlab::WebHooks::GITLAB_EVENT_HEADER}-UUID" + + include Singleton + + attr_accessor :request_uuid + + def initialize + self.new_uuids_for_hooks = {} + end + + class << self + # Back the Singleton with RequestStore so it is isolated to this request. + def instance + Gitlab::SafeRequestStore[:web_hook_recursion_detection_uuid] ||= new + end + end + + # Returns a UUID, which will be either: + # + # - The UUID that was recycled from the request headers if the request was made by a webhook. + # - A new UUID initialized for the webhook. + def uuid_for_hook(hook) + request_uuid || new_uuid_for_hook(hook) + end + + def header(hook) + { HEADER => uuid_for_hook(hook) } + end + + private + + attr_accessor :new_uuids_for_hooks + + def new_uuid_for_hook(hook) + new_uuids_for_hooks[hook.id] ||= SecureRandom.uuid + end + end + end + end +end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 3a905a2e1c5..19d30daa577 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -203,11 +203,11 @@ module Gitlab end def verify_api_request!(request_headers) - decode_jwt(request_headers[INTERNAL_API_REQUEST_HEADER]) + decode_jwt_with_issuer(request_headers[INTERNAL_API_REQUEST_HEADER]) end - def decode_jwt(encoded_message) - decode_jwt_for_issuer('gitlab-workhorse', encoded_message) + def decode_jwt_with_issuer(encoded_message) + decode_jwt(encoded_message, issuer: 'gitlab-workhorse') end def secret_path -- cgit v1.2.3