From edaa33dee2ff2f7ea3fac488d41558eb5f86d68c Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Thu, 20 Jan 2022 09:16:11 +0000 Subject: Add latest changes from gitlab-org/gitlab@14-7-stable-ee --- lib/api/api.rb | 1 + lib/api/ci/helpers/runner.rb | 8 - lib/api/ci/job_artifacts.rb | 11 + lib/api/ci/runner.rb | 20 +- lib/api/ci/runners.rb | 7 +- lib/api/ci/triggers.rb | 2 +- lib/api/debian_project_packages.rb | 1 - lib/api/deployments.rb | 2 + lib/api/entities/group_detail.rb | 2 +- lib/api/entities/issue_basic.rb | 2 +- lib/api/entities/merge_request_basic.rb | 4 + lib/api/entities/project.rb | 3 + lib/api/entities/project_with_access.rb | 2 +- lib/api/entities/resource_access_token.rb | 2 +- lib/api/helpers/integrations_helpers.rb | 16 +- lib/api/helpers/members_helpers.rb | 2 +- lib/api/helpers/projects_helpers.rb | 2 + lib/api/helpers/rate_limiter.rb | 5 + lib/api/integrations.rb | 9 +- lib/api/internal/base.rb | 4 + lib/api/internal/kubernetes.rb | 2 +- lib/api/internal/mail_room.rb | 51 +++++ lib/api/issues.rb | 4 +- lib/api/package_files.rb | 19 +- lib/api/project_container_repositories.rb | 1 - lib/api/projects.rb | 1 + lib/api/resource_access_tokens.rb | 10 +- lib/api/rubygem_packages.rb | 9 +- lib/api/search.rb | 14 +- lib/api/terraform/modules/v1/packages.rb | 6 +- lib/api/users.rb | 8 +- lib/api/v3/github.rb | 8 +- lib/backup.rb | 13 -- lib/backup/database.rb | 2 +- lib/backup/files.rb | 8 +- lib/backup/gitaly_backup.rb | 47 +++-- lib/backup/gitaly_rpc_backup.rb | 2 +- lib/backup/manager.rb | 2 +- lib/backup/packages.rb | 13 ++ lib/backup/repositories.rb | 4 +- lib/backup/terraform_state.rb | 13 ++ lib/banzai/filter/base_sanitization_filter.rb | 2 +- lib/banzai/filter/footnote_filter.rb | 62 ++---- lib/banzai/filter/markdown_engines/common_mark.rb | 36 +--- lib/banzai/filter/markdown_post_escape_filter.rb | 18 +- lib/banzai/filter/plantuml_filter.rb | 7 +- .../filter/references/abstract_reference_filter.rb | 2 + lib/banzai/filter/sanitization_filter.rb | 19 +- lib/banzai/filter/syntax_highlight_filter.rb | 14 +- .../reference_parser/merge_request_parser.rb | 2 - lib/banzai/renderer/common_mark/html.rb | 21 -- .../common/extractors/ndjson_extractor.rb | 34 ++- .../common/pipelines/uploads_pipeline.rb | 14 +- lib/bulk_imports/ndjson_pipeline.rb | 2 +- .../pipelines/project_attributes_pipeline.rb | 31 +-- lib/feature.rb | 9 + lib/gitlab.rb | 50 +---- lib/gitlab/anonymous_session.rb | 8 +- lib/gitlab/application_rate_limiter.rb | 10 +- .../syntax_highlighter/html_pipeline_adapter.rb | 6 +- lib/gitlab/auth.rb | 12 +- lib/gitlab/auth/auth_finders.rb | 2 +- lib/gitlab/auth/ldap/config.rb | 3 +- lib/gitlab/auth/o_auth/user.rb | 4 +- .../backfill_ci_namespace_mirrors.rb | 77 +++++++ .../backfill_ci_project_mirrors.rb | 52 +++++ .../backfill_incident_issue_escalation_statuses.rb | 32 +++ lib/gitlab/background_migration/base_job.rb | 23 +++ .../cleanup_concurrent_rename.rb | 14 -- .../cleanup_concurrent_schema_change.rb | 56 ----- .../cleanup_concurrent_type_change.rb | 14 -- lib/gitlab/background_migration/copy_column.rb | 41 ---- .../encrypt_static_object_token.rb | 70 +++++++ ...lity_occurrences_with_hashes_as_raw_metadata.rb | 124 +++++++++++ lib/gitlab/background_migration/job_coordinator.rb | 14 +- .../migrate_legacy_artifacts.rb | 130 ------------ .../populate_test_reports_issue_id.rb | 14 ++ ...recalculate_vulnerabilities_occurrences_uuid.rb | 148 +++++++++++-- .../remove_duplicate_services.rb | 58 ------ lib/gitlab/checks/changes_access.rb | 35 +++- lib/gitlab/ci/build/policy/refs.rb | 5 +- lib/gitlab/ci/build/status/reason.rb | 37 ++++ lib/gitlab/ci/config.rb | 2 +- lib/gitlab/ci/config/entry/root.rb | 8 +- lib/gitlab/ci/jwt_v2.rb | 17 ++ lib/gitlab/ci/pipeline/chain/create.rb | 48 ++--- lib/gitlab/ci/pipeline/chain/create_deployments.rb | 15 +- lib/gitlab/ci/pipeline/chain/seed.rb | 9 +- lib/gitlab/ci/pipeline/logger.rb | 36 +++- lib/gitlab/ci/pipeline/seed/build.rb | 16 +- lib/gitlab/ci/pipeline/seed/context.rb | 11 +- lib/gitlab/ci/queue/metrics.rb | 37 +++- lib/gitlab/ci/status/build/factory.rb | 3 +- lib/gitlab/ci/status/build/waiting_for_approval.rb | 24 +++ lib/gitlab/ci/tags/bulk_insert.rb | 20 +- .../ci/templates/Jobs/Code-Quality.gitlab-ci.yml | 2 +- .../templates/Jobs/Secret-Detection.gitlab-ci.yml | 16 +- lib/gitlab/ci/templates/Ruby.gitlab-ci.yml | 2 +- .../Security/Coverage-Fuzzing.gitlab-ci.yml | 1 + .../Security/DAST-On-Demand-API-Scan.gitlab-ci.yml | 27 +++ .../ci/templates/Terraform.latest.gitlab-ci.yml | 2 + lib/gitlab/ci/trace/remote_checksum.rb | 1 - lib/gitlab/ci/trace/stream.rb | 5 +- lib/gitlab/ci/variables/builder.rb | 64 ++++++ lib/gitlab/ci/yaml_processor.rb | 8 + lib/gitlab/color_schemes.rb | 28 +-- lib/gitlab/config/entry/configurable.rb | 3 +- lib/gitlab/config/entry/factory.rb | 5 + lib/gitlab/config/entry/node.rb | 20 +- .../content_security_policy/config_loader.rb | 2 +- lib/gitlab/data_builder/archive_trace.rb | 19 ++ lib/gitlab/data_builder/deployment.rb | 3 +- .../database/background_migration/batched_job.rb | 18 +- .../background_migration/batched_migration.rb | 2 +- lib/gitlab/database/background_migration_job.rb | 2 +- lib/gitlab/database/batch_counter.rb | 29 +-- lib/gitlab/database/gitlab_loose_foreign_keys.yml | 93 ++++++++- lib/gitlab/database/gitlab_schemas.yml | 12 +- lib/gitlab/database/grant.rb | 2 +- lib/gitlab/database/load_balancing/setup.rb | 4 +- .../database/loose_index_scan_distinct_count.rb | 102 --------- lib/gitlab/database/migration_helpers.rb | 180 ---------------- .../migrations/background_migration_helpers.rb | 70 +++---- .../database/partitioning/partition_manager.rb | 4 + .../database/partitioning/sliding_list_strategy.rb | 28 ++- .../backfill_partitioned_table.rb | 20 +- .../table_management_helpers.rb | 3 +- lib/gitlab/database/reflection.rb | 29 +++ lib/gitlab/database/reindexing.rb | 15 +- lib/gitlab/database/reindexing/coordinator.rb | 19 ++ .../work_items/base_type_importer.rb | 4 +- lib/gitlab/email.rb | 1 + lib/gitlab/email/failure_handler.rb | 46 +++++ .../error_tracking/processor/sidekiq_processor.rb | 2 + lib/gitlab/event_store.rb | 42 ++++ lib/gitlab/event_store/event.rb | 54 +++++ lib/gitlab/event_store/store.rb | 54 +++++ lib/gitlab/event_store/subscriber.rb | 36 ++++ lib/gitlab/event_store/subscription.rb | 37 ++++ lib/gitlab/exceptions_app.rb | 43 ++++ lib/gitlab/experimentation.rb | 8 - lib/gitlab/gon_helper.rb | 1 + lib/gitlab/gpg/commit.rb | 2 +- lib/gitlab/http.rb | 3 +- lib/gitlab/i18n.rb | 22 +- lib/gitlab/import/set_async_jid.rb | 2 +- lib/gitlab/import_export/base/relation_factory.rb | 2 +- .../import_export/group/relation_tree_restorer.rb | 10 +- lib/gitlab/import_export/project/import_export.yml | 2 + lib/gitlab/jwt_authenticatable.rb | 36 ++-- lib/gitlab/kas.rb | 2 +- lib/gitlab/lfs/client.rb | 81 ++++++-- lib/gitlab/logger.rb | 6 +- lib/gitlab/mail_room.rb | 16 +- lib/gitlab/mail_room/authenticator.rb | 50 +++++ .../merge_requests/commit_message_generator.rb | 72 ++++--- lib/gitlab/metrics/exporter/base_exporter.rb | 45 ++-- .../metrics/exporter/gc_request_middleware.rb | 19 ++ .../metrics/exporter/health_checks_middleware.rb | 35 ++++ lib/gitlab/metrics/exporter/metrics_middleware.rb | 41 ++++ lib/gitlab/metrics/exporter/sidekiq_exporter.rb | 11 +- lib/gitlab/metrics/exporter/web_exporter.rb | 8 +- .../metrics/samplers/action_cable_sampler.rb | 4 +- lib/gitlab/metrics/samplers/base_sampler.rb | 14 +- lib/gitlab/metrics/samplers/ruby_sampler.rb | 4 +- lib/gitlab/middleware/multipart.rb | 4 +- .../middleware/webhook_recursion_detection.rb | 19 ++ lib/gitlab/pages.rb | 2 +- .../pagination/keyset/column_order_definition.rb | 25 ++- .../keyset/in_operator_optimization/column_data.rb | 19 +- .../order_by_column_data.rb | 37 ++++ .../in_operator_optimization/order_by_columns.rb | 6 +- .../in_operator_optimization/query_builder.rb | 4 +- .../strategies/order_values_loader_strategy.rb | 15 +- .../strategies/record_loader_strategy.rb | 16 ++ .../pagination/keyset/sql_type_missing_error.rb | 19 ++ lib/gitlab/password.rb | 14 ++ lib/gitlab/redis/multi_store.rb | 229 --------------------- lib/gitlab/redis/sessions.rb | 36 +--- lib/gitlab/redis/sessions_store_helper.rb | 27 --- lib/gitlab/regex.rb | 26 ++- lib/gitlab/repository_archive_rate_limiter.rb | 2 +- lib/gitlab/search/params.rb | 11 +- lib/gitlab/sherlock.rb | 21 -- lib/gitlab/sherlock/collection.rb | 51 ----- lib/gitlab/sherlock/file_sample.rb | 33 --- lib/gitlab/sherlock/line_profiler.rb | 100 --------- lib/gitlab/sherlock/line_sample.rb | 38 ---- lib/gitlab/sherlock/location.rb | 28 --- lib/gitlab/sherlock/middleware.rb | 43 ---- lib/gitlab/sherlock/query.rb | 112 ---------- lib/gitlab/sherlock/transaction.rb | 140 ------------- lib/gitlab/sidekiq_logging/json_formatter.rb | 1 + lib/gitlab/sidekiq_logging/structured_logger.rb | 2 + lib/gitlab/sidekiq_middleware/monitor.rb | 2 + lib/gitlab/sidekiq_status.rb | 22 +- lib/gitlab/sidekiq_status/client_middleware.rb | 4 +- lib/gitlab/sourcegraph.rb | 7 +- lib/gitlab/ssh_public_key.rb | 26 ++- lib/gitlab/themes.rb | 40 ++-- lib/gitlab/tracking/standard_context.rb | 5 +- lib/gitlab/untrusted_regexp/ruby_syntax.rb | 16 +- lib/gitlab/usage_data.rb | 33 +-- .../counter_events/package_events.yml | 7 - .../usage_data_counters/hll_redis_counter.rb | 2 +- .../known_events/ci_templates.yml | 8 + .../usage_data_counters/known_events/common.yml | 10 + .../known_events/package_events.yml | 32 --- .../known_events/quickactions.yml | 8 + lib/gitlab/utils/sanitize_node_link.rb | 6 + lib/gitlab/utils/usage_data.rb | 12 +- lib/gitlab/web_hooks.rb | 7 + lib/gitlab/web_hooks/recursion_detection.rb | 94 +++++++++ lib/gitlab/web_hooks/recursion_detection/uuid.rb | 46 +++++ lib/gitlab/workhorse.rb | 6 +- lib/gitlab_edition.rb | 50 +++++ lib/sidebars/groups/menus/ci_cd_menu.rb | 4 +- lib/sidebars/groups/menus/settings_menu.rb | 14 ++ lib/sidebars/projects/menus/infrastructure_menu.rb | 2 +- lib/sidebars/projects/menus/issues_menu.rb | 6 +- lib/tasks/gitlab/backup.rake | 109 +++++++--- lib/tasks/gitlab/cleanup.rake | 4 +- lib/tasks/gitlab/db.rake | 2 +- lib/tasks/gitlab/docs/compile_deprecations.rake | 39 +++- lib/tasks/gitlab/docs/redirect.rake | 2 +- lib/tasks/gitlab/gitaly.rake | 38 ---- lib/tasks/gitlab/seed/group_seed.rake | 2 +- lib/version_check.rb | 8 - 228 files changed, 2884 insertions(+), 2424 deletions(-) create mode 100644 lib/api/internal/mail_room.rb create mode 100644 lib/backup/packages.rb create mode 100644 lib/backup/terraform_state.rb delete mode 100644 lib/banzai/renderer/common_mark/html.rb create mode 100644 lib/gitlab/background_migration/backfill_ci_namespace_mirrors.rb create mode 100644 lib/gitlab/background_migration/backfill_ci_project_mirrors.rb create mode 100644 lib/gitlab/background_migration/backfill_incident_issue_escalation_statuses.rb create mode 100644 lib/gitlab/background_migration/base_job.rb delete mode 100644 lib/gitlab/background_migration/cleanup_concurrent_rename.rb delete mode 100644 lib/gitlab/background_migration/cleanup_concurrent_schema_change.rb delete mode 100644 lib/gitlab/background_migration/cleanup_concurrent_type_change.rb delete mode 100644 lib/gitlab/background_migration/copy_column.rb create mode 100644 lib/gitlab/background_migration/encrypt_static_object_token.rb create mode 100644 lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb delete mode 100644 lib/gitlab/background_migration/migrate_legacy_artifacts.rb create mode 100644 lib/gitlab/background_migration/populate_test_reports_issue_id.rb delete mode 100644 lib/gitlab/background_migration/remove_duplicate_services.rb create mode 100644 lib/gitlab/ci/build/status/reason.rb create mode 100644 lib/gitlab/ci/jwt_v2.rb create mode 100644 lib/gitlab/ci/status/build/waiting_for_approval.rb create mode 100644 lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml create mode 100644 lib/gitlab/data_builder/archive_trace.rb delete mode 100644 lib/gitlab/database/loose_index_scan_distinct_count.rb create mode 100644 lib/gitlab/email/failure_handler.rb create mode 100644 lib/gitlab/event_store.rb create mode 100644 lib/gitlab/event_store/event.rb create mode 100644 lib/gitlab/event_store/store.rb create mode 100644 lib/gitlab/event_store/subscriber.rb create mode 100644 lib/gitlab/event_store/subscription.rb create mode 100644 lib/gitlab/exceptions_app.rb create mode 100644 lib/gitlab/mail_room/authenticator.rb create mode 100644 lib/gitlab/metrics/exporter/gc_request_middleware.rb create mode 100644 lib/gitlab/metrics/exporter/health_checks_middleware.rb create mode 100644 lib/gitlab/metrics/exporter/metrics_middleware.rb create mode 100644 lib/gitlab/middleware/webhook_recursion_detection.rb create mode 100644 lib/gitlab/pagination/keyset/in_operator_optimization/order_by_column_data.rb create mode 100644 lib/gitlab/pagination/keyset/sql_type_missing_error.rb create mode 100644 lib/gitlab/password.rb delete mode 100644 lib/gitlab/redis/multi_store.rb delete mode 100644 lib/gitlab/redis/sessions_store_helper.rb delete mode 100644 lib/gitlab/sherlock.rb delete mode 100644 lib/gitlab/sherlock/collection.rb delete mode 100644 lib/gitlab/sherlock/file_sample.rb delete mode 100644 lib/gitlab/sherlock/line_profiler.rb delete mode 100644 lib/gitlab/sherlock/line_sample.rb delete mode 100644 lib/gitlab/sherlock/location.rb delete mode 100644 lib/gitlab/sherlock/middleware.rb delete mode 100644 lib/gitlab/sherlock/query.rb delete mode 100644 lib/gitlab/sherlock/transaction.rb create mode 100644 lib/gitlab/web_hooks.rb create mode 100644 lib/gitlab/web_hooks/recursion_detection.rb create mode 100644 lib/gitlab/web_hooks/recursion_detection/uuid.rb create mode 100644 lib/gitlab_edition.rb (limited to 'lib') diff --git a/lib/api/api.rb b/lib/api/api.rb index dcecaeae558..5984879413f 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -299,6 +299,7 @@ module API mount ::API::Internal::Lfs mount ::API::Internal::Pages mount ::API::Internal::Kubernetes + mount ::API::Internal::MailRoom version 'v3', using: :path do # Although the following endpoints are kept behind V3 namespace, diff --git a/lib/api/ci/helpers/runner.rb b/lib/api/ci/helpers/runner.rb index 72c388160b4..43ed35b99fd 100644 --- a/lib/api/ci/helpers/runner.rb +++ b/lib/api/ci/helpers/runner.rb @@ -11,14 +11,6 @@ module API JOB_TOKEN_HEADER = 'HTTP_JOB_TOKEN' JOB_TOKEN_PARAM = :token - def runner_registration_token_valid? - ActiveSupport::SecurityUtils.secure_compare(params[:token], Gitlab::CurrentSettings.runners_registration_token) - end - - def runner_registrar_valid?(type) - Feature.disabled?(:runner_registration_control) || Gitlab::CurrentSettings.valid_runner_registrars.include?(type) - end - def authenticate_runner! forbidden! unless current_runner diff --git a/lib/api/ci/job_artifacts.rb b/lib/api/ci/job_artifacts.rb index 6431436b50d..ca76d2664f8 100644 --- a/lib/api/ci/job_artifacts.rb +++ b/lib/api/ci/job_artifacts.rb @@ -137,6 +137,17 @@ module API status :no_content end + + desc 'Expire the artifacts files from a project' + delete ':id/artifacts' do + not_found! unless Feature.enabled?(:bulk_expire_project_artifacts, default_enabled: :yaml) + + authorize_destroy_artifacts! + + ::Ci::JobArtifacts::DeleteProjectArtifactsService.new(project: user_project).execute + + accepted! + end end end end diff --git a/lib/api/ci/runner.rb b/lib/api/ci/runner.rb index 4317789f7aa..fef6a7891c2 100644 --- a/lib/api/ci/runner.rb +++ b/lib/api/ci/runner.rb @@ -15,6 +15,7 @@ module API params do requires :token, type: String, desc: 'Registration token' optional :description, type: String, desc: %q(Runner's description) + optional :maintainer_note, type: String, desc: %q(Runner's maintainer notes) optional :info, type: Hash, desc: %q(Runner's metadata) optional :active, type: Boolean, desc: 'Should Runner be active' optional :locked, type: Boolean, desc: 'Should Runner be locked for current project' @@ -25,24 +26,11 @@ module API optional :maximum_timeout, type: Integer, desc: 'Maximum timeout set when this Runner will handle the job' end post '/', feature_category: :runner do - attributes = attributes_for_keys([:description, :active, :locked, :run_untagged, :tag_list, :access_level, :maximum_timeout]) + attributes = attributes_for_keys(%i[description maintainer_note active locked run_untagged tag_list access_level maximum_timeout]) .merge(get_runner_details_from_request) - attributes = - if runner_registration_token_valid? - # Create shared runner. Requires admin access - attributes.merge(runner_type: :instance_type) - elsif runner_registrar_valid?('project') && @project = Project.find_by_runners_token(params[:token]) - # Create a specific runner for the project - attributes.merge(runner_type: :project_type, projects: [@project]) - elsif runner_registrar_valid?('group') && @group = Group.find_by_runners_token(params[:token]) - # Create a specific runner for the group - attributes.merge(runner_type: :group_type, groups: [@group]) - else - forbidden! - end - - @runner = ::Ci::Runner.create(attributes) + @runner = ::Ci::RegisterRunnerService.new.execute(params[:token], attributes) + forbidden! unless @runner if @runner.persisted? present @runner, with: Entities::Ci::RunnerRegistrationDetails diff --git a/lib/api/ci/runners.rb b/lib/api/ci/runners.rb index ef712c84804..f21782a698f 100644 --- a/lib/api/ci/runners.rb +++ b/lib/api/ci/runners.rb @@ -229,7 +229,12 @@ module API use :pagination end get ':id/runners' do - runners = ::Ci::Runner.belonging_to_group(user_group.id, include_ancestors: true) + runners = if ::Feature.enabled?(:ci_find_runners_by_ci_mirrors, user_group, default_enabled: :yaml) + ::Ci::Runner.belonging_to_group_and_ancestors(user_group.id) + else + ::Ci::Runner.legacy_belonging_to_group(user_group.id, include_ancestors: true) + end + runners = apply_filter(runners, params) present paginate(runners), with: Entities::Ci::Runner diff --git a/lib/api/ci/triggers.rb b/lib/api/ci/triggers.rb index 6a2b16e1568..ae89b475ef8 100644 --- a/lib/api/ci/triggers.rb +++ b/lib/api/ci/triggers.rb @@ -5,7 +5,7 @@ module API class Triggers < ::API::Base include PaginationParams - HTTP_GITLAB_EVENT_HEADER = "HTTP_#{WebHookService::GITLAB_EVENT_HEADER}".underscore.upcase + HTTP_GITLAB_EVENT_HEADER = "HTTP_#{::Gitlab::WebHooks::GITLAB_EVENT_HEADER}".underscore.upcase feature_category :continuous_integration diff --git a/lib/api/debian_project_packages.rb b/lib/api/debian_project_packages.rb index 497ce2f4356..5fb11db8938 100644 --- a/lib/api/debian_project_packages.rb +++ b/lib/api/debian_project_packages.rb @@ -83,7 +83,6 @@ module API ::Packages::Debian::ProcessChangesWorker.perform_async(package_file.id, current_user.id) # rubocop:disable CodeReuse/Worker end - track_package_event('push_package', :debian, user: current_user, project: authorized_user_project, namespace: authorized_user_project.namespace) created! rescue ObjectStorage::RemoteStoreError => e Gitlab::ErrorTracking.track_exception(e, extra: { file_name: params[:file_name], project_id: authorized_user_project.id }) diff --git a/lib/api/deployments.rb b/lib/api/deployments.rb index 80a50ded522..486ff5d89bc 100644 --- a/lib/api/deployments.rb +++ b/lib/api/deployments.rb @@ -165,3 +165,5 @@ module API end end end + +API::Deployments.prepend_mod diff --git a/lib/api/entities/group_detail.rb b/lib/api/entities/group_detail.rb index 5eaccbc7154..e6872709432 100644 --- a/lib/api/entities/group_detail.rb +++ b/lib/api/entities/group_detail.rb @@ -4,7 +4,7 @@ module API module Entities class GroupDetail < Group expose :shared_with_groups do |group, options| - SharedGroupWithGroup.represent(group.shared_with_group_links.public_or_visible_to_user(group, options[:current_user])) + SharedGroupWithGroup.represent(group.shared_with_group_links_visible_to_user(options[:current_user])) end expose :runners_token, if: lambda { |group, options| options[:user_can_admin_group] } expose :prevent_sharing_groups_outside_hierarchy, if: ->(group) { group.root? } diff --git a/lib/api/entities/issue_basic.rb b/lib/api/entities/issue_basic.rb index 6125dc05a6e..20f66c026e6 100644 --- a/lib/api/entities/issue_basic.rb +++ b/lib/api/entities/issue_basic.rb @@ -23,7 +23,7 @@ module API expose :issue_type, as: :type, format_with: :upcase, - documentation: { type: "String", desc: "One of #{::WorkItem::Type.allowed_types_for_issues.map(&:upcase)}" } + documentation: { type: "String", desc: "One of #{::WorkItems::Type.allowed_types_for_issues.map(&:upcase)}" } expose :assignee, using: ::API::Entities::UserBasic do |issue| issue.assignees.first diff --git a/lib/api/entities/merge_request_basic.rb b/lib/api/entities/merge_request_basic.rb index d5cf2f653db..55d58166590 100644 --- a/lib/api/entities/merge_request_basic.rb +++ b/lib/api/entities/merge_request_basic.rb @@ -3,9 +3,13 @@ module API module Entities class MergeRequestBasic < IssuableEntity + # Deprecated in favour of merge_user expose :merged_by, using: Entities::UserBasic do |merge_request, _options| merge_request.metrics&.merged_by end + expose :merge_user, using: Entities::UserBasic do |merge_request| + merge_request.metrics&.merged_by || merge_request.merge_user + end expose :merged_at do |merge_request, _options| merge_request.metrics&.merged_at end diff --git a/lib/api/entities/project.rb b/lib/api/entities/project.rb index 1b9299ed17e..74097dc2883 100644 --- a/lib/api/entities/project.rb +++ b/lib/api/entities/project.rb @@ -82,6 +82,8 @@ module API expose :forked_from_project, using: Entities::BasicProjectDetails, if: ->(project, options) do project.forked? && Ability.allowed?(options[:current_user], :read_project, project.forked_from_project) end + expose :mr_default_target_self, if: -> (project) { project.forked? } + expose :import_status expose :import_error, if: lambda { |_project, options| options[:user_can_admin_project] } do |project| @@ -130,6 +132,7 @@ module API Ability.allowed?(options[:current_user], :change_repository_storage, project) } expose :keep_latest_artifacts_available?, as: :keep_latest_artifact + expose :runner_token_expiration_interval # rubocop: disable CodeReuse/ActiveRecord def self.preload_resource(project) diff --git a/lib/api/entities/project_with_access.rb b/lib/api/entities/project_with_access.rb index ac89cb52e43..b541ccbadcf 100644 --- a/lib/api/entities/project_with_access.rb +++ b/lib/api/entities/project_with_access.rb @@ -8,7 +8,7 @@ module API if options[:project_members] options[:project_members].find { |member| member.source_id == project.id } else - project.project_member(options[:current_user]) + project.member(options[:current_user]) end end diff --git a/lib/api/entities/resource_access_token.rb b/lib/api/entities/resource_access_token.rb index a1c7b28af45..569fd16f488 100644 --- a/lib/api/entities/resource_access_token.rb +++ b/lib/api/entities/resource_access_token.rb @@ -4,7 +4,7 @@ module API module Entities class ResourceAccessToken < Entities::PersonalAccessToken expose :access_level do |token, options| - options[:project].project_member(token.user).access_level + options[:resource].member(token.user).access_level end end end diff --git a/lib/api/helpers/integrations_helpers.rb b/lib/api/helpers/integrations_helpers.rb index e7fdb6645a5..3af0dd4c532 100644 --- a/lib/api/helpers/integrations_helpers.rb +++ b/lib/api/helpers/integrations_helpers.rb @@ -314,25 +314,33 @@ module API required: false, name: :datadog_site, type: String, - desc: 'Choose the Datadog site to send data to. Set to "datadoghq.eu" to send data to the EU site' + desc: 'The Datadog site to send data to. To send data to the EU site, use datadoghq.eu' }, { required: false, name: :api_url, type: String, - desc: '(Advanced) Define the full URL for your Datadog site directly' + desc: '(Advanced) The full URL for your Datadog site' }, + # TODO: uncomment this field once :datadog_integration_logs_collection is rolled out + # https://gitlab.com/gitlab-org/gitlab/-/issues/346339 + # { + # required: false, + # name: :archive_trace_events, + # type: Boolean, + # desc: 'When enabled, job logs will be collected by Datadog and shown along pipeline execution traces' + # }, { required: false, name: :datadog_service, type: String, - desc: 'Name of this GitLab instance that all data will be tagged with' + desc: 'Tag all data from this GitLab instance in Datadog. Useful when managing several self-managed deployments' }, { required: false, name: :datadog_env, type: String, - desc: 'The environment tag that traces will be tagged with' + desc: 'For self-managed deployments, set the env tag for all the data sent to Datadog. How do I use tags?' } ], 'discord' => [ diff --git a/lib/api/helpers/members_helpers.rb b/lib/api/helpers/members_helpers.rb index c2710be6c03..6c20993431d 100644 --- a/lib/api/helpers/members_helpers.rb +++ b/lib/api/helpers/members_helpers.rb @@ -50,7 +50,7 @@ module API end def find_all_members_for_group(group) - GroupMembersFinder.new(group).execute + GroupMembersFinder.new(group, current_user).execute(include_relations: [:inherited, :direct, :shared_from_groups]) end def present_members(members) diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb index d7de8bd8b8b..00f745067e7 100644 --- a/lib/api/helpers/projects_helpers.rb +++ b/lib/api/helpers/projects_helpers.rb @@ -71,6 +71,7 @@ module API optional :repository_storage, type: String, desc: 'Which storage shard the repository is on. Available only to admins' optional :packages_enabled, type: Boolean, desc: 'Enable project packages feature' optional :squash_option, type: String, values: %w(never always default_on default_off), desc: 'Squash default for project. One of `never`, `always`, `default_on`, or `default_off`.' + optional :mr_default_target_self, Boolean, desc: 'Merge requests of this forked project targets itself by default' end params :optional_project_params_ee do @@ -169,6 +170,7 @@ module API :packages_enabled, :service_desk_enabled, :keep_latest_artifact, + :mr_default_target_self, # TODO: remove in API v5, replaced by *_access_level :issues_enabled, diff --git a/lib/api/helpers/rate_limiter.rb b/lib/api/helpers/rate_limiter.rb index 7d87c74097d..0ad4f089907 100644 --- a/lib/api/helpers/rate_limiter.rb +++ b/lib/api/helpers/rate_limiter.rb @@ -10,6 +10,7 @@ module API # See app/controllers/concerns/check_rate_limit.rb for Rails controllers version module RateLimiter def check_rate_limit!(key, scope:, **options) + return if bypass_header_set? return unless rate_limiter.throttled?(key, scope: scope, **options) rate_limiter.log_request(request, "#{key}_request_limit".to_sym, current_user) @@ -24,6 +25,10 @@ module API def rate_limiter ::Gitlab::ApplicationRateLimiter end + + def bypass_header_set? + ::Gitlab::Throttle.bypass_header.present? && request.get_header(Gitlab::Throttle.bypass_header) == '1' + end end end end diff --git a/lib/api/integrations.rb b/lib/api/integrations.rb index bab8e556a73..ff1d88e35f0 100644 --- a/lib/api/integrations.rb +++ b/lib/api/integrations.rb @@ -111,7 +111,14 @@ module API integration = user_project.find_or_initialize_integration(params[:slug].underscore) destroy_conditionally!(integration) do - attrs = integration_attributes(integration).index_with { nil }.merge(active: false) + attrs = integration_attributes(integration).index_with do |attr| + column = integration.column_for_attribute(attr) + if column.is_a?(ActiveRecord::ConnectionAdapters::NullColumn) + nil + else + column.default + end + end.merge(active: false) render_api_error!('400 Bad Request', 400) unless integration.update(attrs) end diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb index d8e39d089e4..48157a91477 100644 --- a/lib/api/internal/base.rb +++ b/lib/api/internal/base.rb @@ -43,6 +43,10 @@ module API # This is a separate method so that EE can alter its behaviour more # easily. + if Feature.enabled?(:rate_limit_gitlab_shell, default_enabled: :yaml) + check_rate_limit!(:gitlab_shell_operation, scope: [params[:action], params[:project], actor.key_or_user]) + end + # Stores some Git-specific env thread-safely env = parse_env Gitlab::Git::HookEnv.set(gl_repository, env) if container diff --git a/lib/api/internal/kubernetes.rb b/lib/api/internal/kubernetes.rb index f3974236fe3..3977da4bda4 100644 --- a/lib/api/internal/kubernetes.rb +++ b/lib/api/internal/kubernetes.rb @@ -53,7 +53,7 @@ module API def check_agent_token unauthorized! unless agent_token - agent_token.track_usage + Clusters::AgentTokens::TrackUsageService.new(agent_token).execute end end diff --git a/lib/api/internal/mail_room.rb b/lib/api/internal/mail_room.rb new file mode 100644 index 00000000000..6e24cf6e7c5 --- /dev/null +++ b/lib/api/internal/mail_room.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module API + # This internal endpoint receives webhooks sent from the MailRoom component. + # This component constantly listens to configured email accounts. When it + # finds any incoming email or service desk email, it makes a POST request to + # this endpoint. The target mailbox type is indicated in the request path. + # The email raw content is attached to the request body. + # + # For more information, please visit https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/644 + module Internal + class MailRoom < ::API::Base + feature_category :service_desk + + before do + authenticate_gitlab_mailroom_request! + end + + helpers do + def authenticate_gitlab_mailroom_request! + unauthorized! unless Gitlab::MailRoom::Authenticator.verify_api_request(headers, params[:mailbox_type]) + end + end + + namespace 'internal' do + namespace 'mail_room' do + params do + requires :mailbox_type, type: String, + desc: 'The destination mailbox type configuration. Must either be incoming_email or service_desk_email' + end + post "/*mailbox_type" do + worker = Gitlab::MailRoom.worker_for(params[:mailbox_type]) + raw = request.body.read + begin + worker.perform_async(raw) + rescue Gitlab::SidekiqMiddleware::SizeLimiter::ExceedLimitError + receiver = Gitlab::Email::Receiver.new(raw) + reason = Gitlab::Email::FailureHandler.handle(receiver, Gitlab::Email::EmailTooLarge.new) + + status 400 + break { success: false, message: reason } + end + + status 200 + { success: true } + end + end + end + end + end +end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 4d67cbd1272..46124a74e9d 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -82,7 +82,7 @@ module API desc: 'Return issues sorted in `asc` or `desc` order.' optional :due_date, type: String, values: %w[0 overdue week month next_month_and_previous_two_weeks] << '', desc: 'Return issues that have no due date (`0`), or whose due date is this week, this month, between two weeks ago and next month, or which are overdue. Accepts: `overdue`, `week`, `month`, `next_month_and_previous_two_weeks`, `0`' - optional :issue_type, type: String, values: WorkItem::Type.allowed_types_for_issues, desc: "The type of the issue. Accepts: #{WorkItem::Type.allowed_types_for_issues.join(', ')}" + optional :issue_type, type: String, values: WorkItems::Type.allowed_types_for_issues, desc: "The type of the issue. Accepts: #{WorkItems::Type.allowed_types_for_issues.join(', ')}" use :issues_stats_params use :pagination @@ -99,7 +99,7 @@ module API optional :due_date, type: String, desc: 'Date string in the format YEAR-MONTH-DAY' optional :confidential, type: Boolean, desc: 'Boolean parameter if the issue should be confidential' optional :discussion_locked, type: Boolean, desc: " Boolean parameter indicating if the issue's discussion is locked" - optional :issue_type, type: String, values: WorkItem::Type.allowed_types_for_issues, desc: "The type of the issue. Accepts: #{WorkItem::Type.allowed_types_for_issues.join(', ')}" + optional :issue_type, type: String, values: WorkItems::Type.allowed_types_for_issues, desc: "The type of the issue. Accepts: #{WorkItems::Type.allowed_types_for_issues.join(', ')}" use :optional_issue_params_ee end diff --git a/lib/api/package_files.rb b/lib/api/package_files.rb index 79ebf18ff27..5e421da2c55 100644 --- a/lib/api/package_files.rb +++ b/lib/api/package_files.rb @@ -28,10 +28,15 @@ module API package = ::Packages::PackageFinder .new(user_project, params[:package_id]).execute - files = package.package_files - .preload_pipelines + package_files = if Feature.enabled?(:packages_installable_package_files, default_enabled: :yaml) + package.installable_package_files + else + package.package_files + end - present paginate(files), with: ::API::Entities::PackageFile + package_files = package_files.preload_pipelines + + present paginate(package_files), with: ::API::Entities::PackageFile end desc 'Remove a package file' do @@ -50,7 +55,13 @@ module API not_found! unless package - package_file = package.package_files.find_by_id(params[:package_file_id]) + package_files = if Feature.enabled?(:packages_installable_package_files, default_enabled: :yaml) + package.installable_package_files + else + package.package_files + end + + package_file = package_files.find_by_id(params[:package_file_id]) not_found! unless package_file diff --git a/lib/api/project_container_repositories.rb b/lib/api/project_container_repositories.rb index 82b6082c3fe..d4efca6e8f2 100644 --- a/lib/api/project_container_repositories.rb +++ b/lib/api/project_container_repositories.rb @@ -123,7 +123,6 @@ module API end delete ':id/registry/repositories/:repository_id/tags/:tag_name', requirements: REPOSITORY_ENDPOINT_REQUIREMENTS do authorize_destroy_container_image! - validate_tag! result = ::Projects::ContainerRepository::DeleteTagsService .new(repository.project, current_user, tags: [declared_params[:tag_name]]) diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 887c76941cf..d772079372c 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -363,6 +363,7 @@ module API optional :name, type: String, desc: 'The name that will be assigned to the fork' optional :description, type: String, desc: 'The description that will be assigned to the fork' optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the fork' + optional :mr_default_target_self, Boolean, desc: 'Merge requests of this forked project targets itself by default' end post ':id/fork', feature_category: :source_code_management do Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20759') diff --git a/lib/api/resource_access_tokens.rb b/lib/api/resource_access_tokens.rb index f42acc6b2eb..e52f8fd9111 100644 --- a/lib/api/resource_access_tokens.rb +++ b/lib/api/resource_access_tokens.rb @@ -8,7 +8,7 @@ module API feature_category :authentication_and_authorization - %w[project].each do |source_type| + %w[project group].each do |source_type| resource source_type.pluralize, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get list of all access tokens for the specified resource' do detail 'This feature was introduced in GitLab 13.9.' @@ -23,8 +23,8 @@ module API tokens = PersonalAccessTokensFinder.new({ user: resource.bots, impersonation: false }).execute.preload_users - resource.project_members.load - present paginate(tokens), with: Entities::ResourceAccessToken, project: resource + resource.members.load + present paginate(tokens), with: Entities::ResourceAccessToken, resource: resource end desc 'Revoke a resource access token' do @@ -58,7 +58,7 @@ module API requires :id, type: String, desc: "The #{source_type} ID" requires :name, type: String, desc: "Resource access token name" requires :scopes, type: Array[String], desc: "The permissions of the token" - optional :access_level, type: Integer, desc: "The access level of the token in the project" + optional :access_level, type: Integer, desc: "The access level of the token in the #{source_type}" optional :expires_at, type: Date, desc: "The expiration date of the token" end post ':id/access_tokens' do @@ -71,7 +71,7 @@ module API ).execute if token_response.success? - present token_response.payload[:access_token], with: Entities::ResourceAccessTokenWithToken, project: resource + present token_response.payload[:access_token], with: Entities::ResourceAccessTokenWithToken, resource: resource else bad_request!(token_response.message) end diff --git a/lib/api/rubygem_packages.rb b/lib/api/rubygem_packages.rb index 9ef6ec03a41..3effa370e84 100644 --- a/lib/api/rubygem_packages.rb +++ b/lib/api/rubygem_packages.rb @@ -66,9 +66,12 @@ module API get "gems/:file_name", requirements: FILE_NAME_REQUIREMENTS do authorize!(:read_package, user_project) - package_file = ::Packages::PackageFile.for_rubygem_with_file_name( - user_project, params[:file_name] - ).last! + package_files = ::Packages::PackageFile + .for_rubygem_with_file_name(user_project, params[:file_name]) + + package_files = package_files.installable if Feature.enabled?(:packages_installable_package_files, default_enabled: :yaml) + + package_file = package_files.last! track_package_event('pull_package', :rubygems, project: user_project, namespace: user_project.namespace) diff --git a/lib/api/search.rb b/lib/api/search.rb index fbdbe3476db..60a7e944b43 100644 --- a/lib/api/search.rb +++ b/lib/api/search.rb @@ -4,7 +4,11 @@ module API class Search < ::API::Base include PaginationParams - before { authenticate! } + before do + authenticate! + + check_rate_limit!(:user_email_lookup, scope: [current_user]) if search_service.params.email_lookup? + end feature_category :global_search @@ -36,7 +40,7 @@ module API }.freeze end - def search(additional_params = {}) + def search_service(additional_params = {}) search_params = { scope: params[:scope], search: params[:search], @@ -50,7 +54,11 @@ module API sort: params[:sort] }.merge(additional_params) - results = SearchService.new(current_user, search_params).search_objects(preload_method) + SearchService.new(current_user, search_params) + end + + def search(additional_params = {}) + results = search_service(additional_params).search_objects(preload_method) Gitlab::UsageDataCounters::SearchCounter.count(:all_searches) diff --git a/lib/api/terraform/modules/v1/packages.rb b/lib/api/terraform/modules/v1/packages.rb index ad5a4ae7ea6..970fdeba734 100644 --- a/lib/api/terraform/modules/v1/packages.rb +++ b/lib/api/terraform/modules/v1/packages.rb @@ -71,7 +71,11 @@ module API def package_file strong_memoize(:package_file) do - package.package_files.first + if Feature.enabled?(:packages_installable_package_files, default_enabled: :yaml) + package.installable_package_files.first + else + package.package_files.first + end end end end diff --git a/lib/api/users.rb b/lib/api/users.rb index ce0a0e9b502..eeb5244466a 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -142,11 +142,15 @@ module API get ":id", feature_category: :users do forbidden!('Not authorized!') unless current_user + if Feature.enabled?(:rate_limit_user_by_id_endpoint, type: :development) + check_rate_limit! :users_get_by_id, scope: current_user unless current_user.admin? + end + user = User.find_by(id: params[:id]) not_found!('User') unless user && can?(current_user, :read_user, user) - opts = { with: current_user&.admin? ? Entities::UserDetailsWithAdmin : Entities::User, current_user: current_user } + opts = { with: current_user.admin? ? Entities::UserDetailsWithAdmin : Entities::User, current_user: current_user } user, opts = with_custom_attributes(user, opts) present user, opts @@ -1072,7 +1076,7 @@ module API attrs = declared_params(include_missing: false) - service = ::Users::UpsertCreditCardValidationService.new(attrs).execute + service = ::Users::UpsertCreditCardValidationService.new(attrs, user).execute if service.success? present user.credit_card_validation, with: Entities::UserCreditCardValidations diff --git a/lib/api/v3/github.rb b/lib/api/v3/github.rb index d6c026963e1..c86b7785ce2 100644 --- a/lib/api/v3/github.rb +++ b/lib/api/v3/github.rb @@ -183,7 +183,9 @@ module API params do use :project_full_path end - get ':namespace/:project/pulls' do + # TODO Remove the custom Apdex SLO target `urgency: :low` when this endpoint has been optimised. + # https://gitlab.com/gitlab-org/gitlab/-/issues/337269 + get ':namespace/:project/pulls', urgency: :low do user_project = find_project_with_access(params) merge_requests = authorized_merge_requests_for_project(user_project) @@ -236,7 +238,9 @@ module API use :project_full_path use :pagination end - get ':namespace/:project/branches' do + # TODO Remove the custom Apdex SLO target `urgency: :low` when this endpoint has been optimised. + # https://gitlab.com/gitlab-org/gitlab/-/issues/337268 + get ':namespace/:project/branches', urgency: :low do user_project = find_project_with_access(params) update_project_feature_usage_for(user_project) diff --git a/lib/backup.rb b/lib/backup.rb index 91682645a9a..95b595d885a 100644 --- a/lib/backup.rb +++ b/lib/backup.rb @@ -16,19 +16,6 @@ module Backup end end - class RepositoryBackupError < Backup::Error - attr_reader :container, :backup_repos_path - - def initialize(container, backup_repos_path) - @container = container - @backup_repos_path = backup_repos_path - end - - def message - "Failed to create compressed file '#{backup_repos_path}' when trying to backup the following paths: '#{container.disk_path}'" - end - end - class DatabaseBackupError < Backup::Error attr_reader :config, :db_file_name diff --git a/lib/backup/database.rb b/lib/backup/database.rb index f07fd786b4b..a4ac404d245 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -61,7 +61,7 @@ module Backup report_success(success) progress.flush - raise Backup::Error, 'Backup failed' unless success + raise DatabaseBackupError.new(config, db_file_name) unless success end def restore diff --git a/lib/backup/files.rb b/lib/backup/files.rb index 42cfff98239..4e51dcfb79e 100644 --- a/lib/backup/files.rb +++ b/lib/backup/files.rb @@ -37,7 +37,7 @@ module Backup unless status == 0 puts output - raise Backup::Error, 'Backup failed' + raise_custom_error end tar_cmd = [tar, exclude_dirs(:tar), %W[-C #{@backup_files_dir} -cf - .]].flatten @@ -49,7 +49,7 @@ module Backup end unless pipeline_succeeded?(tar_status: status_list[0], gzip_status: status_list[1], output: output) - raise Backup::Error, "Backup operation failed: #{output}" + raise_custom_error end end @@ -143,5 +143,9 @@ module Backup end end end + + def raise_custom_error + raise FileBackupError.new(app_files_dir, backup_tarball) + end end end diff --git a/lib/backup/gitaly_backup.rb b/lib/backup/gitaly_backup.rb index b104beed39c..8ac09e94004 100644 --- a/lib/backup/gitaly_backup.rb +++ b/lib/backup/gitaly_backup.rb @@ -2,11 +2,17 @@ module Backup # Backup and restores repositories using gitaly-backup + # + # gitaly-backup can work in parallel and accepts a list of repositories + # through input pipe using a specific json format for both backup and restore class GitalyBackup - def initialize(progress, parallel: nil, parallel_storage: nil) + # @param [StringIO] progress IO interface to output progress + # @param [Integer] max_parallelism max parallelism when running backups + # @param [Integer] storage_parallelism max parallelism per storage (is affected by max_parallelism) + def initialize(progress, max_parallelism: nil, storage_parallelism: nil) @progress = progress - @parallel = parallel - @parallel_storage = parallel_storage + @max_parallelism = max_parallelism + @storage_parallelism = storage_parallelism end def start(type) @@ -22,20 +28,20 @@ module Backup end args = [] - args += ['-parallel', @parallel.to_s] if @parallel - args += ['-parallel-storage', @parallel_storage.to_s] if @parallel_storage + args += ['-parallel', @max_parallelism.to_s] if @max_parallelism + args += ['-parallel-storage', @storage_parallelism.to_s] if @storage_parallelism - @stdin, stdout, @thread = Open3.popen2(build_env, bin_path, command, '-path', backup_repos_path, *args) + @input_stream, stdout, @thread = Open3.popen2(build_env, bin_path, command, '-path', backup_repos_path, *args) @out_reader = Thread.new do IO.copy_stream(stdout, @progress) end end - def wait + def finish! return unless started? - @stdin.close + @input_stream.close [@thread, @out_reader].each(&:join) status = @thread.value @@ -49,12 +55,7 @@ module Backup repository = repo_type.repository_for(container) - @stdin.puts({ - storage_name: repository.storage, - relative_path: repository.relative_path, - gl_project_path: repository.gl_project_path, - always_create: repo_type.project? - }.merge(Gitlab::GitalyClient.connection_data(repository.storage)).to_json) + schedule_backup_job(repository, always_create: repo_type.project?) end def parallel_enqueue? @@ -63,6 +64,24 @@ module Backup private + # Schedule a new backup job through a non-blocking JSON based pipe protocol + # + # @see https://gitlab.com/gitlab-org/gitaly/-/blob/master/doc/gitaly-backup.md + def schedule_backup_job(repository, always_create:) + connection_params = Gitlab::GitalyClient.connection_data(repository.storage) + + json_job = { + address: connection_params['address'], + token: connection_params['token'], + storage_name: repository.storage, + relative_path: repository.relative_path, + gl_project_path: repository.gl_project_path, + always_create: always_create + }.to_json + + @input_stream.puts(json_job) + end + def build_env { 'SSL_CERT_FILE' => OpenSSL::X509::DEFAULT_CERT_FILE, diff --git a/lib/backup/gitaly_rpc_backup.rb b/lib/backup/gitaly_rpc_backup.rb index baac4eb26ca..bbd83cd2157 100644 --- a/lib/backup/gitaly_rpc_backup.rb +++ b/lib/backup/gitaly_rpc_backup.rb @@ -23,7 +23,7 @@ module Backup end end - def wait + def finish! @type = nil end diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 1bdc4965e5d..ed2e001cefc 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -2,7 +2,7 @@ module Backup class Manager - ARCHIVES_TO_BACKUP = %w[uploads builds artifacts pages lfs registry].freeze + ARCHIVES_TO_BACKUP = %w[uploads builds artifacts pages lfs terraform_state registry packages].freeze FOLDERS_TO_BACKUP = %w[repositories db].freeze FILE_NAME_SUFFIX = '_gitlab_backup.tar' diff --git a/lib/backup/packages.rb b/lib/backup/packages.rb new file mode 100644 index 00000000000..7b6a8f086ed --- /dev/null +++ b/lib/backup/packages.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Backup + class Packages < Backup::Files + attr_reader :progress + + def initialize(progress) + @progress = progress + + super('packages', Settings.packages.storage_path, excludes: ['tmp']) + end + end +end diff --git a/lib/backup/repositories.rb b/lib/backup/repositories.rb index 0b5a62529b4..4c39e58c87d 100644 --- a/lib/backup/repositories.rb +++ b/lib/backup/repositories.rb @@ -40,7 +40,7 @@ module Backup raise errors.pop unless errors.empty? ensure - strategy.wait + strategy.finish! end def restore @@ -48,7 +48,7 @@ module Backup enqueue_consecutive ensure - strategy.wait + strategy.finish! cleanup_snippets_without_repositories restore_object_pools diff --git a/lib/backup/terraform_state.rb b/lib/backup/terraform_state.rb new file mode 100644 index 00000000000..5f71e18f1b4 --- /dev/null +++ b/lib/backup/terraform_state.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Backup + class TerraformState < Backup::Files + attr_reader :progress + + def initialize(progress) + @progress = progress + + super('terraform_state', Settings.terraform_state.storage_path, excludes: ['tmp']) + end + end +end diff --git a/lib/banzai/filter/base_sanitization_filter.rb b/lib/banzai/filter/base_sanitization_filter.rb index 7ea32c4b1e7..4e350a59fa0 100644 --- a/lib/banzai/filter/base_sanitization_filter.rb +++ b/lib/banzai/filter/base_sanitization_filter.rb @@ -42,7 +42,7 @@ module Banzai # Allow any protocol in `a` elements # and then remove links with unsafe protocols allowlist[:protocols].delete('a') - allowlist[:transformers].push(self.class.method(:remove_unsafe_links)) + allowlist[:transformers].push(self.class.method(:sanitize_unsafe_links)) # Remove `rel` attribute from `a` elements allowlist[:transformers].push(self.class.remove_rel) diff --git a/lib/banzai/filter/footnote_filter.rb b/lib/banzai/filter/footnote_filter.rb index 00a38f02141..537b7c80d91 100644 --- a/lib/banzai/filter/footnote_filter.rb +++ b/lib/banzai/filter/footnote_filter.rb @@ -7,13 +7,14 @@ module Banzai # Footnotes are supported in CommonMark. However we were stripping # the ids during sanitization. Those are now allowed. # - # Footnotes are numbered the same - the first one has `id=fn1`, the - # second is `id=fn2`, etc. In order to allow footnotes when rendering - # multiple markdown blocks on a page, we need to make each footnote - # reference unique. - # + # Footnotes are numbered as an increasing integer starting at `1`. + # The `id` associated with a footnote is based on the footnote reference + # string. For example, `[^foot]` will generate `id="fn-foot"`. + # In order to allow footnotes when rendering multiple markdown blocks + # on a page, we need to make each footnote reference unique. + # This filter adds a random number to each footnote (the same number - # can be used for a single render). So you get `id=fn1-4335` and `id=fn2-4335`. + # can be used for a single render). So you get `id=fn-1-4335` and `id=fn-foot-4335`. # class FootnoteFilter < HTML::Pipeline::Filter FOOTNOTE_ID_PREFIX = 'fn-' @@ -26,53 +27,24 @@ module Banzai CSS_FOOTNOTE = 'sup > a[data-footnote-ref]' XPATH_FOOTNOTE = Gitlab::Utils::Nokogiri.css_to_xpath(CSS_FOOTNOTE).freeze - # only needed when feature flag use_cmark_renderer is turned off - INTEGER_PATTERN = /\A\d+\z/.freeze - FOOTNOTE_ID_PREFIX_OLD = 'fn' - FOOTNOTE_LINK_ID_PREFIX_OLD = 'fnref' - FOOTNOTE_LI_REFERENCE_PATTERN_OLD = /\A#{FOOTNOTE_ID_PREFIX_OLD}\d+\z/.freeze - FOOTNOTE_LINK_REFERENCE_PATTERN_OLD = /\A#{FOOTNOTE_LINK_ID_PREFIX_OLD}\d+\z/.freeze - FOOTNOTE_START_NUMBER = 1 - CSS_SECTION_OLD = "ol > li[id=#{FOOTNOTE_ID_PREFIX_OLD}#{FOOTNOTE_START_NUMBER}]" - XPATH_SECTION_OLD = Gitlab::Utils::Nokogiri.css_to_xpath(CSS_SECTION_OLD).freeze - def call - if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) - # Sanitization stripped off the section class - add it back in - return doc unless section_node = doc.at_xpath(XPATH_SECTION) + # Sanitization stripped off the section class - add it back in + return doc unless section_node = doc.at_xpath(XPATH_SECTION) - section_node.append_class('footnotes') - else - return doc unless first_footnote = doc.at_xpath(XPATH_SECTION_OLD) - return doc unless first_footnote.parent - - first_footnote.parent.wrap('
') - end + section_node.append_class('footnotes') rand_suffix = "-#{random_number}" modified_footnotes = {} - xpath_footnote = if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) - XPATH_FOOTNOTE - else - Gitlab::Utils::Nokogiri.css_to_xpath('sup > a[id]') - end - - doc.xpath(xpath_footnote).each do |link_node| - if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) - ref_num = link_node[:id].delete_prefix(FOOTNOTE_LINK_ID_PREFIX) - ref_num.gsub!(/[[:punct:]]/, '\\\\\&') - else - ref_num = link_node[:id].delete_prefix(FOOTNOTE_LINK_ID_PREFIX_OLD) - end + doc.xpath(XPATH_FOOTNOTE).each do |link_node| + ref_num = link_node[:id].delete_prefix(FOOTNOTE_LINK_ID_PREFIX) + ref_num.gsub!(/[[:punct:]]/, '\\\\\&') - css = Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) ? "section[data-footnotes] li[id=#{fn_id(ref_num)}]" : "li[id=#{fn_id(ref_num)}]" + css = "section[data-footnotes] li[id=#{fn_id(ref_num)}]" node_xpath = Gitlab::Utils::Nokogiri.css_to_xpath(css) footnote_node = doc.at_xpath(node_xpath) if footnote_node || modified_footnotes[ref_num] - next if Feature.disabled?(:use_cmark_renderer, default_enabled: :yaml) && !INTEGER_PATTERN.match?(ref_num) - link_node[:href] += rand_suffix link_node[:id] += rand_suffix @@ -103,13 +75,11 @@ module Banzai end def fn_id(num) - prefix = Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) ? FOOTNOTE_ID_PREFIX : FOOTNOTE_ID_PREFIX_OLD - "#{prefix}#{num}" + "#{FOOTNOTE_ID_PREFIX}#{num}" end def fnref_id(num) - prefix = Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) ? FOOTNOTE_LINK_ID_PREFIX : FOOTNOTE_LINK_ID_PREFIX_OLD - "#{prefix}#{num}" + "#{FOOTNOTE_LINK_ID_PREFIX}#{num}" end end end diff --git a/lib/banzai/filter/markdown_engines/common_mark.rb b/lib/banzai/filter/markdown_engines/common_mark.rb index dc94e3c925a..cf368e28beb 100644 --- a/lib/banzai/filter/markdown_engines/common_mark.rb +++ b/lib/banzai/filter/markdown_engines/common_mark.rb @@ -4,8 +4,8 @@ # This module is used in Banzai::Filter::MarkdownFilter. # Used gem is `commonmarker` which is a ruby wrapper for libcmark (CommonMark parser) # including GitHub's GFM extensions. +# We now utilize the renderer built in `C`, rather than the ruby based renderer. # Homepage: https://github.com/gjtorikian/commonmarker - module Banzai module Filter module MarkdownEngines @@ -22,57 +22,29 @@ module Banzai :VALIDATE_UTF8 # replace illegal sequences with the replacement character U+FFFD. ].freeze - RENDER_OPTIONS_C = [ + RENDER_OPTIONS = [ :GITHUB_PRE_LANG, # use GitHub-style 
 for fenced code blocks.
           :FOOTNOTES,        # render footnotes.
           :FULL_INFO_STRING, # include full info strings of code blocks in separate attribute.
           :UNSAFE            # allow raw/custom HTML and unsafe links.
         ].freeze
 
-        # The `:GITHUB_PRE_LANG` option is not used intentionally because
-        # it renders a fence block with language as `
some code\n
`
-        # while GitLab's syntax is `
some code\n
`.
-        # If in the future the syntax is about to be made GitHub-compatible, please, add `:GITHUB_PRE_LANG` render option below
-        # and remove `code_block` method from `lib/banzai/renderer/common_mark/html.rb`.
-        RENDER_OPTIONS_RUBY = [
-          # as of commonmarker 0.18.0, we need to use :UNSAFE to get the same as the original :DEFAULT
-          # https://github.com/gjtorikian/commonmarker/pull/81
-          :UNSAFE # allow raw/custom HTML and unsafe links.
-        ].freeze
-
         def initialize(context)
           @context = context
-          @renderer = Banzai::Renderer::CommonMark::HTML.new(options: render_options) if Feature.disabled?(:use_cmark_renderer, default_enabled: :yaml)
         end
 
         def render(text)
-          if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml)
-            CommonMarker.render_html(text, render_options, extensions)
-          else
-            doc = CommonMarker.render_doc(text, PARSE_OPTIONS, extensions)
-
-            @renderer.render(doc)
-          end
+          CommonMarker.render_html(text, render_options, EXTENSIONS)
         end
 
         private
 
-        def extensions
-          if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml)
-            EXTENSIONS
-          else
-            EXTENSIONS + [
-              :tagfilter # strips out several "unsafe" HTML tags from being used: https://github.github.com/gfm/#disallowed-raw-html-extension-
-            ].freeze
-          end
-        end
-
         def render_options
           @context[:no_sourcepos] ? render_options_no_sourcepos : render_options_sourcepos
         end
 
         def render_options_no_sourcepos
-          Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) ? RENDER_OPTIONS_C : RENDER_OPTIONS_RUBY
+          RENDER_OPTIONS
         end
 
         def render_options_sourcepos
diff --git a/lib/banzai/filter/markdown_post_escape_filter.rb b/lib/banzai/filter/markdown_post_escape_filter.rb
index b979b7573ae..09ae09a22ae 100644
--- a/lib/banzai/filter/markdown_post_escape_filter.rb
+++ b/lib/banzai/filter/markdown_post_escape_filter.rb
@@ -8,8 +8,10 @@ module Banzai
       NOT_LITERAL_REGEX = %r{#{LITERAL_KEYWORD}-((%5C|\\).+?)-#{LITERAL_KEYWORD}}.freeze
       SPAN_REGEX        = %r{(.*?)}.freeze
 
-      CSS_A   = 'a'
-      XPATH_A = Gitlab::Utils::Nokogiri.css_to_xpath(CSS_A).freeze
+      CSS_A          = 'a'
+      XPATH_A        = Gitlab::Utils::Nokogiri.css_to_xpath(CSS_A).freeze
+      CSS_LANG_TAG   = 'pre'
+      XPATH_LANG_TAG = Gitlab::Utils::Nokogiri.css_to_xpath(CSS_LANG_TAG).freeze
 
       def call
         return doc unless result[:escaped_literals]
@@ -32,22 +34,12 @@ module Banzai
           node.attributes['title'].value = node.attributes['title'].value.gsub(SPAN_REGEX, '\1') if node.attributes['title']
         end
 
-        doc.xpath(lang_tag).each do |node|
+        doc.xpath(XPATH_LANG_TAG).each do |node|
           node.attributes['lang'].value  = node.attributes['lang'].value.gsub(SPAN_REGEX, '\1') if node.attributes['lang']
         end
 
         doc
       end
-
-      private
-
-      def lang_tag
-        if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml)
-          Gitlab::Utils::Nokogiri.css_to_xpath('pre')
-        else
-          Gitlab::Utils::Nokogiri.css_to_xpath('code')
-        end
-      end
     end
   end
 end
diff --git a/lib/banzai/filter/plantuml_filter.rb b/lib/banzai/filter/plantuml_filter.rb
index 3f160960d23..68a99702d6f 100644
--- a/lib/banzai/filter/plantuml_filter.rb
+++ b/lib/banzai/filter/plantuml_filter.rb
@@ -25,12 +25,7 @@ module Banzai
       private
 
       def lang_tag
-        @lang_tag ||=
-          if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml)
-            Gitlab::Utils::Nokogiri.css_to_xpath('pre[lang="plantuml"] > code').freeze
-          else
-            Gitlab::Utils::Nokogiri.css_to_xpath('pre > code[lang="plantuml"]').freeze
-          end
+        @lang_tag ||= Gitlab::Utils::Nokogiri.css_to_xpath('pre[lang="plantuml"] > code').freeze
       end
 
       def settings
diff --git a/lib/banzai/filter/references/abstract_reference_filter.rb b/lib/banzai/filter/references/abstract_reference_filter.rb
index 7a23326bafa..a34519799d5 100644
--- a/lib/banzai/filter/references/abstract_reference_filter.rb
+++ b/lib/banzai/filter/references/abstract_reference_filter.rb
@@ -216,6 +216,8 @@ module Banzai
                   url_for_object_cached(object, parent)
                 end
 
+              url.chomp!(matches[:format]) if matches.names.include?("format")
+
               content = link_content || object_link_text(object, matches)
 
               link = %(')
-            out(escape_html(node.string_content))
-            out('
') - end - end - end - end - end -end diff --git a/lib/bulk_imports/common/extractors/ndjson_extractor.rb b/lib/bulk_imports/common/extractors/ndjson_extractor.rb index ecd7c08bd25..04febebff8e 100644 --- a/lib/bulk_imports/common/extractors/ndjson_extractor.rb +++ b/lib/bulk_imports/common/extractors/ndjson_extractor.rb @@ -4,49 +4,47 @@ module BulkImports module Common module Extractors class NdjsonExtractor - include Gitlab::ImportExport::CommandLineUtil - include Gitlab::Utils::StrongMemoize - def initialize(relation:) @relation = relation - @tmp_dir = Dir.mktmpdir + @tmpdir = Dir.mktmpdir end def extract(context) - download_service(tmp_dir, context).execute - decompression_service(tmp_dir).execute - relations = ndjson_reader(tmp_dir).consume_relation('', relation) + download_service(context).execute + decompression_service.execute + + records = ndjson_reader.consume_relation('', relation) - BulkImports::Pipeline::ExtractedData.new(data: relations) + BulkImports::Pipeline::ExtractedData.new(data: records) end - def remove_tmp_dir - FileUtils.remove_entry(tmp_dir) + def remove_tmpdir + FileUtils.remove_entry(tmpdir) if Dir.exist?(tmpdir) end private - attr_reader :relation, :tmp_dir + attr_reader :relation, :tmpdir def filename - @filename ||= "#{relation}.ndjson.gz" + "#{relation}.ndjson.gz" end - def download_service(tmp_dir, context) + def download_service(context) @download_service ||= BulkImports::FileDownloadService.new( configuration: context.configuration, relative_url: context.entity.relation_download_url_path(relation), - dir: tmp_dir, + tmpdir: tmpdir, filename: filename ) end - def decompression_service(tmp_dir) - @decompression_service ||= BulkImports::FileDecompressionService.new(dir: tmp_dir, filename: filename) + def decompression_service + @decompression_service ||= BulkImports::FileDecompressionService.new(tmpdir: tmpdir, filename: filename) end - def ndjson_reader(tmp_dir) - @ndjson_reader ||= Gitlab::ImportExport::Json::NdjsonReader.new(tmp_dir) + def ndjson_reader + @ndjson_reader ||= Gitlab::ImportExport::Json::NdjsonReader.new(tmpdir) end end end diff --git a/lib/bulk_imports/common/pipelines/uploads_pipeline.rb b/lib/bulk_imports/common/pipelines/uploads_pipeline.rb index 2ac4e533c1d..d7b9d6920ea 100644 --- a/lib/bulk_imports/common/pipelines/uploads_pipeline.rb +++ b/lib/bulk_imports/common/pipelines/uploads_pipeline.rb @@ -15,7 +15,7 @@ module BulkImports decompression_service.execute extraction_service.execute - upload_file_paths = Dir.glob(File.join(tmp_dir, '**', '*')) + upload_file_paths = Dir.glob(File.join(tmpdir, '**', '*')) BulkImports::Pipeline::ExtractedData.new(data: upload_file_paths) end @@ -37,7 +37,7 @@ module BulkImports end def after_run(_) - FileUtils.remove_entry(tmp_dir) if Dir.exist?(tmp_dir) + FileUtils.remove_entry(tmpdir) if Dir.exist?(tmpdir) end private @@ -46,17 +46,17 @@ module BulkImports BulkImports::FileDownloadService.new( configuration: context.configuration, relative_url: context.entity.relation_download_url_path(relation), - dir: tmp_dir, + tmpdir: tmpdir, filename: targz_filename ) end def decompression_service - BulkImports::FileDecompressionService.new(dir: tmp_dir, filename: targz_filename) + BulkImports::FileDecompressionService.new(tmpdir: tmpdir, filename: targz_filename) end def extraction_service - BulkImports::ArchiveExtractionService.new(tmpdir: tmp_dir, filename: tar_filename) + BulkImports::ArchiveExtractionService.new(tmpdir: tmpdir, filename: tar_filename) end def relation @@ -71,8 +71,8 @@ module BulkImports "#{tar_filename}.gz" end - def tmp_dir - @tmp_dir ||= Dir.mktmpdir('bulk_imports') + def tmpdir + @tmpdir ||= Dir.mktmpdir('bulk_imports') end def file_uploader diff --git a/lib/bulk_imports/ndjson_pipeline.rb b/lib/bulk_imports/ndjson_pipeline.rb index d5475a8b324..d85e51984df 100644 --- a/lib/bulk_imports/ndjson_pipeline.rb +++ b/lib/bulk_imports/ndjson_pipeline.rb @@ -68,7 +68,7 @@ module BulkImports end def after_run(_) - extractor.remove_tmp_dir if extractor.respond_to?(:remove_tmp_dir) + extractor.remove_tmpdir if extractor.respond_to?(:remove_tmpdir) end def relation_class(relation_key) diff --git a/lib/bulk_imports/projects/pipelines/project_attributes_pipeline.rb b/lib/bulk_imports/projects/pipelines/project_attributes_pipeline.rb index 4d742225ff7..2492a023cbe 100644 --- a/lib/bulk_imports/projects/pipelines/project_attributes_pipeline.rb +++ b/lib/bulk_imports/projects/pipelines/project_attributes_pipeline.rb @@ -8,15 +8,16 @@ module BulkImports transformer ::BulkImports::Common::Transformers::ProhibitedAttributesTransformer - def extract(context) - download_service(tmp_dir, context).execute - decompression_service(tmp_dir).execute + def extract(_context) + download_service.execute + decompression_service.execute + project_attributes = json_decode(json_attributes) BulkImports::Pipeline::ExtractedData.new(data: project_attributes) end - def transform(_, data) + def transform(_context, data) subrelations = config.portable_relations_tree.keys.map(&:to_s) Gitlab::ImportExport::AttributeCleaner.clean( @@ -26,42 +27,42 @@ module BulkImports ).except(*subrelations) end - def load(_, data) + def load(_context, data) portable.assign_attributes(data) portable.reconcile_shared_runners_setting! portable.drop_visibility_level! portable.save! end - def after_run(_) - FileUtils.remove_entry(tmp_dir) + def after_run(_context) + FileUtils.remove_entry(tmpdir) if Dir.exist?(tmpdir) end def json_attributes - @json_attributes ||= File.read(File.join(tmp_dir, filename)) + @json_attributes ||= File.read(File.join(tmpdir, filename)) end private - def tmp_dir - @tmp_dir ||= Dir.mktmpdir + def tmpdir + @tmpdir ||= Dir.mktmpdir('bulk_imports') end def config @config ||= BulkImports::FileTransfer.config_for(portable) end - def download_service(tmp_dir, context) + def download_service @download_service ||= BulkImports::FileDownloadService.new( configuration: context.configuration, - relative_url: context.entity.relation_download_url_path(BulkImports::FileTransfer::BaseConfig::SELF_RELATION), - dir: tmp_dir, + relative_url: context.entity.relation_download_url_path(BulkImports::FileTransfer::BaseConfig::SELF_RELATION), + tmpdir: tmpdir, filename: compressed_filename ) end - def decompression_service(tmp_dir) - @decompression_service ||= BulkImports::FileDecompressionService.new(dir: tmp_dir, filename: compressed_filename) + def decompression_service + @decompression_service ||= BulkImports::FileDecompressionService.new(tmpdir: tmpdir, filename: compressed_filename) end def compressed_filename diff --git a/lib/feature.rb b/lib/feature.rb index f301f206b46..12b4ef07dd6 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -29,6 +29,15 @@ class Feature class << self delegate :group, to: :flipper + def feature_flags_available? + # When the DBMS is not available, an exception (e.g. PG::ConnectionBad) is raised + active_db_connection = ActiveRecord::Base.connection.active? rescue false # rubocop:disable Database/MultipleDatabases + + active_db_connection && Feature::FlipperFeature.table_exists? + rescue ActiveRecord::NoDatabaseError + false + end + def all flipper.features.to_a end diff --git a/lib/gitlab.rb b/lib/gitlab.rb index d93d7acbaad..2449554d3c0 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -1,10 +1,15 @@ # frozen_string_literal: true require 'pathname' +require 'forwardable' + +require_relative 'gitlab_edition' module Gitlab - def self.root - Pathname.new(File.expand_path('..', __dir__)) + class << self + extend Forwardable + + def_delegators :GitlabEdition, :root, :extensions, :ee?, :ee, :jh?, :jh end def self.version_info @@ -89,47 +94,6 @@ module Gitlab Rails.env.development? || Rails.env.test? end - def self.extensions - if jh? - %w[ee jh] - elsif ee? - %w[ee] - else - %w[] - end - end - - def self.ee? - @is_ee ||= - # We use this method when the Rails environment is not loaded. This - # means that checking the presence of the License class could result in - # this method returning `false`, even for an EE installation. - # - # The `FOSS_ONLY` is always `string` or `nil` - # Thus the nil or empty string will result - # in using default value: false - # - # The behavior needs to be synchronised with - # config/helpers/is_ee_env.js - root.join('ee/app/models/license.rb').exist? && - !%w[true 1].include?(ENV['FOSS_ONLY'].to_s) - end - - def self.jh? - @is_jh ||= - ee? && - root.join('jh').exist? && - !%w[true 1].include?(ENV['EE_ONLY'].to_s) - end - - def self.ee - yield if ee? - end - - def self.jh - yield if jh? - end - def self.http_proxy_env? HTTP_PROXY_ENV_VARS.any? { |name| ENV[name] } end diff --git a/lib/gitlab/anonymous_session.rb b/lib/gitlab/anonymous_session.rb index e58240e16b4..6904945a755 100644 --- a/lib/gitlab/anonymous_session.rb +++ b/lib/gitlab/anonymous_session.rb @@ -2,14 +2,12 @@ module Gitlab class AnonymousSession - include ::Gitlab::Redis::SessionsStoreHelper - def initialize(remote_ip) @remote_ip = remote_ip end def count_session_ip - redis_store_class.with do |redis| + Gitlab::Redis::Sessions.with do |redis| redis.pipelined do |pipeline| pipeline.incr(session_lookup_name) pipeline.expire(session_lookup_name, 24.hours) @@ -18,13 +16,13 @@ module Gitlab end def session_count - redis_store_class.with do |redis| + Gitlab::Redis::Sessions.with do |redis| redis.get(session_lookup_name).to_i end end def cleanup_session_per_ip_count - redis_store_class.with do |redis| + Gitlab::Redis::Sessions.with do |redis| redis.del(session_lookup_name) end end diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index fb90ad9e275..12f1b15f820 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -49,9 +49,15 @@ module Gitlab group_testing_hook: { threshold: 5, interval: 1.minute }, profile_add_new_email: { threshold: 5, interval: 1.minute }, web_hook_calls: { interval: 1.minute }, + users_get_by_id: { threshold: 10, interval: 1.minute }, + username_exists: { threshold: 20, interval: 1.minute }, + user_sign_up: { threshold: 20, interval: 1.minute }, profile_resend_email_confirmation: { threshold: 5, interval: 1.minute }, + profile_update_username: { threshold: 10, interval: 1.minute }, update_environment_canary_ingress: { threshold: 1, interval: 1.minute }, - auto_rollback_deployment: { threshold: 1, interval: 3.minutes } + auto_rollback_deployment: { threshold: 1, interval: 3.minutes }, + user_email_lookup: { threshold: -> { application_settings.user_email_lookup_limit }, interval: 1.minute }, + gitlab_shell_operation: { threshold: 600, interval: 1.minute } }.freeze end @@ -59,7 +65,7 @@ module Gitlab # be throttled. # # @param key [Symbol] Key attribute registered in `.rate_limits` - # @param scope [Array] Array of ActiveRecord models to scope throttling to a specific request (e.g. per user per project) + # @param scope [Array] Array of ActiveRecord models, Strings or Symbols to scope throttling to a specific request (e.g. per user per project) # @param threshold [Integer] Optional threshold value to override default one registered in `.rate_limits` # @param users_allowlist [Array] Optional list of usernames to exclude from the limit. This param will only be functional if Scope includes a current user. # @param peek [Boolean] Optional. When true the key will not be incremented but the current throttled state will be returned. diff --git a/lib/gitlab/asciidoc/syntax_highlighter/html_pipeline_adapter.rb b/lib/gitlab/asciidoc/syntax_highlighter/html_pipeline_adapter.rb index 3ada3f947ee..b2e1c9e2379 100644 --- a/lib/gitlab/asciidoc/syntax_highlighter/html_pipeline_adapter.rb +++ b/lib/gitlab/asciidoc/syntax_highlighter/html_pipeline_adapter.rb @@ -7,11 +7,7 @@ module Gitlab register_for 'gitlab-html-pipeline' def format(node, lang, opts) - if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) - %(
#{node.content}
) - else - %(
#{node.content}
) - end + %(
#{node.content}
) end end end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 3e982168339..38bc50a2cb8 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -84,7 +84,7 @@ module Gitlab Gitlab::Auth::UniqueIpsLimiter.limit_user! do user = User.by_login(login) - break if user && !can_user_login_with_non_expired_password?(user) + break if user && !user.can_log_in_with_non_expired_password? authenticators = [] @@ -187,7 +187,7 @@ module Gitlab if valid_oauth_token?(token) user = User.id_in(token.resource_owner_id).first - return unless user && can_user_login_with_non_expired_password?(user) + return unless user && user.can_log_in_with_non_expired_password? Gitlab::Auth::Result.new(user, nil, :oauth, abilities_for_scopes(token.scopes)) end @@ -210,7 +210,7 @@ module Gitlab return unless token_bot_in_project?(token.user, project) || token_bot_in_group?(token.user, project) end - if can_user_login_with_non_expired_password?(token.user) || token.user.project_bot? + if token.user.can_log_in_with_non_expired_password? || token.user.project_bot? Gitlab::Auth::Result.new(token.user, nil, :personal_access_token, abilities_for_scopes(token.scopes)) end end @@ -309,7 +309,7 @@ module Gitlab return unless build.project.builds_enabled? if build.user - return unless can_user_login_with_non_expired_password?(build.user) || (build.user.project_bot? && build.project.bots&.include?(build.user)) + return unless build.user.can_log_in_with_non_expired_password? || (build.user.project_bot? && build.project.bots&.include?(build.user)) # If user is assigned to build, use restricted credentials of user Gitlab::Auth::Result.new(build.user, build.project, :build, build_authentication_abilities) @@ -406,10 +406,6 @@ module Gitlab user.increment_failed_attempts! end - - def can_user_login_with_non_expired_password?(user) - user.can?(:log_in) && !user.password_expired_if_applicable? - end end end end diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb index 9c33a5fc872..ecda96af403 100644 --- a/lib/gitlab/auth/auth_finders.rb +++ b/lib/gitlab/auth/auth_finders.rb @@ -165,7 +165,7 @@ module Gitlab authorization_token, _options = token_and_options(current_request) - ::Clusters::AgentToken.find_by_token(authorization_token) + ::Clusters::AgentToken.active.find_by_token(authorization_token) end def find_runner_from_token diff --git a/lib/gitlab/auth/ldap/config.rb b/lib/gitlab/auth/ldap/config.rb index 7bfe776fed0..82c6411c712 100644 --- a/lib/gitlab/auth/ldap/config.rb +++ b/lib/gitlab/auth/ldap/config.rb @@ -206,7 +206,8 @@ module Gitlab def base_options { host: options['host'], - port: options['port'] + port: options['port'], + hosts: options['hosts'] } end diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index feb5fea4c85..9f142727ebb 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -230,8 +230,8 @@ module Gitlab name: name.strip.presence || valid_username, username: valid_username, email: email, - password: auth_hash.password, - password_confirmation: auth_hash.password, + password: Gitlab::Password.test_default(21), + password_confirmation: Gitlab::Password.test_default(21), password_automatically_set: true } end diff --git a/lib/gitlab/background_migration/backfill_ci_namespace_mirrors.rb b/lib/gitlab/background_migration/backfill_ci_namespace_mirrors.rb new file mode 100644 index 00000000000..2247747ba08 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_ci_namespace_mirrors.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job to create ci_namespace_mirrors entries in batches + class BackfillCiNamespaceMirrors + class Namespace < ActiveRecord::Base # rubocop:disable Style/Documentation + include ::EachBatch + + self.table_name = 'namespaces' + self.inheritance_column = nil + + scope :base_query, -> do + select(:id, :parent_id) + end + end + + PAUSE_SECONDS = 0.1 + SUB_BATCH_SIZE = 500 + + def perform(start_id, end_id) + batch_query = Namespace.base_query.where(id: start_id..end_id) + batch_query.each_batch(of: SUB_BATCH_SIZE) do |sub_batch| + first, last = sub_batch.pluck(Arel.sql('MIN(id), MAX(id)')).first + ranged_query = Namespace.unscoped.base_query.where(id: first..last) + + update_sql = <<~SQL + INSERT INTO ci_namespace_mirrors (namespace_id, traversal_ids) + #{insert_values(ranged_query)} + ON CONFLICT (namespace_id) DO NOTHING + SQL + # We do nothing on conflict because we consider they were already filled. + + Namespace.connection.execute(update_sql) + + sleep PAUSE_SECONDS + end + + mark_job_as_succeeded(start_id, end_id) + end + + private + + def insert_values(batch) + calculated_traversal_ids( + batch.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336433') + ) + end + + # Copied from lib/gitlab/background_migration/backfill_namespace_traversal_ids_children.rb + def calculated_traversal_ids(batch) + <<~SQL + WITH RECURSIVE cte(source_id, namespace_id, parent_id, height) AS ( + ( + SELECT batch.id, batch.id, batch.parent_id, 1 + FROM (#{batch.to_sql}) AS batch + ) + UNION ALL + ( + SELECT cte.source_id, n.id, n.parent_id, cte.height+1 + FROM namespaces n, cte + WHERE n.id = cte.parent_id + ) + ) + SELECT flat_hierarchy.source_id as namespace_id, + array_agg(flat_hierarchy.namespace_id ORDER BY flat_hierarchy.height DESC) as traversal_ids + FROM (SELECT * FROM cte FOR UPDATE) flat_hierarchy + GROUP BY flat_hierarchy.source_id + SQL + end + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded('BackfillCiNamespaceMirrors', arguments) + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_ci_project_mirrors.rb b/lib/gitlab/background_migration/backfill_ci_project_mirrors.rb new file mode 100644 index 00000000000..ff6ab9928b0 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_ci_project_mirrors.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job to create ci_project_mirrors entries in batches + class BackfillCiProjectMirrors + class Project < ActiveRecord::Base # rubocop:disable Style/Documentation + include ::EachBatch + + self.table_name = 'projects' + + scope :base_query, -> do + select(:id, :namespace_id) + end + end + + PAUSE_SECONDS = 0.1 + SUB_BATCH_SIZE = 500 + + def perform(start_id, end_id) + batch_query = Project.base_query.where(id: start_id..end_id) + batch_query.each_batch(of: SUB_BATCH_SIZE) do |sub_batch| + first, last = sub_batch.pluck(Arel.sql('MIN(id), MAX(id)')).first + ranged_query = Project.unscoped.base_query.where(id: first..last) + + update_sql = <<~SQL + INSERT INTO ci_project_mirrors (project_id, namespace_id) + #{insert_values(ranged_query)} + ON CONFLICT (project_id) DO NOTHING + SQL + # We do nothing on conflict because we consider they were already filled. + + Project.connection.execute(update_sql) + + sleep PAUSE_SECONDS + end + + mark_job_as_succeeded(start_id, end_id) + end + + private + + def insert_values(batch) + batch.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336433').to_sql + end + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded('BackfillCiProjectMirrors', arguments) + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_incident_issue_escalation_statuses.rb b/lib/gitlab/background_migration/backfill_incident_issue_escalation_statuses.rb new file mode 100644 index 00000000000..2d46ff6b933 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_incident_issue_escalation_statuses.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # BackfillIncidentIssueEscalationStatuses adds + # IncidentManagement::IssuableEscalationStatus records for existing Incident issues. + # They will be added with no policy, and escalations_started_at as nil. + class BackfillIncidentIssueEscalationStatuses + def perform(start_id, stop_id) + ActiveRecord::Base.connection.execute <<~SQL + INSERT INTO incident_management_issuable_escalation_statuses (issue_id, created_at, updated_at) + SELECT issues.id, current_timestamp, current_timestamp + FROM issues + WHERE issues.issue_type = 1 + AND issues.id BETWEEN #{start_id} AND #{stop_id} + ON CONFLICT (issue_id) DO NOTHING; + SQL + + mark_job_as_succeeded(start_id, stop_id) + end + + private + + def mark_job_as_succeeded(*arguments) + ::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + self.class.name.demodulize, + arguments + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/base_job.rb b/lib/gitlab/background_migration/base_job.rb new file mode 100644 index 00000000000..e21e7e0e4a3 --- /dev/null +++ b/lib/gitlab/background_migration/base_job.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Simple base class for background migration job classes which are executed through the sidekiq queue. + # + # Any job class that inherits from the base class will have connection to the tracking database set on + # initialization. + class BaseJob + def initialize(connection:) + @connection = connection + end + + def perform(*arguments) + raise NotImplementedError, "subclasses of #{self.class.name} must implement #{__method__}" + end + + private + + attr_reader :connection + end + end +end diff --git a/lib/gitlab/background_migration/cleanup_concurrent_rename.rb b/lib/gitlab/background_migration/cleanup_concurrent_rename.rb deleted file mode 100644 index d3f366f3480..00000000000 --- a/lib/gitlab/background_migration/cleanup_concurrent_rename.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Background migration for cleaning up a concurrent column rename. - class CleanupConcurrentRename < CleanupConcurrentSchemaChange - RESCHEDULE_DELAY = 10.minutes - - def cleanup_concurrent_schema_change(table, old_column, new_column) - cleanup_concurrent_column_rename(table, old_column, new_column) - end - end - end -end diff --git a/lib/gitlab/background_migration/cleanup_concurrent_schema_change.rb b/lib/gitlab/background_migration/cleanup_concurrent_schema_change.rb deleted file mode 100644 index 91b50c1a493..00000000000 --- a/lib/gitlab/background_migration/cleanup_concurrent_schema_change.rb +++ /dev/null @@ -1,56 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Base class for background migration for rename/type changes. - class CleanupConcurrentSchemaChange - include Database::MigrationHelpers - - # table - The name of the table the migration is performed for. - # old_column - The name of the old (to drop) column. - # new_column - The name of the new column. - def perform(table, old_column, new_column) - return unless column_exists?(table, new_column) && column_exists?(table, old_column) - - rows_to_migrate = define_model_for(table) - .where(new_column => nil) - .where - .not(old_column => nil) - - if rows_to_migrate.any? - BackgroundMigrationWorker.perform_in( - RESCHEDULE_DELAY, - self.class.name, - [table, old_column, new_column] - ) - else - cleanup_concurrent_schema_change(table, old_column, new_column) - end - end - - def cleanup_concurrent_schema_change(_table, _old_column, _new_column) - raise NotImplementedError - end - - # These methods are necessary so we can re-use the migration helpers in - # this class. - def connection - ActiveRecord::Base.connection - end - - def method_missing(name, *args, &block) - connection.__send__(name, *args, &block) # rubocop: disable GitlabSecurity/PublicSend - end - - def respond_to_missing?(*args) - connection.respond_to?(*args) || super - end - - def define_model_for(table) - Class.new(ActiveRecord::Base) do - self.table_name = table - end - end - end - end -end diff --git a/lib/gitlab/background_migration/cleanup_concurrent_type_change.rb b/lib/gitlab/background_migration/cleanup_concurrent_type_change.rb deleted file mode 100644 index 48411095dbb..00000000000 --- a/lib/gitlab/background_migration/cleanup_concurrent_type_change.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Background migration for cleaning up a concurrent column type changeb. - class CleanupConcurrentTypeChange < CleanupConcurrentSchemaChange - RESCHEDULE_DELAY = 10.minutes - - def cleanup_concurrent_schema_change(table, old_column, new_column) - cleanup_concurrent_column_type_change(table, old_column) - end - end - end -end diff --git a/lib/gitlab/background_migration/copy_column.rb b/lib/gitlab/background_migration/copy_column.rb deleted file mode 100644 index ef70f37d5eb..00000000000 --- a/lib/gitlab/background_migration/copy_column.rb +++ /dev/null @@ -1,41 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # CopyColumn is a simple (reusable) background migration that can be used to - # update the value of a column based on the value of another column in the - # same table. - # - # For this background migration to work the table that is migrated _has_ to - # have an `id` column as the primary key. - class CopyColumn - # table - The name of the table that contains the columns. - # copy_from - The column containing the data to copy. - # copy_to - The column to copy the data to. - # start_id - The start ID of the range of rows to update. - # end_id - The end ID of the range of rows to update. - def perform(table, copy_from, copy_to, start_id, end_id) - return unless connection.column_exists?(table, copy_to) - - quoted_table = connection.quote_table_name(table) - quoted_copy_from = connection.quote_column_name(copy_from) - quoted_copy_to = connection.quote_column_name(copy_to) - - # We're using raw SQL here since this job may be frequently executed. As - # a result dynamically defining models would lead to many unnecessary - # schema information queries. - connection.execute <<-SQL.strip_heredoc - UPDATE #{quoted_table} - SET #{quoted_copy_to} = #{quoted_copy_from} - WHERE id BETWEEN #{start_id} AND #{end_id} - AND #{quoted_copy_from} IS NOT NULL - AND #{quoted_copy_to} IS NULL - SQL - end - - def connection - ActiveRecord::Base.connection - end - end - end -end diff --git a/lib/gitlab/background_migration/encrypt_static_object_token.rb b/lib/gitlab/background_migration/encrypt_static_object_token.rb new file mode 100644 index 00000000000..80931353e2f --- /dev/null +++ b/lib/gitlab/background_migration/encrypt_static_object_token.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Populates "static_object_token_encrypted" field with encrypted versions + # of values from "static_object_token" field + class EncryptStaticObjectToken + # rubocop:disable Style/Documentation + class User < ActiveRecord::Base + include ::EachBatch + self.table_name = 'users' + scope :with_static_object_token, -> { where.not(static_object_token: nil) } + scope :without_static_object_token_encrypted, -> { where(static_object_token_encrypted: nil) } + end + # rubocop:enable Style/Documentation + + BATCH_SIZE = 100 + + def perform(start_id, end_id) + ranged_query = User + .where(id: start_id..end_id) + .with_static_object_token + .without_static_object_token_encrypted + + ranged_query.each_batch(of: BATCH_SIZE) do |sub_batch| + first, last = sub_batch.pluck(Arel.sql('min(id), max(id)')).first + + batch_query = User.unscoped + .where(id: first..last) + .with_static_object_token + .without_static_object_token_encrypted + + user_tokens = batch_query.pluck(:id, :static_object_token) + + user_encrypted_tokens = user_tokens.map do |(id, plaintext_token)| + next if plaintext_token.blank? + + [id, Gitlab::CryptoHelper.aes256_gcm_encrypt(plaintext_token)] + end + + encrypted_tokens_sql = user_encrypted_tokens.compact.map { |(id, token)| "(#{id}, '#{token}')" }.join(',') + + if user_encrypted_tokens.present? + User.connection.execute(<<~SQL) + WITH cte(cte_id, cte_token) AS #{::Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( + SELECT * + FROM (VALUES #{encrypted_tokens_sql}) AS t (id, token) + ) + UPDATE #{User.table_name} + SET static_object_token_encrypted = cte_token + FROM cte + WHERE cte_id = id + SQL + end + + mark_job_as_succeeded(start_id, end_id) + end + end + + private + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + self.class.name.demodulize, + arguments + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb b/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb new file mode 100644 index 00000000000..2b049ea2d2f --- /dev/null +++ b/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb @@ -0,0 +1,124 @@ +# frozen_string_literal: true + +require 'parser/ruby27' + +module Gitlab + module BackgroundMigration + # This migration fixes raw_metadata entries which have incorrectly been passed a Ruby Hash instead of JSON data. + class FixVulnerabilityOccurrencesWithHashesAsRawMetadata + CLUSTER_IMAGE_SCANNING_REPORT_TYPE = 7 + GENERIC_REPORT_TYPE = 99 + + # Type error is used to handle unexpected types when parsing stringified hashes. + class TypeError < ::StandardError + attr_reader :message, :type + + def initialize(message, type) + @message = message + @type = type + end + end + + # Migration model namespace isolated from application code. + class Finding < ActiveRecord::Base + include EachBatch + + self.table_name = 'vulnerability_occurrences' + + scope :by_api_report_types, -> { where(report_type: [CLUSTER_IMAGE_SCANNING_REPORT_TYPE, GENERIC_REPORT_TYPE]) } + end + + def perform(start_id, end_id) + Finding.by_api_report_types.where(id: start_id..end_id).each do |finding| + next if valid_json?(finding.raw_metadata) + + metadata = hash_from_s(finding.raw_metadata) + + finding.update(raw_metadata: metadata.to_json) if metadata + end + mark_job_as_succeeded(start_id, end_id) + end + + def hash_from_s(str_hash) + ast = Parser::Ruby27.parse(str_hash) + + unless ast.type == :hash + ::Gitlab::AppLogger.error(message: "expected raw_metadata to be a hash", type: ast.type) + return + end + + parse_hash(ast) + rescue Parser::SyntaxError => e + ::Gitlab::AppLogger.error(message: "error parsing raw_metadata", error: e.message) + nil + rescue TypeError => e + ::Gitlab::AppLogger.error(message: "error parsing raw_metadata", error: e.message, type: e.type) + nil + end + + private + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + 'FixVulnerabilityOccurrencesWithHashesAsRawMetadata', + arguments + ) + end + + def valid_json?(metadata) + Oj.load(metadata) + true + rescue Oj::ParseError, Encoding::UndefinedConversionError + false + end + + def parse_hash(hash) + out = {} + hash.children.each do |node| + unless node.type == :pair + raise TypeError.new("expected child of hash to be a `pair`", node.type) + end + + key, value = node.children + + key = parse_key(key) + value = parse_value(value) + + out[key] = value + end + + out + end + + def parse_key(key) + case key.type + when :sym, :str, :int + key.children.first + else + raise TypeError.new("expected key to be either symbol, string, or integer", key.type) + end + end + + def parse_value(value) + case value.type + when :sym, :str, :int + value.children.first + # rubocop:disable Lint/BooleanSymbol + when :true + true + when :false + false + # rubocop:enable Lint/BooleanSymbol + when :nil + nil + when :array + value.children.map { |c| parse_value(c) } + when :hash + parse_hash(value) + else + raise TypeError.new("value of a pair was an unexpected type", value.type) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/job_coordinator.rb b/lib/gitlab/background_migration/job_coordinator.rb index cfbe7167677..5dc77f935e3 100644 --- a/lib/gitlab/background_migration/job_coordinator.rb +++ b/lib/gitlab/background_migration/job_coordinator.rb @@ -36,6 +36,8 @@ module Gitlab attr_reader :worker_class + delegate :minimum_interval, :perform_in, to: :worker_class + def queue @queue ||= worker_class.sidekiq_options['queue'] end @@ -79,7 +81,7 @@ module Gitlab def perform(class_name, arguments) with_shared_connection do - migration_class_for(class_name).new.perform(*arguments) + migration_instance_for(class_name).perform(*arguments) end end @@ -113,6 +115,16 @@ module Gitlab enqueued_job?([retry_set], migration_class) end + def migration_instance_for(class_name) + migration_class = migration_class_for(class_name) + + if migration_class < Gitlab::BackgroundMigration::BaseJob + migration_class.new(connection: connection) + else + migration_class.new + end + end + def migration_class_for(class_name) Gitlab::BackgroundMigration.const_get(class_name, false) end diff --git a/lib/gitlab/background_migration/migrate_legacy_artifacts.rb b/lib/gitlab/background_migration/migrate_legacy_artifacts.rb deleted file mode 100644 index 23d99274232..00000000000 --- a/lib/gitlab/background_migration/migrate_legacy_artifacts.rb +++ /dev/null @@ -1,130 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Metrics/ClassLength - -module Gitlab - module BackgroundMigration - ## - # The class to migrate job artifacts from `ci_builds` to `ci_job_artifacts` - class MigrateLegacyArtifacts - FILE_LOCAL_STORE = 1 # equal to ObjectStorage::Store::LOCAL - ARCHIVE_FILE_TYPE = 1 # equal to Ci::JobArtifact.file_types['archive'] - METADATA_FILE_TYPE = 2 # equal to Ci::JobArtifact.file_types['metadata'] - LEGACY_PATH_FILE_LOCATION = 1 # equal to Ci::JobArtifact.file_location['legacy_path'] - - def perform(start_id, stop_id) - ActiveRecord::Base.transaction do - insert_archives(start_id, stop_id) - insert_metadatas(start_id, stop_id) - delete_legacy_artifacts(start_id, stop_id) - end - end - - private - - def insert_archives(start_id, stop_id) - ActiveRecord::Base.connection.execute <<~SQL - INSERT INTO - ci_job_artifacts ( - project_id, - job_id, - expire_at, - file_location, - created_at, - updated_at, - file, - size, - file_store, - file_type - ) - SELECT - project_id, - id, - artifacts_expire_at #{add_missing_db_timezone}, - #{LEGACY_PATH_FILE_LOCATION}, - created_at #{add_missing_db_timezone}, - created_at #{add_missing_db_timezone}, - artifacts_file, - artifacts_size, - COALESCE(artifacts_file_store, #{FILE_LOCAL_STORE}), - #{ARCHIVE_FILE_TYPE} - FROM - ci_builds - WHERE - id BETWEEN #{start_id.to_i} AND #{stop_id.to_i} - AND artifacts_file <> '' - AND NOT EXISTS ( - SELECT - 1 - FROM - ci_job_artifacts - WHERE - ci_builds.id = ci_job_artifacts.job_id - AND ci_job_artifacts.file_type = #{ARCHIVE_FILE_TYPE}) - SQL - end - - def insert_metadatas(start_id, stop_id) - ActiveRecord::Base.connection.execute <<~SQL - INSERT INTO - ci_job_artifacts ( - project_id, - job_id, - expire_at, - file_location, - created_at, - updated_at, - file, - size, - file_store, - file_type - ) - SELECT - project_id, - id, - artifacts_expire_at #{add_missing_db_timezone}, - #{LEGACY_PATH_FILE_LOCATION}, - created_at #{add_missing_db_timezone}, - created_at #{add_missing_db_timezone}, - artifacts_metadata, - NULL, - COALESCE(artifacts_metadata_store, #{FILE_LOCAL_STORE}), - #{METADATA_FILE_TYPE} - FROM - ci_builds - WHERE - id BETWEEN #{start_id.to_i} AND #{stop_id.to_i} - AND artifacts_file <> '' - AND artifacts_metadata <> '' - AND NOT EXISTS ( - SELECT - 1 - FROM - ci_job_artifacts - WHERE - ci_builds.id = ci_job_artifacts.job_id - AND ci_job_artifacts.file_type = #{METADATA_FILE_TYPE}) - SQL - end - - def delete_legacy_artifacts(start_id, stop_id) - ActiveRecord::Base.connection.execute <<~SQL - UPDATE - ci_builds - SET - artifacts_file = NULL, - artifacts_file_store = NULL, - artifacts_size = NULL, - artifacts_metadata = NULL, - artifacts_metadata_store = NULL - WHERE - id BETWEEN #{start_id.to_i} AND #{stop_id.to_i} - AND artifacts_file <> '' - SQL - end - - def add_missing_db_timezone - 'at time zone \'UTC\'' - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_test_reports_issue_id.rb b/lib/gitlab/background_migration/populate_test_reports_issue_id.rb new file mode 100644 index 00000000000..301efd0c943 --- /dev/null +++ b/lib/gitlab/background_migration/populate_test_reports_issue_id.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true +# rubocop: disable Style/Documentation + +module Gitlab + module BackgroundMigration + class PopulateTestReportsIssueId + def perform(start_id, stop_id) + # NO OP + end + end + end +end + +Gitlab::BackgroundMigration::PopulateTestReportsIssueId.prepend_mod diff --git a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb index 84ff7423254..c1b8de1f6aa 100644 --- a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb +++ b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # rubocop: disable Style/Documentation -class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid +class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid # rubocop:disable Metrics/ClassLength # rubocop: disable Gitlab/NamespacedClass class VulnerabilitiesIdentifier < ActiveRecord::Base self.table_name = "vulnerability_identifiers" @@ -9,10 +9,14 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid end class VulnerabilitiesFinding < ActiveRecord::Base + include EachBatch include ShaAttribute self.table_name = "vulnerability_occurrences" + + has_many :signatures, foreign_key: 'finding_id', class_name: 'VulnerabilityFindingSignature', inverse_of: :finding belongs_to :primary_identifier, class_name: 'VulnerabilitiesIdentifier', inverse_of: :primary_findings, foreign_key: 'primary_identifier_id' + REPORT_TYPES = { sast: 0, dependency_scanning: 1, @@ -20,7 +24,9 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid dast: 3, secret_detection: 4, coverage_fuzzing: 5, - api_fuzzing: 6 + api_fuzzing: 6, + cluster_image_scanning: 7, + generic: 99 }.with_indifferent_access.freeze enum report_type: REPORT_TYPES @@ -28,6 +34,25 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid sha_attribute :location_fingerprint end + class VulnerabilityFindingSignature < ActiveRecord::Base + include ShaAttribute + + self.table_name = 'vulnerability_finding_signatures' + belongs_to :finding, foreign_key: 'finding_id', inverse_of: :signatures, class_name: 'VulnerabilitiesFinding' + + sha_attribute :signature_sha + end + + class VulnerabilitiesFindingPipeline < ActiveRecord::Base + include EachBatch + self.table_name = "vulnerability_occurrence_pipelines" + end + + class Vulnerability < ActiveRecord::Base + include EachBatch + self.table_name = "vulnerabilities" + end + class CalculateFindingUUID FINDING_NAMESPACES_IDS = { development: "a143e9e2-41b3-47bc-9a19-081d089229f4", @@ -52,35 +77,122 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid end # rubocop: enable Gitlab/NamespacedClass + # rubocop: disable Metrics/AbcSize,Metrics/MethodLength,Metrics/BlockLength def perform(start_id, end_id) - findings = VulnerabilitiesFinding - .joins(:primary_identifier) - .select(:id, :report_type, :fingerprint, :location_fingerprint, :project_id) - .where(id: start_id..end_id) - - mappings = findings.each_with_object({}) do |finding, hash| - hash[finding] = { uuid: calculate_uuid_v5_for_finding(finding) } + unless Feature.enabled?(:migrate_vulnerability_finding_uuids, default_enabled: true) + return log_info('Migration is disabled by the feature flag', start_id: start_id, end_id: end_id) end - ::Gitlab::Database::BulkUpdate.execute(%i[uuid], mappings) + log_info('Migration started', start_id: start_id, end_id: end_id) - logger.info(message: 'RecalculateVulnerabilitiesOccurrencesUuid Migration: recalculation is done for:', - finding_ids: mappings.keys.pluck(:id)) + VulnerabilitiesFinding + .joins(:primary_identifier) + .includes(:signatures) + .select(:id, :report_type, :primary_identifier_id, :fingerprint, :location_fingerprint, :project_id, :created_at, :vulnerability_id, :uuid) + .where(id: start_id..end_id) + .each_batch(of: 50) do |relation| + duplicates = find_duplicates(relation) + remove_findings(ids: duplicates) if duplicates.present? + + to_update = relation.reject { |finding| duplicates.include?(finding.id) } + + begin + known_uuids = Set.new + to_be_deleted = [] + + mappings = to_update.each_with_object({}) do |finding, hash| + uuid = calculate_uuid_v5_for_finding(finding) + + if known_uuids.add?(uuid) + hash[finding] = { uuid: uuid } + else + to_be_deleted << finding.id + end + end + + # It is technically still possible to have duplicate uuids + # if the data integrity is broken somehow and the primary identifiers of + # the findings are pointing to different projects with the same fingerprint values. + if to_be_deleted.present? + log_info('Conflicting UUIDs found within the batch', finding_ids: to_be_deleted) + + remove_findings(ids: to_be_deleted) + end + + ::Gitlab::Database::BulkUpdate.execute(%i[uuid], mappings) if mappings.present? + + log_info('Recalculation is done', finding_ids: mappings.keys.pluck(:id)) + rescue ActiveRecord::RecordNotUnique => error + log_info('RecordNotUnique error received') + + match_data = /\(uuid\)=\((?\S{36})\)/.match(error.message) + + # This exception returns the **correct** UUIDv5 which probably comes from a later record + # and it's the one we can drop in the easiest way before retrying the UPDATE query + if match_data + uuid = match_data[:uuid] + log_info('Conflicting UUID found', uuid: uuid) + + id = VulnerabilitiesFinding.find_by(uuid: uuid)&.id + remove_findings(ids: id) if id + retry + else + log_error('Couldnt find conflicting uuid') + + Gitlab::ErrorTracking.track_and_raise_exception(error) + end + end + end mark_job_as_succeeded(start_id, end_id) rescue StandardError => error - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) + log_error('An exception happened') + + Gitlab::ErrorTracking.track_and_raise_exception(error) end + # rubocop: disable Metrics/AbcSize,Metrics/MethodLength,Metrics/BlockLength private + def find_duplicates(relation) + to_exclude = [] + relation.flat_map do |record| + # Assuming we're scanning id 31 and the duplicate is id 40 + # first we'd process 31 and add 40 to the list of ids to remove + # then we would process record 40 and add 31 to the list of removals + # so we would drop both records + to_exclude << record.id + + VulnerabilitiesFinding.where( + report_type: record.report_type, + location_fingerprint: record.location_fingerprint, + primary_identifier_id: record.primary_identifier_id, + project_id: record.project_id + ).where.not(id: to_exclude).pluck(:id) + end + end + + def remove_findings(ids:) + ids = Array(ids) + log_info('Removing Findings and associated records', ids: ids) + + vulnerability_ids = VulnerabilitiesFinding.where(id: ids).pluck(:vulnerability_id).uniq.compact + + VulnerabilitiesFindingPipeline.where(occurrence_id: ids).each_batch { |batch| batch.delete_all } + Vulnerability.where(id: vulnerability_ids).each_batch { |batch| batch.delete_all } + VulnerabilitiesFinding.where(id: ids).delete_all + end + def calculate_uuid_v5_for_finding(vulnerability_finding) return unless vulnerability_finding + signatures = vulnerability_finding.signatures.sort_by { |signature| signature.algorithm_type_before_type_cast } + location_fingerprint = signatures.last&.signature_sha || vulnerability_finding.location_fingerprint + uuid_v5_name_components = { report_type: vulnerability_finding.report_type, primary_identifier_fingerprint: vulnerability_finding.fingerprint, - location_fingerprint: vulnerability_finding.location_fingerprint, + location_fingerprint: location_fingerprint, project_id: vulnerability_finding.project_id } @@ -89,6 +201,14 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid CalculateFindingUUID.call(name) end + def log_info(message, **extra) + logger.info(migrator: 'RecalculateVulnerabilitiesOccurrencesUuid', message: message, **extra) + end + + def log_error(message, **extra) + logger.error(migrator: 'RecalculateVulnerabilitiesOccurrencesUuid', message: message, **extra) + end + def logger @logger ||= Gitlab::BackgroundMigration::Logger.build end diff --git a/lib/gitlab/background_migration/remove_duplicate_services.rb b/lib/gitlab/background_migration/remove_duplicate_services.rb deleted file mode 100644 index 59fb9143a72..00000000000 --- a/lib/gitlab/background_migration/remove_duplicate_services.rb +++ /dev/null @@ -1,58 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Remove duplicated service records with the same project and type. - # These were created in the past for unknown reasons, and should be blocked - # now by the uniqueness validation in the Service model. - class RemoveDuplicateServices - # See app/models/service - class Service < ActiveRecord::Base - include EachBatch - - self.table_name = 'services' - self.inheritance_column = :_type_disabled - - scope :project_ids_with_duplicates, -> do - select(:project_id) - .distinct - .where.not(project_id: nil) - .group(:project_id, :type) - .having('count(*) > 1') - end - - scope :types_with_duplicates, -> (project_ids) do - select(:project_id, :type) - .where(project_id: project_ids) - .group(:project_id, :type) - .having('count(*) > 1') - end - end - - def perform(*project_ids) - types_with_duplicates = Service.types_with_duplicates(project_ids).pluck(:project_id, :type) - - types_with_duplicates.each do |project_id, type| - remove_duplicates(project_id, type) - end - end - - private - - def remove_duplicates(project_id, type) - scope = Service.where(project_id: project_id, type: type) - - # Build a subquery to determine which service record is actually in use, - # by querying for it without specifying an order. - # - # This should match the record returned by `Project#find_service`, - # and the `has_one` service associations on `Project`. - correct_service = scope.select(:id).limit(1) - - # Delete all other services with the same `project_id` and `type` - duplicate_services = scope.where.not(id: correct_service) - duplicate_services.delete_all - end - end - end -end diff --git a/lib/gitlab/checks/changes_access.rb b/lib/gitlab/checks/changes_access.rb index 3ce2e50c548..84c01cf4baf 100644 --- a/lib/gitlab/checks/changes_access.rb +++ b/lib/gitlab/checks/changes_access.rb @@ -33,18 +33,33 @@ module Gitlab # changes. This set may also contain commits which are not referenced by # any of the new revisions. def commits + allow_quarantine = true + newrevs = @changes.map do |change| + oldrev = change[:oldrev] newrev = change[:newrev] - newrev unless newrev.blank? || Gitlab::Git.blank_ref?(newrev) + + next if blank_rev?(newrev) + + # In case any of the old revisions is blank, then we cannot reliably + # detect which commits are new for a given change when enumerating + # objects via the object quarantine directory given that the client + # may have pushed too many commits, and we don't know when to + # terminate the walk. We thus fall back to using `git rev-list --not + # --all`, which is a lot less efficient but at least can only ever + # returns commits which really are new. + allow_quarantine = false if allow_quarantine && blank_rev?(oldrev) + + newrev end.compact return [] if newrevs.empty? - @commits ||= project.repository.new_commits(newrevs, allow_quarantine: true) + @commits ||= project.repository.new_commits(newrevs, allow_quarantine: allow_quarantine) end # All commits which have been newly introduced via the given revision. - def commits_for(newrev) + def commits_for(oldrev, newrev) commits_by_id = commits.index_by(&:id) result = [] @@ -65,9 +80,11 @@ module Gitlab # Only add the parent ID to the pending set if we actually know its # commit to guards us against readding an ID which we have already - # queued up before. + # queued up before. Furthermore, we stop walking as soon as we hit + # `oldrev` such that we do not include any commits in our checks + # which have been "over-pushed" by the client. commit.parent_ids.each do |parent_id| - pending.add(parent_id) if commits_by_id.has_key?(parent_id) + pending.add(parent_id) if commits_by_id.has_key?(parent_id) && parent_id != oldrev end result << commit @@ -80,10 +97,10 @@ module Gitlab @single_changes_accesses ||= changes.map do |change| commits = - if change[:newrev].blank? || Gitlab::Git.blank_ref?(change[:newrev]) + if blank_rev?(change[:newrev]) [] else - Gitlab::Lazy.new { commits_for(change[:newrev]) } + Gitlab::Lazy.new { commits_for(change[:oldrev], change[:newrev]) } end Checks::SingleChangeAccess.new( @@ -109,6 +126,10 @@ module Gitlab def bulk_access_checks! Gitlab::Checks::LfsCheck.new(self).validate! end + + def blank_rev?(rev) + rev.blank? || Gitlab::Git.blank_ref?(rev) + end end end end diff --git a/lib/gitlab/ci/build/policy/refs.rb b/lib/gitlab/ci/build/policy/refs.rb index afe0ccb361e..7ade9ca5085 100644 --- a/lib/gitlab/ci/build/policy/refs.rb +++ b/lib/gitlab/ci/build/policy/refs.rb @@ -35,7 +35,10 @@ module Gitlab # patterns can be matched only when branch or tag is used # the pattern matching does not work for merge requests pipelines if pipeline.branch? || pipeline.tag? - if regexp = Gitlab::UntrustedRegexp::RubySyntax.fabricate(pattern, fallback: true) + regexp = Gitlab::UntrustedRegexp::RubySyntax + .fabricate(pattern, fallback: true, project: pipeline.project) + + if regexp regexp.match?(pipeline.ref) else pattern == pipeline.ref diff --git a/lib/gitlab/ci/build/status/reason.rb b/lib/gitlab/ci/build/status/reason.rb new file mode 100644 index 00000000000..82e07faef63 --- /dev/null +++ b/lib/gitlab/ci/build/status/reason.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + module Status + class Reason + attr_reader :build, :failure_reason, :exit_code + + def initialize(build, failure_reason, exit_code = nil) + @build = build + @failure_reason = failure_reason + @exit_code = exit_code + end + + def failure_reason_enum + ::CommitStatus.failure_reasons[failure_reason] + end + + def force_allow_failure? + return false if exit_code.nil? + + !build.allow_failure? && build.allowed_to_fail_with_code?(exit_code) + end + + def self.fabricate(build, reason) + if reason.is_a?(self) + new(build, reason.failure_reason, reason.exit_code) + else + new(build, reason) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 42b487fdf81..4c98941e032 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -36,7 +36,7 @@ module Gitlab end @root = self.logger.instrument(:config_compose) do - Entry::Root.new(@config).tap(&:compose!) + Entry::Root.new(@config, project: project, user: user).tap(&:compose!) end rescue *rescue_errors => e raise Config::ConfigError, e.message diff --git a/lib/gitlab/ci/config/entry/root.rb b/lib/gitlab/ci/config/entry/root.rb index e6290ef2479..41a3c87037b 100644 --- a/lib/gitlab/ci/config/entry/root.rb +++ b/lib/gitlab/ci/config/entry/root.rb @@ -59,7 +59,8 @@ module Gitlab entry :types, Entry::Stages, description: 'Deprecated: stages for this pipeline.', - reserved: true + reserved: true, + deprecation: { deprecated: '9.0', warning: '14.8', removed: '15.0', documentation: 'https://docs.gitlab.com/ee/ci/yaml/#deprecated-keywords' } entry :cache, Entry::Caches, description: 'Configure caching between build jobs.', @@ -122,8 +123,9 @@ module Gitlab # Deprecated `:types` key workaround - if types are defined and # stages are not defined we use types definition as stages. # - if types_defined? && !stages_defined? - @entries[:stages] = @entries[:types] + if types_defined? + @entries[:stages] = @entries[:types] unless stages_defined? + log_and_warn_deprecated_entry(@entries[:types]) end @entries.delete(:types) diff --git a/lib/gitlab/ci/jwt_v2.rb b/lib/gitlab/ci/jwt_v2.rb new file mode 100644 index 00000000000..278353220e4 --- /dev/null +++ b/lib/gitlab/ci/jwt_v2.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class JwtV2 < Jwt + private + + def reserved_claims + super.merge( + iss: Settings.gitlab.base_url, + sub: "project_path:#{project.full_path}:ref_type:#{ref_type}:ref:#{source_ref}", + aud: Settings.gitlab.base_url + ) + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index 15b0ff3c04d..54b54bd0514 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -9,13 +9,13 @@ module Gitlab include Gitlab::Utils::StrongMemoize def perform! - logger.instrument(:pipeline_save) do + logger.instrument_with_sql(:pipeline_save) do BulkInsertableAssociations.with_bulk_insert do - tags = extract_tag_list_by_status - - pipeline.transaction do - pipeline.save! - CommitStatus.bulk_insert_tags!(statuses, tags) if bulk_insert_tags? + with_bulk_insert_tags do + pipeline.transaction do + pipeline.save! + CommitStatus.bulk_insert_tags!(statuses) if bulk_insert_tags? + end end end end @@ -29,32 +29,26 @@ module Gitlab private - def statuses - strong_memoize(:statuses) do - pipeline.stages.flat_map(&:statuses) + def bulk_insert_tags? + strong_memoize(:bulk_insert_tags) do + ::Feature.enabled?(:ci_bulk_insert_tags, project, default_enabled: :yaml) end end - # We call `job.tag_list=` to assign tags to the jobs from the - # Chain::Seed step which uses the `@tag_list` instance variable to - # store them on the record. We remove them here because we want to - # bulk insert them, otherwise they would be inserted and assigned one - # by one with callbacks. We must use `remove_instance_variable` - # because having the instance variable defined would still run the callbacks - def extract_tag_list_by_status - return {} unless bulk_insert_tags? - - statuses.each.with_object({}) do |job, acc| - tag_list = job.clear_memoization(:tag_list) - next unless tag_list - - acc[job.name] = tag_list - end + def with_bulk_insert_tags + previous = Thread.current['ci_bulk_insert_tags'] + Thread.current['ci_bulk_insert_tags'] = bulk_insert_tags? + yield + ensure + Thread.current['ci_bulk_insert_tags'] = previous end - def bulk_insert_tags? - strong_memoize(:bulk_insert_tags) do - ::Feature.enabled?(:ci_bulk_insert_tags, project, default_enabled: :yaml) + def statuses + strong_memoize(:statuses) do + pipeline + .stages + .flat_map(&:statuses) + .select { |status| status.respond_to?(:tag_list) } end end end diff --git a/lib/gitlab/ci/pipeline/chain/create_deployments.rb b/lib/gitlab/ci/pipeline/chain/create_deployments.rb index b92aa89d62d..b913ba3c87d 100644 --- a/lib/gitlab/ci/pipeline/chain/create_deployments.rb +++ b/lib/gitlab/ci/pipeline/chain/create_deployments.rb @@ -5,8 +5,6 @@ module Gitlab module Pipeline module Chain class CreateDeployments < Chain::Base - DeploymentCreationError = Class.new(StandardError) - def perform! return unless pipeline.create_deployment_in_separate_transaction? @@ -24,18 +22,7 @@ module Gitlab end def create_deployment(build) - return unless build.instance_of?(::Ci::Build) && build.persisted_environment.present? - - deployment = ::Gitlab::Ci::Pipeline::Seed::Deployment - .new(build, build.persisted_environment).to_resource - - return unless deployment - - deployment.deployable = build - deployment.save! - rescue ActiveRecord::RecordInvalid => e - Gitlab::ErrorTracking.track_and_raise_for_dev_exception( - DeploymentCreationError.new(e.message), build_id: build.id) + ::Deployments::CreateForBuildService.new.execute(build) end end end diff --git a/lib/gitlab/ci/pipeline/chain/seed.rb b/lib/gitlab/ci/pipeline/chain/seed.rb index 356eeb76908..feae123f216 100644 --- a/lib/gitlab/ci/pipeline/chain/seed.rb +++ b/lib/gitlab/ci/pipeline/chain/seed.rb @@ -53,13 +53,18 @@ module Gitlab end def context - Gitlab::Ci::Pipeline::Seed::Context.new(pipeline, root_variables: root_variables) + Gitlab::Ci::Pipeline::Seed::Context.new( + pipeline, + root_variables: root_variables, + logger: logger + ) end def root_variables logger.instrument(:pipeline_seed_merge_variables) do ::Gitlab::Ci::Variables::Helpers.merge_variables( - @command.yaml_processor_result.root_variables, @command.workflow_rules_result.variables + @command.yaml_processor_result.root_variables, + @command.workflow_rules_result.variables ) end end diff --git a/lib/gitlab/ci/pipeline/logger.rb b/lib/gitlab/ci/pipeline/logger.rb index 97f7dddd09a..fbba12c11a9 100644 --- a/lib/gitlab/ci/pipeline/logger.rb +++ b/lib/gitlab/ci/pipeline/logger.rb @@ -37,6 +37,16 @@ module Gitlab result end + def instrument_with_sql(operation, &block) + op_start_db_counters = current_db_counter_payload + + result = instrument(operation, &block) + + observe_sql_counters(operation, op_start_db_counters, current_db_counter_payload) + + result + end + def observe(operation, value) return unless enabled? @@ -50,11 +60,20 @@ module Gitlab class: self.class.name.to_s, pipeline_creation_caller: caller, project_id: project.id, - pipeline_id: pipeline.id, pipeline_persisted: pipeline.persisted?, pipeline_source: pipeline.source, pipeline_creation_service_duration_s: age - }.stringify_keys.merge(observations_hash) + } + + if pipeline.persisted? + attributes[:pipeline_builds_tags_count] = pipeline.tags_count + attributes[:pipeline_builds_distinct_tags_count] = pipeline.distinct_tags_count + attributes[:pipeline_id] = pipeline.id + end + + attributes.compact! + attributes.stringify_keys! + attributes.merge!(observations_hash) destination.info(attributes) end @@ -97,6 +116,19 @@ module Gitlab def observations @observations ||= Hash.new { |hash, key| hash[key] = [] } end + + def observe_sql_counters(operation, start_db_counters, end_db_counters) + end_db_counters.each do |key, value| + result = value - start_db_counters.fetch(key, 0) + next if result == 0 + + observe("#{operation}_#{key}", result) + end + end + + def current_db_counter_payload + ::Gitlab::Metrics::Subscribers::ActiveRecord.db_counter_payload + end end end end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 762292f0fa3..5a0ad695741 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -41,12 +41,14 @@ module Gitlab def included? strong_memoize(:inclusion) do - if @using_rules - rules_result.pass? - elsif @using_only || @using_except - all_of_only? && none_of_except? - else - true + logger.instrument(:pipeline_seed_build_inclusion) do + if @using_rules + rules_result.pass? + elsif @using_only || @using_except + all_of_only? && none_of_except? + else + true + end end end end @@ -122,6 +124,8 @@ module Gitlab private + delegate :logger, to: :@context + def all_of_only? @only.all? { |spec| spec.satisfied_by?(@pipeline, evaluate_context) } end diff --git a/lib/gitlab/ci/pipeline/seed/context.rb b/lib/gitlab/ci/pipeline/seed/context.rb index 6194a78f682..c0b8ebeb833 100644 --- a/lib/gitlab/ci/pipeline/seed/context.rb +++ b/lib/gitlab/ci/pipeline/seed/context.rb @@ -5,11 +5,18 @@ module Gitlab module Pipeline module Seed class Context - attr_reader :pipeline, :root_variables + attr_reader :pipeline, :root_variables, :logger - def initialize(pipeline, root_variables: []) + def initialize(pipeline, root_variables: [], logger: nil) @pipeline = pipeline @root_variables = root_variables + @logger = logger || build_logger + end + + private + + def build_logger + ::Gitlab::Ci::Pipeline::Logger.new(project: pipeline.project) end end end diff --git a/lib/gitlab/ci/queue/metrics.rb b/lib/gitlab/ci/queue/metrics.rb index 7f45d626922..54fb1d19ea8 100644 --- a/lib/gitlab/ci/queue/metrics.rb +++ b/lib/gitlab/ci/queue/metrics.rb @@ -69,17 +69,6 @@ module Gitlab self.class.attempt_counter.increment end - # rubocop: disable CodeReuse/ActiveRecord - def jobs_running_for_project(job) - return '+Inf' unless runner.instance_type? - - # excluding currently started job - running_jobs_count = job.project.builds.running.where(runner: ::Ci::Runner.instance_type) - .limit(JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET + 1).count - 1 - running_jobs_count < JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET ? running_jobs_count : "#{JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET}+" - end - # rubocop: enable CodeReuse/ActiveRecord - def increment_queue_operation(operation) self.class.increment_queue_operation(operation) end @@ -242,6 +231,32 @@ module Gitlab Gitlab::Metrics.histogram(name, comment, labels, buckets) end end + + private + + # rubocop: disable CodeReuse/ActiveRecord + def jobs_running_for_project(job) + return '+Inf' unless runner.instance_type? + + # excluding currently started job + running_jobs_count = running_jobs_relation(job) + .limit(JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET + 1).count - 1 + + if running_jobs_count < JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET + running_jobs_count + else + "#{JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET}+" + end + end + + def running_jobs_relation(job) + if ::Feature.enabled?(:ci_pending_builds_maintain_denormalized_data, default_enabled: :yaml) + ::Ci::RunningBuild.instance_type.where(project_id: job.project_id) + else + job.project.builds.running.where(runner: ::Ci::Runner.instance_type) + end + end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/ci/status/build/factory.rb b/lib/gitlab/ci/status/build/factory.rb index 7e5afbad806..a4434e2c144 100644 --- a/lib/gitlab/ci/status/build/factory.rb +++ b/lib/gitlab/ci/status/build/factory.rb @@ -14,7 +14,8 @@ module Gitlab Status::Build::WaitingForResource, Status::Build::Preparing, Status::Build::Pending, - Status::Build::Skipped], + Status::Build::Skipped, + Status::Build::WaitingForApproval], [Status::Build::Cancelable, Status::Build::Retryable], [Status::Build::FailedUnmetPrerequisites, diff --git a/lib/gitlab/ci/status/build/waiting_for_approval.rb b/lib/gitlab/ci/status/build/waiting_for_approval.rb new file mode 100644 index 00000000000..59869a947a9 --- /dev/null +++ b/lib/gitlab/ci/status/build/waiting_for_approval.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Status + module Build + class WaitingForApproval < Status::Extended + def illustration + { + image: 'illustrations/manual_action.svg', + size: 'svg-394', + title: 'Waiting for approval', + content: "This job deploys to the protected environment \"#{subject.deployment&.environment&.name}\" which requires approvals. Use the Deployments API to approve or reject the deployment." + } + end + + def self.matches?(build, user) + build.waiting_for_deployment_approval? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/tags/bulk_insert.rb b/lib/gitlab/ci/tags/bulk_insert.rb index a299df7e2d9..29f3731a9b4 100644 --- a/lib/gitlab/ci/tags/bulk_insert.rb +++ b/lib/gitlab/ci/tags/bulk_insert.rb @@ -4,12 +4,13 @@ module Gitlab module Ci module Tags class BulkInsert + include Gitlab::Utils::StrongMemoize + TAGGINGS_BATCH_SIZE = 1000 TAGS_BATCH_SIZE = 500 - def initialize(statuses, tag_list_by_status) + def initialize(statuses) @statuses = statuses - @tag_list_by_status = tag_list_by_status end def insert! @@ -20,7 +21,18 @@ module Gitlab private - attr_reader :statuses, :tag_list_by_status + attr_reader :statuses + + def tag_list_by_status + strong_memoize(:tag_list_by_status) do + statuses.each.with_object({}) do |status, acc| + tag_list = status.tag_list + next unless tag_list + + acc[status] = tag_list + end + end + end def persist_build_tags! all_tags = tag_list_by_status.values.flatten.uniq.reject(&:blank?) @@ -54,7 +66,7 @@ module Gitlab def build_taggings_attributes(tag_records_by_name) taggings = statuses.flat_map do |status| - tag_list = tag_list_by_status[status.name] + tag_list = tag_list_by_status[status] next unless tag_list tags = tag_records_by_name.values_at(*tag_list) diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 00b771f1e5c..6942631a97f 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -7,7 +7,7 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.24-gitlab.1" + CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.26" needs: [] script: - export SOURCE_CODE=$PWD diff --git a/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml index 18f0f20203d..42487cc0c67 100644 --- a/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml @@ -14,6 +14,8 @@ variables: image: "$SECURE_ANALYZERS_PREFIX/secrets:$SECRETS_ANALYZER_VERSION" services: [] allow_failure: true + variables: + GIT_DEPTH: "50" # `rules` must be overridden explicitly by each child job # see https://gitlab.com/gitlab-org/gitlab/-/issues/218444 artifacts: @@ -29,8 +31,16 @@ secret_detection: script: - if [ -n "$CI_COMMIT_TAG" ]; then echo "Skipping Secret Detection for tags. No code changes have occurred."; exit 0; fi - if [ "$CI_COMMIT_BRANCH" = "$CI_DEFAULT_BRANCH" ]; then echo "Running Secret Detection on default branch."; /analyzer run; exit 0; fi - - git fetch origin $CI_DEFAULT_BRANCH $CI_COMMIT_REF_NAME - - git log --left-right --cherry-pick --pretty=format:"%H" refs/remotes/origin/$CI_DEFAULT_BRANCH...refs/remotes/origin/$CI_COMMIT_REF_NAME > "$CI_COMMIT_SHA"_commit_list.txt - - export SECRET_DETECTION_COMMITS_FILE="$CI_COMMIT_SHA"_commit_list.txt + - | + git fetch origin $CI_DEFAULT_BRANCH $CI_COMMIT_REF_NAME + git log --left-right --cherry-pick --pretty=format:"%H" refs/remotes/origin/${CI_DEFAULT_BRANCH}..refs/remotes/origin/${CI_COMMIT_REF_NAME} >${CI_COMMIT_SHA}_commit_list.txt + if [[ $(wc -l <${CI_COMMIT_SHA}_commit_list.txt) -eq "0" ]]; then + # if git log produces 0 or 1 commits we should scan $CI_COMMIT_SHA only + export SECRET_DETECTION_COMMITS=$CI_COMMIT_SHA + else + # +1 because busybox wc only counts \n and there is no trailing \n + echo "scanning $(($(wc -l <${CI_COMMIT_SHA}_commit_list.txt) + 1)) commits" + export SECRET_DETECTION_COMMITS_FILE=${CI_COMMIT_SHA}_commit_list.txt + fi - /analyzer run - rm "$CI_COMMIT_SHA"_commit_list.txt diff --git a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml index 0c8b98dc1cf..1660a9250e3 100644 --- a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml @@ -29,7 +29,7 @@ before_script: - ruby -v # Print out ruby version for debugging # Uncomment next line if your rails app needs a JS runtime: # - apt-get update -q && apt-get install nodejs -yqq - - bundle config set path 'vendor' # Install dependencies into ./vendor/ruby + - bundle config set path 'vendor' # Install dependencies into ./vendor/ruby - bundle install -j $(nproc) # Optional - Delete if not using `rubocop` diff --git a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml index 7243f240eed..f7f016b5e57 100644 --- a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml @@ -33,6 +33,7 @@ coverage_fuzzing_unlicensed: before_script: - export COVFUZZ_JOB_TOKEN=$CI_JOB_TOKEN - export COVFUZZ_PRIVATE_TOKEN=$CI_PRIVATE_TOKEN + - export COVFUZZ_PROJECT_PATH=$CI_PROJECT_PATH - export COVFUZZ_PROJECT_ID=$CI_PROJECT_ID - if [ -x "$(command -v apt-get)" ] ; then apt-get update && apt-get install -y wget; fi - wget -O gitlab-cov-fuzz "${COVFUZZ_URL_PREFIX}"/"${COVFUZZ_VERSION}"/binaries/gitlab-cov-fuzz_Linux_x86_64 diff --git a/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml new file mode 100644 index 00000000000..6888e955467 --- /dev/null +++ b/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml @@ -0,0 +1,27 @@ +stages: + - build + - test + - deploy + - dast + +variables: + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + DAST_API_VERSION: "1" + DAST_API_IMAGE: $SECURE_ANALYZERS_PREFIX/api-fuzzing:$DAST_API_VERSION + +dast: + stage: dast + image: $DAST_API_IMAGE + variables: + GIT_STRATEGY: none + allow_failure: true + script: + - /peach/analyzer-dast-api + artifacts: + when: always + paths: + - gl-assets + - gl-dast-api-report.json + - gl-*.log + reports: + dast: gl-dast-api-report.json diff --git a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml index e554742735c..12c987a8d37 100644 --- a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml @@ -5,9 +5,11 @@ include: - template: Terraform/Base.latest.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml + - template: Jobs/SAST-IaC.latest.gitlab-ci.yml stages: - validate + - test - build - deploy diff --git a/lib/gitlab/ci/trace/remote_checksum.rb b/lib/gitlab/ci/trace/remote_checksum.rb index d57f3888ec0..7f43d91e6d7 100644 --- a/lib/gitlab/ci/trace/remote_checksum.rb +++ b/lib/gitlab/ci/trace/remote_checksum.rb @@ -26,7 +26,6 @@ module Gitlab delegate :aws?, :google?, to: :object_store_config, prefix: :provider def fetch_md5_checksum - return unless Feature.enabled?(:ci_archived_build_trace_checksum, trace_artifact.project, default_enabled: :yaml) return unless object_store_config.enabled? return if trace_artifact.local_store? diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index 2d31049a0c9..dd435ba05b7 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -11,10 +11,6 @@ module Gitlab delegate :close, :tell, :seek, :size, :url, :truncate, to: :stream, allow_nil: true - delegate :valid?, to: :stream, allow_nil: true - - alias_method :present?, :valid? - def initialize(metrics = Trace::Metrics.new) @stream = yield @stream&.binmode @@ -24,6 +20,7 @@ module Gitlab def valid? self.stream.present? end + alias_method :present?, :valid? def file? self.path.present? diff --git a/lib/gitlab/ci/variables/builder.rb b/lib/gitlab/ci/variables/builder.rb index 3e2c2c7fc1a..4c777527ebc 100644 --- a/lib/gitlab/ci/variables/builder.rb +++ b/lib/gitlab/ci/variables/builder.rb @@ -13,12 +13,76 @@ module Gitlab def scoped_variables(job, environment:, dependencies:) Gitlab::Ci::Variables::Collection.new.tap do |variables| variables.concat(predefined_variables(job)) + + next variables unless pipeline.use_variables_builder_definitions? + + variables.concat(project.predefined_variables) + variables.concat(pipeline.predefined_variables) + variables.concat(job.runner.predefined_variables) if job.runnable? && job.runner + variables.concat(kubernetes_variables(job)) + variables.concat(deployment_variables(environment: environment, job: job)) + variables.concat(job.yaml_variables) + variables.concat(user_variables(job.user)) + variables.concat(job.dependency_variables) if dependencies + variables.concat(secret_instance_variables(ref: job.git_ref)) + variables.concat(secret_group_variables(environment: environment, ref: job.git_ref)) + variables.concat(secret_project_variables(environment: environment, ref: job.git_ref)) + variables.concat(job.trigger_request.user_variables) if job.trigger_request + variables.concat(pipeline.variables) + variables.concat(pipeline.pipeline_schedule.job_variables) if pipeline.pipeline_schedule + end + end + + def kubernetes_variables(job) + ::Gitlab::Ci::Variables::Collection.new.tap do |collection| + # Should get merged with the cluster kubeconfig in deployment_variables, see + # https://gitlab.com/gitlab-org/gitlab/-/issues/335089 + template = ::Ci::GenerateKubeconfigService.new(job).execute + + if template.valid? + collection.append(key: 'KUBECONFIG', value: template.to_yaml, public: false, file: true) + end end end + def deployment_variables(environment:, job:) + return [] unless environment + + project.deployment_variables( + environment: environment, + kubernetes_namespace: job.expanded_kubernetes_namespace + ) + end + + def user_variables(user) + Gitlab::Ci::Variables::Collection.new.tap do |variables| + break variables if user.blank? + + variables.append(key: 'GITLAB_USER_ID', value: user.id.to_s) + variables.append(key: 'GITLAB_USER_EMAIL', value: user.email) + variables.append(key: 'GITLAB_USER_LOGIN', value: user.username) + variables.append(key: 'GITLAB_USER_NAME', value: user.name) + end + end + + def secret_instance_variables(ref:) + project.ci_instance_variables_for(ref: ref) + end + + def secret_group_variables(environment:, ref:) + return [] unless project.group + + project.group.ci_variables_for(ref, project, environment: environment) + end + + def secret_project_variables(environment:, ref:) + project.ci_variables_for(ref: ref, environment: environment) + end + private attr_reader :pipeline + delegate :project, to: :pipeline def predefined_variables(job) Gitlab::Ci::Variables::Collection.new.tap do |variables| diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 296b0cfded2..553508c8638 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -86,11 +86,19 @@ module Gitlab def validate_job_needs!(name, job) return unless needs = job.dig(:needs, :job) + validate_duplicate_needs!(name, needs) + needs.each do |need| validate_job_dependency!(name, need[:name], 'need') end end + def validate_duplicate_needs!(name, needs) + unless needs.uniq == needs + error!("#{name} has duplicate entries in the needs section.") + end + end + def validate_job_dependency!(name, dependency, dependency_type = 'dependency') unless @jobs[dependency.to_sym] error!("#{name} job: undefined #{dependency_type}: #{dependency}") diff --git a/lib/gitlab/color_schemes.rb b/lib/gitlab/color_schemes.rb index 620b4a8aee6..3884f5f0428 100644 --- a/lib/gitlab/color_schemes.rb +++ b/lib/gitlab/color_schemes.rb @@ -7,21 +7,23 @@ module Gitlab # Struct class representing a single Scheme Scheme = Struct.new(:id, :name, :css_class) - SCHEMES = [ - Scheme.new(1, 'White', 'white'), - Scheme.new(2, 'Dark', 'dark'), - Scheme.new(3, 'Solarized Light', 'solarized-light'), - Scheme.new(4, 'Solarized Dark', 'solarized-dark'), - Scheme.new(5, 'Monokai', 'monokai'), - Scheme.new(6, 'None', 'none') - ].freeze + def self.available_schemes + [ + Scheme.new(1, s_('SynthaxHighlightingTheme|Light'), 'white'), + Scheme.new(2, s_('SynthaxHighlightingTheme|Dark'), 'dark'), + Scheme.new(3, s_('SynthaxHighlightingTheme|Solarized Light'), 'solarized-light'), + Scheme.new(4, s_('SynthaxHighlightingTheme|Solarized Dark'), 'solarized-dark'), + Scheme.new(5, s_('SynthaxHighlightingTheme|Monokai'), 'monokai'), + Scheme.new(6, s_('SynthaxHighlightingTheme|None'), 'none') + ] + end # Convenience method to get a space-separated String of all the color scheme # classes that might be applied to a code block. # # Returns a String def self.body_classes - SCHEMES.collect(&:css_class).uniq.join(' ') + available_schemes.collect(&:css_class).uniq.join(' ') end # Get a Scheme by its ID @@ -32,12 +34,12 @@ module Gitlab # # Returns a Scheme def self.by_id(id) - SCHEMES.detect { |s| s.id == id } || default + available_schemes.detect { |s| s.id == id } || default end # Returns the number of defined Schemes def self.count - SCHEMES.size + available_schemes.size end # Get the default Scheme @@ -51,7 +53,7 @@ module Gitlab # # Yields the Scheme object def self.each(&block) - SCHEMES.each(&block) + available_schemes.each(&block) end # Get the Scheme for the specified user, or the default @@ -68,7 +70,7 @@ module Gitlab end def self.valid_ids - SCHEMES.map(&:id) + available_schemes.map(&:id) end end end diff --git a/lib/gitlab/config/entry/configurable.rb b/lib/gitlab/config/entry/configurable.rb index 6bf77ebaa5b..aa6c724c2a3 100644 --- a/lib/gitlab/config/entry/configurable.rb +++ b/lib/gitlab/config/entry/configurable.rb @@ -76,7 +76,7 @@ module Gitlab private # rubocop: disable CodeReuse/ActiveRecord - def entry(key, entry, description: nil, default: nil, inherit: nil, reserved: nil, metadata: {}) + def entry(key, entry, description: nil, default: nil, inherit: nil, reserved: nil, deprecation: nil, metadata: {}) entry_name = key.to_sym raise ArgumentError, "Entry '#{key}' already defined in '#{name}'" if @nodes.to_h[entry_name] @@ -85,6 +85,7 @@ module Gitlab .with(default: default) .with(inherit: inherit) .with(reserved: reserved) + .with(deprecation: deprecation) .metadata(metadata) @nodes ||= {} diff --git a/lib/gitlab/config/entry/factory.rb b/lib/gitlab/config/entry/factory.rb index f76c98f7cbf..61f2071b62f 100644 --- a/lib/gitlab/config/entry/factory.rb +++ b/lib/gitlab/config/entry/factory.rb @@ -32,6 +32,10 @@ module Gitlab self end + def deprecation + @attributes[:deprecation] + end + def description @attributes[:description] end @@ -84,6 +88,7 @@ module Gitlab node.parent = @attributes[:parent] node.default = @attributes[:default] node.description = @attributes[:description] + node.deprecation = @attributes[:deprecation] end end end diff --git a/lib/gitlab/config/entry/node.rb b/lib/gitlab/config/entry/node.rb index 32912cb1046..6ce7046262b 100644 --- a/lib/gitlab/config/entry/node.rb +++ b/lib/gitlab/config/entry/node.rb @@ -10,7 +10,7 @@ module Gitlab InvalidError = Class.new(StandardError) attr_reader :config, :metadata - attr_accessor :key, :parent, :default, :description + attr_accessor :key, :parent, :default, :description, :deprecation def initialize(config, **metadata) @config = config @@ -128,6 +128,24 @@ module Gitlab private attr_reader :entries + + def log_and_warn_deprecated_entry(entry) + user = metadata[:user] + project = metadata[:project] + + if project && user + Gitlab::AppJsonLogger.info(event: 'ci_used_deprecated_keyword', + entry: entry.key.to_s, + user_id: user.id, + project_id: project.id) + end + + deprecation = entry.deprecation + add_warning( + "`#{entry.key}` is deprecated in " \ + "#{deprecation[:deprecated]} and will be removed in #{deprecation[:removed]}." + ) + end end end end diff --git a/lib/gitlab/content_security_policy/config_loader.rb b/lib/gitlab/content_security_policy/config_loader.rb index 87bc2ace204..78ba0916808 100644 --- a/lib/gitlab/content_security_policy/config_loader.rb +++ b/lib/gitlab/content_security_policy/config_loader.rb @@ -147,7 +147,7 @@ module Gitlab # Using 'self' in the CSP introduces several CSP bypass opportunities # for this reason we list the URLs where GitLab frames itself instead def self.allow_framed_gitlab_paths(directives) - ['/admin/', '/assets/', '/-/speedscope/index.html'].map do |path| + ['/admin/', '/assets/', '/-/speedscope/index.html', '/-/sandbox/mermaid'].map do |path| append_to_directive(directives, 'frame_src', Gitlab::Utils.append_path(Gitlab.config.gitlab.url, path)) end end diff --git a/lib/gitlab/data_builder/archive_trace.rb b/lib/gitlab/data_builder/archive_trace.rb new file mode 100644 index 00000000000..f6dd6130104 --- /dev/null +++ b/lib/gitlab/data_builder/archive_trace.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module DataBuilder + module ArchiveTrace + extend self + + def build(job) + { + object_kind: 'archive_trace', + trace_url: job.job_artifacts_trace.file.url, + build_id: job.id, + pipeline_id: job.pipeline_id, + project: job.project.hook_attrs + } + end + end + end +end diff --git a/lib/gitlab/data_builder/deployment.rb b/lib/gitlab/data_builder/deployment.rb index 267c2d32ca9..a4508bc93c5 100644 --- a/lib/gitlab/data_builder/deployment.rb +++ b/lib/gitlab/data_builder/deployment.rb @@ -25,7 +25,8 @@ module Gitlab user: deployment.deployed_by.hook_attrs, user_url: Gitlab::UrlBuilder.build(deployment.deployed_by), commit_url: Gitlab::UrlBuilder.build(deployment.commit), - commit_title: deployment.commit.title + commit_title: deployment.commit.title, + ref: deployment.ref } end end diff --git a/lib/gitlab/database/background_migration/batched_job.rb b/lib/gitlab/database/background_migration/batched_job.rb index 503172dd750..290fa51692a 100644 --- a/lib/gitlab/database/background_migration/batched_job.rb +++ b/lib/gitlab/database/background_migration/batched_job.rb @@ -12,17 +12,6 @@ module Gitlab MAX_ATTEMPTS = 3 STUCK_JOBS_TIMEOUT = 1.hour.freeze - belongs_to :batched_migration, foreign_key: :batched_background_migration_id - - scope :active, -> { where(status: [:pending, :running]) } - scope :stuck, -> { active.where('updated_at <= ?', STUCK_JOBS_TIMEOUT.ago) } - scope :retriable, -> { - failed_jobs = where(status: :failed).where('attempts < ?', MAX_ATTEMPTS) - - from_union([failed_jobs, self.stuck]) - } - scope :except_succeeded, -> { where(status: self.statuses.except(:succeeded).values) } - enum status: { pending: 0, running: 1, @@ -30,7 +19,14 @@ module Gitlab succeeded: 3 } + belongs_to :batched_migration, foreign_key: :batched_background_migration_id + + scope :active, -> { where(status: [:pending, :running]) } + scope :stuck, -> { active.where('updated_at <= ?', STUCK_JOBS_TIMEOUT.ago) } + scope :retriable, -> { from_union([failed.where('attempts < ?', MAX_ATTEMPTS), self.stuck]) } + scope :except_succeeded, -> { where(status: self.statuses.except(:succeeded).values) } scope :successful_in_execution_order, -> { where.not(finished_at: nil).succeeded.order(:finished_at) } + scope :with_preloads, -> { preload(:batched_migration) } delegate :job_class, :table_name, :column_name, :job_arguments, to: :batched_migration, prefix: :migration diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb index 2844cbe4a74..2f066039874 100644 --- a/lib/gitlab/database/background_migration/batched_migration.rb +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -113,7 +113,7 @@ module Gitlab end def smoothed_time_efficiency(number_of_jobs: 10, alpha: 0.2) - jobs = batched_jobs.successful_in_execution_order.reverse_order.limit(number_of_jobs) + jobs = batched_jobs.successful_in_execution_order.reverse_order.limit(number_of_jobs).with_preloads return if jobs.size < number_of_jobs diff --git a/lib/gitlab/database/background_migration_job.rb b/lib/gitlab/database/background_migration_job.rb index c046571a111..c0e3016fd3d 100644 --- a/lib/gitlab/database/background_migration_job.rb +++ b/lib/gitlab/database/background_migration_job.rb @@ -2,7 +2,7 @@ module Gitlab module Database - class BackgroundMigrationJob < ActiveRecord::Base # rubocop:disable Rails/ApplicationRecord + class BackgroundMigrationJob < SharedModel include EachBatch include BulkInsertSafe diff --git a/lib/gitlab/database/batch_counter.rb b/lib/gitlab/database/batch_counter.rb index 6c0ce9e481a..417511618e4 100644 --- a/lib/gitlab/database/batch_counter.rb +++ b/lib/gitlab/database/batch_counter.rb @@ -52,12 +52,7 @@ module Gitlab batch_end = [batch_start + batch_size, finish].min batch_relation = build_relation_batch(batch_start, batch_end, mode) - op_args = @operation_args - if @operation == :count && @operation_args.blank? && use_loose_index_scan_for_distinct_values?(mode) - op_args = [Gitlab::Database::LooseIndexScanDistinctCount::COLUMN_ALIAS] - end - - results = merge_results(results, batch_relation.send(@operation, *op_args)) # rubocop:disable GitlabSecurity/PublicSend + results = merge_results(results, batch_relation.send(@operation, *@operation_args)) # rubocop:disable GitlabSecurity/PublicSend batch_start = batch_end rescue ActiveRecord::QueryCanceled => error # retry with a safe batch size & warmer cache @@ -67,18 +62,6 @@ module Gitlab log_canceled_batch_fetch(batch_start, mode, batch_relation.to_sql, error) return FALLBACK end - rescue Gitlab::Database::LooseIndexScanDistinctCount::ColumnConfigurationError => error - Gitlab::AppJsonLogger - .error( - event: 'batch_count', - relation: @relation.table_name, - operation: @operation, - operation_args: @operation_args, - mode: mode, - message: "LooseIndexScanDistinctCount column error: #{error.message}" - ) - - return FALLBACK end sleep(SLEEP_TIME_IN_SECONDS) @@ -104,11 +87,7 @@ module Gitlab private def build_relation_batch(start, finish, mode) - if use_loose_index_scan_for_distinct_values?(mode) - Gitlab::Database::LooseIndexScanDistinctCount.new(@relation, @column).build_query(from: start, to: finish) - else - @relation.select(@column).public_send(mode).where(between_condition(start, finish)) # rubocop:disable GitlabSecurity/PublicSend - end + @relation.select(@column).public_send(mode).where(between_condition(start, finish)) # rubocop:disable GitlabSecurity/PublicSend end def batch_size_for_mode_and_operation(mode, operation) @@ -151,10 +130,6 @@ module Gitlab ) end - def use_loose_index_scan_for_distinct_values?(mode) - Feature.enabled?(:loose_index_scan_for_distinct_values) && not_group_by_query? && mode == :distinct - end - def not_group_by_query? !@relation.is_a?(ActiveRecord::Relation) || @relation.group_values.blank? end diff --git a/lib/gitlab/database/gitlab_loose_foreign_keys.yml b/lib/gitlab/database/gitlab_loose_foreign_keys.yml index 0343c054f23..d694165574d 100644 --- a/lib/gitlab/database/gitlab_loose_foreign_keys.yml +++ b/lib/gitlab/database/gitlab_loose_foreign_keys.yml @@ -1,3 +1,12 @@ +--- +dast_site_profiles_pipelines: + - table: ci_pipelines + column: ci_pipeline_id + on_delete: async_delete +vulnerability_feedback: + - table: ci_pipelines + column: pipeline_id + on_delete: async_nullify ci_pipeline_chat_data: - table: chat_names column: chat_name_id @@ -6,7 +15,7 @@ dast_scanner_profiles_builds: - table: ci_builds column: ci_build_id on_delete: async_delete -dast_scanner_profiles_builds: +dast_site_profiles_builds: - table: ci_builds column: ci_build_id on_delete: async_delete @@ -18,10 +27,48 @@ clusters_applications_runners: - table: ci_runners column: runner_id on_delete: async_nullify +ci_job_token_project_scope_links: + - table: users + column: added_by_id + on_delete: async_nullify +ci_daily_build_group_report_results: + - table: namespaces + column: group_id + on_delete: async_delete + - table: projects + column: project_id + on_delete: async_delete +ci_freeze_periods: + - table: projects + column: project_id + on_delete: async_delete +ci_pending_builds: + - table: namespaces + column: namespace_id + on_delete: async_delete + - table: projects + column: project_id + on_delete: async_delete +ci_resource_groups: + - table: projects + column: project_id + on_delete: async_delete +ci_runner_namespaces: + - table: namespaces + column: namespace_id + on_delete: async_delete +ci_running_builds: + - table: projects + column: project_id + on_delete: async_delete ci_namespace_mirrors: - table: namespaces column: namespace_id on_delete: async_delete +ci_build_report_results: + - table: projects + column: project_id + on_delete: async_delete ci_builds: - table: users column: user_id @@ -43,6 +90,22 @@ ci_project_mirrors: - table: namespaces column: namespace_id on_delete: async_delete +ci_unit_tests: + - table: projects + column: project_id + on_delete: async_delete +merge_requests: + - table: ci_pipelines + column: head_pipeline_id + on_delete: async_nullify +vulnerability_statistics: + - table: ci_pipelines + column: latest_pipeline_id + on_delete: async_nullify +vulnerability_occurrence_pipelines: + - table: ci_pipelines + column: pipeline_id + on_delete: async_delete packages_build_infos: - table: ci_pipelines column: pipeline_id @@ -67,3 +130,31 @@ project_pages_metadata: - table: ci_job_artifacts column: artifacts_archive_id on_delete: async_nullify +ci_pipeline_schedules: + - table: users + column: owner_id + on_delete: async_nullify +ci_group_variables: + - table: namespaces + column: group_id + on_delete: async_delete +ci_minutes_additional_packs: + - table: namespaces + column: namespace_id + on_delete: async_delete +requirements_management_test_reports: + - table: ci_builds + column: build_id + on_delete: async_nullify +security_scans: + - table: ci_builds + column: build_id + on_delete: async_delete +ci_secure_files: + - table: projects + column: project_id + on_delete: async_delete +ci_pipeline_artifacts: + - table: projects + column: project_id + on_delete: async_delete diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml index 24c2d634780..fb5d8cfa32f 100644 --- a/lib/gitlab/database/gitlab_schemas.yml +++ b/lib/gitlab/database/gitlab_schemas.yml @@ -107,6 +107,7 @@ ci_runner_projects: :gitlab_ci ci_runners: :gitlab_ci ci_running_builds: :gitlab_ci ci_sources_pipelines: :gitlab_ci +ci_secure_files: :gitlab_ci ci_sources_projects: :gitlab_ci ci_stages: :gitlab_ci ci_subscriptions_projects: :gitlab_ci @@ -200,7 +201,7 @@ experiment_subjects: :gitlab_main experiment_users: :gitlab_main external_approval_rules: :gitlab_main external_approval_rules_protected_branches: :gitlab_main -external_pull_requests: :gitlab_main +external_pull_requests: :gitlab_ci external_status_checks: :gitlab_main external_status_checks_protected_branches: :gitlab_main feature_gates: :gitlab_main @@ -231,6 +232,7 @@ gpg_key_subkeys: :gitlab_main gpg_signatures: :gitlab_main grafana_integrations: :gitlab_main group_custom_attributes: :gitlab_main +group_crm_settings: :gitlab_main group_deletion_schedules: :gitlab_main group_deploy_keys: :gitlab_main group_deploy_keys_groups: :gitlab_main @@ -460,6 +462,8 @@ security_findings: :gitlab_main security_orchestration_policy_configurations: :gitlab_main security_orchestration_policy_rule_schedules: :gitlab_main security_scans: :gitlab_main +security_training_providers: :gitlab_main +security_trainings: :gitlab_main self_managed_prometheus_alert_events: :gitlab_main sent_notifications: :gitlab_main sentry_issues: :gitlab_main @@ -521,13 +525,7 @@ vulnerabilities: :gitlab_main vulnerability_exports: :gitlab_main vulnerability_external_issue_links: :gitlab_main vulnerability_feedback: :gitlab_main -vulnerability_finding_evidence_assets: :gitlab_main -vulnerability_finding_evidence_headers: :gitlab_main -vulnerability_finding_evidence_requests: :gitlab_main -vulnerability_finding_evidence_responses: :gitlab_main vulnerability_finding_evidences: :gitlab_main -vulnerability_finding_evidence_sources: :gitlab_main -vulnerability_finding_evidence_supporting_messages: :gitlab_main vulnerability_finding_links: :gitlab_main vulnerability_finding_signatures: :gitlab_main vulnerability_findings_remediations: :gitlab_main diff --git a/lib/gitlab/database/grant.rb b/lib/gitlab/database/grant.rb index c8a30c68bc6..0093848ee6f 100644 --- a/lib/gitlab/database/grant.rb +++ b/lib/gitlab/database/grant.rb @@ -10,7 +10,7 @@ module Gitlab # We _must not_ use quote_table_name as this will produce double # quotes on PostgreSQL and for "has_table_privilege" we need single # quotes. - connection = ActiveRecord::Base.connection # rubocop: disable Database/MultipleDatabases + connection = ApplicationRecord.connection quoted_table = connection.quote(table) begin diff --git a/lib/gitlab/database/load_balancing/setup.rb b/lib/gitlab/database/load_balancing/setup.rb index ef38f42f50b..126c8bb2aa6 100644 --- a/lib/gitlab/database/load_balancing/setup.rb +++ b/lib/gitlab/database/load_balancing/setup.rb @@ -104,11 +104,9 @@ module Gitlab end end - # rubocop:disable Database/MultipleDatabases def connection - use_model_load_balancing? ? super : ActiveRecord::Base.connection + use_model_load_balancing? ? super : ApplicationRecord.connection end - # rubocop:enable Database/MultipleDatabases end end end diff --git a/lib/gitlab/database/loose_index_scan_distinct_count.rb b/lib/gitlab/database/loose_index_scan_distinct_count.rb deleted file mode 100644 index 26be07f91c4..00000000000 --- a/lib/gitlab/database/loose_index_scan_distinct_count.rb +++ /dev/null @@ -1,102 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - # This class builds efficient batched distinct query by using loose index scan. - # Consider the following example: - # > Issue.distinct(:project_id).where(project_id: (1...100)).count - # - # Note: there is an index on project_id - # - # This query will read each element in the index matching the project_id filter. - # If for a project_id has 100_000 issues, all 100_000 elements will be read. - # - # A loose index scan will only read one entry from the index for each project_id to reduce the number of disk reads. - # - # Usage: - # - # Gitlab::Database::LooseIndexScanDisctinctCount.new(Issue, :project_id).count(from: 1, to: 100) - # - # The query will return the number of distinct projects_ids between 1 and 100 - # - # Getting the Arel query: - # - # Gitlab::Database::LooseIndexScanDisctinctCount.new(Issue, :project_id).build_query(from: 1, to: 100) - class LooseIndexScanDistinctCount - COLUMN_ALIAS = 'distinct_count_column' - - ColumnConfigurationError = Class.new(StandardError) - - def initialize(scope, column) - if scope.is_a?(ActiveRecord::Relation) - @scope = scope - @model = scope.model - else - @scope = scope.where({}) - @model = scope - end - - @column = transform_column(column) - end - - def count(from:, to:) - build_query(from: from, to: to).count(COLUMN_ALIAS) - end - - def build_query(from:, to:) # rubocop:disable Metrics/AbcSize - cte = Gitlab::SQL::RecursiveCTE.new(:counter_cte, union_args: { remove_order: false }) - table = model.arel_table - - cte << @scope - .dup - .select(column.as(COLUMN_ALIAS)) - .where(column.gteq(from)) - .where(column.lt(to)) - .order(column) - .limit(1) - - inner_query = @scope - .dup - .where(column.gt(cte.table[COLUMN_ALIAS])) - .where(column.lt(to)) - .select(column.as(COLUMN_ALIAS)) - .order(column) - .limit(1) - - cte << cte.table - .project(Arel::Nodes::Grouping.new(Arel.sql(inner_query.to_sql)).as(COLUMN_ALIAS)) - .where(cte.table[COLUMN_ALIAS].lt(to)) - - model - .with - .recursive(cte.to_arel) - .from(cte.alias_to(table)) - .unscope(where: :source_type) - .unscope(where: model.inheritance_column) # Remove STI query, not needed here - end - - private - - attr_reader :column, :model - - # Transforms the column so it can be used in Arel expressions - # - # 'table.column' => 'table.column' - # 'column' => 'table_name.column' - # :column => 'table_name.column' - # Arel::Attributes::Attribute => name of the column - def transform_column(column) - if column.is_a?(String) || column.is_a?(Symbol) - column_as_string = column.to_s - column_as_string = "#{model.table_name}.#{column_as_string}" unless column_as_string.include?('.') - - Arel.sql(column_as_string) - elsif column.is_a?(Arel::Attributes::Attribute) - column - else - raise ColumnConfigurationError, "Cannot transform the column: #{column.inspect}, please provide the column name as string" - end - end - end - end -end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 4245dd80714..aa5ac1e3486 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -778,186 +778,6 @@ module Gitlab install_rename_triggers(table, old, new) end - # Changes the column type of a table using a background migration. - # - # Because this method uses a background migration it's more suitable for - # large tables. For small tables it's better to use - # `change_column_type_concurrently` since it can complete its work in a - # much shorter amount of time and doesn't rely on Sidekiq. - # - # Example usage: - # - # class Issue < ActiveRecord::Base - # self.table_name = 'issues' - # - # include EachBatch - # - # def self.to_migrate - # where('closed_at IS NOT NULL') - # end - # end - # - # change_column_type_using_background_migration( - # Issue.to_migrate, - # :closed_at, - # :datetime_with_timezone - # ) - # - # Reverting a migration like this is done exactly the same way, just with - # a different type to migrate to (e.g. `:datetime` in the above example). - # - # relation - An ActiveRecord relation to use for scheduling jobs and - # figuring out what table we're modifying. This relation _must_ - # have the EachBatch module included. - # - # column - The name of the column for which the type will be changed. - # - # new_type - The new type of the column. - # - # batch_size - The number of rows to schedule in a single background - # migration. - # - # interval - The time interval between every background migration. - def change_column_type_using_background_migration( - relation, - column, - new_type, - batch_size: 10_000, - interval: 10.minutes - ) - - unless relation.model < EachBatch - raise TypeError, 'The relation must include the EachBatch module' - end - - temp_column = "#{column}_for_type_change" - table = relation.table_name - max_index = 0 - - add_column(table, temp_column, new_type) - install_rename_triggers(table, column, temp_column) - - # Schedule the jobs that will copy the data from the old column to the - # new one. Rows with NULL values in our source column are skipped since - # the target column is already NULL at this point. - relation.where.not(column => nil).each_batch(of: batch_size) do |batch, index| - start_id, end_id = batch.pluck('MIN(id), MAX(id)').first - max_index = index - - migrate_in( - index * interval, - 'CopyColumn', - [table, column, temp_column, start_id, end_id] - ) - end - - # Schedule the renaming of the column to happen (initially) 1 hour after - # the last batch finished. - migrate_in( - (max_index * interval) + 1.hour, - 'CleanupConcurrentTypeChange', - [table, column, temp_column] - ) - - if perform_background_migration_inline? - # To ensure the schema is up to date immediately we perform the - # migration inline in dev / test environments. - Gitlab::BackgroundMigration.steal('CopyColumn') - Gitlab::BackgroundMigration.steal('CleanupConcurrentTypeChange') - end - end - - # Renames a column using a background migration. - # - # Because this method uses a background migration it's more suitable for - # large tables. For small tables it's better to use - # `rename_column_concurrently` since it can complete its work in a much - # shorter amount of time and doesn't rely on Sidekiq. - # - # Example usage: - # - # rename_column_using_background_migration( - # :users, - # :feed_token, - # :rss_token - # ) - # - # table - The name of the database table containing the column. - # - # old - The old column name. - # - # new - The new column name. - # - # type - The type of the new column. If no type is given the old column's - # type is used. - # - # batch_size - The number of rows to schedule in a single background - # migration. - # - # interval - The time interval between every background migration. - def rename_column_using_background_migration( - table, - old_column, - new_column, - type: nil, - batch_size: 10_000, - interval: 10.minutes - ) - - check_trigger_permissions!(table) - - old_col = column_for(table, old_column) - new_type = type || old_col.type - max_index = 0 - - add_column(table, new_column, new_type, - limit: old_col.limit, - precision: old_col.precision, - scale: old_col.scale) - - # We set the default value _after_ adding the column so we don't end up - # updating any existing data with the default value. This isn't - # necessary since we copy over old values further down. - change_column_default(table, new_column, old_col.default) if old_col.default - - install_rename_triggers(table, old_column, new_column) - - model = Class.new(ActiveRecord::Base) do - self.table_name = table - - include ::EachBatch - end - - # Schedule the jobs that will copy the data from the old column to the - # new one. Rows with NULL values in our source column are skipped since - # the target column is already NULL at this point. - model.where.not(old_column => nil).each_batch(of: batch_size) do |batch, index| - start_id, end_id = batch.pluck('MIN(id), MAX(id)').first - max_index = index - - migrate_in( - index * interval, - 'CopyColumn', - [table, old_column, new_column, start_id, end_id] - ) - end - - # Schedule the renaming of the column to happen (initially) 1 hour after - # the last batch finished. - migrate_in( - (max_index * interval) + 1.hour, - 'CleanupConcurrentRename', - [table, old_column, new_column] - ) - - if perform_background_migration_inline? - # To ensure the schema is up to date immediately we perform the - # migration inline in dev / test environments. - Gitlab::BackgroundMigration.steal('CopyColumn') - Gitlab::BackgroundMigration.steal('CleanupConcurrentRename') - end - end - def convert_to_bigint_column(column) "#{column}_convert_to_bigint" end diff --git a/lib/gitlab/database/migrations/background_migration_helpers.rb b/lib/gitlab/database/migrations/background_migration_helpers.rb index 8c33c41ce77..4f1b490cc8f 100644 --- a/lib/gitlab/database/migrations/background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/background_migration_helpers.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true - module Gitlab module Database module Migrations @@ -45,11 +44,11 @@ module Gitlab raise "#{model_class} does not have an ID column of #{primary_column_name} to use for batch ranges" unless model_class.column_names.include?(primary_column_name.to_s) raise "#{primary_column_name} is not an integer column" unless model_class.columns_hash[primary_column_name.to_s].type == :integer + job_coordinator = coordinator_for_tracking_database + # To not overload the worker too much we enforce a minimum interval both # when scheduling and performing jobs. - if delay_interval < BackgroundMigrationWorker.minimum_interval - delay_interval = BackgroundMigrationWorker.minimum_interval - end + delay_interval = [delay_interval, job_coordinator.minimum_interval].max final_delay = 0 batch_counter = 0 @@ -60,14 +59,14 @@ module Gitlab start_id, end_id = relation.pluck(min, max).first - # `BackgroundMigrationWorker.bulk_perform_in` schedules all jobs for + # `SingleDatabaseWorker.bulk_perform_in` schedules all jobs for # the same time, which is not helpful in most cases where we wish to # spread the work over time. final_delay = initial_delay + delay_interval * index full_job_arguments = [start_id, end_id] + other_job_arguments track_in_database(job_class_name, full_job_arguments) if track_jobs - migrate_in(final_delay, job_class_name, full_job_arguments) + migrate_in(final_delay, job_class_name, full_job_arguments, coordinator: job_coordinator) batch_counter += 1 end @@ -91,9 +90,11 @@ module Gitlab # delay_interval - The duration between each job's scheduled time # batch_size - The maximum number of jobs to fetch to memory from the database. def requeue_background_migration_jobs_by_range_at_intervals(job_class_name, delay_interval, batch_size: BATCH_SIZE, initial_delay: 0) + job_coordinator = coordinator_for_tracking_database + # To not overload the worker too much we enforce a minimum interval both # when scheduling and performing jobs. - delay_interval = [delay_interval, BackgroundMigrationWorker.minimum_interval].max + delay_interval = [delay_interval, job_coordinator.minimum_interval].max final_delay = 0 job_counter = 0 @@ -103,7 +104,7 @@ module Gitlab job_batch.each do |job| final_delay = initial_delay + delay_interval * job_counter - migrate_in(final_delay, job_class_name, job.arguments) + migrate_in(final_delay, job_class_name, job.arguments, coordinator: job_coordinator) job_counter += 1 end @@ -132,56 +133,33 @@ module Gitlab # This method does not garauntee that all jobs completed successfully. # It can only be used if the previous background migration used the queue_background_migration_jobs_by_range_at_intervals helper. def finalize_background_migration(class_name, delete_tracking_jobs: ['succeeded']) + job_coordinator = coordinator_for_tracking_database + # Empty the sidekiq queue. - Gitlab::BackgroundMigration.steal(class_name) + job_coordinator.steal(class_name) # Process pending tracked jobs. jobs = Gitlab::Database::BackgroundMigrationJob.pending.for_migration_class(class_name) + jobs.find_each do |job| - BackgroundMigrationWorker.new.perform(job.class_name, job.arguments) + job_coordinator.perform(job.class_name, job.arguments) end # Empty the sidekiq queue. - Gitlab::BackgroundMigration.steal(class_name) + job_coordinator.steal(class_name) # Delete job tracking rows. delete_job_tracking(class_name, status: delete_tracking_jobs) if delete_tracking_jobs end - def perform_background_migration_inline? - Rails.env.test? || Rails.env.development? - end - - def migrate_async(*args) - with_migration_context do - BackgroundMigrationWorker.perform_async(*args) - end - end - - def migrate_in(*args) - with_migration_context do - BackgroundMigrationWorker.perform_in(*args) - end - end - - def bulk_migrate_in(*args) + def migrate_in(*args, coordinator: coordinator_for_tracking_database) with_migration_context do - BackgroundMigrationWorker.bulk_perform_in(*args) + coordinator.perform_in(*args) end end - def bulk_migrate_async(*args) - with_migration_context do - BackgroundMigrationWorker.bulk_perform_async(*args) - end - end - - def with_migration_context(&block) - Gitlab::ApplicationContext.with_context(caller_id: self.class.to_s, &block) - end - def delete_queued_jobs(class_name) - Gitlab::BackgroundMigration.steal(class_name) do |job| + coordinator_for_tracking_database.steal(class_name) do |job| job.delete false @@ -196,9 +174,21 @@ module Gitlab private + def with_migration_context(&block) + Gitlab::ApplicationContext.with_context(caller_id: self.class.to_s, &block) + end + def track_in_database(class_name, arguments) Gitlab::Database::BackgroundMigrationJob.create!(class_name: class_name, arguments: arguments) end + + def coordinator_for_tracking_database + Gitlab::BackgroundMigration.coordinator_for_database(tracking_database) + end + + def tracking_database + Gitlab::BackgroundMigration::DEFAULT_TRACKING_DATABASE + end end end end diff --git a/lib/gitlab/database/partitioning/partition_manager.rb b/lib/gitlab/database/partitioning/partition_manager.rb index aa824dfbd2f..ba6fa0cf278 100644 --- a/lib/gitlab/database/partitioning/partition_manager.rb +++ b/lib/gitlab/database/partitioning/partition_manager.rb @@ -64,6 +64,10 @@ module Gitlab # with_lock_retries starts a requires_new transaction most of the time, but not on the last iteration with_lock_retries do connection.transaction(requires_new: false) do # so we open a transaction here if not already in progress + # Partitions might not get created (IF NOT EXISTS) so explicit locking will not happen. + # This LOCK TABLE ensures to have exclusive lock as the first step. + connection.execute "LOCK TABLE #{connection.quote_table_name(model.table_name)} IN ACCESS EXCLUSIVE MODE" + partitions.each do |partition| connection.execute partition.to_sql diff --git a/lib/gitlab/database/partitioning/sliding_list_strategy.rb b/lib/gitlab/database/partitioning/sliding_list_strategy.rb index 21b86b43ae7..e9865fb91d6 100644 --- a/lib/gitlab/database/partitioning/sliding_list_strategy.rb +++ b/lib/gitlab/database/partitioning/sliding_list_strategy.rb @@ -44,7 +44,18 @@ module Gitlab def extra_partitions possibly_extra = current_partitions[0...-1] # Never consider the most recent partition - possibly_extra.take_while { |p| detach_partition_if.call(p.value) } + extra = possibly_extra.take_while { |p| detach_partition_if.call(p.value) } + + default_value = current_default_value + if extra.any? { |p| p.value == default_value } + Gitlab::AppLogger.error(message: "Inconsistent partition detected: partition with value #{current_default_value} should not be deleted because it's used as the default value.", + partition_number: current_default_value, + table_name: model.table_name) + + extra = extra.reject { |p| p.value == default_value } + end + + extra end def after_adding_partitions @@ -64,6 +75,21 @@ module Gitlab private + def current_default_value + column_name = model.connection.quote(partitioning_key) + table_name = model.connection.quote(model.table_name) + + value = model.connection.select_value <<~SQL + SELECT columns.column_default AS default_value + FROM information_schema.columns columns + WHERE columns.column_name = #{column_name} AND columns.table_name = #{table_name} + SQL + + raise "No default value found for the #{partitioning_key} column within #{model.name}" if value.nil? + + Integer(value) + end + def ensure_partitioning_column_ignored! unless model.ignored_columns.include?(partitioning_key.to_s) raise "Add #{partitioning_key} to #{model.name}.ignored_columns to use it with SlidingListStrategy" diff --git a/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb b/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb index 17a42d997e6..f551fa06cad 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb @@ -4,7 +4,7 @@ module Gitlab module Database module PartitioningMigrationHelpers # Class that will generically copy data from a given table into its corresponding partitioned table - class BackfillPartitionedTable + class BackfillPartitionedTable < ::Gitlab::BackgroundMigration::BaseJob include ::Gitlab::Database::DynamicModelHelpers SUB_BATCH_SIZE = 2_500 @@ -21,7 +21,7 @@ module Gitlab return end - bulk_copy = BulkCopy.new(source_table, partitioned_table, source_column) + bulk_copy = BulkCopy.new(source_table, partitioned_table, source_column, connection: connection) parent_batch_relation = relation_scoped_to_range(source_table, source_column, start_id, stop_id) parent_batch_relation.each_batch(of: SUB_BATCH_SIZE) do |sub_batch| @@ -36,10 +36,6 @@ module Gitlab private - def connection - ActiveRecord::Base.connection - end - def transaction_open? connection.transaction_open? end @@ -53,7 +49,8 @@ module Gitlab end def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) - define_batchable_model(source_table).where(source_key_column => start_id..stop_id) + define_batchable_model(source_table) + .where(source_key_column => start_id..stop_id) end def mark_jobs_as_succeeded(*arguments) @@ -64,12 +61,13 @@ module Gitlab class BulkCopy DELIMITER = ', ' - attr_reader :source_table, :destination_table, :source_column + attr_reader :source_table, :destination_table, :source_column, :connection - def initialize(source_table, destination_table, source_column) + def initialize(source_table, destination_table, source_column, connection:) @source_table = source_table @destination_table = destination_table @source_column = source_column + @connection = connection end def copy_between(start_id, stop_id) @@ -85,10 +83,6 @@ module Gitlab private - def connection - @connection ||= ActiveRecord::Base.connection - end - def column_listing @column_listing ||= connection.columns(source_table).map(&:name).join(DELIMITER) end diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb index c382d2f0715..984c708aa48 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb @@ -406,7 +406,8 @@ module Gitlab end def copy_missed_records(source_table_name, partitioned_table_name, source_column) - backfill_table = BackfillPartitionedTable.new + backfill_table = BackfillPartitionedTable.new(connection: connection) + relation = ::Gitlab::Database::BackgroundMigrationJob.pending .for_partitioning_migration(MIGRATION_CLASS_NAME, source_table_name) diff --git a/lib/gitlab/database/reflection.rb b/lib/gitlab/database/reflection.rb index 48a4de28541..3ea7277571f 100644 --- a/lib/gitlab/database/reflection.rb +++ b/lib/gitlab/database/reflection.rb @@ -105,6 +105,35 @@ module Gitlab row['system_identifier'] end + def flavor + { + # Based on https://aws.amazon.com/premiumsupport/knowledge-center/aurora-version-number/ + 'Amazon Aurora PostgreSQL' => { statement: 'SELECT AURORA_VERSION()', error: /PG::UndefinedFunction/ }, + # Based on https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_PostgreSQL.html#PostgreSQL.Concepts.General.FeatureSupport.Extensions, + # this is also available for both Aurora and RDS, so we need to check for the former first. + 'PostgreSQL on Amazon RDS' => { statement: 'SHOW rds.extensions', error: /PG::UndefinedObject/ }, + # Based on https://cloud.google.com/sql/docs/postgres/flags#postgres-c this should be specific + # to Cloud SQL for PostgreSQL + 'Cloud SQL for PostgreSQL' => { statement: 'SHOW cloudsql.iam_authentication', error: /PG::UndefinedObject/ }, + # Based on + # - https://docs.microsoft.com/en-us/azure/postgresql/flexible-server/concepts-extensions + # - https://docs.microsoft.com/en-us/azure/postgresql/concepts-extensions + # this should be available only for Azure Database for PostgreSQL - Flexible Server. + 'Azure Database for PostgreSQL - Flexible Server' => { statement: 'SHOW azure.extensions', error: /PG::UndefinedObject/ }, + # Based on + # - https://docs.microsoft.com/en-us/azure/postgresql/flexible-server/concepts-servers + # - https://docs.microsoft.com/en-us/azure/postgresql/concepts-servers#managing-your-server + # this database is present on both Flexible and Single server, so we should check the former first. + 'Azure Database for PostgreSQL - Single Server' => { statement: "SELECT datname FROM pg_database WHERE datname = 'azure_maintenance'" } + }.each do |flavor, conditions| + return flavor if connection.execute(conditions[:statement]).to_a.present? + rescue ActiveRecord::StatementInvalid => e + raise if conditions[:error] && !e.message.match?(conditions[:error]) + end + + nil + end + private def connection diff --git a/lib/gitlab/database/reindexing.rb b/lib/gitlab/database/reindexing.rb index 6ffe14249f0..91c3fcc7d72 100644 --- a/lib/gitlab/database/reindexing.rb +++ b/lib/gitlab/database/reindexing.rb @@ -76,20 +76,7 @@ module Gitlab def self.cleanup_leftovers! PostgresIndex.reindexing_leftovers.each do |index| - Gitlab::AppLogger.info("Removing index #{index.identifier} which is a leftover, temporary index from previous reindexing activity") - - retries = Gitlab::Database::WithLockRetriesOutsideTransaction.new( - connection: index.connection, - timing_configuration: REMOVE_INDEX_RETRY_CONFIG, - klass: self.class, - logger: Gitlab::AppLogger - ) - - retries.run(raise_on_exhaustion: false) do - index.connection.tap do |conn| - conn.execute("DROP INDEX CONCURRENTLY IF EXISTS #{conn.quote_table_name(index.schema)}.#{conn.quote_table_name(index.name)}") - end - end + Coordinator.new(index).drop end end end diff --git a/lib/gitlab/database/reindexing/coordinator.rb b/lib/gitlab/database/reindexing/coordinator.rb index 3e4a83aa2e7..b4f7da999df 100644 --- a/lib/gitlab/database/reindexing/coordinator.rb +++ b/lib/gitlab/database/reindexing/coordinator.rb @@ -31,6 +31,25 @@ module Gitlab end end + def drop + try_obtain_lease do + Gitlab::AppLogger.info("Removing index #{index.identifier} which is a leftover, temporary index from previous reindexing activity") + + retries = Gitlab::Database::WithLockRetriesOutsideTransaction.new( + connection: index.connection, + timing_configuration: REMOVE_INDEX_RETRY_CONFIG, + klass: self.class, + logger: Gitlab::AppLogger + ) + + retries.run(raise_on_exhaustion: false) do + index.connection.tap do |conn| + conn.execute("DROP INDEX CONCURRENTLY IF EXISTS #{conn.quote_table_name(index.schema)}.#{conn.quote_table_name(index.name)}") + end + end + end + end + private def with_notifications(action) diff --git a/lib/gitlab/database_importers/work_items/base_type_importer.rb b/lib/gitlab/database_importers/work_items/base_type_importer.rb index c5acdb41de5..2d9700cb2bc 100644 --- a/lib/gitlab/database_importers/work_items/base_type_importer.rb +++ b/lib/gitlab/database_importers/work_items/base_type_importer.rb @@ -5,8 +5,8 @@ module Gitlab module WorkItems module BaseTypeImporter def self.import - WorkItem::Type::BASE_TYPES.each do |type, attributes| - WorkItem::Type.create!(base_type: type, **attributes.slice(:name, :icon_name)) + ::WorkItems::Type::BASE_TYPES.each do |type, attributes| + ::WorkItems::Type.create!(base_type: type, **attributes.slice(:name, :icon_name)) end end end diff --git a/lib/gitlab/email.rb b/lib/gitlab/email.rb index 5f935880764..2e8f076c5d8 100644 --- a/lib/gitlab/email.rb +++ b/lib/gitlab/email.rb @@ -18,5 +18,6 @@ module Gitlab InvalidMergeRequestError = Class.new(InvalidRecordError) UnknownIncomingEmail = Class.new(ProcessingError) InvalidAttachment = Class.new(ProcessingError) + EmailTooLarge = Class.new(ProcessingError) end end diff --git a/lib/gitlab/email/failure_handler.rb b/lib/gitlab/email/failure_handler.rb new file mode 100644 index 00000000000..1079a9c2bb6 --- /dev/null +++ b/lib/gitlab/email/failure_handler.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Email + module FailureHandler + def self.handle(receiver, error) + can_retry = false + reason = + case error + when Gitlab::Email::UnknownIncomingEmail + s_("EmailError|We couldn't figure out what the email is for. Please create your issue or comment through the web interface.") + when Gitlab::Email::SentNotificationNotFoundError + s_("EmailError|We couldn't figure out what the email is in reply to. Please create your comment through the web interface.") + when Gitlab::Email::ProjectNotFound + s_("EmailError|We couldn't find the project. Please check if there's any typo.") + when Gitlab::Email::EmptyEmailError + can_retry = true + s_("EmailError|It appears that the email is blank. Make sure your reply is at the top of the email, we can't process inline replies.") + when Gitlab::Email::UserNotFoundError + s_("EmailError|We couldn't figure out what user corresponds to the email. Please create your comment through the web interface.") + when Gitlab::Email::UserBlockedError + s_("EmailError|Your account has been blocked. If you believe this is in error, contact a staff member.") + when Gitlab::Email::UserNotAuthorizedError + s_("EmailError|You are not allowed to perform this action. If you believe this is in error, contact a staff member.") + when Gitlab::Email::NoteableNotFoundError + s_("EmailError|The thread you are replying to no longer exists, perhaps it was deleted? If you believe this is in error, contact a staff member.") + when Gitlab::Email::InvalidAttachment + error.message + when Gitlab::Email::InvalidRecordError + can_retry = true + error.message + when Gitlab::Email::EmailTooLarge + s_("EmailError|We couldn't process your email because it is too large. Please create your issue or comment through the web interface.") + end + + if reason + receiver.mail.body = nil + + EmailRejectionMailer.rejection(reason, receiver.mail.encoded, can_retry).deliver_later + end + + reason + end + end + end +end diff --git a/lib/gitlab/error_tracking/processor/sidekiq_processor.rb b/lib/gitlab/error_tracking/processor/sidekiq_processor.rb index 0d2f673d73c..cc8cfd827f1 100644 --- a/lib/gitlab/error_tracking/processor/sidekiq_processor.rb +++ b/lib/gitlab/error_tracking/processor/sidekiq_processor.rb @@ -53,6 +53,8 @@ module Gitlab # 'args' in :job => from default error handler job_holder = sidekiq.key?('args') ? sidekiq : sidekiq[:job] + return event unless job_holder + if job_holder['args'] job_holder['args'] = filter_arguments(job_holder['args'], job_holder['class']).to_a end diff --git a/lib/gitlab/event_store.rb b/lib/gitlab/event_store.rb new file mode 100644 index 00000000000..3d7b6b27eb0 --- /dev/null +++ b/lib/gitlab/event_store.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +# Gitlab::EventStore is a simple pub-sub mechanism that lets you publish +# domain events and use Sidekiq workers as event handlers. +# +# It can be used to decouple domains from different bounded contexts +# by publishing domain events and let any interested parties subscribe +# to them. +# +module Gitlab + module EventStore + Error = Class.new(StandardError) + InvalidEvent = Class.new(Error) + InvalidSubscriber = Class.new(Error) + + def self.publish(event) + instance.publish(event) + end + + def self.instance + @instance ||= configure! + end + + # Define all event subscriptions using: + # + # store.subscribe(DomainA::SomeWorker, to: DomainB::SomeEvent) + # + # It is possible to subscribe to a subset of events matching a condition: + # + # store.subscribe(DomainA::SomeWorker, to: DomainB::SomeEvent), if: ->(event) { event.data == :some_value } + # + def self.configure! + Store.new do |store| + ### + # Add subscriptions here: + + store.subscribe ::MergeRequests::UpdateHeadPipelineWorker, to: ::Ci::PipelineCreatedEvent + end + end + private_class_method :configure! + end +end diff --git a/lib/gitlab/event_store/event.rb b/lib/gitlab/event_store/event.rb new file mode 100644 index 00000000000..ee0c329b8e8 --- /dev/null +++ b/lib/gitlab/event_store/event.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +# An Event object represents a domain event that occurred in a bounded context. +# By publishing events we notify other bounded contexts about something +# that happened, so that they can react to it. +# +# Define new event classes under `app/events//` with a name +# representing something that happened in the past: +# +# class Projects::ProjectCreatedEvent < Gitlab::EventStore::Event +# def schema +# { +# 'type' => 'object', +# 'properties' => { +# 'project_id' => { 'type' => 'integer' } +# } +# } +# end +# end +# +# To publish it: +# +# Gitlab::EventStore.publish( +# Projects::ProjectCreatedEvent.new(data: { project_id: project.id }) +# ) +# +module Gitlab + module EventStore + class Event + attr_reader :data + + def initialize(data:) + validate_schema!(data) + @data = data + end + + def schema + raise NotImplementedError, 'must specify schema to validate the event' + end + + private + + def validate_schema!(data) + unless data.is_a?(Hash) + raise Gitlab::EventStore::InvalidEvent, "Event data must be a Hash" + end + + unless JSONSchemer.schema(schema).valid?(data.deep_stringify_keys) + raise Gitlab::EventStore::InvalidEvent, "Data for event #{self.class} does not match the defined schema: #{schema}" + end + end + end + end +end diff --git a/lib/gitlab/event_store/store.rb b/lib/gitlab/event_store/store.rb new file mode 100644 index 00000000000..ecf3cd7e562 --- /dev/null +++ b/lib/gitlab/event_store/store.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Gitlab + module EventStore + class Store + attr_reader :subscriptions + + def initialize + @subscriptions = Hash.new { |h, k| h[k] = [] } + + yield(self) if block_given? + + # freeze the subscriptions as safety measure to avoid further + # subcriptions after initialization. + lock! + end + + def subscribe(worker, to:, if: nil) + condition = binding.local_variable_get('if') + + Array(to).each do |event| + validate_subscription!(worker, event) + subscriptions[event] << Gitlab::EventStore::Subscription.new(worker, condition) + end + end + + def publish(event) + unless event.is_a?(Event) + raise InvalidEvent, "Event being published is not an instance of Gitlab::EventStore::Event: got #{event.inspect}" + end + + subscriptions[event.class].each do |subscription| + subscription.consume_event(event) + end + end + + private + + def lock! + @subscriptions.freeze + end + + def validate_subscription!(subscriber, event_class) + unless event_class < Event + raise InvalidEvent, "Event being subscribed to is not a subclass of Gitlab::EventStore::Event: got #{event_class}" + end + + unless subscriber.respond_to?(:perform_async) + raise InvalidSubscriber, "Subscriber is not an ApplicationWorker: got #{subscriber}" + end + end + end + end +end diff --git a/lib/gitlab/event_store/subscriber.rb b/lib/gitlab/event_store/subscriber.rb new file mode 100644 index 00000000000..cf326d1f9e4 --- /dev/null +++ b/lib/gitlab/event_store/subscriber.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +# This module should be included in order to turn an ApplicationWorker +# into a Subscriber. +# This module overrides the `perform` method and provides a better and +# safer interface for handling events via `handle_event` method. +# +# @example: +# class SomeEventSubscriber +# include ApplicationWorker +# include Gitlab::EventStore::Subscriber +# +# def handle_event(event) +# # ... +# end +# end + +module Gitlab + module EventStore + module Subscriber + def perform(event_type, data) + raise InvalidEvent, event_type unless self.class.const_defined?(event_type) + + event = event_type.constantize.new( + data: data.with_indifferent_access + ) + + handle_event(event) + end + + def handle_event(event) + raise NotImplementedError, 'you must implement this methods in order to handle events' + end + end + end +end diff --git a/lib/gitlab/event_store/subscription.rb b/lib/gitlab/event_store/subscription.rb new file mode 100644 index 00000000000..e5c92ab969f --- /dev/null +++ b/lib/gitlab/event_store/subscription.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module EventStore + class Subscription + attr_reader :worker, :condition + + def initialize(worker, condition) + @worker = worker + @condition = condition + end + + def consume_event(event) + return unless condition_met?(event) + + worker.perform_async(event.class.name, event.data) + # TODO: Log dispatching of events to subscriber + + # We rescue and track any exceptions here because we don't want to + # impact other subscribers if one is faulty. + # The method `condition_met?`, since it can run a block, it might encounter + # a bug. By raising an exception here we could interrupt the publishing + # process, preventing other subscribers from consuming the event. + rescue StandardError => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, event_class: event.class.name, event_data: event.data) + end + + private + + def condition_met?(event) + return true unless condition + + condition.call(event) + end + end + end +end diff --git a/lib/gitlab/exceptions_app.rb b/lib/gitlab/exceptions_app.rb new file mode 100644 index 00000000000..de07b788fb9 --- /dev/null +++ b/lib/gitlab/exceptions_app.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +require_relative 'utils/override' + +module Gitlab + class ExceptionsApp < ActionDispatch::PublicExceptions + extend ::Gitlab::Utils::Override + + REQUEST_ID_PLACEHOLDER = '' + REQUEST_ID_PARAGRAPH = '

Request ID: %s

' + + override :call + def call(env) + status, headers, body = super + + if html_rendered? && body.first&.include?(REQUEST_ID_PLACEHOLDER) + body = [insert_request_id(env, body.first)] + headers['X-GitLab-Custom-Error'] = '1' + end + + [status, headers, body] + end + + private + + override :render_html + def render_html(status) + @html_rendered = true + + super + end + + def html_rendered? + !!@html_rendered + end + + def insert_request_id(env, body) + request_id = ERB::Util.html_escape(ActionDispatch::Request.new(env).request_id) + + body.gsub(REQUEST_ID_PLACEHOLDER, REQUEST_ID_PARAGRAPH % [request_id]) + end + end +end diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index 4cc653bec43..7edda290204 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -30,14 +30,6 @@ module Gitlab module Experimentation EXPERIMENTS = { - remove_known_trial_form_fields_welcoming: { - tracking_category: 'Growth::Conversion::Experiment::RemoveKnownTrialFormFieldsWelcoming', - rollout_strategy: :user - }, - remove_known_trial_form_fields_noneditable: { - tracking_category: 'Growth::Conversion::Experiment::RemoveKnownTrialFormFieldsNoneditable', - rollout_strategy: :user - } }.freeze class << self diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index bb3ba1129fc..ac3b4de0988 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -57,6 +57,7 @@ module Gitlab push_frontend_feature_flag(:improved_emoji_picker, default_enabled: :yaml) push_frontend_feature_flag(:new_header_search, default_enabled: :yaml) push_frontend_feature_flag(:bootstrap_confirmation_modals, default_enabled: :yaml) + push_frontend_feature_flag(:sandboxed_mermaid, default_enabled: :yaml) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb index 59882e8d4f8..ab7de14b07a 100644 --- a/lib/gitlab/gpg/commit.rb +++ b/lib/gitlab/gpg/commit.rb @@ -102,7 +102,7 @@ module Gitlab end def verification_status(gpg_key) - return :multiple_signatures if multiple_signatures? && Feature.enabled?(:multiple_gpg_signatures, @commit.project, default_enabled: :yaml) + return :multiple_signatures if multiple_signatures? return :unknown_key unless gpg_key return :unverified_key unless gpg_key.verified? return :unverified unless verified_signature&.valid? diff --git a/lib/gitlab/http.rb b/lib/gitlab/http.rb index 1b860001ac0..d0918fc39bc 100644 --- a/lib/gitlab/http.rb +++ b/lib/gitlab/http.rb @@ -49,11 +49,12 @@ module Gitlab return httparty_perform_request(http_method, path, options_with_timeouts, &block) end - start_time = Gitlab::Metrics::System.monotonic_time + start_time = nil read_total_timeout = options.fetch(:timeout, DEFAULT_READ_TOTAL_TIMEOUT) tracked_timeout_error = false httparty_perform_request(http_method, path, options_with_timeouts) do |fragment| + start_time ||= Gitlab::Metrics::System.monotonic_time elapsed = Gitlab::Metrics::System.monotonic_time - start_time if elapsed > read_total_timeout diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 12203cab8c8..f056381b86a 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -43,27 +43,27 @@ module Gitlab TRANSLATION_LEVELS = { 'bg' => 0, 'cs_CZ' => 0, - 'da_DK' => 51, + 'da_DK' => 49, 'de' => 15, 'en' => 100, 'eo' => 0, - 'es' => 39, + 'es' => 38, 'fil_PH' => 0, - 'fr' => 12, + 'fr' => 11, 'gl_ES' => 0, 'id_ID' => 0, 'it' => 2, - 'ja' => 35, - 'ko' => 11, - 'nb_NO' => 33, + 'ja' => 36, + 'ko' => 12, + 'nb_NO' => 32, 'nl_NL' => 0, 'pl_PL' => 5, - 'pt_BR' => 49, - 'ro_RO' => 23, - 'ru' => 25, - 'tr_TR' => 15, + 'pt_BR' => 50, + 'ro_RO' => 22, + 'ru' => 26, + 'tr_TR' => 14, 'uk' => 45, - 'zh_CN' => 95, + 'zh_CN' => 98, 'zh_HK' => 2, 'zh_TW' => 3 }.freeze diff --git a/lib/gitlab/import/set_async_jid.rb b/lib/gitlab/import/set_async_jid.rb index 054fcdb433f..527d84477fe 100644 --- a/lib/gitlab/import/set_async_jid.rb +++ b/lib/gitlab/import/set_async_jid.rb @@ -13,7 +13,7 @@ module Gitlab def self.set_jid(import_state) jid = generate_jid(import_state) - Gitlab::SidekiqStatus.set(jid, Gitlab::Import::StuckImportJob::IMPORT_JOBS_EXPIRATION, value: 2) + Gitlab::SidekiqStatus.set(jid, Gitlab::Import::StuckImportJob::IMPORT_JOBS_EXPIRATION) import_state.update_column(:jid, jid) end diff --git a/lib/gitlab/import_export/base/relation_factory.rb b/lib/gitlab/import_export/base/relation_factory.rb index 6749ef4e276..8a8c74c302d 100644 --- a/lib/gitlab/import_export/base/relation_factory.rb +++ b/lib/gitlab/import_export/base/relation_factory.rb @@ -196,7 +196,7 @@ module Gitlab end def use_attributes_permitter? - Feature.enabled?(:permitted_attributes_for_import_export, default_enabled: :yaml) + true end def existing_or_new_object diff --git a/lib/gitlab/import_export/group/relation_tree_restorer.rb b/lib/gitlab/import_export/group/relation_tree_restorer.rb index cbc8ee9e18b..c2cbd2fdf47 100644 --- a/lib/gitlab/import_export/group/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/group/relation_tree_restorer.rb @@ -118,7 +118,7 @@ module Gitlab end def filter_attributes(params) - if use_attributes_permitter? && attributes_permitter.permitted_attributes_defined?(importable_class_sym) + if attributes_permitter.permitted_attributes_defined?(importable_class_sym) attributes_permitter.permit(importable_class_sym, params) else Gitlab::ImportExport::AttributeCleaner.clean( @@ -132,10 +132,6 @@ module Gitlab @attributes_permitter ||= Gitlab::ImportExport::AttributesPermitter.new end - def use_attributes_permitter? - Feature.enabled?(:permitted_attributes_for_import_export, default_enabled: :yaml) - end - def present_override_params # we filter out the empty strings from the overrides # keeping the default values configured @@ -264,14 +260,12 @@ module Gitlab @relation_reader.sort_ci_pipelines_by_id end - # Enable logging of each top-level relation creation when Importing - # into a Group if feature flag is enabled + # Enable logging of each top-level relation creation when Importing into a Group def log_relation_creation(importable, relation_key, relation_object) root_ancestor_group = importable.try(:root_ancestor) return unless root_ancestor_group return unless root_ancestor_group.instance_of?(::Group) - return unless Feature.enabled?(:log_import_export_relation_creation, root_ancestor_group) @shared.logger.info( importable_type: importable.class.to_s, diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index ef146359da9..059f6bd42e3 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -120,6 +120,7 @@ included_attributes: - :name ci_cd_settings: - :group_runners_enabled + - :runner_token_expiration_interval metrics_setting: - :dashboard_timezone - :external_dashboard_url @@ -904,6 +905,7 @@ excluded_attributes: - :release_id project_members: - :source_id + - :member_namespace_id - :invite_email_success - :state group_members: diff --git a/lib/gitlab/jwt_authenticatable.rb b/lib/gitlab/jwt_authenticatable.rb index 1270a148e8d..08d9f69497e 100644 --- a/lib/gitlab/jwt_authenticatable.rb +++ b/lib/gitlab/jwt_authenticatable.rb @@ -13,26 +13,38 @@ module Gitlab module ClassMethods include Gitlab::Utils::StrongMemoize - def decode_jwt_for_issuer(issuer, encoded_message) - JWT.decode( - encoded_message, - secret, - true, - { iss: issuer, verify_iss: true, algorithm: 'HS256' } - ) + def decode_jwt(encoded_message, jwt_secret = secret, issuer: nil, iat_after: nil) + options = { algorithm: 'HS256' } + options = options.merge(iss: issuer, verify_iss: true) if issuer.present? + options = options.merge(verify_iat: true) if iat_after.present? + + decoded_message = JWT.decode(encoded_message, jwt_secret, true, options) + payload = decoded_message[0] + if iat_after.present? + raise JWT::DecodeError, "JWT iat claim is missing" if payload['iat'].blank? + + iat = payload['iat'].to_i + raise JWT::ExpiredSignature, 'Token has expired' if iat < iat_after.to_i + end + + decoded_message end def secret strong_memoize(:secret) do - Base64.strict_decode64(File.read(secret_path).chomp).tap do |bytes| - raise "#{secret_path} does not contain #{SECRET_LENGTH} bytes" if bytes.length != SECRET_LENGTH - end + read_secret(secret_path) + end + end + + def read_secret(path) + Base64.strict_decode64(File.read(path).chomp).tap do |bytes| + raise "#{path} does not contain #{SECRET_LENGTH} bytes" if bytes.length != SECRET_LENGTH end end - def write_secret + def write_secret(path = secret_path) bytes = SecureRandom.random_bytes(SECRET_LENGTH) - File.open(secret_path, 'w:BINARY', 0600) do |f| + File.open(path, 'w:BINARY', 0600) do |f| f.chmod(0600) # If the file already existed, the '0600' passed to 'open' above was a no-op. f.write(Base64.strict_encode64(bytes)) end diff --git a/lib/gitlab/kas.rb b/lib/gitlab/kas.rb index 408b3afc128..ed7787ffc49 100644 --- a/lib/gitlab/kas.rb +++ b/lib/gitlab/kas.rb @@ -11,7 +11,7 @@ module Gitlab class << self def verify_api_request(request_headers) - decode_jwt_for_issuer(JWT_ISSUER, request_headers[INTERNAL_API_REQUEST_HEADER]) + decode_jwt(request_headers[INTERNAL_API_REQUEST_HEADER], issuer: JWT_ISSUER) rescue JWT::DecodeError nil end diff --git a/lib/gitlab/lfs/client.rb b/lib/gitlab/lfs/client.rb index a05e8107cad..10df9262cca 100644 --- a/lib/gitlab/lfs/client.rb +++ b/lib/gitlab/lfs/client.rb @@ -36,7 +36,7 @@ module Gitlab headers: build_request_headers ) - raise BatchSubmitError unless rsp.success? + raise BatchSubmitError.new(http_response: rsp) unless rsp.success? # HTTParty provides rsp.parsed_response, but it only kicks in for the # application/json content type in the response, which we can't rely on @@ -53,19 +53,13 @@ module Gitlab params = { body_stream: file, - headers: { - 'Content-Length' => object.size.to_s, - 'Content-Type' => 'application/octet-stream', - 'User-Agent' => GIT_LFS_USER_AGENT - }.merge(upload_action['header'] || {}) + headers: upload_headers(object, upload_action) } - authenticated = true if params[:headers].key?('Authorization') - params[:basic_auth] = basic_auth unless authenticated - - rsp = Gitlab::HTTP.put(upload_action['href'], params) + url = set_basic_auth_and_extract_lfs_url!(params, upload_action['href']) + rsp = Gitlab::HTTP.put(url, params) - raise ObjectUploadError unless rsp.success? + raise ObjectUploadError.new(http_response: rsp) unless rsp.success? ensure file&.close end @@ -76,20 +70,51 @@ module Gitlab headers: build_request_headers(verify_action['header']) } - authenticated = true if params[:headers].key?('Authorization') - params[:basic_auth] = basic_auth unless authenticated - - rsp = Gitlab::HTTP.post(verify_action['href'], params) + url = set_basic_auth_and_extract_lfs_url!(params, verify_action['href']) + rsp = Gitlab::HTTP.post(url, params) - raise ObjectVerifyError unless rsp.success? + raise ObjectVerifyError.new(http_response: rsp) unless rsp.success? end private + def set_basic_auth_and_extract_lfs_url!(params, raw_url) + authenticated = true if params[:headers].key?('Authorization') + params[:basic_auth] = basic_auth unless authenticated + strip_userinfo = authenticated || params[:basic_auth].present? + lfs_url(raw_url, strip_userinfo) + end + def build_request_headers(extra_headers = nil) DEFAULT_HEADERS.merge(extra_headers || {}) end + def upload_headers(object, upload_action) + # This uses the httprb library to handle case-insensitive HTTP headers + headers = ::HTTP::Headers.new + headers.merge!(upload_action['header']) + transfer_encodings = Array(headers['Transfer-Encoding']&.split(',')).map(&:strip) + + headers['Content-Length'] = object.size.to_s unless transfer_encodings.include?('chunked') + headers['Content-Type'] = 'application/octet-stream' + headers['User-Agent'] = GIT_LFS_USER_AGENT + + headers.to_h + end + + def lfs_url(raw_url, strip_userinfo) + # HTTParty will give precedence to the username/password + # specified in the URL. This causes problems with Azure DevOps, + # which includes a username in the URL. Stripping the userinfo + # from the URL allows the provided HTTP Basic Authentication + # credentials to be used. + if strip_userinfo + Gitlab::UrlSanitizer.new(raw_url).sanitized_url + else + raw_url + end + end + attr_reader :credentials def batch_url @@ -105,9 +130,21 @@ module Gitlab { username: credentials[:user], password: credentials[:password] } end - class BatchSubmitError < StandardError + class HttpError < StandardError + def initialize(http_response:) + super + + @http_response = http_response + end + + def http_error + "HTTP status #{@http_response.code}" + end + end + + class BatchSubmitError < HttpError def message - "Failed to submit batch" + "Failed to submit batch: #{http_error}" end end @@ -122,15 +159,15 @@ module Gitlab end end - class ObjectUploadError < StandardError + class ObjectUploadError < HttpError def message - "Failed to upload object" + "Failed to upload object: #{http_error}" end end - class ObjectVerifyError < StandardError + class ObjectVerifyError < HttpError def message - "Failed to verify object" + "Failed to verify object: #{http_error}" end end end diff --git a/lib/gitlab/logger.rb b/lib/gitlab/logger.rb index 89a4e36a232..53ad0d9cb4d 100644 --- a/lib/gitlab/logger.rb +++ b/lib/gitlab/logger.rb @@ -33,7 +33,11 @@ module Gitlab def self.build Gitlab::SafeRequestStore[self.cache_key] ||= - new(self.full_log_path, level: ::Logger::DEBUG) + new(self.full_log_path, level: log_level) + end + + def self.log_level(fallback: ::Logger::DEBUG) + ENV.fetch('GITLAB_LOG_LEVEL', fallback) end def self.full_log_path diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb index 75d27ed8cc1..e93a297cee4 100644 --- a/lib/gitlab/mail_room.rb +++ b/lib/gitlab/mail_room.rb @@ -25,7 +25,7 @@ module Gitlab # Email specific configuration which is merged with configuration # fetched from YML config file. - ADDRESS_SPECIFIC_CONFIG = { + MAILBOX_SPECIFIC_CONFIGS = { incoming_email: { queue: 'email_receiver', worker: 'EmailReceiverWorker' @@ -38,7 +38,15 @@ module Gitlab class << self def enabled_configs - @enabled_configs ||= configs.select { |config| enabled?(config) } + @enabled_configs ||= configs.select { |_key, config| enabled?(config) } + end + + def enabled_mailbox_types + enabled_configs.keys.map(&:to_s) + end + + def worker_for(mailbox_type) + MAILBOX_SPECIFIC_CONFIGS.try(:[], mailbox_type.to_sym).try(:[], :worker).try(:safe_constantize) end private @@ -48,7 +56,7 @@ module Gitlab end def configs - ADDRESS_SPECIFIC_CONFIG.keys.map { |key| fetch_config(key) } + MAILBOX_SPECIFIC_CONFIGS.to_h { |key, _value| [key, fetch_config(key)] } end def fetch_config(config_key) @@ -63,7 +71,7 @@ module Gitlab def merged_configs(config_key) yml_config = load_yaml.fetch(config_key, {}) - specific_config = ADDRESS_SPECIFIC_CONFIG.fetch(config_key, {}) + specific_config = MAILBOX_SPECIFIC_CONFIGS.fetch(config_key, {}) DEFAULT_CONFIG.merge(specific_config, yml_config) do |_key, oldval, newval| newval.nil? ? oldval : newval end diff --git a/lib/gitlab/mail_room/authenticator.rb b/lib/gitlab/mail_room/authenticator.rb new file mode 100644 index 00000000000..26ebdca8beb --- /dev/null +++ b/lib/gitlab/mail_room/authenticator.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + module MailRoom + class Authenticator + include JwtAuthenticatable + + SecretConfigurationError = Class.new(StandardError) + INTERNAL_API_REQUEST_HEADER = 'Gitlab-Mailroom-Api-Request' + INTERNAL_API_REQUEST_JWT_ISSUER = 'gitlab-mailroom' + + # Only allow token generated within the last 5 minutes + EXPIRATION = 5.minutes + + class << self + def verify_api_request(request_headers, mailbox_type) + mailbox_type = mailbox_type.to_sym + return false if enabled_configs[mailbox_type].blank? + + decode_jwt( + request_headers[INTERNAL_API_REQUEST_HEADER], + secret(mailbox_type), + issuer: INTERNAL_API_REQUEST_JWT_ISSUER, iat_after: Time.current - EXPIRATION + ) + rescue JWT::DecodeError => e + ::Gitlab::AppLogger.warn("Fail to decode MailRoom JWT token: #{e.message}") if Rails.env.development? + + false + end + + def secret(mailbox_type) + strong_memoize("jwt_secret_#{mailbox_type}".to_sym) do + secret_path = enabled_configs[mailbox_type][:secret_file] + raise SecretConfigurationError, "#{mailbox_type}'s secret_file configuration is missing" if secret_path.blank? + + begin + read_secret(secret_path) + rescue StandardError => e + raise SecretConfigurationError, "Fail to read #{mailbox_type}'s secret: #{e.message}" + end + end + end + + def enabled_configs + Gitlab::MailRoom.enabled_configs + end + end + end + end +end diff --git a/lib/gitlab/merge_requests/commit_message_generator.rb b/lib/gitlab/merge_requests/commit_message_generator.rb index 0e9ec6f5cb3..0515c17fe5d 100644 --- a/lib/gitlab/merge_requests/commit_message_generator.rb +++ b/lib/gitlab/merge_requests/commit_message_generator.rb @@ -2,8 +2,9 @@ module Gitlab module MergeRequests class CommitMessageGenerator - def initialize(merge_request:) + def initialize(merge_request:, current_user:) @merge_request = merge_request + @current_user = @merge_request.metrics&.merged_by || @merge_request.merge_user || current_user end def merge_message @@ -15,57 +16,66 @@ module Gitlab def squash_message return unless @merge_request.target_project.squash_commit_template.present? - replace_placeholders(@merge_request.target_project.squash_commit_template) + replace_placeholders(@merge_request.target_project.squash_commit_template, squash: true) end private attr_reader :merge_request + attr_reader :current_user PLACEHOLDERS = { - 'source_branch' => ->(merge_request) { merge_request.source_branch.to_s }, - 'target_branch' => ->(merge_request) { merge_request.target_branch.to_s }, - 'title' => ->(merge_request) { merge_request.title }, - 'issues' => ->(merge_request) do - return "" if merge_request.visible_closing_issues_for.blank? + 'source_branch' => ->(merge_request, _, _) { merge_request.source_branch.to_s }, + 'target_branch' => ->(merge_request, _, _) { merge_request.target_branch.to_s }, + 'title' => ->(merge_request, _, _) { merge_request.title }, + 'issues' => ->(merge_request, _, _) do + return if merge_request.visible_closing_issues_for.blank? closes_issues_references = merge_request.visible_closing_issues_for.map do |issue| issue.to_reference(merge_request.target_project) end "Closes #{closes_issues_references.to_sentence}" end, - 'description' => ->(merge_request) { merge_request.description.presence || '' }, - 'reference' => ->(merge_request) { merge_request.to_reference(full: true) }, - 'first_commit' => -> (merge_request) { merge_request.first_commit&.safe_message&.strip.presence || '' }, - 'first_multiline_commit' => -> (merge_request) { merge_request.first_multiline_commit&.safe_message&.strip.presence || merge_request.title } + 'description' => ->(merge_request, _, _) { merge_request.description }, + 'reference' => ->(merge_request, _, _) { merge_request.to_reference(full: true) }, + 'first_commit' => -> (merge_request, _, _) { merge_request.first_commit&.safe_message&.strip }, + 'first_multiline_commit' => -> (merge_request, _, _) { merge_request.first_multiline_commit&.safe_message&.strip.presence || merge_request.title }, + 'url' => ->(merge_request, _, _) { Gitlab::UrlBuilder.build(merge_request) }, + 'approved_by' => ->(merge_request, _, _) { merge_request.approved_by_users.map { |user| "Approved-by: #{user.name} <#{user.commit_email_or_default}>" }.join("\n") }, + 'merged_by' => ->(_, user, _) { "#{user&.name} <#{user&.commit_email_or_default}>" }, + 'co_authored_by' => ->(merge_request, merged_by, squash) do + commit_author = squash ? merge_request.author : merged_by + merge_request.recent_commits + .to_h { |commit| [commit.author_email, commit.author_name] } + .except(commit_author&.commit_email_or_default) + .map { |author_email, author_name| "Co-authored-by: #{author_name} <#{author_email}>" } + .join("\n") + end }.freeze - PLACEHOLDERS_REGEX = Regexp.union(PLACEHOLDERS.keys.map do |key| - Regexp.new(Regexp.escape(key)) - end).freeze - - BLANK_PLACEHOLDERS_REGEXES = (PLACEHOLDERS.map do |key, value| - [key, Regexp.new("[\n\r]+%{#{Regexp.escape(key)}}$")] - end).to_h.freeze + PLACEHOLDERS_COMBINED_REGEX = /%{(#{Regexp.union(PLACEHOLDERS.keys)})}/.freeze - def replace_placeholders(message) - # convert CRLF to LF + def replace_placeholders(message, squash: false) + # Convert CRLF to LF. message = message.delete("\r") - # Remove placeholders that correspond to empty values and are the last word in the line - # along with all whitespace characters preceding them. - # This allows us to recreate previous default merge commit message behaviour - we skipped new line character - # before empty description and before closed issues when none were present. - PLACEHOLDERS.each do |key, value| - unless value.call(merge_request).present? - message = message.gsub(BLANK_PLACEHOLDERS_REGEXES[key], '') - end + used_variables = message.scan(PLACEHOLDERS_COMBINED_REGEX).map { |value| value[0] }.uniq + values = used_variables.to_h do |variable_name| + ["%{#{variable_name}}", PLACEHOLDERS[variable_name].call(merge_request, current_user, squash)] end + names_of_empty_variables = values.filter_map { |name, value| name if value.blank? } - Gitlab::StringPlaceholderReplacer - .replace_string_placeholders(message, PLACEHOLDERS_REGEX) do |key| - PLACEHOLDERS[key].call(merge_request) + # Remove lines that contain empty variable placeholder and nothing else. + if names_of_empty_variables.present? + # If there is blank line or EOF after it, remove blank line before it as well. + message = message.gsub(/\n\n#{Regexp.union(names_of_empty_variables)}(\n\n|\Z)/, '\1') + # Otherwise, remove only the line it is in. + message = message.gsub(/^#{Regexp.union(names_of_empty_variables)}\n/, '') end + # Substitute all variables with their values. + message = message.gsub(Regexp.union(values.keys), values) if values.present? + + message end end end diff --git a/lib/gitlab/metrics/exporter/base_exporter.rb b/lib/gitlab/metrics/exporter/base_exporter.rb index 47c862c0232..190d3d3fd2f 100644 --- a/lib/gitlab/metrics/exporter/base_exporter.rb +++ b/lib/gitlab/metrics/exporter/base_exporter.rb @@ -11,44 +11,41 @@ module Gitlab attr_accessor :readiness_checks - def initialize(settings, **options) + def initialize(settings, log_enabled:, log_file:, gc_requests: false, **options) super(**options) @settings = settings + @gc_requests = gc_requests + + # log_enabled does not exist for all exporters + log_sink = log_enabled ? File.join(Rails.root, 'log', log_file) : File::NULL + @logger = WEBrick::Log.new(log_sink) + @logger.time_format = "[%Y-%m-%dT%H:%M:%S.%L%z]" end def enabled? settings.enabled end - def log_filename - raise NotImplementedError - end - private - attr_reader :settings + attr_reader :settings, :logger def start_working - logger = WEBrick::Log.new(log_filename) - logger.time_format = "[%Y-%m-%dT%H:%M:%S.%L%z]" - access_log = [ [logger, WEBrick::AccessLog::COMBINED_LOG_FORMAT] ] @server = ::WEBrick::HTTPServer.new( Port: settings.port, BindAddress: settings.address, - Logger: logger, AccessLog: access_log) - server.mount_proc '/readiness' do |req, res| - render_probe(readiness_probe, req, res) - end - server.mount_proc '/liveness' do |req, res| - render_probe(liveness_probe, req, res) - end + Logger: logger, AccessLog: access_log + ) server.mount '/', Rack::Handler::WEBrick, rack_app true + rescue StandardError => e + logger.error(e) + false end def run_thread @@ -72,8 +69,16 @@ module Gitlab end def rack_app + readiness = readiness_probe + liveness = liveness_probe + pid = thread_name + gc_requests = @gc_requests + Rack::Builder.app do use Rack::Deflater + use Gitlab::Metrics::Exporter::MetricsMiddleware, pid + use Gitlab::Metrics::Exporter::HealthChecksMiddleware, readiness, liveness + use Gitlab::Metrics::Exporter::GcRequestMiddleware if gc_requests use ::Prometheus::Client::Rack::Exporter if ::Gitlab::Metrics.metrics_folder_present? run -> (env) { [404, {}, ['']] } end @@ -86,14 +91,6 @@ module Gitlab def liveness_probe ::Gitlab::HealthChecks::Probes::Collection.new end - - def render_probe(probe, req, res) - result = probe.execute - - res.status = result.http_status - res.content_type = 'application/json; charset=utf-8' - res.body = result.json.to_json - end end end end diff --git a/lib/gitlab/metrics/exporter/gc_request_middleware.rb b/lib/gitlab/metrics/exporter/gc_request_middleware.rb new file mode 100644 index 00000000000..3806b0e2bd1 --- /dev/null +++ b/lib/gitlab/metrics/exporter/gc_request_middleware.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Exporter + class GcRequestMiddleware + def initialize(app) + @app = app + end + + def call(env) + @app.call(env).tap do + GC.start + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/exporter/health_checks_middleware.rb b/lib/gitlab/metrics/exporter/health_checks_middleware.rb new file mode 100644 index 00000000000..c43b8004b72 --- /dev/null +++ b/lib/gitlab/metrics/exporter/health_checks_middleware.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Exporter + class HealthChecksMiddleware + def initialize(app, readiness_probe, liveness_probe) + @app = app + @readiness_probe = readiness_probe + @liveness_probe = liveness_probe + end + + def call(env) + case env['PATH_INFO'] + when '/readiness' then render_probe(@readiness_probe) + when '/liveness' then render_probe(@liveness_probe) + else @app.call(env) + end + end + + private + + def render_probe(probe) + result = probe.execute + + [ + result.http_status, + { 'Content-Type' => 'application/json; charset=utf-8' }, + [result.json.to_json] + ] + end + end + end + end +end diff --git a/lib/gitlab/metrics/exporter/metrics_middleware.rb b/lib/gitlab/metrics/exporter/metrics_middleware.rb new file mode 100644 index 00000000000..e17f1c13cf0 --- /dev/null +++ b/lib/gitlab/metrics/exporter/metrics_middleware.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Exporter + class MetricsMiddleware + def initialize(app, pid) + @app = app + default_labels = { + pid: pid + } + @requests_total = Gitlab::Metrics.counter( + :exporter_http_requests_total, 'Total number of HTTP requests', default_labels + ) + @request_durations = Gitlab::Metrics.histogram( + :exporter_http_request_duration_seconds, + 'HTTP request duration histogram (seconds)', + default_labels, + [0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10] + ) + end + + def call(env) + start = Gitlab::Metrics::System.monotonic_time + @app.call(env).tap do |response| + duration = Gitlab::Metrics::System.monotonic_time - start + + labels = { + method: env['REQUEST_METHOD'].downcase, + path: env['PATH_INFO'].to_s, + code: response.first.to_s + } + + @requests_total.increment(labels) + @request_durations.observe(labels, duration) + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/exporter/sidekiq_exporter.rb b/lib/gitlab/metrics/exporter/sidekiq_exporter.rb index eea71fda6a0..afecf6546f8 100644 --- a/lib/gitlab/metrics/exporter/sidekiq_exporter.rb +++ b/lib/gitlab/metrics/exporter/sidekiq_exporter.rb @@ -4,12 +4,11 @@ module Gitlab module Metrics module Exporter class SidekiqExporter < BaseExporter - def log_filename - if settings['log_enabled'] - File.join(Rails.root, 'log', 'sidekiq_exporter.log') - else - File::NULL - end + def initialize(settings, **options) + super(settings, + log_enabled: settings['log_enabled'], + log_file: 'sidekiq_exporter.log', + **options) end end end diff --git a/lib/gitlab/metrics/exporter/web_exporter.rb b/lib/gitlab/metrics/exporter/web_exporter.rb index d41484aaaa7..c05ad8ccf42 100644 --- a/lib/gitlab/metrics/exporter/web_exporter.rb +++ b/lib/gitlab/metrics/exporter/web_exporter.rb @@ -26,8 +26,8 @@ module Gitlab attr_reader :running # This exporter is always run on master process - def initialize - super(Settings.monitoring.web_exporter) + def initialize(**options) + super(Settings.monitoring.web_exporter, log_enabled: true, log_file: 'web_exporter.log', **options) # DEPRECATED: # these `readiness_checks` are deprecated @@ -39,10 +39,6 @@ module Gitlab ] end - def log_filename - File.join(Rails.root, 'log', 'web_exporter.log') - end - def mark_as_not_running! @running = false end diff --git a/lib/gitlab/metrics/samplers/action_cable_sampler.rb b/lib/gitlab/metrics/samplers/action_cable_sampler.rb index adce3030d0d..1f50371cae9 100644 --- a/lib/gitlab/metrics/samplers/action_cable_sampler.rb +++ b/lib/gitlab/metrics/samplers/action_cable_sampler.rb @@ -6,8 +6,8 @@ module Gitlab class ActionCableSampler < BaseSampler DEFAULT_SAMPLING_INTERVAL_SECONDS = 5 - def initialize(interval = nil, action_cable: ::ActionCable.server) - super(interval) + def initialize(action_cable: ::ActionCable.server, **options) + super(**options) @action_cable = action_cable end diff --git a/lib/gitlab/metrics/samplers/base_sampler.rb b/lib/gitlab/metrics/samplers/base_sampler.rb index 52d80c3c27e..b2a9de21145 100644 --- a/lib/gitlab/metrics/samplers/base_sampler.rb +++ b/lib/gitlab/metrics/samplers/base_sampler.rb @@ -9,7 +9,10 @@ module Gitlab attr_reader :interval # interval - The sampling interval in seconds. - def initialize(interval = nil) + # warmup - When true, takes a single sample eagerly before entering the sampling loop. + # This can be useful to ensure that all metrics files exist after `start` returns, + # since prometheus-client-mmap creates them lazily upon first access. + def initialize(interval: nil, logger: Logger.new($stdout), warmup: false, **options) interval ||= ENV[interval_env_key]&.to_i interval ||= self.class::DEFAULT_SAMPLING_INTERVAL_SECONDS interval_half = interval.to_f / 2 @@ -17,13 +20,16 @@ module Gitlab @interval = interval @interval_steps = (-interval_half..interval_half).step(0.1).to_a - super() + @logger = logger + @warmup = warmup + + super(**options) end def safe_sample sample rescue StandardError => e - ::Gitlab::AppLogger.warn("#{self.class}: #{e}, stopping") + @logger.warn("#{self.class}: #{e}, stopping") stop end @@ -63,6 +69,8 @@ module Gitlab def start_working @running = true + safe_sample if @warmup + true end diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index b1c5e9800da..d71ee671b8d 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -7,12 +7,12 @@ module Gitlab DEFAULT_SAMPLING_INTERVAL_SECONDS = 60 GC_REPORT_BUCKETS = [0.01, 0.05, 0.1, 0.2, 0.3, 0.5, 1].freeze - def initialize(*) + def initialize(...) GC::Profiler.clear metrics[:process_start_time_seconds].set(labels, Time.now.to_i) - super + super(...) end def metrics diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb index a047015e54f..0b7b5e23b75 100644 --- a/lib/gitlab/middleware/multipart.rb +++ b/lib/gitlab/middleware/multipart.rb @@ -116,7 +116,7 @@ module Gitlab jwt_token = params[param_key] raise "Empty JWT param: #{param_key}" if jwt_token.blank? - payload = Gitlab::Workhorse.decode_jwt(jwt_token).first + payload = Gitlab::Workhorse.decode_jwt_with_issuer(jwt_token).first raise "Invalid JWT payload: not a Hash" unless payload.is_a?(Hash) upload_params = payload.fetch(JWT_PARAM_FIXED_KEY, {}) @@ -172,7 +172,7 @@ module Gitlab encoded_message = env.delete(RACK_ENV_KEY) return @app.call(env) if encoded_message.blank? - message = ::Gitlab::Workhorse.decode_jwt(encoded_message)[0] + message = ::Gitlab::Workhorse.decode_jwt_with_issuer(encoded_message)[0] ::Gitlab::Middleware::Multipart::Handler.new(env, message).with_open_files do @app.call(env) diff --git a/lib/gitlab/middleware/webhook_recursion_detection.rb b/lib/gitlab/middleware/webhook_recursion_detection.rb new file mode 100644 index 00000000000..2677445852c --- /dev/null +++ b/lib/gitlab/middleware/webhook_recursion_detection.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Middleware + class WebhookRecursionDetection + def initialize(app) + @app = app + end + + def call(env) + headers = ActionDispatch::Request.new(env).headers + + ::Gitlab::WebHooks::RecursionDetection.set_from_headers(headers) + + @app.call(env) + end + end + end +end diff --git a/lib/gitlab/pages.rb b/lib/gitlab/pages.rb index 98e87e9e915..4e0e5102bec 100644 --- a/lib/gitlab/pages.rb +++ b/lib/gitlab/pages.rb @@ -10,7 +10,7 @@ module Gitlab class << self def verify_api_request(request_headers) - decode_jwt_for_issuer('gitlab-pages', request_headers[INTERNAL_API_REQUEST_HEADER]) + decode_jwt(request_headers[INTERNAL_API_REQUEST_HEADER], issuer: 'gitlab-pages') rescue JWT::DecodeError false end diff --git a/lib/gitlab/pagination/keyset/column_order_definition.rb b/lib/gitlab/pagination/keyset/column_order_definition.rb index 2b968c4253f..302e7b406b1 100644 --- a/lib/gitlab/pagination/keyset/column_order_definition.rb +++ b/lib/gitlab/pagination/keyset/column_order_definition.rb @@ -114,6 +114,20 @@ module Gitlab # - When the order is a calculated expression or the column is in another table (JOIN-ed) # # If the add_to_projections is true, the query builder will automatically add the column to the SELECT values + # + # **sql_type** + # + # The SQL type of the column or SQL expression. This is an optional field which is only required when using the + # column with the InOperatorOptimization class. + # + # Example: When the order expression is a calculated SQL expression. + # + # { + # attribute_name: 'id_times_count', + # order_expression: Arel.sql('(id * count)').asc, + # sql_type: 'integer' # the SQL type here must match with the type of the produced data by the order_expression. Putting 'text' here would be incorrect. + # } + # class ColumnOrderDefinition REVERSED_ORDER_DIRECTIONS = { asc: :desc, desc: :asc }.freeze REVERSED_NULL_POSITIONS = { nulls_first: :nulls_last, nulls_last: :nulls_first }.freeze @@ -122,7 +136,8 @@ module Gitlab attr_reader :attribute_name, :column_expression, :order_expression, :add_to_projections, :order_direction - def initialize(attribute_name:, order_expression:, column_expression: nil, reversed_order_expression: nil, nullable: :not_nullable, distinct: true, order_direction: nil, add_to_projections: false) + # rubocop: disable Metrics/ParameterLists + def initialize(attribute_name:, order_expression:, column_expression: nil, reversed_order_expression: nil, nullable: :not_nullable, distinct: true, order_direction: nil, sql_type: nil, add_to_projections: false) @attribute_name = attribute_name @order_expression = order_expression @column_expression = column_expression || calculate_column_expression(order_expression) @@ -130,8 +145,10 @@ module Gitlab @reversed_order_expression = reversed_order_expression || calculate_reversed_order(order_expression) @nullable = parse_nullable(nullable, distinct) @order_direction = parse_order_direction(order_expression, order_direction) + @sql_type = sql_type @add_to_projections = add_to_projections end + # rubocop: enable Metrics/ParameterLists def reverse self.class.new( @@ -185,6 +202,12 @@ module Gitlab sql_string end + def sql_type + raise Gitlab::Pagination::Keyset::SqlTypeMissingError.for_column(self) if @sql_type.nil? + + @sql_type + end + private attr_reader :reversed_order_expression, :nullable, :distinct diff --git a/lib/gitlab/pagination/keyset/in_operator_optimization/column_data.rb b/lib/gitlab/pagination/keyset/in_operator_optimization/column_data.rb index 3f620f74eca..93b28661bb0 100644 --- a/lib/gitlab/pagination/keyset/in_operator_optimization/column_data.rb +++ b/lib/gitlab/pagination/keyset/in_operator_optimization/column_data.rb @@ -4,23 +4,35 @@ module Gitlab module Pagination module Keyset module InOperatorOptimization + # This class is used for wrapping an Arel column with + # convenient helper methods in order to make the query + # building for the InOperatorOptimization a bit cleaner. class ColumnData attr_reader :original_column_name, :as, :arel_table - def initialize(original_column_name, as, arel_table) - @original_column_name = original_column_name.to_s + # column - name of the DB column + # as - custom alias for the column + # arel_table - relation where the column is located + def initialize(column, as, arel_table) + @original_column_name = column @as = as.to_s @arel_table = arel_table end + # Generates: `issues.name AS my_alias` def projection arel_column.as(as) end + # Generates: issues.name` def arel_column arel_table[original_column_name] end + # overridden in OrderByColumnData class + alias_method :column_expression, :arel_column + + # Generates: `issues.my_alias` def arel_column_as arel_table[as] end @@ -29,8 +41,9 @@ module Gitlab "#{arel_table.name}_#{original_column_name}_array" end + # Generates: SELECT ARRAY_AGG(...) AS issues_name_array def array_aggregated_column - Arel::Nodes::NamedFunction.new('ARRAY_AGG', [arel_column]).as(array_aggregated_column_name) + Arel::Nodes::NamedFunction.new('ARRAY_AGG', [column_expression]).as(array_aggregated_column_name) end end end diff --git a/lib/gitlab/pagination/keyset/in_operator_optimization/order_by_column_data.rb b/lib/gitlab/pagination/keyset/in_operator_optimization/order_by_column_data.rb new file mode 100644 index 00000000000..9cb1ba1542d --- /dev/null +++ b/lib/gitlab/pagination/keyset/in_operator_optimization/order_by_column_data.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module Pagination + module Keyset + module InOperatorOptimization + class OrderByColumnData < ColumnData + extend ::Gitlab::Utils::Override + + attr_reader :column + + # column - a ColumnOrderDefinition object + # as - custom alias for the column + # arel_table - relation where the column is located + def initialize(column, as, arel_table) + super(column.attribute_name.to_s, as, arel_table) + @column = column + end + + override :arel_column + def arel_column + column.column_expression + end + + override :column_expression + def column_expression + arel_table[original_column_name] + end + + def column_for_projection + column.column_expression.as(original_column_name) + end + end + end + end + end +end diff --git a/lib/gitlab/pagination/keyset/in_operator_optimization/order_by_columns.rb b/lib/gitlab/pagination/keyset/in_operator_optimization/order_by_columns.rb index d8c69a74e6b..d6513114d08 100644 --- a/lib/gitlab/pagination/keyset/in_operator_optimization/order_by_columns.rb +++ b/lib/gitlab/pagination/keyset/in_operator_optimization/order_by_columns.rb @@ -9,16 +9,16 @@ module Gitlab # This class exposes collection methods for the order by columns # - # Example: by modelling the `issues.created_at ASC, issues.id ASC` ORDER BY + # Example: by modeling the `issues.created_at ASC, issues.id ASC` ORDER BY # SQL clause, this class will receive two ColumnOrderDefinition objects def initialize(columns, arel_table) @columns = columns.map do |column| - ColumnData.new(column.attribute_name, "order_by_columns_#{column.attribute_name}", arel_table) + OrderByColumnData.new(column, "order_by_columns_#{column.attribute_name}", arel_table) end end def arel_columns - columns.map(&:arel_column) + columns.map(&:column_for_projection) end def array_aggregated_columns diff --git a/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder.rb b/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder.rb index 53faf8469f2..065a3a0cf20 100644 --- a/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder.rb +++ b/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder.rb @@ -120,7 +120,7 @@ module Gitlab .from(array_cte) .join(Arel.sql("LEFT JOIN LATERAL (#{initial_keyset_query.to_sql}) #{table_name} ON TRUE")) - order_by_columns.each { |column| q.where(column.arel_column.not_eq(nil)) } + order_by_columns.each { |column| q.where(column.column_expression.not_eq(nil)) } q.as('array_scope_lateral_query') end @@ -231,7 +231,7 @@ module Gitlab order .apply_cursor_conditions(keyset_scope, cursor_values, use_union_optimization: true) - .reselect(*order_by_columns.arel_columns) + .reselect(*order_by_columns.map(&:column_for_projection)) .limit(1) end diff --git a/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/order_values_loader_strategy.rb b/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/order_values_loader_strategy.rb index fc2b56048f6..932aa0c2d28 100644 --- a/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/order_values_loader_strategy.rb +++ b/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/order_values_loader_strategy.rb @@ -12,11 +12,7 @@ module Gitlab end def initializer_columns - order_by_columns.map do |column| - column_name = column.original_column_name.to_s - type = model.columns_hash[column_name].sql_type - "NULL::#{type} AS #{column_name}" - end + order_by_columns.map { |column_data| null_with_type_cast(column_data) } end def columns @@ -30,6 +26,15 @@ module Gitlab private attr_reader :model, :order_by_columns + + def null_with_type_cast(column_data) + column_name = column_data.original_column_name.to_s + active_record_column = model.columns_hash[column_name] + + type = active_record_column ? active_record_column.sql_type : column_data.column.sql_type + + "NULL::#{type} AS #{column_name}" + end end end end diff --git a/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy.rb b/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy.rb index b12c33d6e51..51f38c1da58 100644 --- a/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy.rb +++ b/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy.rb @@ -9,6 +9,8 @@ module Gitlab RECORDS_COLUMN = 'records' def initialize(finder_query, model, order_by_columns) + verify_order_by_attributes_on_model!(model, order_by_columns) + @finder_query = finder_query @order_by_columns = order_by_columns @table_name = model.table_name @@ -34,6 +36,20 @@ module Gitlab private attr_reader :finder_query, :order_by_columns, :table_name + + def verify_order_by_attributes_on_model!(model, order_by_columns) + order_by_columns.map(&:column).each do |column| + unless model.columns_hash[column.attribute_name.to_s] + text = <<~TEXT + The "RecordLoaderStrategy" does not support the following ORDER BY column because + it's not available on the \"#{model.table_name}\" table: #{column.attribute_name} + + Omit the "finder_query" parameter to use the "OrderValuesLoaderStrategy". + TEXT + raise text + end + end + end end end end diff --git a/lib/gitlab/pagination/keyset/sql_type_missing_error.rb b/lib/gitlab/pagination/keyset/sql_type_missing_error.rb new file mode 100644 index 00000000000..0525ae13e9c --- /dev/null +++ b/lib/gitlab/pagination/keyset/sql_type_missing_error.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true +module Gitlab + module Pagination + module Keyset + class SqlTypeMissingError < StandardError + def self.for_column(column) + message = <<~TEXT + The "sql_type" attribute is not set for the following column definition: + #{column.attribute_name} + + See the ColumnOrderDefinition class for more context. + TEXT + + new(message) + end + end + end + end +end diff --git a/lib/gitlab/password.rb b/lib/gitlab/password.rb new file mode 100644 index 00000000000..00aef8754d6 --- /dev/null +++ b/lib/gitlab/password.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +# This module is used to return fake strong password for tests + +module Gitlab + module Password + DEFAULT_LENGTH = 12 + TEST_DEFAULT = "123qweQWE!@#" + "0" * (User.password_length.max - DEFAULT_LENGTH) + def self.test_default(length = 12) + password_length = [[User.password_length.min, length].max, User.password_length.max].min + TEST_DEFAULT[...password_length] + end + end +end diff --git a/lib/gitlab/redis/multi_store.rb b/lib/gitlab/redis/multi_store.rb deleted file mode 100644 index 500b62bf0e8..00000000000 --- a/lib/gitlab/redis/multi_store.rb +++ /dev/null @@ -1,229 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Redis - class MultiStore - include Gitlab::Utils::StrongMemoize - - class ReadFromPrimaryError < StandardError - def message - 'Value not found on the redis primary store. Read from the redis secondary store successful.' - end - end - class MethodMissingError < StandardError - def message - 'Method missing. Falling back to execute method on the redis secondary store.' - end - end - - attr_reader :primary_store, :secondary_store, :instance_name - - FAILED_TO_READ_ERROR_MESSAGE = 'Failed to read from the redis primary_store.' - FAILED_TO_WRITE_ERROR_MESSAGE = 'Failed to write to the redis primary_store.' - - SKIP_LOG_METHOD_MISSING_FOR_COMMANDS = %i(info).freeze - - READ_COMMANDS = %i( - get - mget - smembers - scard - ).freeze - - WRITE_COMMANDS = %i( - set - setnx - setex - sadd - srem - del - pipelined - flushdb - ).freeze - - def initialize(primary_store, secondary_store, instance_name) - @primary_store = primary_store - @secondary_store = secondary_store - @instance_name = instance_name - - validate_stores! - end - # rubocop:disable GitlabSecurity/PublicSend - READ_COMMANDS.each do |name| - define_method(name) do |*args, &block| - if use_primary_and_secondary_stores? - read_command(name, *args, &block) - else - default_store.send(name, *args, &block) - end - end - end - - WRITE_COMMANDS.each do |name| - define_method(name) do |*args, &block| - if use_primary_and_secondary_stores? - write_command(name, *args, &block) - else - default_store.send(name, *args, &block) - end - end - end - - def method_missing(...) - return @instance.send(...) if @instance - - log_method_missing(...) - - default_store.send(...) - end - # rubocop:enable GitlabSecurity/PublicSend - - def respond_to_missing?(command_name, include_private = false) - true - end - - # This is needed because of Redis::Rack::Connection is requiring Redis::Store - # https://github.com/redis-store/redis-rack/blob/a833086ba494083b6a384a1a4e58b36573a9165d/lib/redis/rack/connection.rb#L15 - # Done similarly in https://github.com/lsegal/yard/blob/main/lib/yard/templates/template.rb#L122 - def is_a?(klass) - return true if klass == default_store.class - - super(klass) - end - alias_method :kind_of?, :is_a? - - def to_s - use_primary_and_secondary_stores? ? primary_store.to_s : default_store.to_s - end - - def use_primary_and_secondary_stores? - feature_flags_available? && - Feature.enabled?("use_primary_and_secondary_stores_for_#{instance_name.underscore}", default_enabled: :yaml) && - !same_redis_store? - end - - def use_primary_store_as_default? - feature_flags_available? && - Feature.enabled?("use_primary_store_as_default_for_#{instance_name.underscore}", default_enabled: :yaml) && - !same_redis_store? - end - - private - - def default_store - use_primary_store_as_default? ? primary_store : secondary_store - end - - def log_method_missing(command_name, *_args) - return if SKIP_LOG_METHOD_MISSING_FOR_COMMANDS.include?(command_name) - - log_error(MethodMissingError.new, command_name) - increment_method_missing_count(command_name) - end - - def read_command(command_name, *args, &block) - if @instance - send_command(@instance, command_name, *args, &block) - else - read_one_with_fallback(command_name, *args, &block) - end - end - - def write_command(command_name, *args, &block) - if @instance - send_command(@instance, command_name, *args, &block) - else - write_both(command_name, *args, &block) - end - end - - def read_one_with_fallback(command_name, *args, &block) - begin - value = send_command(primary_store, command_name, *args, &block) - rescue StandardError => e - log_error(e, command_name, - multi_store_error_message: FAILED_TO_READ_ERROR_MESSAGE) - end - - value ||= fallback_read(command_name, *args, &block) - - value - end - - def fallback_read(command_name, *args, &block) - value = send_command(secondary_store, command_name, *args, &block) - - if value - log_error(ReadFromPrimaryError.new, command_name) - increment_read_fallback_count(command_name) - end - - value - end - - def write_both(command_name, *args, &block) - begin - send_command(primary_store, command_name, *args, &block) - rescue StandardError => e - log_error(e, command_name, - multi_store_error_message: FAILED_TO_WRITE_ERROR_MESSAGE) - end - - send_command(secondary_store, command_name, *args, &block) - end - - def same_redis_store? - strong_memoize(:same_redis_store) do - # " - primary_store.inspect == secondary_store.inspect - end - end - - # rubocop:disable GitlabSecurity/PublicSend - def send_command(redis_instance, command_name, *args, &block) - if block_given? - # Make sure that block is wrapped and executed only on the redis instance that is executing the block - redis_instance.send(command_name, *args) do |*params| - with_instance(redis_instance, *params, &block) - end - else - redis_instance.send(command_name, *args) - end - end - # rubocop:enable GitlabSecurity/PublicSend - - def with_instance(instance, *params) - @instance = instance - - yield(*params) - ensure - @instance = nil - end - - def increment_read_fallback_count(command_name) - @read_fallback_counter ||= Gitlab::Metrics.counter(:gitlab_redis_multi_store_read_fallback_total, 'Client side Redis MultiStore reading fallback') - @read_fallback_counter.increment(command: command_name, instance_name: instance_name) - end - - def increment_method_missing_count(command_name) - @method_missing_counter ||= Gitlab::Metrics.counter(:gitlab_redis_multi_store_method_missing_total, 'Client side Redis MultiStore method missing') - @method_missing_counter.increment(command: command_name, instance_name: instance_name) - end - - def validate_stores! - raise ArgumentError, 'primary_store is required' unless primary_store - raise ArgumentError, 'secondary_store is required' unless secondary_store - raise ArgumentError, 'instance_name is required' unless instance_name - raise ArgumentError, 'invalid primary_store' unless primary_store.is_a?(::Redis) - raise ArgumentError, 'invalid secondary_store' unless secondary_store.is_a?(::Redis) - end - - def log_error(exception, command_name, extra = {}) - Gitlab::ErrorTracking.log_exception( - exception, - command_name: command_name, - extra: extra.merge(instance_name: instance_name)) - end - end - end -end diff --git a/lib/gitlab/redis/sessions.rb b/lib/gitlab/redis/sessions.rb index c547828d907..ddcfdf6e798 100644 --- a/lib/gitlab/redis/sessions.rb +++ b/lib/gitlab/redis/sessions.rb @@ -9,39 +9,9 @@ module Gitlab IP_SESSIONS_LOOKUP_NAMESPACE = 'session:lookup:ip:gitlab2' OTP_SESSIONS_NAMESPACE = 'session:otp' - class << self - # The data we store on Sessions used to be stored on SharedState. - def config_fallback - SharedState - end - - private - - def redis - # Don't use multistore if redis.sessions configuration is not provided - return super if config_fallback? - - primary_store = ::Redis.new(params) - secondary_store = ::Redis.new(config_fallback.params) - - MultiStore.new(primary_store, secondary_store, store_name) - end - end - - def store(extras = {}) - # Don't use multistore if redis.sessions configuration is not provided - return super if self.class.config_fallback? - - primary_store = create_redis_store(redis_store_options, extras) - secondary_store = create_redis_store(self.class.config_fallback.params, extras) - - MultiStore.new(primary_store, secondary_store, self.class.store_name) - end - - private - - def create_redis_store(options, extras) - ::Redis::Store.new(options.merge(extras)) + # The data we store on Sessions used to be stored on SharedState. + def self.config_fallback + SharedState end end end diff --git a/lib/gitlab/redis/sessions_store_helper.rb b/lib/gitlab/redis/sessions_store_helper.rb deleted file mode 100644 index c80442847f1..00000000000 --- a/lib/gitlab/redis/sessions_store_helper.rb +++ /dev/null @@ -1,27 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Redis - module SessionsStoreHelper - extend ActiveSupport::Concern - - module StoreMethods - def redis_store_class - use_redis_session_store? ? Gitlab::Redis::Sessions : Gitlab::Redis::SharedState - end - - private - - def use_redis_session_store? - Gitlab::Utils.to_boolean(ENV['GITLAB_USE_REDIS_SESSIONS_STORE'], default: true) - end - end - - include StoreMethods - - included do - extend StoreMethods - end - end - end -end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 8e139ae0709..b07b9c79858 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -79,7 +79,11 @@ module Gitlab def nuget_version_regex @nuget_version_regex ||= / - \A#{_semver_major_minor_patch_regex}(\.\d*)?#{_semver_prerelease_build_regex}\z + \A#{_semver_major_regex} + \.#{_semver_minor_regex} + (\.#{_semver_patch_regex})? + (\.\d*)? + #{_semver_prerelease_build_regex}\z /x.freeze end @@ -167,9 +171,25 @@ module Gitlab # regexes rather than being used alone. def _semver_major_minor_patch_regex @_semver_major_minor_patch_regex ||= / + #{_semver_major_regex}\.#{_semver_minor_regex}\.#{_semver_patch_regex} + /x.freeze + end + + def _semver_major_regex + @_semver_major_regex ||= / (?0|[1-9]\d*) - \.(?0|[1-9]\d*) - \.(?0|[1-9]\d*) + /x.freeze + end + + def _semver_minor_regex + @_semver_minor_regex ||= / + (?0|[1-9]\d*) + /x.freeze + end + + def _semver_patch_regex + @_semver_patch_regex ||= / + (?0|[1-9]\d*) /x.freeze end diff --git a/lib/gitlab/repository_archive_rate_limiter.rb b/lib/gitlab/repository_archive_rate_limiter.rb index 31a3dc34bf6..d395b1aba7f 100644 --- a/lib/gitlab/repository_archive_rate_limiter.rb +++ b/lib/gitlab/repository_archive_rate_limiter.rb @@ -3,7 +3,7 @@ module Gitlab module RepositoryArchiveRateLimiter def check_archive_rate_limit!(current_user, project, &block) - return unless Feature.enabled?(:archive_rate_limit) + return unless Feature.enabled?(:archive_rate_limit, default_enabled: :yaml) threshold = current_user ? nil : 100 diff --git a/lib/gitlab/search/params.rb b/lib/gitlab/search/params.rb index e6a1305a82a..1ae14e5e618 100644 --- a/lib/gitlab/search/params.rb +++ b/lib/gitlab/search/params.rb @@ -7,6 +7,7 @@ module Gitlab SEARCH_CHAR_LIMIT = 4096 SEARCH_TERM_LIMIT = 64 + MIN_TERM_LENGTH = 3 # Generic validation validates :query_string, length: { maximum: SEARCH_CHAR_LIMIT } @@ -53,6 +54,10 @@ module Gitlab errors[:query_string].none? { |msg| msg.include? SEARCH_TERM_LIMIT.to_s } end + def email_lookup? + search_terms.any? { |term| term =~ URI::MailTo::EMAIL_REGEXP } + end + def validate if detect_abuse? abuse_detection.validate @@ -75,8 +80,12 @@ module Gitlab @detect_abuse end + def search_terms + @search_terms ||= query_string.split.select { |word| word.length >= MIN_TERM_LENGTH } + end + def not_too_many_terms - if query_string.split.count { |word| word.length >= 3 } > SEARCH_TERM_LIMIT + if search_terms.count > SEARCH_TERM_LIMIT errors.add :query_string, "has too many search terms (maximum is #{SEARCH_TERM_LIMIT})" end end diff --git a/lib/gitlab/sherlock.rb b/lib/gitlab/sherlock.rb deleted file mode 100644 index a1471c9de47..00000000000 --- a/lib/gitlab/sherlock.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -require 'securerandom' - -module Gitlab - module Sherlock - @collection = Collection.new - - class << self - attr_reader :collection - end - - def self.enabled? - Rails.env.development? && !!ENV['ENABLE_SHERLOCK'] - end - - def self.enable_line_profiler? - RUBY_ENGINE == 'ruby' - end - end -end diff --git a/lib/gitlab/sherlock/collection.rb b/lib/gitlab/sherlock/collection.rb deleted file mode 100644 index ce3a376cf75..00000000000 --- a/lib/gitlab/sherlock/collection.rb +++ /dev/null @@ -1,51 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Sherlock - # A collection of transactions recorded by Sherlock. - # - # Method calls for this class are synchronized using a mutex to allow - # sharing of a single Collection instance between threads (e.g. when using - # Puma as a webserver). - class Collection - include Enumerable - - def initialize - @transactions = [] - @mutex = Mutex.new - end - - def add(transaction) - synchronize { @transactions << transaction } - end - - alias_method :<<, :add - - def each(&block) - synchronize { @transactions.each(&block) } - end - - def clear - synchronize { @transactions.clear } - end - - def empty? - synchronize { @transactions.empty? } - end - - def find_transaction(id) - find { |trans| trans.id == id } - end - - def newest_first - sort { |a, b| b.finished_at <=> a.finished_at } - end - - private - - def synchronize(&block) - @mutex.synchronize(&block) - end - end - end -end diff --git a/lib/gitlab/sherlock/file_sample.rb b/lib/gitlab/sherlock/file_sample.rb deleted file mode 100644 index 5d10d8c4877..00000000000 --- a/lib/gitlab/sherlock/file_sample.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Sherlock - class FileSample - attr_reader :id, :file, :line_samples, :events, :duration - - # file - The full path to the file this sample belongs to. - # line_samples - An array of LineSample objects. - # duration - The total execution time in milliseconds. - # events - The total amount of events. - def initialize(file, line_samples, duration, events) - @id = SecureRandom.uuid - @file = file - @line_samples = line_samples - @duration = duration - @events = events - end - - def relative_path - @relative_path ||= @file.gsub(%r{^#{Rails.root}/?}, '') - end - - def to_param - @id - end - - def source - @source ||= File.read(@file) - end - end - end -end diff --git a/lib/gitlab/sherlock/line_profiler.rb b/lib/gitlab/sherlock/line_profiler.rb deleted file mode 100644 index aa25eb5a571..00000000000 --- a/lib/gitlab/sherlock/line_profiler.rb +++ /dev/null @@ -1,100 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Sherlock - # Class for profiling code on a per line basis. - # - # The LineProfiler class can be used to profile code on per line basis - # without littering your code with Ruby implementation specific profiling - # methods. - # - # This profiler only includes samples taking longer than a given threshold - # and those that occur in the actual application (e.g. files from Gems are - # ignored). - class LineProfiler - # The minimum amount of time that has to be spent in a file for it to be - # included in a list of samples. - MINIMUM_DURATION = 10.0 - - # Profiles the given block. - # - # Example: - # - # profiler = LineProfiler.new - # - # retval, samples = profiler.profile do - # "cats are amazing" - # end - # - # retval # => "cats are amazing" - # samples # => [#, ...] - # - # Returns an Array containing the block's return value and an Array of - # FileSample objects. - def profile(&block) - if mri? - profile_mri(&block) - else - raise NotImplementedError, - 'Line profiling is not supported on this platform' - end - end - - # Profiles the given block using rblineprof (MRI only). - def profile_mri - require 'rblineprof' - - retval = nil - samples = lineprof(/^#{Rails.root}/) { retval = yield } - - file_samples = aggregate_rblineprof(samples) - - [retval, file_samples] - end - - # Returns an Array of file samples based on the output of rblineprof. - # - # lineprof_stats - A Hash containing rblineprof statistics on a per file - # basis. - # - # Returns an Array of FileSample objects. - def aggregate_rblineprof(lineprof_stats) - samples = [] - - lineprof_stats.each do |(file, stats)| - source_lines = File.read(file).each_line.to_a - line_samples = [] - - total_duration = microsec_to_millisec(stats[0][0]) - total_events = stats[0][2] - - next if total_duration <= MINIMUM_DURATION - - stats[1..].each_with_index do |data, index| - next unless source_lines[index] - - duration = microsec_to_millisec(data[0]) - events = data[2] - - line_samples << LineSample.new(duration, events) - end - - samples << FileSample - .new(file, line_samples, total_duration, total_events) - end - - samples - end - - private - - def microsec_to_millisec(microsec) - microsec / 1000.0 - end - - def mri? - RUBY_ENGINE == 'ruby' - end - end - end -end diff --git a/lib/gitlab/sherlock/line_sample.rb b/lib/gitlab/sherlock/line_sample.rb deleted file mode 100644 index c92fa9ea1ff..00000000000 --- a/lib/gitlab/sherlock/line_sample.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Sherlock - class LineSample - attr_reader :duration, :events - - # duration - The execution time in milliseconds. - # events - The amount of events. - def initialize(duration, events) - @duration = duration - @events = events - end - - # Returns the sample duration percentage relative to the given duration. - # - # Example: - # - # sample.duration # => 150 - # sample.percentage_of(1500) # => 10.0 - # - # total_duration - The total duration to compare with. - # - # Returns a float - def percentage_of(total_duration) - (duration.to_f / total_duration) * 100.0 - end - - # Returns true if the current sample takes up the majority of the given - # duration. - # - # total_duration - The total duration to compare with. - def majority_of?(total_duration) - percentage_of(total_duration) >= 30 - end - end - end -end diff --git a/lib/gitlab/sherlock/location.rb b/lib/gitlab/sherlock/location.rb deleted file mode 100644 index 4bba60f3490..00000000000 --- a/lib/gitlab/sherlock/location.rb +++ /dev/null @@ -1,28 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Sherlock - class Location - attr_reader :path, :line - - SHERLOCK_DIR = File.dirname(__FILE__) - - # Creates a new Location from a `Thread::Backtrace::Location`. - def self.from_ruby_location(location) - new(location.path, location.lineno) - end - - # path - The full path of the frame as a String. - # line - The line number of the frame as a Fixnum. - def initialize(path, line) - @path = path - @line = line - end - - # Returns true if the current frame originated from the application. - def application? - @path.start_with?(Rails.root.to_s) && !path.start_with?(SHERLOCK_DIR) - end - end - end -end diff --git a/lib/gitlab/sherlock/middleware.rb b/lib/gitlab/sherlock/middleware.rb deleted file mode 100644 index f7b08d58e49..00000000000 --- a/lib/gitlab/sherlock/middleware.rb +++ /dev/null @@ -1,43 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Sherlock - # Rack middleware used for tracking request metrics. - class Middleware - CONTENT_TYPES = %r{text/html|application/json}i.freeze - - IGNORE_PATHS = %r{^/sherlock}.freeze - - def initialize(app) - @app = app - end - - # env - A Hash containing Rack environment details. - def call(env) - if instrument?(env) - call_with_instrumentation(env) - else - @app.call(env) - end - end - - def call_with_instrumentation(env) - trans = transaction_from_env(env) - retval = trans.run { @app.call(env) } - - Sherlock.collection.add(trans) - - retval - end - - def instrument?(env) - !!(env['HTTP_ACCEPT'] =~ CONTENT_TYPES && - env['REQUEST_URI'] !~ IGNORE_PATHS) - end - - def transaction_from_env(env) - Transaction.new(env['REQUEST_METHOD'], env['REQUEST_URI']) - end - end - end -end diff --git a/lib/gitlab/sherlock/query.rb b/lib/gitlab/sherlock/query.rb deleted file mode 100644 index 6f1d2ad23c1..00000000000 --- a/lib/gitlab/sherlock/query.rb +++ /dev/null @@ -1,112 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Sherlock - class Query - attr_reader :id, :query, :started_at, :finished_at, :backtrace - - # SQL identifiers that should be prefixed with newlines. - PREFIX_NEWLINE = %r{ - \s+(FROM - |(LEFT|RIGHT)?INNER\s+JOIN - |(LEFT|RIGHT)?OUTER\s+JOIN - |WHERE - |AND - |GROUP\s+BY - |ORDER\s+BY - |LIMIT - |OFFSET)\s+}ix.freeze # Vim indent breaks when this is on a newline :< - - # Creates a new Query using a String and a separate Array of bindings. - # - # query - A String containing a SQL query, optionally with numeric - # placeholders (`$1`, `$2`, etc). - # - # bindings - An Array of ActiveRecord columns and their values. - # started_at - The start time of the query as a Time-like object. - # finished_at - The completion time of the query as a Time-like object. - # - # Returns a new Query object. - def self.new_with_bindings(query, bindings, started_at, finished_at) - bindings.each_with_index do |(_, value), index| - quoted_value = ActiveRecord::Base.connection.quote(value) - - query = query.gsub("$#{index + 1}", quoted_value) - end - - new(query, started_at, finished_at) - end - - # query - The SQL query as a String (without placeholders). - # started_at - The start time of the query as a Time-like object. - # finished_at - The completion time of the query as a Time-like object. - def initialize(query, started_at, finished_at) - @id = SecureRandom.uuid - @query = query - @started_at = started_at - @finished_at = finished_at - @backtrace = caller_locations.map do |loc| - Location.from_ruby_location(loc) - end - - unless @query.end_with?(';') - @query = "#{@query};" - end - end - - # Returns the query duration in milliseconds. - def duration - @duration ||= (@finished_at - @started_at) * 1000.0 - end - - def to_param - @id - end - - # Returns a human readable version of the query. - def formatted_query - @formatted_query ||= format_sql(@query) - end - - # Returns the last application frame of the backtrace. - def last_application_frame - @last_application_frame ||= @backtrace.find(&:application?) - end - - # Returns an Array of application frames (excluding Gems and the likes). - def application_backtrace - @application_backtrace ||= @backtrace.select(&:application?) - end - - # Returns the query plan as a String. - def explain - unless @explain - ActiveRecord::Base.connection.transaction do - @explain = raw_explain(@query).values.flatten.join("\n") - - # Roll back any queries that mutate data so we don't mess up - # anything when running explain on an INSERT, UPDATE, DELETE, etc. - raise ActiveRecord::Rollback - end - end - - @explain - end - - private - - def raw_explain(query) - explain = "EXPLAIN ANALYZE #{query};" - - ActiveRecord::Base.connection.execute(explain) - end - - def format_sql(query) - query.each_line - .map { |line| line.strip } - .join("\n") - .gsub(PREFIX_NEWLINE) { "\n#{Regexp.last_match(1)} " } - end - end - end -end diff --git a/lib/gitlab/sherlock/transaction.rb b/lib/gitlab/sherlock/transaction.rb deleted file mode 100644 index d04624977dc..00000000000 --- a/lib/gitlab/sherlock/transaction.rb +++ /dev/null @@ -1,140 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Sherlock - class Transaction - attr_reader :id, :type, :path, :queries, :file_samples, :started_at, - :finished_at, :view_counts - - # type - The type of transaction (e.g. "GET", "POST", etc) - # path - The path of the transaction (e.g. the HTTP request path) - def initialize(type, path) - @id = SecureRandom.uuid - @type = type - @path = path - @queries = [] - @file_samples = [] - @started_at = nil - @finished_at = nil - @thread = Thread.current - @view_counts = Hash.new(0) - end - - # Runs the transaction and returns the block's return value. - def run - @started_at = Time.now - - retval = with_subscriptions do - profile_lines { yield } - end - - @finished_at = Time.now - - retval - end - - # Returns the duration in seconds. - def duration - @duration ||= started_at && finished_at ? finished_at - started_at : 0 - end - - # Returns the total query duration in seconds. - def query_duration - @query_duration ||= @queries.map { |q| q.duration }.inject(:+) / 1000.0 - end - - def to_param - @id - end - - # Returns the queries sorted in descending order by their durations. - def sorted_queries - @queries.sort { |a, b| b.duration <=> a.duration } - end - - # Returns the file samples sorted in descending order by their durations. - def sorted_file_samples - @file_samples.sort { |a, b| b.duration <=> a.duration } - end - - # Finds a query by the given ID. - # - # id - The query ID as a String. - # - # Returns a Query object if one could be found, nil otherwise. - def find_query(id) - @queries.find { |query| query.id == id } - end - - # Finds a file sample by the given ID. - # - # id - The query ID as a String. - # - # Returns a FileSample object if one could be found, nil otherwise. - def find_file_sample(id) - @file_samples.find { |sample| sample.id == id } - end - - def profile_lines - retval = nil - - if Sherlock.enable_line_profiler? - retval, @file_samples = LineProfiler.new.profile { yield } - else - retval = yield - end - - retval - end - - def subscribe_to_active_record - ActiveSupport::Notifications.subscribe('sql.active_record') do |_, start, finish, _, data| - next unless same_thread? - - unless data.fetch(:cached, data[:name] == 'CACHE') - track_query(data[:sql].strip, data[:binds], start, finish) - end - end - end - - def subscribe_to_action_view - regex = /render_(template|partial)\.action_view/ - - ActiveSupport::Notifications.subscribe(regex) do |_, start, finish, _, data| - next unless same_thread? - - track_view(data[:identifier]) - end - end - - private - - def track_query(query, bindings, start, finish) - @queries << Query.new_with_bindings(query, bindings, start, finish) - end - - def track_view(path) - @view_counts[path] += 1 - end - - def with_subscriptions - ar_subscriber = subscribe_to_active_record - av_subscriber = subscribe_to_action_view - - retval = yield - - ActiveSupport::Notifications.unsubscribe(ar_subscriber) - ActiveSupport::Notifications.unsubscribe(av_subscriber) - - retval - end - - # In case somebody uses a multi-threaded server locally (e.g. Puma) we - # _only_ want to track notifications that originate from the transaction - # thread. - def same_thread? - Thread.current == @thread - end - end - end -end diff --git a/lib/gitlab/sidekiq_logging/json_formatter.rb b/lib/gitlab/sidekiq_logging/json_formatter.rb index a6281bbdf26..dd50fef8c3d 100644 --- a/lib/gitlab/sidekiq_logging/json_formatter.rb +++ b/lib/gitlab/sidekiq_logging/json_formatter.rb @@ -2,6 +2,7 @@ # This is needed for sidekiq-cluster require 'json' +require 'sidekiq/job_retry' module Gitlab module SidekiqLogging diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb index 3438bc0f3ef..a9bfcce2e0a 100644 --- a/lib/gitlab/sidekiq_logging/structured_logger.rb +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -2,6 +2,8 @@ require 'active_record' require 'active_record/log_subscriber' +require 'sidekiq/job_logger' +require 'sidekiq/job_retry' module Gitlab module SidekiqLogging diff --git a/lib/gitlab/sidekiq_middleware/monitor.rb b/lib/gitlab/sidekiq_middleware/monitor.rb index ed825dbfd60..d38fed3b768 100644 --- a/lib/gitlab/sidekiq_middleware/monitor.rb +++ b/lib/gitlab/sidekiq_middleware/monitor.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'sidekiq/job_retry' + module Gitlab module SidekiqMiddleware class Monitor diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb index 120d18f63f2..66417b3697e 100644 --- a/lib/gitlab/sidekiq_status.rb +++ b/lib/gitlab/sidekiq_status.rb @@ -29,16 +29,15 @@ module Gitlab # for most jobs. DEFAULT_EXPIRATION = 30.minutes.to_i - DEFAULT_VALUE = 1 - DEFAULT_VALUE_MESSAGE = 'Keys using the default value for SidekiqStatus detected' - # Starts tracking of the given job. # # jid - The Sidekiq job ID # expire - The expiration time of the Redis key. - def self.set(jid, expire = DEFAULT_EXPIRATION, value: DEFAULT_VALUE) + def self.set(jid, expire = DEFAULT_EXPIRATION) + return unless expire + Sidekiq.redis do |redis| - redis.set(key_for(jid), value, ex: expire) + redis.set(key_for(jid), 1, ex: expire) end end @@ -94,17 +93,10 @@ module Gitlab return [] if job_ids.empty? keys = job_ids.map { |jid| key_for(jid) } - results = Sidekiq.redis { |redis| redis.mget(*keys) } - - if Feature.enabled?(:log_implicit_sidekiq_status_calls, default_enabled: :yaml) - to_log = keys.zip(results).select do |_key, result| - result == DEFAULT_VALUE.to_s - end.map(&:first) - - Sidekiq.logger.info(message: DEFAULT_VALUE_MESSAGE, keys: to_log) if to_log.any? - end - results.map { |result| !result.nil? } + Sidekiq + .redis { |redis| redis.mget(*keys) } + .map { |result| !result.nil? } end # Returns the JIDs that are completed diff --git a/lib/gitlab/sidekiq_status/client_middleware.rb b/lib/gitlab/sidekiq_status/client_middleware.rb index cee7270f2fb..8471696dcea 100644 --- a/lib/gitlab/sidekiq_status/client_middleware.rb +++ b/lib/gitlab/sidekiq_status/client_middleware.rb @@ -4,10 +4,8 @@ module Gitlab module SidekiqStatus class ClientMiddleware def call(_, job, _, _) - status_expiration = job['status_expiration'] || Gitlab::SidekiqStatus::DEFAULT_EXPIRATION - value = job['status_expiration'] ? 2 : Gitlab::SidekiqStatus::DEFAULT_VALUE + Gitlab::SidekiqStatus.set(job['jid'], job['status_expiration']) - Gitlab::SidekiqStatus.set(job['jid'], status_expiration, value: value) yield end end diff --git a/lib/gitlab/sourcegraph.rb b/lib/gitlab/sourcegraph.rb index 7ef6ab32bd4..892c4468107 100644 --- a/lib/gitlab/sourcegraph.rb +++ b/lib/gitlab/sourcegraph.rb @@ -8,13 +8,14 @@ module Gitlab end def feature_available? - # The sourcegraph_bundle feature could be conditionally applied, so check if `!off?` - !feature.off? + # The sourcegraph feature could be conditionally applied, so check if `!off?` + # We also can't just check !off? because the ActiveRecord might not exist yet + self.feature_enabled? || !feature.off? end def feature_enabled?(actor = nil) # Some CI jobs grep for Feature.enabled? in our codebase, so it is important this reference stays around. - Feature.enabled?(:sourcegraph, actor) + Feature.enabled?(:sourcegraph, actor, default_enabled: :yaml) end private diff --git a/lib/gitlab/ssh_public_key.rb b/lib/gitlab/ssh_public_key.rb index 6df54852d02..314cc5e2db6 100644 --- a/lib/gitlab/ssh_public_key.rb +++ b/lib/gitlab/ssh_public_key.rb @@ -2,13 +2,15 @@ module Gitlab class SSHPublicKey - Technology = Struct.new(:name, :key_class, :supported_sizes) + Technology = Struct.new(:name, :key_class, :supported_sizes, :supported_algorithms) + # See https://man.openbsd.org/sshd#AUTHORIZED_KEYS_FILE_FORMAT for the list of + # supported algorithms. TECHNOLOGIES = [ - Technology.new(:rsa, OpenSSL::PKey::RSA, [1024, 2048, 3072, 4096]), - Technology.new(:dsa, OpenSSL::PKey::DSA, [1024, 2048, 3072]), - Technology.new(:ecdsa, OpenSSL::PKey::EC, [256, 384, 521]), - Technology.new(:ed25519, Net::SSH::Authentication::ED25519::PubKey, [256]) + Technology.new(:rsa, OpenSSL::PKey::RSA, [1024, 2048, 3072, 4096], %w(ssh-rsa)), + Technology.new(:dsa, OpenSSL::PKey::DSA, [1024, 2048, 3072], %w(ssh-dss)), + Technology.new(:ecdsa, OpenSSL::PKey::EC, [256, 384, 521], %w(ecdsa-sha2-nistp256 ecdsa-sha2-nistp384 ecdsa-sha2-nistp521)), + Technology.new(:ed25519, Net::SSH::Authentication::ED25519::PubKey, [256], %w(ssh-ed25519)) ].freeze def self.technology(name) @@ -19,8 +21,20 @@ module Gitlab TECHNOLOGIES.find { |tech| key.is_a?(tech.key_class) } end + def self.supported_types + TECHNOLOGIES.map(&:name) + end + def self.supported_sizes(name) - technology(name)&.supported_sizes + technology(name).supported_sizes + end + + def self.supported_algorithms + TECHNOLOGIES.flat_map { |tech| tech.supported_algorithms } + end + + def self.supported_algorithms_for_name(name) + technology(name).supported_algorithms end def self.sanitize(key_content) diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb index ac1522b8a6c..228da9ee370 100644 --- a/lib/gitlab/themes.rb +++ b/lib/gitlab/themes.rb @@ -13,26 +13,28 @@ module Gitlab Theme = Struct.new(:id, :name, :css_class, :css_filename, :primary_color) # All available Themes - THEMES = [ - Theme.new(1, 'Indigo', 'ui-indigo', 'theme_indigo', '#292961'), - Theme.new(6, 'Light Indigo', 'ui-light-indigo', 'theme_light_indigo', '#4b4ba3'), - Theme.new(4, 'Blue', 'ui-blue', 'theme_blue', '#1a3652'), - Theme.new(7, 'Light Blue', 'ui-light-blue', 'theme_light_blue', '#2261a1'), - Theme.new(5, 'Green', 'ui-green', 'theme_green', '#0d4524'), - Theme.new(8, 'Light Green', 'ui-light-green', 'theme_light_green', '#156b39'), - Theme.new(9, 'Red', 'ui-red', 'theme_red', '#691a16'), - Theme.new(10, 'Light Red', 'ui-light-red', 'theme_light_red', '#a62e21'), - Theme.new(2, 'Dark', 'ui-dark', 'theme_dark', '#303030'), - Theme.new(3, 'Light', 'ui-light', 'theme_light', '#666'), - Theme.new(11, 'Dark Mode (alpha)', 'gl-dark', nil, '#303030') - ].freeze + def available_themes + [ + Theme.new(1, s_('NavigationTheme|Indigo'), 'ui-indigo', 'theme_indigo', '#292961'), + Theme.new(6, s_('NavigationTheme|Light Indigo'), 'ui-light-indigo', 'theme_light_indigo', '#4b4ba3'), + Theme.new(4, s_('NavigationTheme|Blue'), 'ui-blue', 'theme_blue', '#1a3652'), + Theme.new(7, s_('NavigationTheme|Light Blue'), 'ui-light-blue', 'theme_light_blue', '#2261a1'), + Theme.new(5, s_('NavigationTheme|Green'), 'ui-green', 'theme_green', '#0d4524'), + Theme.new(8, s_('NavigationTheme|Light Green'), 'ui-light-green', 'theme_light_green', '#156b39'), + Theme.new(9, s_('NavigationTheme|Red'), 'ui-red', 'theme_red', '#691a16'), + Theme.new(10, s_('NavigationTheme|Light Red'), 'ui-light-red', 'theme_light_red', '#a62e21'), + Theme.new(2, s_('NavigationTheme|Dark'), 'ui-dark', 'theme_dark', '#303030'), + Theme.new(3, s_('NavigationTheme|Light'), 'ui-light', 'theme_light', '#666'), + Theme.new(11, s_('NavigationTheme|Dark Mode (alpha)'), 'gl-dark', nil, '#303030') + ] + end # Convenience method to get a space-separated String of all the theme # classes that might be applied to the `body` element # # Returns a String def body_classes - THEMES.collect(&:css_class).uniq.join(' ') + available_themes.collect(&:css_class).uniq.join(' ') end # Get a Theme by its ID @@ -43,12 +45,12 @@ module Gitlab # # Returns a Theme def by_id(id) - THEMES.detect { |t| t.id == id } || default + available_themes.detect { |t| t.id == id } || default end # Returns the number of defined Themes def count - THEMES.size + available_themes.size end # Get the default Theme @@ -62,7 +64,7 @@ module Gitlab # # Yields the Theme object def each(&block) - THEMES.each(&block) + available_themes.each(&block) end # Get the Theme for the specified user, or the default @@ -79,7 +81,7 @@ module Gitlab end def self.valid_ids - THEMES.map(&:id) + available_themes.map(&:id) end private @@ -87,7 +89,7 @@ module Gitlab def default_id @default_id ||= begin id = Gitlab.config.gitlab.default_theme.to_i - theme_ids = THEMES.map(&:id) + theme_ids = available_themes.map(&:id) theme_ids.include?(id) ? id : APPLICATION_DEFAULT end diff --git a/lib/gitlab/tracking/standard_context.rb b/lib/gitlab/tracking/standard_context.rb index 837390b91fb..542dc476526 100644 --- a/lib/gitlab/tracking/standard_context.rb +++ b/lib/gitlab/tracking/standard_context.rb @@ -3,7 +3,7 @@ module Gitlab module Tracking class StandardContext - GITLAB_STANDARD_SCHEMA_URL = 'iglu:com.gitlab/gitlab_standard/jsonschema/1-0-7' + GITLAB_STANDARD_SCHEMA_URL = 'iglu:com.gitlab/gitlab_standard/jsonschema/1-0-8' GITLAB_RAILS_SOURCE = 'gitlab-rails' def initialize(namespace: nil, project: nil, user: nil, **extra) @@ -46,7 +46,8 @@ module Gitlab extra: extra, user_id: user&.id, namespace_id: namespace&.id, - project_id: project_id + project_id: project_id, + context_generated_at: Time.current } end diff --git a/lib/gitlab/untrusted_regexp/ruby_syntax.rb b/lib/gitlab/untrusted_regexp/ruby_syntax.rb index 6adf119aa75..010214cf295 100644 --- a/lib/gitlab/untrusted_regexp/ruby_syntax.rb +++ b/lib/gitlab/untrusted_regexp/ruby_syntax.rb @@ -20,13 +20,13 @@ module Gitlab !!self.fabricate(pattern, fallback: fallback) end - def self.fabricate(pattern, fallback: false) - self.fabricate!(pattern, fallback: fallback) + def self.fabricate(pattern, fallback: false, project: nil) + self.fabricate!(pattern, fallback: fallback, project: project) rescue RegexpError nil end - def self.fabricate!(pattern, fallback: false) + def self.fabricate!(pattern, fallback: false, project: nil) raise RegexpError, 'Pattern is not string!' unless pattern.is_a?(String) matches = pattern.match(PATTERN) @@ -38,6 +38,16 @@ module Gitlab raise unless fallback && Feature.enabled?(:allow_unsafe_ruby_regexp, default_enabled: false) + if Feature.enabled?(:ci_unsafe_regexp_logger, type: :ops, default_enabled: :yaml) + Gitlab::AppJsonLogger.info( + class: self.class.name, + regexp: pattern.to_s, + fabricated: 'unsafe ruby regexp', + project_id: project&.id, + project_path: project&.full_path + ) + end + create_ruby_regexp(matches[:regexp], matches[:flags]) end end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 917c273d3f6..adf920d8b52 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -304,7 +304,8 @@ module Gitlab # rubocop: disable UsageData/LargeTable adapter: alt_usage_data { ApplicationRecord.database.adapter_name }, version: alt_usage_data { ApplicationRecord.database.version }, - pg_system_id: alt_usage_data { ApplicationRecord.database.system_id } + pg_system_id: alt_usage_data { ApplicationRecord.database.system_id }, + flavor: alt_usage_data { ApplicationRecord.database.flavor } # rubocop: enable UsageData/LargeTable }, mail: { @@ -521,11 +522,7 @@ module Gitlab projects_with_disable_overriding_approvers_per_merge_request: count(::Project.where(time_period.merge(disable_overriding_approvers_per_merge_request: true))), projects_without_disable_overriding_approvers_per_merge_request: count(::Project.where(time_period.merge(disable_overriding_approvers_per_merge_request: [false, nil]))), remote_mirrors: distinct_count(::Project.with_remote_mirrors.where(time_period), :creator_id), - snippets: distinct_count(::Snippet.where(time_period), :author_id), - suggestions: distinct_count(::Note.with_suggestions.where(time_period), - :author_id, - start: minimum_id(::User), - finish: maximum_id(::User)) + snippets: distinct_count(::Snippet.where(time_period), :author_id) }.tap do |h| if time_period.present? h[:merge_requests_users] = merge_requests_users(time_period) @@ -552,33 +549,11 @@ module Gitlab user_auth_by_provider: distinct_count_user_auth_by_provider(time_period), unique_users_all_imports: unique_users_all_imports(time_period), bulk_imports: { - gitlab: DEPRECATED_VALUE, gitlab_v1: count(::BulkImport.where(**time_period, source_type: :gitlab)) }, project_imports: project_imports(time_period), issue_imports: issue_imports(time_period), - group_imports: group_imports(time_period), - - # Deprecated data to be removed - projects_imported: { - total: DEPRECATED_VALUE, - gitlab_project: DEPRECATED_VALUE, - gitlab: DEPRECATED_VALUE, - github: DEPRECATED_VALUE, - bitbucket: DEPRECATED_VALUE, - bitbucket_server: DEPRECATED_VALUE, - gitea: DEPRECATED_VALUE, - git: DEPRECATED_VALUE, - manifest: DEPRECATED_VALUE - }, - issues_imported: { - jira: DEPRECATED_VALUE, - fogbugz: DEPRECATED_VALUE, - phabricator: DEPRECATED_VALUE, - csv: DEPRECATED_VALUE - }, - groups_imported: DEPRECATED_VALUE - # End of deprecated keys + group_imports: group_imports(time_period) } end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/gitlab/usage_data_counters/counter_events/package_events.yml b/lib/gitlab/usage_data_counters/counter_events/package_events.yml index 21d637e7152..a64d0ff7e24 100644 --- a/lib/gitlab/usage_data_counters/counter_events/package_events.yml +++ b/lib/gitlab/usage_data_counters/counter_events/package_events.yml @@ -5,12 +5,8 @@ - i_package_conan_delete_package - i_package_conan_pull_package - i_package_conan_push_package -- i_package_container_delete_package -- i_package_container_pull_package -- i_package_container_push_package - i_package_debian_delete_package - i_package_debian_pull_package -- i_package_debian_push_package - i_package_delete_package - i_package_delete_package_by_deploy_token - i_package_delete_package_by_guest @@ -56,9 +52,6 @@ - i_package_rubygems_delete_package - i_package_rubygems_pull_package - i_package_rubygems_push_package -- i_package_tag_delete_package -- i_package_tag_pull_package -- i_package_tag_push_package - i_package_terraform_module_delete_package - i_package_terraform_module_pull_package - i_package_terraform_module_push_package diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index 8fc8bb5d344..c6e9db6a314 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -104,7 +104,7 @@ module Gitlab events_names = events_for_category(category) event_results = events_names.each_with_object({}) do |event, hash| - hash["#{event}_weekly"] = unique_events(**weekly_time_range.merge(event_names: [event])) + hash["#{event}_weekly"] = unique_events(**weekly_time_range.merge(event_names: [event])) unless event == "i_package_composer_deploy_token" hash["#{event}_monthly"] = unique_events(**monthly_time_range.merge(event_names: [event])) end diff --git a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml index d90960b344c..55ed9a42512 100644 --- a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml +++ b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml @@ -103,6 +103,10 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_security_dast_on_demand_api_scan + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_security_coverage_fuzzing category: ci_templates redis_slot: ci_templates @@ -539,6 +543,10 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_implicit_security_dast_on_demand_api_scan + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_implicit_security_coverage_fuzzing category: ci_templates redis_slot: ci_templates diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index bb98a0b262a..fc610f1e2d6 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -368,8 +368,18 @@ redis_slot: testing category: testing aggregation: weekly +- name: users_clicking_license_testing_visiting_external_website + redis_slot: testing + category: testing + aggregation: weekly # Container Security - Network Policies - name: clusters_using_network_policies_ui redis_slot: network_policies category: network_policies aggregation: weekly +# Geo group +- name: g_geo_proxied_requests + category: geo + redis_slot: geo + aggregation: daily + feature_flag: track_geo_proxy_events diff --git a/lib/gitlab/usage_data_counters/known_events/package_events.yml b/lib/gitlab/usage_data_counters/known_events/package_events.yml index e5031599dd0..debdbd8614f 100644 --- a/lib/gitlab/usage_data_counters/known_events/package_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/package_events.yml @@ -15,22 +15,6 @@ category: user_packages aggregation: weekly redis_slot: package -- name: i_package_container_deploy_token - category: deploy_token_packages - aggregation: weekly - redis_slot: package -- name: i_package_container_user - category: user_packages - aggregation: weekly - redis_slot: package -- name: i_package_debian_deploy_token - category: deploy_token_packages - aggregation: weekly - redis_slot: package -- name: i_package_debian_user - category: user_packages - aggregation: weekly - redis_slot: package - name: i_package_generic_deploy_token category: deploy_token_packages aggregation: weekly @@ -39,14 +23,6 @@ category: user_packages aggregation: weekly redis_slot: package -- name: i_package_golang_deploy_token - category: deploy_token_packages - aggregation: weekly - redis_slot: package -- name: i_package_golang_user - category: user_packages - aggregation: weekly - redis_slot: package - name: i_package_helm_deploy_token category: deploy_token_packages aggregation: weekly @@ -95,14 +71,6 @@ category: user_packages aggregation: weekly redis_slot: package -- name: i_package_tag_deploy_token - category: deploy_token_packages - aggregation: weekly - redis_slot: package -- name: i_package_tag_user - category: user_packages - aggregation: weekly - redis_slot: package - name: i_package_terraform_module_deploy_token category: deploy_token_packages aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/quickactions.yml b/lib/gitlab/usage_data_counters/known_events/quickactions.yml index d831ac02dd1..44f6b42d584 100644 --- a/lib/gitlab/usage_data_counters/known_events/quickactions.yml +++ b/lib/gitlab/usage_data_counters/known_events/quickactions.yml @@ -39,6 +39,10 @@ category: quickactions redis_slot: quickactions aggregation: weekly +- name: i_quickactions_clear_health_status + category: quickactions + redis_slot: quickactions + aggregation: weekly - name: i_quickactions_clone category: quickactions redis_slot: quickactions @@ -263,6 +267,10 @@ category: quickactions redis_slot: quickactions aggregation: weekly +- name: i_quickactions_health_status + category: quickactions + redis_slot: quickactions + aggregation: weekly - name: i_quickactions_wip category: quickactions redis_slot: quickactions diff --git a/lib/gitlab/utils/sanitize_node_link.rb b/lib/gitlab/utils/sanitize_node_link.rb index ab5d18e9c8a..b0dfa087fcf 100644 --- a/lib/gitlab/utils/sanitize_node_link.rb +++ b/lib/gitlab/utils/sanitize_node_link.rb @@ -8,6 +8,12 @@ module Gitlab UNSAFE_PROTOCOLS = %w(data javascript vbscript).freeze ATTRS_TO_SANITIZE = %w(href src data-src data-canonical-src).freeze + # sanitize 6.0 requires only a context argument. Do not add any default + # arguments to this method. + def sanitize_unsafe_links(env) + remove_unsafe_links(env) + end + def remove_unsafe_links(env, remove_invalid_links: true) node = env[:node] diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb index e347168f419..6c182f98dd0 100644 --- a/lib/gitlab/utils/usage_data.rb +++ b/lib/gitlab/utils/usage_data.rb @@ -169,7 +169,8 @@ module Gitlab return -1 if args.any?(&:negative?) args.sum - rescue StandardError + rescue StandardError => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) FALLBACK end @@ -179,7 +180,8 @@ module Gitlab else value end - rescue StandardError + rescue StandardError => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) fallback end @@ -295,13 +297,15 @@ module Gitlab def redis_usage_counter yield - rescue ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent, Gitlab::UsageDataCounters::HLLRedisCounter::EventError + rescue ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent, Gitlab::UsageDataCounters::HLLRedisCounter::EventError => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) FALLBACK end def redis_usage_data_totals(counter) counter.totals - rescue ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent + rescue ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) counter.fallback_totals end end diff --git a/lib/gitlab/web_hooks.rb b/lib/gitlab/web_hooks.rb new file mode 100644 index 00000000000..349c7a020cc --- /dev/null +++ b/lib/gitlab/web_hooks.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +module Gitlab + module WebHooks + GITLAB_EVENT_HEADER = 'X-Gitlab-Event' + end +end diff --git a/lib/gitlab/web_hooks/recursion_detection.rb b/lib/gitlab/web_hooks/recursion_detection.rb new file mode 100644 index 00000000000..1b5350d4a4e --- /dev/null +++ b/lib/gitlab/web_hooks/recursion_detection.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +# This module detects and blocks recursive webhook requests. +# +# Recursion can happen when a webhook has been configured to make a call +# to its own GitLab instance (i.e., its API), and during the execution of +# the call the webhook is triggered again to create an infinite loop of +# being triggered. +# +# Additionally the module blocks a webhook once the number of requests to +# the instance made by a series of webhooks triggering other webhooks reaches +# a limit. +# +# Blocking recursive webhooks allows GitLab to continue to support workflows +# that use webhooks to call the API non-recursively, or do not go on to +# trigger an unreasonable number of other webhooks. +module Gitlab + module WebHooks + module RecursionDetection + COUNT_LIMIT = 100 + TOUCH_CACHE_TTL = 30.minutes + + class << self + def set_from_headers(headers) + uuid = headers[UUID::HEADER] + + return unless uuid + + set_request_uuid(uuid) + end + + def set_request_uuid(uuid) + UUID.instance.request_uuid = uuid + end + + # Before a webhook is executed, `.register!` should be called. + # Adds the webhook ID to a cache (see `#cache_key_for_hook` for + # details of the cache). + def register!(hook) + cache_key = cache_key_for_hook(hook) + + ::Gitlab::Redis::SharedState.with do |redis| + redis.multi do + redis.sadd(cache_key, hook.id) + redis.expire(cache_key, TOUCH_CACHE_TTL) + end + end + end + + # Returns true if the webhook ID is present in the cache, or if the + # number of IDs in the cache exceeds the limit (see + # `#cache_key_for_hook` for details of the cache). + def block?(hook) + # If a request UUID has not been set then we know the request was not + # made by a webhook, and no recursion is possible. + return false unless UUID.instance.request_uuid + + cache_key = cache_key_for_hook(hook) + + ::Gitlab::Redis::SharedState.with do |redis| + redis.sismember(cache_key, hook.id) || + redis.scard(cache_key) >= COUNT_LIMIT + end + end + + def header(hook) + UUID.instance.header(hook) + end + + def to_log(hook) + { + uuid: UUID.instance.uuid_for_hook(hook), + ids: ::Gitlab::Redis::SharedState.with { |redis| redis.smembers(cache_key_for_hook(hook)).map(&:to_i) } + } + end + + private + + # Returns a cache key scoped to a UUID. + # + # The particular UUID will be either: + # + # - A UUID that was recycled from the request headers if the request was made by a webhook. + # - a new UUID initialized for the webhook. + # + # This means that cycles of webhooks that are triggered from other webhooks + # will share the same cache, and other webhooks will use a new cache. + def cache_key_for_hook(hook) + [:webhook_recursion_detection, UUID.instance.uuid_for_hook(hook)].join(':') + end + end + end + end +end diff --git a/lib/gitlab/web_hooks/recursion_detection/uuid.rb b/lib/gitlab/web_hooks/recursion_detection/uuid.rb new file mode 100644 index 00000000000..9c52399818d --- /dev/null +++ b/lib/gitlab/web_hooks/recursion_detection/uuid.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module WebHooks + module RecursionDetection + class UUID + HEADER = "#{::Gitlab::WebHooks::GITLAB_EVENT_HEADER}-UUID" + + include Singleton + + attr_accessor :request_uuid + + def initialize + self.new_uuids_for_hooks = {} + end + + class << self + # Back the Singleton with RequestStore so it is isolated to this request. + def instance + Gitlab::SafeRequestStore[:web_hook_recursion_detection_uuid] ||= new + end + end + + # Returns a UUID, which will be either: + # + # - The UUID that was recycled from the request headers if the request was made by a webhook. + # - A new UUID initialized for the webhook. + def uuid_for_hook(hook) + request_uuid || new_uuid_for_hook(hook) + end + + def header(hook) + { HEADER => uuid_for_hook(hook) } + end + + private + + attr_accessor :new_uuids_for_hooks + + def new_uuid_for_hook(hook) + new_uuids_for_hooks[hook.id] ||= SecureRandom.uuid + end + end + end + end +end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 3a905a2e1c5..19d30daa577 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -203,11 +203,11 @@ module Gitlab end def verify_api_request!(request_headers) - decode_jwt(request_headers[INTERNAL_API_REQUEST_HEADER]) + decode_jwt_with_issuer(request_headers[INTERNAL_API_REQUEST_HEADER]) end - def decode_jwt(encoded_message) - decode_jwt_for_issuer('gitlab-workhorse', encoded_message) + def decode_jwt_with_issuer(encoded_message) + decode_jwt(encoded_message, issuer: 'gitlab-workhorse') end def secret_path diff --git a/lib/gitlab_edition.rb b/lib/gitlab_edition.rb new file mode 100644 index 00000000000..6eb6b52c357 --- /dev/null +++ b/lib/gitlab_edition.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +require 'pathname' + +module GitlabEdition + def self.root + Pathname.new(File.expand_path('..', __dir__)) + end + + def self.extensions + if jh? + %w[ee jh] + elsif ee? + %w[ee] + else + %w[] + end + end + + def self.ee? + @is_ee ||= + # We use this method when the Rails environment is not loaded. This + # means that checking the presence of the License class could result in + # this method returning `false`, even for an EE installation. + # + # The `FOSS_ONLY` is always `string` or `nil` + # Thus the nil or empty string will result + # in using default value: false + # + # The behavior needs to be synchronised with + # config/helpers/is_ee_env.js + root.join('ee/app/models/license.rb').exist? && + !%w[true 1].include?(ENV['FOSS_ONLY'].to_s) + end + + def self.jh? + @is_jh ||= + ee? && + root.join('jh').exist? && + !%w[true 1].include?(ENV['EE_ONLY'].to_s) + end + + def self.ee + yield if ee? + end + + def self.jh + yield if jh? + end +end diff --git a/lib/sidebars/groups/menus/ci_cd_menu.rb b/lib/sidebars/groups/menus/ci_cd_menu.rb index f5bce57f496..a1f98b918e6 100644 --- a/lib/sidebars/groups/menus/ci_cd_menu.rb +++ b/lib/sidebars/groups/menus/ci_cd_menu.rb @@ -34,10 +34,8 @@ module Sidebars ) end - # TODO Proper policies, such as `read_group_runners`, should be implemented per - # See https://gitlab.com/gitlab-org/gitlab/-/issues/334802 def show_runners? - can?(context.current_user, :admin_group, context.group) && + can?(context.current_user, :read_group_runners, context.group) && Feature.enabled?(:runner_list_group_view_vue_ui, context.group, default_enabled: :yaml) end end diff --git a/lib/sidebars/groups/menus/settings_menu.rb b/lib/sidebars/groups/menus/settings_menu.rb index f0239ca6a1a..810b467ed2d 100644 --- a/lib/sidebars/groups/menus/settings_menu.rb +++ b/lib/sidebars/groups/menus/settings_menu.rb @@ -10,6 +10,7 @@ module Sidebars add_item(general_menu_item) add_item(integrations_menu_item) + add_item(access_tokens_menu_item) add_item(group_projects_menu_item) add_item(repository_menu_item) add_item(ci_cd_menu_item) @@ -56,6 +57,19 @@ module Sidebars ) end + def access_tokens_menu_item + unless can?(context.current_user, :read_resource_access_tokens, context.group) + return ::Sidebars::NilMenuItem.new(item_id: :access_tokens) + end + + ::Sidebars::MenuItem.new( + title: _('Access Tokens'), + link: group_settings_access_tokens_path(context.group), + active_routes: { path: 'access_tokens#index' }, + item_id: :access_tokens + ) + end + def group_projects_menu_item ::Sidebars::MenuItem.new( title: _('Projects'), diff --git a/lib/sidebars/projects/menus/infrastructure_menu.rb b/lib/sidebars/projects/menus/infrastructure_menu.rb index 1018bdd545b..060a5be5f57 100644 --- a/lib/sidebars/projects/menus/infrastructure_menu.rb +++ b/lib/sidebars/projects/menus/infrastructure_menu.rb @@ -100,7 +100,7 @@ module Sidebars ::Sidebars::MenuItem.new( title: _('Google Cloud'), link: project_google_cloud_index_path(context.project), - active_routes: { controller: [:google_cloud, :service_accounts] }, + active_routes: { controller: [:google_cloud, :service_accounts, :deployments] }, item_id: :google_cloud ) end diff --git a/lib/sidebars/projects/menus/issues_menu.rb b/lib/sidebars/projects/menus/issues_menu.rb index 3774bec2f13..51eea3d850d 100644 --- a/lib/sidebars/projects/menus/issues_menu.rb +++ b/lib/sidebars/projects/menus/issues_menu.rb @@ -8,7 +8,7 @@ module Sidebars override :configure_menu_items def configure_menu_items - return unless can?(context.current_user, :read_issue, context.project) + return false unless show_issues_menu_items? add_item(list_menu_item) add_item(boards_menu_item) @@ -70,6 +70,10 @@ module Sidebars private + def show_issues_menu_items? + can?(context.current_user, :read_issue, context.project) + end + def list_menu_item ::Sidebars::MenuItem.new( title: _('List'), diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index cc10d73f76a..0bca63a64f5 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -9,14 +9,9 @@ namespace :gitlab do task create: :gitlab_environment do warn_user_is_not_gitlab - Rake::Task['gitlab:backup:db:create'].invoke - Rake::Task['gitlab:backup:repo:create'].invoke - Rake::Task['gitlab:backup:uploads:create'].invoke - Rake::Task['gitlab:backup:builds:create'].invoke - Rake::Task['gitlab:backup:artifacts:create'].invoke - Rake::Task['gitlab:backup:pages:create'].invoke - Rake::Task['gitlab:backup:lfs:create'].invoke - Rake::Task['gitlab:backup:registry:create'].invoke + %w(db repo uploads builds artifacts pages lfs terraform_state registry packages).each do |type| + Rake::Task["gitlab:backup:#{type}:create"].invoke + end backup = Backup::Manager.new(progress) backup.write_info @@ -83,7 +78,9 @@ namespace :gitlab do Rake::Task['gitlab:backup:artifacts:restore'].invoke unless backup.skipped?('artifacts') Rake::Task['gitlab:backup:pages:restore'].invoke unless backup.skipped?('pages') Rake::Task['gitlab:backup:lfs:restore'].invoke unless backup.skipped?('lfs') + Rake::Task['gitlab:backup:terraform_state:restore'].invoke unless backup.skipped?('terraform_state') Rake::Task['gitlab:backup:registry:restore'].invoke unless backup.skipped?('registry') + Rake::Task['gitlab:backup:packages:restore'].invoke unless backup.skipped?('packages') Rake::Task['gitlab:shell:setup'].invoke Rake::Task['cache:clear'].invoke @@ -133,8 +130,12 @@ namespace :gitlab do if ENV["SKIP"] && ENV["SKIP"].include?("db") puts_time "[SKIPPED]".color(:cyan) else - Backup::Database.new(progress).dump - puts_time "done".color(:green) + begin + Backup::Database.new(progress).dump + puts_time "done".color(:green) + rescue Backup::DatabaseBackupError => e + progress.puts "#{e.message}" + end end end @@ -166,8 +167,12 @@ namespace :gitlab do if ENV["SKIP"] && ENV["SKIP"].include?("builds") puts_time "[SKIPPED]".color(:cyan) else - Backup::Builds.new(progress).dump - puts_time "done".color(:green) + begin + Backup::Builds.new(progress).dump + puts_time "done".color(:green) + rescue Backup::FileBackupError => e + progress.puts "#{e.message}" + end end end @@ -185,8 +190,12 @@ namespace :gitlab do if ENV["SKIP"] && ENV["SKIP"].include?("uploads") puts_time "[SKIPPED]".color(:cyan) else - Backup::Uploads.new(progress).dump - puts_time "done".color(:green) + begin + Backup::Uploads.new(progress).dump + puts_time "done".color(:green) + rescue Backup::FileBackupError => e + progress.puts "#{e.message}" + end end end @@ -204,8 +213,12 @@ namespace :gitlab do if ENV["SKIP"] && ENV["SKIP"].include?("artifacts") puts_time "[SKIPPED]".color(:cyan) else - Backup::Artifacts.new(progress).dump - puts_time "done".color(:green) + begin + Backup::Artifacts.new(progress).dump + puts_time "done".color(:green) + rescue Backup::FileBackupError => e + progress.puts "#{e.message}" + end end end @@ -223,8 +236,12 @@ namespace :gitlab do if ENV["SKIP"] && ENV["SKIP"].include?("pages") puts_time "[SKIPPED]".color(:cyan) else - Backup::Pages.new(progress).dump - puts_time "done".color(:green) + begin + Backup::Pages.new(progress).dump + puts_time "done".color(:green) + rescue Backup::FileBackupError => e + progress.puts "#{e.message}" + end end end @@ -242,8 +259,12 @@ namespace :gitlab do if ENV["SKIP"] && ENV["SKIP"].include?("lfs") puts_time "[SKIPPED]".color(:cyan) else - Backup::Lfs.new(progress).dump - puts_time "done".color(:green) + begin + Backup::Lfs.new(progress).dump + puts_time "done".color(:green) + rescue Backup::FileBackupError => e + progress.puts "#{e.message}" + end end end @@ -254,6 +275,25 @@ namespace :gitlab do end end + namespace :terraform_state do + task create: :gitlab_environment do + puts_time "Dumping terraform states ... ".color(:blue) + + if ENV["SKIP"] && ENV["SKIP"].include?("terraform_state") + puts_time "[SKIPPED]".color(:cyan) + else + Backup::TerraformState.new(progress).dump + puts_time "done".color(:green) + end + end + + task restore: :gitlab_environment do + puts_time "Restoring terraform states ... ".color(:blue) + Backup::TerraformState.new(progress).restore + puts_time "done".color(:green) + end + end + namespace :registry do task create: :gitlab_environment do puts_time "Dumping container registry images ... ".color(:blue) @@ -262,8 +302,12 @@ namespace :gitlab do if ENV["SKIP"] && ENV["SKIP"].include?("registry") puts_time "[SKIPPED]".color(:cyan) else - Backup::Registry.new(progress).dump - puts_time "done".color(:green) + begin + Backup::Registry.new(progress).dump + puts_time "done".color(:green) + rescue Backup::FileBackupError => e + progress.puts "#{e.message}" + end end else puts_time "[DISABLED]".color(:cyan) @@ -282,6 +326,25 @@ namespace :gitlab do end end + namespace :packages do + task create: :gitlab_environment do + puts_time "Dumping packages ... ".color(:blue) + + if ENV['SKIP'] && ENV['SKIP'].include?('packages') + puts_time "[SKIPPED]".color(:cyan) + else + Backup::Packages.new(progress).dump + puts_time "done".color(:green) + end + end + + task restore: :gitlab_environment do + puts_time "Restoring packages ...".color(:blue) + Backup::Packages.new(progress).restore + puts_time "done".color(:green) + end + end + def puts_time(msg) progress.puts "#{Time.now} -- #{msg}" Gitlab::BackupLogger.info(message: "#{Rainbow.uncolor(msg)}") @@ -302,7 +365,7 @@ namespace :gitlab do if Feature.enabled?(:gitaly_backup, default_enabled: :yaml) max_concurrency = ENV['GITLAB_BACKUP_MAX_CONCURRENCY'].presence max_storage_concurrency = ENV['GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY'].presence - Backup::GitalyBackup.new(progress, parallel: max_concurrency, parallel_storage: max_storage_concurrency) + Backup::GitalyBackup.new(progress, max_parallelism: max_concurrency, storage_parallelism: max_storage_concurrency) else Backup::GitalyRpcBackup.new(progress) end diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index 8f033a41e3d..f908a7606fa 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -100,15 +100,13 @@ namespace :gitlab do namespace :sessions do desc "GitLab | Cleanup | Sessions | Clean ActiveSession lookup keys" task active_sessions_lookup_keys: :gitlab_environment do - use_redis_session_store = Gitlab::Utils.to_boolean(ENV['GITLAB_USE_REDIS_SESSIONS_STORE'], default: true) - redis_store_class = use_redis_session_store ? Gitlab::Redis::Sessions : Gitlab::Redis::SharedState session_key_pattern = "#{Gitlab::Redis::Sessions::USER_SESSIONS_LOOKUP_NAMESPACE}:*" last_save_check = Time.at(0) wait_time = 10.seconds cursor = 0 total_users_scanned = 0 - redis_store_class.with do |redis| + Gitlab::Redis::Sessions.with do |redis| begin cursor, keys = redis.scan(cursor, match: session_key_pattern) total_users_scanned += keys.count diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 9e733fc3a0f..efb0e1ef1e1 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -170,7 +170,7 @@ namespace :gitlab do # the `ActiveRecord::Base.connection` might be switched to another one # This is due to `if should_reconnect`: # https://github.com/rails/rails/blob/a81aeb63a007ede2fe606c50539417dada9030c7/activerecord/lib/active_record/railties/databases.rake#L622 - ActiveRecord::Base.establish_connection :main + ActiveRecord::Base.establish_connection :main # rubocop: disable Database/EstablishConnection Rake::Task['gitlab:db:create_dynamic_partitions'].invoke end diff --git a/lib/tasks/gitlab/docs/compile_deprecations.rake b/lib/tasks/gitlab/docs/compile_deprecations.rake index dc9788cb0b2..4ac68a9f850 100644 --- a/lib/tasks/gitlab/docs/compile_deprecations.rake +++ b/lib/tasks/gitlab/docs/compile_deprecations.rake @@ -4,19 +4,19 @@ namespace :gitlab do namespace :docs do desc "Generate deprecation list from individual files" task :compile_deprecations do - require_relative '../../../../tooling/deprecations/docs' - - File.write(Deprecations::Docs.path, Deprecations::Docs.render) - - puts "Deprecations compiled to #{Deprecations::Docs.path}" + require_relative '../../../../tooling/docs/deprecation_handling' + path = Rails.root.join("doc/update/deprecations.md") + File.write(path, Docs::DeprecationHandling.new('deprecation').render) + puts "Deprecations compiled to #{path}" end desc "Check that the deprecation doc is up to date" task :check_deprecations do - require_relative '../../../../tooling/deprecations/docs' + require_relative '../../../../tooling/docs/deprecation_handling' + path = Rails.root.join("doc/update/deprecations.md") - contents = Deprecations::Docs.render - doc = File.read(Deprecations::Docs.path) + contents = Docs::DeprecationHandling.new('deprecation').render + doc = File.read(path) if doc == contents puts "Deprecations doc is up to date." @@ -25,5 +25,28 @@ namespace :gitlab do abort end end + + desc "Generate removal list from individual files" + task :compile_removals do + require_relative '../../../../tooling/docs/deprecation_handling' + path = Rails.root.join("doc/update/removals.md") + File.write(path, Docs::DeprecationHandling.new('removal').render) + puts "Removals compiled to #{path}" + end + + desc "Check that the removal doc is up to date" + task :check_removals do + require_relative '../../../../tooling/docs/deprecation_handling' + path = Rails.root.join("doc/update/removals.md") + contents = Docs::DeprecationHandling.new('removal').render + doc = File.read(path) + + if doc == contents + puts "Removals doc is up to date." + else + format_output('Removals doc is outdated! You (or your technical writer) can update it by running `bin/rake gitlab:docs:compile_removals`.') + abort + end + end end end diff --git a/lib/tasks/gitlab/docs/redirect.rake b/lib/tasks/gitlab/docs/redirect.rake index 123a4775605..e7ece9e0fdd 100644 --- a/lib/tasks/gitlab/docs/redirect.rake +++ b/lib/tasks/gitlab/docs/redirect.rake @@ -51,7 +51,7 @@ namespace :gitlab do post.puts "remove_date: '#{date}'" post.puts '---' post.puts - post.puts "This file was moved to [another location](#{new_path})." + post.puts "This document was moved to [another location](#{new_path})." post.puts post.puts "" post.puts "" diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index b01a7902bf2..18c68615637 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -2,41 +2,6 @@ namespace :gitlab do namespace :gitaly do - desc 'Installs gitaly for running tests within gitlab-development-kit' - task :test_install, [:dir, :storage_path, :repo] => :gitlab_environment do |t, args| - inside_gdk = Rails.env.test? && File.exist?(Rails.root.join('../GDK_ROOT')) - - if ENV['FORCE_GITALY_INSTALL'] || !inside_gdk - Rake::Task["gitlab:gitaly:install"].invoke(*args) - - next - end - - gdk_gitaly_dir = ENV.fetch('GDK_GITALY', Rails.root.join('../gitaly')) - - # Our test setup expects a git repo, so clone rather than copy - clone_repo(gdk_gitaly_dir, args.dir, clone_opts: %w[--depth 1]) unless Dir.exist?(args.dir) - - # We assume the GDK gitaly already compiled binaries - build_dir = File.join(gdk_gitaly_dir, '_build') - FileUtils.cp_r(build_dir, args.dir) - - # We assume the GDK gitaly already ran bundle install - bundle_dir = File.join(gdk_gitaly_dir, 'ruby', '.bundle') - FileUtils.cp_r(bundle_dir, File.join(args.dir, 'ruby')) - - # For completeness we copy this for gitaly's make target - ruby_bundle_file = File.join(gdk_gitaly_dir, '.ruby-bundle') - FileUtils.cp_r(ruby_bundle_file, args.dir) - - gitaly_binary = File.join(build_dir, 'bin', 'gitaly') - warn_gitaly_out_of_date!(gitaly_binary, Gitlab::GitalyClient.expected_server_version) - rescue Errno::ENOENT => e - puts "Could not copy files, did you run `gdk update`? Error: #{e.message}" - - raise - end - desc 'GitLab | Gitaly | Clone and checkout gitaly' task :clone, [:dir, :storage_path, :repo] => :gitlab_environment do |t, args| warn_user_is_not_gitlab @@ -60,9 +25,6 @@ Usage: rake "gitlab:gitaly:install[/installation/dir,/storage/path]") storage_paths = { 'default' => args.storage_path } Gitlab::SetupHelper::Gitaly.create_configuration(args.dir, storage_paths) - # In CI we run scripts/gitaly-test-build - next if ENV['CI'].present? - Dir.chdir(args.dir) do Bundler.with_original_env do env = { "RUBYOPT" => nil, "BUNDLE_GEMFILE" => nil } diff --git a/lib/tasks/gitlab/seed/group_seed.rake b/lib/tasks/gitlab/seed/group_seed.rake index a9a350fb6c3..491cf782985 100644 --- a/lib/tasks/gitlab/seed/group_seed.rake +++ b/lib/tasks/gitlab/seed/group_seed.rake @@ -125,7 +125,7 @@ class GroupSeeder name: FFaker::Name.name, email: FFaker::Internet.email, confirmed_at: DateTime.now, - password: Devise.friendly_token + password: Gitlab::Password.test_default ) end diff --git a/lib/version_check.rb b/lib/version_check.rb index e5a4c244c7a..2d132001f54 100644 --- a/lib/version_check.rb +++ b/lib/version_check.rb @@ -16,14 +16,6 @@ class VersionCheck { "REFERER": Gitlab.config.gitlab.url } end - # This is temporary and will be removed when the new UI is hooked up - # to the version_check.json endpoint. - def self.image_url - encoded_data = Base64.urlsafe_encode64(data.to_json) - - "#{host}/check.svg?gitlab_info=#{encoded_data}" - end - def self.url encoded_data = Base64.urlsafe_encode64(data.to_json) -- cgit v1.2.3