Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2023-10-19 15:57:54 +0300
committerGitLab Bot <gitlab-bot@gitlab.com>2023-10-19 15:57:54 +0300
commit419c53ec62de6e97a517abd5fdd4cbde3a942a34 (patch)
tree1f43a548b46bca8a5fb8fe0c31cef1883d49c5b6 /lib/gitlab
parent1da20d9135b3ad9e75e65b028bffc921aaf8deb7 (diff)
Add latest changes from gitlab-org/gitlab@16-5-stable-eev16.5.0-rc42
Diffstat (limited to 'lib/gitlab')
-rw-r--r--lib/gitlab/analytics/cycle_analytics/aggregated/base_query_builder.rb20
-rw-r--r--lib/gitlab/analytics/cycle_analytics/request_params.rb27
-rw-r--r--lib/gitlab/application_rate_limiter.rb3
-rw-r--r--lib/gitlab/auth.rb10
-rw-r--r--lib/gitlab/auth/auth_finders.rb17
-rw-r--r--lib/gitlab/auth/ldap/auth_hash.rb8
-rw-r--r--lib/gitlab/auth/ldap/config.rb8
-rw-r--r--lib/gitlab/auth/o_auth/auth_hash.rb7
-rw-r--r--lib/gitlab/auth/o_auth/user.rb4
-rw-r--r--lib/gitlab/background_migration/backfill_finding_id_in_vulnerabilities.rb28
-rw-r--r--lib/gitlab/background_migration/backfill_has_remediations_of_vulnerability_reads.rb43
-rw-r--r--lib/gitlab/background_migration/backfill_integrations_enable_ssl_verification.rb4
-rw-r--r--lib/gitlab/background_migration/delete_orphans_approval_merge_request_rules2.rb36
-rw-r--r--lib/gitlab/background_migration/delete_orphans_approval_project_rules2.rb38
-rw-r--r--lib/gitlab/background_migration/migrate_pages_to_zip_storage.rb16
-rw-r--r--lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb2
-rw-r--r--lib/gitlab/background_migration/update_workspaces_config_version.rb13
-rw-r--r--lib/gitlab/base_doorkeeper_controller.rb4
-rw-r--r--lib/gitlab/bitbucket_import/error_tracking.rb16
-rw-r--r--lib/gitlab/bitbucket_import/importer.rb46
-rw-r--r--lib/gitlab/bitbucket_import/importers/issue_importer.rb68
-rw-r--r--lib/gitlab/bitbucket_import/importers/issue_notes_importer.rb51
-rw-r--r--lib/gitlab/bitbucket_import/importers/issues_importer.rb60
-rw-r--r--lib/gitlab/bitbucket_import/importers/issues_notes_importer.rb45
-rw-r--r--lib/gitlab/bitbucket_import/importers/lfs_object_importer.rb32
-rw-r--r--lib/gitlab/bitbucket_import/importers/lfs_objects_importer.rb57
-rw-r--r--lib/gitlab/bitbucket_import/importers/pull_request_importer.rb3
-rw-r--r--lib/gitlab/bitbucket_import/importers/pull_request_notes_importer.rb34
-rw-r--r--lib/gitlab/bitbucket_import/importers/pull_requests_notes_importer.rb45
-rw-r--r--lib/gitlab/bitbucket_import/importers/repository_importer.rb6
-rw-r--r--lib/gitlab/bitbucket_import/parallel_scheduling.rb10
-rw-r--r--lib/gitlab/bitbucket_import/ref_converter.rb48
-rw-r--r--lib/gitlab/bitbucket_server_import/importers/pull_request_importer.rb10
-rw-r--r--lib/gitlab/bitbucket_server_import/importers/pull_requests_importer.rb25
-rw-r--r--lib/gitlab/bitbucket_server_import/project_creator.rb7
-rw-r--r--lib/gitlab/changelog/generator.rb2
-rw-r--r--lib/gitlab/chat.rb10
-rw-r--r--lib/gitlab/checks/global_file_size_check.rb30
-rw-r--r--lib/gitlab/checks/security/policy_check.rb13
-rw-r--r--lib/gitlab/checks/single_change_access.rb1
-rw-r--r--lib/gitlab/checks/tag_check.rb13
-rw-r--r--lib/gitlab/ci/build/artifacts/metadata.rb4
-rw-r--r--lib/gitlab/ci/build/context/build.rb10
-rw-r--r--lib/gitlab/ci/build/duration_parser.rb2
-rw-r--r--lib/gitlab/ci/components/instance_path.rb57
-rw-r--r--lib/gitlab/ci/config/entry/artifacts.rb2
-rw-r--r--lib/gitlab/ci/config/entry/job.rb2
-rw-r--r--lib/gitlab/ci/config/external/file/base.rb6
-rw-r--r--lib/gitlab/ci/config/external/file/component.rb18
-rw-r--r--lib/gitlab/ci/config/header/input.rb8
-rw-r--r--lib/gitlab/ci/config/interpolation/block.rb2
-rw-r--r--lib/gitlab/ci/config/interpolation/context.rb23
-rw-r--r--lib/gitlab/ci/config/interpolation/functions/base.rb6
-rw-r--r--lib/gitlab/ci/config/interpolation/functions/expand_vars.rb33
-rw-r--r--lib/gitlab/ci/config/interpolation/functions_stack.rb10
-rw-r--r--lib/gitlab/ci/config/interpolation/inputs/base_input.rb10
-rw-r--r--lib/gitlab/ci/config/interpolation/inputs/string_input.rb18
-rw-r--r--lib/gitlab/ci/config/interpolation/interpolator.rb7
-rw-r--r--lib/gitlab/ci/config/yaml/loader.rb13
-rw-r--r--lib/gitlab/ci/config/yaml/result.rb4
-rw-r--r--lib/gitlab/ci/lint.rb4
-rw-r--r--lib/gitlab/ci/parsers/security/common.rb5
-rw-r--r--lib/gitlab/ci/parsers/test/junit.rb9
-rw-r--r--lib/gitlab/ci/pipeline/chain/skip.rb2
-rw-r--r--lib/gitlab/ci/pipeline/chain/validate/abilities.rb2
-rw-r--r--lib/gitlab/ci/pipeline/expression.rb1
-rw-r--r--lib/gitlab/ci/pipeline/expression/lexeme/and.rb2
-rw-r--r--lib/gitlab/ci/pipeline/expression/lexeme/equals.rb2
-rw-r--r--lib/gitlab/ci/pipeline/expression/lexeme/matches.rb2
-rw-r--r--lib/gitlab/ci/pipeline/expression/lexeme/not_equals.rb2
-rw-r--r--lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb2
-rw-r--r--lib/gitlab/ci/pipeline/expression/lexeme/null.rb2
-rw-r--r--lib/gitlab/ci/pipeline/expression/lexeme/or.rb2
-rw-r--r--lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_close.rb2
-rw-r--r--lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_open.rb2
-rw-r--r--lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb18
-rw-r--r--lib/gitlab/ci/pipeline/expression/lexeme/pattern/regular_expression.rb37
-rw-r--r--lib/gitlab/ci/pipeline/expression/lexeme/string.rb2
-rw-r--r--lib/gitlab/ci/pipeline/expression/lexeme/variable.rb2
-rw-r--r--lib/gitlab/ci/reports/security/finding.rb4
-rw-r--r--lib/gitlab/ci/status/canceled.rb2
-rw-r--r--lib/gitlab/ci/status/core.rb4
-rw-r--r--lib/gitlab/ci/status/created.rb2
-rw-r--r--lib/gitlab/ci/status/failed.rb2
-rw-r--r--lib/gitlab/ci/status/manual.rb2
-rw-r--r--lib/gitlab/ci/status/pending.rb2
-rw-r--r--lib/gitlab/ci/status/pipeline/blocked.rb2
-rw-r--r--lib/gitlab/ci/status/pipeline/delayed.rb2
-rw-r--r--lib/gitlab/ci/status/preparing.rb2
-rw-r--r--lib/gitlab/ci/status/running.rb4
-rw-r--r--lib/gitlab/ci/status/scheduled.rb2
-rw-r--r--lib/gitlab/ci/status/skipped.rb2
-rw-r--r--lib/gitlab/ci/status/success.rb2
-rw-r--r--lib/gitlab/ci/status/success_warning.rb6
-rw-r--r--lib/gitlab/ci/status/waiting_for_resource.rb6
-rw-r--r--lib/gitlab/ci/templates/Code-Quality.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Cosign.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Docker.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Python.gitlab-ci.yml5
-rw-r--r--lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Security/Container-Scanning.latest.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Security/SAST-IaC.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Security/SAST-IaC.latest.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/trace/section_parser.rb2
-rw-r--r--lib/gitlab/ci/variables/collection/item.rb6
-rw-r--r--lib/gitlab/ci/yaml_processor.rb3
-rw-r--r--lib/gitlab/cleanup/project_uploads.rb2
-rw-r--r--lib/gitlab/color.rb2
-rw-r--r--lib/gitlab/config/entry/legacy_validation_helpers.rb9
-rw-r--r--lib/gitlab/config/loader/multi_doc_yaml.rb2
-rw-r--r--lib/gitlab/content_security_policy/config_loader.rb2
-rw-r--r--lib/gitlab/database/background_migration/batch_optimizer.rb2
-rw-r--r--lib/gitlab/database/gitlab_schema.rb35
-rw-r--r--lib/gitlab/database/gitlab_schema_info.rb72
-rw-r--r--lib/gitlab/database/load_balancing/service_discovery.rb10
-rw-r--r--lib/gitlab/database/migration.rb6
-rw-r--r--lib/gitlab/database/migration_helpers.rb19
-rw-r--r--lib/gitlab/database/migration_helpers/swapping.rb50
-rw-r--r--lib/gitlab/database/migration_helpers/v2.rb2
-rw-r--r--lib/gitlab/database/migrations/batched_background_migration_helpers.rb60
-rw-r--r--lib/gitlab/database/migrations/milestone_mixin.rb35
-rw-r--r--lib/gitlab/database/migrations/observers/query_statistics.rb8
-rw-r--r--lib/gitlab/database/migrations/runner.rb2
-rw-r--r--lib/gitlab/database/migrations/swap_columns.rb39
-rw-r--r--lib/gitlab/database/migrations/swap_columns_default.rb62
-rw-r--r--lib/gitlab/database/migrations/version.rb76
-rw-r--r--lib/gitlab/database/partitioning/partition_manager.rb1
-rw-r--r--lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb2
-rw-r--r--lib/gitlab/database/rename_reserved_paths_migration/v1.rb42
-rw-r--r--lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb99
-rw-r--r--lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb196
-rw-r--r--lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb106
-rw-r--r--lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb78
-rw-r--r--lib/gitlab/database_importers/work_items/hierarchy_restrictions_importer.rb61
-rw-r--r--lib/gitlab/database_importers/work_items/related_links_restrictions_importer.rb74
-rw-r--r--lib/gitlab/dependency_linker/base_linker.rb6
-rw-r--r--lib/gitlab/dependency_linker/gemfile_linker.rb4
-rw-r--r--lib/gitlab/dependency_linker/godeps_json_linker.rb2
-rw-r--r--lib/gitlab/dependency_linker/podspec_linker.rb2
-rw-r--r--lib/gitlab/deploy_key_access.rb11
-rw-r--r--lib/gitlab/diff/file.rb2
-rw-r--r--lib/gitlab/diff/highlight.rb2
-rw-r--r--lib/gitlab/diff/pair_selector.rb2
-rw-r--r--lib/gitlab/diff/position_tracer.rb5
-rw-r--r--lib/gitlab/diff/suggestions_parser.rb2
-rw-r--r--lib/gitlab/doctor/reset_tokens.rb66
-rw-r--r--lib/gitlab/email/handler/base_handler.rb2
-rw-r--r--lib/gitlab/email/handler/create_issue_handler.rb4
-rw-r--r--lib/gitlab/email/handler/create_merge_request_handler.rb4
-rw-r--r--lib/gitlab/email/handler/create_note_on_issuable_handler.rb2
-rw-r--r--lib/gitlab/email/handler/service_desk_handler.rb14
-rw-r--r--lib/gitlab/email/message/build_ios_app_guide.rb57
-rw-r--r--lib/gitlab/email/message/in_product_marketing/helper.rb97
-rw-r--r--lib/gitlab/email/receiver.rb57
-rw-r--r--lib/gitlab/encoding_helper.rb4
-rw-r--r--lib/gitlab/error_tracking/error_repository/open_api_strategy.rb2
-rw-r--r--lib/gitlab/exclusive_lease.rb95
-rw-r--r--lib/gitlab/exclusive_lease_helpers.rb2
-rw-r--r--lib/gitlab/experiment/rollout/feature.rb7
-rw-r--r--lib/gitlab/git.rb2
-rw-r--r--lib/gitlab/git/base_error.rb2
-rw-r--r--lib/gitlab/git/blame.rb15
-rw-r--r--lib/gitlab/git/diff.rb12
-rw-r--r--lib/gitlab/git/pre_receive_error.rb2
-rw-r--r--lib/gitlab/git/repository.rb22
-rw-r--r--lib/gitlab/git/rugged_impl/use_rugged.rb15
-rw-r--r--lib/gitlab/git_audit_event.rb28
-rw-r--r--lib/gitlab/gitaly_client.rb121
-rw-r--r--lib/gitlab/gitaly_client/commit_service.rb3
-rw-r--r--lib/gitlab/gitaly_client/namespace_service.rb57
-rw-r--r--lib/gitlab/gitaly_client/repository_service.rb13
-rw-r--r--lib/gitlab/github_import/bulk_importing.rb12
-rw-r--r--lib/gitlab/github_import/client.rb4
-rw-r--r--lib/gitlab/github_import/clients/proxy.rb24
-rw-r--r--lib/gitlab/github_import/exceptions.rb2
-rw-r--r--lib/gitlab/github_import/importer/attachments/issues_importer.rb2
-rw-r--r--lib/gitlab/github_import/importer/attachments/merge_requests_importer.rb2
-rw-r--r--lib/gitlab/github_import/importer/attachments/notes_importer.rb2
-rw-r--r--lib/gitlab/github_import/importer/attachments/releases_importer.rb2
-rw-r--r--lib/gitlab/github_import/importer/diff_note_importer.rb10
-rw-r--r--lib/gitlab/github_import/importer/issue_importer.rb5
-rw-r--r--lib/gitlab/github_import/importer/note_importer.rb15
-rw-r--r--lib/gitlab/github_import/importer/pull_request_importer.rb2
-rw-r--r--lib/gitlab/github_import/importer/pull_requests/review_requests_importer.rb7
-rw-r--r--lib/gitlab/github_import/importer/pull_requests_importer.rb2
-rw-r--r--lib/gitlab/github_import/parallel_scheduling.rb6
-rw-r--r--lib/gitlab/github_import/representation/diff_note.rb2
-rw-r--r--lib/gitlab/github_import/representation/diff_notes/discussion_id.rb2
-rw-r--r--lib/gitlab/github_import/representation/diff_notes/suggestion_formatter.rb2
-rw-r--r--lib/gitlab/github_import/representation/note.rb2
-rw-r--r--lib/gitlab/github_import/settings.rb7
-rw-r--r--lib/gitlab/github_import/user_finder.rb3
-rw-r--r--lib/gitlab/golang.rb4
-rw-r--r--lib/gitlab/gon_helper.rb25
-rw-r--r--lib/gitlab/graphql/authorize/connection_filter_extension.rb7
-rw-r--r--lib/gitlab/graphql/deprecations.rb15
-rw-r--r--lib/gitlab/graphql/deprecations/deprecation.rb6
-rw-r--r--lib/gitlab/graphql/pagination/active_record_array_connection.rb1
-rw-r--r--lib/gitlab/graphql/queries.rb16
-rw-r--r--lib/gitlab/harbor/query.rb2
-rw-r--r--lib/gitlab/hashed_storage/migrator.rb125
-rw-r--r--lib/gitlab/hashed_storage/rake_helper.rb129
-rw-r--r--lib/gitlab/health_checks/puma_check.rb2
-rw-r--r--lib/gitlab/http.rb116
-rw-r--r--lib/gitlab/http_connection_adapter.rb2
-rw-r--r--lib/gitlab/i18n.rb19
-rw-r--r--lib/gitlab/i18n/po_linter.rb2
-rw-r--r--lib/gitlab/i18n/translation_entry.rb4
-rw-r--r--lib/gitlab/import/import_failure_service.rb20
-rw-r--r--lib/gitlab/import_export/project/import_export.yml1
-rw-r--r--lib/gitlab/import_export/project/relation_factory.rb1
-rw-r--r--lib/gitlab/internal_events.rb30
-rw-r--r--lib/gitlab/internal_events/event_definitions.rb4
-rw-r--r--lib/gitlab/jira/dvcs.rb2
-rw-r--r--lib/gitlab/legacy_http.rb78
-rw-r--r--lib/gitlab/mail_room.rb14
-rw-r--r--lib/gitlab/merge_requests/mergeability/check_result.rb4
-rw-r--r--lib/gitlab/merge_requests/message_generator.rb2
-rw-r--r--lib/gitlab/metrics/exporter/base_exporter.rb2
-rw-r--r--lib/gitlab/metrics/requests_rack_middleware.rb2
-rw-r--r--lib/gitlab/metrics/samplers/puma_sampler.rb2
-rw-r--r--lib/gitlab/metrics/subscribers/active_record.rb4
-rw-r--r--lib/gitlab/metrics/system.rb14
-rw-r--r--lib/gitlab/metrics/web_transaction.rb17
-rw-r--r--lib/gitlab/middleware/compressed_json.rb8
-rw-r--r--lib/gitlab/middleware/go.rb2
-rw-r--r--lib/gitlab/middleware/handle_malformed_strings.rb2
-rw-r--r--lib/gitlab/middleware/path_traversal_check.rb58
-rw-r--r--lib/gitlab/middleware/read_only.rb2
-rw-r--r--lib/gitlab/middleware/sidekiq_web_static.rb2
-rw-r--r--lib/gitlab/middleware/static.rb2
-rw-r--r--lib/gitlab/observability.rb132
-rw-r--r--lib/gitlab/pages/cache_control.rb105
-rw-r--r--lib/gitlab/pagination/cursor_based_keyset.rb25
-rw-r--r--lib/gitlab/patch/hangouts_chat_http_override.rb21
-rw-r--r--lib/gitlab/path_regex.rb26
-rw-r--r--lib/gitlab/path_traversal.rb4
-rw-r--r--lib/gitlab/prometheus/metric_group.rb34
-rw-r--r--lib/gitlab/prometheus/parsing_error.rb7
-rw-r--r--lib/gitlab/prometheus/queries/base_query.rb33
-rw-r--r--lib/gitlab/prometheus/queries/deployment_query.rb40
-rw-r--r--lib/gitlab/prometheus/queries/environment_query.rb34
-rw-r--r--lib/gitlab/prometheus/queries/matched_metric_query.rb82
-rw-r--r--lib/gitlab/prometheus/queries/query_additional_metrics.rb101
-rw-r--r--lib/gitlab/prometheus/queries/validate_query.rb21
-rw-r--r--lib/gitlab/prometheus/query_variables.rb31
-rw-r--r--lib/gitlab/puma/error_handler.rb41
-rw-r--r--lib/gitlab/push_options.rb2
-rw-r--r--lib/gitlab/query_limiting/transaction.rb2
-rw-r--r--lib/gitlab/quick_actions/extractor.rb10
-rw-r--r--lib/gitlab/quick_actions/spend_time_and_date_separator.rb2
-rw-r--r--lib/gitlab/quick_actions/timeline_text_and_date_time_separator.rb6
-rw-r--r--lib/gitlab/quick_actions/work_item_actions.rb56
-rw-r--r--lib/gitlab/rack_attack/request.rb11
-rw-r--r--lib/gitlab/redis/hll.rb2
-rw-r--r--lib/gitlab/redis/multi_store.rb13
-rw-r--r--lib/gitlab/redis/queues_metadata.rb9
-rw-r--r--lib/gitlab/redis/workhorse.rb9
-rw-r--r--lib/gitlab/redis/wrapper.rb22
-rw-r--r--lib/gitlab/regex.rb51
-rw-r--r--lib/gitlab/regex/packages.rb70
-rw-r--r--lib/gitlab/request_forgery_protection.rb4
-rw-r--r--lib/gitlab/robots_txt/parser.rb4
-rw-r--r--lib/gitlab/runtime.rb8
-rw-r--r--lib/gitlab/search_results.rb4
-rw-r--r--lib/gitlab/shell.rb104
-rw-r--r--lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb7
-rw-r--r--lib/gitlab/sidekiq_middleware/extra_done_log_metadata.rb2
-rw-r--r--lib/gitlab/sidekiq_middleware/skip_jobs.rb2
-rw-r--r--lib/gitlab/slash_commands/run.rb2
-rw-r--r--lib/gitlab/time_tracking_formatter.rb7
-rw-r--r--lib/gitlab/url_blocker.rb50
-rw-r--r--lib/gitlab/url_builder.rb2
-rw-r--r--lib/gitlab/usage/metric_definition.rb56
-rw-r--r--lib/gitlab/usage/metrics/instrumentations/base_metric.rb2
-rw-r--r--lib/gitlab/usage/metrics/instrumentations/container_registry_db_enabled_metric.rb15
-rw-r--r--lib/gitlab/usage/metrics/instrumentations/count_csv_imports_metric.rb15
-rw-r--r--lib/gitlab/usage/metrics/instrumentations/count_jira_imports_metric.rb15
-rw-r--r--lib/gitlab/usage/metrics/instrumentations/count_packages_metric.rb15
-rw-r--r--lib/gitlab/usage/metrics/instrumentations/count_projects_metric.rb18
-rw-r--r--lib/gitlab/usage/metrics/instrumentations/total_count_metric.rb35
-rw-r--r--lib/gitlab/usage_data.rb39
-rw-r--r--lib/gitlab/usage_data_counters/ci_template_unique_counter.rb6
-rw-r--r--lib/gitlab/workhorse.rb7
294 files changed, 2752 insertions, 2780 deletions
diff --git a/lib/gitlab/analytics/cycle_analytics/aggregated/base_query_builder.rb b/lib/gitlab/analytics/cycle_analytics/aggregated/base_query_builder.rb
index 41f94e79f91..fc0e4ab5a0d 100644
--- a/lib/gitlab/analytics/cycle_analytics/aggregated/base_query_builder.rb
+++ b/lib/gitlab/analytics/cycle_analytics/aggregated/base_query_builder.rb
@@ -50,8 +50,7 @@ module Gitlab
def filter_author(query)
return query if params[:author_username].blank?
- user = User.by_username(params[:author_username]).first
-
+ user = find_user(params[:author_username])
return query.none if user.blank?
query.authored(user)
@@ -60,11 +59,7 @@ module Gitlab
def filter_milestone_ids(query)
return query if params[:milestone_title].blank?
- milestone = MilestonesFinder
- .new(group_ids: root_ancestor.self_and_descendant_ids, project_ids: root_ancestor.all_projects.select(:id), title: params[:milestone_title])
- .execute
- .first
-
+ milestone = find_milestone(params[:milestone_title])
return query.none if milestone.blank?
query.with_milestone_id(milestone.id)
@@ -115,6 +110,17 @@ module Gitlab
private
attr_reader :stage, :params, :root_ancestor, :stage_event_model
+
+ def find_milestone(title)
+ MilestonesFinder
+ .new(group_ids: root_ancestor.self_and_descendant_ids, project_ids: root_ancestor.all_projects.select(:id), title: title)
+ .execute
+ .first
+ end
+
+ def find_user(username)
+ User.by_username(username).first
+ end
end
# rubocop: enable CodeReuse/ActiveRecord
end
diff --git a/lib/gitlab/analytics/cycle_analytics/request_params.rb b/lib/gitlab/analytics/cycle_analytics/request_params.rb
index 2c4b0215307..cea25ba2db4 100644
--- a/lib/gitlab/analytics/cycle_analytics/request_params.rb
+++ b/lib/gitlab/analytics/cycle_analytics/request_params.rb
@@ -12,6 +12,24 @@ module Gitlab
MAX_RANGE_DAYS = 180.days.freeze
DEFAULT_DATE_RANGE = 29.days # 30 including Date.today
+ NEGATABLE_PARAMS = [
+ :assignee_username,
+ :author_username,
+ :epic_id,
+ :iteration_id,
+ :label_name,
+ :milestone_title,
+ :my_reaction_emoji,
+ :weight
+ ].freeze
+
+ LICENSED_PARAMS = [
+ :weight,
+ :epic_id,
+ :my_reaction_emoji,
+ :iteration_id
+ ].freeze
+
STRONG_PARAMS_DEFINITION = [
:created_before,
:created_after,
@@ -22,9 +40,11 @@ module Gitlab
:page,
:stage_id,
:end_event_filter,
+ *LICENSED_PARAMS,
label_name: [].freeze,
assignee_username: [].freeze,
- project_ids: [].freeze
+ project_ids: [].freeze,
+ not: NEGATABLE_PARAMS
].freeze
FINDER_PARAM_NAMES = [
@@ -46,6 +66,11 @@ module Gitlab
attribute :page
attribute :stage_id
attribute :end_event_filter
+ attribute :weight
+ attribute :epic_id
+ attribute :my_reaction_emoji
+ attribute :iteration_id
+ attribute :not, default: -> { {} }
FINDER_PARAM_NAMES.each do |param_name|
attribute param_name
diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb
index 8d7712951e1..bf3f5b61825 100644
--- a/lib/gitlab/application_rate_limiter.rb
+++ b/lib/gitlab/application_rate_limiter.rb
@@ -58,7 +58,8 @@ module Gitlab
fetch_google_ip_list: { threshold: 10, interval: 1.minute },
project_fork_sync: { threshold: 10, interval: 30.minutes },
ai_action: { threshold: 160, interval: 8.hours },
- jobs_index: { threshold: 600, interval: 1.minute },
+ vertex_embeddings_api: { threshold: 450, interval: 1.minute },
+ jobs_index: { threshold: -> { application_settings.project_jobs_api_rate_limit }, interval: 1.minute },
bulk_import: { threshold: 6, interval: 1.minute },
projects_api_rate_limit_unauthenticated: {
threshold: -> { application_settings.projects_api_rate_limit_unauthenticated }, interval: 10.minutes
diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb
index 9ddfc995535..fc1f7a1583c 100644
--- a/lib/gitlab/auth.rb
+++ b/lib/gitlab/auth.rb
@@ -426,15 +426,21 @@ module Gitlab
end
def unavailable_scopes_for_resource(resource)
- unavailable_observability_scopes_for_resource(resource)
+ unavailable_observability_scopes_for_resource(resource) +
+ unavailable_ai_features_scopes_for_resource(resource)
end
def unavailable_observability_scopes_for_resource(resource)
- return [] if resource.is_a?(Group) && Gitlab::Observability.enabled?(resource)
+ return [] if (resource.is_a?(Project) || resource.is_a?(Group)) &&
+ Gitlab::Observability.should_enable_observability_auth_scopes?(resource)
OBSERVABILITY_SCOPES
end
+ def unavailable_ai_features_scopes_for_resource(_resource)
+ AI_FEATURES_SCOPES
+ end
+
def non_admin_available_scopes
API_SCOPES + REPOSITORY_SCOPES + registry_scopes + OBSERVABILITY_SCOPES + AI_FEATURES_SCOPES
end
diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb
index a715f17ecd6..25465e73b95 100644
--- a/lib/gitlab/auth/auth_finders.rb
+++ b/lib/gitlab/auth/auth_finders.rb
@@ -32,6 +32,17 @@ module Gitlab
RUNNER_JOB_TOKEN_PARAM = :token
PATH_DEPENDENT_FEED_TOKEN_REGEX = /\A#{User::FEED_TOKEN_PREFIX}(\h{64})-(\d+)\z/
+ PARAM_TOKEN_KEYS = [
+ PRIVATE_TOKEN_PARAM,
+ JOB_TOKEN_PARAM,
+ RUNNER_JOB_TOKEN_PARAM
+ ].map(&:to_s).freeze
+ HEADER_TOKEN_KEYS = [
+ PRIVATE_TOKEN_HEADER,
+ JOB_TOKEN_HEADER,
+ DEPLOY_TOKEN_HEADER
+ ].freeze
+
# Check the Rails session for valid authentication details
def find_user_from_warden
current_request.env['warden']&.authenticate if verified_request?
@@ -204,6 +215,12 @@ module Gitlab
end
end
+ def authentication_token_present?
+ PARAM_TOKEN_KEYS.intersection(current_request.params.keys).any? ||
+ HEADER_TOKEN_KEYS.intersection(current_request.env.keys).any? ||
+ parsed_oauth_token.present?
+ end
+
private
def find_user_from_job_bearer_token
diff --git a/lib/gitlab/auth/ldap/auth_hash.rb b/lib/gitlab/auth/ldap/auth_hash.rb
index 5435355f136..6d1d1519fc2 100644
--- a/lib/gitlab/auth/ldap/auth_hash.rb
+++ b/lib/gitlab/auth/ldap/auth_hash.rb
@@ -6,6 +6,8 @@ module Gitlab
module Auth
module Ldap
class AuthHash < Gitlab::Auth::OAuth::AuthHash
+ extend ::Gitlab::Utils::Override
+
def uid
@uid ||= Gitlab::Auth::Ldap::Person.normalize_dn(super)
end
@@ -44,6 +46,12 @@ module Gitlab
def ldap_config
@ldap_config ||= Gitlab::Auth::Ldap::Config.new(self.provider)
end
+
+ # Overrding this method as LDAP allows email as the username !
+ override :get_username
+ def get_username
+ username_claims.map { |claim| get_from_auth_hash_or_info(claim) }.find(&:presence)
+ end
end
end
end
diff --git a/lib/gitlab/auth/ldap/config.rb b/lib/gitlab/auth/ldap/config.rb
index ed7caf84558..15e8cb04ea4 100644
--- a/lib/gitlab/auth/ldap/config.rb
+++ b/lib/gitlab/auth/ldap/config.rb
@@ -94,7 +94,7 @@ module Gitlab
def omniauth_options
opts = base_options.merge(
base: base,
- encryption: options['encryption'],
+ encryption: encryption,
filter: omniauth_user_filter,
name_proc: name_proc,
disable_verify_certificates: !options['verify_certificates'],
@@ -188,6 +188,10 @@ module Gitlab
options['sync_name']
end
+ def encryption
+ options['encryption'] || 'plain'
+ end
+
def name_proc
if allow_username_or_email_login
proc { |name| name.gsub(/@.*\z/, '') }
@@ -235,7 +239,7 @@ module Gitlab
end
def translate_method
- NET_LDAP_ENCRYPTION_METHOD[options['encryption']&.to_sym]
+ NET_LDAP_ENCRYPTION_METHOD[encryption.to_sym]
end
def tls_options
diff --git a/lib/gitlab/auth/o_auth/auth_hash.rb b/lib/gitlab/auth/o_auth/auth_hash.rb
index cce08750296..c2b49c1c068 100644
--- a/lib/gitlab/auth/o_auth/auth_hash.rb
+++ b/lib/gitlab/auth/o_auth/auth_hash.rb
@@ -68,7 +68,7 @@ module Gitlab
end
def provider_config
- Gitlab::Auth::OAuth::Provider.config_for(@provider) || {}
+ Gitlab::Auth::OAuth::Provider.config_for(provider) || {}
end
def provider_args
@@ -96,7 +96,10 @@ module Gitlab
end
def get_username
- username_claims.map { |claim| get_from_auth_hash_or_info(claim) }.find { |name| name.presence }
+ username_claims.map { |claim| get_from_auth_hash_or_info(claim) }
+ .find { |name| name.presence }
+ &.split("@")
+ &.first
end
def username_and_email
diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb
index 3981594478d..d70c788dac8 100644
--- a/lib/gitlab/auth/o_auth/user.rb
+++ b/lib/gitlab/auth/o_auth/user.rb
@@ -225,7 +225,7 @@ module Gitlab
if creating_linked_ldap_user?
username = ldap_person.username.presence
name = ldap_person.name.presence
- email = ldap_person.email.first.presence
+ email = ldap_person.email&.first.presence
end
username ||= auth_hash.username
@@ -272,7 +272,7 @@ module Gitlab
if creating_linked_ldap_user?
metadata.set_attribute_synced(:name, true) if gl_user.name == ldap_person.name
- metadata.set_attribute_synced(:email, true) if gl_user.email == ldap_person.email.first
+ metadata.set_attribute_synced(:email, true) if gl_user.email == ldap_person.email&.first
metadata.provider = ldap_person.provider
end
end
diff --git a/lib/gitlab/background_migration/backfill_finding_id_in_vulnerabilities.rb b/lib/gitlab/background_migration/backfill_finding_id_in_vulnerabilities.rb
new file mode 100644
index 00000000000..e3b77e3c7cd
--- /dev/null
+++ b/lib/gitlab/background_migration/backfill_finding_id_in_vulnerabilities.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # Backfills vulnerabilities.finding_id column based on vulnerability_occurrences.vulnerability_id column
+ class BackfillFindingIdInVulnerabilities < BatchedMigrationJob
+ operation_name :backfill_finding_id_in_vulnerabilities_table
+ scope_to ->(relation) { relation.where(finding_id: nil) }
+ feature_category :vulnerability_management
+
+ class VulnerabilitiesFindings < ApplicationRecord # rubocop:disable Style/Documentation
+ self.table_name = "vulnerability_occurrences"
+ end
+
+ def perform
+ each_sub_batch do |sub_batch|
+ connection.execute <<~SQL
+ UPDATE vulnerabilities
+ SET finding_id = vulnerability_occurrences.id
+ FROM vulnerability_occurrences
+ WHERE vulnerabilities.id IN (#{sub_batch.select(:id).to_sql})
+ AND vulnerabilities.id = vulnerability_occurrences.vulnerability_id
+ SQL
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/backfill_has_remediations_of_vulnerability_reads.rb b/lib/gitlab/background_migration/backfill_has_remediations_of_vulnerability_reads.rb
new file mode 100644
index 00000000000..83acd8a27f7
--- /dev/null
+++ b/lib/gitlab/background_migration/backfill_has_remediations_of_vulnerability_reads.rb
@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # Backfills has_remediations column for vulnerability_reads table.
+ class BackfillHasRemediationsOfVulnerabilityReads < BatchedMigrationJob
+ operation_name :set_has_remediations
+ feature_category :database
+
+ UPDATE_SQL = <<~SQL
+ UPDATE
+ vulnerability_reads
+ SET
+ has_remediations = true
+ FROM
+ (%<subquery>s) as sub_query
+ WHERE
+ vulnerability_reads.vulnerability_id = sub_query.vulnerability_id
+ SQL
+
+ def perform
+ each_sub_batch do |sub_batch|
+ update_query = update_query_for(sub_batch)
+
+ connection.execute(update_query)
+ end
+ end
+
+ private
+
+ def update_query_for(sub_batch)
+ subquery = sub_batch.joins("
+ INNER JOIN vulnerability_occurrences ON
+ vulnerability_reads.vulnerability_id = vulnerability_occurrences.vulnerability_id")
+ .select("vulnerability_reads.vulnerability_id, vulnerability_occurrences.id")
+ .joins("INNER JOIN vulnerability_findings_remediations ON
+ vulnerability_occurrences.id = vulnerability_findings_remediations.vulnerability_occurrence_id")
+
+ format(UPDATE_SQL, subquery: subquery.to_sql)
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/backfill_integrations_enable_ssl_verification.rb b/lib/gitlab/background_migration/backfill_integrations_enable_ssl_verification.rb
index 878f89a8b3d..c8e6841c2ae 100644
--- a/lib/gitlab/background_migration/backfill_integrations_enable_ssl_verification.rb
+++ b/lib/gitlab/background_migration/backfill_integrations_enable_ssl_verification.rb
@@ -10,14 +10,14 @@ module Gitlab
# - https://docs.drone.io/pipeline/environment/reference/drone-system-hostname/
'Integrations::DroneCi' => [
:drone_url,
- /\Acloud\.drone\.io\z/i.freeze
+ /\Acloud\.drone\.io\z/i
],
# This matches the logic in `Integrations::Teamcity#url_is_saas?`
# - https://gitlab.com/gitlab-org/gitlab/blob/65b7fc1ad1ad33247890324e9a3396993b7718a1/app/models/integrations/teamcity.rb#L117-122
# - https://www.jetbrains.com/help/teamcity/cloud/migrate-from-teamcity-on-premises-to-teamcity-cloud.html#Migration+Process
'Integrations::Teamcity' => [
:teamcity_url,
- /\A[^\.]+\.teamcity\.com\z/i.freeze
+ /\A[^\.]+\.teamcity\.com\z/i
]
# Other CI integrations which don't seem to have a SaaS offering:
diff --git a/lib/gitlab/background_migration/delete_orphans_approval_merge_request_rules2.rb b/lib/gitlab/background_migration/delete_orphans_approval_merge_request_rules2.rb
new file mode 100644
index 00000000000..a882b61c67d
--- /dev/null
+++ b/lib/gitlab/background_migration/delete_orphans_approval_merge_request_rules2.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # Deletes orphans records whenever report_type equals to scan_finding (4) or license_scanning (2)
+ class DeleteOrphansApprovalMergeRequestRules2 < BatchedMigrationJob
+ LICENSE_SCANNING_REPORT_TYPE = 2
+ SCAN_FINDING_REPORT_TYPE = 4
+
+ scope_to ->(relation) {
+ relation.where(report_type: [LICENSE_SCANNING_REPORT_TYPE, SCAN_FINDING_REPORT_TYPE],
+ security_orchestration_policy_configuration_id: nil)
+ }
+
+ operation_name :delete_all
+ feature_category :database
+
+ # rubocop: disable Style/Documentation
+ class ApprovalMergeRequestRuleSource < ::ApplicationRecord
+ # rubocop: enable Style/Documentation
+
+ self.table_name = 'approval_merge_request_rule_sources'
+ end
+
+ def perform
+ each_sub_batch do |sub_batch|
+ ApprovalMergeRequestRuleSource
+ .where(approval_merge_request_rule_id: sub_batch.distinct.select(:id))
+ .delete_all
+
+ sub_batch.delete_all
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/delete_orphans_approval_project_rules2.rb b/lib/gitlab/background_migration/delete_orphans_approval_project_rules2.rb
new file mode 100644
index 00000000000..f6b7e184811
--- /dev/null
+++ b/lib/gitlab/background_migration/delete_orphans_approval_project_rules2.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # Deletes orphans records whenever report_type equals to scan_finding (4) or license_scanning (2)
+ # rubocop: disable CodeReuse/ActiveRecord
+ class DeleteOrphansApprovalProjectRules2 < BatchedMigrationJob
+ LICENSE_SCANNING_REPORT_TYPE = 2
+ SCAN_FINDING_REPORT_TYPE = 4
+
+ scope_to ->(relation) {
+ relation.where(report_type: [LICENSE_SCANNING_REPORT_TYPE, SCAN_FINDING_REPORT_TYPE],
+ security_orchestration_policy_configuration_id: nil)
+ }
+
+ operation_name :delete_all
+ feature_category :database
+
+ # rubocop: disable Style/Documentation
+ class ApprovalMergeRequestRuleSource < ::ApplicationRecord
+ # rubocop: enable Style/Documentation
+
+ self.table_name = 'approval_merge_request_rule_sources'
+ end
+
+ def perform
+ each_sub_batch do |sub_batch|
+ ApprovalMergeRequestRuleSource
+ .where(approval_project_rule_id: sub_batch.distinct.select(:id))
+ .delete_all
+
+ sub_batch.delete_all
+ end
+ end
+ end
+ # rubocop: enable CodeReuse/ActiveRecord
+ end
+end
diff --git a/lib/gitlab/background_migration/migrate_pages_to_zip_storage.rb b/lib/gitlab/background_migration/migrate_pages_to_zip_storage.rb
deleted file mode 100644
index f53f2e8ee79..00000000000
--- a/lib/gitlab/background_migration/migrate_pages_to_zip_storage.rb
+++ /dev/null
@@ -1,16 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # migrates pages from legacy storage to zip format
- # we intentionally use application code here because
- # it has a lot of dependencies including models, carrierwave uploaders and service objects
- # and copying all or part of this code in the background migration doesn't add much value
- # see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/54578 for discussion
- class MigratePagesToZipStorage
- def perform(start_id, stop_id)
- # no-op
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb b/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb
index 6d59a5c8651..de3c52719c3 100644
--- a/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb
+++ b/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb
@@ -20,7 +20,7 @@ module Gitlab
# rubocop: enable Gitlab/NamespacedClass
# https://rubular.com/r/uwgK7k9KH23efa
- JIRA_CLOUD_REGEX = %r{^https?://[A-Za-z0-9](?:[A-Za-z0-9\-]{0,61}[A-Za-z0-9])?\.atlassian\.net$}ix.freeze
+ JIRA_CLOUD_REGEX = %r{^https?://[A-Za-z0-9](?:[A-Za-z0-9\-]{0,61}[A-Za-z0-9])?\.atlassian\.net$}ix
def perform
cloud = []
diff --git a/lib/gitlab/background_migration/update_workspaces_config_version.rb b/lib/gitlab/background_migration/update_workspaces_config_version.rb
new file mode 100644
index 00000000000..77a7fc1bcca
--- /dev/null
+++ b/lib/gitlab/background_migration/update_workspaces_config_version.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # No op on ce
+ class UpdateWorkspacesConfigVersion < BatchedMigrationJob
+ feature_category :remote_development
+ def perform; end
+ end
+ end
+end
+
+Gitlab::BackgroundMigration::UpdateWorkspacesConfigVersion.prepend_mod_with('Gitlab::BackgroundMigration::UpdateWorkspacesConfigVersion') # rubocop:disable Layout/LineLength
diff --git a/lib/gitlab/base_doorkeeper_controller.rb b/lib/gitlab/base_doorkeeper_controller.rb
index c8520993b8e..91994c2fa95 100644
--- a/lib/gitlab/base_doorkeeper_controller.rb
+++ b/lib/gitlab/base_doorkeeper_controller.rb
@@ -3,8 +3,7 @@
# This is a base controller for doorkeeper.
# It adds the `can?` helper used in the views.
module Gitlab
- # rubocop:disable Rails/ApplicationController
- class BaseDoorkeeperController < ActionController::Base
+ class BaseDoorkeeperController < BaseActionController
include Gitlab::Allowable
include EnforcesTwoFactorAuthentication
include SessionsHelper
@@ -13,5 +12,4 @@ module Gitlab
helper_method :can?
end
- # rubocop:enable Rails/ApplicationController
end
diff --git a/lib/gitlab/bitbucket_import/error_tracking.rb b/lib/gitlab/bitbucket_import/error_tracking.rb
new file mode 100644
index 00000000000..eaffe34daf8
--- /dev/null
+++ b/lib/gitlab/bitbucket_import/error_tracking.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BitbucketImport
+ module ErrorTracking
+ def track_import_failure!(project, exception:, **args)
+ Gitlab::Import::ImportFailureService.track(
+ project_id: project.id,
+ error_source: self.class.name,
+ exception: exception,
+ **args
+ )
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb
index 7f228c19b6b..9f87bb2347c 100644
--- a/lib/gitlab/bitbucket_import/importer.rb
+++ b/lib/gitlab/bitbucket_import/importer.rb
@@ -16,6 +16,7 @@ module Gitlab
@project = project
@client = Bitbucket::Client.new(project.import_data.credentials)
@formatter = Gitlab::ImportFormatter.new
+ @ref_converter = Gitlab::BitbucketImport::RefConverter.new(project)
@labels = {}
@errors = []
@users = {}
@@ -31,6 +32,26 @@ module Gitlab
true
end
+ def create_labels
+ LABELS.each do |label_params|
+ label = ::Labels::FindOrCreateService.new(nil, project, label_params).execute(skip_authorization: true)
+ if label.valid?
+ @labels[label_params[:title]] = label
+ else
+ raise "Failed to create label \"#{label_params[:title]}\" for project \"#{project.full_name}\""
+ end
+ end
+ end
+
+ def import_pull_request_comments(pull_request, merge_request)
+ comments = client.pull_request_comments(repo, pull_request.iid)
+
+ inline_comments, pr_comments = comments.partition(&:inline?)
+
+ import_inline_comments(inline_comments, pull_request, merge_request)
+ import_standalone_pr_comments(pr_comments, merge_request)
+ end
+
private
def already_imported?(collection, iid)
@@ -166,7 +187,7 @@ module Gitlab
note = ''
note += @formatter.author_line(comment.author) unless find_user_id(comment.author)
- note += comment.note
+ note += @ref_converter.convert_note(comment.note.to_s)
begin
gitlab_issue.notes.create!(
@@ -182,17 +203,6 @@ module Gitlab
end
end
- def create_labels
- LABELS.each do |label_params|
- label = ::Labels::FindOrCreateService.new(nil, project, label_params).execute(skip_authorization: true)
- if label.valid?
- @labels[label_params[:title]] = label
- else
- raise "Failed to create label \"#{label_params[:title]}\" for project \"#{project.full_name}\""
- end
- end
- end
-
def import_pull_requests
pull_requests = client.pull_requests(repo)
@@ -242,15 +252,6 @@ module Gitlab
store_pull_request_error(pull_request, e)
end
- def import_pull_request_comments(pull_request, merge_request)
- comments = client.pull_request_comments(repo, pull_request.iid)
-
- inline_comments, pr_comments = comments.partition(&:inline?)
-
- import_inline_comments(inline_comments, pull_request, merge_request)
- import_standalone_pr_comments(pr_comments, merge_request)
- end
-
def import_inline_comments(inline_comments, pull_request, merge_request)
position_map = {}
discussion_map = {}
@@ -319,8 +320,7 @@ module Gitlab
def comment_note(comment)
author = @formatter.author_line(comment.author) unless find_user_id(comment.author)
-
- author.to_s + comment.note.to_s
+ author.to_s + @ref_converter.convert_note(comment.note.to_s)
end
def log_base_data
diff --git a/lib/gitlab/bitbucket_import/importers/issue_importer.rb b/lib/gitlab/bitbucket_import/importers/issue_importer.rb
new file mode 100644
index 00000000000..2c3be67eabc
--- /dev/null
+++ b/lib/gitlab/bitbucket_import/importers/issue_importer.rb
@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BitbucketImport
+ module Importers
+ class IssueImporter
+ include Loggable
+ include ErrorTracking
+
+ def initialize(project, hash)
+ @project = project
+ @formatter = Gitlab::ImportFormatter.new
+ @user_finder = UserFinder.new(project)
+ @object = hash.with_indifferent_access
+ end
+
+ def execute
+ log_info(import_stage: 'import_issue', message: 'starting', iid: object[:iid])
+
+ description = ''
+ description += author_line
+ description += object[:description] if object[:description]
+
+ milestone = object[:milestone] ? project.milestones.find_or_create_by(title: object[:milestone]) : nil # rubocop: disable CodeReuse/ActiveRecord
+
+ attributes = {
+ iid: object[:iid],
+ title: object[:title],
+ description: description,
+ state_id: Issue.available_states[object[:state]],
+ author_id: author_id,
+ assignee_ids: [author_id],
+ namespace_id: project.project_namespace_id,
+ milestone: milestone,
+ work_item_type_id: object[:issue_type_id],
+ label_ids: [object[:label_id]].compact,
+ created_at: object[:created_at],
+ updated_at: object[:updated_at]
+ }
+
+ project.issues.create!(attributes)
+
+ log_info(import_stage: 'import_issue', message: 'finished', iid: object[:iid])
+ rescue StandardError => e
+ track_import_failure!(project, exception: e)
+ end
+
+ private
+
+ attr_reader :object, :project, :formatter, :user_finder
+
+ def author_line
+ return '' if find_user_id
+
+ formatter.author_line(object[:author])
+ end
+
+ def find_user_id
+ user_finder.find_user_id(object[:author])
+ end
+
+ def author_id
+ user_finder.gitlab_user_id(project, object[:author])
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/bitbucket_import/importers/issue_notes_importer.rb b/lib/gitlab/bitbucket_import/importers/issue_notes_importer.rb
new file mode 100644
index 00000000000..ac0e039939f
--- /dev/null
+++ b/lib/gitlab/bitbucket_import/importers/issue_notes_importer.rb
@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BitbucketImport
+ module Importers
+ class IssueNotesImporter
+ include ParallelScheduling
+
+ def initialize(project, hash)
+ @project = project
+ @formatter = Gitlab::ImportFormatter.new
+ @user_finder = UserFinder.new(project)
+ @ref_converter = Gitlab::BitbucketImport::RefConverter.new(project)
+ @object = hash.with_indifferent_access
+ end
+
+ def execute
+ log_info(import_stage: 'import_issue_notes', message: 'starting', iid: object[:iid])
+
+ issue = project.issues.find_by(iid: object[:iid]) # rubocop: disable CodeReuse/ActiveRecord
+
+ if issue
+ client.issue_comments(project.import_source, issue.iid).each do |comment|
+ next unless comment.note.present?
+
+ note = ''
+ note += formatter.author_line(comment.author) unless user_finder.find_user_id(comment.author)
+ note += ref_converter.convert_note(comment.note)
+
+ issue.notes.create!(
+ project: project,
+ note: note,
+ author_id: user_finder.gitlab_user_id(project, comment.author),
+ created_at: comment.created_at,
+ updated_at: comment.updated_at
+ )
+ end
+ end
+
+ log_info(import_stage: 'import_issue_notes', message: 'finished', iid: object[:iid])
+ rescue StandardError => e
+ track_import_failure!(project, exception: e)
+ end
+
+ private
+
+ attr_reader :object, :project, :formatter, :user_finder, :ref_converter
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/bitbucket_import/importers/issues_importer.rb b/lib/gitlab/bitbucket_import/importers/issues_importer.rb
new file mode 100644
index 00000000000..6162433e701
--- /dev/null
+++ b/lib/gitlab/bitbucket_import/importers/issues_importer.rb
@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BitbucketImport
+ module Importers
+ class IssuesImporter
+ include ParallelScheduling
+
+ def execute
+ log_info(import_stage: 'import_issues', message: 'importing issues')
+
+ issues = client.issues(project.import_source)
+
+ labels = build_labels_hash
+
+ issues.each do |issue|
+ job_waiter.jobs_remaining += 1
+
+ next if already_enqueued?(issue)
+
+ job_delay = calculate_job_delay(job_waiter.jobs_remaining)
+
+ issue_hash = issue.to_hash.merge({ issue_type_id: default_issue_type_id, label_id: labels[issue.kind] })
+ sidekiq_worker_class.perform_in(job_delay, project.id, issue_hash, job_waiter.key)
+
+ mark_as_enqueued(issue)
+ end
+
+ job_waiter
+ rescue StandardError => e
+ track_import_failure!(project, exception: e)
+ end
+
+ private
+
+ def sidekiq_worker_class
+ ImportIssueWorker
+ end
+
+ def collection_method
+ :issues
+ end
+
+ def id_for_already_enqueued_cache(object)
+ object.iid
+ end
+
+ def default_issue_type_id
+ ::WorkItems::Type.default_issue_type.id
+ end
+
+ def build_labels_hash
+ labels = {}
+ project.labels.each { |l| labels[l.title.to_s] = l.id }
+ labels
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/bitbucket_import/importers/issues_notes_importer.rb b/lib/gitlab/bitbucket_import/importers/issues_notes_importer.rb
new file mode 100644
index 00000000000..03dcc645f07
--- /dev/null
+++ b/lib/gitlab/bitbucket_import/importers/issues_notes_importer.rb
@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BitbucketImport
+ module Importers
+ class IssuesNotesImporter
+ include ParallelScheduling
+
+ def execute
+ project.issues.find_each do |issue|
+ job_waiter.jobs_remaining += 1
+
+ next if already_enqueued?(issue)
+
+ job_delay = calculate_job_delay(job_waiter.jobs_remaining)
+
+ sidekiq_class.perform_in(job_delay, project.id, { iid: issue.iid }, job_waiter.key)
+
+ mark_as_enqueued(issue)
+ end
+
+ job_waiter
+ rescue StandardError => e
+ track_import_failure!(project, exception: e)
+ end
+
+ private
+
+ attr_reader :project
+
+ def sidekiq_class
+ ImportIssueNotesWorker
+ end
+
+ def id_for_already_enqueued_cache(object)
+ object.iid
+ end
+
+ def collection_method
+ :issues_notes
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/bitbucket_import/importers/lfs_object_importer.rb b/lib/gitlab/bitbucket_import/importers/lfs_object_importer.rb
new file mode 100644
index 00000000000..06b30c7b496
--- /dev/null
+++ b/lib/gitlab/bitbucket_import/importers/lfs_object_importer.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BitbucketImport
+ module Importers
+ class LfsObjectImporter
+ include Loggable
+ include ErrorTracking
+
+ def initialize(project, lfs_attributes)
+ @project = project
+ @lfs_download_object = LfsDownloadObject.new(**lfs_attributes.symbolize_keys)
+ end
+
+ def execute
+ log_info(import_stage: 'import_lfs_object', message: 'starting', oid: lfs_download_object.oid)
+
+ lfs_download_object.validate!
+ Projects::LfsPointers::LfsDownloadService.new(project, lfs_download_object).execute
+
+ log_info(import_stage: 'import_lfs_object', message: 'finished', oid: lfs_download_object.oid)
+ rescue StandardError => e
+ track_import_failure!(project, exception: e)
+ end
+
+ private
+
+ attr_reader :lfs_download_object, :project
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/bitbucket_import/importers/lfs_objects_importer.rb b/lib/gitlab/bitbucket_import/importers/lfs_objects_importer.rb
new file mode 100644
index 00000000000..aa9ff7000f1
--- /dev/null
+++ b/lib/gitlab/bitbucket_import/importers/lfs_objects_importer.rb
@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BitbucketImport
+ module Importers
+ class LfsObjectsImporter
+ include ParallelScheduling
+
+ def execute
+ log_info(import_stage: 'import_lfs_objects', message: 'starting')
+
+ download_service = Projects::LfsPointers::LfsObjectDownloadListService.new(project)
+
+ begin
+ queue_workers(download_service) if project.lfs_enabled?
+ rescue StandardError => e
+ track_import_failure!(project, exception: e)
+ end
+
+ log_info(import_stage: 'import_lfs_objects', message: 'finished')
+
+ job_waiter
+ end
+
+ private
+
+ def sidekiq_worker_class
+ ImportLfsObjectWorker
+ end
+
+ def collection_method
+ :lfs_objects
+ end
+
+ def id_for_already_enqueued_cache(object)
+ object.oid
+ end
+
+ def queue_workers(download_service)
+ download_service.each_list_item do |lfs_download_object|
+ # Needs to come before `already_enqueued?` as `jobs_remaining` resets to zero when the job restarts and
+ # jobs_remaining needs to be the total amount of enqueued jobs
+ job_waiter.jobs_remaining += 1
+
+ next if already_enqueued?(lfs_download_object)
+
+ job_delay = calculate_job_delay(job_waiter.jobs_remaining)
+
+ sidekiq_worker_class.perform_in(job_delay, project.id, lfs_download_object.to_hash, job_waiter.key)
+
+ mark_as_enqueued(lfs_download_object)
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/bitbucket_import/importers/pull_request_importer.rb b/lib/gitlab/bitbucket_import/importers/pull_request_importer.rb
index d76e08e1039..a18d50e8fce 100644
--- a/lib/gitlab/bitbucket_import/importers/pull_request_importer.rb
+++ b/lib/gitlab/bitbucket_import/importers/pull_request_importer.rb
@@ -5,6 +5,7 @@ module Gitlab
module Importers
class PullRequestImporter
include Loggable
+ include ErrorTracking
def initialize(project, hash)
@project = project
@@ -48,7 +49,7 @@ module Gitlab
log_info(import_stage: 'import_pull_request', message: 'finished', iid: object[:iid])
rescue StandardError => e
- Gitlab::Import::ImportFailureService.track(project_id: project.id, exception: e)
+ track_import_failure!(project, exception: e)
end
private
diff --git a/lib/gitlab/bitbucket_import/importers/pull_request_notes_importer.rb b/lib/gitlab/bitbucket_import/importers/pull_request_notes_importer.rb
new file mode 100644
index 00000000000..8ea8b1562f2
--- /dev/null
+++ b/lib/gitlab/bitbucket_import/importers/pull_request_notes_importer.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BitbucketImport
+ module Importers
+ class PullRequestNotesImporter
+ include Loggable
+ include ErrorTracking
+
+ def initialize(project, hash)
+ @project = project
+ @importer = Gitlab::BitbucketImport::Importer.new(project)
+ @object = hash.with_indifferent_access
+ end
+
+ def execute
+ log_info(import_stage: 'import_pull_request_notes', message: 'starting', iid: object[:iid])
+
+ merge_request = project.merge_requests.find_by(iid: object[:iid]) # rubocop: disable CodeReuse/ActiveRecord
+
+ importer.import_pull_request_comments(merge_request, merge_request) if merge_request
+
+ log_info(import_stage: 'import_pull_request_notes', message: 'finished', iid: object[:iid])
+ rescue StandardError => e
+ track_import_failure!(project, exception: e)
+ end
+
+ private
+
+ attr_reader :object, :project, :importer
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/bitbucket_import/importers/pull_requests_notes_importer.rb b/lib/gitlab/bitbucket_import/importers/pull_requests_notes_importer.rb
new file mode 100644
index 00000000000..a1b0c2a5afe
--- /dev/null
+++ b/lib/gitlab/bitbucket_import/importers/pull_requests_notes_importer.rb
@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BitbucketImport
+ module Importers
+ class PullRequestsNotesImporter
+ include ParallelScheduling
+
+ def execute
+ project.merge_requests.find_each do |merge_request|
+ job_waiter.jobs_remaining += 1
+
+ next if already_enqueued?(merge_request)
+
+ job_delay = calculate_job_delay(job_waiter.jobs_remaining)
+
+ sidekiq_worker_class.perform_in(job_delay, project.id, { iid: merge_request.iid }, job_waiter.key)
+
+ mark_as_enqueued(merge_request)
+ end
+
+ job_waiter
+ rescue StandardError => e
+ track_import_failure!(project, exception: e)
+ end
+
+ private
+
+ attr_reader :project
+
+ def sidekiq_worker_class
+ ImportPullRequestNotesWorker
+ end
+
+ def id_for_already_enqueued_cache(object)
+ object.iid
+ end
+
+ def collection_method
+ :merge_requests_notes
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/bitbucket_import/importers/repository_importer.rb b/lib/gitlab/bitbucket_import/importers/repository_importer.rb
index 7b0362b6ec6..b8c0ba69d37 100644
--- a/lib/gitlab/bitbucket_import/importers/repository_importer.rb
+++ b/lib/gitlab/bitbucket_import/importers/repository_importer.rb
@@ -23,6 +23,7 @@ module Gitlab
end
import_wiki
+ create_labels
log_info(import_stage: 'import_repository', message: 'finished import')
@@ -59,6 +60,11 @@ module Gitlab
)
end
+ def create_labels
+ importer = Gitlab::BitbucketImport::Importer.new(project)
+ importer.create_labels
+ end
+
def wiki
WikiFormatter.new(project)
end
diff --git a/lib/gitlab/bitbucket_import/parallel_scheduling.rb b/lib/gitlab/bitbucket_import/parallel_scheduling.rb
index f4df9a35526..ca2597ea5cf 100644
--- a/lib/gitlab/bitbucket_import/parallel_scheduling.rb
+++ b/lib/gitlab/bitbucket_import/parallel_scheduling.rb
@@ -4,6 +4,7 @@ module Gitlab
module BitbucketImport
module ParallelScheduling
include Loggable
+ include ErrorTracking
attr_reader :project, :already_enqueued_cache_key, :job_waiter_cache_key
@@ -79,15 +80,6 @@ module Gitlab
(multiplier * 1.minute) + 1.second
end
-
- def track_import_failure!(project, exception:, **args)
- Gitlab::Import::ImportFailureService.track(
- project_id: project.id,
- error_source: self.class.name,
- exception: exception,
- **args
- )
- end
end
end
end
diff --git a/lib/gitlab/bitbucket_import/ref_converter.rb b/lib/gitlab/bitbucket_import/ref_converter.rb
new file mode 100644
index 00000000000..1159159a76d
--- /dev/null
+++ b/lib/gitlab/bitbucket_import/ref_converter.rb
@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BitbucketImport
+ class RefConverter
+ REPO_MATCHER = 'https://bitbucket.org/%s'
+ PR_NOTE_ISSUE_NAME_REGEX = '(?<=/)[^/\)]+(?=\)[^/]*$)'
+ UNWANTED_NOTE_REF_HTML = "{: data-inline-card='' }"
+
+ attr_reader :project
+
+ def initialize(project)
+ @project = project
+ end
+
+ def convert_note(note)
+ repo_matcher = REPO_MATCHER % project.import_source
+
+ return note unless note.match?(repo_matcher)
+
+ note = note.gsub(repo_matcher, url_helpers.project_url(project))
+ .gsub(UNWANTED_NOTE_REF_HTML, '')
+ .strip
+
+ if note.match?('issues')
+ note.gsub!('issues', '-/issues')
+ note.gsub!(issue_name(note), '')
+ else
+ note.gsub!('pull-requests', '-/merge_requests')
+ note.gsub!('src', '-/blob')
+ note.gsub!('lines-', 'L')
+ end
+
+ note
+ end
+
+ private
+
+ def url_helpers
+ Rails.application.routes.url_helpers
+ end
+
+ def issue_name(note)
+ note.match(PR_NOTE_ISSUE_NAME_REGEX)[0]
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/bitbucket_server_import/importers/pull_request_importer.rb b/lib/gitlab/bitbucket_server_import/importers/pull_request_importer.rb
index 34963452192..0d4de385f5e 100644
--- a/lib/gitlab/bitbucket_server_import/importers/pull_request_importer.rb
+++ b/lib/gitlab/bitbucket_server_import/importers/pull_request_importer.rb
@@ -30,7 +30,7 @@ module Gitlab
reviewer_ids: reviewers,
source_project_id: project.id,
source_branch: Gitlab::Git.ref_name(object[:source_branch_name]),
- source_branch_sha: object[:source_branch_sha],
+ source_branch_sha: source_branch_sha,
target_project_id: project.id,
target_branch: Gitlab::Git.ref_name(object[:target_branch_name]),
target_branch_sha: object[:target_branch_sha],
@@ -68,6 +68,14 @@ module Gitlab
end
end
end
+
+ def source_branch_sha
+ source_branch_sha = project.repository.commit(object[:source_branch_sha])&.sha
+
+ return source_branch_sha if source_branch_sha
+
+ project.repository.find_commits_by_message(object[:source_branch_sha])&.first&.sha
+ end
end
end
end
diff --git a/lib/gitlab/bitbucket_server_import/importers/pull_requests_importer.rb b/lib/gitlab/bitbucket_server_import/importers/pull_requests_importer.rb
index 92ec10bf037..ae73681f7f8 100644
--- a/lib/gitlab/bitbucket_server_import/importers/pull_requests_importer.rb
+++ b/lib/gitlab/bitbucket_server_import/importers/pull_requests_importer.rb
@@ -20,6 +20,22 @@ module Gitlab
break if pull_requests.empty?
+ commits_to_fetch = pull_requests.filter_map do |pull_request|
+ next if already_processed?(pull_request)
+ next unless pull_request.merged? || pull_request.closed?
+
+ [pull_request.source_branch_sha, pull_request.target_branch_sha]
+ end.flatten
+
+ # Bitbucket Server keeps tracks of references for open pull requests in
+ # refs/heads/pull-requests, but closed and merged requests get moved
+ # into hidden internal refs under stash-refs/pull-requests. As a result,
+ # they are not fetched by default.
+ #
+ # This method call explicitly fetches head and start commits for affected pull requests.
+ # That allows us to correctly assign diffs and commits to merge requests.
+ fetch_missing_commits(commits_to_fetch)
+
pull_requests.each do |pull_request|
# Needs to come before `already_processed?` as `jobs_remaining` resets to zero when the job restarts and
# jobs_remaining needs to be the total amount of enqueued jobs
@@ -42,6 +58,15 @@ module Gitlab
private
+ def fetch_missing_commits(commits_to_fetch)
+ return if commits_to_fetch.blank?
+ return unless Feature.enabled?(:fetch_commits_for_bitbucket_server, project.group)
+
+ project.repository.fetch_remote(project.import_url, refmap: commits_to_fetch, prune: false)
+ rescue StandardError => e
+ track_import_failure!(project, exception: e)
+ end
+
def sidekiq_worker_class
ImportPullRequestWorker
end
diff --git a/lib/gitlab/bitbucket_server_import/project_creator.rb b/lib/gitlab/bitbucket_server_import/project_creator.rb
index ddc678abdd8..be60e431b80 100644
--- a/lib/gitlab/bitbucket_server_import/project_creator.rb
+++ b/lib/gitlab/bitbucket_server_import/project_creator.rb
@@ -3,9 +3,9 @@
module Gitlab
module BitbucketServerImport
class ProjectCreator
- attr_reader :project_key, :repo_slug, :repo, :name, :namespace, :current_user, :session_data
+ attr_reader :project_key, :repo_slug, :repo, :name, :namespace, :current_user, :session_data, :timeout_strategy
- def initialize(project_key, repo_slug, repo, name, namespace, current_user, session_data)
+ def initialize(project_key, repo_slug, repo, name, namespace, current_user, session_data, timeout_strategy)
@project_key = project_key
@repo_slug = repo_slug
@repo = repo
@@ -13,6 +13,7 @@ module Gitlab
@namespace = namespace
@current_user = current_user
@session_data = session_data
+ @timeout_strategy = timeout_strategy
end
def execute
@@ -28,7 +29,7 @@ module Gitlab
import_url: repo.clone_url,
import_data: {
credentials: session_data,
- data: { project_key: project_key, repo_slug: repo_slug }
+ data: { project_key: project_key, repo_slug: repo_slug, timeout_strategy: timeout_strategy }
},
skip_wiki: true
).execute
diff --git a/lib/gitlab/changelog/generator.rb b/lib/gitlab/changelog/generator.rb
index a80ca0728f9..0e546c5eb60 100644
--- a/lib/gitlab/changelog/generator.rb
+++ b/lib/gitlab/changelog/generator.rb
@@ -6,7 +6,7 @@ module Gitlab
class Generator
# The regex used to parse a release header.
RELEASE_REGEX =
- /^##\s+(?<version>#{Gitlab::Regex.unbounded_semver_regex})/.freeze
+ /^##\s+(?<version>#{Gitlab::Regex.unbounded_semver_regex})/
# The `input` argument must be a `String` containing the existing
# changelog Markdown. If no changelog exists, this should be an empty
diff --git a/lib/gitlab/chat.rb b/lib/gitlab/chat.rb
deleted file mode 100644
index 30e9989d270..00000000000
--- a/lib/gitlab/chat.rb
+++ /dev/null
@@ -1,10 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Chat
- # Returns `true` if Chatops is available for the current instance.
- def self.available?
- ::Feature.enabled?(:chatops)
- end
- end
-end
diff --git a/lib/gitlab/checks/global_file_size_check.rb b/lib/gitlab/checks/global_file_size_check.rb
index 62facf52239..ff24467e9cc 100644
--- a/lib/gitlab/checks/global_file_size_check.rb
+++ b/lib/gitlab/checks/global_file_size_check.rb
@@ -17,16 +17,34 @@ module Gitlab
).find
if oversized_blobs.present?
+
+ blob_details = {}
+ blob_id_size_msg = ""
+ oversized_blobs.each do |blob|
+ blob_details[blob.id] = { "size" => blob.size }
+
+ # blob size is in byte, divide it by "/ 1024.0 / 1024.0" to get MiB
+ blob_id_size_msg += "- #{blob.id} (#{(blob.size / 1024.0 / 1024.0).round(2)} MiB) \n"
+ end
+
+ oversize_err_msg = <<~OVERSIZE_ERR_MSG
+ You are attempting to check in one or more blobs which exceed the #{file_size_limit}MiB limit:
+
+ #{blob_id_size_msg}
+ To resolve this error, you must either reduce the size of the above blobs, or utilize LFS.
+ You may use "git ls-tree -r HEAD | grep $BLOB_ID" to see the file path.
+ Please refer to #{Rails.application.routes.url_helpers.help_page_url('user/free_push_limit')} and
+ #{Rails.application.routes.url_helpers.help_page_url('administration/settings/account_and_limit_settings')}
+ for further information.
+ OVERSIZE_ERR_MSG
+
Gitlab::AppJsonLogger.info(
message: 'Found blob over global limit',
- blob_sizes: oversized_blobs.map(&:size)
+ blob_sizes: oversized_blobs.map(&:size),
+ blob_details: blob_details
)
- if enforce_global_file_size_limit?
- raise ::Gitlab::GitAccess::ForbiddenError,
- "Changes include a file that is larger than the allowed size of #{file_size_limit} MiB. " \
- "Use Git LFS to manage this file.)"
- end
+ raise ::Gitlab::GitAccess::ForbiddenError, oversize_err_msg if enforce_global_file_size_limit?
end
end
diff --git a/lib/gitlab/checks/security/policy_check.rb b/lib/gitlab/checks/security/policy_check.rb
new file mode 100644
index 00000000000..b2be393351a
--- /dev/null
+++ b/lib/gitlab/checks/security/policy_check.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Checks
+ module Security
+ class PolicyCheck < BaseSingleChecker
+ def validate!; end
+ end
+ end
+ end
+end
+
+Gitlab::Checks::Security::PolicyCheck.prepend_mod
diff --git a/lib/gitlab/checks/single_change_access.rb b/lib/gitlab/checks/single_change_access.rb
index 9f427e98e55..625524cf2bc 100644
--- a/lib/gitlab/checks/single_change_access.rb
+++ b/lib/gitlab/checks/single_change_access.rb
@@ -54,6 +54,7 @@ module Gitlab
Gitlab::Checks::PushCheck.new(self).validate!
Gitlab::Checks::BranchCheck.new(self).validate!
Gitlab::Checks::TagCheck.new(self).validate!
+ Gitlab::Checks::Security::PolicyCheck.new(self).validate!
end
def commits_check
diff --git a/lib/gitlab/checks/tag_check.rb b/lib/gitlab/checks/tag_check.rb
index 4505bcb5411..d5addab74b8 100644
--- a/lib/gitlab/checks/tag_check.rb
+++ b/lib/gitlab/checks/tag_check.rb
@@ -11,7 +11,8 @@ module Gitlab
delete_protected_tag_non_web: 'You can only delete protected tags using the web interface.',
create_protected_tag: 'You are not allowed to create this tag as it is protected.',
default_branch_collision: 'You cannot use default branch name to create a tag',
- prohibited_tag_name: 'You cannot create a tag with a prohibited pattern.'
+ prohibited_tag_name: 'You cannot create a tag with a prohibited pattern.',
+ prohibited_tag_name_encoding: 'Tag names must be valid when converted to UTF-8 encoding'
}.freeze
LOG_MESSAGES = {
@@ -46,6 +47,16 @@ module Gitlab
if tag_name.start_with?("refs/tags/") # rubocop: disable Style/GuardClause
raise GitAccess::ForbiddenError, ERROR_MESSAGES[:prohibited_tag_name]
end
+
+ # rubocop: disable Style/GuardClause
+ # rubocop: disable Style/SoleNestedConditional
+ if Feature.enabled?(:prohibited_tag_name_encoding_check, project)
+ unless Gitlab::EncodingHelper.force_encode_utf8(tag_name).valid_encoding?
+ raise GitAccess::ForbiddenError, ERROR_MESSAGES[:prohibited_tag_name_encoding]
+ end
+ end
+ # rubocop: enable Style/SoleNestedConditional
+ # rubocop: enable Style/GuardClause
end
def protected_tag_checks
diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb
index 5748b8e34cf..7d9235ac460 100644
--- a/lib/gitlab/ci/build/artifacts/metadata.rb
+++ b/lib/gitlab/ci/build/artifacts/metadata.rb
@@ -11,8 +11,8 @@ module Gitlab
ParserError = Class.new(StandardError)
InvalidStreamError = Class.new(StandardError)
- VERSION_PATTERN = /^[\w\s]+(\d+\.\d+\.\d+)/.freeze
- INVALID_PATH_PATTERN = %r{(^\.?\.?/)|(/\.?\.?/)}.freeze
+ VERSION_PATTERN = /^[\w\s]+(\d+\.\d+\.\d+)/
+ INVALID_PATH_PATTERN = %r{(^\.?\.?/)|(/\.?\.?/)}
attr_reader :stream, :path, :full_version
diff --git a/lib/gitlab/ci/build/context/build.rb b/lib/gitlab/ci/build/context/build.rb
index 81efbdb297b..48b138b0258 100644
--- a/lib/gitlab/ci/build/context/build.rb
+++ b/lib/gitlab/ci/build/context/build.rb
@@ -30,8 +30,16 @@ module Gitlab
::Ci::Build.new(build_attributes)
end
+ # Assigning tags and needs is slow and they are not needed for rules
+ # evaluation since we don't use them to compute the variables at this point.
def build_attributes
- attributes.merge(pipeline_attributes, ci_stage_attributes)
+ if pipeline.reduced_build_attributes_list_for_rules?
+ attributes
+ .except(:tag_list, :needs_attributes)
+ .merge!(pipeline_attributes, ci_stage_attributes)
+ else
+ attributes.merge(pipeline_attributes, ci_stage_attributes)
+ end
end
def ci_stage_attributes
diff --git a/lib/gitlab/ci/build/duration_parser.rb b/lib/gitlab/ci/build/duration_parser.rb
index 97049a4f876..9385dccd5f3 100644
--- a/lib/gitlab/ci/build/duration_parser.rb
+++ b/lib/gitlab/ci/build/duration_parser.rb
@@ -41,7 +41,7 @@ module Gitlab
def parse
return if never?
- ChronicDuration.parse(value, use_complete_matcher: true)
+ ChronicDuration.parse(value)
end
def validation_cache
diff --git a/lib/gitlab/ci/components/instance_path.rb b/lib/gitlab/ci/components/instance_path.rb
index 17c784c4d54..551284d9099 100644
--- a/lib/gitlab/ci/components/instance_path.rb
+++ b/lib/gitlab/ci/components/instance_path.rb
@@ -7,19 +7,17 @@ module Gitlab
include Gitlab::Utils::StrongMemoize
LATEST_VERSION_KEYWORD = '~latest'
- TEMPLATES_DIR = 'templates'
def self.match?(address)
address.include?('@') && address.start_with?(Settings.gitlab_ci['component_fqdn'])
end
- attr_reader :host, :project_file_path
+ attr_reader :host
- def initialize(address:, content_filename:)
+ def initialize(address:)
@full_path, @version = address.to_s.split('@', 2)
- @content_filename = content_filename
@host = Settings.gitlab_ci['component_fqdn']
- @project_file_path = nil
+ @component_project = ::Ci::Catalog::ComponentsProject.new(project, sha)
end
def fetch_content!(current_user:)
@@ -28,7 +26,7 @@ module Gitlab
raise Gitlab::Access::AccessDeniedError unless Ability.allowed?(current_user, :download_code, project)
- content(simple_template_path) || content(complex_template_path) || content(legacy_template_path)
+ @component_project.fetch_component(component_name)
end
def project
@@ -46,16 +44,7 @@ module Gitlab
private
- attr_reader :version, :path
-
- def instance_path
- @full_path.delete_prefix(host)
- end
-
- def component_path
- instance_path.delete_prefix(project.full_path).delete_prefix('/')
- end
- strong_memoize_attr :component_path
+ attr_reader :version
# Given a path like "my-org/sub-group/the-project/path/to/component"
# find the project "my-org/sub-group/the-project" by looking at all possible paths.
@@ -65,45 +54,23 @@ module Gitlab
while index = path.rindex('/') # find index of last `/` in a path
possible_paths << (path = path[0..index - 1])
end
-
# remove shortest path as it is group
possible_paths.pop
::Project.where_full_path_in(possible_paths).take # rubocop: disable CodeReuse/ActiveRecord
end
- def latest_version_sha
- project.releases.latest&.sha
- end
-
- # A simple template consists of a single file
- def simple_template_path
- # Extract this line and move to fetch_content once we remove legacy fetching
- return unless templates_dir_exists? && component_path.index('/').nil?
-
- @project_file_path = File.join(TEMPLATES_DIR, "#{component_path}.yml")
- end
-
- # A complex template is directory-based and may consist of multiple files.
- # Given a path like "my-org/sub-group/the-project/templates/component"
- # returns the entry point path: "templates/component/template.yml".
- def complex_template_path
- # Extract this line and move to fetch_content once we remove legacy fetching
- return unless templates_dir_exists? && component_path.index('/').nil?
-
- @project_file_path = File.join(TEMPLATES_DIR, component_path, @content_filename)
- end
-
- def legacy_template_path
- @project_file_path = File.join(component_path, @content_filename).delete_prefix('/')
+ def instance_path
+ @full_path.delete_prefix(host)
end
- def templates_dir_exists?
- project.repository.tree.trees.map(&:name).include?(TEMPLATES_DIR)
+ def component_name
+ instance_path.delete_prefix(project.full_path).delete_prefix('/')
end
+ strong_memoize_attr :component_name
- def content(path)
- project.repository.blob_data_at(sha, path)
+ def latest_version_sha
+ project.releases.latest&.sha
end
end
end
diff --git a/lib/gitlab/ci/config/entry/artifacts.rb b/lib/gitlab/ci/config/entry/artifacts.rb
index 27206d7e3a8..3fd07811daf 100644
--- a/lib/gitlab/ci/config/entry/artifacts.rb
+++ b/lib/gitlab/ci/config/entry/artifacts.rb
@@ -14,7 +14,7 @@ module Gitlab
ALLOWED_WHEN = %w[on_success on_failure always].freeze
ALLOWED_KEYS = %i[name untracked paths reports when expire_in expose_as exclude public].freeze
- EXPOSE_AS_REGEX = /\A\w[-\w ]*\z/.freeze
+ EXPOSE_AS_REGEX = /\A\w[-\w ]*\z/
EXPOSE_AS_ERROR_MESSAGE = "can contain only letters, digits, '-', '_' and spaces"
attributes ALLOWED_KEYS
diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb
index c40d665f320..bf8a99ef45e 100644
--- a/lib/gitlab/ci/config/entry/job.rb
+++ b/lib/gitlab/ci/config/entry/job.rb
@@ -177,7 +177,7 @@ module Gitlab
def parsed_timeout
return unless has_timeout?
- ChronicDuration.parse(timeout.to_s, use_complete_matcher: true)
+ ChronicDuration.parse(timeout.to_s)
end
def ignored?
diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb
index efba81c7420..b3c802e5657 100644
--- a/lib/gitlab/ci/config/external/file/base.rb
+++ b/lib/gitlab/ci/config/external/file/base.rb
@@ -10,7 +10,7 @@ module Gitlab
attr_reader :location, :params, :context, :errors
- YAML_WHITELIST_EXTENSION = /.+\.(yml|yaml)$/i.freeze
+ YAML_WHITELIST_EXTENSION = /.+\.(yml|yaml)$/i
def initialize(params, context)
@params = params
@@ -114,7 +114,9 @@ module Gitlab
def content_result
context.logger.instrument(:config_file_fetch_content_hash) do
- ::Gitlab::Ci::Config::Yaml::Loader.new(content, inputs: content_inputs).load
+ ::Gitlab::Ci::Config::Yaml::Loader.new(
+ content, inputs: content_inputs, variables: context.variables
+ ).load
end
end
strong_memoize_attr :content_result
diff --git a/lib/gitlab/ci/config/external/file/component.rb b/lib/gitlab/ci/config/external/file/component.rb
index de6de1bb7a8..03063e76dde 100644
--- a/lib/gitlab/ci/config/external/file/component.rb
+++ b/lib/gitlab/ci/config/external/file/component.rb
@@ -20,7 +20,7 @@ module Gitlab
::Gitlab::UsageDataCounters::HLLRedisCounter.track_event('cicd_component_usage', values: context.user.id)
- component_result.payload.fetch(:content)
+ component_payload.fetch(:content)
end
strong_memoize_attr :content
@@ -65,30 +65,30 @@ module Gitlab
override :expand_context_attrs
def expand_context_attrs
{
- project: component_path.project,
- sha: component_path.sha,
+ project: component_payload.fetch(:project),
+ sha: component_payload.fetch(:sha),
user: context.user,
variables: context.variables
}
end
def masked_blob
- return unless component_path
+ return unless component_payload
context.mask_variables_from(
Gitlab::Routing.url_helpers.project_blob_url(
- component_path.project,
- ::File.join(component_path.sha, component_path.project_file_path))
+ component_payload.fetch(:project),
+ ::File.join(component_payload.fetch(:sha), component_payload.fetch(:path)))
)
end
strong_memoize_attr :masked_blob
- def component_path
+ def component_payload
return unless component_result.success?
- component_result.payload.fetch(:path)
+ component_result.payload
end
- strong_memoize_attr :component_path
+ strong_memoize_attr :component_payload
end
end
end
diff --git a/lib/gitlab/ci/config/header/input.rb b/lib/gitlab/ci/config/header/input.rb
index 76a89a3080e..dcb96006459 100644
--- a/lib/gitlab/ci/config/header/input.rb
+++ b/lib/gitlab/ci/config/header/input.rb
@@ -11,12 +11,16 @@ module Gitlab
include ::Gitlab::Config::Entry::Validatable
include ::Gitlab::Config::Entry::Attributable
- attributes :default, :type, prefix: :input
+ ALLOWED_KEYS = %i[default description regex type].freeze
+
+ attributes ALLOWED_KEYS, prefix: :input
validations do
- validates :config, type: Hash, allowed_keys: [:default, :type]
+ validates :config, type: Hash, allowed_keys: ALLOWED_KEYS
validates :key, alphanumeric: true
validates :input_default, alphanumeric: true, allow_nil: true
+ validates :input_description, alphanumeric: true, allow_nil: true
+ validates :input_regex, type: String, allow_nil: true
validates :input_type, allow_nil: true, allowed_values: Interpolation::Inputs.input_types
end
end
diff --git a/lib/gitlab/ci/config/interpolation/block.rb b/lib/gitlab/ci/config/interpolation/block.rb
index cf8420f924e..aec19299e86 100644
--- a/lib/gitlab/ci/config/interpolation/block.rb
+++ b/lib/gitlab/ci/config/interpolation/block.rb
@@ -62,7 +62,7 @@ module Gitlab
return @errors.concat(access.errors) unless access.valid?
return @errors.push('too many functions in interpolation block') if functions.count > MAX_FUNCTIONS
- result = Interpolation::FunctionsStack.new(functions).evaluate(access.value)
+ result = Interpolation::FunctionsStack.new(functions, ctx).evaluate(access.value)
if result.success?
@value = result.value
diff --git a/lib/gitlab/ci/config/interpolation/context.rb b/lib/gitlab/ci/config/interpolation/context.rb
index f5e7db03291..19ea619f7da 100644
--- a/lib/gitlab/ci/config/interpolation/context.rb
+++ b/lib/gitlab/ci/config/interpolation/context.rb
@@ -14,8 +14,11 @@ module Gitlab
MAX_DEPTH = 3
- def initialize(hash)
- @context = hash
+ attr_reader :variables
+
+ def initialize(data, variables: [])
+ @data = data
+ @variables = Ci::Variables::Collection.fabricate(variables)
raise ContextTooComplexError if depth > MAX_DEPTH
end
@@ -32,25 +35,25 @@ module Gitlab
end
def depth
- deep_depth(@context)
+ deep_depth(@data)
end
def fetch(field)
- @context.fetch(field)
+ @data.fetch(field)
end
def key?(name)
- @context.key?(name)
+ @data.key?(name)
end
def to_h
- @context.to_h
+ @data.to_h
end
private
- def deep_depth(context, depth = 0)
- values = context.values.map do |value|
+ def deep_depth(data, depth = 0)
+ values = data.values.map do |value|
if value.is_a?(Hash)
deep_depth(value, depth + 1)
else
@@ -61,10 +64,10 @@ module Gitlab
values.max.to_i
end
- def self.fabricate(context)
+ def self.fabricate(context, variables: [])
case context
when Hash
- new(context)
+ new(context, variables: variables)
when Interpolation::Context
context
else
diff --git a/lib/gitlab/ci/config/interpolation/functions/base.rb b/lib/gitlab/ci/config/interpolation/functions/base.rb
index b9ce8cdc5bc..b04152a1558 100644
--- a/lib/gitlab/ci/config/interpolation/functions/base.rb
+++ b/lib/gitlab/ci/config/interpolation/functions/base.rb
@@ -20,9 +20,10 @@ module Gitlab
function_expression_pattern.match?(function_expression)
end
- def initialize(function_expression)
+ def initialize(function_expression, ctx)
@errors = []
@function_args = parse_args(function_expression)
+ @ctx = ctx
end
def valid?
@@ -35,10 +36,11 @@ module Gitlab
private
- attr_reader :function_args
+ attr_reader :function_args, :ctx
def error(message)
errors << "error in `#{self.class.name}` function: #{message}"
+ nil
end
def parse_args(function_expression)
diff --git a/lib/gitlab/ci/config/interpolation/functions/expand_vars.rb b/lib/gitlab/ci/config/interpolation/functions/expand_vars.rb
new file mode 100644
index 00000000000..658964018b5
--- /dev/null
+++ b/lib/gitlab/ci/config/interpolation/functions/expand_vars.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ class Config
+ module Interpolation
+ module Functions
+ class ExpandVars < Base
+ def self.function_expression_pattern
+ /^#{name}$/
+ end
+
+ def self.name
+ 'expand_vars'
+ end
+
+ def execute(input_value)
+ unless input_value.is_a?(String)
+ error("invalid input type: #{self.class.name} can only be used with string inputs")
+ return
+ end
+
+ ExpandVariables.expand_existing(input_value, ctx.variables, fail_on_masked: true)
+ rescue ExpandVariables::VariableExpansionError => e
+ error("variable expansion error: #{e.message}")
+ nil
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/config/interpolation/functions_stack.rb b/lib/gitlab/ci/config/interpolation/functions_stack.rb
index 951d1121d4f..4cb3e67b3e3 100644
--- a/lib/gitlab/ci/config/interpolation/functions_stack.rb
+++ b/lib/gitlab/ci/config/interpolation/functions_stack.rb
@@ -16,12 +16,14 @@ module Gitlab
end
FUNCTIONS = [
- Functions::Truncate
+ Functions::Truncate,
+ Functions::ExpandVars
].freeze
attr_reader :errors
- def initialize(function_expressions)
+ def initialize(function_expressions, ctx)
+ @ctx = ctx
@errors = []
@functions = build_stack(function_expressions)
end
@@ -48,14 +50,14 @@ module Gitlab
private
- attr_reader :functions
+ attr_reader :functions, :ctx
def build_stack(function_expressions)
function_expressions.map do |function_expression|
matching_function = FUNCTIONS.find { |function| function.matches?(function_expression) }
if matching_function.present?
- matching_function.new(function_expression)
+ matching_function.new(function_expression, ctx)
else
message = "no function matching `#{function_expression}`: " \
'check that the function name, arguments, and types are correct'
diff --git a/lib/gitlab/ci/config/interpolation/inputs/base_input.rb b/lib/gitlab/ci/config/interpolation/inputs/base_input.rb
index 5648c4d31ea..ba519776635 100644
--- a/lib/gitlab/ci/config/interpolation/inputs/base_input.rb
+++ b/lib/gitlab/ci/config/interpolation/inputs/base_input.rb
@@ -62,7 +62,15 @@ module Gitlab
end
# validate provided value
- error("provided value is not a #{self.class.type_name}") unless valid_value?(actual_value)
+ return error("provided value is not a #{self.class.type_name}") unless valid_value?(actual_value)
+
+ validate_regex!
+ end
+
+ def validate_regex!
+ return unless spec.key?(:regex)
+
+ error('RegEx validation can only be used with string inputs')
end
def error(message)
diff --git a/lib/gitlab/ci/config/interpolation/inputs/string_input.rb b/lib/gitlab/ci/config/interpolation/inputs/string_input.rb
index 39870582d0c..3f40e851f11 100644
--- a/lib/gitlab/ci/config/interpolation/inputs/string_input.rb
+++ b/lib/gitlab/ci/config/interpolation/inputs/string_input.rb
@@ -25,6 +25,24 @@ module Gitlab
def valid_value?(value)
value.nil? || value.is_a?(String)
end
+
+ private
+
+ def validate_regex!
+ return unless spec.key?(:regex)
+
+ safe_regex = ::Gitlab::UntrustedRegexp.new(spec[:regex])
+
+ return if safe_regex.match?(actual_value)
+
+ if value.nil?
+ error('default value does not match required RegEx pattern')
+ else
+ error('provided value does not match required RegEx pattern')
+ end
+ rescue RegexpError
+ error('invalid regular expression')
+ end
end
end
end
diff --git a/lib/gitlab/ci/config/interpolation/interpolator.rb b/lib/gitlab/ci/config/interpolation/interpolator.rb
index 95c419d7427..5b21b777c1d 100644
--- a/lib/gitlab/ci/config/interpolation/interpolator.rb
+++ b/lib/gitlab/ci/config/interpolation/interpolator.rb
@@ -8,11 +8,12 @@ module Gitlab
# Performs CI config file interpolation, and surfaces all possible interpolation errors.
#
class Interpolator
- attr_reader :config, :args, :errors
+ attr_reader :config, :args, :variables, :errors
- def initialize(config, args)
+ def initialize(config, args, variables)
@config = config
@args = args.to_h
+ @variables = variables
@errors = []
@interpolated = false
end
@@ -86,7 +87,7 @@ module Gitlab
end
def context
- @context ||= Context.new({ inputs: inputs.to_hash })
+ @context ||= Context.new({ inputs: inputs.to_hash }, variables: variables)
end
def template
diff --git a/lib/gitlab/ci/config/yaml/loader.rb b/lib/gitlab/ci/config/yaml/loader.rb
index 5d56061a8bb..1e9ac2b3dd5 100644
--- a/lib/gitlab/ci/config/yaml/loader.rb
+++ b/lib/gitlab/ci/config/yaml/loader.rb
@@ -10,9 +10,10 @@ module Gitlab
AVAILABLE_TAGS = [Config::Yaml::Tags::Reference].freeze
MAX_DOCUMENTS = 2
- def initialize(content, inputs: {})
+ def initialize(content, inputs: {}, variables: [])
@content = content
@inputs = inputs
+ @variables = variables
end
def load
@@ -20,7 +21,7 @@ module Gitlab
return yaml_result unless yaml_result.valid?
- interpolator = Interpolation::Interpolator.new(yaml_result, inputs)
+ interpolator = Interpolation::Interpolator.new(yaml_result, inputs, variables)
interpolator.interpolate!
@@ -32,16 +33,16 @@ module Gitlab
end
end
- private
-
- attr_reader :content, :inputs
-
def load_uninterpolated_yaml
Yaml::Result.new(config: load_yaml!, error: nil)
rescue ::Gitlab::Config::Loader::FormatError => e
Yaml::Result.new(error: e.message, error_class: e)
end
+ private
+
+ attr_reader :content, :inputs, :variables
+
def load_yaml!
ensure_custom_tags
diff --git a/lib/gitlab/ci/config/yaml/result.rb b/lib/gitlab/ci/config/yaml/result.rb
index a68cfde6653..0e7e9230467 100644
--- a/lib/gitlab/ci/config/yaml/result.rb
+++ b/lib/gitlab/ci/config/yaml/result.rb
@@ -39,6 +39,10 @@ module Gitlab
@config.first || {}
end
+
+ def inputs
+ (has_header? && header[:spec][:inputs]) || {}
+ end
end
end
end
diff --git a/lib/gitlab/ci/lint.rb b/lib/gitlab/ci/lint.rb
index 54861e2769e..f213bc83d90 100644
--- a/lib/gitlab/ci/lint.rb
+++ b/lib/gitlab/ci/lint.rb
@@ -25,12 +25,12 @@ module Gitlab
LOG_MAX_DURATION_THRESHOLD = 2.seconds
- def initialize(project:, current_user:, sha: nil)
+ def initialize(project:, current_user:, sha: nil, verify_project_sha: true)
@project = project
@current_user = current_user
# If the `sha` is not provided, the default is the project's head commit (or nil). In such case, we
# don't need to call `YamlProcessor.verify_project_sha!`, which prevents redundant calls to Gitaly.
- @verify_project_sha = sha.present?
+ @verify_project_sha = verify_project_sha && sha.present?
@sha = sha || project&.repository&.commit&.sha
end
diff --git a/lib/gitlab/ci/parsers/security/common.rb b/lib/gitlab/ci/parsers/security/common.rb
index ee1da82f285..9032faa66d4 100644
--- a/lib/gitlab/ci/parsers/security/common.rb
+++ b/lib/gitlab/ci/parsers/security/common.rb
@@ -140,7 +140,10 @@ module Gitlab
signatures: signatures,
project_id: @project.id,
found_by_pipeline: report.pipeline,
- vulnerability_finding_signatures_enabled: @signatures_enabled))
+ vulnerability_finding_signatures_enabled: @signatures_enabled,
+ cvss: data['cvss'] || []
+ )
+ )
end
def create_signatures(tracking)
diff --git a/lib/gitlab/ci/parsers/test/junit.rb b/lib/gitlab/ci/parsers/test/junit.rb
index d95ecff85cd..5b8abccc6d4 100644
--- a/lib/gitlab/ci/parsers/test/junit.rb
+++ b/lib/gitlab/ci/parsers/test/junit.rb
@@ -6,7 +6,7 @@ module Gitlab
module Test
class Junit
JunitParserError = Class.new(Gitlab::Ci::Parsers::ParserError)
- ATTACHMENT_TAG_REGEX = /\[\[ATTACHMENT\|(?<path>.+?)\]\]/.freeze
+ ATTACHMENT_TAG_REGEX = /\[\[ATTACHMENT\|(?<path>.+?)\]\]/
def parse!(xml_data, test_report, job:)
test_suite = test_report.get_suite(job.test_suite_name)
@@ -64,13 +64,16 @@ module Gitlab
end
def create_test_case(data, test_suite, job)
+ system_out = data.key?('system_out') ? "System Out:\n\n#{data['system_out']}" : nil
+ system_err = data.key?('system_err') ? "System Err:\n\n#{data['system_err']}" : nil
+
if data.key?('failure')
status = ::Gitlab::Ci::Reports::TestCase::STATUS_FAILED
- system_output = data['failure'] || data['system_err']
+ system_output = [data['failure'], system_out, system_err].compact.join("\n\n")
attachment = attachment_path(data['system_out'])
elsif data.key?('error')
status = ::Gitlab::Ci::Reports::TestCase::STATUS_ERROR
- system_output = data['error'] || data['system_err']
+ system_output = [data['error'], system_out, system_err].compact.join("\n\n")
attachment = attachment_path(data['system_out'])
elsif data.key?('skipped')
status = ::Gitlab::Ci::Reports::TestCase::STATUS_SKIPPED
diff --git a/lib/gitlab/ci/pipeline/chain/skip.rb b/lib/gitlab/ci/pipeline/chain/skip.rb
index 76dfb4cbd87..152ea700eb7 100644
--- a/lib/gitlab/ci/pipeline/chain/skip.rb
+++ b/lib/gitlab/ci/pipeline/chain/skip.rb
@@ -7,7 +7,7 @@ module Gitlab
class Skip < Chain::Base
include ::Gitlab::Utils::StrongMemoize
- SKIP_PATTERN = /\[(ci[ _-]skip|skip[ _-]ci)\]/i.freeze
+ SKIP_PATTERN = /\[(ci[ _-]skip|skip[ _-]ci)\]/i
def perform!
if skipped?
diff --git a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb
index 1939b1ff395..c89f9933616 100644
--- a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb
+++ b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb
@@ -19,7 +19,7 @@ module Gitlab
end
if project.import_in_progress?
- return error('Import in progress')
+ return error('You cannot run pipelines before project import is complete.')
end
unless allowed_to_create_pipeline?
diff --git a/lib/gitlab/ci/pipeline/expression.rb b/lib/gitlab/ci/pipeline/expression.rb
index 61d392121d8..a7b82395b6d 100644
--- a/lib/gitlab/ci/pipeline/expression.rb
+++ b/lib/gitlab/ci/pipeline/expression.rb
@@ -5,7 +5,6 @@ module Gitlab
module Pipeline
module Expression
ExpressionError = Class.new(StandardError)
- RuntimeError = Class.new(ExpressionError)
end
end
end
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/and.rb b/lib/gitlab/ci/pipeline/expression/lexeme/and.rb
index 422735bd104..70d439e2d20 100644
--- a/lib/gitlab/ci/pipeline/expression/lexeme/and.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/and.rb
@@ -6,7 +6,7 @@ module Gitlab
module Expression
module Lexeme
class And < Lexeme::LogicalOperator
- PATTERN = /&&/.freeze
+ PATTERN = /&&/
def evaluate(variables = {})
@left.evaluate(variables) && @right.evaluate(variables)
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb b/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb
index d35be12c996..9a45105eeaf 100644
--- a/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb
@@ -6,7 +6,7 @@ module Gitlab
module Expression
module Lexeme
class Equals < Lexeme::LogicalOperator
- PATTERN = /==/.freeze
+ PATTERN = /==/
def evaluate(variables = {})
@left.evaluate(variables) == @right.evaluate(variables)
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb
index c4f06c4686d..35e08776820 100644
--- a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb
@@ -6,7 +6,7 @@ module Gitlab
module Expression
module Lexeme
class Matches < Lexeme::LogicalOperator
- PATTERN = /=~/.freeze
+ PATTERN = /=~/
def evaluate(variables = {})
text = @left.evaluate(variables)
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/not_equals.rb b/lib/gitlab/ci/pipeline/expression/lexeme/not_equals.rb
index 64485a7e6b3..54ae3b0c369 100644
--- a/lib/gitlab/ci/pipeline/expression/lexeme/not_equals.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/not_equals.rb
@@ -6,7 +6,7 @@ module Gitlab
module Expression
module Lexeme
class NotEquals < Lexeme::LogicalOperator
- PATTERN = /!=/.freeze
+ PATTERN = /!=/
def evaluate(variables = {})
@left.evaluate(variables) != @right.evaluate(variables)
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb b/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb
index 99d9206da74..4cd9e3f3572 100644
--- a/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb
@@ -6,7 +6,7 @@ module Gitlab
module Expression
module Lexeme
class NotMatches < Lexeme::LogicalOperator
- PATTERN = /\!~/.freeze
+ PATTERN = /\!~/
def evaluate(variables = {})
text = @left.evaluate(variables)
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/null.rb b/lib/gitlab/ci/pipeline/expression/lexeme/null.rb
index e7f7945532b..89b7e0b102e 100644
--- a/lib/gitlab/ci/pipeline/expression/lexeme/null.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/null.rb
@@ -6,7 +6,7 @@ module Gitlab
module Expression
module Lexeme
class Null < Lexeme::Value
- PATTERN = /null/.freeze
+ PATTERN = /null/
def initialize(value = nil)
super
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/or.rb b/lib/gitlab/ci/pipeline/expression/lexeme/or.rb
index c7d653ac859..1a7b619c49c 100644
--- a/lib/gitlab/ci/pipeline/expression/lexeme/or.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/or.rb
@@ -6,7 +6,7 @@ module Gitlab
module Expression
module Lexeme
class Or < Lexeme::LogicalOperator
- PATTERN = /\|\|/.freeze
+ PATTERN = /\|\|/
def evaluate(variables = {})
@left.evaluate(variables) || @right.evaluate(variables)
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_close.rb b/lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_close.rb
index b0ca26c9f5d..29b5e47a65f 100644
--- a/lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_close.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_close.rb
@@ -6,7 +6,7 @@ module Gitlab
module Expression
module Lexeme
class ParenthesisClose < Lexeme::Operator
- PATTERN = /\)/.freeze
+ PATTERN = /\)/
def self.type
:parenthesis_close
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_open.rb b/lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_open.rb
index 924fe0663ab..80f92609154 100644
--- a/lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_open.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_open.rb
@@ -6,7 +6,7 @@ module Gitlab
module Expression
module Lexeme
class ParenthesisOpen < Lexeme::Operator
- PATTERN = /\(/.freeze
+ PATTERN = /\(/
def self.type
:parenthesis_open
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb
index cd4106b16bb..17fe82b2236 100644
--- a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb
@@ -5,23 +5,17 @@ module Gitlab
module Pipeline
module Expression
module Lexeme
- require_dependency 're2'
-
class Pattern < Lexeme::Value
- PATTERN = %r{^\/([^\/]|\\/)+[^\\]\/[ismU]*}.freeze
+ PATTERN = %r{^\/([^\/]|\\/)+[^\\]\/[ismU]*}
def initialize(regexp)
super(regexp.gsub(%r{\\/}, '/'))
- unless Gitlab::UntrustedRegexp::RubySyntax.valid?(@value)
- raise Lexer::SyntaxError, 'Invalid regular expression!'
- end
+ raise Lexer::SyntaxError, 'Invalid regular expression!' unless cached_regexp.valid?
end
def evaluate(variables = {})
- Gitlab::UntrustedRegexp::RubySyntax.fabricate!(@value)
- rescue RegexpError
- raise Expression::RuntimeError, 'Invalid regular expression!'
+ cached_regexp.expression
end
def inspect
@@ -47,6 +41,12 @@ module Gitlab
new_pattern.evaluate(variables)
end
+
+ private
+
+ def cached_regexp
+ @cached_regexp ||= RegularExpression.new(@value)
+ end
end
end
end
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/pattern/regular_expression.rb b/lib/gitlab/ci/pipeline/expression/lexeme/pattern/regular_expression.rb
new file mode 100644
index 00000000000..5b771abf4ba
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/pattern/regular_expression.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Pipeline
+ module Expression
+ module Lexeme
+ class Pattern
+ require_dependency 're2'
+ class RegularExpression
+ include Gitlab::Utils::StrongMemoize
+
+ attr_reader :value
+
+ def initialize(value)
+ @value = value
+ end
+
+ def expression
+ Gitlab::SafeRequestStore.fetch("#{self.class}#unsafe_regexp:#{value}") do
+ Gitlab::UntrustedRegexp::RubySyntax.fabricate!(value)
+ end
+ end
+ strong_memoize_attr :expression
+
+ def valid?
+ !!expression
+ rescue RegexpError
+ false
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/string.rb b/lib/gitlab/ci/pipeline/expression/lexeme/string.rb
index 798cea34db6..c43150125b7 100644
--- a/lib/gitlab/ci/pipeline/expression/lexeme/string.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/string.rb
@@ -6,7 +6,7 @@ module Gitlab
module Expression
module Lexeme
class String < Lexeme::Value
- PATTERN = /("(?<string>.*?)")|('(?<string>.*?)')/.freeze
+ PATTERN = /("(?<string>.*?)")|('(?<string>.*?)')/
def evaluate(variables = {})
@value.to_s
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb
index 6da88fd287e..2ecd50d32e4 100644
--- a/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb
@@ -6,7 +6,7 @@ module Gitlab
module Expression
module Lexeme
class Variable < Lexeme::Value
- PATTERN = /\$(?<name>\w+)/.freeze
+ PATTERN = /\$(?<name>\w+)/
def evaluate(variables = {})
unless variables.is_a?(ActiveSupport::HashWithIndifferentAccess)
diff --git a/lib/gitlab/ci/reports/security/finding.rb b/lib/gitlab/ci/reports/security/finding.rb
index d439149158a..fa8494483d3 100644
--- a/lib/gitlab/ci/reports/security/finding.rb
+++ b/lib/gitlab/ci/reports/security/finding.rb
@@ -30,12 +30,13 @@ module Gitlab
attr_reader :project_id
attr_reader :original_data
attr_reader :found_by_pipeline
+ attr_reader :cvss
delegate :file_path, :start_line, :end_line, to: :location
alias_method :cve, :compare_key
- def initialize(compare_key:, identifiers:, flags: [], links: [], remediations: [], location:, evidence:, metadata_version:, name:, original_data:, report_type:, scanner:, scan:, uuid:, confidence: nil, severity: nil, details: {}, signatures: [], project_id: nil, vulnerability_finding_signatures_enabled: false, found_by_pipeline: nil) # rubocop:disable Metrics/ParameterLists
+ def initialize(compare_key:, identifiers:, flags: [], links: [], remediations: [], location:, evidence:, metadata_version:, name:, original_data:, report_type:, scanner:, scan:, uuid:, confidence: nil, severity: nil, details: {}, signatures: [], project_id: nil, vulnerability_finding_signatures_enabled: false, found_by_pipeline: nil, cvss: []) # rubocop:disable Metrics/ParameterLists
@compare_key = compare_key
@confidence = confidence
@identifiers = identifiers
@@ -57,6 +58,7 @@ module Gitlab
@project_id = project_id
@vulnerability_finding_signatures_enabled = vulnerability_finding_signatures_enabled
@found_by_pipeline = found_by_pipeline
+ @cvss = cvss
@project_fingerprint = generate_project_fingerprint
end
diff --git a/lib/gitlab/ci/status/canceled.rb b/lib/gitlab/ci/status/canceled.rb
index f173964b36c..a3376692570 100644
--- a/lib/gitlab/ci/status/canceled.rb
+++ b/lib/gitlab/ci/status/canceled.rb
@@ -5,7 +5,7 @@ module Gitlab
module Status
class Canceled < Status::Core
def text
- s_('CiStatusText|canceled')
+ s_('CiStatusText|Canceled')
end
def label
diff --git a/lib/gitlab/ci/status/core.rb b/lib/gitlab/ci/status/core.rb
index f60f5243666..c5306de830b 100644
--- a/lib/gitlab/ci/status/core.rb
+++ b/lib/gitlab/ci/status/core.rb
@@ -38,6 +38,10 @@ module Gitlab
raise NotImplementedError
end
+ def name
+ self.class.name.demodulize.underscore.upcase
+ end
+
def group
self.class.name.demodulize.underscore
end
diff --git a/lib/gitlab/ci/status/created.rb b/lib/gitlab/ci/status/created.rb
index 33e67314d93..9ad4b2f079e 100644
--- a/lib/gitlab/ci/status/created.rb
+++ b/lib/gitlab/ci/status/created.rb
@@ -5,7 +5,7 @@ module Gitlab
module Status
class Created < Status::Core
def text
- s_('CiStatusText|created')
+ s_('CiStatusText|Created')
end
def label
diff --git a/lib/gitlab/ci/status/failed.rb b/lib/gitlab/ci/status/failed.rb
index 215d27734a7..cb498f72ffe 100644
--- a/lib/gitlab/ci/status/failed.rb
+++ b/lib/gitlab/ci/status/failed.rb
@@ -5,7 +5,7 @@ module Gitlab
module Status
class Failed < Status::Core
def text
- s_('CiStatusText|failed')
+ s_('CiStatusText|Failed')
end
def label
diff --git a/lib/gitlab/ci/status/manual.rb b/lib/gitlab/ci/status/manual.rb
index eb376df5f22..02e65dd1f4c 100644
--- a/lib/gitlab/ci/status/manual.rb
+++ b/lib/gitlab/ci/status/manual.rb
@@ -5,7 +5,7 @@ module Gitlab
module Status
class Manual < Status::Core
def text
- s_('CiStatusText|manual')
+ s_('CiStatusText|Manual')
end
def label
diff --git a/lib/gitlab/ci/status/pending.rb b/lib/gitlab/ci/status/pending.rb
index 4280ad84534..ddbdf94c089 100644
--- a/lib/gitlab/ci/status/pending.rb
+++ b/lib/gitlab/ci/status/pending.rb
@@ -5,7 +5,7 @@ module Gitlab
module Status
class Pending < Status::Core
def text
- s_('CiStatusText|pending')
+ s_('CiStatusText|Pending')
end
def label
diff --git a/lib/gitlab/ci/status/pipeline/blocked.rb b/lib/gitlab/ci/status/pipeline/blocked.rb
index ed13a439be0..2e01f4948a9 100644
--- a/lib/gitlab/ci/status/pipeline/blocked.rb
+++ b/lib/gitlab/ci/status/pipeline/blocked.rb
@@ -6,7 +6,7 @@ module Gitlab
module Pipeline
class Blocked < Status::Extended
def text
- s_('CiStatusText|blocked')
+ s_('CiStatusText|Blocked')
end
def label
diff --git a/lib/gitlab/ci/status/pipeline/delayed.rb b/lib/gitlab/ci/status/pipeline/delayed.rb
index e61acdcd167..47048afbe1d 100644
--- a/lib/gitlab/ci/status/pipeline/delayed.rb
+++ b/lib/gitlab/ci/status/pipeline/delayed.rb
@@ -6,7 +6,7 @@ module Gitlab
module Pipeline
class Delayed < Status::Extended
def text
- s_('CiStatusText|delayed')
+ s_('CiStatusText|Delayed')
end
def label
diff --git a/lib/gitlab/ci/status/preparing.rb b/lib/gitlab/ci/status/preparing.rb
index e59d1d2eed1..e29b5416e8d 100644
--- a/lib/gitlab/ci/status/preparing.rb
+++ b/lib/gitlab/ci/status/preparing.rb
@@ -5,7 +5,7 @@ module Gitlab
module Status
class Preparing < Status::Core
def text
- s_('CiStatusText|preparing')
+ s_('CiStatusText|Preparing')
end
def label
diff --git a/lib/gitlab/ci/status/running.rb b/lib/gitlab/ci/status/running.rb
index eed1983e60e..dc36e62e2a3 100644
--- a/lib/gitlab/ci/status/running.rb
+++ b/lib/gitlab/ci/status/running.rb
@@ -5,11 +5,11 @@ module Gitlab
module Status
class Running < Status::Core
def text
- s_('CiStatus|running')
+ s_('CiStatusText|Running')
end
def label
- s_('CiStatus|running')
+ s_('CiStatusLabel|running')
end
def icon
diff --git a/lib/gitlab/ci/status/scheduled.rb b/lib/gitlab/ci/status/scheduled.rb
index 8526becfef9..a3797c5c8d7 100644
--- a/lib/gitlab/ci/status/scheduled.rb
+++ b/lib/gitlab/ci/status/scheduled.rb
@@ -5,7 +5,7 @@ module Gitlab
module Status
class Scheduled < Status::Core
def text
- s_('CiStatusText|scheduled')
+ s_('CiStatusText|Scheduled')
end
def label
diff --git a/lib/gitlab/ci/status/skipped.rb b/lib/gitlab/ci/status/skipped.rb
index 238aa3ab4f9..4263536552b 100644
--- a/lib/gitlab/ci/status/skipped.rb
+++ b/lib/gitlab/ci/status/skipped.rb
@@ -5,7 +5,7 @@ module Gitlab
module Status
class Skipped < Status::Core
def text
- s_('CiStatusText|skipped')
+ s_('CiStatusText|Skipped')
end
def label
diff --git a/lib/gitlab/ci/status/success.rb b/lib/gitlab/ci/status/success.rb
index 2a10e60414e..9389138e034 100644
--- a/lib/gitlab/ci/status/success.rb
+++ b/lib/gitlab/ci/status/success.rb
@@ -5,7 +5,7 @@ module Gitlab
module Status
class Success < Status::Core
def text
- s_('CiStatusText|passed')
+ s_('CiStatusText|Passed')
end
def label
diff --git a/lib/gitlab/ci/status/success_warning.rb b/lib/gitlab/ci/status/success_warning.rb
index 84a0e52f518..91f0ba1a58f 100644
--- a/lib/gitlab/ci/status/success_warning.rb
+++ b/lib/gitlab/ci/status/success_warning.rb
@@ -9,7 +9,7 @@ module Gitlab
#
class SuccessWarning < Status::Extended
def text
- s_('CiStatusText|warning')
+ s_('CiStatusText|Warning')
end
def label
@@ -20,6 +20,10 @@ module Gitlab
'status_warning'
end
+ def name
+ 'SUCCESS_WITH_WARNINGS'
+ end
+
def group
'success-with-warnings'
end
diff --git a/lib/gitlab/ci/status/waiting_for_resource.rb b/lib/gitlab/ci/status/waiting_for_resource.rb
index 9ced0aadb88..5714a68cac8 100644
--- a/lib/gitlab/ci/status/waiting_for_resource.rb
+++ b/lib/gitlab/ci/status/waiting_for_resource.rb
@@ -5,7 +5,7 @@ module Gitlab
module Status
class WaitingForResource < Status::Core
def text
- s_('CiStatusText|waiting')
+ s_('CiStatusText|Waiting')
end
def label
@@ -20,6 +20,10 @@ module Gitlab
'favicon_status_pending'
end
+ def name
+ 'WAITING_FOR_RESOURCE'
+ end
+
def group
'waiting-for-resource'
end
diff --git a/lib/gitlab/ci/templates/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Code-Quality.gitlab-ci.yml
index b4ccf96b859..3132535ef6b 100644
--- a/lib/gitlab/ci/templates/Code-Quality.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Code-Quality.gitlab-ci.yml
@@ -1,2 +1,2 @@
include:
- template: Jobs/Code-Quality.gitlab-ci.yml
+ - template: Jobs/Code-Quality.gitlab-ci.yml
diff --git a/lib/gitlab/ci/templates/Cosign.gitlab-ci.yml b/lib/gitlab/ci/templates/Cosign.gitlab-ci.yml
index 48c9422b469..356062c734e 100644
--- a/lib/gitlab/ci/templates/Cosign.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Cosign.gitlab-ci.yml
@@ -8,7 +8,7 @@
# See https://docs.gitlab.com/ee/ci/yaml/signing_examples.html for more details.
include:
- template: Docker.gitlab-ci.yml
+ - template: Docker.gitlab-ci.yml
docker-build:
variables:
diff --git a/lib/gitlab/ci/templates/Docker.gitlab-ci.yml b/lib/gitlab/ci/templates/Docker.gitlab-ci.yml
index 1aa346aec67..416f424dfa5 100644
--- a/lib/gitlab/ci/templates/Docker.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Docker.gitlab-ci.yml
@@ -11,7 +11,7 @@
docker-build:
# Use the official docker image.
- image: docker:latest
+ image: docker:cli
stage: build
services:
- docker:dind
diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml
index 07bc3fbe795..2d04c97b32e 100644
--- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml
@@ -1,5 +1,5 @@
variables:
- AUTO_BUILD_IMAGE_VERSION: 'v1.41.0'
+ AUTO_BUILD_IMAGE_VERSION: 'v1.44.0'
build:
stage: build
diff --git a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml
index 07bc3fbe795..2d04c97b32e 100644
--- a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml
@@ -1,5 +1,5 @@
variables:
- AUTO_BUILD_IMAGE_VERSION: 'v1.41.0'
+ AUTO_BUILD_IMAGE_VERSION: 'v1.44.0'
build:
stage: build
diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
index f9440bfe904..45547b87eb6 100644
--- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
@@ -7,7 +7,9 @@ code_quality:
command: ['--tls=false', '--host=tcp://0.0.0.0:2375']
variables:
DOCKER_DRIVER: overlay2
+ DOCKER_CERT_PATH: ""
DOCKER_TLS_CERTDIR: ""
+ DOCKER_TLS_VERIFY: ""
CODE_QUALITY_IMAGE_TAG: "0.96.0"
CODE_QUALITY_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/gitlab-org/ci-cd/codequality:$CODE_QUALITY_IMAGE_TAG"
DOCKER_SOCKET_PATH: /var/run/docker.sock
diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
index e9ba938142d..4d53b92763a 100644
--- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
@@ -1,5 +1,5 @@
variables:
- DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.56.0'
+ DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.59.1'
.dast-auto-deploy:
image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}"
diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
index eaaf171e4b5..390824e8e49 100644
--- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
@@ -1,5 +1,5 @@
variables:
- AUTO_DEPLOY_IMAGE_VERSION: 'v2.56.0'
+ AUTO_DEPLOY_IMAGE_VERSION: 'v2.59.1'
.auto-deploy:
image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}"
diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml
index d2e448fb6a1..a9681c0f927 100644
--- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml
@@ -1,5 +1,5 @@
variables:
- AUTO_DEPLOY_IMAGE_VERSION: 'v2.56.0'
+ AUTO_DEPLOY_IMAGE_VERSION: 'v2.59.1'
.auto-deploy:
image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}"
diff --git a/lib/gitlab/ci/templates/Python.gitlab-ci.yml b/lib/gitlab/ci/templates/Python.gitlab-ci.yml
index d53f3ddcad4..c19a08bd11d 100644
--- a/lib/gitlab/ci/templates/Python.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Python.gitlab-ci.yml
@@ -12,15 +12,10 @@ image: python:latest
variables:
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
-# Pip's cache doesn't store the python packages
# https://pip.pypa.io/en/stable/topics/caching/
-#
-# If you want to also cache the installed packages, you have to install
-# them in a virtualenv and cache it as well.
cache:
paths:
- .cache/pip
- - venv/
before_script:
- python --version ; pip --version # For debugging
diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml
index 879d6a7a468..d6384f59bc1 100644
--- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml
@@ -2,4 +2,4 @@
# Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/381665
include:
- template: Jobs/Container-Scanning.gitlab-ci.yml
+ - template: Jobs/Container-Scanning.gitlab-ci.yml
diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.latest.gitlab-ci.yml
index 7a4f451314e..f4fd9e97665 100644
--- a/lib/gitlab/ci/templates/Security/Container-Scanning.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/Container-Scanning.latest.gitlab-ci.yml
@@ -2,4 +2,4 @@
# Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/381665
include:
- template: Jobs/Container-Scanning.latest.gitlab-ci.yml
+ - template: Jobs/Container-Scanning.latest.gitlab-ci.yml
diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml
index 1785d4216e7..2055b5e181f 100644
--- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml
@@ -2,4 +2,4 @@
# Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/292977
include:
- template: Jobs/Dependency-Scanning.gitlab-ci.yml
+ - template: Jobs/Dependency-Scanning.gitlab-ci.yml
diff --git a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml
index a99fe4a6dcf..0fe544b2c84 100644
--- a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml
@@ -2,4 +2,4 @@
# Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/292977
include:
- template: Jobs/License-Scanning.gitlab-ci.yml
+ - template: Jobs/License-Scanning.gitlab-ci.yml
diff --git a/lib/gitlab/ci/templates/Security/SAST-IaC.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST-IaC.gitlab-ci.yml
index 2207d4ec17a..4cc51c01b63 100644
--- a/lib/gitlab/ci/templates/Security/SAST-IaC.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/SAST-IaC.gitlab-ci.yml
@@ -1,2 +1,2 @@
include:
- template: Jobs/SAST-IaC.gitlab-ci.yml
+ - template: Jobs/SAST-IaC.gitlab-ci.yml
diff --git a/lib/gitlab/ci/templates/Security/SAST-IaC.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST-IaC.latest.gitlab-ci.yml
index 8c0d72ff282..a411fc03122 100644
--- a/lib/gitlab/ci/templates/Security/SAST-IaC.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/SAST-IaC.latest.gitlab-ci.yml
@@ -1,2 +1,2 @@
include:
- template: Jobs/SAST-IaC.latest.gitlab-ci.yml
+ - template: Jobs/SAST-IaC.latest.gitlab-ci.yml
diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
index 77ce813dd4f..6c25d628d55 100644
--- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
@@ -2,4 +2,4 @@
# Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/292977
include:
- template: Jobs/SAST.gitlab-ci.yml
+ - template: Jobs/SAST.gitlab-ci.yml
diff --git a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml
index d4ea7165d0a..353d523daf3 100644
--- a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml
@@ -2,4 +2,4 @@
# Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/292977
include:
- template: Jobs/Secret-Detection.gitlab-ci.yml
+ - template: Jobs/Secret-Detection.gitlab-ci.yml
diff --git a/lib/gitlab/ci/trace/section_parser.rb b/lib/gitlab/ci/trace/section_parser.rb
index f33f8cc56c1..a6c1bf28f24 100644
--- a/lib/gitlab/ci/trace/section_parser.rb
+++ b/lib/gitlab/ci/trace/section_parser.rb
@@ -74,7 +74,7 @@ module Gitlab
end
def beginning_of_section_regex
- @beginning_of_section_regex ||= /section_/.freeze
+ @beginning_of_section_regex ||= /section_/
end
def find_next_marker(scanner)
diff --git a/lib/gitlab/ci/variables/collection/item.rb b/lib/gitlab/ci/variables/collection/item.rb
index 73452d83bce..2334db0718f 100644
--- a/lib/gitlab/ci/variables/collection/item.rb
+++ b/lib/gitlab/ci/variables/collection/item.rb
@@ -7,7 +7,7 @@ module Gitlab
class Item
include Gitlab::Utils::StrongMemoize
- VARIABLES_REGEXP = /\$\$|%%|\$(?<key>[a-zA-Z_][a-zA-Z0-9_]*)|\${\g<key>?}|%\g<key>%/.freeze.freeze
+ VARIABLES_REGEXP = /\$\$|%%|\$(?<key>[a-zA-Z_][a-zA-Z0-9_]*)|\${\g<key>?}|%\g<key>%/
VARIABLE_REF_CHARS = %w[$ %].freeze
def initialize(key:, value:, public: true, file: false, masked: false, raw: false)
@@ -34,6 +34,10 @@ module Gitlab
@variable.fetch(:file)
end
+ def masked?
+ @variable.fetch(:masked)
+ end
+
def [](key)
@variable.fetch(key)
end
diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb
index 289f41b4ec7..cf5755242e2 100644
--- a/lib/gitlab/ci/yaml_processor.rb
+++ b/lib/gitlab/ci/yaml_processor.rb
@@ -209,7 +209,8 @@ module Gitlab
return unless project && sha && project.repository_exists? && project.commit(sha)
unless project_ref_contains_sha?
- error!('Could not validate configuration. Config originates from external project')
+ error!('Could not validate configuration. The configuration originates from an external ' \
+ 'project or a commit not associated with a Git reference (a detached commit)')
end
end
diff --git a/lib/gitlab/cleanup/project_uploads.rb b/lib/gitlab/cleanup/project_uploads.rb
index 7f24b2f78b0..6feaab2a791 100644
--- a/lib/gitlab/cleanup/project_uploads.rb
+++ b/lib/gitlab/cleanup/project_uploads.rb
@@ -93,7 +93,7 @@ module Gitlab
end
class ProjectUploadPath
- PROJECT_FULL_PATH_REGEX = %r{\A#{FileUploader.root}/(.+)/(\h+/[^/]+)\z}.freeze
+ PROJECT_FULL_PATH_REGEX = %r{\A#{FileUploader.root}/(.+)/(\h+/[^/]+)\z}
attr_reader :full_path, :upload_path
diff --git a/lib/gitlab/color.rb b/lib/gitlab/color.rb
index c31c8cb5876..4be78f59bd3 100644
--- a/lib/gitlab/color.rb
+++ b/lib/gitlab/color.rb
@@ -2,7 +2,7 @@
module Gitlab
class Color
- PATTERN = /\A\#(?:[0-9A-Fa-f]{3}){1,2}\Z/.freeze
+ PATTERN = /\A\#(?:[0-9A-Fa-f]{3}){1,2}\Z/
def initialize(value)
@value = value&.strip&.freeze
diff --git a/lib/gitlab/config/entry/legacy_validation_helpers.rb b/lib/gitlab/config/entry/legacy_validation_helpers.rb
index ec67d65c526..1f70afbfb75 100644
--- a/lib/gitlab/config/entry/legacy_validation_helpers.rb
+++ b/lib/gitlab/config/entry/legacy_validation_helpers.rb
@@ -12,7 +12,7 @@ module Gitlab
if parser && parser.respond_to?(:validate_duration)
parser.validate_duration(value)
else
- ChronicDuration.parse(value, use_complete_matcher: true)
+ ChronicDuration.parse(value)
end
rescue ChronicDuration::DurationParseError
false
@@ -24,12 +24,7 @@ module Gitlab
if parser && parser.respond_to?(:validate_duration_limit)
parser.validate_duration_limit(value, limit)
else
- ChronicDuration.parse(
- value, use_complete_matcher: true
- ).second.from_now <
- ChronicDuration.parse(
- limit, use_complete_matcher: true
- ).second.from_now
+ ChronicDuration.parse(value).second.from_now < ChronicDuration.parse(limit).second.from_now
end
rescue ChronicDuration::DurationParseError
false
diff --git a/lib/gitlab/config/loader/multi_doc_yaml.rb b/lib/gitlab/config/loader/multi_doc_yaml.rb
index 084d32a85bc..5db1cc9a5d5 100644
--- a/lib/gitlab/config/loader/multi_doc_yaml.rb
+++ b/lib/gitlab/config/loader/multi_doc_yaml.rb
@@ -6,7 +6,7 @@ module Gitlab
class MultiDocYaml
include Gitlab::Utils::StrongMemoize
- MULTI_DOC_DIVIDER = /^---\s+/.freeze
+ MULTI_DOC_DIVIDER = /^---\s+/
def initialize(config, max_documents:, additional_permitted_classes: [], reject_empty: false)
@config = config
diff --git a/lib/gitlab/content_security_policy/config_loader.rb b/lib/gitlab/content_security_policy/config_loader.rb
index 87a6d4ada70..f7c9d95c53c 100644
--- a/lib/gitlab/content_security_policy/config_loader.rb
+++ b/lib/gitlab/content_security_policy/config_loader.rb
@@ -86,7 +86,7 @@ module Gitlab
def add_browsersdk_tracking(directives)
return if directives.blank?
- return unless Gitlab.com? && Feature.enabled?(:browsersdk_tracking) && ENV['GITLAB_ANALYTICS_URL'].present?
+ return unless Gitlab.com? && ENV['GITLAB_ANALYTICS_URL'].present?
default_connect_src = directives['connect-src'] || directives['default-src']
connect_src_values = Array.wrap(default_connect_src) | [ENV['GITLAB_ANALYTICS_URL']]
diff --git a/lib/gitlab/database/background_migration/batch_optimizer.rb b/lib/gitlab/database/background_migration/batch_optimizer.rb
index 9eb456f6e2e..e99b63c0e4b 100644
--- a/lib/gitlab/database/background_migration/batch_optimizer.rb
+++ b/lib/gitlab/database/background_migration/batch_optimizer.rb
@@ -17,7 +17,7 @@ module Gitlab
class BatchOptimizer
# Target time efficiency for a job
# Time efficiency is defined as: job duration / interval
- TARGET_EFFICIENCY = (0.9..0.95).freeze
+ TARGET_EFFICIENCY = (0.9..0.95)
# Lower and upper bound for the batch size
MIN_BATCH_SIZE = 1_000
diff --git a/lib/gitlab/database/gitlab_schema.rb b/lib/gitlab/database/gitlab_schema.rb
index 0bd357b7730..31ceb898eee 100644
--- a/lib/gitlab/database/gitlab_schema.rb
+++ b/lib/gitlab/database/gitlab_schema.rb
@@ -87,24 +87,37 @@ module Gitlab
# rubocop:enable Gitlab/DocUrl
end
- private_class_method def self.cross_access_allowed?(type, table_schemas)
+ def self.cross_joins_allowed?(table_schemas, all_tables)
+ return true unless table_schemas.many?
+
table_schemas.any? do |schema|
- extra_schemas = table_schemas - [schema]
- extra_schemas -= Gitlab::Database.all_gitlab_schemas[schema]&.public_send(type) || [] # rubocop:disable GitlabSecurity/PublicSend
- extra_schemas.empty?
+ schema_info = Gitlab::Database.all_gitlab_schemas[schema]
+ next false unless schema_info
+
+ schema_info.allow_cross_joins?(table_schemas, all_tables)
end
end
- def self.cross_joins_allowed?(table_schemas)
- table_schemas.empty? || self.cross_access_allowed?(:allow_cross_joins, table_schemas)
- end
+ def self.cross_transactions_allowed?(table_schemas, all_tables)
+ return true unless table_schemas.many?
+
+ table_schemas.any? do |schema|
+ schema_info = Gitlab::Database.all_gitlab_schemas[schema]
+ next false unless schema_info
- def self.cross_transactions_allowed?(table_schemas)
- table_schemas.empty? || self.cross_access_allowed?(:allow_cross_transactions, table_schemas)
+ schema_info.allow_cross_transactions?(table_schemas, all_tables)
+ end
end
- def self.cross_foreign_key_allowed?(table_schemas)
- self.cross_access_allowed?(:allow_cross_foreign_keys, table_schemas)
+ def self.cross_foreign_key_allowed?(table_schemas, all_tables)
+ return true if table_schemas.one?
+
+ table_schemas.any? do |schema|
+ schema_info = Gitlab::Database.all_gitlab_schemas[schema]
+ next false unless schema_info
+
+ schema_info.allow_cross_foreign_keys?(table_schemas, all_tables)
+ end
end
def self.dictionary_paths
diff --git a/lib/gitlab/database/gitlab_schema_info.rb b/lib/gitlab/database/gitlab_schema_info.rb
index 34b89cb9006..20d2b31a65c 100644
--- a/lib/gitlab/database/gitlab_schema_info.rb
+++ b/lib/gitlab/database/gitlab_schema_info.rb
@@ -2,6 +2,11 @@
module Gitlab
module Database
+ GitlabSchemaInfoAllowCross = Struct.new(
+ :specific_tables,
+ keyword_init: true
+ )
+
GitlabSchemaInfo = Struct.new(
:name,
:description,
@@ -14,15 +19,76 @@ module Gitlab
def initialize(*)
super
self.name = name.to_sym
- self.allow_cross_joins = allow_cross_joins&.map(&:to_sym)&.freeze
- self.allow_cross_transactions = allow_cross_transactions&.map(&:to_sym)&.freeze
- self.allow_cross_foreign_keys = allow_cross_foreign_keys&.map(&:to_sym)&.freeze
+ self.allow_cross_joins = convert_array_to_hash(allow_cross_joins)
+ self.allow_cross_transactions = convert_array_to_hash(allow_cross_transactions)
+ self.allow_cross_foreign_keys = convert_array_to_hash(allow_cross_foreign_keys)
end
def self.load_file(yaml_file)
content = YAML.load_file(yaml_file)
new(**content.deep_symbolize_keys.merge(file_path: yaml_file))
end
+
+ def allow_cross_joins?(table_schemas, all_tables)
+ allowed_schemas = allow_cross_joins || {}
+
+ allowed_for?(allowed_schemas, table_schemas, all_tables)
+ end
+
+ def allow_cross_transactions?(table_schemas, all_tables)
+ allowed_schemas = allow_cross_transactions || {}
+
+ allowed_for?(allowed_schemas, table_schemas, all_tables)
+ end
+
+ def allow_cross_foreign_keys?(table_schemas, all_tables)
+ allowed_schemas = allow_cross_foreign_keys || {}
+
+ allowed_for?(allowed_schemas, table_schemas, all_tables)
+ end
+
+ private
+
+ def allowed_for?(allowed_schemas, table_schemas, all_tables)
+ denied_schemas = table_schemas - [name]
+ denied_schemas -= allowed_schemas.keys
+ return false unless denied_schemas.empty?
+
+ all_tables.all? do |table|
+ table_schema = ::Gitlab::Database::GitlabSchema.table_schema!(table)
+ allowed_tables = allowed_schemas[table_schema]
+
+ allowed_tables.nil? || allowed_tables.specific_tables.include?(table)
+ end
+ end
+
+ # Convert from:
+ # - schema_a
+ # - schema_b:
+ # specific_tables:
+ # - table_b_of_schema_b
+ # - table_c_of_schema_b
+ #
+ # To:
+ # { :schema_a => nil,
+ # :schema_b => { specific_tables : [:table_b_of_schema_b, :table_c_of_schema_b] }
+ # }
+ #
+ def convert_array_to_hash(subject)
+ result = {}
+
+ subject&.each do |item|
+ if item.is_a?(Hash)
+ item.each do |key, value|
+ result[key.to_sym] = GitlabSchemaInfoAllowCross.new(value || {})
+ end
+ else
+ result[item.to_sym] = nil
+ end
+ end
+
+ result.freeze
+ end
end
end
end
diff --git a/lib/gitlab/database/load_balancing/service_discovery.rb b/lib/gitlab/database/load_balancing/service_discovery.rb
index 1f9ab1cfe98..3d11f0f88c1 100644
--- a/lib/gitlab/database/load_balancing/service_discovery.rb
+++ b/lib/gitlab/database/load_balancing/service_discovery.rb
@@ -24,7 +24,7 @@ module Gitlab
MAX_DISCOVERY_RETRIES = 3
DISCOVERY_THREAD_REFRESH_DELTA = 5
- RETRY_DELAY_RANGE = (0.1..0.2).freeze
+ RETRY_DELAY_RANGE = (0.1..0.2)
RECORD_TYPES = {
'A' => Net::DNS::A,
@@ -151,13 +151,7 @@ module Gitlab
# started just before we added the new hosts it will use an old
# host/connection. While this connection will be checked in and out,
# it won't be explicitly disconnected.
- if Gitlab::Utils.to_boolean(ENV['LOAD_BALANCER_PARALLEL_DISCONNECT'], default: false)
- disconnect_old_hosts(old_hosts)
- else
- old_hosts.each do |host|
- host.disconnect!(timeout: disconnect_timeout)
- end
- end
+ disconnect_old_hosts(old_hosts)
end
# Returns an Array containing:
diff --git a/lib/gitlab/database/migration.rb b/lib/gitlab/database/migration.rb
index fbb71c1ccfd..41044816de9 100644
--- a/lib/gitlab/database/migration.rb
+++ b/lib/gitlab/database/migration.rb
@@ -34,7 +34,7 @@ module Gitlab
# to indicate backwards-compatible or otherwise minor changes (e.g. a Rails version bump).
# However, this hasn't been strictly formalized yet.
- class V1_0 < ActiveRecord::Migration[6.1] # rubocop:disable Naming/ClassAndModuleCamelCase
+ class V1_0 < ActiveRecord::Migration[6.1]
include LockRetriesConcern
include Gitlab::Database::MigrationHelpers::V2
include Gitlab::Database::MigrationHelpers::AnnounceDatabase
@@ -47,11 +47,11 @@ module Gitlab
end
end
- class V2_0 < V1_0 # rubocop:disable Naming/ClassAndModuleCamelCase
+ class V2_0 < V1_0
include Gitlab::Database::MigrationHelpers::RestrictGitlabSchema
end
- class V2_1 < V2_0 # rubocop:disable Naming/ClassAndModuleCamelCase
+ class V2_1 < V2_0
include Gitlab::Database::MigrationHelpers::AutomaticLockWritesOnTables
include Gitlab::Database::Migrations::RunnerBackoff::MigrationHelpers
end
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index 60cec12b4b5..efcceafda90 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -567,8 +567,8 @@ module Gitlab
# table - The table containing the column.
# column - The name of the column to change.
# new_type - The new column type.
- def cleanup_concurrent_column_type_change(table, column)
- temp_column = "#{column}_for_type_change"
+ def cleanup_concurrent_column_type_change(table, column, temp_column: nil)
+ temp_column ||= "#{column}_for_type_change"
transaction do
# This has to be performed in a transaction as otherwise we might have
@@ -586,10 +586,10 @@ module Gitlab
# type_cast_function - Required if the conversion back to the original type is not automatic
# batch_column_name - option for tables without a primary key, in this case
# another unique integer column can be used. Example: :user_id
- def undo_cleanup_concurrent_column_type_change(table, column, old_type, type_cast_function: nil, batch_column_name: :id, limit: nil)
+ def undo_cleanup_concurrent_column_type_change(table, column, old_type, type_cast_function: nil, batch_column_name: :id, limit: nil, temp_column: nil)
Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode!
- temp_column = "#{column}_for_type_change"
+ temp_column ||= "#{column}_for_type_change"
# Using a descriptive name that includes orinal column's name risks
# taking us above the 63 character limit, so we use a hash
@@ -751,7 +751,7 @@ module Gitlab
trigger_name = rename_trigger_name(table, columns, temporary_columns)
remove_rename_triggers(table, trigger_name)
- temporary_columns.each { |column| remove_column(table, column) }
+ temporary_columns.each { |column| remove_column(table, column, if_exists: true) }
end
alias_method :cleanup_conversion_of_integer_to_bigint, :revert_initialize_conversion_of_integer_to_bigint
@@ -909,6 +909,11 @@ module Gitlab
name = index.name.gsub(old, new)
+ if name.length > 63
+ digest = Digest::SHA256.hexdigest(name).first(10)
+ name = "idx_copy_#{digest}"
+ end
+
options = {
unique: index.unique,
name: name,
@@ -1204,6 +1209,10 @@ into similar problems in the future (e.g. when new tables are created).
end
end
+ def lock_tables(*tables, mode: :access_exclusive)
+ execute("LOCK TABLE #{tables.join(', ')} IN #{mode.to_s.upcase.tr('_', ' ')} MODE")
+ end
+
private
def multiple_columns(columns, separator: ', ')
diff --git a/lib/gitlab/database/migration_helpers/swapping.rb b/lib/gitlab/database/migration_helpers/swapping.rb
new file mode 100644
index 00000000000..6d19f8002d8
--- /dev/null
+++ b/lib/gitlab/database/migration_helpers/swapping.rb
@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module MigrationHelpers
+ module Swapping
+ def reset_trigger_function(function_name)
+ execute("ALTER FUNCTION #{quote_table_name(function_name)} RESET ALL")
+ end
+
+ def swap_columns(table, column1, column2)
+ ::Gitlab::Database::Migrations::SwapColumns.new(
+ migration_context: self,
+ table: table,
+ column1: column1,
+ column2: column2
+ ).execute
+ end
+
+ def swap_columns_default(table, column1, column2)
+ ::Gitlab::Database::Migrations::SwapColumnsDefault.new(
+ migration_context: self,
+ table: table,
+ column1: column1,
+ column2: column2
+ ).execute
+ end
+
+ def swap_foreign_keys(table, foreign_key1, foreign_key2)
+ rename_constraint(table, foreign_key1, :temp_name_for_renaming)
+ rename_constraint(table, foreign_key2, foreign_key1)
+ rename_constraint(table, :temp_name_for_renaming, foreign_key2)
+ end
+
+ def swap_indexes(table, index1, index2)
+ identifier = "index_#{index1}_on_#{table}"
+ # Check Gitlab::Database::MigrationHelpers#concurrent_foreign_key_name()
+ # for info on why we use a hash
+ hashed_identifier = Digest::SHA256.hexdigest(identifier).first(10)
+
+ temp_index = "temp_#{hashed_identifier}"
+
+ rename_index(table, index1, temp_index)
+ rename_index(table, index2, index1)
+ rename_index(table, temp_index, index2)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/migration_helpers/v2.rb b/lib/gitlab/database/migration_helpers/v2.rb
index 07e22963177..7cfafa1a6a6 100644
--- a/lib/gitlab/database/migration_helpers/v2.rb
+++ b/lib/gitlab/database/migration_helpers/v2.rb
@@ -43,7 +43,7 @@ module Gitlab
end
end
- t.instance_eval(&block) unless block.nil?
+ yield t unless block.nil?
end
end
diff --git a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb
index efb1957d5e7..64cde273a59 100644
--- a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb
+++ b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb
@@ -38,6 +38,10 @@ module Gitlab
# batch_class_name - The name of the class that will be called to find the range of each next batch
# batch_size - The maximum number of rows per job
# sub_batch_size - The maximum number of rows processed per "iteration" within the job
+ # queued_migration_version - Version of the migration that queues the BBM, this is used to establish dependecies
+ #
+ # queued_migration_version is made optional temporarily to allow prior migrations to not fail,
+ # https://gitlab.com/gitlab-org/gitlab/-/issues/426417 will make it mandatory.
#
# *Returns the created BatchedMigration record*
#
@@ -63,6 +67,7 @@ module Gitlab
batch_column_name,
*job_arguments,
job_interval:,
+ queued_migration_version: nil,
batch_min_value: BATCH_MIN_VALUE,
batch_max_value: nil,
batch_class_name: BATCH_CLASS_NAME,
@@ -113,27 +118,13 @@ module Gitlab
"(given #{job_arguments.count}, expected #{migration.job_class.job_arguments_count})"
end
- # Below `BatchedMigration` attributes were introduced after the
- # initial `batched_background_migrations` table was created, so any
- # migrations that ran relying on initial table schema would not know
- # about columns introduced later on because this model is not
- # isolated in migrations, which is why we need to check for existence
- # of these columns first.
- if migration.respond_to?(:max_batch_size)
- migration.max_batch_size = max_batch_size
- end
-
- if migration.respond_to?(:total_tuple_count)
- # We keep track of the estimated number of tuples to reason later
- # about the overall progress of a migration.
- migration.total_tuple_count = Gitlab::Database::SharedModel.using_connection(connection) do
- Gitlab::Database::PgClass.for_table(batch_table_name)&.cardinality_estimate
- end
- end
-
- if migration.respond_to?(:gitlab_schema)
- migration.gitlab_schema = gitlab_schema
- end
+ assign_attribtues_safely(
+ migration,
+ max_batch_size,
+ batch_table_name,
+ gitlab_schema,
+ queued_migration_version
+ )
migration.save!
migration
@@ -244,6 +235,33 @@ module Gitlab
"\n\n" \
"\thttps://docs.gitlab.com/ee/update/background_migrations.html#database-migrations-failing-because-of-batched-background-migration-not-finished"
end
+
+ private
+
+ # Below `BatchedMigration` attributes were introduced after the
+ # initial `batched_background_migrations` table was created, so any
+ # migrations that ran relying on initial table schema would not know
+ # about columns introduced later on because this model is not
+ # isolated in migrations, which is why we need to check for existence
+ # of these columns first.
+ def assign_attribtues_safely(migration, max_batch_size, batch_table_name, gitlab_schema, queued_migration_version)
+ # We keep track of the estimated number of tuples in 'total_tuple_count' to reason later
+ # about the overall progress of a migration.
+ safe_attributes_value = {
+ max_batch_size: max_batch_size,
+ total_tuple_count: Gitlab::Database::SharedModel.using_connection(connection) do
+ Gitlab::Database::PgClass.for_table(batch_table_name)&.cardinality_estimate
+ end,
+ gitlab_schema: gitlab_schema,
+ queued_migration_version: queued_migration_version
+ }
+
+ # rubocop:disable GitlabSecurity/PublicSend
+ safe_attributes_value.each do |safe_attribute, value|
+ migration.public_send("#{safe_attribute}=", value) if migration.respond_to?(safe_attribute)
+ end
+ # rubocop:enable GitlabSecurity/PublicSend
+ end
end
end
end
diff --git a/lib/gitlab/database/migrations/milestone_mixin.rb b/lib/gitlab/database/migrations/milestone_mixin.rb
new file mode 100644
index 00000000000..10bc0c192e7
--- /dev/null
+++ b/lib/gitlab/database/migrations/milestone_mixin.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module Migrations
+ module MilestoneMixin
+ extend ActiveSupport::Concern
+ include Gitlab::ClassAttributes
+
+ MilestoneNotSetError = Class.new(StandardError)
+
+ class_methods do
+ def milestone(milestone_str = nil)
+ if milestone_str.present?
+ set_class_attribute(:migration_milestone, milestone_str)
+ else
+ get_class_attribute(:migration_milestone)
+ end
+ end
+ end
+
+ def initialize(name = class_name, version = nil, type = nil)
+ raise MilestoneNotSetError, "Milestone is not set for #{self.class.name}" if milestone.nil?
+
+ super(name, version)
+ @version = Gitlab::Database::Migrations::Version.new(version, milestone, type)
+ end
+
+ def milestone # rubocop:disable Lint/DuplicateMethods
+ @milestone ||= self.class.milestone
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/migrations/observers/query_statistics.rb b/lib/gitlab/database/migrations/observers/query_statistics.rb
index 2d026f0c8d2..b0797cd4f3f 100644
--- a/lib/gitlab/database/migrations/observers/query_statistics.rb
+++ b/lib/gitlab/database/migrations/observers/query_statistics.rb
@@ -20,7 +20,13 @@ module Gitlab
return unless enabled?
observation.query_statistics = connection.execute(<<~SQL)
- SELECT query, calls, total_time, max_time, mean_time, rows
+ SELECT
+ query,
+ calls,
+ total_exec_time + total_plan_time AS total_time,
+ max_exec_time + max_plan_time AS max_time,
+ mean_exec_time + mean_plan_time AS mean_time,
+ "rows"
FROM pg_stat_statements
WHERE pg_get_userbyid(userid) = current_user
ORDER BY total_time DESC
diff --git a/lib/gitlab/database/migrations/runner.rb b/lib/gitlab/database/migrations/runner.rb
index dc9ea304aac..f640d6fcf75 100644
--- a/lib/gitlab/database/migrations/runner.rb
+++ b/lib/gitlab/database/migrations/runner.rb
@@ -7,7 +7,7 @@ module Gitlab
BASE_RESULT_DIR = Rails.root.join('tmp', 'migration-testing').freeze
METADATA_FILENAME = 'metadata.json'
SCHEMA_VERSION = 4 # Version of the output format produced by the runner
- POST_MIGRATION_MATCHER = %r{db/post_migrate/}.freeze
+ POST_MIGRATION_MATCHER = %r{db/post_migrate/}
class << self
def up(database:, legacy_mode: false)
diff --git a/lib/gitlab/database/migrations/swap_columns.rb b/lib/gitlab/database/migrations/swap_columns.rb
new file mode 100644
index 00000000000..6d0347553b3
--- /dev/null
+++ b/lib/gitlab/database/migrations/swap_columns.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module Migrations
+ class SwapColumns
+ delegate :quote_table_name, :quote_column_name, :clear_cache!, to: :@migration_context
+
+ def initialize(migration_context:, table:, column1:, column2:)
+ @migration_context = migration_context
+ @table = table
+ @column_name1 = column1
+ @column_name2 = column2
+ end
+
+ def execute
+ rename_column(@table, @column_name1, :temp_name_for_renaming)
+ rename_column(@table, @column_name2, @column_name1)
+ rename_column(@table, :temp_name_for_renaming, @column_name2)
+ end
+
+ private
+
+ # Rails' `rename_column` will rename related indexes
+ # using a format e.g. `index_{TABLE_NAME}_on_{KEY1}_and_{KEY2}`
+ # This will break the migration if the formated index name is longer than 63 chars, e.g.
+ # `index_ci_pipeline_variables_on_pipeline_id_convert_to_bigint_and_key`
+ # Therefore, we need to duplicate what Rails has done here without the part renaming related indexes
+ def rename_column(table_name, column_name, column2_name)
+ clear_cache!
+ @migration_context.execute <<~SQL
+ ALTER TABLE #{quote_table_name(table_name)}
+ RENAME COLUMN #{quote_column_name(column_name)} TO #{quote_column_name(column2_name)}
+ SQL
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/migrations/swap_columns_default.rb b/lib/gitlab/database/migrations/swap_columns_default.rb
new file mode 100644
index 00000000000..0005c606b87
--- /dev/null
+++ b/lib/gitlab/database/migrations/swap_columns_default.rb
@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module Migrations
+ class SwapColumnsDefault
+ delegate(
+ :change_column_default, :quote_table_name, :quote_column_name, :column_for,
+ to: :@migration_context
+ )
+
+ def initialize(migration_context:, table:, column1:, column2:)
+ @migration_context = migration_context
+ @table = table
+ @column_name1 = column1
+ @column_name2 = column2
+ end
+
+ def execute
+ default1 = find_default_by(@column_name1)
+ default2 = find_default_by(@column_name2)
+ return if default1 == default2
+
+ change_sequence_owner_if(default1[:sequence_name], @column_name2)
+ change_sequence_owner_if(default2[:sequence_name], @column_name1)
+
+ change_column_default(@table, @column_name1, default2[:default])
+ change_column_default(@table, @column_name2, default1[:default])
+ end
+
+ private
+
+ def change_sequence_owner_if(sequence_name, column_name)
+ return if sequence_name.blank?
+
+ @migration_context.execute(<<~SQL.squish)
+ ALTER SEQUENCE #{quote_table_name(sequence_name)}
+ OWNED BY #{quote_table_name(@table)}.#{quote_column_name(column_name)}
+ SQL
+ end
+
+ def find_default_by(name)
+ column = column_for(@table, name)
+ if column.default_function.present?
+ {
+ default: -> { column.default_function },
+ sequence_name: extract_sequence_name_from(column.default_function)
+ }
+ else
+ {
+ default: column.default
+ }
+ end
+ end
+
+ def extract_sequence_name_from(expression)
+ expression[/nextval\('([^']+)'/, 1]
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/migrations/version.rb b/lib/gitlab/database/migrations/version.rb
new file mode 100644
index 00000000000..27c4c7d0746
--- /dev/null
+++ b/lib/gitlab/database/migrations/version.rb
@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module Migrations
+ class Version
+ InvalidTypeError = Class.new(StandardError)
+
+ include Comparable
+
+ TYPE_VALUES = {
+ regular: 0,
+ post: 1
+ }.freeze
+
+ attr_reader :timestamp, :milestone, :type_value
+
+ def initialize(timestamp, milestone, type)
+ @timestamp = timestamp
+ @milestone = milestone
+ self.type = type
+ end
+
+ def type
+ TYPE_VALUES.key(@type_value)
+ end
+
+ def type=(value)
+ @type_value = TYPE_VALUES.fetch(value.to_sym) { raise InvalidTypeError }
+ end
+
+ def regular?
+ @type_value == TYPE_VALUES[:regular]
+ end
+
+ def post_deployment?
+ @type_value == TYPE_VALUES[:post]
+ end
+
+ def <=>(other)
+ return 1 unless other.is_a?(self.class)
+
+ return milestone <=> other.milestone if milestone != other.milestone
+
+ return @type_value <=> other.type_value if @type_value != other.type_value
+
+ @timestamp <=> other.timestamp
+ end
+
+ def to_s
+ @timestamp.to_s
+ end
+
+ def to_i
+ @timestamp.to_i
+ end
+
+ def coerce(_other)
+ [-1, timestamp.to_i]
+ end
+
+ def eql?(other)
+ (self <=> other) == 0
+ end
+
+ def ==(other)
+ eql?(other)
+ end
+
+ def hash
+ [timestamp, milestone, @type_value].hash
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/partitioning/partition_manager.rb b/lib/gitlab/database/partitioning/partition_manager.rb
index cc5c49cc24a..bb70d052e3e 100644
--- a/lib/gitlab/database/partitioning/partition_manager.rb
+++ b/lib/gitlab/database/partitioning/partition_manager.rb
@@ -153,7 +153,6 @@ module Gitlab
end
def run_analyze_on_partitioned_table
- return if Feature.disabled?(:database_analyze_on_partitioned_tables)
return if ineligible_for_analyzing?
primary_transaction(statement_timeout: STATEMENT_TIMEOUT) do
diff --git a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb
index a9f2b963340..fb25cb70e57 100644
--- a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb
+++ b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb
@@ -113,7 +113,7 @@ module Gitlab
schemas = ::Gitlab::Database::GitlabSchema.table_schemas!(all_tables)
schemas += ApplicationRecord.gitlab_transactions_stack
- unless ::Gitlab::Database::GitlabSchema.cross_transactions_allowed?(schemas)
+ unless ::Gitlab::Database::GitlabSchema.cross_transactions_allowed?(schemas, all_tables)
messages = []
messages << "Cross-database data modification of '#{schemas.to_a.join(", ")}' were detected within " \
"a transaction modifying the '#{all_tables.to_a.join(", ")}' tables. "
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1.rb
deleted file mode 100644
index 2314246da55..00000000000
--- a/lib/gitlab/database/rename_reserved_paths_migration/v1.rb
+++ /dev/null
@@ -1,42 +0,0 @@
-# frozen_string_literal: true
-
-# This module can be included in migrations to make it easier to rename paths
-# of `Namespace` & `Project` models certain paths would become `reserved`.
-#
-# If the way things are stored on the filesystem related to namespaces and
-# projects ever changes. Don't update this module, or anything nested in `V1`,
-# since it needs to keep functioning for all migrations using it using the state
-# that the data is in at the time. Instead, create a `V2` module that implements
-# the new way of reserving paths.
-module Gitlab
- module Database
- module RenameReservedPathsMigration
- module V1
- def self.included(kls)
- kls.include(MigrationHelpers)
- end
-
- def rename_wildcard_paths(one_or_more_paths)
- rename_child_paths(one_or_more_paths)
- paths = Array(one_or_more_paths)
- RenameProjects.new(paths, self).rename_projects
- end
-
- def rename_child_paths(one_or_more_paths)
- paths = Array(one_or_more_paths)
- RenameNamespaces.new(paths, self).rename_namespaces(type: :child)
- end
-
- def rename_root_paths(paths)
- paths = Array(paths)
- RenameNamespaces.new(paths, self).rename_namespaces(type: :top_level)
- end
-
- def revert_renames
- RenameProjects.new([], self).revert_renames
- RenameNamespaces.new([], self).revert_renames
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb
deleted file mode 100644
index f1dc3ed74fe..00000000000
--- a/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb
+++ /dev/null
@@ -1,99 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Database
- module RenameReservedPathsMigration
- module V1
- module MigrationClasses
- module Routable
- def full_path
- if route && route.path.present?
- @full_path ||= route.path # rubocop:disable Gitlab/ModuleWithInstanceVariables
- else
- update_route if persisted?
-
- build_full_path
- end
- end
-
- def build_full_path
- if parent && path
- parent.full_path + '/' + path
- else
- path
- end
- end
-
- def update_route
- prepare_route
- route.save
- end
-
- def prepare_route
- route || build_route(source: self)
- route.path = build_full_path
- @full_path = nil # rubocop:disable Gitlab/ModuleWithInstanceVariables
- end
- end
-
- class Namespace < ActiveRecord::Base
- include MigrationClasses::Routable
- self.table_name = 'namespaces'
- self.inheritance_column = :_type_disabled
- belongs_to :parent,
- class_name: "#{MigrationClasses.name}::Namespace"
- has_one :route, as: :source
- has_many :children,
- class_name: "#{MigrationClasses.name}::Namespace",
- foreign_key: :parent_id
-
- # Overridden to have the correct `source_type` for the `route` relation
- def self.name
- 'Namespace'
- end
-
- def kind
- type == 'Group' ? 'group' : 'user'
- end
- end
-
- class User < ActiveRecord::Base
- self.table_name = 'users'
- end
-
- class Route < ActiveRecord::Base
- self.table_name = 'routes'
- belongs_to :source, polymorphic: true
- end
-
- class Project < ActiveRecord::Base
- include MigrationClasses::Routable
- has_one :route, as: :source
- self.table_name = 'projects'
-
- HASHED_STORAGE_FEATURES = {
- repository: 1,
- attachments: 2
- }.freeze
-
- def repository_storage_path
- Gitlab.config.repositories.storages[repository_storage].legacy_disk_path
- end
-
- # Overridden to have the correct `source_type` for the `route` relation
- def self.name
- 'Project'
- end
-
- def hashed_storage?(feature)
- raise ArgumentError, "Invalid feature" unless HASHED_STORAGE_FEATURES.include?(feature)
- return false unless respond_to?(:storage_version)
-
- self.storage_version && self.storage_version >= HASHED_STORAGE_FEATURES[feature]
- end
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb
deleted file mode 100644
index 2c9d0d6c0d1..00000000000
--- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb
+++ /dev/null
@@ -1,196 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Database
- module RenameReservedPathsMigration
- module V1
- class RenameBase
- attr_reader :paths, :migration
-
- delegate :update_column_in_batches,
- :execute,
- :replace_sql,
- :quote_string,
- :say,
- to: :migration
-
- def initialize(paths, migration)
- @paths = paths
- @migration = migration
- end
-
- def path_patterns
- @path_patterns ||= paths.flat_map { |path| ["%/#{path}", path] }
- end
-
- def rename_path_for_routable(routable)
- old_path = routable.path
- old_full_path = routable.full_path
- # Only remove the last occurrence of the path name to get the parent namespace path
- namespace_path = remove_last_occurrence(old_full_path, old_path)
- new_path = rename_path(namespace_path, old_path)
- new_full_path = join_routable_path(namespace_path, new_path)
-
- perform_rename(routable, old_full_path, new_full_path)
-
- [old_full_path, new_full_path]
- end
-
- def perform_rename(routable, old_full_path, new_full_path)
- # skips callbacks & validations
- new_path = new_full_path.split('/').last
- routable.class.where(id: routable)
- .update_all(path: new_path)
-
- rename_routes(old_full_path, new_full_path)
- end
-
- def rename_routes(old_full_path, new_full_path)
- routes = Route.arel_table
-
- quoted_old_full_path = quote_string(old_full_path)
- quoted_old_wildcard_path = quote_string("#{old_full_path}/%")
-
- filter =
- "routes.id IN "\
- "( SELECT routes.id FROM routes WHERE lower(routes.path) = lower('#{quoted_old_full_path}') "\
- "UNION SELECT routes.id FROM routes WHERE routes.path ILIKE '#{quoted_old_wildcard_path}' )"
-
- replace_statement = replace_sql(Route.arel_table[:path],
- old_full_path,
- new_full_path)
-
- update = Arel::UpdateManager.new
- .table(routes)
- .set([[routes[:path], replace_statement]])
- .where(Arel::Nodes::SqlLiteral.new(filter))
-
- execute(update.to_sql)
- end
-
- def rename_path(namespace_path, path_was)
- counter = 0
- path = "#{path_was}#{counter}"
-
- while route_exists?(join_routable_path(namespace_path, path))
- counter += 1
- path = "#{path_was}#{counter}"
- end
-
- path
- end
-
- def remove_last_occurrence(string, pattern)
- string.reverse.sub(pattern.reverse, "").reverse
- end
-
- def join_routable_path(namespace_path, top_level)
- if namespace_path.present?
- File.join(namespace_path, top_level)
- else
- top_level
- end
- end
-
- def route_exists?(full_path)
- MigrationClasses::Route.where(Route.arel_table[:path].matches(full_path)).any?
- end
-
- def move_pages(old_path, new_path)
- move_folders(pages_dir, old_path, new_path)
- end
-
- def move_uploads(old_path, new_path)
- return unless file_storage?
-
- move_folders(uploads_dir, old_path, new_path)
- end
-
- def move_folders(directory, old_relative_path, new_relative_path)
- old_path = File.join(directory, old_relative_path)
- unless File.directory?(old_path)
- say "#{old_path} doesn't exist, skipping"
- return
- end
-
- new_path = File.join(directory, new_relative_path)
- FileUtils.mv(old_path, new_path)
- end
-
- def remove_cached_html_for_projects(project_ids)
- project_ids.each do |project_id|
- update_column_in_batches(:projects, :description_html, nil) do |table, query|
- query.where(table[:id].eq(project_id))
- end
-
- update_column_in_batches(:issues, :description_html, nil) do |table, query|
- query.where(table[:project_id].eq(project_id))
- end
-
- update_column_in_batches(:merge_requests, :description_html, nil) do |table, query|
- query.where(table[:target_project_id].eq(project_id))
- end
-
- update_column_in_batches(:notes, :note_html, nil) do |table, query|
- query.where(table[:project_id].eq(project_id))
- end
-
- update_column_in_batches(:milestones, :description_html, nil) do |table, query|
- query.where(table[:project_id].eq(project_id))
- end
- end
- end
-
- def track_rename(type, old_path, new_path)
- key = redis_key_for_type(type)
- Gitlab::Redis::SharedState.with do |redis|
- redis.lpush(key, [old_path, new_path].to_json)
- redis.expire(key, 2.weeks.to_i)
- end
- say "tracked rename: #{key}: #{old_path} -> #{new_path}"
- end
-
- def reverts_for_type(type)
- key = redis_key_for_type(type)
-
- Gitlab::Redis::SharedState.with do |redis|
- failed_reverts = []
-
- while rename_info = redis.lpop(key)
- path_before_rename, path_after_rename = Gitlab::Json.parse(rename_info)
- say "renaming #{type} from #{path_after_rename} back to #{path_before_rename}"
- begin
- yield(path_before_rename, path_after_rename)
- rescue StandardError => e
- failed_reverts << rename_info
- say "Renaming #{type} from #{path_after_rename} back to "\
- "#{path_before_rename} failed. Review the error and try "\
- "again by running the `down` action. \n"\
- "#{e.message}: \n #{e.backtrace.join("\n")}"
- end
- end
-
- failed_reverts.each { |rename_info| redis.lpush(key, rename_info) }
- end
- end
-
- def redis_key_for_type(type)
- "rename:#{migration.name}:#{type}"
- end
-
- def file_storage?
- CarrierWave::Uploader::Base.storage == CarrierWave::Storage::File
- end
-
- def uploads_dir
- File.join(CarrierWave.root, "uploads")
- end
-
- def pages_dir
- Settings.pages.path
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb
deleted file mode 100644
index 72ae2849911..00000000000
--- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb
+++ /dev/null
@@ -1,106 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Database
- module RenameReservedPathsMigration
- module V1
- class RenameNamespaces < RenameBase
- include Gitlab::ShellAdapter
-
- def rename_namespaces(type:)
- namespaces_for_paths(type: type).each do |namespace|
- rename_namespace(namespace)
- end
- end
-
- def namespaces_for_paths(type:)
- namespaces = case type
- when :child
- MigrationClasses::Namespace.where.not(parent_id: nil)
- when :top_level
- MigrationClasses::Namespace.where(parent_id: nil)
- end
- with_paths = MigrationClasses::Route.arel_table[:path]
- .matches_any(path_patterns)
- namespaces.joins(:route).where(with_paths)
- .allow_cross_joins_across_databases(url: "https://gitlab.com/gitlab-org/gitlab/-/issues/420046")
- end
-
- def rename_namespace(namespace)
- old_full_path, new_full_path = rename_path_for_routable(namespace)
-
- track_rename('namespace', old_full_path, new_full_path)
-
- rename_namespace_dependencies(namespace, old_full_path, new_full_path)
- end
-
- def rename_namespace_dependencies(namespace, old_full_path, new_full_path)
- move_repositories(namespace, old_full_path, new_full_path)
- move_uploads(old_full_path, new_full_path)
- move_pages(old_full_path, new_full_path)
- rename_user(old_full_path, new_full_path) if namespace.kind == 'user'
- remove_cached_html_for_projects(projects_for_namespace(namespace).map(&:id))
- end
-
- def revert_renames
- reverts_for_type('namespace') do |path_before_rename, current_path|
- matches_path = MigrationClasses::Route.arel_table[:path].matches(current_path)
- namespace = MigrationClasses::Namespace.joins(:route)
- .allow_cross_joins_across_databases(url: "https://gitlab.com/gitlab-org/gitlab/-/issues/420046")
- .find_by(matches_path)&.becomes(MigrationClasses::Namespace) # rubocop: disable Cop/AvoidBecomes
-
- if namespace
- perform_rename(namespace, current_path, path_before_rename)
-
- rename_namespace_dependencies(namespace, current_path, path_before_rename)
- else
- say "Couldn't rename namespace from #{current_path} back to #{path_before_rename}, "\
- "namespace was renamed, or no longer exists at the expected path"
- end
- end
- end
-
- def rename_user(old_username, new_username)
- MigrationClasses::User.where(username: old_username)
- .update_all(username: new_username)
- end
-
- def move_repositories(namespace, old_full_path, new_full_path)
- repo_shards_for_namespace(namespace).each do |repository_storage|
- # Ensure old directory exists before moving it
- Gitlab::GitalyClient::NamespaceService.allow do
- gitlab_shell.add_namespace(repository_storage, old_full_path)
-
- unless gitlab_shell.mv_namespace(repository_storage, old_full_path, new_full_path)
- message = "Exception moving on shard #{repository_storage} from #{old_full_path} to #{new_full_path}"
- Gitlab::AppLogger.error message
- end
- end
- end
- end
-
- def repo_shards_for_namespace(namespace)
- projects_for_namespace(namespace).distinct.select(:repository_storage)
- .map(&:repository_storage)
- end
-
- def projects_for_namespace(namespace)
- namespace_ids = child_ids_for_parent(namespace, ids: [namespace.id])
- namespace_or_children = MigrationClasses::Project
- .arel_table[:namespace_id]
- .in(namespace_ids)
- MigrationClasses::Project.where(namespace_or_children)
- end
-
- def child_ids_for_parent(namespace, ids: [])
- namespace.children.each do |child|
- ids << child.id
- child_ids_for_parent(child, ids: ids) if child.children.any?
- end
- ids
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb
deleted file mode 100644
index 155e35b64f4..00000000000
--- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb
+++ /dev/null
@@ -1,78 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Database
- module RenameReservedPathsMigration
- module V1
- class RenameProjects < RenameBase
- include Gitlab::ShellAdapter
-
- def rename_projects
- projects_for_paths.each do |project|
- rename_project(project)
- end
-
- remove_cached_html_for_projects(projects_for_paths.map(&:id))
- end
-
- def rename_project(project)
- old_full_path, new_full_path = rename_path_for_routable(project)
-
- track_rename('project', old_full_path, new_full_path)
-
- move_project_folders(project, old_full_path, new_full_path)
- end
-
- def move_project_folders(project, old_full_path, new_full_path)
- unless project.hashed_storage?(:repository)
- move_repository(project, old_full_path, new_full_path)
- move_repository(project, "#{old_full_path}.wiki", "#{new_full_path}.wiki")
- end
-
- move_uploads(old_full_path, new_full_path) unless project.hashed_storage?(:attachments)
- move_pages(old_full_path, new_full_path)
- end
-
- def revert_renames
- reverts_for_type('project') do |path_before_rename, current_path|
- matches_path = MigrationClasses::Route.arel_table[:path].matches(current_path)
- project = MigrationClasses::Project.joins(:route)
- .allow_cross_joins_across_databases(url:
- 'https://gitlab.com/gitlab-org/gitlab/-/issues/421843')
- .find_by(matches_path)
-
- if project
- perform_rename(project, current_path, path_before_rename)
-
- move_project_folders(project, current_path, path_before_rename)
- else
- say "Couldn't rename project from #{current_path} back to "\
- "#{path_before_rename}, project was renamed or no longer "\
- "exists at the expected path."
-
- end
- end
- end
-
- def move_repository(project, old_path, new_path)
- unless gitlab_shell.mv_repository(project.repository_storage,
- old_path,
- new_path)
- Gitlab::AppLogger.error "Error moving #{old_path} to #{new_path}"
- end
- end
-
- def projects_for_paths
- return @projects_for_paths if @projects_for_paths
-
- with_paths = MigrationClasses::Route.arel_table[:path]
- .matches_any(path_patterns)
-
- @projects_for_paths = MigrationClasses::Project.joins(:route).where(with_paths)
- .allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/421843')
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/database_importers/work_items/hierarchy_restrictions_importer.rb b/lib/gitlab/database_importers/work_items/hierarchy_restrictions_importer.rb
index 4e7a4ec748b..4e3b685c06c 100644
--- a/lib/gitlab/database_importers/work_items/hierarchy_restrictions_importer.rb
+++ b/lib/gitlab/database_importers/work_items/hierarchy_restrictions_importer.rb
@@ -14,17 +14,52 @@ module Gitlab
ticket = find_or_create_type(::WorkItems::Type::TYPE_NAMES[:ticket])
restrictions = [
- { parent_type_id: objective.id, child_type_id: objective.id, maximum_depth: 9 },
- { parent_type_id: objective.id, child_type_id: key_result.id, maximum_depth: 1 },
- { parent_type_id: issue.id, child_type_id: task.id, maximum_depth: 1 },
- { parent_type_id: incident.id, child_type_id: task.id, maximum_depth: 1 },
- { parent_type_id: epic.id, child_type_id: epic.id, maximum_depth: 9 },
- { parent_type_id: epic.id, child_type_id: issue.id, maximum_depth: 1 },
- { parent_type_id: ticket.id, child_type_id: task.id, maximum_depth: 1 }
+ {
+ parent_type_id: objective.id,
+ child_type_id: objective.id,
+ maximum_depth: 9,
+ cross_hierarchy_enabled: false
+ },
+ {
+ parent_type_id: objective.id,
+ child_type_id: key_result.id,
+ maximum_depth: 1,
+ cross_hierarchy_enabled: false
+ },
+ {
+ parent_type_id: issue.id,
+ child_type_id: task.id,
+ maximum_depth: 1,
+ cross_hierarchy_enabled: false
+ },
+ {
+ parent_type_id: incident.id,
+ child_type_id: task.id,
+ maximum_depth: 1,
+ cross_hierarchy_enabled: false
+ },
+ {
+ parent_type_id: epic.id,
+ child_type_id: epic.id,
+ maximum_depth: 9,
+ cross_hierarchy_enabled: true
+ },
+ {
+ parent_type_id: epic.id,
+ child_type_id: issue.id,
+ maximum_depth: 1,
+ cross_hierarchy_enabled: true
+ },
+ {
+ parent_type_id: ticket.id,
+ child_type_id: task.id,
+ maximum_depth: 1,
+ cross_hierarchy_enabled: false
+ }
]
::WorkItems::HierarchyRestriction.upsert_all(
- restrictions,
+ filtered_restrictions(restrictions),
unique_by: :index_work_item_hierarchy_restrictions_on_parent_and_child
)
end
@@ -36,6 +71,16 @@ module Gitlab
Gitlab::DatabaseImporters::WorkItems::BaseTypeImporter.upsert_types
::WorkItems::Type.find_by_name_and_namespace_id(name, nil)
end
+
+ def self.filtered_restrictions(restrictions)
+ missing_columns = restrictions.first.keys.select do |attribute|
+ ::WorkItems::HierarchyRestriction.column_names.exclude?(attribute.to_s)
+ end
+
+ return restrictions if missing_columns.empty?
+
+ restrictions.map { |restriction| restriction.except(*missing_columns) }
+ end
end
end
end
diff --git a/lib/gitlab/database_importers/work_items/related_links_restrictions_importer.rb b/lib/gitlab/database_importers/work_items/related_links_restrictions_importer.rb
new file mode 100644
index 00000000000..692764bd16d
--- /dev/null
+++ b/lib/gitlab/database_importers/work_items/related_links_restrictions_importer.rb
@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module DatabaseImporters
+ module WorkItems
+ module RelatedLinksRestrictionsImporter
+ # This importer populates the default link restrictions for the base work item types that support this feature.
+ # These rules are documented in https://docs.gitlab.com/ee/development/work_items.html#write-a-database-migration
+
+ # rubocop:disable Metrics/AbcSize
+ def self.upsert_restrictions
+ epic = find_or_create_type(::WorkItems::Type::TYPE_NAMES[:epic])
+ issue = find_or_create_type(::WorkItems::Type::TYPE_NAMES[:issue])
+ task = find_or_create_type(::WorkItems::Type::TYPE_NAMES[:task])
+ objective = find_or_create_type(::WorkItems::Type::TYPE_NAMES[:objective])
+ key_result = find_or_create_type(::WorkItems::Type::TYPE_NAMES[:key_result])
+
+ restrictions = [
+ # Source can relate to target and target can relate to source
+ { source_type_id: epic.id, target_type_id: epic.id, link_type: 0 },
+ { source_type_id: epic.id, target_type_id: issue.id, link_type: 0 },
+ { source_type_id: epic.id, target_type_id: task.id, link_type: 0 },
+ { source_type_id: epic.id, target_type_id: objective.id, link_type: 0 },
+ { source_type_id: epic.id, target_type_id: key_result.id, link_type: 0 },
+ { source_type_id: issue.id, target_type_id: issue.id, link_type: 0 },
+ { source_type_id: issue.id, target_type_id: task.id, link_type: 0 },
+ { source_type_id: issue.id, target_type_id: objective.id, link_type: 0 },
+ { source_type_id: issue.id, target_type_id: key_result.id, link_type: 0 },
+ { source_type_id: task.id, target_type_id: task.id, link_type: 0 },
+ { source_type_id: task.id, target_type_id: objective.id, link_type: 0 },
+ { source_type_id: task.id, target_type_id: key_result.id, link_type: 0 },
+ { source_type_id: objective.id, target_type_id: objective.id, link_type: 0 },
+ { source_type_id: objective.id, target_type_id: key_result.id, link_type: 0 },
+ { source_type_id: key_result.id, target_type_id: key_result.id, link_type: 0 },
+ # Source can block target and target can be blocked by source
+ { source_type_id: epic.id, target_type_id: epic.id, link_type: 1 },
+ { source_type_id: epic.id, target_type_id: issue.id, link_type: 1 },
+ { source_type_id: epic.id, target_type_id: task.id, link_type: 1 },
+ { source_type_id: epic.id, target_type_id: objective.id, link_type: 1 },
+ { source_type_id: epic.id, target_type_id: key_result.id, link_type: 1 },
+ { source_type_id: issue.id, target_type_id: issue.id, link_type: 1 },
+ { source_type_id: issue.id, target_type_id: epic.id, link_type: 1 },
+ { source_type_id: issue.id, target_type_id: task.id, link_type: 1 },
+ { source_type_id: issue.id, target_type_id: objective.id, link_type: 1 },
+ { source_type_id: issue.id, target_type_id: key_result.id, link_type: 1 },
+ { source_type_id: task.id, target_type_id: task.id, link_type: 1 },
+ { source_type_id: task.id, target_type_id: epic.id, link_type: 1 },
+ { source_type_id: task.id, target_type_id: issue.id, link_type: 1 },
+ { source_type_id: task.id, target_type_id: objective.id, link_type: 1 },
+ { source_type_id: task.id, target_type_id: key_result.id, link_type: 1 },
+ { source_type_id: objective.id, target_type_id: objective.id, link_type: 1 },
+ { source_type_id: objective.id, target_type_id: key_result.id, link_type: 1 },
+ { source_type_id: key_result.id, target_type_id: key_result.id, link_type: 1 },
+ { source_type_id: key_result.id, target_type_id: objective.id, link_type: 1 }
+ ]
+
+ ::WorkItems::RelatedLinkRestriction.upsert_all(
+ restrictions,
+ unique_by: :index_work_item_link_restrictions_on_source_link_type_target
+ )
+ end
+ # rubocop:enable Metrics/AbcSize
+
+ def self.find_or_create_type(name)
+ type = ::WorkItems::Type.find_by_name_and_namespace_id(name, nil)
+ return type if type
+
+ Gitlab::DatabaseImporters::WorkItems::BaseTypeImporter.upsert_types
+ ::WorkItems::Type.find_by_name_and_namespace_id(name, nil)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/dependency_linker/base_linker.rb b/lib/gitlab/dependency_linker/base_linker.rb
index ff17010c11c..74bec55253f 100644
--- a/lib/gitlab/dependency_linker/base_linker.rb
+++ b/lib/gitlab/dependency_linker/base_linker.rb
@@ -3,9 +3,9 @@
module Gitlab
module DependencyLinker
class BaseLinker
- URL_REGEX = %r{https?://[^'" ]+}.freeze
- GIT_INVALID_URL_REGEX = /^git\+#{URL_REGEX}/.freeze
- REPO_REGEX = %r{[^/'" ]+/[^/'" ]+}.freeze
+ URL_REGEX = %r{https?://[^'" ]+}
+ GIT_INVALID_URL_REGEX = /^git\+#{URL_REGEX}/
+ REPO_REGEX = %r{[^/'" ]+/[^/'" ]+}
class_attribute :file_type
diff --git a/lib/gitlab/dependency_linker/gemfile_linker.rb b/lib/gitlab/dependency_linker/gemfile_linker.rb
index c6e02248b0a..baba6511d62 100644
--- a/lib/gitlab/dependency_linker/gemfile_linker.rb
+++ b/lib/gitlab/dependency_linker/gemfile_linker.rb
@@ -8,8 +8,8 @@ module Gitlab
self.package_keyword = :gem
self.file_type = :gemfile
- GITHUB_REGEX = /(github:|:github\s*=>)\s*['"](?<name>[^'"]+)['"]/.freeze
- GIT_REGEX = /(git:|:git\s*=>)\s*['"](?<name>#{URL_REGEX})['"]/.freeze
+ GITHUB_REGEX = /(github:|:github\s*=>)\s*['"](?<name>[^'"]+)['"]/
+ GIT_REGEX = /(git:|:git\s*=>)\s*['"](?<name>#{URL_REGEX})['"]/
private
diff --git a/lib/gitlab/dependency_linker/godeps_json_linker.rb b/lib/gitlab/dependency_linker/godeps_json_linker.rb
index 049a807b760..1d12198d637 100644
--- a/lib/gitlab/dependency_linker/godeps_json_linker.rb
+++ b/lib/gitlab/dependency_linker/godeps_json_linker.rb
@@ -3,7 +3,7 @@
module Gitlab
module DependencyLinker
class GodepsJsonLinker < JsonLinker
- NESTED_REPO_REGEX = %r{([^/]+/)+[^/]+?}.freeze
+ NESTED_REPO_REGEX = %r{([^/]+/)+[^/]+?}
self.file_type = :godeps_json
diff --git a/lib/gitlab/dependency_linker/podspec_linker.rb b/lib/gitlab/dependency_linker/podspec_linker.rb
index f6da17efff4..55149877d15 100644
--- a/lib/gitlab/dependency_linker/podspec_linker.rb
+++ b/lib/gitlab/dependency_linker/podspec_linker.rb
@@ -5,7 +5,7 @@ module Gitlab
class PodspecLinker < MethodLinker
include Cocoapods
- STRING_REGEX = /['"](?<name>[^'"]+)['"]/.freeze
+ STRING_REGEX = /['"](?<name>[^'"]+)['"]/
self.file_type = :podspec
diff --git a/lib/gitlab/deploy_key_access.rb b/lib/gitlab/deploy_key_access.rb
index a582c978be7..caf2a73c338 100644
--- a/lib/gitlab/deploy_key_access.rb
+++ b/lib/gitlab/deploy_key_access.rb
@@ -16,17 +16,6 @@ module Gitlab
attr_reader :deploy_key
- def protected_tag_accessible_to?(ref, action:)
- if Feature.enabled?(:deploy_key_for_protected_tags, project)
- super
- else
- assert_project!
- # a deploy key can always push a protected tag
- # (which is not always the case when pushing to a protected branch)
- true
- end
- end
-
def can_collaborate?(_ref)
assert_project!
diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb
index d5c0b187f92..de7be6efd72 100644
--- a/lib/gitlab/diff/file.rb
+++ b/lib/gitlab/diff/file.rb
@@ -7,7 +7,7 @@ module Gitlab
attr_reader :diff, :repository, :diff_refs, :fallback_diff_refs, :unique_identifier
- delegate :new_file?, :deleted_file?, :renamed_file?,
+ delegate :new_file?, :deleted_file?, :renamed_file?, :unidiff,
:old_path, :new_path, :a_mode, :b_mode, :mode_changed?,
:submodule?, :expanded?, :too_large?, :collapsed?, :line_count, :has_binary_notice?, to: :diff, prefix: false
diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb
index 95ea3fe9f0f..d2524ae1761 100644
--- a/lib/gitlab/diff/highlight.rb
+++ b/lib/gitlab/diff/highlight.rb
@@ -3,7 +3,7 @@
module Gitlab
module Diff
class Highlight
- PREFIX_REGEXP = /\A(.)/.freeze
+ PREFIX_REGEXP = /\A(.)/
attr_reader :diff_file, :diff_lines, :repository, :project
diff --git a/lib/gitlab/diff/pair_selector.rb b/lib/gitlab/diff/pair_selector.rb
index 2e5ee3a7363..e848f5107ae 100644
--- a/lib/gitlab/diff/pair_selector.rb
+++ b/lib/gitlab/diff/pair_selector.rb
@@ -20,7 +20,7 @@ module Gitlab
# Runs end at the end of the string (the last line) or before a space (for an unchanged line)
(?=\s|\z)
- }x.freeze
+ }x
# rubocop: enable Lint/MixedRegexpCaptureTypes
def initialize(lines)
diff --git a/lib/gitlab/diff/position_tracer.rb b/lib/gitlab/diff/position_tracer.rb
index a8c0108fa34..e847d05ae71 100644
--- a/lib/gitlab/diff/position_tracer.rb
+++ b/lib/gitlab/diff/position_tracer.rb
@@ -59,8 +59,11 @@ module Gitlab
end
def compare(start_sha, head_sha, straight: false)
+ include_stats = !Feature.enabled?(:remove_request_stats_for_tracing, project)
+
compare = CompareService.new(project, head_sha).execute(project, start_sha, straight: straight)
- compare.diffs(paths: paths, expanded: true, ignore_whitespace_change: @ignore_whitespace_change)
+ compare.diffs(paths: paths, expanded: true, ignore_whitespace_change: @ignore_whitespace_change, include_stats:
+ include_stats)
end
end
end
diff --git a/lib/gitlab/diff/suggestions_parser.rb b/lib/gitlab/diff/suggestions_parser.rb
index 6f126147113..85f7a5c7527 100644
--- a/lib/gitlab/diff/suggestions_parser.rb
+++ b/lib/gitlab/diff/suggestions_parser.rb
@@ -4,7 +4,7 @@ module Gitlab
module Diff
class SuggestionsParser
# Matches for instance "-1", "+1" or "-1+2".
- SUGGESTION_CONTEXT = /^(\-(?<above>\d+))?(\+(?<below>\d+))?$/.freeze
+ SUGGESTION_CONTEXT = /^(\-(?<above>\d+))?(\+(?<below>\d+))?$/
CSS = 'pre.language-suggestion'
XPATH = Gitlab::Utils::Nokogiri.css_to_xpath(CSS).freeze
diff --git a/lib/gitlab/doctor/reset_tokens.rb b/lib/gitlab/doctor/reset_tokens.rb
new file mode 100644
index 00000000000..45333e2effb
--- /dev/null
+++ b/lib/gitlab/doctor/reset_tokens.rb
@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Doctor
+ class ResetTokens
+ attr_reader :logger
+
+ PRINT_PROGRESS_EVERY = 1000
+
+ def initialize(logger, model_names:, token_names:, dry_run: true)
+ @logger = logger
+ @model_names = model_names
+ @token_names = token_names
+ @dry_run = dry_run
+ end
+
+ def run!
+ logger.info "Resetting #{@token_names.join(', ')} on #{@model_names.join(', ')} if they can not be read"
+ logger.info "Executing in DRY RUN mode, no records will actually be updated" if @dry_run
+ Rails.application.eager_load!
+
+ models_with_encrypted_tokens.each do |model|
+ fix_model(model)
+ end
+ logger.info "Done!"
+ end
+
+ private
+
+ def fix_model(model)
+ matched_token_names = @token_names & model.encrypted_token_authenticatable_fields.map(&:to_s)
+
+ return if matched_token_names.empty?
+
+ total_count = model.count
+
+ model.find_each.with_index do |instance, index|
+ matched_token_names.each do |attribute_name|
+ fix_attribute(instance, attribute_name)
+ end
+
+ logger.info "Checked #{index + 1}/#{total_count} #{model.name.pluralize}" if index % PRINT_PROGRESS_EVERY == 0
+ end
+ logger.info "Checked #{total_count} #{model.name.pluralize}"
+ end
+
+ def fix_attribute(instance, attribute_name)
+ instance.public_send(attribute_name) # rubocop:disable GitlabSecurity/PublicSend
+ rescue OpenSSL::Cipher::CipherError, TypeError
+ logger.debug "> Fix #{instance.class.name}[#{instance.id}].#{attribute_name}"
+ instance.public_send("reset_#{attribute_name}!") unless @dry_run # rubocop:disable GitlabSecurity/PublicSend
+ rescue StandardError => e
+ logger.debug(
+ "> Something went wrong for #{instance.class.name}[#{instance.id}].#{attribute_name}: #{e}".color(:red))
+
+ false
+ end
+
+ def models_with_encrypted_tokens
+ ApplicationRecord.descendants.select do |model|
+ @model_names.include?(model.name) && model.include?(TokenAuthenticatable)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/email/handler/base_handler.rb b/lib/gitlab/email/handler/base_handler.rb
index 2e487c42cb5..19826138075 100644
--- a/lib/gitlab/email/handler/base_handler.rb
+++ b/lib/gitlab/email/handler/base_handler.rb
@@ -6,7 +6,7 @@ module Gitlab
class BaseHandler
attr_reader :mail, :mail_key
- HANDLER_ACTION_BASE_REGEX ||= /(?<project_slug>.+)-(?<project_id>\d+)/.freeze
+ HANDLER_ACTION_BASE_REGEX ||= /(?<project_slug>.+)-(?<project_id>\d+)/
def initialize(mail, mail_key)
@mail = mail
diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb
index 869bcc6e2be..cac6c29f10b 100644
--- a/lib/gitlab/email/handler/create_issue_handler.rb
+++ b/lib/gitlab/email/handler/create_issue_handler.rb
@@ -11,8 +11,8 @@ module Gitlab
class CreateIssueHandler < BaseHandler
include ReplyProcessing
- HANDLER_REGEX = /\A#{HANDLER_ACTION_BASE_REGEX}-(?<incoming_email_token>.+)-issue\z/.freeze
- HANDLER_REGEX_LEGACY = /\A(?<project_path>[^\+]*)\+(?<incoming_email_token>.*)\z/.freeze
+ HANDLER_REGEX = /\A#{HANDLER_ACTION_BASE_REGEX}-(?<incoming_email_token>.+)-issue\z/
+ HANDLER_REGEX_LEGACY = /\A(?<project_path>[^\+]*)\+(?<incoming_email_token>.*)\z/
def initialize(mail, mail_key)
super(mail, mail_key)
diff --git a/lib/gitlab/email/handler/create_merge_request_handler.rb b/lib/gitlab/email/handler/create_merge_request_handler.rb
index c723c2762c7..6e25202241c 100644
--- a/lib/gitlab/email/handler/create_merge_request_handler.rb
+++ b/lib/gitlab/email/handler/create_merge_request_handler.rb
@@ -12,8 +12,8 @@ module Gitlab
class CreateMergeRequestHandler < BaseHandler
include ReplyProcessing
- HANDLER_REGEX = /\A#{HANDLER_ACTION_BASE_REGEX}-(?<incoming_email_token>.+)-merge-request\z/.freeze
- HANDLER_REGEX_LEGACY = /\A(?<project_path>[^\+]*)\+merge-request\+(?<incoming_email_token>.*)/.freeze
+ HANDLER_REGEX = /\A#{HANDLER_ACTION_BASE_REGEX}-(?<incoming_email_token>.+)-merge-request\z/
+ HANDLER_REGEX_LEGACY = /\A(?<project_path>[^\+]*)\+merge-request\+(?<incoming_email_token>.*)/
def initialize(mail, mail_key)
super(mail, mail_key)
diff --git a/lib/gitlab/email/handler/create_note_on_issuable_handler.rb b/lib/gitlab/email/handler/create_note_on_issuable_handler.rb
index aed3647744a..8fea0593c78 100644
--- a/lib/gitlab/email/handler/create_note_on_issuable_handler.rb
+++ b/lib/gitlab/email/handler/create_note_on_issuable_handler.rb
@@ -15,7 +15,7 @@ module Gitlab
attr_reader :issuable_iid
- HANDLER_REGEX = /\A#{HANDLER_ACTION_BASE_REGEX}-(?<incoming_email_token>.+)-issue-(?<issuable_iid>\d+)\z/.freeze
+ HANDLER_REGEX = /\A#{HANDLER_ACTION_BASE_REGEX}-(?<incoming_email_token>.+)-issue-(?<issuable_iid>\d+)\z/
def initialize(mail, mail_key)
super(mail, mail_key)
diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb
index 949fa554aeb..ebc4e9c2c8c 100644
--- a/lib/gitlab/email/handler/service_desk_handler.rb
+++ b/lib/gitlab/email/handler/service_desk_handler.rb
@@ -10,9 +10,9 @@ module Gitlab
include ReplyProcessing
include Gitlab::Utils::StrongMemoize
- HANDLER_REGEX = /\A#{HANDLER_ACTION_BASE_REGEX}-issue-\z/.freeze
- HANDLER_REGEX_LEGACY = /\A(?<project_path>[^\+]*)\z/.freeze
- PROJECT_KEY_PATTERN = /\A(?<slug>.+)-(?<key>[a-z0-9_]+)\z/.freeze
+ HANDLER_REGEX = /\A#{HANDLER_ACTION_BASE_REGEX}-issue-\z/
+ HANDLER_REGEX_LEGACY = /\A(?<project_path>[^\+]*)\z/
+ PROJECT_KEY_PATTERN = /\A(?<slug>.+)-(?<key>[a-z0-9_]+)\z/
def initialize(mail, mail_key, service_desk_key: nil)
if service_desk_key
@@ -75,9 +75,10 @@ module Gitlab
def contains_custom_email_address_verification_subaddress?
return false unless Feature.enabled?(:service_desk_custom_email, project)
+ return false unless to_address.present?
# Verification email only has one recipient
- mail.to.first.include?(ServiceDeskSetting::CUSTOM_EMAIL_VERIFICATION_SUBADDRESS)
+ to_address.include?(ServiceDeskSetting::CUSTOM_EMAIL_VERIFICATION_SUBADDRESS)
end
def handled_custom_email_address_verification?
@@ -209,6 +210,11 @@ module Gitlab
(mail.reply_to || []).first || mail.from.first || mail.sender
end
+ def to_address
+ mail.to&.first
+ end
+ strong_memoize_attr :to_address
+
def can_handle_legacy_format?
project_path && project_path.include?('/') && !mail_key.include?('+')
end
diff --git a/lib/gitlab/email/message/build_ios_app_guide.rb b/lib/gitlab/email/message/build_ios_app_guide.rb
deleted file mode 100644
index 4acf558a6a2..00000000000
--- a/lib/gitlab/email/message/build_ios_app_guide.rb
+++ /dev/null
@@ -1,57 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Email
- module Message
- class BuildIosAppGuide
- include Gitlab::Email::Message::InProductMarketing::Helper
- include Gitlab::Routing
-
- attr_accessor :format
-
- def initialize(format: :html)
- @format = format
- end
-
- def subject_line
- s_('InProductMarketing|Get set up to build for iOS')
- end
-
- def title
- s_("InProductMarketing|Building for iOS? We've got you covered.")
- end
-
- def body_line1
- s_(
- 'InProductMarketing|Want to get your iOS app up and running, including publishing all the way to ' \
- 'TestFlight? Follow our guide to set up GitLab and fastlane to publish iOS apps to the App Store.'
- )
- end
-
- def cta_text
- s_('InProductMarketing|Learn how to build for iOS')
- end
-
- def cta_link
- action_link(cta_text, 'https://about.gitlab.com/blog/2019/03/06/ios-publishing-with-gitlab-and-fastlane/')
- end
-
- def cta2_text
- s_('InProductMarketing|Watch iOS building in action.')
- end
-
- def cta2_link
- action_link(cta2_text, 'https://www.youtube.com/watch?v=325FyJt7ZG8')
- end
-
- def logo_path
- 'mailers/in_product_marketing/create-0.png'
- end
-
- def unsubscribe
- unsubscribe_message
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/email/message/in_product_marketing/helper.rb b/lib/gitlab/email/message/in_product_marketing/helper.rb
deleted file mode 100644
index 0770e5f4d76..00000000000
--- a/lib/gitlab/email/message/in_product_marketing/helper.rb
+++ /dev/null
@@ -1,97 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Email
- module Message
- module InProductMarketing
- module Helper
- include ActionView::Context
- include ActionView::Helpers::TagHelper
-
- def footer_links
- links = [
- [s_('InProductMarketing|Blog'), 'https://about.gitlab.com/blog'],
- [s_('InProductMarketing|Twitter'), 'https://twitter.com/gitlab'],
- [s_('InProductMarketing|Facebook'), 'https://www.facebook.com/gitlab'],
- [s_('InProductMarketing|YouTube'), 'https://www.youtube.com/channel/UCnMGQ8QHMAnVIsI3xJrihhg']
- ]
- case format
- when :html
- links.map do |text, link|
- ActionController::Base.helpers.link_to(text, link)
- end
- else
- '| ' + links.map do |text, link|
- [text, link].join(' ')
- end.join("\n| ")
- end
- end
-
- def address
- s_('InProductMarketing|%{strong_start}GitLab Inc.%{strong_end} 268 Bush Street, #350, San Francisco, CA 94104, USA').html_safe % strong_options
- end
-
- def unsubscribe_message(self_managed_preferences_link = nil)
- parts = Gitlab.com? ? unsubscribe_com : unsubscribe_self_managed(self_managed_preferences_link)
-
- case format
- when :html
- parts.join(' ')
- else
- parts.join("\n" + ' ' * 16)
- end
- end
-
- private
-
- def unsubscribe_link
- unsubscribe_url = Gitlab.com? ? '%tag_unsubscribe_url%' : profile_notifications_url
-
- link(s_('InProductMarketing|unsubscribe'), unsubscribe_url)
- end
-
- def unsubscribe_com
- [
- s_('InProductMarketing|If you no longer wish to receive marketing emails from us,'),
- s_('InProductMarketing|you may %{unsubscribe_link} at any time.') % { unsubscribe_link: unsubscribe_link }
- ]
- end
-
- def unsubscribe_self_managed(preferences_link)
- [
- s_('InProductMarketing|To opt out of these onboarding emails, %{unsubscribe_link}.') % { unsubscribe_link: unsubscribe_link },
- s_("InProductMarketing|If you don't want to receive marketing emails directly from GitLab, %{marketing_preference_link}.") % { marketing_preference_link: preferences_link }
- ]
- end
-
- def strong_options
- case format
- when :html
- { strong_start: '<b>'.html_safe, strong_end: '</b>'.html_safe }
- else
- { strong_start: '', strong_end: '' }
- end
- end
-
- def link(text, link)
- case format
- when :html
- ActionController::Base.helpers.link_to text, link
- else
- "#{text} (#{link})"
- end
- end
-
- def action_link(text, link)
- case format
- when :html
- ActionController::Base.helpers.link_to text, link, target: '_blank', rel: 'noopener noreferrer'
- else
- [text, link].join(' >> ')
- end
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb
index ee11105537b..d5877234c3a 100644
--- a/lib/gitlab/email/receiver.rb
+++ b/lib/gitlab/email/receiver.rb
@@ -8,7 +8,7 @@ module Gitlab
class Receiver
include Gitlab::Utils::StrongMemoize
- RECEIVED_HEADER_REGEX = /for\s+\<([^<]+)\>/.freeze
+ RECEIVED_HEADER_REGEX = /for\s+\<([^<]+)\>/
# Errors that are purely from users and not anything we can control
USER_ERRORS = [
@@ -50,6 +50,7 @@ module Gitlab
delivered_to: delivered_to.map(&:value),
envelope_to: envelope_to.map(&:value),
x_envelope_to: x_envelope_to.map(&:value),
+ cc_address: cc,
# reduced down to what looks like an email in the received headers
received_recipients: recipients_from_received_headers,
meta: {
@@ -84,23 +85,27 @@ module Gitlab
def mail_key
strong_memoize(:mail_key) do
- key_from_to_header || key_from_additional_headers
+ find_first_key_from(to) || key_from_additional_headers
end
end
- def key_from_to_header
- to.find do |address|
- key = email_class.key_from_address(address)
- break key if key
+ def find_first_key_from(items)
+ items.each do |item|
+ email = item.is_a?(Mail::Field) ? item.value : item
+
+ key = email_class.key_from_address(email)
+ return key if key
end
+ nil
end
def key_from_additional_headers
find_key_from_references ||
- find_key_from_delivered_to_header ||
- find_key_from_envelope_to_header ||
- find_key_from_x_envelope_to_header ||
- find_first_key_from_received_headers
+ find_first_key_from(delivered_to) ||
+ find_first_key_from(envelope_to) ||
+ find_first_key_from(x_envelope_to) ||
+ find_first_key_from(recipients_from_received_headers) ||
+ find_first_key_from(cc)
end
def ensure_references_array(references)
@@ -131,6 +136,10 @@ module Gitlab
Array(mail.to)
end
+ def cc
+ Array(mail.cc)
+ end
+
def delivered_to
Array(mail[:delivered_to])
end
@@ -147,34 +156,6 @@ module Gitlab
Array(mail[:received])
end
- def find_key_from_delivered_to_header
- delivered_to.find do |header|
- key = email_class.key_from_address(header.value)
- break key if key
- end
- end
-
- def find_key_from_envelope_to_header
- envelope_to.find do |header|
- key = email_class.key_from_address(header.value)
- break key if key
- end
- end
-
- def find_key_from_x_envelope_to_header
- x_envelope_to.find do |header|
- key = email_class.key_from_address(header.value)
- break key if key
- end
- end
-
- def find_first_key_from_received_headers
- recipients_from_received_headers.find do |email|
- key = email_class.key_from_address(email)
- break key if key
- end
- end
-
def recipients_from_received_headers
strong_memoize :emails_from_received_headers do
received.filter_map { |header| header.value[RECEIVED_HEADER_REGEX, 1] }
diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb
index 99240f2ad48..b080cb197d4 100644
--- a/lib/gitlab/encoding_helper.rb
+++ b/lib/gitlab/encoding_helper.rb
@@ -152,8 +152,6 @@ module Gitlab
message.delete_prefix(BOM_UTF8)
end
- private
-
def force_encode_utf8(message)
raise ArgumentError unless message.respond_to?(:force_encoding)
return message if message.encoding == Encoding::UTF_8 && message.valid_encoding?
@@ -163,6 +161,8 @@ module Gitlab
message.force_encoding("UTF-8")
end
+ private
+
# Escapes \x80 - \xFF characters not supported by UTF-8
def escape_chars(char)
bytes = char.bytes
diff --git a/lib/gitlab/error_tracking/error_repository/open_api_strategy.rb b/lib/gitlab/error_tracking/error_repository/open_api_strategy.rb
index 398ddebd355..3b0b4c6e935 100644
--- a/lib/gitlab/error_tracking/error_repository/open_api_strategy.rb
+++ b/lib/gitlab/error_tracking/error_repository/open_api_strategy.rb
@@ -177,7 +177,7 @@ module Gitlab
ErrorRepository::Pagination.new(pagination_hash['next'], pagination_hash['prev'])
end
- LINK_PATTERN = %r{cursor=(?<cursor>[^&]+).*; rel="(?<direction>\w+)"}.freeze
+ LINK_PATTERN = %r{cursor=(?<cursor>[^&]+).*; rel="(?<direction>\w+)"}
def parse_pagination_link(content)
match = LINK_PATTERN.match(content)
diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb
index 8679f17eb9b..e887e455792 100644
--- a/lib/gitlab/exclusive_lease.rb
+++ b/lib/gitlab/exclusive_lease.rb
@@ -12,8 +12,6 @@ module Gitlab
# ExclusiveLease.
#
class ExclusiveLease
- include Gitlab::Utils::StrongMemoize
-
PREFIX = 'gitlab:exclusive_lease'
NoKey = Class.new(ArgumentError)
@@ -33,7 +31,7 @@ module Gitlab
EOS
def self.get_uuid(key)
- with_read_redis do |redis|
+ Gitlab::Redis::ClusterSharedState.with do |redis|
redis.get(redis_shared_state_key(key)) || false
end
end
@@ -63,7 +61,7 @@ module Gitlab
def self.cancel(key, uuid)
return unless key.present?
- with_write_redis do |redis|
+ Gitlab::Redis::ClusterSharedState.with do |redis|
redis.eval(LUA_CANCEL_SCRIPT, keys: [ensure_prefixed_key(key)], argv: [uuid])
end
end
@@ -81,12 +79,6 @@ module Gitlab
# Removes any existing exclusive_lease from redis
# Don't run this in a live system without making sure no one is using the leases
def self.reset_all!(scope = '*')
- Gitlab::Redis::SharedState.with do |redis|
- redis.scan_each(match: redis_shared_state_key(scope)).each do |key|
- redis.del(key)
- end
- end
-
Gitlab::Redis::ClusterSharedState.with do |redis|
redis.scan_each(match: redis_shared_state_key(scope)).each do |key|
redis.del(key)
@@ -94,15 +86,6 @@ module Gitlab
end
end
- def self.use_cluster_shared_state?
- Gitlab::SafeRequestStore[:use_cluster_shared_state] ||=
- Feature.enabled?(:use_cluster_shared_state_for_exclusive_lease)
- end
-
- def self.use_double_lock?
- Gitlab::SafeRequestStore[:use_double_lock] ||= Feature.enabled?(:enable_exclusive_lease_double_lock_rw)
- end
-
def initialize(key, uuid: nil, timeout:)
@redis_shared_state_key = self.class.redis_shared_state_key(key)
@timeout = timeout
@@ -112,23 +95,10 @@ module Gitlab
# Try to obtain the lease. Return lease UUID on success,
# false if the lease is already taken.
def try_obtain
- return try_obtain_with_new_lock if self.class.use_cluster_shared_state?
-
# Performing a single SET is atomic
- obtained = set_lease(Gitlab::Redis::SharedState) && @uuid
-
- # traffic to new store is minimal since only the first lock holder can run SETNX in ClusterSharedState
- return false unless obtained
- return obtained unless self.class.use_double_lock?
- return obtained if same_store # 2nd setnx will surely fail if store are the same
-
- second_lock_obtained = set_lease(Gitlab::Redis::ClusterSharedState) && @uuid
-
- # cancel is safe since it deletes key only if value matches uuid
- # i.e. it will not delete the held lock on ClusterSharedState
- cancel unless second_lock_obtained
-
- second_lock_obtained
+ Gitlab::Redis::ClusterSharedState.with do |redis|
+ redis.set(@redis_shared_state_key, @uuid, nx: true, ex: @timeout) && @uuid
+ end
end
# This lease is waiting to obtain
@@ -139,7 +109,7 @@ module Gitlab
# Try to renew an existing lease. Return lease UUID on success,
# false if the lease is taken by a different UUID or inexistent.
def renew
- self.class.with_write_redis do |redis|
+ Gitlab::Redis::ClusterSharedState.with do |redis|
result = redis.eval(LUA_RENEW_SCRIPT, keys: [@redis_shared_state_key], argv: [@uuid, @timeout])
result == @uuid
end
@@ -147,7 +117,7 @@ module Gitlab
# Returns true if the key for this lease is set.
def exists?
- self.class.with_read_redis do |redis|
+ Gitlab::Redis::ClusterSharedState.with do |redis|
redis.exists?(@redis_shared_state_key) # rubocop:disable CodeReuse/ActiveRecord
end
end
@@ -156,66 +126,17 @@ module Gitlab
#
# This method will return `nil` if no TTL could be obtained.
def ttl
- self.class.with_read_redis do |redis|
+ Gitlab::Redis::ClusterSharedState.with do |redis|
ttl = redis.ttl(@redis_shared_state_key)
ttl if ttl > 0
end
end
- # rubocop:disable CodeReuse/ActiveRecord
- def self.with_write_redis(&blk)
- if use_cluster_shared_state?
- result = Gitlab::Redis::ClusterSharedState.with(&blk)
- Gitlab::Redis::SharedState.with(&blk)
-
- result
- elsif use_double_lock?
- result = Gitlab::Redis::SharedState.with(&blk)
- Gitlab::Redis::ClusterSharedState.with(&blk)
-
- result
- else
- Gitlab::Redis::SharedState.with(&blk)
- end
- end
-
- def self.with_read_redis(&blk)
- if use_cluster_shared_state?
- Gitlab::Redis::ClusterSharedState.with(&blk)
- elsif use_double_lock?
- Gitlab::Redis::SharedState.with(&blk) || Gitlab::Redis::ClusterSharedState.with(&blk)
- else
- Gitlab::Redis::SharedState.with(&blk)
- end
- end
- # rubocop:enable CodeReuse/ActiveRecord
-
# Gives up this lease, allowing it to be obtained by others.
def cancel
self.class.cancel(@redis_shared_state_key, @uuid)
end
-
- private
-
- def set_lease(redis_class)
- redis_class.with do |redis|
- redis.set(@redis_shared_state_key, @uuid, nx: true, ex: @timeout)
- end
- end
-
- def try_obtain_with_new_lock
- # checks shared-state to avoid 2 versions of the application acquiring 1 lock
- # wait for held lock to expire or yielded in case any process on old version is running
- return false if Gitlab::Redis::SharedState.with { |c| c.exists?(@redis_shared_state_key) } # rubocop:disable CodeReuse/ActiveRecord
-
- set_lease(Gitlab::Redis::ClusterSharedState) && @uuid
- end
-
- def same_store
- Gitlab::Redis::ClusterSharedState.with(&:id) == Gitlab::Redis::SharedState.with(&:id) # rubocop:disable CodeReuse/ActiveRecord
- end
- strong_memoize_attr :same_store
end
end
diff --git a/lib/gitlab/exclusive_lease_helpers.rb b/lib/gitlab/exclusive_lease_helpers.rb
index 7cf0232fbf2..8836cb34745 100644
--- a/lib/gitlab/exclusive_lease_helpers.rb
+++ b/lib/gitlab/exclusive_lease_helpers.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module Gitlab
- # This module provides helper methods which are intregrated with GitLab::ExclusiveLease
+ # This module provides helper methods which are integrated with GitLab::ExclusiveLease
module ExclusiveLeaseHelpers
FailedToObtainLockError = Class.new(StandardError)
diff --git a/lib/gitlab/experiment/rollout/feature.rb b/lib/gitlab/experiment/rollout/feature.rb
index bf31dfe08a0..4ff61aa3551 100644
--- a/lib/gitlab/experiment/rollout/feature.rb
+++ b/lib/gitlab/experiment/rollout/feature.rb
@@ -27,7 +27,8 @@ module Gitlab
#
# If the `Feature.enabled?` check is false, we return nil implicitly,
# which will assign the control. Otherwise we call super, which will
- # assign a variant evenly, or based on our provided distribution rules.
+ # assign a variant based on our provided distribution rules.
+ # Otherwise we will assign a variant evenly across the behaviours without control.
def execute_assignment
super if ::Feature.enabled?(feature_flag_name, self, type: :experiment)
end
@@ -67,6 +68,10 @@ module Gitlab
def feature_flag_name
experiment.name.tr('/', '_')
end
+
+ def behavior_names
+ super - [:control]
+ end
end
end
end
diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb
index 9150213020e..37f593ed551 100644
--- a/lib/gitlab/git.rb
+++ b/lib/gitlab/git.rb
@@ -8,7 +8,7 @@ module Gitlab
# https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012
EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'
BLANK_SHA = ('0' * 40).freeze
- COMMIT_ID = /\A#{Gitlab::Git::Commit::RAW_FULL_SHA_PATTERN}\z/.freeze
+ COMMIT_ID = /\A#{Gitlab::Git::Commit::RAW_FULL_SHA_PATTERN}\z/
TAG_REF_PREFIX = "refs/tags/"
BRANCH_REF_PREFIX = "refs/heads/"
diff --git a/lib/gitlab/git/base_error.rb b/lib/gitlab/git/base_error.rb
index 330e947844c..48235b30d90 100644
--- a/lib/gitlab/git/base_error.rb
+++ b/lib/gitlab/git/base_error.rb
@@ -5,7 +5,7 @@ module Gitlab
module Git
class BaseError < StandardError
METADATA_KEY = :gitaly_error_metadata
- DEBUG_ERROR_STRING_REGEX = /(.*?) debug_error_string:.*$/m.freeze
+ DEBUG_ERROR_STRING_REGEX = /(.*?) debug_error_string:.*$/m
GRPC_CODES = {
'0' => 'ok',
'1' => 'cancelled',
diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb
index 21d2eaec041..3d2bde6f0a7 100644
--- a/lib/gitlab/git/blame.rb
+++ b/lib/gitlab/git/blame.rb
@@ -4,6 +4,7 @@ module Gitlab
module Git
class Blame
include Gitlab::EncodingHelper
+ include Gitlab::Git::WrapsGitalyErrors
attr_reader :lines, :blames, :range
@@ -29,13 +30,19 @@ module Gitlab
end
def load_blame
- output = encode_utf8(
- @repo.gitaly_commit_client.raw_blame(@sha, @path, range: range_spec)
- )
-
+ output = encode_utf8(fetch_raw_blame)
process_raw_blame(output)
end
+ def fetch_raw_blame
+ wrapped_gitaly_errors do
+ @repo.gitaly_commit_client.raw_blame(@sha, @path, range: range_spec)
+ end
+ # Return empty result when blame range is out-of-range
+ rescue ArgumentError
+ ""
+ end
+
def process_raw_blame(output)
start_line = nil
lines = []
diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb
index de25fa7e099..743bac62764 100644
--- a/lib/gitlab/git/diff.rb
+++ b/lib/gitlab/git/diff.rb
@@ -33,7 +33,7 @@ module Gitlab
SERIALIZE_KEYS = %i[diff new_path old_path a_mode b_mode new_file renamed_file deleted_file too_large].freeze
- BINARY_NOTICE_PATTERN = %r{Binary files a\/(.*) and b\/(.*) differ}.freeze
+ BINARY_NOTICE_PATTERN = %r{Binary files (.*) and (.*) differ}
class << self
def between(repo, head, base, options = {}, *paths)
@@ -183,6 +183,16 @@ module Gitlab
a_mode == '160000' || b_mode == '160000'
end
+ def unidiff
+ return diff if diff.blank?
+ return json_safe_diff if detect_binary?(@diff) || has_binary_notice?
+
+ old_path_header = new_file? ? '/dev/null' : "a/#{old_path}"
+ new_path_header = deleted_file? ? '/dev/null' : "b/#{new_path}"
+
+ "--- #{old_path_header}\n+++ #{new_path_header}\n" + diff
+ end
+
def line_count
@line_count ||= Util.count_lines(@diff)
end
diff --git a/lib/gitlab/git/pre_receive_error.rb b/lib/gitlab/git/pre_receive_error.rb
index b84ac656927..49116775215 100644
--- a/lib/gitlab/git/pre_receive_error.rb
+++ b/lib/gitlab/git/pre_receive_error.rb
@@ -14,7 +14,7 @@ module Gitlab
'GL-HOOK-ERR:' # Messages marked as safe by user
].freeze
- SAFE_MESSAGE_REGEX = /^(#{SAFE_MESSAGE_PREFIXES.join('|')})\s*(?<safe_message>.+)/.freeze
+ SAFE_MESSAGE_REGEX = /^(#{SAFE_MESSAGE_PREFIXES.join('|')})\s*(?<safe_message>.+)/
attr_reader :raw_message
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index dfbf8292f54..a98cf95edf4 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -163,12 +163,6 @@ module Gitlab
branch_names.count
end
- def rename(new_relative_path)
- wrapped_gitaly_errors do
- gitaly_repository_client.rename(new_relative_path)
- end
- end
-
def remove
wrapped_gitaly_errors do
gitaly_repository_client.remove
@@ -261,12 +255,8 @@ module Gitlab
def archive_metadata(ref, storage_path, project_path, format = "tar.gz", append_sha:, path: nil)
ref ||= root_ref
- if Feature.enabled?(:resolve_ambiguous_archives, container)
- commit_id = extract_commit_id_from_ref(ref)
- return {} if commit_id.nil?
- else
- commit_id = ref
- end
+ commit_id = extract_commit_id_from_ref(ref)
+ return {} if commit_id.nil?
commit = Gitlab::Git::Commit.find(self, commit_id)
return {} if commit.nil?
@@ -1220,6 +1210,14 @@ module Gitlab
end
end
+ def get_file_attributes(revision, file_paths, attributes)
+ wrapped_gitaly_errors do
+ gitaly_repository_client
+ .get_file_attributes(revision, file_paths, attributes)
+ .attribute_infos
+ end
+ end
+
private
def repository_info_size_megabytes
diff --git a/lib/gitlab/git/rugged_impl/use_rugged.rb b/lib/gitlab/git/rugged_impl/use_rugged.rb
index 632b4133f2e..57cced97d02 100644
--- a/lib/gitlab/git/rugged_impl/use_rugged.rb
+++ b/lib/gitlab/git/rugged_impl/use_rugged.rb
@@ -4,15 +4,8 @@ module Gitlab
module Git
module RuggedImpl
module UseRugged
- def use_rugged?(repo, feature_key)
- return Feature.enabled?(feature_key) if Feature.persisted_name?(feature_key)
-
- # Disable Rugged auto-detect(can_use_disk?) when Puma threads>1
- # https://gitlab.com/gitlab-org/gitlab/issues/119326
- return false if running_puma_with_multiple_threads?
- return false if Feature.enabled?(:skip_rugged_auto_detect, type: :ops)
-
- Gitlab::GitalyClient.can_use_disk?(repo.storage)
+ def use_rugged?(_, _)
+ false
end
def execute_rugged_call(method_name, *args)
@@ -49,9 +42,7 @@ module Gitlab
end
def rugged_enabled_through_feature_flag?
- rugged_feature_keys.any? do |feature_key|
- Feature.enabled?(feature_key)
- end
+ false
end
end
end
diff --git a/lib/gitlab/git_audit_event.rb b/lib/gitlab/git_audit_event.rb
new file mode 100644
index 00000000000..b8365bdf41f
--- /dev/null
+++ b/lib/gitlab/git_audit_event.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Gitlab
+ class GitAuditEvent # rubocop:disable Gitlab/NamespacedClass
+ attr_reader :project, :user, :author
+
+ def initialize(player, project)
+ @project = project
+ @author = player.is_a?(::API::Support::GitAccessActor) ? player.deploy_key_or_user : player
+ @user = player.is_a?(::API::Support::GitAccessActor) ? player.user : player
+ end
+
+ def send_audit_event(msg)
+ return if user.blank? || project.blank?
+
+ audit_context = {
+ name: 'repository_git_operation',
+ stream_only: true,
+ author: author,
+ scope: project,
+ target: project,
+ message: msg
+ }
+
+ ::Gitlab::Audit::Auditor.audit(audit_context)
+ end
+ end
+end
diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
index f2ea6f17d90..5ec58fc4f44 100644
--- a/lib/gitlab/gitaly_client.rb
+++ b/lib/gitlab/gitaly_client.rb
@@ -56,13 +56,6 @@ module Gitlab
# https://gitlab.com/gitlab-org/gitaly/-/blob/bf9f52bc/client/dial.go#L78
'grpc.keepalive_time_ms': 20000,
'grpc.keepalive_permit_without_calls': 1,
- # Enable client-side automatic retry. After enabled, gRPC requests will be retried when there are connectivity
- # problems with the target host. Only transparent failures, which mean requests fail before leaving clients, are
- # eligible. Other cases are configurable via retry policy in service config (below). In theory, we can auto-retry
- # read-only RPCs. Gitaly defines a custom field in service proto. Unfortunately, gRPC ruby doesn't support
- # descriptor reflection.
- # For more information please visit https://github.com/grpc/proposal/blob/master/A6-client-retries.md
- 'grpc.enable_retries': 1,
# Service config is a mechanism for grpc to control the behavior of gRPC client. It defines the client-side
# balancing strategy and retry policy. The config receives a raw JSON string. The format is defined here:
# https://github.com/grpc/grpc-proto/blob/master/grpc/service_config/service_config.proto
@@ -72,7 +65,119 @@ module Gitlab
# grpc creates multiple subchannels to all targets retrurned by the resolver. Requests are distributed to
# those subchannels in a round-robin fashion.
# More about client-side load-balancing: https://gitlab.com/groups/gitlab-org/-/epics/8971#note_1207008162
- "loadBalancingConfig": [{ "round_robin": {} }]
+ "loadBalancingConfig": [{ "round_robin": {} }],
+ # Enable retries for read-only RPCs. With this setting the client to will resend requests that fail with
+ # the following conditions:
+ # 1. An `UNAVAILABLE` status code was received.
+ # 2. No response-headers were received from the server.
+ # This allows the client to handle momentary service interruptions without user-facing errors. gRPC's
+ # automatic 'transparent retries' may also be sent.
+ # For more information please visit https://github.com/grpc/proposal/blob/master/A6-client-retries.md
+ 'methodConfig': [
+ {
+ # Gitaly sets an `op_type` `MethodOption` on RPCs to note if it mutates a repository. We cannot
+ # programatically detect read-only RPCs, i.e. those safe to retry, because Ruby's protobuf
+ # implementation does not provide access to `MethodOptions`. That feature is being tracked under
+ # https://github.com/protocolbuffers/protobuf/issues/1198. When that is complete we can replace this
+ # table.
+ 'name': [
+ { 'service': 'gitaly.BlobService', 'method': 'GetBlob' },
+ { 'service': 'gitaly.BlobService', 'method': 'GetBlobs' },
+ { 'service': 'gitaly.BlobService', 'method': 'GetLFSPointers' },
+ { 'service': 'gitaly.BlobService', 'method': 'GetAllLFSPointers' },
+ { 'service': 'gitaly.BlobService', 'method': 'ListAllBlobs' },
+ { 'service': 'gitaly.BlobService', 'method': 'ListAllLFSPointers' },
+ { 'service': 'gitaly.BlobService', 'method': 'ListBlobs' },
+ { 'service': 'gitaly.BlobService', 'method': 'ListLFSPointers' },
+ { 'service': 'gitaly.CommitService', 'method': 'CheckObjectsExist' },
+ { 'service': 'gitaly.CommitService', 'method': 'CommitIsAncestor' },
+ { 'service': 'gitaly.CommitService', 'method': 'CommitLanguages' },
+ { 'service': 'gitaly.CommitService', 'method': 'CommitStats' },
+ { 'service': 'gitaly.CommitService', 'method': 'CommitsByMessage' },
+ { 'service': 'gitaly.CommitService', 'method': 'CountCommits' },
+ { 'service': 'gitaly.CommitService', 'method': 'CountDivergingCommits' },
+ { 'service': 'gitaly.CommitService', 'method': 'FilterShasWithSignatures' },
+ { 'service': 'gitaly.CommitService', 'method': 'FindAllCommits' },
+ { 'service': 'gitaly.CommitService', 'method': 'FindCommit' },
+ { 'service': 'gitaly.CommitService', 'method': 'FindCommits' },
+ { 'service': 'gitaly.CommitService', 'method': 'GetCommitMessages' },
+ { 'service': 'gitaly.CommitService', 'method': 'GetCommitSignatures' },
+ { 'service': 'gitaly.CommitService', 'method': 'GetTreeEntries' },
+ { 'service': 'gitaly.CommitService', 'method': 'LastCommitForPath' },
+ { 'service': 'gitaly.CommitService', 'method': 'ListAllCommits' },
+ { 'service': 'gitaly.CommitService', 'method': 'ListCommits' },
+ { 'service': 'gitaly.CommitService', 'method': 'ListCommitsByOid' },
+ { 'service': 'gitaly.CommitService', 'method': 'ListCommitsByRefName' },
+ { 'service': 'gitaly.CommitService', 'method': 'ListFiles' },
+ { 'service': 'gitaly.CommitService', 'method': 'ListLastCommitsForTree' },
+ { 'service': 'gitaly.CommitService', 'method': 'RawBlame' },
+ { 'service': 'gitaly.CommitService', 'method': 'TreeEntry' },
+ { 'service': 'gitaly.ConflictsService', 'method': 'ListConflictFiles' },
+ { 'service': 'gitaly.DiffService', 'method': 'CommitDelta' },
+ { 'service': 'gitaly.DiffService', 'method': 'CommitDiff' },
+ { 'service': 'gitaly.DiffService', 'method': 'DiffStats' },
+ { 'service': 'gitaly.DiffService', 'method': 'FindChangedPaths' },
+ { 'service': 'gitaly.DiffService', 'method': 'GetPatchID' },
+ { 'service': 'gitaly.DiffService', 'method': 'RawDiff' },
+ { 'service': 'gitaly.DiffService', 'method': 'RawPatch' },
+ { 'service': 'gitaly.ObjectPoolService', 'method': 'GetObjectPool' },
+ { 'service': 'gitaly.RefService', 'method': 'FindAllBranches' },
+ { 'service': 'gitaly.RefService', 'method': 'FindAllRemoteBranches' },
+ { 'service': 'gitaly.RefService', 'method': 'FindAllTags' },
+ { 'service': 'gitaly.RefService', 'method': 'FindBranch' },
+ { 'service': 'gitaly.RefService', 'method': 'FindDefaultBranchName' },
+ { 'service': 'gitaly.RefService', 'method': 'FindLocalBranches' },
+ { 'service': 'gitaly.RefService', 'method': 'FindRefsByOID' },
+ { 'service': 'gitaly.RefService', 'method': 'FindTag' },
+ { 'service': 'gitaly.RefService', 'method': 'GetTagMessages' },
+ { 'service': 'gitaly.RefService', 'method': 'GetTagSignatures' },
+ { 'service': 'gitaly.RefService', 'method': 'ListBranchNamesContainingCommit' },
+ { 'service': 'gitaly.RefService', 'method': 'ListRefs' },
+ { 'service': 'gitaly.RefService', 'method': 'ListTagNamesContainingCommit' },
+ { 'service': 'gitaly.RefService', 'method': 'RefExists' },
+ { 'service': 'gitaly.RemoteService', 'method': 'FindRemoteRepository' },
+ { 'service': 'gitaly.RemoteService', 'method': 'FindRemoteRootRef' },
+ { 'service': 'gitaly.RemoteService', 'method': 'UpdateRemoteMirror' },
+ { 'service': 'gitaly.RepositoryService', 'method': 'BackupCustomHooks' },
+ { 'service': 'gitaly.RepositoryService', 'method': 'BackupRepository' },
+ { 'service': 'gitaly.RepositoryService', 'method': 'CalculateChecksum' },
+ { 'service': 'gitaly.RepositoryService', 'method': 'CreateBundle' },
+ { 'service': 'gitaly.RepositoryService', 'method': 'Fsck' },
+ { 'service': 'gitaly.RepositoryService', 'method': 'FindLicense' },
+ { 'service': 'gitaly.RepositoryService', 'method': 'FindMergeBase' },
+ { 'service': 'gitaly.RepositoryService', 'method': 'FullPath' },
+ { 'service': 'gitaly.RepositoryService', 'method': 'HasLocalBranches' },
+ { 'service': 'gitaly.RepositoryService', 'method': 'GetArchive' },
+ { 'service': 'gitaly.RepositoryService', 'method': 'GetConfig' },
+ { 'service': 'gitaly.RepositoryService', 'method': 'GetCustomHooks' },
+ { 'service': 'gitaly.RepositoryService', 'method': 'GetFileAttributes' },
+ { 'service': 'gitaly.RepositoryService', 'method': 'GetInfoAttributes' },
+ { 'service': 'gitaly.RepositoryService', 'method': 'GetObject' },
+ { 'service': 'gitaly.RepositoryService', 'method': 'GetObjectDirectorySize' },
+ { 'service': 'gitaly.RepositoryService', 'method': 'GetRawChanges' },
+ { 'service': 'gitaly.RepositoryService', 'method': 'GetSnapshot' },
+ { 'service': 'gitaly.RepositoryService', 'method': 'ObjectSize' },
+ { 'service': 'gitaly.RepositoryService', 'method': 'ObjectFormat' },
+ { 'service': 'gitaly.RepositoryService', 'method': 'RepositoryExists' },
+ { 'service': 'gitaly.RepositoryService', 'method': 'RepositoryInfo' },
+ { 'service': 'gitaly.RepositoryService', 'method': 'RepositorySize' },
+ { 'service': 'gitaly.RepositoryService', 'method': 'SearchFilesByContent' },
+ { 'service': 'gitaly.RepositoryService', 'method': 'SearchFilesByName' },
+ { 'service': 'gitaly.ServerService', 'method': 'ClockSynced' },
+ { 'service': 'gitaly.ServerService', 'method': 'DiskStatistics' },
+ { 'service': 'gitaly.ServerService', 'method': 'ReadinessCheck' },
+ { 'service': 'gitaly.ServerService', 'method': 'ServerInfo' },
+ { 'service': 'grpc.health.v1.Health', 'method': 'Check' }
+ ],
+ 'retryPolicy': {
+ 'maxAttempts': 3, # Initial request, plus up to two retries.
+ 'initialBackoff': '0.25s',
+ 'maxBackoff': '1s',
+ 'backoffMultiplier': 2, # Minimum retry duration is 750ms.
+ 'retryableStatusCodes': ['UNAVAILABLE']
+ }
+ }
+ ]
}.to_json
}
end
diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb
index 573e3547202..1ef5b0f96c2 100644
--- a/lib/gitlab/gitaly_client/commit_service.rb
+++ b/lib/gitlab/gitaly_client/commit_service.rb
@@ -418,9 +418,6 @@ module Gitlab
response = gitaly_client_call(@repository.storage, :commit_service, :raw_blame, request, timeout: GitalyClient.medium_timeout)
response.reduce([]) { |memo, msg| memo << msg.data }.join
- # Temporary fix, use structured errors when they are available: https://gitlab.com/gitlab-org/gitaly/-/issues/5594
- rescue GRPC::Internal
- ""
end
def find_commit(revision)
diff --git a/lib/gitlab/gitaly_client/namespace_service.rb b/lib/gitlab/gitaly_client/namespace_service.rb
deleted file mode 100644
index 05aee2fa55d..00000000000
--- a/lib/gitlab/gitaly_client/namespace_service.rb
+++ /dev/null
@@ -1,57 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module GitalyClient
- class NamespaceService
- extend Gitlab::TemporarilyAllow
-
- NamespaceServiceAccessError = Class.new(StandardError)
- ALLOW_KEY = :allow_namespace
-
- def self.allow
- temporarily_allow(ALLOW_KEY) { yield }
- end
-
- def self.denied?
- !temporarily_allowed?(ALLOW_KEY)
- end
-
- def initialize(storage)
- raise NamespaceServiceAccessError if self.class.denied?
-
- @storage = storage
- end
-
- def add(name)
- request = Gitaly::AddNamespaceRequest.new(storage_name: @storage, name: name)
-
- gitaly_client_call(:add_namespace, request, timeout: GitalyClient.fast_timeout)
- end
-
- def remove(name)
- request = Gitaly::RemoveNamespaceRequest.new(storage_name: @storage, name: name)
-
- gitaly_client_call(:remove_namespace, request, timeout: GitalyClient.long_timeout)
- end
-
- def rename(from, to)
- request = Gitaly::RenameNamespaceRequest.new(storage_name: @storage, from: from, to: to)
-
- gitaly_client_call(:rename_namespace, request, timeout: GitalyClient.fast_timeout)
- end
-
- def exists?(name)
- request = Gitaly::NamespaceExistsRequest.new(storage_name: @storage, name: name)
-
- response = gitaly_client_call(:namespace_exists, request, timeout: GitalyClient.fast_timeout)
- response.exists
- end
-
- private
-
- def gitaly_client_call(type, request, timeout: nil)
- GitalyClient.call(@storage, :namespace_service, type, request, timeout: timeout)
- end
- end
- end
-end
diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb
index 9ea541e083d..d92bf5263f1 100644
--- a/lib/gitlab/gitaly_client/repository_service.rb
+++ b/lib/gitlab/gitaly_client/repository_service.rb
@@ -319,12 +319,6 @@ module Gitlab
gitaly_client_call(@storage, :object_pool_service, :disconnect_git_alternates, request, timeout: GitalyClient.long_timeout)
end
- def rename(relative_path)
- request = Gitaly::RenameRepositoryRequest.new(repository: @gitaly_repo, relative_path: relative_path)
-
- gitaly_client_call(@storage, :repository_service, :rename_repository, request, timeout: GitalyClient.fast_timeout)
- end
-
def remove
request = Gitaly::RemoveRepositoryRequest.new(repository: @gitaly_repo)
@@ -359,6 +353,13 @@ module Gitlab
)
end
+ def get_file_attributes(revision, paths, attributes)
+ request = Gitaly::GetFileAttributesRequest
+ .new(repository: @gitaly_repo, revision: revision, paths: paths, attributes: attributes)
+
+ gitaly_client_call(@repository.storage, :repository_service, :get_file_attributes, request, timeout: GitalyClient.fast_timeout)
+ end
+
private
def search_results_from_response(gitaly_response, options = {})
diff --git a/lib/gitlab/github_import/bulk_importing.rb b/lib/gitlab/github_import/bulk_importing.rb
index d16f4d7587b..47080ea1979 100644
--- a/lib/gitlab/github_import/bulk_importing.rb
+++ b/lib/gitlab/github_import/bulk_importing.rb
@@ -32,7 +32,7 @@ module Gitlab
log_error(github_identifiers, build_record.errors.full_messages)
errors << {
validation_errors: build_record.errors,
- github_identifiers: github_identifiers
+ external_identifiers: github_identifiers
}
next
end
@@ -69,7 +69,7 @@ module Gitlab
correlation_id_value: correlation_id_value,
retry_count: nil,
created_at: Time.zone.now,
- external_identifiers: error[:github_identifiers]
+ external_identifiers: error[:external_identifiers]
}
end
@@ -79,8 +79,7 @@ module Gitlab
private
def log_and_increment_counter(value, operation)
- Gitlab::Import::Logger.info(
- import_type: :github,
+ Logger.info(
project_id: project.id,
importer: self.class.name,
message: "#{value} #{object_type.to_s.pluralize} #{operation}"
@@ -95,12 +94,11 @@ module Gitlab
end
def log_error(github_identifiers, messages)
- Gitlab::Import::Logger.error(
- import_type: :github,
+ Logger.error(
project_id: project.id,
importer: self.class.name,
message: messages,
- github_identifiers: github_identifiers
+ external_identifiers: github_identifiers
)
end
diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb
index 23d4faa3dde..5a0ae680ab8 100644
--- a/lib/gitlab/github_import/client.rb
+++ b/lib/gitlab/github_import/client.rb
@@ -284,10 +284,10 @@ module Gitlab
def on_retry
proc do |exception, try, elapsed_time, next_interval|
- Gitlab::Import::Logger.info(
+ Logger.info(
message: "GitHub connection retry triggered",
'error.class': exception.class,
- 'error.message': exception.message,
+ 'exception.message': exception.message,
try_count: try,
elapsed_time_s: elapsed_time,
wait_to_retry_s: next_interval
diff --git a/lib/gitlab/github_import/clients/proxy.rb b/lib/gitlab/github_import/clients/proxy.rb
index 27030f5382a..a95a8cddc8d 100644
--- a/lib/gitlab/github_import/clients/proxy.rb
+++ b/lib/gitlab/github_import/clients/proxy.rb
@@ -10,19 +10,15 @@ module Gitlab
REPOS_COUNT_CACHE_KEY = 'github-importer/provider-repo-count/%{type}/%{user_id}'
- def initialize(access_token, client_options)
- @client = pick_client(access_token, client_options)
+ def initialize(access_token)
+ @client = Gitlab::GithubImport::Client.new(access_token)
end
def repos(search_text, options)
- return { repos: filtered(client.repos, search_text) } if use_legacy?
-
fetch_repos_via_graphql(search_text, options)
end
def count_repos_by(relation_type, user_id)
- return if use_legacy?
-
key = format(REPOS_COUNT_CACHE_KEY, type: relation_type, user_id: user_id)
::Gitlab::Cache::Import::Caching.read_integer(key, timeout: 5.minutes) ||
@@ -40,22 +36,6 @@ module Gitlab
}
end
- def pick_client(access_token, client_options)
- return Gitlab::GithubImport::Client.new(access_token) unless use_legacy?
-
- Gitlab::LegacyGithubImport::Client.new(access_token, **client_options)
- end
-
- def filtered(collection, search_text)
- return collection if search_text.blank?
-
- collection.select { |item| item[:name].to_s.downcase.include?(search_text) }
- end
-
- def use_legacy?
- Feature.disabled?(:remove_legacy_github_client)
- end
-
def fetch_and_cache_repos_count_via_graphql(relation_type, key)
response = client.count_repos_by_relation_type_graphql(relation_type: relation_type)
count = response.dig(:data, :search, :repositoryCount)
diff --git a/lib/gitlab/github_import/exceptions.rb b/lib/gitlab/github_import/exceptions.rb
index 3a36b64a11b..b7d93182603 100644
--- a/lib/gitlab/github_import/exceptions.rb
+++ b/lib/gitlab/github_import/exceptions.rb
@@ -6,6 +6,8 @@ module Gitlab
# Sometimes it's not clear which of not implemented interfaces caused this error.
# We need custom exception to be able to add text that gives extra context.
NotImplementedError = Class.new(StandardError)
+
+ NoteableNotFound = Class.new(StandardError)
end
end
end
diff --git a/lib/gitlab/github_import/importer/attachments/issues_importer.rb b/lib/gitlab/github_import/importer/attachments/issues_importer.rb
index c8f0b59fd18..a0e1a3f2d25 100644
--- a/lib/gitlab/github_import/importer/attachments/issues_importer.rb
+++ b/lib/gitlab/github_import/importer/attachments/issues_importer.rb
@@ -24,7 +24,7 @@ module Gitlab
private
def collection
- project.issues.select(:id, :description, :iid)
+ project.issues.id_not_in(already_imported_ids).select(:id, :description, :iid)
end
def ordering_column
diff --git a/lib/gitlab/github_import/importer/attachments/merge_requests_importer.rb b/lib/gitlab/github_import/importer/attachments/merge_requests_importer.rb
index cd3a327a846..22b3e7c640b 100644
--- a/lib/gitlab/github_import/importer/attachments/merge_requests_importer.rb
+++ b/lib/gitlab/github_import/importer/attachments/merge_requests_importer.rb
@@ -24,7 +24,7 @@ module Gitlab
private
def collection
- project.merge_requests.select(:id, :description, :iid)
+ project.merge_requests.id_not_in(already_imported_ids).select(:id, :description, :iid)
end
def ordering_column
diff --git a/lib/gitlab/github_import/importer/attachments/notes_importer.rb b/lib/gitlab/github_import/importer/attachments/notes_importer.rb
index aa38a7a3a3f..5ab0cf5b6b0 100644
--- a/lib/gitlab/github_import/importer/attachments/notes_importer.rb
+++ b/lib/gitlab/github_import/importer/attachments/notes_importer.rb
@@ -26,7 +26,7 @@ module Gitlab
# TODO: exclude :system, :noteable_type from select after removing override Note#note method
# https://gitlab.com/gitlab-org/gitlab/-/issues/369923
def collection
- project.notes.user.select(:id, :note, :system, :noteable_type)
+ project.notes.id_not_in(already_imported_ids).user.select(:id, :note, :system, :noteable_type)
end
end
end
diff --git a/lib/gitlab/github_import/importer/attachments/releases_importer.rb b/lib/gitlab/github_import/importer/attachments/releases_importer.rb
index 7d6dbeb901e..0527170f5e1 100644
--- a/lib/gitlab/github_import/importer/attachments/releases_importer.rb
+++ b/lib/gitlab/github_import/importer/attachments/releases_importer.rb
@@ -24,7 +24,7 @@ module Gitlab
private
def collection
- project.releases.select(:id, :description, :tag)
+ project.releases.id_not_in(already_imported_ids).select(:id, :description, :tag)
end
end
end
diff --git a/lib/gitlab/github_import/importer/diff_note_importer.rb b/lib/gitlab/github_import/importer/diff_note_importer.rb
index 44ffcd7a1e4..d49180e6927 100644
--- a/lib/gitlab/github_import/importer/diff_note_importer.rb
+++ b/lib/gitlab/github_import/importer/diff_note_importer.rb
@@ -36,13 +36,6 @@ module Gitlab
Logger.warn(message: e.message, 'error.class': e.class.name)
import_with_legacy_diff_note
- rescue ActiveRecord::InvalidForeignKey => e
- # It's possible the project and the issue have been deleted since
- # scheduling this job. In this case we'll just skip creating the note
- Logger.info(
- message: e.message,
- github_identifiers: note.github_identifiers
- )
end
private
@@ -71,6 +64,7 @@ module Gitlab
discussion_id: note.discussion_id,
noteable_id: merge_request_id,
project_id: project.id,
+ namespace_id: project.project_namespace_id,
author_id: author_id,
note: note_body,
commit_id: note.original_commit_id,
@@ -132,7 +126,7 @@ module Gitlab
Logger.info(
project_id: project.id,
importer: self.class.name,
- github_identifiers: note.github_identifiers,
+ external_identifiers: note.github_identifiers,
model: model
)
end
diff --git a/lib/gitlab/github_import/importer/issue_importer.rb b/lib/gitlab/github_import/importer/issue_importer.rb
index a537841ecf3..3cf67d7df96 100644
--- a/lib/gitlab/github_import/importer/issue_importer.rb
+++ b/lib/gitlab/github_import/importer/issue_importer.rb
@@ -29,8 +29,8 @@ module Gitlab
def execute
Issue.transaction do
if (issue_id = create_issue)
- create_assignees(issue_id)
issuable_finder.cache_database_id(issue_id)
+ create_assignees(issue_id)
update_search_data(issue_id)
end
end
@@ -64,9 +64,6 @@ module Gitlab
issue.validate!
insert_and_return_id(attributes, project.issues)
- rescue ActiveRecord::InvalidForeignKey
- # It's possible the project has been deleted since scheduling this
- # job. In this case we'll just skip creating the issue.
end
# Stores all issue assignees in the database.
diff --git a/lib/gitlab/github_import/importer/note_importer.rb b/lib/gitlab/github_import/importer/note_importer.rb
index 04da015a33f..dc5c6e49b55 100644
--- a/lib/gitlab/github_import/importer/note_importer.rb
+++ b/lib/gitlab/github_import/importer/note_importer.rb
@@ -17,7 +17,9 @@ module Gitlab
end
def execute
- return unless (noteable_id = find_noteable_id)
+ noteable_id = find_noteable_id
+
+ raise Exceptions::NoteableNotFound, 'Error to find noteable_id for note' unless noteable_id
author_id, author_found = user_finder.author_id_for(note)
@@ -25,6 +27,7 @@ module Gitlab
noteable_type: note.noteable_type,
noteable_id: noteable_id,
project_id: project.id,
+ namespace_id: project.project_namespace_id,
author_id: author_id,
note: note_body(author_found),
discussion_id: note.discussion_id,
@@ -33,19 +36,15 @@ module Gitlab
updated_at: note.updated_at
}
- note = Note.new(attributes.merge(importing: true))
- note.validate!
+ Note.new(attributes.merge(importing: true)).validate!
- # We're using bulk_insert here so we can bypass any validations and
- # callbacks. Running these would result in a lot of unnecessary SQL
+ # We're using bulk_insert here so we can bypass any callbacks.
+ # Running these would result in a lot of unnecessary SQL
# queries being executed when importing large projects.
# Note: if you're going to replace `legacy_bulk_insert` with something that trigger callback
# to generate HTML version - you also need to regenerate it in
# Gitlab::GithubImport::Importer::NoteAttachmentsImporter.
ApplicationRecord.legacy_bulk_insert(Note.table_name, [attributes]) # rubocop:disable Gitlab/BulkInsert
- rescue ActiveRecord::InvalidForeignKey
- # It's possible the project and the issue have been deleted since
- # scheduling this job. In this case we'll just skip creating the note.
end
# Returns the ID of the issue or merge request to create the note for.
diff --git a/lib/gitlab/github_import/importer/pull_request_importer.rb b/lib/gitlab/github_import/importer/pull_request_importer.rb
index 5690a2cc997..acdafef670c 100644
--- a/lib/gitlab/github_import/importer/pull_request_importer.rb
+++ b/lib/gitlab/github_import/importer/pull_request_importer.rb
@@ -27,9 +27,9 @@ module Gitlab
mr, already_exists = create_merge_request
if mr
+ issuable_finder.cache_database_id(mr.id)
set_merge_request_assignees(mr)
insert_git_data(mr, already_exists)
- issuable_finder.cache_database_id(mr.id)
end
end
diff --git a/lib/gitlab/github_import/importer/pull_requests/review_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests/review_requests_importer.rb
index 0a92aee801d..7f78df615a2 100644
--- a/lib/gitlab/github_import/importer/pull_requests/review_requests_importer.rb
+++ b/lib/gitlab/github_import/importer/pull_requests/review_requests_importer.rb
@@ -19,6 +19,9 @@ module Gitlab
review_requests = client.pull_request_review_requests(repo, merge_request.iid)
review_requests[:merge_request_id] = merge_request.id
review_requests[:merge_request_iid] = merge_request.iid
+
+ Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched)
+
yield review_requests
mark_merge_request_imported(merge_request)
@@ -42,6 +45,10 @@ module Gitlab
:pull_request_review_requests
end
+ def object_type
+ :pull_request_review_request
+ end
+
# rubocop:disable CodeReuse/ActiveRecord
def merge_request_collection
project.merge_requests
diff --git a/lib/gitlab/github_import/importer/pull_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests_importer.rb
index 62863ba67fd..671e023e90b 100644
--- a/lib/gitlab/github_import/importer/pull_requests_importer.rb
+++ b/lib/gitlab/github_import/importer/pull_requests_importer.rb
@@ -44,7 +44,7 @@ module Gitlab
pname = project.path_with_namespace
- Gitlab::Import::Logger.info(
+ Logger.info(
message: 'GitHub importer finished updating repository',
project_name: pname
)
diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb
index cfc1ec526b0..cccd99f48b1 100644
--- a/lib/gitlab/github_import/parallel_scheduling.rb
+++ b/lib/gitlab/github_import/parallel_scheduling.rb
@@ -211,6 +211,12 @@ module Gitlab
private
+ # Returns the set used to track "already imported" objects.
+ # Items are the values returned by `#id_for_already_imported_cache`.
+ def already_imported_ids
+ Gitlab::Cache::Import::Caching.values_from_set(already_imported_cache_key)
+ end
+
def additional_object_data
{}
end
diff --git a/lib/gitlab/github_import/representation/diff_note.rb b/lib/gitlab/github_import/representation/diff_note.rb
index 191e15962a6..e8e515d1f87 100644
--- a/lib/gitlab/github_import/representation/diff_note.rb
+++ b/lib/gitlab/github_import/representation/diff_note.rb
@@ -7,7 +7,7 @@ module Gitlab
include ToHash
include ExposeAttribute
- NOTEABLE_ID_REGEX = %r{/pull/(?<iid>\d+)}i.freeze
+ NOTEABLE_ID_REGEX = %r{/pull/(?<iid>\d+)}i
expose_attribute :noteable_id, :commit_id, :file_path,
:diff_hunk, :author, :created_at, :updated_at,
diff --git a/lib/gitlab/github_import/representation/diff_notes/discussion_id.rb b/lib/gitlab/github_import/representation/diff_notes/discussion_id.rb
index 38b560f21c0..db36e81c5b8 100644
--- a/lib/gitlab/github_import/representation/diff_notes/discussion_id.rb
+++ b/lib/gitlab/github_import/representation/diff_notes/discussion_id.rb
@@ -6,7 +6,7 @@ module Gitlab
module DiffNotes
class DiscussionId
NOTEABLE_TYPE = 'MergeRequest'
- DISCUSSION_CACHE_REGEX = %r{/(?<repo>[^/]*)/pull/(?<iid>\d+)}i.freeze
+ DISCUSSION_CACHE_REGEX = %r{/(?<repo>[^/]*)/pull/(?<iid>\d+)}i
DISCUSSION_CACHE_KEY = 'github-importer/discussion-id-map/%{project}/%{noteable_id}/%{original_note_id}'
def initialize(note)
diff --git a/lib/gitlab/github_import/representation/diff_notes/suggestion_formatter.rb b/lib/gitlab/github_import/representation/diff_notes/suggestion_formatter.rb
index 38b15c4b5bb..fdf74fd9c9f 100644
--- a/lib/gitlab/github_import/representation/diff_notes/suggestion_formatter.rb
+++ b/lib/gitlab/github_import/representation/diff_notes/suggestion_formatter.rb
@@ -16,7 +16,7 @@ module Gitlab
# - the ```suggestion tag must be the first text of the line
# - it might have up to 3 spaces before the ```suggestion tag
# - extra text on the ```suggestion tag line will be ignored
- GITHUB_SUGGESTION = /^\ {,3}(?<suggestion>```suggestion\b).*(?<eol>\R)/.freeze
+ GITHUB_SUGGESTION = /^\ {,3}(?<suggestion>```suggestion\b).*(?<eol>\R)/
def initialize(note:, start_line: nil, end_line: nil)
@note = note
diff --git a/lib/gitlab/github_import/representation/note.rb b/lib/gitlab/github_import/representation/note.rb
index 7a8bdfb1c64..76adbb651af 100644
--- a/lib/gitlab/github_import/representation/note.rb
+++ b/lib/gitlab/github_import/representation/note.rb
@@ -12,7 +12,7 @@ module Gitlab
expose_attribute :noteable_id, :noteable_type, :author, :note,
:created_at, :updated_at, :note_id
- NOTEABLE_TYPE_REGEX = %r{/(?<type>(pull|issues))/(?<iid>\d+)}i.freeze
+ NOTEABLE_TYPE_REGEX = %r{/(?<type>(pull|issues))/(?<iid>\d+)}i
# Builds a note from a GitHub API response.
#
diff --git a/lib/gitlab/github_import/settings.rb b/lib/gitlab/github_import/settings.rb
index 73a5f49a9e3..a4170f4147f 100644
--- a/lib/gitlab/github_import/settings.rb
+++ b/lib/gitlab/github_import/settings.rb
@@ -61,8 +61,11 @@ module Gitlab
additional_access_tokens: user_settings[:additional_access_tokens]
)
- import_data = project.create_or_update_import_data(
- data: { optional_stages: optional_stages },
+ import_data = project.build_or_assign_import_data(
+ data: {
+ optional_stages: optional_stages,
+ timeout_strategy: user_settings[:timeout_strategy]
+ },
credentials: credentials
)
diff --git a/lib/gitlab/github_import/user_finder.rb b/lib/gitlab/github_import/user_finder.rb
index 1832f071a44..4bf2d8a0aca 100644
--- a/lib/gitlab/github_import/user_finder.rb
+++ b/lib/gitlab/github_import/user_finder.rb
@@ -271,8 +271,7 @@ module Gitlab
end
def log(message, username: nil)
- Gitlab::Import::Logger.info(
- import_type: :github,
+ Logger.info(
project_id: project.id,
class: self.class.name,
username: username,
diff --git a/lib/gitlab/golang.rb b/lib/gitlab/golang.rb
index 1b625a3a514..f0afe493019 100644
--- a/lib/gitlab/golang.rb
+++ b/lib/gitlab/golang.rb
@@ -32,7 +32,7 @@ module Gitlab
# vX.Y.(Z+1)-0.yyyymmddhhmmss-sha1337beef0, when most recent prior tag is vX.Y.Z
if version.minor != 0 || version.patch != 0
- m = /\A(.*\.)?0\./.freeze.match pre
+ m = /\A(.*\.)?0\./.match pre
return false unless m
pre = pre[m[0].length..]
@@ -40,7 +40,7 @@ module Gitlab
# This pattern is intentionally more forgiving than the patterns
# above. Correctness is verified by #validate_pseudo_version.
- /\A\d{14}-\h+\z/.freeze.match? pre
+ /\A\d{14}-\h+\z/.match? pre
end
def parse_pseudo_version(semver)
diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb
index eefa23142af..e057b4bb6f1 100644
--- a/lib/gitlab/gon_helper.rb
+++ b/lib/gitlab/gon_helper.rb
@@ -7,15 +7,16 @@ module Gitlab
include WebpackHelper
def add_gon_variables
- gon.api_version = 'v4'
- gon.default_avatar_url = default_avatar_url
- gon.max_file_size = Gitlab::CurrentSettings.max_attachment_size
- gon.asset_host = ActionController::Base.asset_host
- gon.webpack_public_path = webpack_public_path
- gon.relative_url_root = Gitlab.config.gitlab.relative_url_root
- gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class
- gon.markdown_surround_selection = current_user&.markdown_surround_selection
- gon.markdown_automatic_lists = current_user&.markdown_automatic_lists
+ gon.api_version = 'v4'
+ gon.default_avatar_url = default_avatar_url
+ gon.max_file_size = Gitlab::CurrentSettings.max_attachment_size
+ gon.asset_host = ActionController::Base.asset_host
+ gon.webpack_public_path = webpack_public_path
+ gon.relative_url_root = Gitlab.config.gitlab.relative_url_root
+ gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class
+ gon.markdown_surround_selection = current_user&.markdown_surround_selection
+ gon.markdown_automatic_lists = current_user&.markdown_automatic_lists
+ gon.math_rendering_limits_enabled = Gitlab::CurrentSettings.math_rendering_limits_enabled
add_browsersdk_tracking
@@ -75,10 +76,8 @@ module Gitlab
push_frontend_feature_flag(:source_editor_toolbar)
push_frontend_feature_flag(:vscode_web_ide, current_user)
push_frontend_feature_flag(:unbatch_graphql_queries, current_user)
- push_frontend_feature_flag(:server_side_frecent_namespaces, current_user)
# To be removed with https://gitlab.com/gitlab-org/gitlab/-/issues/399248
push_frontend_feature_flag(:remove_monitor_metrics)
- push_frontend_feature_flag(:gitlab_duo, current_user)
push_frontend_feature_flag(:custom_emoji)
end
@@ -121,7 +120,9 @@ module Gitlab
end
def add_browsersdk_tracking
- return unless Gitlab.com? && Feature.enabled?(:browsersdk_tracking)
+ return unless Gitlab.com? && Feature.enabled?(:browsersdk_tracking) && Feature.enabled?(:gl_analytics_tracking,
+Feature.current_request)
+
return if ENV['GITLAB_ANALYTICS_URL'].blank? || ENV['GITLAB_ANALYTICS_ID'].blank?
gon.analytics_url = ENV['GITLAB_ANALYTICS_URL']
diff --git a/lib/gitlab/graphql/authorize/connection_filter_extension.rb b/lib/gitlab/graphql/authorize/connection_filter_extension.rb
index 889c024ab5e..9b6398fc498 100644
--- a/lib/gitlab/graphql/authorize/connection_filter_extension.rb
+++ b/lib/gitlab/graphql/authorize/connection_filter_extension.rb
@@ -46,12 +46,15 @@ module Gitlab
end
def after_resolve(value:, context:, **rest)
- return value if value.is_a?(GraphQL::Execution::Execute::Skip)
+ return value if value.is_a?(GraphQL::Execution::Skip)
if @field.connection?
redact_connection(value, context)
elsif @field.type.list?
- redact_list(value.to_a, context) unless value.nil?
+ unless value.nil?
+ value = value.to_a
+ redact_list(value, context)
+ end
end
value
diff --git a/lib/gitlab/graphql/deprecations.rb b/lib/gitlab/graphql/deprecations.rb
index 221b19bf8a3..61a49bd7473 100644
--- a/lib/gitlab/graphql/deprecations.rb
+++ b/lib/gitlab/graphql/deprecations.rb
@@ -11,6 +11,14 @@ module Gitlab
attr_accessor :deprecation
end
+ def initialize(*args, **kwargs, &block)
+ init_gitlab_deprecation(kwargs)
+
+ super
+
+ update_deprecation_description
+ end
+
def visible?(ctx)
super && ctx[:remove_deprecated] == true ? deprecation.nil? : true
end
@@ -37,7 +45,12 @@ module Gitlab
end
kwargs[:deprecation_reason] = deprecation.deprecation_reason
- kwargs[:description] = deprecation.edit_description(kwargs[:description])
+ end
+
+ def update_deprecation_description
+ return if deprecation.nil?
+
+ description(deprecation.edit_description(description))
end
end
end
diff --git a/lib/gitlab/graphql/deprecations/deprecation.rb b/lib/gitlab/graphql/deprecations/deprecation.rb
index 0cf555b0e34..6821c7b0ab9 100644
--- a/lib/gitlab/graphql/deprecations/deprecation.rb
+++ b/lib/gitlab/graphql/deprecations/deprecation.rb
@@ -74,10 +74,10 @@ module Gitlab
end
def edit_description(original_description)
- @original_description = original_description
- return unless original_description
+ @original_description = original_description&.strip
+ return unless @original_description
- original_description + description_suffix
+ @original_description + description_suffix
end
def original_description
diff --git a/lib/gitlab/graphql/pagination/active_record_array_connection.rb b/lib/gitlab/graphql/pagination/active_record_array_connection.rb
index 9e40f79b2fd..ce16693cf89 100644
--- a/lib/gitlab/graphql/pagination/active_record_array_connection.rb
+++ b/lib/gitlab/graphql/pagination/active_record_array_connection.rb
@@ -59,6 +59,7 @@ module Gitlab
def dup
self.class.new(
items.dup,
+ context: context,
first: first,
after: after,
max_page_size: max_page_size,
diff --git a/lib/gitlab/graphql/queries.rb b/lib/gitlab/graphql/queries.rb
index 9cdc84ffaa3..fc569bdc5dc 100644
--- a/lib/gitlab/graphql/queries.rb
+++ b/lib/gitlab/graphql/queries.rb
@@ -5,14 +5,14 @@ require 'find'
module Gitlab
module Graphql
module Queries
- IMPORT_RE = /^#\s*import "(?<path>[^"]+)"$/m.freeze
- EE_ELSE_CE = /^ee_else_ce/.freeze
- HOME_RE = /^~/.freeze
- HOME_EE = %r{^ee/}.freeze
- DOTS_RE = %r{^(\.\./)+}.freeze
- DOT_RE = %r{^\./}.freeze
- IMPLICIT_ROOT = %r{^app/}.freeze
- CONN_DIRECTIVE = /@connection\(key: "\w+"\)/.freeze
+ IMPORT_RE = /^#\s*import "(?<path>[^"]+)"$/m
+ EE_ELSE_CE = /^ee_else_ce/
+ HOME_RE = /^~/
+ HOME_EE = %r{^ee/}
+ DOTS_RE = %r{^(\.\./)+}
+ DOT_RE = %r{^\./}
+ IMPLICIT_ROOT = %r{^app/}
+ CONN_DIRECTIVE = /@connection\(key: "\w+"\)/
class WrappedError
delegate :message, to: :@error
diff --git a/lib/gitlab/harbor/query.rb b/lib/gitlab/harbor/query.rb
index fc0ac539e07..2d03412fde2 100644
--- a/lib/gitlab/harbor/query.rb
+++ b/lib/gitlab/harbor/query.rb
@@ -8,7 +8,7 @@ module Gitlab
attr_reader :client, :repository_id, :artifact_id, :search, :limit, :sort, :page
DEFAULT_LIMIT = 10
- SORT_REGEX = %r{\A(creation_time|update_time|name) (asc|desc)\z}.freeze
+ SORT_REGEX = %r{\A(creation_time|update_time|name) (asc|desc)\z}
validates :page, numericality: { greater_than: 0, integer: true }, allow_blank: true
validates :limit, numericality: { greater_than: 0, less_than_or_equal_to: 25, integer: true }, allow_blank: true
diff --git a/lib/gitlab/hashed_storage/migrator.rb b/lib/gitlab/hashed_storage/migrator.rb
deleted file mode 100644
index 912e2ee99e9..00000000000
--- a/lib/gitlab/hashed_storage/migrator.rb
+++ /dev/null
@@ -1,125 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module HashedStorage
- # Hashed Storage Migrator
- #
- # This is responsible for scheduling and flagging projects
- # to be migrated from Legacy to Hashed storage, either one by one or in bulk.
- class Migrator
- BATCH_SIZE = 100
-
- # Schedule a range of projects to be bulk migrated with #bulk_migrate asynchronously
- #
- # @param [Integer] start first project id for the range
- # @param [Integer] finish last project id for the range
- def bulk_schedule_migration(start:, finish:)
- ::HashedStorage::MigratorWorker.perform_async(start, finish)
- end
-
- # Schedule a range of projects to be bulk rolledback with #bulk_rollback asynchronously
- #
- # @param [Integer] start first project id for the range
- # @param [Integer] finish last project id for the range
- def bulk_schedule_rollback(start:, finish:)
- ::HashedStorage::RollbackerWorker.perform_async(start, finish)
- end
-
- # Start migration of projects from specified range
- #
- # Flagging a project to be migrated is a synchronous action
- # but the migration runs through async jobs
- #
- # @param [Integer] start first project id for the range
- # @param [Integer] finish last project id for the range
- # rubocop: disable CodeReuse/ActiveRecord
- def bulk_migrate(start:, finish:)
- projects = build_relation(start, finish)
-
- projects.with_route.find_each(batch_size: BATCH_SIZE) do |project|
- migrate(project)
- end
- end
- # rubocop: enable CodeReuse/ActiveRecord
-
- # Start rollback of projects from specified range
- #
- # Flagging a project to be rolled back is a synchronous action
- # but the rollback runs through async jobs
- #
- # @param [Integer] start first project id for the range
- # @param [Integer] finish last project id for the range
- # rubocop: disable CodeReuse/ActiveRecord
- def bulk_rollback(start:, finish:)
- projects = build_relation(start, finish)
-
- projects.with_route.find_each(batch_size: BATCH_SIZE) do |project|
- rollback(project)
- end
- end
- # rubocop: enable CodeReuse/ActiveRecord
-
- # Flag a project to be migrated to Hashed Storage
- #
- # @param [Project] project that will be migrated
- def migrate(project)
- Gitlab::AppLogger.info "Starting storage migration of #{project.full_path} (ID=#{project.id})..."
-
- project.migrate_to_hashed_storage!
- rescue StandardError => err
- Gitlab::AppLogger.error("#{err.message} migrating storage of #{project.full_path} (ID=#{project.id}), trace - #{err.backtrace}")
- end
-
- # Flag a project to be rolled-back to Legacy Storage
- #
- # @param [Project] project that will be rolled-back
- def rollback(project)
- Gitlab::AppLogger.info "Starting storage rollback of #{project.full_path} (ID=#{project.id})..."
-
- project.rollback_to_legacy_storage!
- rescue StandardError => err
- Gitlab::AppLogger.error("#{err.message} rolling-back storage of #{project.full_path} (ID=#{project.id}), trace - #{err.backtrace}")
- end
-
- # Returns whether we have any pending storage migration
- #
- def migration_pending?
- any_non_empty_queue?(::HashedStorage::MigratorWorker, ::HashedStorage::ProjectMigrateWorker)
- end
-
- # Returns whether we have any pending storage rollback
- #
- def rollback_pending?
- any_non_empty_queue?(::HashedStorage::RollbackerWorker, ::HashedStorage::ProjectRollbackWorker)
- end
-
- # Remove all remaining scheduled rollback operations
- #
- def abort_rollback!
- [::HashedStorage::RollbackerWorker, ::HashedStorage::ProjectRollbackWorker].each do |worker|
- Sidekiq::Queue.new(worker.queue).clear
- end
- end
-
- private
-
- def any_non_empty_queue?(*workers)
- workers.any? do |worker|
- Sidekiq::Queue.new(worker.queue).size != 0 # rubocop:disable Style/ZeroLengthPredicate
- end
- end
-
- # rubocop: disable CodeReuse/ActiveRecord
- def build_relation(start, finish)
- relation = Project
- table = Project.arel_table
-
- relation = relation.where(table[:id].gteq(start)) if start
- relation = relation.where(table[:id].lteq(finish)) if finish
-
- relation
- end
- # rubocop: enable CodeReuse/ActiveRecord
- end
- end
-end
diff --git a/lib/gitlab/hashed_storage/rake_helper.rb b/lib/gitlab/hashed_storage/rake_helper.rb
deleted file mode 100644
index d3468569e5e..00000000000
--- a/lib/gitlab/hashed_storage/rake_helper.rb
+++ /dev/null
@@ -1,129 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module HashedStorage
- module RakeHelper
- def self.batch_size
- ENV.fetch('BATCH', 200).to_i
- end
-
- def self.listing_limit
- ENV.fetch('LIMIT', 500).to_i
- end
-
- def self.range_from
- ENV['ID_FROM']
- end
-
- def self.range_to
- ENV['ID_TO']
- end
-
- def self.using_ranges?
- !range_from.nil? && !range_to.nil?
- end
-
- def self.range_single_item?
- using_ranges? && range_from == range_to
- end
-
- # rubocop: disable CodeReuse/ActiveRecord
- def self.project_id_batches_migration(&block)
- Project.with_unmigrated_storage.in_batches(of: batch_size, start: range_from, finish: range_to) do |relation| # rubocop: disable Cop/InBatches
- ids = relation.pluck(:id)
-
- yield ids.min, ids.max
- end
- end
- # rubocop: enable CodeReuse/ActiveRecord
-
- # rubocop: disable CodeReuse/ActiveRecord
- def self.project_id_batches_rollback(&block)
- Project.with_storage_feature(:repository).in_batches(of: batch_size, start: range_from, finish: range_to) do |relation| # rubocop: disable Cop/InBatches
- ids = relation.pluck(:id)
-
- yield ids.min, ids.max
- end
- end
- # rubocop: enable CodeReuse/ActiveRecord
-
- def self.legacy_attachments_relation
- Upload.inner_join_local_uploads_projects.merge(Project.without_storage_feature(:attachments))
- end
-
- def self.hashed_attachments_relation
- Upload.inner_join_local_uploads_projects.merge(Project.with_storage_feature(:attachments))
- end
-
- def self.relation_summary(relation_name, relation)
- relation_count = relation.count
- $stdout.puts "* Found #{relation_count} #{relation_name}".color(:green)
-
- relation_count
- end
-
- def self.projects_list(relation_name, relation)
- listing(relation_name, relation.with_route) do |project|
- $stdout.puts " - #{project.full_path} (id: #{project.id})".color(:red)
- $stdout.puts " #{project.repository.disk_path}"
- end
- end
-
- def self.attachments_list(relation_name, relation)
- listing(relation_name, relation) do |upload|
- $stdout.puts " - #{upload.path} (id: #{upload.id})".color(:red)
- end
- end
-
- # rubocop: disable CodeReuse/ActiveRecord
- def self.listing(relation_name, relation)
- relation_count = relation_summary(relation_name, relation)
- return unless relation_count > 0
-
- limit = listing_limit
-
- if relation_count > limit
- $stdout.puts " ! Displaying first #{limit} #{relation_name}..."
- end
-
- relation.find_each(batch_size: batch_size).with_index do |element, index|
- yield element
-
- break if index + 1 >= limit
- end
- end
- # rubocop: enable CodeReuse/ActiveRecord
-
- def self.prune(relation_name, relation, dry_run: true, root: nil)
- root ||= '../repositories'
-
- known_paths = Set.new
- listing(relation_name, relation) { |p| known_paths << "#{root}/#{p.repository.disk_path}" }
-
- marked_for_deletion = Set.new(Dir["#{root}/@hashed/*/*/*"])
- marked_for_deletion.reject! do |path|
- base = path.gsub(/\.(\w+\.)?git$/, '')
- known_paths.include?(base)
- end
-
- if marked_for_deletion.empty?
- $stdout.puts "No orphaned directories found. Nothing to do!"
- else
- n = marked_for_deletion.size
- $stdout.puts "Found #{n} orphaned #{'directory'.pluralize(n)}"
- $stdout.puts "Dry run. (Run again with FORCE=1 to delete). We would have deleted:" if dry_run
- end
-
- marked_for_deletion.each do |p|
- p = Pathname.new(p)
- if dry_run
- $stdout.puts " - #{p}"
- else
- $stdout.puts "Removing #{p}"
- p.rmtree
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/health_checks/puma_check.rb b/lib/gitlab/health_checks/puma_check.rb
index 2dc8a093572..efe3d65db91 100644
--- a/lib/gitlab/health_checks/puma_check.rb
+++ b/lib/gitlab/health_checks/puma_check.rb
@@ -20,7 +20,7 @@ module Gitlab
def check
return unless Gitlab::Runtime.puma?
- stats = Puma.stats
+ stats = ::Puma.stats
stats = Gitlab::Json.parse(stats)
# If `workers` is missing this means that
diff --git a/lib/gitlab/http.rb b/lib/gitlab/http.rb
index feb54fcca0c..9c9816e142e 100644
--- a/lib/gitlab/http.rb
+++ b/lib/gitlab/http.rb
@@ -10,21 +10,14 @@ require_relative 'http_connection_adapter'
module Gitlab
class HTTP
- BlockedUrlError = Class.new(StandardError)
- RedirectionTooDeep = Class.new(StandardError)
- ReadTotalTimeout = Class.new(Net::ReadTimeout)
- HeaderReadTimeout = Class.new(Net::ReadTimeout)
- SilentModeBlockedError = Class.new(StandardError)
+ BlockedUrlError = Gitlab::HTTP_V2::BlockedUrlError
+ RedirectionTooDeep = Gitlab::HTTP_V2::RedirectionTooDeep
+ ReadTotalTimeout = Gitlab::HTTP_V2::ReadTotalTimeout
+ HeaderReadTimeout = Gitlab::HTTP_V2::HeaderReadTimeout
+ SilentModeBlockedError = Gitlab::HTTP_V2::SilentModeBlockedError
- HTTP_TIMEOUT_ERRORS = [
- Net::OpenTimeout, Net::ReadTimeout, Net::WriteTimeout, Gitlab::HTTP::ReadTotalTimeout
- ].freeze
- HTTP_ERRORS = HTTP_TIMEOUT_ERRORS + [
- EOFError, SocketError, OpenSSL::SSL::SSLError, OpenSSL::OpenSSLError,
- Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ENETUNREACH,
- Gitlab::HTTP::BlockedUrlError, Gitlab::HTTP::RedirectionTooDeep,
- Net::HTTPBadResponse
- ].freeze
+ HTTP_TIMEOUT_ERRORS = Gitlab::HTTP_V2::HTTP_TIMEOUT_ERRORS
+ HTTP_ERRORS = Gitlab::HTTP_V2::HTTP_ERRORS
DEFAULT_TIMEOUT_OPTIONS = {
open_timeout: 10,
@@ -40,70 +33,63 @@ module Gitlab
Net::HTTP::Trace
].freeze
- include HTTParty # rubocop:disable Gitlab/HTTParty
+ # We are explicitly assigning these constants because they are used in the codebase.
+ Error = HTTParty::Error
+ Response = HTTParty::Response
+ ResponseError = HTTParty::ResponseError
+ CookieHash = HTTParty::CookieHash
class << self
- alias_method :httparty_perform_request, :perform_request
- end
-
- connection_adapter ::Gitlab::HTTPConnectionAdapter
-
- def self.perform_request(http_method, path, options, &block)
- raise_if_blocked_by_silent_mode(http_method)
-
- log_info = options.delete(:extra_log_info)
- options_with_timeouts =
- if !options.has_key?(:timeout)
- options.with_defaults(DEFAULT_TIMEOUT_OPTIONS)
- else
- options
+ ::Gitlab::HTTP_V2::SUPPORTED_HTTP_METHODS.each do |method|
+ define_method(method) do |path, options = {}, &block|
+ if ::Feature.enabled?(:use_gitlab_http_v2, Feature.current_request)
+ ::Gitlab::HTTP_V2.public_send(method, path, http_v2_options(options), &block) # rubocop:disable GitlabSecurity/PublicSend
+ else
+ ::Gitlab::LegacyHTTP.public_send(method, path, options, &block) # rubocop:disable GitlabSecurity/PublicSend
+ end
end
+ end
- if options[:stream_body]
- return httparty_perform_request(http_method, path, options_with_timeouts, &block)
+ def try_get(path, options = {}, &block)
+ get(path, options, &block)
+ rescue *HTTP_ERRORS
+ nil
end
- start_time = nil
- read_total_timeout = options.fetch(:timeout, DEFAULT_READ_TOTAL_TIMEOUT)
+ # TODO: This method is subject to be removed
+ # We have this for now because we explicitly use the `perform_request` method in some places.
+ def perform_request(http_method, path, options, &block)
+ if ::Feature.enabled?(:use_gitlab_http_v2, Feature.current_request)
+ method_name = http_method::METHOD.downcase.to_sym
- httparty_perform_request(http_method, path, options_with_timeouts) do |fragment|
- start_time ||= Gitlab::Metrics::System.monotonic_time
- elapsed = Gitlab::Metrics::System.monotonic_time - start_time
+ unless ::Gitlab::HTTP_V2::SUPPORTED_HTTP_METHODS.include?(method_name)
+ raise ArgumentError, "Unsupported HTTP method: '#{method_name}'."
+ end
- if elapsed > read_total_timeout
- raise ReadTotalTimeout, "Request timed out after #{elapsed} seconds"
+ # Use `::Gitlab::HTTP_V2.get/post/...` methods
+ ::Gitlab::HTTP_V2.public_send(method_name, path, http_v2_options(options), &block) # rubocop:disable GitlabSecurity/PublicSend
+ else
+ ::Gitlab::LegacyHTTP.perform_request(http_method, path, options, &block)
end
-
- yield fragment if block
end
- rescue HTTParty::RedirectionTooDeep
- raise RedirectionTooDeep
- rescue *HTTP_ERRORS => e
- extra_info = log_info || {}
- extra_info = log_info.call(e, path, options) if log_info.respond_to?(:call)
- Gitlab::ErrorTracking.log_exception(e, extra_info)
- raise e
- end
-
- def self.try_get(path, options = {}, &block)
- self.get(path, options, &block)
- rescue *HTTP_ERRORS
- nil
- end
- def self.raise_if_blocked_by_silent_mode(http_method)
- return unless blocked_by_silent_mode?(http_method)
+ private
- ::Gitlab::SilentMode.log_info(
- message: 'Outbound HTTP request blocked',
- outbound_http_request_method: http_method.to_s
- )
-
- raise SilentModeBlockedError, 'only get, head, options, and trace methods are allowed in silent mode'
- end
+ def http_v2_options(options)
+ # TODO: until we remove `allow_object_storage` from all places.
+ if options.delete(:allow_object_storage)
+ options[:extra_allowed_uris] = ObjectStoreSettings.enabled_endpoint_uris
+ end
- def self.blocked_by_silent_mode?(http_method)
- ::Gitlab::SilentMode.enabled? && SILENT_MODE_ALLOWED_METHODS.exclude?(http_method)
+ # Configure HTTP_V2 Client
+ {
+ allow_local_requests: Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services?,
+ deny_all_requests_except_allowed: Gitlab::CurrentSettings.deny_all_requests_except_allowed?,
+ dns_rebinding_protection_enabled: Gitlab::CurrentSettings.dns_rebinding_protection_enabled?,
+ outbound_local_requests_allowlist: Gitlab::CurrentSettings.outbound_local_requests_whitelist, # rubocop:disable Naming/InclusiveLanguage
+ silent_mode_enabled: Gitlab::SilentMode.enabled?
+ }.merge(options)
+ end
end
end
end
diff --git a/lib/gitlab/http_connection_adapter.rb b/lib/gitlab/http_connection_adapter.rb
index 822b8a9f8d9..8e9a63a9f7f 100644
--- a/lib/gitlab/http_connection_adapter.rb
+++ b/lib/gitlab/http_connection_adapter.rb
@@ -56,7 +56,7 @@ module Gitlab
allow_object_storage: allow_object_storage?,
dns_rebind_protection: dns_rebind_protection?,
schemes: %w[http https])
- rescue Gitlab::UrlBlocker::BlockedUrlError => e
+ rescue Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError => e
raise Gitlab::HTTP::BlockedUrlError, "URL is blocked: #{e.message}"
end
diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb
index f8e7e66a8a5..96e3d90c139 100644
--- a/lib/gitlab/i18n.rb
+++ b/lib/gitlab/i18n.rb
@@ -45,7 +45,7 @@ module Gitlab
'bg' => 0,
'cs_CZ' => 0,
'da_DK' => 29,
- 'de' => 96,
+ 'de' => 97,
'en' => 100,
'eo' => 0,
'es' => 28,
@@ -54,20 +54,20 @@ module Gitlab
'gl_ES' => 0,
'id_ID' => 0,
'it' => 1,
- 'ja' => 99,
+ 'ja' => 98,
'ko' => 23,
'nb_NO' => 21,
'nl_NL' => 0,
'pl_PL' => 3,
- 'pt_BR' => 55,
+ 'pt_BR' => 57,
'ro_RO' => 76,
- 'ru' => 22,
+ 'ru' => 21,
'si_LK' => 12,
'tr_TR' => 8,
- 'uk' => 51,
+ 'uk' => 52,
'zh_CN' => 99,
'zh_HK' => 1,
- 'zh_TW' => 99
+ 'zh_TW' => 100
}.freeze
private_constant :TRANSLATION_LEVELS
@@ -81,6 +81,13 @@ module Gitlab
TRANSLATION_LEVELS.fetch(code, 0)
end
+ def trimmed_language_name(code)
+ language_name = AVAILABLE_LANGUAGES[code]
+ return if language_name.blank?
+
+ language_name.sub(/\s-\s.*/, '')
+ end
+
def available_locales
AVAILABLE_LANGUAGES.keys
end
diff --git a/lib/gitlab/i18n/po_linter.rb b/lib/gitlab/i18n/po_linter.rb
index 3ad01ef2257..8092b3c72ea 100644
--- a/lib/gitlab/i18n/po_linter.rb
+++ b/lib/gitlab/i18n/po_linter.rb
@@ -9,7 +9,7 @@ module Gitlab
attr_reader :po_path, :translation_entries, :metadata_entry, :locale
- VARIABLE_REGEX = /%{\w*}|%[a-z]/.freeze
+ VARIABLE_REGEX = /%{\w*}|%[a-z]/
def initialize(po_path:, locale: I18n.locale.to_s)
@po_path = po_path
diff --git a/lib/gitlab/i18n/translation_entry.rb b/lib/gitlab/i18n/translation_entry.rb
index 6623d42f526..6f4451d42ee 100644
--- a/lib/gitlab/i18n/translation_entry.rb
+++ b/lib/gitlab/i18n/translation_entry.rb
@@ -3,8 +3,8 @@
module Gitlab
module I18n
class TranslationEntry
- PERCENT_REGEX = /(?:^|[^%])%(?!{\w*}|[a-z%])/.freeze
- ANGLE_BRACKET_REGEX = /[<>]/.freeze
+ PERCENT_REGEX = /(?:^|[^%])%(?!{\w*}|[a-z%])/
+ ANGLE_BRACKET_REGEX = /[<>]/
attr_reader :nplurals, :entry_data
diff --git a/lib/gitlab/import/import_failure_service.rb b/lib/gitlab/import/import_failure_service.rb
index 714d9b3edd9..a8ccf24b780 100644
--- a/lib/gitlab/import/import_failure_service.rb
+++ b/lib/gitlab/import/import_failure_service.rb
@@ -9,7 +9,8 @@ module Gitlab
project_id: nil,
error_source: nil,
fail_import: false,
- metrics: false
+ metrics: false,
+ external_identifiers: {}
)
new(
exception: exception,
@@ -17,7 +18,8 @@ module Gitlab
project_id: project_id,
error_source: error_source,
fail_import: fail_import,
- metrics: metrics
+ metrics: metrics,
+ external_identifiers: external_identifiers
).execute
end
@@ -27,7 +29,8 @@ module Gitlab
project_id: nil,
error_source: nil,
fail_import: false,
- metrics: false
+ metrics: false,
+ external_identifiers: {}
)
if import_state.blank? && project_id.blank?
@@ -46,6 +49,7 @@ module Gitlab
@error_source = error_source
@fail_import = fail_import
@metrics = metrics
+ @external_identifiers = external_identifiers
end
def execute
@@ -58,19 +62,20 @@ module Gitlab
private
- attr_reader :exception, :import_state, :project, :error_source, :fail_import, :metrics
+ attr_reader :exception, :import_state, :project, :error_source, :fail_import, :metrics, :external_identifiers
def track_exception
attributes = {
import_type: project.import_type,
project_id: project.id,
- source: error_source
+ source: error_source,
+ external_identifiers: external_identifiers
}
Gitlab::Import::Logger.error(
attributes.merge(
message: 'importer failed',
- 'error.message': exception.message
+ 'exception.message': exception.message
)
)
@@ -85,7 +90,8 @@ module Gitlab
exception_class: exception.class.to_s,
exception_message: exception.message.truncate(255),
correlation_id_value: Labkit::Correlation::CorrelationId.current_or_new_id,
- retry_count: fail_import ? 0 : nil
+ retry_count: fail_import ? 0 : nil,
+ external_identifiers: external_identifiers
)
end
diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml
index 7130fd8d7d6..6f3601e9a21 100644
--- a/lib/gitlab/import_export/project/import_export.yml
+++ b/lib/gitlab/import_export/project/import_export.yml
@@ -172,6 +172,7 @@ included_attributes:
- :project_id
- :ref
- :updated_at
+ - :owner_id
error_tracking_setting:
- :api_url
- :organization_name
diff --git a/lib/gitlab/import_export/project/relation_factory.rb b/lib/gitlab/import_export/project/relation_factory.rb
index 78d0735bbb5..943c997a056 100644
--- a/lib/gitlab/import_export/project/relation_factory.rb
+++ b/lib/gitlab/import_export/project/relation_factory.rb
@@ -194,6 +194,7 @@ module Gitlab
def setup_pipeline_schedule
@relation_hash['active'] = false
+ @relation_hash['owner_id'] = @user.id
end
def setup_merge_request
diff --git a/lib/gitlab/internal_events.rb b/lib/gitlab/internal_events.rb
index 92bf2a826ff..2790bc8ee24 100644
--- a/lib/gitlab/internal_events.rb
+++ b/lib/gitlab/internal_events.rb
@@ -9,10 +9,31 @@ module Gitlab
class << self
include Gitlab::Tracking::Helpers
- def track_event(event_name, **kwargs)
+ def track_event(event_name, send_snowplow_event: true, **kwargs)
raise UnknownEventError, "Unknown event: #{event_name}" unless EventDefinitions.known_event?(event_name)
+ increase_total_counter(event_name)
+ update_unique_counter(event_name, kwargs)
+ trigger_snowplow_event(event_name, kwargs) if send_snowplow_event
+ rescue StandardError => e
+ Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, event_name: event_name, kwargs: kwargs)
+ nil
+ end
+
+ private
+
+ def increase_total_counter(event_name)
+ return unless ::ServicePing::ServicePingSettings.enabled?
+
+ redis_counter_key =
+ Gitlab::Usage::Metrics::Instrumentations::TotalCountMetric.redis_key(event_name)
+ Gitlab::Redis::SharedState.with { |redis| redis.incr(redis_counter_key) }
+ end
+
+ def update_unique_counter(event_name, kwargs)
unique_property = EventDefinitions.unique_property(event_name)
+ return unless unique_property
+
unique_method = :id
unless kwargs.has_key?(unique_property)
@@ -26,7 +47,9 @@ module Gitlab
unique_value = kwargs[unique_property].public_send(unique_method) # rubocop:disable GitlabSecurity/PublicSend
UsageDataCounters::HLLRedisCounter.track_event(event_name, values: unique_value)
+ end
+ def trigger_snowplow_event(event_name, kwargs)
user = kwargs[:user]
project = kwargs[:project]
namespace = kwargs[:namespace]
@@ -44,13 +67,8 @@ module Gitlab
).to_context
track_struct_event(event_name, contexts: [standard_context, service_ping_context])
- rescue StandardError => e
- Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, event_name: event_name, kwargs: kwargs)
- nil
end
- private
-
def track_struct_event(event_name, contexts:)
category = 'InternalEventTracking'
tracker = Gitlab::Tracking.tracker
diff --git a/lib/gitlab/internal_events/event_definitions.rb b/lib/gitlab/internal_events/event_definitions.rb
index f3c8092bcb0..9fd58ae0cb0 100644
--- a/lib/gitlab/internal_events/event_definitions.rb
+++ b/lib/gitlab/internal_events/event_definitions.rb
@@ -16,7 +16,7 @@ module Gitlab
def unique_property(event_name)
unique_value = events[event_name]&.to_s
- raise(InvalidMetricConfiguration, "Unique property not defined for #{event_name}") unless unique_value
+ return unless unique_value
unless VALID_UNIQUE_VALUES.include?(unique_value)
raise(InvalidMetricConfiguration, "Invalid unique value '#{unique_value}' for #{event_name}")
@@ -32,7 +32,7 @@ module Gitlab
private
def events
- load_configurations if @events.nil? || Gitlab::Usage::MetricDefinition.metric_definitions_changed?
+ load_configurations if @events.nil?
@events
end
diff --git a/lib/gitlab/jira/dvcs.rb b/lib/gitlab/jira/dvcs.rb
index 41a039674b3..020c31fa281 100644
--- a/lib/gitlab/jira/dvcs.rb
+++ b/lib/gitlab/jira/dvcs.rb
@@ -5,7 +5,7 @@ module Gitlab
module Dvcs
ENCODED_SLASH = '@'
SLASH = '/'
- ENCODED_ROUTE_REGEX = /[a-zA-Z0-9_\-\.#{ENCODED_SLASH}]+/.freeze
+ ENCODED_ROUTE_REGEX = /[a-zA-Z0-9_\-\.#{ENCODED_SLASH}]+/
def self.encode_slash(path)
path.gsub(SLASH, ENCODED_SLASH)
diff --git a/lib/gitlab/legacy_http.rb b/lib/gitlab/legacy_http.rb
new file mode 100644
index 00000000000..f38b2819c15
--- /dev/null
+++ b/lib/gitlab/legacy_http.rb
@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+#
+# IMPORTANT: With the new development of the 'gitlab-http' gem (https://gitlab.com/gitlab-org/gitlab/-/issues/415686),
+# no additional change should be implemented in this class. This class will be removed after migrating all
+# the usages to the new gem.
+#
+
+require_relative 'http_connection_adapter'
+
+module Gitlab
+ class LegacyHTTP # rubocop:disable Gitlab/NamespacedClass
+ include HTTParty # rubocop:disable Gitlab/HTTParty
+
+ class << self
+ alias_method :httparty_perform_request, :perform_request
+ end
+
+ connection_adapter ::Gitlab::HTTPConnectionAdapter
+
+ def self.perform_request(http_method, path, options, &block)
+ raise_if_blocked_by_silent_mode(http_method)
+
+ log_info = options.delete(:extra_log_info)
+ options_with_timeouts =
+ if !options.has_key?(:timeout)
+ options.with_defaults(Gitlab::HTTP::DEFAULT_TIMEOUT_OPTIONS)
+ else
+ options
+ end
+
+ return httparty_perform_request(http_method, path, options_with_timeouts, &block) if options[:stream_body]
+
+ start_time = nil
+ read_total_timeout = options.fetch(:timeout, Gitlab::HTTP::DEFAULT_READ_TOTAL_TIMEOUT)
+
+ httparty_perform_request(http_method, path, options_with_timeouts) do |fragment|
+ start_time ||= Gitlab::Metrics::System.monotonic_time
+ elapsed = Gitlab::Metrics::System.monotonic_time - start_time
+
+ if elapsed > read_total_timeout
+ raise Gitlab::HTTP::ReadTotalTimeout, "Request timed out after #{elapsed} seconds"
+ end
+
+ yield fragment if block
+ end
+ rescue HTTParty::RedirectionTooDeep
+ raise Gitlab::HTTP::RedirectionTooDeep
+ rescue *Gitlab::HTTP::HTTP_ERRORS => e
+ extra_info = log_info || {}
+ extra_info = log_info.call(e, path, options) if log_info.respond_to?(:call)
+ Gitlab::ErrorTracking.log_exception(e, extra_info)
+ raise e
+ end
+
+ def self.try_get(path, options = {}, &block)
+ self.get(path, options, &block) # rubocop:disable Style/RedundantSelf
+ rescue *Gitlab::HTTP::HTTP_ERRORS
+ nil
+ end
+
+ def self.raise_if_blocked_by_silent_mode(http_method)
+ return unless blocked_by_silent_mode?(http_method)
+
+ ::Gitlab::SilentMode.log_info(
+ message: 'Outbound HTTP request blocked',
+ outbound_http_request_method: http_method.to_s
+ )
+
+ raise Gitlab::HTTP::SilentModeBlockedError,
+ 'only get, head, options, and trace methods are allowed in silent mode'
+ end
+
+ def self.blocked_by_silent_mode?(http_method)
+ ::Gitlab::SilentMode.enabled? && Gitlab::HTTP::SILENT_MODE_ALLOWED_METHODS.exclude?(http_method)
+ end
+ end
+end
diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb
index 5f760e764c8..426d449abbe 100644
--- a/lib/gitlab/mail_room.rb
+++ b/lib/gitlab/mail_room.rb
@@ -11,20 +11,6 @@ require_relative 'redis/queues' unless defined?(Gitlab::Redis::Queues)
# This service is run independently of the main Rails process,
# therefore the `Rails` class and its methods are unavailable.
-# TODO: Remove this once we're on Ruby 3
-# https://gitlab.com/gitlab-org/gitlab/-/issues/393651
-unless YAML.respond_to?(:safe_load_file)
- module YAML
- # Temporary Ruby 2 back-compat workaround.
- #
- # This method only exists as of stdlib 3.0.0:
- # https://ruby-doc.org/stdlib-3.0.0/libdoc/psych/rdoc/Psych.html
- def self.safe_load_file(path, **options)
- YAML.safe_load(File.read(path), **options)
- end
- end
-end
-
module Gitlab
module MailRoom
RAILS_ROOT_DIR = Pathname.new('../..').expand_path(__dir__).freeze
diff --git a/lib/gitlab/merge_requests/mergeability/check_result.rb b/lib/gitlab/merge_requests/mergeability/check_result.rb
index fae4b721e1a..e18909d8f17 100644
--- a/lib/gitlab/merge_requests/mergeability/check_result.rb
+++ b/lib/gitlab/merge_requests/mergeability/check_result.rb
@@ -35,6 +35,10 @@ module Gitlab
{ status: status, payload: payload }
end
+ def identifier
+ payload&.fetch(:identifier)&.to_sym
+ end
+
def failed?
status == FAILED_STATUS
end
diff --git a/lib/gitlab/merge_requests/message_generator.rb b/lib/gitlab/merge_requests/message_generator.rb
index 5ca26fdae86..2307d4fc64e 100644
--- a/lib/gitlab/merge_requests/message_generator.rb
+++ b/lib/gitlab/merge_requests/message_generator.rb
@@ -111,7 +111,7 @@ module Gitlab
all_commits
].freeze
- PLACEHOLDERS_COMBINED_REGEX = /%{(#{Regexp.union(PLACEHOLDERS.keys)})}/.freeze
+ PLACEHOLDERS_COMBINED_REGEX = /%{(#{Regexp.union(PLACEHOLDERS.keys)})}/
def replace_placeholders(message, allowed_placeholders: [], squash: false, keep_carriage_return: false)
# Convert CRLF to LF.
diff --git a/lib/gitlab/metrics/exporter/base_exporter.rb b/lib/gitlab/metrics/exporter/base_exporter.rb
index 858a0a120cc..e22b9c2a761 100644
--- a/lib/gitlab/metrics/exporter/base_exporter.rb
+++ b/lib/gitlab/metrics/exporter/base_exporter.rb
@@ -7,7 +7,7 @@ module Gitlab
module Metrics
module Exporter
class BaseExporter < Daemon
- CERT_REGEX = /-----BEGIN CERTIFICATE-----(?:.|\n)+?-----END CERTIFICATE-----/.freeze
+ CERT_REGEX = /-----BEGIN CERTIFICATE-----(?:.|\n)+?-----END CERTIFICATE-----/
attr_reader :server
diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb
index c299fa37e7a..b00459de17e 100644
--- a/lib/gitlab/metrics/requests_rack_middleware.rb
+++ b/lib/gitlab/metrics/requests_rack_middleware.rb
@@ -13,7 +13,7 @@ module Gitlab
"put" => %w[200 202 204 400 401 403 404 405 406 409 410 422 500]
}.freeze
- HEALTH_ENDPOINT = %r{^/-/(liveness|readiness|health|metrics)/?$}.freeze
+ HEALTH_ENDPOINT = %r{^/-/(liveness|readiness|health|metrics)/?$}
FEATURE_CATEGORY_DEFAULT = ::Gitlab::FeatureCategories::FEATURE_CATEGORY_DEFAULT
ENDPOINT_MISSING = 'unknown'
diff --git a/lib/gitlab/metrics/samplers/puma_sampler.rb b/lib/gitlab/metrics/samplers/puma_sampler.rb
index d818aa43853..c806db09e66 100644
--- a/lib/gitlab/metrics/samplers/puma_sampler.rb
+++ b/lib/gitlab/metrics/samplers/puma_sampler.rb
@@ -40,7 +40,7 @@ module Gitlab
private
def puma_stats
- Puma.stats
+ ::Puma.stats
rescue NoMethodError
Gitlab::AppLogger.info "PumaSampler: stats are not available yet, waiting for Puma to boot"
nil
diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb
index f9749b65888..c7c54efc50e 100644
--- a/lib/gitlab/metrics/subscribers/active_record.rb
+++ b/lib/gitlab/metrics/subscribers/active_record.rb
@@ -10,7 +10,7 @@ module Gitlab
attach_to :active_record
DB_COUNTERS = %i[count write_count cached_count].freeze
- SQL_COMMANDS_WITH_COMMENTS_REGEX = %r{\A(/\*.*\*/\s)?((?!(.*[^\w'"](DELETE|UPDATE|INSERT INTO)[^\w'"])))(WITH.*)?(SELECT)((?!(FOR UPDATE|FOR SHARE)).)*$}i.freeze
+ SQL_COMMANDS_WITH_COMMENTS_REGEX = %r{\A(/\*.*\*/\s)?((?!(.*[^\w'"](DELETE|UPDATE|INSERT INTO)[^\w'"])))(WITH.*)?(SELECT)((?!(FOR UPDATE|FOR SHARE)).)*$}i
SQL_DURATION_BUCKET = [0.05, 0.1, 0.25].freeze
TRANSACTION_DURATION_BUCKET = [0.1, 0.25, 1].freeze
@@ -19,7 +19,7 @@ module Gitlab
DB_LOAD_BALANCING_COUNTERS = %i[count cached_count wal_count wal_cached_count].freeze
DB_LOAD_BALANCING_DURATIONS = %i[duration_s].freeze
- SQL_WAL_LOCATION_REGEX = /(pg_current_wal_insert_lsn\(\)::text|pg_last_wal_replay_lsn\(\)::text)/.freeze
+ SQL_WAL_LOCATION_REGEX = /(pg_current_wal_insert_lsn\(\)::text|pg_last_wal_replay_lsn\(\)::text)/
# This event is published from ActiveRecordBaseTransactionMetrics and
# used to record a database transaction duration when calling
diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb
index 9b0ae84dec2..80ce155321b 100644
--- a/lib/gitlab/metrics/system.rb
+++ b/lib/gitlab/metrics/system.rb
@@ -16,13 +16,13 @@ module Gitlab
PROC_FD_GLOB = '/proc/self/fd/*'
PROC_MEM_INFO = '/proc/meminfo'
- PRIVATE_PAGES_PATTERN = /^(Private_Clean|Private_Dirty|Private_Hugetlb):\s+(?<value>\d+)/.freeze
- PSS_PATTERN = /^Pss:\s+(?<value>\d+)/.freeze
- RSS_TOTAL_PATTERN = /^VmRSS:\s+(?<value>\d+)/.freeze
- RSS_ANON_PATTERN = /^RssAnon:\s+(?<value>\d+)/.freeze
- RSS_FILE_PATTERN = /^RssFile:\s+(?<value>\d+)/.freeze
- MAX_OPEN_FILES_PATTERN = /Max open files\s*(?<value>\d+)/.freeze
- MEM_TOTAL_PATTERN = /^MemTotal:\s+(?<value>\d+) (.+)/.freeze
+ PRIVATE_PAGES_PATTERN = /^(Private_Clean|Private_Dirty|Private_Hugetlb):\s+(?<value>\d+)/
+ PSS_PATTERN = /^Pss:\s+(?<value>\d+)/
+ RSS_TOTAL_PATTERN = /^VmRSS:\s+(?<value>\d+)/
+ RSS_ANON_PATTERN = /^RssAnon:\s+(?<value>\d+)/
+ RSS_FILE_PATTERN = /^RssFile:\s+(?<value>\d+)/
+ MAX_OPEN_FILES_PATTERN = /Max open files\s*(?<value>\d+)/
+ MEM_TOTAL_PATTERN = /^MemTotal:\s+(?<value>\d+) (.+)/
def summary
proportional_mem = memory_usage_uss_pss
diff --git a/lib/gitlab/metrics/web_transaction.rb b/lib/gitlab/metrics/web_transaction.rb
index f3c1e6897af..ad635a9e376 100644
--- a/lib/gitlab/metrics/web_transaction.rb
+++ b/lib/gitlab/metrics/web_transaction.rb
@@ -9,7 +9,7 @@ module Gitlab
# etc.
class WebTransaction < Transaction
THREAD_KEY = :_gitlab_metrics_transaction
- BASE_LABEL_KEYS = %i[controller action feature_category].freeze
+ BASE_LABEL_KEYS = %i[controller action feature_category endpoint_id].freeze
CONTROLLER_KEY = 'action_controller.instance'
ENDPOINT_KEY = 'api.endpoint'
@@ -95,7 +95,13 @@ module Gitlab
action = "#{action}.#{suffix}"
end
- { controller: controller.class.name, action: action, feature_category: feature_category }
+ {
+ controller: controller.class.name,
+ action: action,
+ feature_category: feature_category,
+ # inline endpoint_id_for_action as not all controllers extend ApplicationController
+ endpoint_id: "#{controller.class.name}##{controller.action_name}"
+ }
end
def labels_from_endpoint
@@ -112,7 +118,12 @@ module Gitlab
if route
path = endpoint_paths_cache[route.request_method][route.path]
- { controller: 'Grape', action: "#{route.request_method} #{path}", feature_category: feature_category }
+ {
+ controller: 'Grape',
+ action: "#{route.request_method} #{path}",
+ feature_category: feature_category,
+ endpoint_id: API::Base.endpoint_id_for_route(route)
+ }
end
end
diff --git a/lib/gitlab/middleware/compressed_json.rb b/lib/gitlab/middleware/compressed_json.rb
index 1f15f1d5857..1131e34b73c 100644
--- a/lib/gitlab/middleware/compressed_json.rb
+++ b/lib/gitlab/middleware/compressed_json.rb
@@ -6,23 +6,23 @@ module Gitlab
INSTANCE_PACKAGES_PATH = %r{
\A/api/v4/packages/npm/-/npm/v1/security/
(?:(?:advisories/bulk)|(?:audits/quick))\z (?# end)
- }xi.freeze
+ }xi
GROUP_PACKAGES_PATH = %r{
\A/api/v4/groups/
(?<id>
[a-zA-Z0-9%-._]{1,255}
)/-/packages/npm/-/npm/v1/security/
(?:(?:advisories/bulk)|(?:audits/quick))\z (?# end)
- }xi.freeze
+ }xi
PROJECT_PACKAGES_PATH = %r{
\A/api/v4/projects/
(?<id>
[a-zA-Z0-9%-._]{1,255}
)/packages/npm/-/npm/v1/security/
(?:(?:advisories/bulk)|(?:audits/quick))\z (?# end)
- }xi.freeze
+ }xi
MAXIMUM_BODY_SIZE = 200.kilobytes.to_i
- UNSAFE_CHARACTERS = %r{[!"#&'()*+,./:;<>=?@\[\]^`{}|~$]}xi.freeze
+ UNSAFE_CHARACTERS = %r{[!"#&'()*+,./:;<>=?@\[\]^`{}|~$]}xi
def initialize(app)
@app = app
diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb
index 4da5fef9fd7..d2336ec4bb2 100644
--- a/lib/gitlab/middleware/go.rb
+++ b/lib/gitlab/middleware/go.rb
@@ -8,7 +8,7 @@ module Gitlab
include ActionView::Helpers::TagHelper
include ActionController::HttpAuthentication::Basic
- PROJECT_PATH_REGEX = %r{\A(#{Gitlab::PathRegex.full_namespace_route_regex}/#{Gitlab::PathRegex.project_route_regex})/}.freeze
+ PROJECT_PATH_REGEX = %r{\A(#{Gitlab::PathRegex.full_namespace_route_regex}/#{Gitlab::PathRegex.project_route_regex})/}
def initialize(app)
@app = app
diff --git a/lib/gitlab/middleware/handle_malformed_strings.rb b/lib/gitlab/middleware/handle_malformed_strings.rb
index b966395ee32..e0f38b63cc1 100644
--- a/lib/gitlab/middleware/handle_malformed_strings.rb
+++ b/lib/gitlab/middleware/handle_malformed_strings.rb
@@ -37,7 +37,7 @@ module Gitlab
request.params.values.any? do |value|
param_has_null_byte?(value)
end
- rescue ActionController::BadRequest
+ rescue ActionController::BadRequest, ActionDispatch::Http::Parameters::ParseError
# If we can't build an ActionDispatch::Request something's wrong
# This would also happen if `#params` contains invalid UTF-8
# in this case we'll return a 400
diff --git a/lib/gitlab/middleware/path_traversal_check.rb b/lib/gitlab/middleware/path_traversal_check.rb
new file mode 100644
index 00000000000..79465f3cb30
--- /dev/null
+++ b/lib/gitlab/middleware/path_traversal_check.rb
@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Middleware
+ class PathTraversalCheck
+ PATH_TRAVERSAL_MESSAGE = 'Potential path traversal attempt detected'
+
+ def initialize(app)
+ @app = app
+ end
+
+ def call(env)
+ if Feature.enabled?(:check_path_traversal_middleware, Feature.current_request)
+ log_params = {}
+
+ execution_time = measure_execution_time do
+ check(env, log_params)
+ end
+
+ log_params[:duration_ms] = execution_time.round(5) if execution_time
+
+ log(log_params) unless log_params.empty?
+ end
+
+ @app.call(env)
+ end
+
+ private
+
+ def measure_execution_time(&blk)
+ if Feature.enabled?(:log_execution_time_path_traversal_middleware, Feature.current_request)
+ Benchmark.ms(&blk)
+ else
+ yield
+
+ nil
+ end
+ end
+
+ def check(env, log_params)
+ request = ::Rack::Request.new(env)
+ fullpath = request.fullpath
+ decoded_fullpath = CGI.unescape(fullpath)
+ ::Gitlab::PathTraversal.check_path_traversal!(decoded_fullpath, skip_decoding: true)
+
+ rescue ::Gitlab::PathTraversal::PathTraversalAttackError
+ log_params[:fullpath] = fullpath
+ log_params[:message] = PATH_TRAVERSAL_MESSAGE
+ end
+
+ def log(payload)
+ Gitlab::AppLogger.warn(
+ payload.merge(class_name: self.class.name)
+ )
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/middleware/read_only.rb b/lib/gitlab/middleware/read_only.rb
index 8e17073abab..83c52a6c6e0 100644
--- a/lib/gitlab/middleware/read_only.rb
+++ b/lib/gitlab/middleware/read_only.rb
@@ -3,7 +3,7 @@
module Gitlab
module Middleware
class ReadOnly
- API_VERSIONS = (3..4).freeze
+ API_VERSIONS = (3..4)
def self.internal_routes
@internal_routes ||=
diff --git a/lib/gitlab/middleware/sidekiq_web_static.rb b/lib/gitlab/middleware/sidekiq_web_static.rb
index c5d2ecbe00e..7640c02fbb5 100644
--- a/lib/gitlab/middleware/sidekiq_web_static.rb
+++ b/lib/gitlab/middleware/sidekiq_web_static.rb
@@ -8,7 +8,7 @@
module Gitlab
module Middleware
class SidekiqWebStatic
- SIDEKIQ_REGEX = %r{\A/admin/sidekiq/}.freeze
+ SIDEKIQ_REGEX = %r{\A/admin/sidekiq/}
def initialize(app)
@app = app
diff --git a/lib/gitlab/middleware/static.rb b/lib/gitlab/middleware/static.rb
index 324d929a93d..e6e36de175d 100644
--- a/lib/gitlab/middleware/static.rb
+++ b/lib/gitlab/middleware/static.rb
@@ -3,7 +3,7 @@
module Gitlab
module Middleware
class Static < ActionDispatch::Static
- UPLOADS_REGEX = %r{\A/uploads(/|\z)}.freeze
+ UPLOADS_REGEX = %r{\A/uploads(/|\z)}
def call(env)
return @app.call(env) if UPLOADS_REGEX.match?(env['PATH_INFO'])
diff --git a/lib/gitlab/observability.rb b/lib/gitlab/observability.rb
index a4e18cc170b..d42d10cd0f4 100644
--- a/lib/gitlab/observability.rb
+++ b/lib/gitlab/observability.rb
@@ -4,15 +4,6 @@ module Gitlab
module Observability
extend self
- ACTION_TO_PERMISSION = {
- explore: :read_observability,
- datasources: :admin_observability,
- manage: :admin_observability,
- dashboards: :read_observability
- }.freeze
-
- EMBEDDABLE_PATHS = %w[explore goto].freeze
-
# Returns the GitLab Observability URL
#
def observability_url
@@ -31,124 +22,13 @@ module Gitlab
"#{Gitlab::Observability.observability_url}/v3/tenant/#{project.id}"
end
- # Returns true if the GitLab Observability UI (GOUI) feature flag is enabled
- #
- # @deprecated
- #
- def enabled?(group = nil)
- return Feature.enabled?(:observability_group_tab, group) if group
-
- Feature.enabled?(:observability_group_tab)
- end
-
- # Returns the embeddable Observability URL of a given URL
- #
- # - Validates the URL
- # - Checks that the path is embeddable
- # - Converts the gitlab.com URL to observe.gitlab.com URL
- #
- # e.g.
- #
- # from: gitlab.com/groups/GROUP_PATH/-/observability/explore?observability_path=/explore
- # to observe.gitlab.com/-/GROUP_ID/explore
- #
- # Returns nil if the URL is not a valid Observability URL or the path is not embeddable
- #
- def embeddable_url(url)
- uri = validate_url(url, Gitlab.config.gitlab.url)
- return unless uri
-
- group = group_from_observability_url(url)
- return unless group
-
- parsed_query = CGI.parse(uri.query.to_s).transform_values(&:first).symbolize_keys
- observability_path = parsed_query[:observability_path]
-
- return build_full_url(group, observability_path, '/') if observability_path_embeddable?(observability_path)
- end
-
- # Returns true if the user is allowed to perform an action within a group
- #
- def allowed_for_action?(user, group, action)
- return false if action.nil?
-
- permission = ACTION_TO_PERMISSION.fetch(action.to_sym, :admin_observability)
- allowed?(user, group, permission)
- end
-
- # Returns true if the user has the specified permission within the group
- def allowed?(user, group, permission = :admin_observability)
- return false unless group && user
-
- observability_url.present? && Ability.allowed?(user, permission, group)
- end
-
- # Builds the full Observability URL given a certan group and path
- #
- # If unsanitized_observability_path is not valid or missing, fallbacks to fallback_path
- #
- def build_full_url(group, unsanitized_observability_path, fallback_path)
- return unless group
-
- # When running Observability UI in standalone mode (i.e. not backed by Observability Backend)
- # the group-id is not required. !!This is only used for local dev!!
- base_url = ENV['STANDALONE_OBSERVABILITY_UI'] == 'true' ? observability_url : "#{observability_url}/-/#{group.id}"
-
- sanitized_path = if unsanitized_observability_path && sanitize(unsanitized_observability_path) != ''
- CGI.unescapeHTML(sanitize(unsanitized_observability_path))
- else
- fallback_path || '/'
- end
-
- sanitized_path.prepend('/') if sanitized_path[0] != '/'
-
- "#{base_url}#{sanitized_path}"
- end
-
- private
-
- def validate_url(url, reference_url)
- uri = URI.parse(url)
- reference_uri = URI.parse(reference_url)
-
- return uri if uri.scheme == reference_uri.scheme &&
- uri.port == reference_uri.port &&
- uri.host.casecmp?(reference_uri.host)
- rescue URI::InvalidURIError
- nil
- end
-
- def link_sanitizer
- @link_sanitizer ||= Rails::Html::Sanitizer.safe_list_sanitizer.new
- end
-
- def sanitize(input)
- link_sanitizer.sanitize(input, {})&.html_safe
- end
-
- def group_from_observability_url(url)
- match = Rails.application.routes.recognize_path(url)
-
- return if match[:unmatched_route].present?
- return if match[:group_id].blank? || match[:action].blank? || match[:controller] != "groups/observability"
-
- group_path = match[:group_id]
- Group.find_by_full_path(group_path)
- rescue ActionController::RoutingError
- nil
- end
-
- def observability_path_embeddable?(observability_path)
- return false unless observability_path
-
- observability_path = observability_path[1..] if observability_path[0] == '/'
-
- parsed_observability_path = URI.parse(observability_path).path.split('/')
-
- base_path = parsed_observability_path[0]
+ def should_enable_observability_auth_scopes?(resource)
+ # Enable the needed oauth scopes if tracing is enabled.
+ if resource.is_a?(Group) || resource.is_a?(Project)
+ return Feature.enabled?(:observability_tracing,
+ resource.root_ancestor)
+ end
- EMBEDDABLE_PATHS.include?(base_path)
- rescue URI::InvalidURIError
false
end
end
diff --git a/lib/gitlab/pages/cache_control.rb b/lib/gitlab/pages/cache_control.rb
deleted file mode 100644
index 81da34f1219..00000000000
--- a/lib/gitlab/pages/cache_control.rb
+++ /dev/null
@@ -1,105 +0,0 @@
-# frozen_string_literal: true
-
-require 'set'
-
-module Gitlab
- module Pages
- class CacheControl
- include Gitlab::Utils::StrongMemoize
-
- EXPIRE = 12.hours
- # To avoid delivering expired deployment URL in the cached payload,
- # use a longer expiration time in the deployment URL
- DEPLOYMENT_EXPIRATION = (EXPIRE + 12.hours)
-
- SETTINGS_CACHE_KEY = 'pages_domain_for_%{type}_%{id}'
- PAYLOAD_CACHE_KEY = '%{settings_cache_key}_%{settings_hash}'
-
- class << self
- def for_domain(domain_id)
- new(type: :domain, id: domain_id)
- end
-
- def for_namespace(namespace_id)
- new(type: :namespace, id: namespace_id)
- end
- end
-
- def initialize(type:, id:)
- raise(ArgumentError, "type must be :namespace or :domain") unless %i[namespace domain].include?(type)
-
- @type = type
- @id = id
- end
-
- def cache_key
- strong_memoize(:payload_cache_key) do
- cache_settings_hash!
-
- payload_cache_key_for(settings_hash)
- end
- end
-
- # Invalidates the cache.
- #
- # Since rails nodes and sidekiq nodes have different application settings,
- # and the invalidation happens in a sidekiq node, we have to use the
- # cached settings hash to build the payload cache key to be invalidated.
- def clear_cache
- keys = cached_settings_hashes
- .map { |hash| payload_cache_key_for(hash) }
- .push(settings_cache_key)
-
- ::Gitlab::AppLogger.info(
- message: 'clear pages cache',
- pages_keys: keys,
- pages_type: @type,
- pages_id: @id
- )
-
- Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do
- Rails.cache.delete_multi(keys)
- end
- end
-
- private
-
- # Since rails nodes and sidekiq nodes have different application settings,
- # we cache the application settings hash when creating the payload cache
- # so we can use these values to invalidate the cache in a sidekiq node later.
- def cache_settings_hash!
- cached = cached_settings_hashes.to_set
- Rails.cache.write(settings_cache_key, cached.add(settings_hash))
- end
-
- def cached_settings_hashes
- Rails.cache.read(settings_cache_key) || []
- end
-
- def payload_cache_key_for(settings_hash)
- PAYLOAD_CACHE_KEY % {
- settings_cache_key: settings_cache_key,
- settings_hash: settings_hash
- }
- end
-
- def settings_cache_key
- strong_memoize(:settings_cache_key) do
- SETTINGS_CACHE_KEY % { type: @type, id: @id }
- end
- end
-
- def settings_hash
- strong_memoize(:settings_hash) do
- values = ::Gitlab.config.pages.dup
-
- values['app_settings'] = ::Gitlab::CurrentSettings.attributes.slice(
- 'force_pages_access_control'
- )
-
- ::Digest::SHA256.hexdigest(values.inspect)
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/pagination/cursor_based_keyset.rb b/lib/gitlab/pagination/cursor_based_keyset.rb
index 592f635c14e..81dcc54ff35 100644
--- a/lib/gitlab/pagination/cursor_based_keyset.rb
+++ b/lib/gitlab/pagination/cursor_based_keyset.rb
@@ -3,13 +3,6 @@
module Gitlab
module Pagination
module CursorBasedKeyset
- SUPPORTED_ORDERING = {
- Group => { name: :asc },
- AuditEvent => { id: :desc },
- ::Ci::Build => { id: :desc },
- ::Packages::BuildInfo => { id: :desc }
- }.freeze
-
SUPPORTED_MULTI_ORDERING = {
Group => { name: [:asc] },
AuditEvent => { id: [:desc] },
@@ -33,11 +26,7 @@ module Gitlab
ENFORCED_TYPES = [Group].freeze
def self.available_for_type?(relation)
- if Feature.enabled?(:api_keyset_pagination_multi_order)
- SUPPORTED_MULTI_ORDERING.key?(relation.klass)
- else
- SUPPORTED_ORDERING.key?(relation.klass)
- end
+ SUPPORTED_MULTI_ORDERING.key?(relation.klass)
end
def self.available?(cursor_based_request_context, relation)
@@ -50,16 +39,10 @@ module Gitlab
end
def self.order_satisfied?(relation, cursor_based_request_context)
- if Feature.enabled?(:api_keyset_pagination_multi_order)
- order_by_from_request = cursor_based_request_context.order
- sort_from_request = cursor_based_request_context.sort
-
- SUPPORTED_MULTI_ORDERING[relation.klass][order_by_from_request]&.include?(sort_from_request)
- else
- order_by_from_request = cursor_based_request_context.order_by
+ order_by_from_request = cursor_based_request_context.order
+ sort_from_request = cursor_based_request_context.sort
- SUPPORTED_ORDERING[relation.klass] == order_by_from_request
- end
+ SUPPORTED_MULTI_ORDERING[relation.klass][order_by_from_request]&.include?(sort_from_request)
end
private_class_method :order_satisfied?
end
diff --git a/lib/gitlab/patch/hangouts_chat_http_override.rb b/lib/gitlab/patch/hangouts_chat_http_override.rb
deleted file mode 100644
index 20dc678e251..00000000000
--- a/lib/gitlab/patch/hangouts_chat_http_override.rb
+++ /dev/null
@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Patch
- module HangoutsChatHTTPOverride
- attr_reader :uri
-
- # See https://github.com/enzinia/hangouts-chat/blob/6a509f61a56e757f8f417578b393b94423831ff7/lib/hangouts_chat/http.rb
- def post(payload)
- httparty_response = Gitlab::HTTP.post(
- uri,
- body: payload.to_json,
- headers: { 'Content-Type' => 'application/json' },
- parse: nil # Disables automatic response parsing
- )
- httparty_response.response
- # The rest of the integration expects a Net::HTTP response
- end
- end
- end
-end
diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb
index 8afcf682d5d..cde621bc9e4 100644
--- a/lib/gitlab/path_regex.rb
+++ b/lib/gitlab/path_regex.rb
@@ -134,10 +134,10 @@ module Gitlab
PATH_REGEX_STR = PATH_START_CHAR + '[a-zA-Z0-9_\-\.]' + "{0,#{Namespace::URL_MAX_LENGTH - 1}}"
NAMESPACE_FORMAT_REGEX_JS = PATH_REGEX_STR + '[a-zA-Z0-9_\-]|[a-zA-Z0-9_]'
- NO_SUFFIX_REGEX = /(?<!\.git|\.atom)/.freeze
- NAMESPACE_FORMAT_REGEX = /(?:#{NAMESPACE_FORMAT_REGEX_JS})#{NO_SUFFIX_REGEX}/.freeze
- PROJECT_PATH_FORMAT_REGEX = /(?:#{PATH_REGEX_STR})#{NO_SUFFIX_REGEX}/.freeze
- FULL_NAMESPACE_FORMAT_REGEX = %r{(#{NAMESPACE_FORMAT_REGEX}/){,#{Namespace::NUMBER_OF_ANCESTORS_ALLOWED}}#{NAMESPACE_FORMAT_REGEX}}.freeze
+ NO_SUFFIX_REGEX = /(?<!\.git|\.atom)/
+ NAMESPACE_FORMAT_REGEX = /(?:#{NAMESPACE_FORMAT_REGEX_JS})#{NO_SUFFIX_REGEX}/
+ PROJECT_PATH_FORMAT_REGEX = /(?:#{PATH_REGEX_STR})#{NO_SUFFIX_REGEX}/
+ FULL_NAMESPACE_FORMAT_REGEX = %r{(#{NAMESPACE_FORMAT_REGEX}/){,#{Namespace::NUMBER_OF_ANCESTORS_ALLOWED}}#{NAMESPACE_FORMAT_REGEX}}
def organization_route_regex
@organization_route_regex ||= begin
@@ -188,19 +188,19 @@ module Gitlab
end
def repository_route_regex
- @repository_route_regex ||= /(#{full_namespace_route_regex}|#{personal_snippet_repository_path_regex})\.*/.freeze
+ @repository_route_regex ||= /(#{full_namespace_route_regex}|#{personal_snippet_repository_path_regex})\.*/
end
def repository_git_route_regex
- @repository_git_route_regex ||= /#{repository_route_regex}\.git/.freeze
+ @repository_git_route_regex ||= /#{repository_route_regex}\.git/
end
def repository_git_lfs_route_regex
- @repository_git_lfs_route_regex ||= %r{#{repository_git_route_regex}\/(info\/lfs|gitlab-lfs)\/}.freeze
+ @repository_git_lfs_route_regex ||= %r{#{repository_git_route_regex}\/(info\/lfs|gitlab-lfs)\/}
end
def repository_wiki_git_route_regex
- @repository_wiki_git_route_regex ||= /#{full_namespace_route_regex}\.*\.wiki\.git/.freeze
+ @repository_wiki_git_route_regex ||= /#{full_namespace_route_regex}\.*\.wiki\.git/
end
def full_namespace_path_regex
@@ -220,7 +220,7 @@ module Gitlab
end
def namespace_format_regex
- @namespace_format_regex ||= /\A#{NAMESPACE_FORMAT_REGEX}\z/o.freeze
+ @namespace_format_regex ||= /\A#{NAMESPACE_FORMAT_REGEX}\z/o
end
def namespace_format_message
@@ -229,7 +229,7 @@ module Gitlab
end
def project_path_format_regex
- @project_path_format_regex ||= /\A#{PROJECT_PATH_FORMAT_REGEX}\z/o.freeze
+ @project_path_format_regex ||= /\A#{PROJECT_PATH_FORMAT_REGEX}\z/o
end
def project_path_format_message
@@ -239,7 +239,7 @@ module Gitlab
def archive_formats_regex
# |zip|tar| tar.gz | tar.bz2 |
- @archive_formats_regex ||= /(zip|tar|tar\.gz|tgz|gz|tar\.bz2|tbz|tbz2|tb2|bz2)/.freeze
+ @archive_formats_regex ||= /(zip|tar|tar\.gz|tgz|gz|tar\.bz2|tbz|tbz2|tb2|bz2)/
end
def git_reference_regex
@@ -270,11 +270,11 @@ module Gitlab
end
def container_image_regex
- @container_image_regex ||= %r{([\w\.-]+\/){0,4}[\w\.-]+}.freeze
+ @container_image_regex ||= %r{([\w\.-]+\/){0,4}[\w\.-]+}
end
def container_image_blob_sha_regex
- @container_image_blob_sha_regex ||= %r{[\w+.-]+:?\w+}.freeze
+ @container_image_blob_sha_regex ||= %r{[\w+.-]+:?\w+}
end
def dependency_proxy_route_regex
diff --git a/lib/gitlab/path_traversal.rb b/lib/gitlab/path_traversal.rb
index d42b5fde615..c8308c9da1c 100644
--- a/lib/gitlab/path_traversal.rb
+++ b/lib/gitlab/path_traversal.rb
@@ -15,13 +15,13 @@ module Gitlab
# We url decode the path to avoid passing invalid paths forward in url encoded format.
# Also see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24223#note_284122580
# It also checks for backslash '\', which is sometimes a File::ALT_SEPARATOR.
- def check_path_traversal!(path)
+ def check_path_traversal!(path, skip_decoding: false)
return unless path
path = path.to_s if path.is_a?(Gitlab::HashedPath)
raise PathTraversalAttackError, 'Invalid path' unless path.is_a?(String)
- path = ::Gitlab::Utils.decode_path(path)
+ path = ::Gitlab::Utils.decode_path(path) unless skip_decoding
if path.match?(PATH_TRAVERSAL_REGEX)
logger.warn(message: "Potential path traversal attempt detected", path: path.to_s)
diff --git a/lib/gitlab/prometheus/metric_group.rb b/lib/gitlab/prometheus/metric_group.rb
deleted file mode 100644
index 020d4cf74a3..00000000000
--- a/lib/gitlab/prometheus/metric_group.rb
+++ /dev/null
@@ -1,34 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Prometheus
- class MetricGroup
- include ActiveModel::Model
-
- attr_accessor :name, :priority, :metrics
-
- validates :name, :priority, :metrics, presence: true
-
- def self.common_metrics
- all_groups = ::PrometheusMetricsFinder.new(common: true).execute
- .group_by(&:group_title)
- .map do |name, metrics|
- MetricGroup.new(
- name: name,
- priority: metrics.map(&:priority).max,
- metrics: metrics.map(&:to_query_metric)
- )
- end
-
- all_groups.sort_by(&:priority).reverse
- end
-
- # EE only
- def self.for_project(_)
- common_metrics
- end
- end
- end
-end
-
-Gitlab::Prometheus::MetricGroup.prepend_mod_with('Gitlab::Prometheus::MetricGroup')
diff --git a/lib/gitlab/prometheus/parsing_error.rb b/lib/gitlab/prometheus/parsing_error.rb
deleted file mode 100644
index 20b5ef5ce55..00000000000
--- a/lib/gitlab/prometheus/parsing_error.rb
+++ /dev/null
@@ -1,7 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Prometheus
- ParsingError = Class.new(StandardError)
- end
-end
diff --git a/lib/gitlab/prometheus/queries/base_query.rb b/lib/gitlab/prometheus/queries/base_query.rb
deleted file mode 100644
index eabac6128b5..00000000000
--- a/lib/gitlab/prometheus/queries/base_query.rb
+++ /dev/null
@@ -1,33 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Prometheus
- module Queries
- class BaseQuery
- attr_accessor :client
-
- delegate :query_range, :query, :label_values, :series, to: :client, prefix: true
-
- def raw_memory_usage_query(environment_slug)
- %{avg(container_memory_usage_bytes{container_name!="POD",environment="#{environment_slug}"}) / 2^20}
- end
-
- def raw_cpu_usage_query(environment_slug)
- %{avg(rate(container_cpu_usage_seconds_total{container_name!="POD",environment="#{environment_slug}"}[2m])) * 100}
- end
-
- def initialize(client)
- @client = client
- end
-
- def query(*args)
- raise NotImplementedError
- end
-
- def self.transform_reactive_result(result)
- result
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/prometheus/queries/deployment_query.rb b/lib/gitlab/prometheus/queries/deployment_query.rb
deleted file mode 100644
index 13d85d33cc0..00000000000
--- a/lib/gitlab/prometheus/queries/deployment_query.rb
+++ /dev/null
@@ -1,40 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Prometheus
- module Queries
- class DeploymentQuery < BaseQuery
- # rubocop: disable CodeReuse/ActiveRecord
- def query(deployment_id)
- Deployment.find_by(id: deployment_id).try do |deployment|
- environment_slug = deployment.environment.slug
-
- memory_query = raw_memory_usage_query(environment_slug)
- memory_avg_query = %{avg(avg_over_time(container_memory_usage_bytes{container_name!="POD",environment="#{environment_slug}"}[30m]))}
- cpu_query = raw_cpu_usage_query(environment_slug)
- cpu_avg_query = %{avg(rate(container_cpu_usage_seconds_total{container_name!="POD",environment="#{environment_slug}"}[30m])) * 100}
-
- timeframe_start = (deployment.created_at - 30.minutes).to_f
- timeframe_end = (deployment.created_at + 30.minutes).to_f
-
- {
- memory_values: client_query_range(memory_query, start_time: timeframe_start, end_time: timeframe_end),
- memory_before: client_query(memory_avg_query, time: deployment.created_at.to_f),
- memory_after: client_query(memory_avg_query, time: timeframe_end),
-
- cpu_values: client_query_range(cpu_query, start_time: timeframe_start, end_time: timeframe_end),
- cpu_before: client_query(cpu_avg_query, time: deployment.created_at.to_f),
- cpu_after: client_query(cpu_avg_query, time: timeframe_end)
- }
- end
- end
- # rubocop: enable CodeReuse/ActiveRecord
-
- def self.transform_reactive_result(result)
- result[:metrics] = result.delete :data
- result
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/prometheus/queries/environment_query.rb b/lib/gitlab/prometheus/queries/environment_query.rb
deleted file mode 100644
index 5f3093eecd4..00000000000
--- a/lib/gitlab/prometheus/queries/environment_query.rb
+++ /dev/null
@@ -1,34 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Prometheus
- module Queries
- class EnvironmentQuery < BaseQuery
- # rubocop: disable CodeReuse/ActiveRecord
- def query(environment_id)
- ::Environment.find_by(id: environment_id).try do |environment|
- environment_slug = environment.slug
- timeframe_start = 8.hours.ago.to_f
- timeframe_end = Time.now.to_f
-
- memory_query = raw_memory_usage_query(environment_slug)
- cpu_query = raw_cpu_usage_query(environment_slug)
-
- {
- memory_values: client_query_range(memory_query, start_time: timeframe_start, end_time: timeframe_end),
- memory_current: client_query(memory_query, time: timeframe_end),
- cpu_values: client_query_range(cpu_query, start_time: timeframe_start, end_time: timeframe_end),
- cpu_current: client_query(cpu_query, time: timeframe_end)
- }
- end
- end
- # rubocop: enable CodeReuse/ActiveRecord
-
- def self.transform_reactive_result(result)
- result[:metrics] = result.delete :data
- result
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/prometheus/queries/matched_metric_query.rb b/lib/gitlab/prometheus/queries/matched_metric_query.rb
deleted file mode 100644
index 73de5a11998..00000000000
--- a/lib/gitlab/prometheus/queries/matched_metric_query.rb
+++ /dev/null
@@ -1,82 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Prometheus
- module Queries
- class MatchedMetricQuery < BaseQuery
- MAX_QUERY_ITEMS = 40
-
- def query
- groups_data.map do |group, data|
- {
- group: group.name,
- priority: group.priority,
- active_metrics: data[:active_metrics],
- metrics_missing_requirements: data[:metrics_missing_requirements]
- }
- end
- end
-
- private
-
- def groups_data
- metrics_groups = groups_with_active_metrics(Gitlab::Prometheus::MetricGroup.common_metrics)
- lookup = active_series_lookup(metrics_groups)
-
- groups = {}
-
- metrics_groups.each do |group|
- groups[group] ||= { active_metrics: 0, metrics_missing_requirements: 0 }
- active_metrics = group.metrics.count { |metric| metric.required_metrics.all?(&lookup.method(:has_key?)) }
-
- groups[group][:active_metrics] += active_metrics
- groups[group][:metrics_missing_requirements] += group.metrics.count - active_metrics
- end
-
- groups
- end
-
- def active_series_lookup(metric_groups)
- timeframe_start = 8.hours.ago
- timeframe_end = Time.now
-
- series = metric_groups.flat_map(&:metrics).flat_map(&:required_metrics).uniq
-
- lookup = series.each_slice(MAX_QUERY_ITEMS).flat_map do |batched_series|
- client_series(*batched_series, start_time: timeframe_start, end_time: timeframe_end)
- .select(&method(:has_matching_label?))
- .map { |series_info| [series_info['__name__'], true] }
- end
- lookup.to_h
- end
-
- def has_matching_label?(series_info)
- series_info.key?('environment')
- end
-
- def available_metrics
- @available_metrics ||= client_label_values || []
- end
-
- def filter_active_metrics(metric_group)
- metric_group.metrics.select! do |metric|
- metric.required_metrics.all?(&available_metrics.method(:include?))
- end
- metric_group
- end
-
- def groups_with_active_metrics(metric_groups)
- metric_groups.map(&method(:filter_active_metrics)).select { |group| group.metrics.any? }
- end
-
- def metrics_with_required_series(metric_groups)
- metric_groups.flat_map do |group|
- group.metrics.select do |metric|
- metric.required_metrics.all?(&available_metrics.method(:include?))
- end
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/prometheus/queries/query_additional_metrics.rb b/lib/gitlab/prometheus/queries/query_additional_metrics.rb
deleted file mode 100644
index a870bb6bc5f..00000000000
--- a/lib/gitlab/prometheus/queries/query_additional_metrics.rb
+++ /dev/null
@@ -1,101 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Prometheus
- module Queries
- module QueryAdditionalMetrics
- def query_metrics(project, environment, query_context)
- matched_metrics(project).map(&query_group(query_context))
- .select(&method(:group_with_any_metrics))
- end
-
- protected
-
- def query_group(query_context)
- query_processor = method(:process_query).curry[query_context]
-
- lambda do |group|
- metrics = group.metrics.map do |metric|
- metric_hsh = {
- title: metric.title,
- weight: metric.weight,
- y_label: metric.y_label,
- queries: metric.queries.map(&query_processor).select(&method(:query_with_result))
- }
-
- metric_hsh[:id] = metric.id if metric.id
-
- metric_hsh
- end
-
- {
- group: group.name,
- priority: group.priority,
- metrics: metrics.select(&method(:metric_with_any_queries))
- }
- end
- end
-
- private
-
- def metric_with_any_queries(metric)
- metric[:queries]&.count&.> 0
- end
-
- def group_with_any_metrics(group)
- group[:metrics]&.count&.> 0
- end
-
- def query_with_result(query)
- query[:result]&.any? do |item|
- item&.[](:values)&.any? || item&.[](:value)&.any?
- end
- end
-
- def process_query(context, query)
- query = query.dup
- result =
- if query.key?(:query_range)
- query[:query_range] %= context
- client_query_range(query[:query_range], start_time: context[:timeframe_start], end_time: context[:timeframe_end])
- else
- query[:query] %= context
- client_query(query[:query], time: context[:timeframe_end])
- end
-
- query[:result] = result&.map(&:deep_symbolize_keys)
- query
- end
-
- def available_metrics
- @available_metrics ||= client_label_values || []
- end
-
- def matched_metrics(project)
- result = Gitlab::Prometheus::MetricGroup.for_project(project).map do |group|
- group.metrics.select! do |metric|
- metric.required_metrics.all?(&available_metrics.method(:include?))
- end
- group
- end
-
- result.select { |group| group.metrics.any? }
- end
-
- def common_query_context(environment, timeframe_start:, timeframe_end:)
- base_query_context(timeframe_start, timeframe_end)
- .merge(QueryVariables.call(environment))
- end
-
- def base_query_context(timeframe_start, timeframe_end)
- {
- timeframe_start: timeframe_start,
- timeframe_end: timeframe_end
- }
- end
- end
- end
- end
-end
-
-Gitlab::Prometheus::Queries::QueryAdditionalMetrics.prepend_mod_with('Gitlab::Prometheus::Queries::QueryAdditionalMetrics')
diff --git a/lib/gitlab/prometheus/queries/validate_query.rb b/lib/gitlab/prometheus/queries/validate_query.rb
deleted file mode 100644
index 160db7d44bc..00000000000
--- a/lib/gitlab/prometheus/queries/validate_query.rb
+++ /dev/null
@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Prometheus
- module Queries
- class ValidateQuery < BaseQuery
- def query(query)
- client_query(query)
- { valid: true }
- rescue Gitlab::PrometheusClient::QueryError, Gitlab::PrometheusClient::ConnectionError => ex
- { valid: false, error: ex.message }
- end
-
- def self.transform_reactive_result(result)
- result[:query] = result.delete :data
- result
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/prometheus/query_variables.rb b/lib/gitlab/prometheus/query_variables.rb
deleted file mode 100644
index 6a6e5c22d63..00000000000
--- a/lib/gitlab/prometheus/query_variables.rb
+++ /dev/null
@@ -1,31 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Prometheus
- module QueryVariables
- # start_time and end_time should be Time objects.
- def self.call(environment, start_time: nil, end_time: nil)
- {
- __range: range(start_time, end_time),
- ci_environment_slug: environment.slug,
- kube_namespace: environment.deployment_namespace || '',
- environment_filter: %(container_name!="POD",environment="#{environment.slug}"),
- ci_project_name: environment.project.name,
- ci_project_namespace: environment.project.namespace.name,
- ci_project_path: environment.project.full_path,
- ci_environment_name: environment.name
- }
- end
-
- private
-
- def self.range(start_time, end_time)
- if start_time && end_time
- range_seconds = (end_time - start_time).to_i
- "#{range_seconds}s"
- end
- end
- private_class_method :range
- end
- end
-end
diff --git a/lib/gitlab/puma/error_handler.rb b/lib/gitlab/puma/error_handler.rb
new file mode 100644
index 00000000000..4efc4866431
--- /dev/null
+++ b/lib/gitlab/puma/error_handler.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Puma
+ class ErrorHandler
+ PROD_ERROR_MESSAGE = "An error has occurred and reported in the system's low-level error handler."
+ DEV_ERROR_MESSAGE = <<~MSG
+ Server Error: An error has been caught by Puma's low-level error handler.
+ Read the Puma section of the troubleshooting docs for next steps - https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/troubleshooting/index.md#puma.
+ MSG
+
+ def initialize(is_production)
+ @is_production = is_production
+ end
+
+ def execute(ex, env, status_code)
+ # Puma v6.4.0 added the status_code argument in
+ # https://github.com/puma/puma/pull/3094
+ status_code ||= 500
+
+ if Raven.configuration.capture_allowed?
+ Raven.capture_exception(ex, tags: { handler: 'puma_low_level' },
+ extra: { puma_env: env, status_code: status_code })
+ end
+
+ # note the below is just a Rack response
+ [status_code, {}, message]
+ end
+
+ private
+
+ def message
+ if @is_production
+ PROD_ERROR_MESSAGE
+ else
+ DEV_ERROR_MESSAGE
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/push_options.rb b/lib/gitlab/push_options.rb
index 8a604c7d8a6..4471d21b9ac 100644
--- a/lib/gitlab/push_options.rb
+++ b/lib/gitlab/push_options.rb
@@ -39,7 +39,7 @@ module Gitlab
mr: :merge_request
}).freeze
- OPTION_MATCHER = /(?<namespace>[^\.]+)\.(?<key>[^=]+)=?(?<value>.*)/.freeze
+ OPTION_MATCHER = /(?<namespace>[^\.]+)\.(?<key>[^=]+)=?(?<value>.*)/
CI_SKIP = 'ci.skip'
diff --git a/lib/gitlab/query_limiting/transaction.rb b/lib/gitlab/query_limiting/transaction.rb
index f44e5383b4f..b9faf05391a 100644
--- a/lib/gitlab/query_limiting/transaction.rb
+++ b/lib/gitlab/query_limiting/transaction.rb
@@ -68,7 +68,7 @@ module Gitlab
GEO_NODES_LOAD = 'SELECT 1 AS one FROM "geo_nodes" LIMIT 1'
LICENSES_LOAD = 'SELECT "licenses".* FROM "licenses" ORDER BY "licenses"."id"'
- SCHEMA_INTROSPECTION = %r{SELECT.*(FROM|JOIN) (pg_attribute|pg_class)}m.freeze
+ SCHEMA_INTROSPECTION = %r{SELECT.*(FROM|JOIN) (pg_attribute|pg_class)}m
# queries can be safely ignored if they are amoritized in regular usage
# (i.e. only requested occasionally and otherwise cached).
diff --git a/lib/gitlab/quick_actions/extractor.rb b/lib/gitlab/quick_actions/extractor.rb
index 015dbe7063c..5cf79db83af 100644
--- a/lib/gitlab/quick_actions/extractor.rb
+++ b/lib/gitlab/quick_actions/extractor.rb
@@ -20,7 +20,7 @@ module Gitlab
.+?
\n```$
)
- }mix.freeze
+ }mix
INLINE_CODE_REGEX = %r{
(?<inline_code>
@@ -31,7 +31,7 @@ module Gitlab
`.+?`
)
- }mix.freeze
+ }mix
HTML_BLOCK_REGEX = %r{
(?<html>
@@ -44,7 +44,7 @@ module Gitlab
.+?
\n<\/[^>]+?>$
)
- }mix.freeze
+ }mix
QUOTE_BLOCK_REGEX = %r{
(?<html>
@@ -57,11 +57,11 @@ module Gitlab
.+?
\n>>>$
)
- }mix.freeze
+ }mix
EXCLUSION_REGEX = %r{
#{CODE_REGEX} | #{INLINE_CODE_REGEX} | #{HTML_BLOCK_REGEX} | #{QUOTE_BLOCK_REGEX}
- }mix.freeze
+ }mix
attr_reader :command_definitions, :keep_actions
diff --git a/lib/gitlab/quick_actions/spend_time_and_date_separator.rb b/lib/gitlab/quick_actions/spend_time_and_date_separator.rb
index 3794f2f8818..a8dd4aa17c5 100644
--- a/lib/gitlab/quick_actions/spend_time_and_date_separator.rb
+++ b/lib/gitlab/quick_actions/spend_time_and_date_separator.rb
@@ -11,7 +11,7 @@ module Gitlab
# if date doesn't present return time with current date
# in other cases return nil
class SpendTimeAndDateSeparator
- DATE_REGEX = %r{(\d{2,4}[/\-.]\d{1,2}[/\-.]\d{1,2})}.freeze
+ DATE_REGEX = %r{(\d{2,4}[/\-.]\d{1,2}[/\-.]\d{1,2})}
def initialize(spend_command_arg)
@spend_arg = spend_command_arg
diff --git a/lib/gitlab/quick_actions/timeline_text_and_date_time_separator.rb b/lib/gitlab/quick_actions/timeline_text_and_date_time_separator.rb
index e8002656ff5..2eca87347b1 100644
--- a/lib/gitlab/quick_actions/timeline_text_and_date_time_separator.rb
+++ b/lib/gitlab/quick_actions/timeline_text_and_date_time_separator.rb
@@ -3,9 +3,9 @@
module Gitlab
module QuickActions
class TimelineTextAndDateTimeSeparator
- DATETIME_REGEX = %r{(\d{2,4}[\-.]\d{1,2}[\-.]\d{1,2} \d{1,2}:\d{2})}.freeze
- MIXED_DELIMITER = %r{([/.])}.freeze
- TIME_REGEX = %r{(\d{1,2}:\d{2})}.freeze
+ DATETIME_REGEX = %r{(\d{2,4}[\-.]\d{1,2}[\-.]\d{1,2} \d{1,2}:\d{2})}
+ MIXED_DELIMITER = %r{([/.])}
+ TIME_REGEX = %r{(\d{1,2}:\d{2})}
def initialize(timeline_event_arg)
@timeline_event_arg = timeline_event_arg
diff --git a/lib/gitlab/quick_actions/work_item_actions.rb b/lib/gitlab/quick_actions/work_item_actions.rb
index 0a96d502862..2adee0f9a9a 100644
--- a/lib/gitlab/quick_actions/work_item_actions.rb
+++ b/lib/gitlab/quick_actions/work_item_actions.rb
@@ -27,6 +27,30 @@ module Gitlab
command :promote_to do |type_name|
@execution_message[:promote_to] = update_type(type_name, :promote_to)
end
+
+ desc { _('Change work item parent') }
+ explanation do |parent_param|
+ format(_("Change work item's parent to %{parent_ref}."), parent_ref: parent_param)
+ end
+ types WorkItem
+ params 'Parent #iid, reference or URL'
+ condition { supports_parent? && can_admin_link? }
+ command :set_parent do |parent_param|
+ @updates[:set_parent] = extract_work_items(parent_param).first
+ @execution_message[:set_parent] = success_msg[:set_parent]
+ end
+
+ desc { _('Add children to work item') }
+ explanation do |child_param|
+ format(_("Add %{child_ref} to this work item as child(ren)."), child_ref: child_param)
+ end
+ types WorkItem
+ params 'Children #iids, references or URLs'
+ condition { supports_children? && can_admin_link? }
+ command :add_child do |child_param|
+ @updates[:add_child] = extract_work_items(child_param)
+ @execution_message[:add_child] = success_msg[:add_child]
+ end
end
private
@@ -52,6 +76,18 @@ module Gitlab
nil
end
+ # rubocop: disable CodeReuse/ActiveRecord
+ def extract_work_items(params)
+ return if params.nil?
+
+ issuable_type = params.include?('work_items') ? :work_item : :issue
+ issuables = extract_references(params, issuable_type)
+ return unless issuables
+
+ WorkItem.find(issuables.pluck(:id))
+ end
+ # rubocop: enable CodeReuse/ActiveRecord
+
def validate_promote_to(type)
return error_msg(:not_found, action: 'promote') unless type && supports_promote_to?(type.name)
return if current_user.can?(:"create_#{type.base_type}", quick_action_target)
@@ -78,8 +114,8 @@ module Gitlab
def error_msg(reason, action: 'convert')
message = {
not_found: 'Provided type is not supported',
- same_type: 'Types are the same',
- forbidden: 'You have insufficient permissions'
+ forbidden: 'You have insufficient permissions',
+ same_type: 'Types are the same'
}.freeze
format(_("Failed to %{action} this work item: %{reason}."), { action: action, reason: message[reason] })
@@ -88,9 +124,23 @@ module Gitlab
def success_msg
{
type: _('Type changed successfully.'),
- promote_to: _("Work item promoted successfully.")
+ promote_to: _("Work item promoted successfully."),
+ set_parent: _('Work item parent set successfully'),
+ add_child: _('Child work item(s) added successfully')
}
end
+
+ def supports_parent?
+ ::WorkItems::HierarchyRestriction.find_by_child_type_id(quick_action_target.work_item_type_id).present?
+ end
+
+ def supports_children?
+ ::WorkItems::HierarchyRestriction.find_by_parent_type_id(quick_action_target.work_item_type_id).present?
+ end
+
+ def can_admin_link?
+ current_user.can?(:admin_issue_link, quick_action_target)
+ end
end
end
end
diff --git a/lib/gitlab/rack_attack/request.rb b/lib/gitlab/rack_attack/request.rb
index a03116f5bb2..e45782b8be0 100644
--- a/lib/gitlab/rack_attack/request.rb
+++ b/lib/gitlab/rack_attack/request.rb
@@ -5,8 +5,9 @@ module Gitlab
module Request
include ::Gitlab::Utils::StrongMemoize
- FILES_PATH_REGEX = %r{^/api/v\d+/projects/[^/]+/repository/files/.+}.freeze
- GROUP_PATH_REGEX = %r{^/api/v\d+/groups/[^/]+/?$}.freeze
+ API_PATH_REGEX = %r{^/api/|/oauth/}
+ FILES_PATH_REGEX = %r{^/api/v\d+/projects/[^/]+/repository/files/.+}
+ GROUP_PATH_REGEX = %r{^/api/v\d+/groups/[^/]+/?$}
def unauthenticated?
!(authenticated_identifier([:api, :rss, :ics]) || authenticated_runner_id)
@@ -32,7 +33,11 @@ module Gitlab
end
def api_request?
- logical_path.start_with?('/api')
+ if ::Feature.enabled?(:rate_limit_oauth_api, ::Feature.current_request)
+ matches?(API_PATH_REGEX)
+ else
+ logical_path.start_with?('/api')
+ end
end
def logical_path
diff --git a/lib/gitlab/redis/hll.rb b/lib/gitlab/redis/hll.rb
index 4d1855e4637..ab144b30796 100644
--- a/lib/gitlab/redis/hll.rb
+++ b/lib/gitlab/redis/hll.rb
@@ -5,7 +5,7 @@ module Gitlab
module Redis
class HLL
BATCH_SIZE = 300
- KEY_REGEX = %r{\A(\w|-|:)*\{\w*\}(\w|-|:)*\z}.freeze
+ KEY_REGEX = %r{\A(\w|-|:)*\{\w*\}(\w|-|:)*\z}
KeyFormatError = Class.new(StandardError)
def self.count(params)
diff --git a/lib/gitlab/redis/multi_store.rb b/lib/gitlab/redis/multi_store.rb
index 9ce030c0bbe..bbe5a8add4b 100644
--- a/lib/gitlab/redis/multi_store.rb
+++ b/lib/gitlab/redis/multi_store.rb
@@ -248,6 +248,19 @@ module Gitlab
end
end
+ # connection_pool gem calls `#close` method:
+ #
+ # https://github.com/mperham/connection_pool/blob/v2.4.1/lib/connection_pool.rb#L63
+ #
+ # Let's define it explicitly instead of propagating it to method_missing
+ def close
+ if use_primary_and_secondary_stores?
+ [primary_store, secondary_store].map(&:close).first
+ else
+ default_store.close
+ end
+ end
+
private
# @return [Boolean]
diff --git a/lib/gitlab/redis/queues_metadata.rb b/lib/gitlab/redis/queues_metadata.rb
index bb83e7709e1..a0344c93ae4 100644
--- a/lib/gitlab/redis/queues_metadata.rb
+++ b/lib/gitlab/redis/queues_metadata.rb
@@ -7,15 +7,6 @@ module Gitlab
def config_fallback
Queues
end
-
- private
-
- def redis
- primary_store = ::Redis.new(params)
- secondary_store = ::Redis.new(config_fallback.params)
-
- MultiStore.new(primary_store, secondary_store, name.demodulize)
- end
end
end
end
diff --git a/lib/gitlab/redis/workhorse.rb b/lib/gitlab/redis/workhorse.rb
index ea0fca515fe..382797bc72a 100644
--- a/lib/gitlab/redis/workhorse.rb
+++ b/lib/gitlab/redis/workhorse.rb
@@ -7,15 +7,6 @@ module Gitlab
def config_fallback
SharedState
end
-
- private
-
- def redis
- primary_store = ::Redis.new(params)
- secondary_store = ::Redis.new(config_fallback.params)
-
- MultiStore.new(primary_store, secondary_store, store_name)
- end
end
end
end
diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb
index 1ec8818d3f5..2bcf4769b5a 100644
--- a/lib/gitlab/redis/wrapper.rb
+++ b/lib/gitlab/redis/wrapper.rb
@@ -167,20 +167,24 @@ module Gitlab
cert_file = config[:ssl_params].delete(:cert_file)
key_file = config[:ssl_params].delete(:key_file)
- unless ::File.exist?(cert_file)
- raise InvalidPathError,
- "Certificate file #{cert_file} specified in in `resque.yml` does not exist."
+ if cert_file
+ unless ::File.exist?(cert_file)
+ raise InvalidPathError,
+ "Certificate file #{cert_file} specified in in `resque.yml` does not exist."
+ end
+
+ config[:ssl_params][:cert] = OpenSSL::X509::Certificate.new(File.read(cert_file))
end
- config[:ssl_params][:cert] = OpenSSL::X509::Certificate.new(File.read(cert_file))
+ if key_file
+ unless ::File.exist?(key_file)
+ raise InvalidPathError,
+ "Key file #{key_file} specified in in `resque.yml` does not exist."
+ end
- unless ::File.exist?(key_file)
- raise InvalidPathError,
- "Key file #{key_file} specified in in `resque.yml` does not exist."
+ config[:ssl_params][:key] = OpenSSL::PKey.read(File.read(key_file))
end
- config[:ssl_params][:key] = OpenSSL::PKey.read(File.read(key_file))
-
config
end
diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb
index 8ef455efe07..2fd9dc9fa09 100644
--- a/lib/gitlab/regex.rb
+++ b/lib/gitlab/regex.rb
@@ -6,26 +6,11 @@ module Gitlab
extend MergeRequests
extend Packages
- def group_path_regex
- # This regexp validates the string conforms to rules for a group slug:
- # i.e does not start with a non-alphanumeric character except for periods or underscores,
- # contains only alphanumeric characters, periods, and underscores,
- # does not end with a period or forward slash, and has no leading or trailing forward slashes
- # eg 'destination-path' or 'destination_pth' not 'example/com/destination/full/path'
- @group_path_regex ||= %r{\A[.]?[^\W]([.]?[0-9a-z][-_]*)+\z}i
- end
-
- def group_path_regex_message
- "cannot start with a non-alphanumeric character except for periods or underscores, " \
- "can contain only alphanumeric characters, periods, and underscores, " \
- "cannot end with a period or forward slash, and has no leading or trailing forward slashes." \
- end
-
def project_name_regex
# The character range \p{Alnum} overlaps with \u{00A9}-\u{1f9ff}
# hence the Ruby warning.
# https://gitlab.com/gitlab-org/gitlab/merge_requests/23165#not-easy-fixable
- @project_name_regex ||= /\A[\p{Alnum}\u{00A9}-\u{1f9ff}_][\p{Alnum}\p{Pd}\u{002B}\u{00A9}-\u{1f9ff}_\. ]*\z/.freeze
+ @project_name_regex ||= /\A[\p{Alnum}\u{00A9}-\u{1f9ff}_][\p{Alnum}\p{Pd}\u{002B}\u{00A9}-\u{1f9ff}_\. ]*\z/
end
def project_name_regex_message
@@ -35,7 +20,7 @@ module Gitlab
# Project path must conform to this regex. See https://gitlab.com/gitlab-org/gitlab/-/issues/27483
def oci_repository_path_regex
- @oci_repository_path_regex ||= %r{\A[a-zA-Z0-9]+([._-][a-zA-Z0-9]+)*\z}.freeze
+ @oci_repository_path_regex ||= %r{\A[a-zA-Z0-9]+([._-][a-zA-Z0-9]+)*\z}
end
def oci_repository_path_regex_message
@@ -43,11 +28,11 @@ module Gitlab
end
def group_name_regex
- @group_name_regex ||= /\A#{group_name_regex_chars}\z/.freeze
+ @group_name_regex ||= /\A#{group_name_regex_chars}\z/
end
def group_name_regex_chars
- @group_name_regex_chars ||= /[\p{Alnum}\u{00A9}-\u{1f9ff}_][\p{Alnum}\p{Pd}\u{00A9}-\u{1f9ff}_()\. ]*/.freeze
+ @group_name_regex_chars ||= /[\p{Alnum}\u{00A9}-\u{1f9ff}_][\p{Alnum}\p{Pd}\u{00A9}-\u{1f9ff}_()\. ]*/
end
def group_name_regex_message
@@ -81,7 +66,7 @@ module Gitlab
end
def environment_name_regex
- @environment_name_regex ||= /\A[#{environment_name_regex_chars_without_slash}]([#{environment_name_regex_chars}]*[#{environment_name_regex_chars_without_slash}])?\z/.freeze
+ @environment_name_regex ||= /\A[#{environment_name_regex_chars_without_slash}]([#{environment_name_regex_chars}]*[#{environment_name_regex_chars_without_slash}])?\z/
end
def environment_name_regex_message
@@ -93,7 +78,7 @@ module Gitlab
end
def environment_scope_regex
- @environment_scope_regex ||= /\A[#{environment_scope_regex_chars}]+\z/.freeze
+ @environment_scope_regex ||= /\A[#{environment_scope_regex_chars}]+\z/
end
def environment_scope_regex_message
@@ -125,7 +110,7 @@ module Gitlab
end
def environment_slug_regex
- @environment_slug_regex ||= /\A[a-z]([a-z0-9-]*[a-z0-9])?\z/.freeze
+ @environment_slug_regex ||= /\A[a-z]([a-z0-9-]*[a-z0-9])?\z/
end
def environment_slug_regex_message
@@ -153,7 +138,7 @@ module Gitlab
#{logs_section_prefix_regex}
#{logs_section_options_regex}
#{logs_section_suffix_regex}
- }x.freeze
+ }x
end
MARKDOWN_CODE_BLOCK_REGEX = %r{
@@ -167,7 +152,7 @@ module Gitlab
.+?
\n```\ *$
)
- }mx.freeze
+ }mx
# Code blocks:
# ```
@@ -178,7 +163,7 @@ module Gitlab
'^```.*?\n' \
'(?:\n|.)*?' \
'\n```\ *$' \
- ')'.freeze
+ ')'
MARKDOWN_HTML_BLOCK_REGEX = %r{
(?<html>
@@ -191,7 +176,7 @@ module Gitlab
.+?
\n<\/[^>]+?>\ *$
)
- }mx.freeze
+ }mx
# HTML block:
# <tag>
@@ -202,28 +187,28 @@ module Gitlab
'^<[^>]+?>\ *\n' \
'(?:\n|.)*?' \
'\n<\/[^>]+?>\ *$' \
- ')'.freeze
+ ')'
# HTML comment line:
# <!-- some commented text -->
MARKDOWN_HTML_COMMENT_LINE_REGEX_UNTRUSTED =
'(?P<html_comment_line>' \
'^<!--\ .*?\ -->\ *$' \
- ')'.freeze
+ ')'
MARKDOWN_HTML_COMMENT_BLOCK_REGEX_UNTRUSTED =
'(?P<html_comment_block>' \
'^<!--.*?\n' \
'(?:\n|.)*?' \
'\n.*?-->\ *$' \
- ')'.freeze
+ ')'
def markdown_code_or_html_blocks
@markdown_code_or_html_blocks ||= %r{
#{MARKDOWN_CODE_BLOCK_REGEX}
|
#{MARKDOWN_HTML_BLOCK_REGEX}
- }mx.freeze
+ }mx
end
def markdown_code_or_html_blocks_untrusted
@@ -292,7 +277,7 @@ module Gitlab
end
def utc_date_regex
- @utc_date_regex ||= /\A[0-9]{4}-[0-9]{2}-[0-9]{2}\z/.freeze
+ @utc_date_regex ||= /\A[0-9]{4}-[0-9]{2}-[0-9]{2}\z/
end
def issue
@@ -304,7 +289,7 @@ module Gitlab
end
def base64_regex
- @base64_regex ||= %r{(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?}.freeze
+ @base64_regex ||= %r{(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?}
end
def feature_flag_regex
@@ -317,7 +302,7 @@ module Gitlab
end
def x509_subject_key_identifier_regex
- @x509_subject_key_identifier_regex ||= /\A(?:\h{2}:)*\h{2}\z/.freeze
+ @x509_subject_key_identifier_regex ||= /\A(?:\h{2}:)*\h{2}\z/
end
def ml_model_name_regex
diff --git a/lib/gitlab/regex/packages.rb b/lib/gitlab/regex/packages.rb
index 107f2070801..6b178933a25 100644
--- a/lib/gitlab/regex/packages.rb
+++ b/lib/gitlab/regex/packages.rb
@@ -9,34 +9,34 @@ module Gitlab
PYPI_NORMALIZED_NAME_REGEX_STRING = '[-_.]+'
# see https://github.com/apache/maven/blob/c1dfb947b509e195c75d4275a113598cf3063c3e/maven-artifact/src/main/java/org/apache/maven/artifact/Artifact.java#L46
- MAVEN_SNAPSHOT_DYNAMIC_PARTS = /\A.{0,1000}(-\d{8}\.\d{6}-\d+).{0,1000}\z/.freeze
+ MAVEN_SNAPSHOT_DYNAMIC_PARTS = /\A.{0,1000}(-\d{8}\.\d{6}-\d+).{0,1000}\z/
- API_PATH_REGEX = %r{^/api/v\d+/(projects/[^/]+/|groups?/[^/]+/-/)?packages/[A-Za-z]+}.freeze
+ API_PATH_REGEX = %r{^/api/v\d+/(projects/[^/]+/|groups?/[^/]+/-/)?packages/[A-Za-z]+}
def conan_package_reference_regex
- @conan_package_reference_regex ||= %r{\A[A-Za-z0-9]+\z}.freeze
+ @conan_package_reference_regex ||= %r{\A[A-Za-z0-9]+\z}
end
def conan_revision_regex
- @conan_revision_regex ||= %r{\A0\z}.freeze
+ @conan_revision_regex ||= %r{\A0\z}
end
def conan_recipe_user_channel_regex
- %r{\A(_|#{conan_name_regex})\z}.freeze
+ %r{\A(_|#{conan_name_regex})\z}
end
def conan_recipe_component_regex
# https://docs.conan.io/en/latest/reference/conanfile/attributes.html#name
- @conan_recipe_component_regex ||= %r{\A#{conan_name_regex}\z}.freeze
+ @conan_recipe_component_regex ||= %r{\A#{conan_name_regex}\z}
end
def composer_package_version_regex
# see https://github.com/composer/semver/blob/31f3ea725711245195f62e54ffa402d8ef2fdba9/src/VersionParser.php#L215
- @composer_package_version_regex ||= %r{\Av?((\d++)(\.(?:\d++|[xX*]))?(\.(?:\d++|[xX*]))?(\.(?:\d++|[xX*]))?)?\z}.freeze
+ @composer_package_version_regex ||= %r{\Av?((\d++)(\.(?:\d++|[xX*]))?(\.(?:\d++|[xX*]))?(\.(?:\d++|[xX*]))?)?\z}
end
def composer_dev_version_regex
- @composer_dev_version_regex ||= %r{(^dev-)|(-dev$)}.freeze
+ @composer_dev_version_regex ||= %r{(^dev-)|(-dev$)}
end
def package_name_regex
@@ -51,23 +51,23 @@ module Gitlab
(([\w\-\.\+]*)\/)*([\w\-\.]*)
)
\z
- }x.freeze
+ }x
end
def maven_file_name_regex
- @maven_file_name_regex ||= %r{\A[A-Za-z0-9\.\_\-\+]+\z}.freeze
+ @maven_file_name_regex ||= %r{\A[A-Za-z0-9\.\_\-\+]+\z}
end
def maven_path_regex
- @maven_path_regex ||= %r{\A\@?(([\w\-\.]*)/)*([\w\-\.\+]*)\z}.freeze
+ @maven_path_regex ||= %r{\A\@?(([\w\-\.]*)/)*([\w\-\.\+]*)\z}
end
def maven_app_name_regex
- @maven_app_name_regex ||= /\A[\w\-\.]+\z/.freeze
+ @maven_app_name_regex ||= /\A[\w\-\.]+\z/
end
def maven_version_regex
- @maven_version_regex ||= /\A(?!.*\.\.)[\w+.-]+\z/.freeze
+ @maven_version_regex ||= /\A(?!.*\.\.)[\w+.-]+\z/
end
def maven_app_group_regex
@@ -83,7 +83,7 @@ module Gitlab
end
def nuget_package_name_regex
- @nuget_package_name_regex ||= %r{\A[-+\.\_a-zA-Z0-9]+\z}.freeze
+ @nuget_package_name_regex ||= %r{\A[-+\.\_a-zA-Z0-9]+\z}
end
def nuget_version_regex
@@ -93,11 +93,11 @@ module Gitlab
(\.#{_semver_patch_regex})?
(\.\d*)?
#{_semver_prerelease_build_regex}\z
- /x.freeze
+ /x
end
def terraform_module_package_name_regex
- @terraform_module_package_name_regex ||= %r{\A[-a-z0-9]+\/[-a-z0-9]+\z}.freeze
+ @terraform_module_package_name_regex ||= %r{\A[-a-z0-9]+\/[-a-z0-9]+\z}
end
def pypi_version_regex
@@ -112,7 +112,7 @@ module Gitlab
((?:-([0-9]+))|(?:[-_\.]?(post|rev|r)[-_\.]?([0-9]+)?))? (?# post release)
([-_\.]?(dev)[-_\.]?([0-9]+)?)? (?# dev release)
(?:\+([a-z0-9]+(?:[-_\.][a-z0-9]+)*))? (?# local version)
- )\z}xi.freeze
+ )\z}xi
end
def debian_package_name_regex
@@ -121,7 +121,7 @@ module Gitlab
# @debian_package_name_regex ||= %r{\A[a-z0-9][-+\._a-z0-9]*\z}i.freeze
# But we prefer a more strict version from Lintian
# https://salsa.debian.org/lintian/lintian/-/blob/5080c0068ffc4a9ddee92022a91d0c2ff53e56d1/lib/Lintian/Util.pm#L116
- @debian_package_name_regex ||= %r{\A[a-z0-9][-+\.a-z0-9]+\z}.freeze
+ @debian_package_name_regex ||= %r{\A[a-z0-9][-+\.a-z0-9]+\z}
end
def debian_version_regex
@@ -132,33 +132,33 @@ module Gitlab
([0-9][0-9a-z\.+~]*) (?# version)
(-[0-9a-z\.+~]+){0,14} (?# -revision)
(?<!-)
- )\z}xi.freeze
+ )\z}xi
end
def debian_architecture_regex
# See official parser: https://git.dpkg.org/cgit/dpkg/dpkg.git/tree/lib/dpkg/arch.c?id=9e0c88ec09475f4d1addde9cdba1ad7849720356#n43
# But we limit to lower case
- @debian_architecture_regex ||= %r{\A#{::Packages::Debian::ARCHITECTURE_REGEX}\z}o.freeze
+ @debian_architecture_regex ||= %r{\A#{::Packages::Debian::ARCHITECTURE_REGEX}\z}o
end
def debian_distribution_regex
- @debian_distribution_regex ||= %r{\A#{::Packages::Debian::DISTRIBUTION_REGEX}\z}io.freeze
+ @debian_distribution_regex ||= %r{\A#{::Packages::Debian::DISTRIBUTION_REGEX}\z}io
end
def debian_component_regex
- @debian_component_regex ||= %r{\A#{::Packages::Debian::COMPONENT_REGEX}\z}o.freeze
+ @debian_component_regex ||= %r{\A#{::Packages::Debian::COMPONENT_REGEX}\z}o
end
def debian_direct_upload_filename_regex
- @debian_direct_upload_filename_regex ||= %r{\A.*\.(deb|udeb|ddeb)\z}o.freeze
+ @debian_direct_upload_filename_regex ||= %r{\A.*\.(deb|udeb|ddeb)\z}o
end
def helm_channel_regex
- @helm_channel_regex ||= %r{\A([a-zA-Z0-9](\.|-|_)?){1,255}(?<!\.|-|_)\z}.freeze
+ @helm_channel_regex ||= %r{\A([a-zA-Z0-9](\.|-|_)?){1,255}(?<!\.|-|_)\z}
end
def helm_package_regex
- @helm_package_regex ||= %r{#{helm_channel_regex}}.freeze
+ @helm_package_regex ||= %r{#{helm_channel_regex}}
end
def helm_version_regex
@@ -174,7 +174,7 @@ module Gitlab
# only partially match "v0.0.0-20201230123456-abcdefabcdef".
@unbounded_semver_regex ||= /
#{_semver_major_minor_patch_regex}#{_semver_prerelease_build_regex}
- /x.freeze
+ /x
end
def semver_regex
@@ -190,32 +190,32 @@ module Gitlab
def _semver_major_minor_patch_regex
@_semver_major_minor_patch_regex ||= /
#{_semver_major_regex}\.#{_semver_minor_regex}\.#{_semver_patch_regex}
- /x.freeze
+ /x
end
def _semver_major_regex
@_semver_major_regex ||= /
(?<major>0|[1-9]\d*)
- /x.freeze
+ /x
end
def _semver_minor_regex
@_semver_minor_regex ||= /
(?<minor>0|[1-9]\d*)
- /x.freeze
+ /x
end
def _semver_patch_regex
@_semver_patch_regex ||= /
(?<patch>0|[1-9]\d*)
- /x.freeze
+ /x
end
def _semver_prerelease_build_regex
@_semver_prerelease_build_regex ||= /
(?:-(?<prerelease>(?:\d*[a-zA-Z-][0-9a-zA-Z-]*|[1-9]\d*|0)(?:\.(?:\d*[a-zA-Z-][0-9a-zA-Z-]*|[1-9]\d*|0))*))?
(?:\+(?<build>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?
- /x.freeze
+ /x
end
def prefixed_semver_regex
@@ -240,7 +240,7 @@ module Gitlab
| %[0-9a-f]{2})* (?# URL encoded character)
)? (?# path)
\b (?# word boundary)
- }ix.freeze
+ }ix
end
def generic_package_version_regex
@@ -256,17 +256,17 @@ module Gitlab
end
def sha256_regex
- @sha256_regex ||= /\A[0-9a-f]{64}\z/i.freeze
+ @sha256_regex ||= /\A[0-9a-f]{64}\z/i
end
def slack_link_regex
- @slack_link_regex ||= /<(.*[|].*)>/i.freeze
+ @slack_link_regex ||= /<(.*[|].*)>/i
end
private
def conan_name_regex
- @conan_name_regex ||= %r{[a-zA-Z0-9_][a-zA-Z0-9_\+\.-]{1,49}}.freeze
+ @conan_name_regex ||= %r{[a-zA-Z0-9_][a-zA-Z0-9_\+\.-]{1,49}}
end
end
end
diff --git a/lib/gitlab/request_forgery_protection.rb b/lib/gitlab/request_forgery_protection.rb
index d5e80053772..3a389d3363f 100644
--- a/lib/gitlab/request_forgery_protection.rb
+++ b/lib/gitlab/request_forgery_protection.rb
@@ -6,8 +6,7 @@
module Gitlab
module RequestForgeryProtection
- # rubocop:disable Rails/ApplicationController
- class Controller < ActionController::Base
+ class Controller < BaseActionController
protect_from_forgery with: :exception, prepend: true
def initialize
@@ -40,6 +39,5 @@ module Gitlab
rescue ActionController::InvalidAuthenticityToken
false
end
- # rubocop:enable Rails/ApplicationController
end
end
diff --git a/lib/gitlab/robots_txt/parser.rb b/lib/gitlab/robots_txt/parser.rb
index 604d2f9b35b..82505ff031a 100644
--- a/lib/gitlab/robots_txt/parser.rb
+++ b/lib/gitlab/robots_txt/parser.rb
@@ -3,8 +3,8 @@
module Gitlab
module RobotsTxt
class Parser
- DISALLOW_REGEX = /^disallow: /i.freeze
- ALLOW_REGEX = /^allow: /i.freeze
+ DISALLOW_REGEX = /^disallow: /i
+ ALLOW_REGEX = /^allow: /i
attr_reader :disallow_rules, :allow_rules
diff --git a/lib/gitlab/runtime.rb b/lib/gitlab/runtime.rb
index f74f1489405..269fb74ceca 100644
--- a/lib/gitlab/runtime.rb
+++ b/lib/gitlab/runtime.rb
@@ -78,16 +78,16 @@ module Gitlab
def puma_in_clustered_mode?
return unless puma?
- return unless Puma.respond_to?(:cli_config)
+ return unless ::Puma.respond_to?(:cli_config)
- Puma.cli_config.options[:workers].to_i > 0
+ ::Puma.cli_config.options[:workers].to_i > 0
end
def max_threads
threads = 1 # main thread
- if puma? && Puma.respond_to?(:cli_config)
- threads += Puma.cli_config.options[:max_threads]
+ if puma? && ::Puma.respond_to?(:cli_config)
+ threads += ::Puma.cli_config.options[:max_threads]
elsif sidekiq?
# 2 extra threads for the pollers in Sidekiq and Sidekiq Cron:
# https://github.com/ondrejbartas/sidekiq-cron#under-the-hood
diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb
index 35e01101b3b..d06f414bd9a 100644
--- a/lib/gitlab/search_results.rb
+++ b/lib/gitlab/search_results.rb
@@ -238,9 +238,7 @@ module Gitlab
def filter_milestones_by_project(milestones)
candidate_project_ids = project_ids_relation
- if Feature.enabled?(:search_milestones_hide_archived_projects, current_user) && !filters[:include_archived]
- candidate_project_ids = candidate_project_ids.non_archived
- end
+ candidate_project_ids = candidate_project_ids.non_archived unless filters[:include_archived]
project_ids = milestones.of_projects(candidate_project_ids).select(:project_id).distinct.pluck(:project_id) # rubocop: disable CodeReuse/ActiveRecord
diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb
index ba822955133..15facc4bb2f 100644
--- a/lib/gitlab/shell.rb
+++ b/lib/gitlab/shell.rb
@@ -14,11 +14,6 @@ module Gitlab
class Shell
Error = Class.new(StandardError)
- PERMITTED_ACTIONS = %w[
- mv_repository remove_repository add_namespace rm_namespace mv_namespace
- repository_exists?
- ].freeze
-
class << self
# Retrieve GitLab Shell secret token
#
@@ -80,105 +75,6 @@ module Gitlab
end
end
- # Move or rename a repository
- #
- # @example Move/rename a repository
- # mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new")
- #
- # @param [String] storage project's storage path
- # @param [String] disk_path current project path on disk
- # @param [String] new_disk_path new project path on disk
- # @return [Boolean] whether repository could be moved/renamed on disk
- #
- # @deprecated
- def mv_repository(storage, disk_path, new_disk_path)
- return false if disk_path.empty? || new_disk_path.empty?
-
- Gitlab::Git::Repository.new(storage, "#{disk_path}.git", nil, nil).rename("#{new_disk_path}.git")
-
- true
- rescue StandardError => e
- Gitlab::ErrorTracking.track_exception(e, path: disk_path, new_path: new_disk_path, storage: storage)
-
- false
- end
-
- # Removes a repository from file system, using rm_diretory which is an alias
- # for rm_namespace. Given the underlying implementation removes the name
- # passed as second argument on the passed storage.
- #
- # @example Remove a repository
- # remove_repository("/path/to/storage", "gitlab/gitlab-ci")
- #
- # @param [String] storage project's storage path
- # @param [String] disk_path current project path on disk
- #
- # @deprecated
- def remove_repository(storage, disk_path)
- return false if disk_path.empty?
-
- Gitlab::Git::Repository.new(storage, "#{disk_path}.git", nil, nil).remove
-
- true
- rescue StandardError => e
- Gitlab::AppLogger.warn("Repository does not exist: #{e} at: #{disk_path}.git")
- Gitlab::ErrorTracking.track_exception(e, path: disk_path, storage: storage)
-
- false
- end
-
- # Add empty directory for storing repositories
- #
- # @example Add new namespace directory
- # add_namespace("default", "gitlab")
- #
- # @param [String] storage project's storage path
- # @param [String] name namespace name
- #
- # @deprecated
- def add_namespace(storage, name)
- Gitlab::GitalyClient.allow_n_plus_1_calls do
- Gitlab::GitalyClient::NamespaceService.new(storage).add(name)
- end
- rescue GRPC::InvalidArgument => e
- raise ArgumentError, e.message
- end
-
- # Remove directory from repositories storage
- # Every repository inside this directory will be removed too
- #
- # @example Remove namespace directory
- # rm_namespace("default", "gitlab")
- #
- # @param [String] storage project's storage path
- # @param [String] name namespace name
- #
- # @deprecated
- def rm_namespace(storage, name)
- Gitlab::GitalyClient::NamespaceService.new(storage).remove(name)
- rescue GRPC::InvalidArgument => e
- raise ArgumentError, e.message
- end
- alias_method :rm_directory, :rm_namespace
-
- # Move namespace directory inside repositories storage
- #
- # @example Move/rename a namespace directory
- # mv_namespace("/path/to/storage", "gitlab", "gitlabhq")
- #
- # @param [String] storage project's storage path
- # @param [String] old_name current namespace name
- # @param [String] new_name new namespace name
- #
- # @deprecated
- def mv_namespace(storage, old_name, new_name)
- Gitlab::GitalyClient::NamespaceService.new(storage).rename(old_name, new_name)
- rescue GRPC::InvalidArgument => e
- Gitlab::ErrorTracking.track_exception(e, old_name: old_name, new_name: new_name, storage: storage)
-
- false
- end
-
# Check if repository exists on disk
#
# @example Check if repository exists
diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb
index 7cc57f9497f..a1363e7b6b2 100644
--- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb
+++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb
@@ -257,12 +257,7 @@ module Gitlab
end
def with_redis(&block)
- if Feature.enabled?(:use_primary_and_secondary_stores_for_queues_metadata) ||
- Feature.enabled?(:use_primary_store_as_default_for_queues_metadata)
- Gitlab::Redis::QueuesMetadata.with(&block) # rubocop:disable CodeReuse/ActiveRecord
- else
- Gitlab::Redis::Queues.with(&block) # rubocop:disable Cop/RedisQueueUsage, CodeReuse/ActiveRecord
- end
+ Gitlab::Redis::QueuesMetadata.with(&block) # rubocop:disable CodeReuse/ActiveRecord
end
end
end
diff --git a/lib/gitlab/sidekiq_middleware/extra_done_log_metadata.rb b/lib/gitlab/sidekiq_middleware/extra_done_log_metadata.rb
index 93c3131d50e..0b1dc9c219e 100644
--- a/lib/gitlab/sidekiq_middleware/extra_done_log_metadata.rb
+++ b/lib/gitlab/sidekiq_middleware/extra_done_log_metadata.rb
@@ -5,7 +5,7 @@ module Gitlab
class ExtraDoneLogMetadata
def call(worker, job, queue)
yield
-
+ ensure
# We needed a way to pass state from a worker in to the
# Gitlab::SidekiqLogging::StructuredLogger . Unfortunately the
# StructuredLogger itself is not a middleware so cannot access the
diff --git a/lib/gitlab/sidekiq_middleware/skip_jobs.rb b/lib/gitlab/sidekiq_middleware/skip_jobs.rb
index 6cc394aa5f4..34ad843e8ee 100644
--- a/lib/gitlab/sidekiq_middleware/skip_jobs.rb
+++ b/lib/gitlab/sidekiq_middleware/skip_jobs.rb
@@ -67,6 +67,7 @@ module Gitlab
# always returns true by default for all workers unless the FF is specifically disabled, e.g. during an incident
Feature.enabled?(
:"#{RUN_FEATURE_FLAG_PREFIX}_#{worker_class.name}",
+ Feature.current_request,
type: :worker,
default_enabled_if_undefined: true
)
@@ -94,6 +95,7 @@ module Gitlab
def drop_job?(worker_class)
Feature.enabled?(
:"#{DROP_FEATURE_FLAG_PREFIX}_#{worker_class.name}",
+ Feature.current_request,
type: :worker,
default_enabled_if_undefined: false
)
diff --git a/lib/gitlab/slash_commands/run.rb b/lib/gitlab/slash_commands/run.rb
index 40fd7ee4f20..c5330c551a1 100644
--- a/lib/gitlab/slash_commands/run.rb
+++ b/lib/gitlab/slash_commands/run.rb
@@ -13,7 +13,7 @@ module Gitlab
end
def self.available?(project)
- Chat.available? && project.builds_enabled?
+ project.builds_enabled?
end
def self.allowed?(project, user)
diff --git a/lib/gitlab/time_tracking_formatter.rb b/lib/gitlab/time_tracking_formatter.rb
index 26efb3b918d..c72a58d1ce0 100644
--- a/lib/gitlab/time_tracking_formatter.rb
+++ b/lib/gitlab/time_tracking_formatter.rb
@@ -15,12 +15,7 @@ module Gitlab
seconds =
begin
- ChronicDuration.parse(
- string,
- CUSTOM_DAY_AND_MONTH_LENGTH.merge(
- default_unit: 'hours', keep_zero: keep_zero,
- use_complete_matcher: true
- ))
+ ChronicDuration.parse(string, CUSTOM_DAY_AND_MONTH_LENGTH.merge(default_unit: 'hours', keep_zero: keep_zero))
rescue StandardError
nil
end
diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb
index 57d3b3ec6f9..8f2dfce67bb 100644
--- a/lib/gitlab/url_blocker.rb
+++ b/lib/gitlab/url_blocker.rb
@@ -1,12 +1,16 @@
# frozen_string_literal: true
+#
+# IMPORTANT: With the new development of the 'gitlab-http' gem (https://gitlab.com/gitlab-org/gitlab/-/issues/415686),
+# no additional change should be implemented in this class. This class will be removed after migrating all
+# the usages to the new gem.
+#
+
require 'resolv'
require 'ipaddress'
module Gitlab
class UrlBlocker
- BlockedUrlError = Class.new(StandardError)
-
DENY_ALL_REQUESTS_EXCEPT_ALLOWED_DEFAULT = proc { deny_all_requests_except_allowed_app_setting }.freeze
# Result stores the validation result:
@@ -77,7 +81,7 @@ module Gitlab
return Result.new(uri, nil, proxy_in_use) unless enforce_address_info_retrievable?(uri, dns_rebind_protection, deny_all_requests_except_allowed)
- raise BlockedUrlError, 'Host cannot be resolved or invalid'
+ raise Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError, 'Host cannot be resolved or invalid'
end
ip_address = ip_address(address_info)
@@ -112,7 +116,7 @@ module Gitlab
validate!(url, **kwargs)
false
- rescue BlockedUrlError
+ rescue Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError
true
end
@@ -173,7 +177,7 @@ module Gitlab
#
# @param uri [Addressable::URI]
#
- # @raise [Gitlab::UrlBlocker::BlockedUrlError, ArgumentError] - BlockedUrlError raised if host is too long.
+ # @raise [Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError, ArgumentError] - raised if host is too long.
#
# @return [Array<Addrinfo>]
def get_address_info(uri)
@@ -184,7 +188,7 @@ module Gitlab
# Addrinfo.getaddrinfo errors if the domain exceeds 1024 characters.
raise unless error.message.include?('hostname too long')
- raise BlockedUrlError, "Host is too long (maximum is 1024 characters)"
+ raise Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError, "Host is too long (maximum is 1024 characters)"
end
def enforce_address_info_retrievable?(uri, dns_rebind_protection, deny_all_requests_except_allowed)
@@ -232,7 +236,7 @@ module Gitlab
netmask = IPAddr.new('100.64.0.0/10')
return unless addrs_info.any? { |addr| netmask.include?(addr.ip_address) }
- raise BlockedUrlError, "Requests to the shared address space are not allowed"
+ raise Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError, "Requests to the shared address space are not allowed"
end
def get_port(uri)
@@ -243,7 +247,7 @@ module Gitlab
uri_str = uri.to_s
sanitized_uri = ActionController::Base.helpers.sanitize(uri_str, tags: [])
if sanitized_uri != uri_str
- raise BlockedUrlError, 'HTML/CSS/JS tags are not allowed'
+ raise Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError, 'HTML/CSS/JS tags are not allowed'
end
end
@@ -252,7 +256,7 @@ module Gitlab
raise Addressable::URI::InvalidURIError if multiline_blocked?(parsed_url)
end
rescue Addressable::URI::InvalidURIError, URI::InvalidURIError
- raise BlockedUrlError, 'URI is invalid'
+ raise Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError, 'URI is invalid'
end
def multiline_blocked?(parsed_url)
@@ -271,12 +275,13 @@ module Gitlab
return if port >= 1024
return if ports.include?(port)
- raise BlockedUrlError, "Only allowed ports are #{ports.join(', ')}, and any over 1024"
+ raise Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError,
+ "Only allowed ports are #{ports.join(', ')}, and any over 1024"
end
def validate_scheme(scheme, schemes)
if scheme.blank? || (schemes.any? && schemes.exclude?(scheme))
- raise BlockedUrlError, "Only allowed schemes are #{schemes.join(', ')}"
+ raise Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError, "Only allowed schemes are #{schemes.join(', ')}"
end
end
@@ -284,7 +289,7 @@ module Gitlab
return if value.blank?
return if /\A\p{Alnum}/.match?(value)
- raise BlockedUrlError, "Username needs to start with an alphanumeric character"
+ raise Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError, "Username needs to start with an alphanumeric character"
end
def validate_hostname(value)
@@ -292,13 +297,13 @@ module Gitlab
return if IPAddress.valid?(value)
return if /\A\p{Alnum}/.match?(value)
- raise BlockedUrlError, "Hostname or IP address invalid"
+ raise Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError, "Hostname or IP address invalid"
end
def validate_unicode_restriction(uri)
return if uri.to_s.ascii_only?
- raise BlockedUrlError, "URI must be ascii only #{uri.to_s.dump}"
+ raise Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError, "URI must be ascii only #{uri.to_s.dump}"
end
def validate_localhost(addrs_info)
@@ -307,38 +312,39 @@ module Gitlab
return if (local_ips & addrs_info.map(&:ip_address)).empty?
- raise BlockedUrlError, "Requests to localhost are not allowed"
+ raise Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError, "Requests to localhost are not allowed"
end
def validate_loopback(addrs_info)
return unless addrs_info.any? { |addr| addr.ipv4_loopback? || addr.ipv6_loopback? }
- raise BlockedUrlError, "Requests to loopback addresses are not allowed"
+ raise Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError, "Requests to loopback addresses are not allowed"
end
def validate_local_network(addrs_info)
return unless addrs_info.any? { |addr| addr.ipv4_private? || addr.ipv6_sitelocal? || addr.ipv6_unique_local? }
- raise BlockedUrlError, "Requests to the local network are not allowed"
+ raise Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError, "Requests to the local network are not allowed"
end
def validate_link_local(addrs_info)
netmask = IPAddr.new('169.254.0.0/16')
return unless addrs_info.any? { |addr| addr.ipv6_linklocal? || netmask.include?(addr.ip_address) }
- raise BlockedUrlError, "Requests to the link local network are not allowed"
+ raise Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError, "Requests to the link local network are not allowed"
end
- # Raises a BlockedUrlError if the instance is configured to deny all requests.
+ # Raises a Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError if the instance is configured to deny all requests.
#
# This should only be called after allow list checks have been made.
def validate_deny_all_requests_except_allowed!(should_deny)
return unless deny_all_requests_except_allowed?(should_deny)
- raise BlockedUrlError, "Requests to hosts and IP addresses not on the Allow List are denied"
+ raise Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError,
+ "Requests to hosts and IP addresses not on the Allow List are denied"
end
- # Raises a BlockedUrlError if any IP in `addrs_info` is the limited
+ # Raises a Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError if any IP in `addrs_info` is the limited
# broadcast address.
# https://datatracker.ietf.org/doc/html/rfc919#section-7
def validate_limited_broadcast_address(addrs_info)
@@ -346,7 +352,7 @@ module Gitlab
return if (blocked_ips & addrs_info.map(&:ip_address)).empty?
- raise BlockedUrlError, "Requests to the limited broadcast address are not allowed"
+ raise Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError, "Requests to the limited broadcast address are not allowed"
end
def internal?(uri)
diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb
index e203fb486e7..1b7dcaa5cf4 100644
--- a/lib/gitlab/url_builder.rb
+++ b/lib/gitlab/url_builder.rb
@@ -91,6 +91,8 @@ module Gitlab
instance.merge_request_url(note.noteable, anchor: dom_id(note), **options)
elsif note.for_snippet?
instance.gitlab_snippet_url(note.noteable, anchor: dom_id(note), **options)
+ elsif note.for_abuse_report?
+ instance.admin_abuse_report_url(note.noteable, anchor: dom_id(note), **options)
end
end
diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb
index 450575b7223..7252283d1b9 100644
--- a/lib/gitlab/usage/metric_definition.rb
+++ b/lib/gitlab/usage/metric_definition.rb
@@ -28,13 +28,7 @@ module Gitlab
def to_context
return unless %w[redis redis_hll].include?(data_source)
- event_name = if data_source == 'redis_hll'
- options[:events].first
- elsif data_source == 'redis'
- Gitlab::Usage::Metrics::Instrumentations::RedisMetric.new(attributes).redis_key
- end
-
- Gitlab::Tracking::ServicePingContext.new(data_source: data_source, event: event_name)
+ Gitlab::Tracking::ServicePingContext.new(data_source: data_source, event: events.each_key.first)
end
def to_h
@@ -58,18 +52,16 @@ module Gitlab
end
def validate!
- unless skip_validation?
- self.class.schemer.validate(attributes.deep_stringify_keys).each do |error|
- error_message = <<~ERROR_MSG
- Error type: #{error['type']}
- Data: #{error['data']}
- Path: #{error['data_pointer']}
- Details: #{error['details']}
- Metric file: #{path}
- ERROR_MSG
-
- Gitlab::ErrorTracking.track_and_raise_for_dev_exception(InvalidError.new(error_message))
- end
+ self.class.schemer.validate(attributes.deep_stringify_keys).each do |error|
+ error_message = <<~ERROR_MSG
+ Error type: #{error['type']}
+ Data: #{error['data']}
+ Path: #{error['data_pointer']}
+ Details: #{error['details']}
+ Metric file: #{path}
+ ERROR_MSG
+
+ Gitlab::ErrorTracking.track_and_raise_for_dev_exception(InvalidError.new(error_message))
end
end
@@ -92,8 +84,7 @@ module Gitlab
@paths ||= [Rails.root.join('config', 'metrics', '[^agg]*', '*.yml')]
end
- def definitions(skip_validation: false)
- @skip_validation = skip_validation
+ def definitions
@definitions ||= load_all!
end
@@ -110,7 +101,7 @@ module Gitlab
end
def context_for(key_path)
- definitions[key_path].to_context
+ definitions[key_path]&.to_context
end
def schemer
@@ -121,19 +112,6 @@ module Gitlab
@metrics_yaml ||= definitions.values.map(&:to_h).map(&:deep_stringify_keys).to_yaml
end
- def metric_definitions_changed?
- return false unless Rails.env.development?
-
- return false if @last_change_check && @last_change_check > 3.seconds.ago
-
- @last_change_check = Time.current
-
- last_change = Dir.glob(paths).map { |f| File.mtime(f) }.max
- did_change = @last_metric_update != last_change
- @last_metric_update = last_change
- did_change
- end
-
private
def load_all!
@@ -147,7 +125,7 @@ module Gitlab
definition = YAML.safe_load(definition)
definition.deep_symbolize_keys!
- self.new(path, definition).tap(&:validate!).tap(&:category_to_lowercase)
+ self.new(path, definition).tap(&:category_to_lowercase)
rescue StandardError => e
Gitlab::ErrorTracking.track_and_raise_for_dev_exception(InvalidError.new(e.message))
end
@@ -175,15 +153,11 @@ module Gitlab
attributes[method].present? || super
end
- def skip_validation?
- !!attributes[:skip_validation] || @skip_validation
- end
-
def events_from_new_structure
events = attributes[:events]
return unless events
- events.to_h { |event| [event[:name], event[:unique].to_sym] }
+ events.to_h { |event| [event[:name], event[:unique]&.to_sym] }
end
def events_from_old_structure
diff --git a/lib/gitlab/usage/metrics/instrumentations/base_metric.rb b/lib/gitlab/usage/metrics/instrumentations/base_metric.rb
index 0c102f0f386..31948e30992 100644
--- a/lib/gitlab/usage/metrics/instrumentations/base_metric.rb
+++ b/lib/gitlab/usage/metrics/instrumentations/base_metric.rb
@@ -10,6 +10,7 @@ module Gitlab
attr_reader :time_frame
attr_reader :options
+ attr_reader :events
class << self
def available?(&block)
@@ -26,6 +27,7 @@ module Gitlab
def initialize(metric_definition)
@time_frame = metric_definition.fetch(:time_frame)
@options = metric_definition.fetch(:options, {})
+ @events = metric_definition.fetch(:events, {})
end
def instrumentation
diff --git a/lib/gitlab/usage/metrics/instrumentations/container_registry_db_enabled_metric.rb b/lib/gitlab/usage/metrics/instrumentations/container_registry_db_enabled_metric.rb
new file mode 100644
index 00000000000..f4306a3c319
--- /dev/null
+++ b/lib/gitlab/usage/metrics/instrumentations/container_registry_db_enabled_metric.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Usage
+ module Metrics
+ module Instrumentations
+ class ContainerRegistryDbEnabledMetric < GenericMetric
+ value do
+ Gitlab::CurrentSettings.container_registry_db_enabled
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/usage/metrics/instrumentations/count_csv_imports_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_csv_imports_metric.rb
new file mode 100644
index 00000000000..291484dd22a
--- /dev/null
+++ b/lib/gitlab/usage/metrics/instrumentations/count_csv_imports_metric.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Usage
+ module Metrics
+ module Instrumentations
+ class CountCsvImportsMetric < DatabaseMetric
+ operation :count
+
+ relation { ::Issues::CsvImport }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/usage/metrics/instrumentations/count_jira_imports_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_jira_imports_metric.rb
new file mode 100644
index 00000000000..7c32854bf4c
--- /dev/null
+++ b/lib/gitlab/usage/metrics/instrumentations/count_jira_imports_metric.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Usage
+ module Metrics
+ module Instrumentations
+ class CountJiraImportsMetric < DatabaseMetric
+ operation :count
+
+ relation { JiraImportState }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/usage/metrics/instrumentations/count_packages_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_packages_metric.rb
new file mode 100644
index 00000000000..61a3dba1942
--- /dev/null
+++ b/lib/gitlab/usage/metrics/instrumentations/count_packages_metric.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Usage
+ module Metrics
+ module Instrumentations
+ class CountPackagesMetric < DatabaseMetric
+ operation :count
+
+ relation { ::Packages::Package }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/usage/metrics/instrumentations/count_projects_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_projects_metric.rb
new file mode 100644
index 00000000000..3844aedf439
--- /dev/null
+++ b/lib/gitlab/usage/metrics/instrumentations/count_projects_metric.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Usage
+ module Metrics
+ module Instrumentations
+ class CountProjectsMetric < DatabaseMetric
+ operation :count
+
+ start { Project.minimum(:id) }
+ finish { Project.maximum(:id) }
+
+ relation { Project }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/usage/metrics/instrumentations/total_count_metric.rb b/lib/gitlab/usage/metrics/instrumentations/total_count_metric.rb
new file mode 100644
index 00000000000..d07438f4bf7
--- /dev/null
+++ b/lib/gitlab/usage/metrics/instrumentations/total_count_metric.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Usage
+ module Metrics
+ module Instrumentations
+ # Usage example
+ #
+ # In metric YAML definition:
+ #
+ # instrumentation_class: TotalCountMetric
+ # options:
+ # event: commit_pushed
+ #
+ class TotalCountMetric < BaseMetric
+ include Gitlab::UsageDataCounters::RedisCounter
+
+ KEY_PREFIX = "{event_counters}_"
+
+ def self.redis_key(event_name)
+ KEY_PREFIX + event_name
+ end
+
+ def value
+ events.sum do |event|
+ redis_usage_data do
+ total_count(self.class.redis_key(event[:name]))
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
index c3378856633..b2027791e9d 100644
--- a/lib/gitlab/usage_data.rb
+++ b/lib/gitlab/usage_data.rb
@@ -135,15 +135,6 @@ module Gitlab
}
end
# rubocop: enable Metrics/AbcSize
-
- def system_usage_data_monthly
- {
- counts_monthly: {
- projects: count(Project.where(monthly_time_range_db_params), start: minimum_id(Project), finish: maximum_id(Project)),
- packages: count(::Packages::Package.where(monthly_time_range_db_params))
- }
- }
- end
# rubocop: enable CodeReuse/ActiveRecord
def system_usage_data_license
@@ -379,8 +370,6 @@ module Gitlab
bulk_imports: {
gitlab_v1: count(::BulkImport.where(**time_period, source_type: :gitlab))
},
- project_imports: project_imports(time_period),
- issue_imports: issue_imports(time_period),
group_imports: group_imports(time_period)
}
end
@@ -498,7 +487,6 @@ module Gitlab
def usage_data_metrics
system_usage_data_license
.merge(system_usage_data)
- .merge(system_usage_data_monthly)
.merge(system_usage_data_weekly)
.merge(features_usage_data)
.merge(components_usage_data)
@@ -569,33 +557,6 @@ module Gitlab
omniauth_provider_names.reject { |name| name.starts_with?('ldap') }
end
- def project_imports(time_period)
- time_frame = metric_time_period(time_period)
- counters = {
- gitlab_project: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'gitlab_project' }),
- github: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'github' }),
- bitbucket: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'bitbucket' }),
- bitbucket_server: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'bitbucket_server' }),
- gitea: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'gitea' }),
- git: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'git' }),
- manifest: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'manifest' }),
- gitlab_migration: add_metric('CountBulkImportsEntitiesMetric', time_frame: time_frame, options: { source_type: :project_entity })
- }
-
- counters[:total] = add_metric('CountImportedProjectsTotalMetric', time_frame: time_frame)
-
- counters
- end
-
- def issue_imports(time_period)
- time_frame = metric_time_period(time_period)
- {
- jira: count(::JiraImportState.where(time_period)), # rubocop: disable CodeReuse/ActiveRecord
- fogbugz: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'fogbugz' }),
- csv: count(::Issues::CsvImport.where(time_period)) # rubocop: disable CodeReuse/ActiveRecord
- }
- end
-
def group_imports(time_period)
time_frame = metric_time_period(time_period)
{
diff --git a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb
index eb141a2e2f6..e0a4f879f48 100644
--- a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb
+++ b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb
@@ -13,11 +13,7 @@ module Gitlab::UsageDataCounters
Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name, values: project.id)
namespace = project.namespace
- context = Gitlab::Tracking::ServicePingContext.new(data_source: :redis_hll,
- event: event_name).to_context
- label = 'redis_hll_counters.ci_templates.ci_templates_total_unique_counts_monthly'
- Gitlab::Tracking.event(name, 'ci_templates_unique', namespace: namespace,
- project: project, context: [context], user: user, label: label)
+ Gitlab::InternalEvents.track_event('ci_template_included', namespace: namespace, project: project, user: user)
end
def ci_templates(relative_base = 'lib/gitlab/ci/templates')
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index d19dd6cd856..f2db7e3c9b9 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -260,12 +260,7 @@ module Gitlab
protected
def with_redis(&blk)
- if Feature.enabled?(:use_primary_and_secondary_stores_for_workhorse) ||
- Feature.enabled?(:use_primary_store_as_default_for_workhorse)
- Gitlab::Redis::Workhorse.with(&blk) # rubocop:disable CodeReuse/ActiveRecord
- else
- Gitlab::Redis::SharedState.with(&blk) # rubocop:disable CodeReuse/ActiveRecord
- end
+ Gitlab::Redis::Workhorse.with(&blk) # rubocop:disable CodeReuse/ActiveRecord
end
# This is the outermost encoding of a senddata: header. It is safe for