From 3b1af5cc7ed2666ff18b718ce5d30fa5a2756674 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Tue, 20 Jun 2023 10:43:29 +0000 Subject: Add latest changes from gitlab-org/gitlab@16-1-stable-ee --- lib/api/admin/batched_background_migrations.rb | 7 +- lib/api/admin/dictionary.rb | 61 +++++ lib/api/admin/migrations.rb | 62 +++++ lib/api/admin/plan_limits.rb | 10 +- lib/api/api.rb | 11 +- lib/api/api_guard.rb | 5 +- lib/api/badges.rb | 2 +- lib/api/ci/helpers/runner.rb | 6 + lib/api/ci/pipelines.rb | 3 +- lib/api/ci/runner.rb | 2 + lib/api/ci/secure_files.rb | 11 +- .../concerns/packages/nuget/private_endpoints.rb | 153 +++++++++++ .../concerns/packages/nuget/public_endpoints.rb | 42 ++++ lib/api/concerns/packages/nuget_endpoints.rb | 159 ------------ lib/api/debian_project_packages.rb | 6 +- lib/api/deploy_keys.rb | 4 +- lib/api/discussions.rb | 4 +- lib/api/entities/deploy_key.rb | 1 + lib/api/entities/dictionary/table.rb | 12 + lib/api/entities/draft_note.rb | 2 +- lib/api/entities/error_tracking.rb | 2 +- lib/api/entities/merge_request_basic.rb | 1 + lib/api/entities/namespace.rb | 8 + lib/api/entities/note.rb | 2 +- lib/api/entities/nuget/metadatum.rb | 6 + .../nuget/package_metadata_catalog_entry.rb | 3 +- lib/api/entities/nuget/search_result.rb | 2 - lib/api/entities/package.rb | 2 +- lib/api/entities/package_version.rb | 2 +- lib/api/entities/plan_limit.rb | 2 + lib/api/entities/project_integration_basic.rb | 2 + lib/api/entities/project_scope_link.rb | 10 + lib/api/error_tracking/collector.rb | 156 ------------ lib/api/group_avatar.rb | 2 +- lib/api/groups.rb | 30 +-- lib/api/helpers.rb | 13 + lib/api/helpers/integrations_helpers.rb | 90 ++++++- lib/api/helpers/notes_helpers.rb | 2 +- lib/api/helpers/packages/basic_auth_helpers.rb | 13 +- lib/api/helpers/packages/conan/api_helpers.rb | 45 ++-- lib/api/helpers/packages/npm.rb | 108 ++++---- lib/api/helpers/packages_helpers.rb | 18 +- lib/api/internal/error_tracking.rb | 2 +- lib/api/internal/kubernetes.rb | 15 +- lib/api/issues.rb | 7 +- lib/api/markdown.rb | 5 + lib/api/maven_packages.rb | 10 - lib/api/members.rb | 4 +- lib/api/ml/mlflow/entrypoint.rb | 2 +- lib/api/ml_model_packages.rb | 111 ++++++++ lib/api/namespaces.rb | 6 +- lib/api/notes.rb | 2 +- lib/api/npm_group_packages.rb | 25 ++ lib/api/npm_instance_packages.rb | 6 + lib/api/npm_project_packages.rb | 6 + lib/api/nuget_group_packages.rb | 39 ++- lib/api/nuget_project_packages.rb | 280 ++++++++++++--------- lib/api/project_hooks.rb | 2 +- lib/api/project_job_token_scope.rb | 130 ++++++++++ lib/api/project_packages.rb | 50 +++- lib/api/project_snippets.rb | 8 +- lib/api/project_templates.rb | 8 +- lib/api/projects.rb | 40 ++- lib/api/release/links.rb | 12 +- lib/api/releases.rb | 15 +- lib/api/resource_access_tokens.rb | 35 ++- lib/api/settings.rb | 9 +- lib/api/snippets.rb | 8 +- lib/api/system_hooks.rb | 2 +- lib/api/topics.rb | 2 +- lib/api/users.rb | 21 +- lib/api/v3/github.rb | 10 +- lib/api/validations/validators/absence.rb | 2 +- lib/api/validations/validators/array_none_any.rb | 2 +- lib/api/validations/validators/bulk_imports.rb | 6 +- .../validators/check_assignees_count.rb | 2 +- .../validations/validators/email_or_email_list.rb | 2 +- lib/api/validations/validators/file_path.rb | 4 +- lib/api/validations/validators/git_ref.rb | 2 +- lib/api/validations/validators/git_sha.rb | 2 +- .../validators/integer_or_custom_value.rb | 4 +- lib/api/validations/validators/limit.rb | 2 +- lib/api/validations/validators/project_portable.rb | 2 +- lib/api/validations/validators/untrusted_regexp.rb | 2 +- lib/atlassian/jira_issue_key_extractor.rb | 4 +- lib/backup/manager.rb | 17 +- lib/backup/repositories.rb | 20 +- lib/banzai/filter/autolink_filter.rb | 2 +- lib/banzai/filter/dollar_math_post_filter.rb | 4 +- lib/banzai/filter/inline_alert_metrics_filter.rb | 47 ---- lib/banzai/filter/inline_cluster_metrics_filter.rb | 40 --- lib/banzai/filter/inline_diff_filter.rb | 4 +- lib/banzai/filter/inline_embeds_filter.rb | 93 ------- lib/banzai/filter/inline_grafana_metrics_filter.rb | 79 ------ lib/banzai/filter/inline_metrics_filter.rb | 36 --- .../filter/inline_metrics_redactor_filter.rb | 154 ------------ lib/banzai/filter/references/reference_filter.rb | 4 +- .../filter/references/user_reference_filter.rb | 2 +- lib/banzai/filter/sanitization_filter.rb | 2 +- lib/banzai/filter/spaced_link_filter.rb | 2 +- lib/banzai/filter/table_of_contents_filter.rb | 4 +- lib/banzai/issuable_extractor.rb | 2 +- lib/banzai/pipeline/gfm_pipeline.rb | 9 - lib/banzai/pipeline/post_process_pipeline.rb | 1 - lib/bitbucket/representation/pull_request.rb | 10 +- lib/bulk_imports/clients/http.rb | 4 +- .../common/pipelines/lfs_objects_pipeline.rb | 4 +- .../common/pipelines/members_pipeline.rb | 2 +- .../common/pipelines/uploads_pipeline.rb | 2 +- .../transformers/member_attributes_transformer.rb | 51 ++++ lib/bulk_imports/file_downloads/validations.rb | 2 +- .../transformers/member_attributes_transformer.rb | 51 ---- .../projects/pipelines/design_bundle_pipeline.rb | 4 +- .../pipelines/repository_bundle_pipeline.rb | 4 +- lib/error_tracking/collector/payload_validator.rb | 13 - lib/error_tracking/collector/sentry_auth_parser.rb | 25 -- .../collector/sentry_request_parser.rb | 30 --- lib/error_tracking/stacktrace_builder.rb | 1 + lib/extracts_ref.rb | 30 ++- lib/feature/shared.rb | 11 + .../gitlab/analytics/internal_events_generator.rb | 278 ++++++++++++++++++++ .../instrumentation_class_spec.rb.template | 2 +- lib/gitlab/access.rb | 6 + lib/gitlab/access/branch_protection.rb | 4 + lib/gitlab/alert_management/payload/base.rb | 1 - lib/gitlab/alert_management/payload/prometheus.rb | 6 + lib/gitlab/analytics/date_filler.rb | 2 +- lib/gitlab/api_authentication/token_locator.rb | 44 +++- lib/gitlab/application_rate_limiter.rb | 2 +- lib/gitlab/asciidoc.rb | 3 +- lib/gitlab/asciidoc/include_processor.rb | 44 +++- lib/gitlab/audit/auditor.rb | 10 +- lib/gitlab/audit/type/definition.rb | 10 + lib/gitlab/auth.rb | 2 +- lib/gitlab/auth/o_auth/auth_hash.rb | 2 +- lib/gitlab/auth/saml/auth_hash.rb | 2 +- lib/gitlab/auth/saml/config.rb | 36 +-- lib/gitlab/auth/saml/user.rb | 2 +- lib/gitlab/authorized_keys.rb | 2 +- lib/gitlab/avatar_cache.rb | 8 +- .../backfill_ci_queuing_tables.rb | 153 ----------- ...backfill_code_suggestions_namespace_settings.rb | 33 +++ .../backfill_group_features.rb | 35 --- .../backfill_namespace_id_for_namespace_route.rb | 38 --- .../backfill_resource_link_events.rb | 71 ++++++ ...l_root_storage_statistics_fork_storage_sizes.rb | 99 ++++++++ .../cleanup_draft_data_from_faulty_regex.rb | 48 ---- .../encrypt_integration_properties.rb | 84 ------- .../encrypt_static_object_token.rb | 70 ------ .../fix_duplicate_project_name_and_path.rb | 82 ------ .../fix_incorrect_max_seats_used.rb | 13 - ...lity_occurrences_with_hashes_as_raw_metadata.rb | 124 --------- .../mark_duplicate_npm_packages_for_destruction.rb | 48 ++++ .../merge_topics_with_same_name.rb | 76 ------ ...rsonal_namespace_project_maintainer_to_owner.rb | 45 ---- ...igrate_shimo_confluence_integration_category.rb | 27 -- ...llify_creator_id_column_of_orphaned_projects.rb | 2 +- .../nullify_orphan_runner_id_on_ci_builds.rb | 44 ---- ...populate_container_repository_migration_plan.rb | 51 ---- .../populate_namespace_statistics.rb | 47 ---- .../populate_test_reports_issue_id.rb | 14 -- .../populate_topics_non_private_projects_count.rb | 36 --- .../populate_vulnerability_reads.rb | 84 ------- ...recalculate_vulnerabilities_occurrences_uuid.rb | 218 ---------------- ...ulnerability_finding_signatures_for_findings.rb | 13 - .../remove_all_trace_expiration_dates.rb | 53 ---- .../remove_invalid_deploy_access_level_groups.rb | 21 ++ ...emove_project_group_link_with_missing_groups.rb | 2 +- .../remove_vulnerability_finding_links.rb | 19 -- ...i_runners_token_encrypted_values_on_projects.rb | 40 --- ...uplicate_ci_runners_token_values_on_projects.rb | 40 --- .../update_timelogs_null_spent_at.rb | 39 --- lib/gitlab/bitbucket_import/importer.rb | 22 ++ lib/gitlab/bitbucket_server_import/importer.rb | 7 +- .../importers/repository_importer.rb | 8 + lib/gitlab/bitbucket_server_import/user_finder.rb | 2 +- lib/gitlab/cache/import/caching.rb | 4 +- lib/gitlab/cache/json_cache.rb | 123 +++++++++ lib/gitlab/cache/json_caches/json_keyed.rb | 41 +++ lib/gitlab/cache/json_caches/redis_keyed.rb | 31 +++ lib/gitlab/checks/branch_check.rb | 7 +- lib/gitlab/checks/diff_check.rb | 5 +- lib/gitlab/ci/badge/release/latest_release.rb | 3 +- lib/gitlab/ci/build/rules.rb | 30 ++- lib/gitlab/ci/config/entry/cache.rb | 8 +- lib/gitlab/ci/config/entry/include/rules/rule.rb | 8 +- lib/gitlab/ci/config/external/file/base.rb | 3 +- lib/gitlab/ci/config/external/file/project.rb | 27 -- lib/gitlab/ci/config/external/interpolator.rb | 127 ---------- lib/gitlab/ci/config/external/mapper/verifier.rb | 28 --- lib/gitlab/ci/config/external/rules.rb | 27 +- lib/gitlab/ci/config/yaml.rb | 45 ---- lib/gitlab/ci/config/yaml/interpolator.rb | 127 ++++++++++ lib/gitlab/ci/config/yaml/loader.rb | 48 ++++ lib/gitlab/ci/config/yaml/result.rb | 2 +- lib/gitlab/ci/decompressed_gzip_size_validator.rb | 2 +- lib/gitlab/ci/jwt_v2.rb | 27 +- lib/gitlab/ci/parsers/security/common.rb | 1 - lib/gitlab/ci/project_config.rb | 2 +- lib/gitlab/ci/project_config/repository.rb | 5 + lib/gitlab/ci/project_config/source.rb | 5 + lib/gitlab/ci/reports/security/finding.rb | 5 - lib/gitlab/ci/runner_instructions.rb | 2 + lib/gitlab/ci/secure_files/migration_helper.rb | 33 +++ lib/gitlab/ci/status/build/waiting_for_approval.rb | 38 +-- lib/gitlab/ci/status/scheduled.rb | 4 +- lib/gitlab/ci/status/success_warning.rb | 2 +- lib/gitlab/ci/templates/Android.gitlab-ci.yml | 36 +-- lib/gitlab/ci/templates/Flutter.gitlab-ci.yml | 8 +- lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml | 2 +- .../ci/templates/Jobs/Build.latest.gitlab-ci.yml | 2 +- .../ci/templates/Jobs/CF-Provision.gitlab-ci.yml | 2 +- .../ci/templates/Jobs/Code-Quality.gitlab-ci.yml | 2 +- .../Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml | 2 +- lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml | 2 +- .../ci/templates/Jobs/Deploy.latest.gitlab-ci.yml | 2 +- lib/gitlab/ci/templates/Kaniko.gitlab-ci.yml | 4 +- lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml | 5 +- .../ci/templates/Pages/Doxygen.gitlab-ci.yml | 5 +- lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml | 13 +- lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml | 5 +- lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml | 5 +- lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml | 13 +- lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml | 11 +- lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml | 21 +- lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml | 15 +- lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml | 43 ++-- lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml | 5 +- .../ci/templates/Pages/Metalsmith.gitlab-ci.yml | 5 +- .../ci/templates/Pages/Middleman.gitlab-ci.yml | 11 +- lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml | 5 +- .../ci/templates/Pages/Octopress.gitlab-ci.yml | 5 +- .../ci/templates/Pages/Pelican.gitlab-ci.yml | 5 +- .../ci/templates/Pages/SwaggerUI.gitlab-ci.yml | 13 +- lib/gitlab/ci/templates/Pages/Zola.gitlab-ci.yml | 30 +++ .../ci/templates/Security/DAST.gitlab-ci.yml | 8 +- .../templates/Security/DAST.latest.gitlab-ci.yml | 15 +- .../templates/Terraform/Base.latest.gitlab-ci.yml | 15 +- lib/gitlab/ci/variables/builder.rb | 8 - lib/gitlab/ci/variables/builder/pipeline.rb | 27 +- .../cluster/puma_worker_killer_initializer.rb | 51 ---- lib/gitlab/cluster/puma_worker_killer_observer.rb | 22 -- lib/gitlab/counters/buffered_counter.rb | 12 +- lib/gitlab/data_builder/pipeline.rb | 4 +- lib/gitlab/database.rb | 121 +++++---- lib/gitlab/database/async_indexes/index_base.rb | 3 +- .../database/async_indexes/postgres_async_index.rb | 24 +- .../database/background_migration/batched_job.rb | 2 +- .../background_migration/batched_migration.rb | 15 +- .../batched_migration_runner.rb | 2 +- .../database/background_migration/health_status.rb | 45 ---- .../health_status/indicators.rb | 12 - .../indicators/autovacuum_active_on_table.rb | 41 --- .../health_status/indicators/patroni_apdex.rb | 90 ------- .../health_status/indicators/write_ahead_log.rb | 74 ------ .../background_migration/health_status/signals.rb | 71 ------ .../convert_feature_category_to_group_label.rb | 37 +++ lib/gitlab/database/database_connection_info.rb | 71 ++++++ lib/gitlab/database/each_database.rb | 1 + lib/gitlab/database/gitlab_schema.rb | 85 +++---- lib/gitlab/database/gitlab_schema_info.rb | 28 +++ lib/gitlab/database/health_status.rb | 40 +++ lib/gitlab/database/health_status/context.rb | 28 +++ lib/gitlab/database/health_status/indicators.rb | 10 + .../indicators/autovacuum_active_on_table.rb | 39 +++ .../health_status/indicators/patroni_apdex.rb | 88 +++++++ .../health_status/indicators/write_ahead_log.rb | 71 ++++++ lib/gitlab/database/health_status/logger.rb | 15 ++ lib/gitlab/database/health_status/signals.rb | 63 +++++ lib/gitlab/database/load_balancing/host.rb | 81 ++++-- lib/gitlab/database/lock_writes_manager.rb | 9 +- .../migration_helpers/wraparound_autovacuum.rb | 38 +++ .../database/migrations/constraints_helpers.rb | 10 + lib/gitlab/database/partitioning.rb | 4 +- .../database/partitioning/list/convert_table.rb | 50 ++-- .../database/partitioning/sliding_list_strategy.rb | 13 + .../table_management_helpers.rb | 3 +- .../database/postgres_autovacuum_activity.rb | 7 +- .../prevent_cross_database_modification.rb | 50 ++-- .../query_analyzers/restrict_allowed_schemas.rb | 22 +- .../adapters/column_structure_sql_adapter.rb | 15 +- .../adapters/foreign_key_database_adapter.rb | 31 +++ .../adapters/foreign_key_structure_sql_adapter.rb | 50 ++++ lib/gitlab/database/schema_validation/database.rb | 38 +++ .../schema_validation/schema_inconsistency.rb | 4 +- .../schema_objects/foreign_key.rb | 34 +++ .../database/schema_validation/structure_sql.rb | 39 +++ .../schema_validation/track_inconsistency.rb | 58 ++++- .../schema_validation/validators/base_validator.rb | 5 +- .../different_definition_foreign_keys.rb | 24 ++ .../validators/extra_foreign_keys.rb | 21 ++ .../validators/missing_foreign_keys.rb | 21 ++ lib/gitlab/database/tables_locker.rb | 6 +- lib/gitlab/database/tables_truncate.rb | 2 +- lib/gitlab/database_importers/common_metrics.rb | 8 - .../database_importers/common_metrics/importer.rb | 78 ------ .../common_metrics/prometheus_metric.rb | 12 - .../common_metrics/prometheus_metric_enums.rb | 45 ---- .../default_organization_importer.rb | 19 ++ .../dependency_linker/requirements_txt_linker.rb | 2 +- lib/gitlab/devise_failure.rb | 8 + lib/gitlab/diff/formatters/base_formatter.rb | 1 + lib/gitlab/diff/formatters/file_formatter.rb | 33 +++ lib/gitlab/diff/formatters/image_formatter.rb | 1 + lib/gitlab/diff/formatters/text_formatter.rb | 4 +- lib/gitlab/diff/position.rb | 15 +- lib/gitlab/diff/position_tracer.rb | 16 +- lib/gitlab/diff/position_tracer/file_strategy.rb | 47 ++++ lib/gitlab/diff/position_tracer/image_strategy.rb | 31 +-- lib/gitlab/diff/position_tracer/line_strategy.rb | 5 +- lib/gitlab/discussions_diff/highlight_cache.rb | 12 +- lib/gitlab/email/handler/create_issue_handler.rb | 2 +- lib/gitlab/email/handler/service_desk_handler.rb | 2 +- lib/gitlab/email/hook/silent_mode_interceptor.rb | 12 +- lib/gitlab/email/reply_parser.rb | 4 +- lib/gitlab/error_tracking/error_repository.rb | 2 +- .../error_repository/open_api_strategy.rb | 10 +- lib/gitlab/etag_caching/middleware.rb | 2 +- lib/gitlab/etag_caching/router/rails.rb | 2 +- lib/gitlab/front_matter.rb | 2 +- lib/gitlab/git/repository.rb | 16 +- lib/gitlab/git/tag.rb | 16 +- lib/gitlab/git/tree.rb | 2 +- lib/gitlab/gitaly_client/commit_service.rb | 87 ++++++- lib/gitlab/gitaly_client/operation_service.rb | 15 +- lib/gitlab/gitaly_client/ref_service.rb | 4 +- .../github_gists_import/importer/gist_importer.rb | 51 +++- .../github_gists_import/representation/gist.rb | 4 + .../github_import/importer/repository_importer.rb | 8 + .../github_import/representation/diff_note.rb | 17 +- lib/gitlab/gl_repository.rb | 2 +- lib/gitlab/gl_repository/identifier.rb | 2 +- lib/gitlab/gon_helper.rb | 7 +- lib/gitlab/graphql/generic_tracing.rb | 6 +- lib/gitlab/hotlinking_detector.rb | 5 + lib/gitlab/http.rb | 25 ++ lib/gitlab/i18n.rb | 16 +- .../decompressed_archive_size_validator.rb | 2 +- lib/gitlab/import_export/group/import_export.yml | 1 - lib/gitlab/import_export/group/tree_restorer.rb | 29 ++- .../import_export/legacy_relation_tree_saver.rb | 23 -- .../project/exported_relations_merger.rb | 4 +- lib/gitlab/import_export/project/import_export.yml | 3 +- .../import_export/project/relation_factory.rb | 1 + .../import_export/recursive_merge_folders.rb | 6 +- lib/gitlab/import_export/repo_restorer.rb | 2 +- lib/gitlab/instrumentation/redis_base.rb | 4 +- .../instrumentation/redis_cluster_validator.rb | 11 +- lib/gitlab/internal_events.rb | 44 ++++ lib/gitlab/json_cache.rb | 118 --------- lib/gitlab/markdown_cache/redis/store.rb | 2 +- lib/gitlab/merge_requests/message_generator.rb | 1 + lib/gitlab/metrics/loose_foreign_keys_slis.rb | 2 +- lib/gitlab/metrics/subscribers/rails_cache.rb | 34 ++- lib/gitlab/middleware/compressed_json.rb | 4 +- lib/gitlab/pagination/cursor_based_keyset.rb | 3 +- lib/gitlab/patch/redis_cache_store.rb | 84 +++++++ lib/gitlab/patch/redis_cluster.rb | 21 ++ lib/gitlab/path_regex.rb | 16 ++ lib/gitlab/path_traversal.rb | 48 ++++ lib/gitlab/project_authorizations.rb | 140 +++++++---- lib/gitlab/quick_actions/relate_actions.rb | 46 +++- lib/gitlab/quick_actions/work_item_actions.rb | 90 +++++-- lib/gitlab/reactive_cache_set_cache.rb | 10 +- lib/gitlab/redis.rb | 4 +- lib/gitlab/redis/cache.rb | 40 ++- lib/gitlab/redis/chat.rb | 13 + lib/gitlab/redis/cluster_cache.rb | 13 + lib/gitlab/redis/cluster_util.rb | 32 +++ lib/gitlab/redis/cross_slot.rb | 141 +++++++++++ lib/gitlab/redis/multi_store.rb | 16 +- lib/gitlab/redis/rate_limiting.rb | 7 - lib/gitlab/regex.rb | 16 +- lib/gitlab/repository_hash_cache.rb | 6 +- .../resource_events/assignment_event_recorder.rb | 2 - lib/gitlab/search/abuse_detection.rb | 4 +- lib/gitlab/search/params.rb | 2 +- lib/gitlab/search_results.rb | 10 +- lib/gitlab/sentence.rb | 12 + lib/gitlab/set_cache.rb | 6 +- lib/gitlab/sidekiq_logging/structured_logger.rb | 22 +- lib/gitlab/sidekiq_middleware.rb | 5 +- lib/gitlab/sidekiq_middleware/defer_jobs.rb | 78 ++++++ lib/gitlab/silent_mode.rb | 21 ++ .../incident_management/incident_command.rb | 8 +- .../incident_management/incident_new.rb | 8 + lib/gitlab/slash_commands/issue_new.rb | 2 +- lib/gitlab/spamcheck/client.rb | 1 + .../template/finders/global_template_finder.rb | 2 +- .../template/finders/repo_template_finder.rb | 2 +- lib/gitlab/template/metrics_dashboard_template.rb | 31 --- lib/gitlab/tracking.rb | 16 +- lib/gitlab/tracking/standard_context.rb | 2 +- .../instrumentations/count_all_ci_builds_metric.rb | 15 ++ .../instrumentations/count_deployments_metric.rb | 40 +++ .../count_imported_projects_metric.rb | 4 +- .../count_personal_snippets_metric.rb | 17 ++ .../count_project_snippets_metric.rb | 17 ++ .../count_projects_with_alerts_created_metric.rb | 17 ++ .../instrumentations/count_snippets_metric.rb | 35 +++ .../installation_creation_date_metric.rb | 15 -- lib/gitlab/usage_data.rb | 57 +---- .../usage_data_counters/hll_redis_counter.rb | 80 ++---- ...rains_bundled_plugin_activity_unique_counter.rb | 29 +++ .../known_events/ci_templates.yml | 2 + .../known_events/code_review_events.yml | 2 + .../known_events/product_analytics.yml | 4 + .../known_events/quickactions.yml | 4 + .../known_events/workspaces.yml | 5 + .../kubernetes_agent_counter.rb | 2 +- .../merge_request_activity_unique_counter.rb | 17 +- lib/gitlab/utils.rb | 46 ---- lib/gitlab/utils/markdown.rb | 2 +- lib/gitlab/utils/sanitize_node_link.rb | 2 + lib/gitlab/verify/ci_secure_files.rb | 39 +++ lib/gitlab/x509/tag.rb | 2 +- lib/google_api/cloud_platform/client.rb | 4 + lib/google_cloud/authentication.rb | 20 ++ lib/google_cloud/logging_service/logger.rb | 41 +++ lib/grafana/validator.rb | 1 - lib/kramdown/parser/atlassian_document_format.rb | 4 +- lib/object_storage/fog_helpers.rb | 51 ++++ lib/object_storage/pending_direct_upload.rb | 88 ++++++- lib/product_analytics/settings.rb | 41 ++- lib/quality/seeders/issues.rb | 2 +- .../groups/menus/packages_registries_menu.rb | 2 +- .../groups/super_sidebar_menus/deploy_menu.rb | 26 ++ .../groups/super_sidebar_menus/operations_menu.rb | 3 +- .../groups/super_sidebar_menus/secure_menu.rb | 1 + lib/sidebars/groups/super_sidebar_panel.rb | 1 + lib/sidebars/projects/menus/deployments_menu.rb | 6 +- lib/sidebars/projects/menus/monitor_menu.rb | 18 -- .../projects/menus/packages_registries_menu.rb | 8 +- .../projects/super_sidebar_menus/analyze_menu.rb | 3 +- .../projects/super_sidebar_menus/build_menu.rb | 3 - .../projects/super_sidebar_menus/deploy_menu.rb | 30 +++ .../projects/super_sidebar_menus/monitor_menu.rb | 1 - .../super_sidebar_menus/operations_menu.rb | 5 +- lib/sidebars/projects/super_sidebar_panel.rb | 1 + lib/sidebars/user_profile/panel.rb | 3 + lib/tasks/cache.rake | 32 +-- lib/tasks/frontend.rake | 18 -- lib/tasks/gitlab/assets.rake | 1 + lib/tasks/gitlab/background_migrations.rake | 2 +- lib/tasks/gitlab/backup.rake | 10 + lib/tasks/gitlab/ci_secure_files/check.rake | 10 + lib/tasks/gitlab/ci_secure_files/migrate.rake | 23 ++ lib/tasks/gitlab/db.rake | 23 +- lib/tasks/gitlab/packages/events.rake | 5 +- lib/tasks/gitlab/tw/codeowners.rake | 10 +- lib/tasks/gitlab/usage_data.rake | 9 +- lib/tasks/tanuki_emoji.rake | 6 +- lib/tasks/tokens.rake | 9 +- 453 files changed, 6002 insertions(+), 4901 deletions(-) create mode 100644 lib/api/admin/dictionary.rb create mode 100644 lib/api/admin/migrations.rb create mode 100644 lib/api/concerns/packages/nuget/private_endpoints.rb create mode 100644 lib/api/concerns/packages/nuget/public_endpoints.rb delete mode 100644 lib/api/concerns/packages/nuget_endpoints.rb create mode 100644 lib/api/entities/dictionary/table.rb create mode 100644 lib/api/entities/project_scope_link.rb delete mode 100644 lib/api/error_tracking/collector.rb create mode 100644 lib/api/ml_model_packages.rb create mode 100644 lib/api/npm_group_packages.rb delete mode 100644 lib/banzai/filter/inline_alert_metrics_filter.rb delete mode 100644 lib/banzai/filter/inline_cluster_metrics_filter.rb delete mode 100644 lib/banzai/filter/inline_embeds_filter.rb delete mode 100644 lib/banzai/filter/inline_grafana_metrics_filter.rb delete mode 100644 lib/banzai/filter/inline_metrics_filter.rb delete mode 100644 lib/banzai/filter/inline_metrics_redactor_filter.rb create mode 100644 lib/bulk_imports/common/transformers/member_attributes_transformer.rb delete mode 100644 lib/bulk_imports/groups/transformers/member_attributes_transformer.rb delete mode 100644 lib/error_tracking/collector/payload_validator.rb delete mode 100644 lib/error_tracking/collector/sentry_auth_parser.rb delete mode 100644 lib/error_tracking/collector/sentry_request_parser.rb create mode 100644 lib/generators/gitlab/analytics/internal_events_generator.rb delete mode 100644 lib/gitlab/background_migration/backfill_ci_queuing_tables.rb create mode 100644 lib/gitlab/background_migration/backfill_code_suggestions_namespace_settings.rb delete mode 100644 lib/gitlab/background_migration/backfill_group_features.rb delete mode 100644 lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route.rb create mode 100644 lib/gitlab/background_migration/backfill_resource_link_events.rb create mode 100644 lib/gitlab/background_migration/backfill_root_storage_statistics_fork_storage_sizes.rb delete mode 100644 lib/gitlab/background_migration/cleanup_draft_data_from_faulty_regex.rb delete mode 100644 lib/gitlab/background_migration/encrypt_integration_properties.rb delete mode 100644 lib/gitlab/background_migration/encrypt_static_object_token.rb delete mode 100644 lib/gitlab/background_migration/fix_duplicate_project_name_and_path.rb delete mode 100644 lib/gitlab/background_migration/fix_incorrect_max_seats_used.rb delete mode 100644 lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb create mode 100644 lib/gitlab/background_migration/mark_duplicate_npm_packages_for_destruction.rb delete mode 100644 lib/gitlab/background_migration/merge_topics_with_same_name.rb delete mode 100644 lib/gitlab/background_migration/migrate_personal_namespace_project_maintainer_to_owner.rb delete mode 100644 lib/gitlab/background_migration/migrate_shimo_confluence_integration_category.rb delete mode 100644 lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds.rb delete mode 100644 lib/gitlab/background_migration/populate_container_repository_migration_plan.rb delete mode 100644 lib/gitlab/background_migration/populate_namespace_statistics.rb delete mode 100644 lib/gitlab/background_migration/populate_test_reports_issue_id.rb delete mode 100644 lib/gitlab/background_migration/populate_topics_non_private_projects_count.rb delete mode 100644 lib/gitlab/background_migration/populate_vulnerability_reads.rb delete mode 100644 lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb delete mode 100644 lib/gitlab/background_migration/recalculate_vulnerability_finding_signatures_for_findings.rb delete mode 100644 lib/gitlab/background_migration/remove_all_trace_expiration_dates.rb create mode 100644 lib/gitlab/background_migration/remove_invalid_deploy_access_level_groups.rb delete mode 100644 lib/gitlab/background_migration/remove_vulnerability_finding_links.rb delete mode 100644 lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values_on_projects.rb delete mode 100644 lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values_on_projects.rb delete mode 100644 lib/gitlab/background_migration/update_timelogs_null_spent_at.rb create mode 100644 lib/gitlab/cache/json_cache.rb create mode 100644 lib/gitlab/cache/json_caches/json_keyed.rb create mode 100644 lib/gitlab/cache/json_caches/redis_keyed.rb delete mode 100644 lib/gitlab/ci/config/external/interpolator.rb create mode 100644 lib/gitlab/ci/config/yaml/interpolator.rb create mode 100644 lib/gitlab/ci/config/yaml/loader.rb create mode 100644 lib/gitlab/ci/secure_files/migration_helper.rb create mode 100644 lib/gitlab/ci/templates/Pages/Zola.gitlab-ci.yml delete mode 100644 lib/gitlab/cluster/puma_worker_killer_initializer.rb delete mode 100644 lib/gitlab/cluster/puma_worker_killer_observer.rb delete mode 100644 lib/gitlab/database/background_migration/health_status.rb delete mode 100644 lib/gitlab/database/background_migration/health_status/indicators.rb delete mode 100644 lib/gitlab/database/background_migration/health_status/indicators/autovacuum_active_on_table.rb delete mode 100644 lib/gitlab/database/background_migration/health_status/indicators/patroni_apdex.rb delete mode 100644 lib/gitlab/database/background_migration/health_status/indicators/write_ahead_log.rb delete mode 100644 lib/gitlab/database/background_migration/health_status/signals.rb create mode 100644 lib/gitlab/database/convert_feature_category_to_group_label.rb create mode 100644 lib/gitlab/database/database_connection_info.rb create mode 100644 lib/gitlab/database/gitlab_schema_info.rb create mode 100644 lib/gitlab/database/health_status.rb create mode 100644 lib/gitlab/database/health_status/context.rb create mode 100644 lib/gitlab/database/health_status/indicators.rb create mode 100644 lib/gitlab/database/health_status/indicators/autovacuum_active_on_table.rb create mode 100644 lib/gitlab/database/health_status/indicators/patroni_apdex.rb create mode 100644 lib/gitlab/database/health_status/indicators/write_ahead_log.rb create mode 100644 lib/gitlab/database/health_status/logger.rb create mode 100644 lib/gitlab/database/health_status/signals.rb create mode 100644 lib/gitlab/database/migration_helpers/wraparound_autovacuum.rb create mode 100644 lib/gitlab/database/schema_validation/adapters/foreign_key_database_adapter.rb create mode 100644 lib/gitlab/database/schema_validation/adapters/foreign_key_structure_sql_adapter.rb create mode 100644 lib/gitlab/database/schema_validation/schema_objects/foreign_key.rb create mode 100644 lib/gitlab/database/schema_validation/validators/different_definition_foreign_keys.rb create mode 100644 lib/gitlab/database/schema_validation/validators/extra_foreign_keys.rb create mode 100644 lib/gitlab/database/schema_validation/validators/missing_foreign_keys.rb delete mode 100644 lib/gitlab/database_importers/common_metrics.rb delete mode 100644 lib/gitlab/database_importers/common_metrics/importer.rb delete mode 100644 lib/gitlab/database_importers/common_metrics/prometheus_metric.rb delete mode 100644 lib/gitlab/database_importers/common_metrics/prometheus_metric_enums.rb create mode 100644 lib/gitlab/database_importers/default_organization_importer.rb create mode 100644 lib/gitlab/diff/formatters/file_formatter.rb create mode 100644 lib/gitlab/diff/position_tracer/file_strategy.rb delete mode 100644 lib/gitlab/import_export/legacy_relation_tree_saver.rb create mode 100644 lib/gitlab/internal_events.rb delete mode 100644 lib/gitlab/json_cache.rb create mode 100644 lib/gitlab/patch/redis_cache_store.rb create mode 100644 lib/gitlab/patch/redis_cluster.rb create mode 100644 lib/gitlab/path_traversal.rb create mode 100644 lib/gitlab/redis/chat.rb create mode 100644 lib/gitlab/redis/cluster_cache.rb create mode 100644 lib/gitlab/redis/cluster_util.rb create mode 100644 lib/gitlab/redis/cross_slot.rb create mode 100644 lib/gitlab/sentence.rb create mode 100644 lib/gitlab/sidekiq_middleware/defer_jobs.rb create mode 100644 lib/gitlab/silent_mode.rb delete mode 100644 lib/gitlab/template/metrics_dashboard_template.rb create mode 100644 lib/gitlab/usage/metrics/instrumentations/count_all_ci_builds_metric.rb create mode 100644 lib/gitlab/usage/metrics/instrumentations/count_deployments_metric.rb create mode 100644 lib/gitlab/usage/metrics/instrumentations/count_personal_snippets_metric.rb create mode 100644 lib/gitlab/usage/metrics/instrumentations/count_project_snippets_metric.rb create mode 100644 lib/gitlab/usage/metrics/instrumentations/count_projects_with_alerts_created_metric.rb create mode 100644 lib/gitlab/usage/metrics/instrumentations/count_snippets_metric.rb delete mode 100644 lib/gitlab/usage/metrics/instrumentations/installation_creation_date_metric.rb create mode 100644 lib/gitlab/usage_data_counters/jetbrains_bundled_plugin_activity_unique_counter.rb create mode 100644 lib/gitlab/usage_data_counters/known_events/workspaces.yml create mode 100644 lib/gitlab/verify/ci_secure_files.rb create mode 100644 lib/google_cloud/authentication.rb create mode 100644 lib/google_cloud/logging_service/logger.rb create mode 100644 lib/object_storage/fog_helpers.rb create mode 100644 lib/sidebars/groups/super_sidebar_menus/deploy_menu.rb create mode 100644 lib/sidebars/projects/super_sidebar_menus/deploy_menu.rb create mode 100644 lib/tasks/gitlab/ci_secure_files/check.rake create mode 100644 lib/tasks/gitlab/ci_secure_files/migrate.rake (limited to 'lib') diff --git a/lib/api/admin/batched_background_migrations.rb b/lib/api/admin/batched_background_migrations.rb index 7e612b5b66a..c0d1ce8767d 100644 --- a/lib/api/admin/batched_background_migrations.rb +++ b/lib/api/admin/batched_background_migrations.rb @@ -142,9 +142,12 @@ module API @base_model ||= Gitlab::Database.database_base_models[database] end + # Force progress evaluation to occur now while we're using the right connection def present_entity(result) - present result, - with: ::API::Entities::BatchedBackgroundMigration + representation = entity_representation_for(::API::Entities::BatchedBackgroundMigration, result, {}) + json_representation = Gitlab::Json.dump(representation) + + body Gitlab::Json::PrecompiledJson.new(json_representation) end end end diff --git a/lib/api/admin/dictionary.rb b/lib/api/admin/dictionary.rb new file mode 100644 index 00000000000..038c122c021 --- /dev/null +++ b/lib/api/admin/dictionary.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +module API + module Admin + class Dictionary < ::API::Base + feature_category :database + urgency :low + + before do + authenticated_as_admin! + end + + namespace 'admin' do + resources 'databases/:database_name/dictionary/tables/:table_name' do + desc 'Retrieve dictionary details' do + success ::API::Entities::Dictionary::Table + failure [ + { code: 401, message: '401 Unauthorized' }, + { code: 403, message: '403 Forbidden' }, + { code: 404, message: '404 Not found' } + ] + end + params do + requires :database_name, + type: String, + values: %w[main ci], + desc: 'The database name' + + requires :table_name, + type: String, + desc: 'The table name' + end + get do + not_found!('Table not found') unless File.exist?(safe_file_path!) + + present table_dictionary, with: Entities::Dictionary::Table + end + end + + helpers do + def table_name + params[:table_name] + end + + def table_dictionary + YAML.load_file(safe_file_path!).with_indifferent_access + end + + def safe_file_path! + dir = Gitlab::Database::GitlabSchema.dictionary_paths.first.to_s + path = Rails.root.join(dir, "#{table_name}.yml").to_s + + Gitlab::PathTraversal.check_allowed_absolute_path_and_path_traversal!(path, [dir]) + + path + end + end + end + end + end +end diff --git a/lib/api/admin/migrations.rb b/lib/api/admin/migrations.rb new file mode 100644 index 00000000000..d4dbdbbb021 --- /dev/null +++ b/lib/api/admin/migrations.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +module API + module Admin + class Migrations < ::API::Base + feature_category :database + urgency :low + + before do + authenticated_as_admin! + end + + namespace 'admin' do + resources 'migrations/:timestamp/mark' do + desc 'Mark the migration as successfully executed' do + success [ + { code: 201, message: '201 Created' } + ] + failure [ + { code: 401, message: '401 Unauthorized' }, + { code: 403, message: '403 Forbidden' }, + { code: 404, message: '404 Not found' }, + { code: 422, message: 'You can mark only pending migrations' } + ] + tags %w[migrations] + end + params do + optional :database, + type: String, + values: Gitlab::Database.all_database_names, + desc: 'The name of the database', + default: 'main' + requires :timestamp, + type: Integer, + desc: 'The migration version timestamp' + end + post do + response = Database::MarkMigrationService.new( + connection: base_model.connection, + version: params[:timestamp] + ).execute + + if response.success? + created! + elsif response.reason == :not_found + not_found! + else + render_api_error!('You can mark only pending migrations', 422) + end + end + end + end + + helpers do + def base_model + database = params[:database] || Gitlab::Database::MAIN_DATABASE_NAME + @base_model ||= Gitlab::Database.database_base_models[database] + end + end + end + end +end diff --git a/lib/api/admin/plan_limits.rb b/lib/api/admin/plan_limits.rb index f1d7b56ad92..017c27cec95 100644 --- a/lib/api/admin/plan_limits.rb +++ b/lib/api/admin/plan_limits.rb @@ -60,15 +60,19 @@ module API optional :ci_registered_group_runners, type: Integer, desc: 'Maximum number of runners registered per group' optional :ci_registered_project_runners, type: Integer, desc: 'Maximum number of runners registered per project' optional :conan_max_file_size, type: Integer, desc: 'Maximum Conan package file size in bytes' + optional :enforcement_limit, type: Integer, + desc: 'Maximum storage size for the root namespace enforcement in MiB' optional :generic_packages_max_file_size, type: Integer, desc: 'Maximum generic package file size in bytes' optional :helm_max_file_size, type: Integer, desc: 'Maximum Helm chart file size in bytes' optional :maven_max_file_size, type: Integer, desc: 'Maximum Maven package file size in bytes' + optional :notification_limit, type: Integer, + desc: 'Maximum storage size for the root namespace notifications in MiB' optional :npm_max_file_size, type: Integer, desc: 'Maximum NPM package file size in bytes' optional :nuget_max_file_size, type: Integer, desc: 'Maximum NuGet package file size in bytes' optional :pypi_max_file_size, type: Integer, desc: 'Maximum PyPI package file size in bytes' optional :terraform_module_max_file_size, type: Integer, desc: 'Maximum Terraform Module package file size in bytes' - optional :storage_size_limit, type: Integer, desc: 'Maximum storage size for the root namespace in megabytes' + optional :storage_size_limit, type: Integer, desc: 'Maximum storage size for the root namespace in MiB' optional :pipeline_hierarchy_size, type: Integer, desc: "Maximum number of downstream pipelines in a pipeline's hierarchy tree" end @@ -76,7 +80,9 @@ module API params = declared_params(include_missing: false) plan = current_plan(params.delete(:plan_name)) - if plan.actual_limits.update(params) + result = ::Admin::PlanLimits::UpdateService.new(params, current_user: current_user, plan: plan).execute + + if result[:status] == :success present plan.actual_limits, with: Entities::PlanLimit else render_validation_error!(plan.actual_limits) diff --git a/lib/api/api.rb b/lib/api/api.rb index f50c705c3ea..090fbaa7f93 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -90,6 +90,10 @@ module API Gitlab::UsageDataCounters::JetBrainsPluginActivityUniqueCounter.track_api_request_when_trackable(user_agent: request&.user_agent, user: @current_user) end + after do + Gitlab::UsageDataCounters::JetBrainsBundledPluginActivityUniqueCounter.track_api_request_when_trackable(user_agent: request&.user_agent, user: @current_user) + end + after do Gitlab::UsageDataCounters::GitLabCliActivityUniqueCounter.track_api_request_when_trackable(user_agent: request&.user_agent, user: @current_user) end @@ -135,7 +139,7 @@ module API end rescue_from Gitlab::Git::ResourceExhaustedError do |exception| - rack_response({ 'message' => exception.message }.to_json, 429, exception.headers) + rack_response({ 'message' => exception.message }.to_json, 503, exception.headers) end rescue_from :all do |exception| @@ -177,7 +181,9 @@ module API mount ::API::AccessRequests mount ::API::Admin::BatchedBackgroundMigrations mount ::API::Admin::Ci::Variables + mount ::API::Admin::Dictionary mount ::API::Admin::InstanceClusters + mount ::API::Admin::Migrations mount ::API::Admin::PlanLimits mount ::API::AlertManagementAlerts mount ::API::Appearance @@ -255,7 +261,9 @@ module API mount ::API::Metadata mount ::API::Metrics::Dashboard::Annotations mount ::API::Metrics::UserStarredDashboards + mount ::API::MlModelPackages mount ::API::Namespaces + mount ::API::NpmGroupPackages mount ::API::NpmInstancePackages mount ::API::NpmProjectPackages mount ::API::NugetGroupPackages @@ -321,7 +329,6 @@ module API mount ::API::Ci::PipelineSchedules mount ::API::Ci::SecureFiles mount ::API::Discussions - mount ::API::ErrorTracking::Collector mount ::API::GroupBoards mount ::API::GroupLabels mount ::API::GroupMilestones diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index 81a640d9a93..0aee0c70203 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -83,10 +83,7 @@ module API private def bypass_session_for_admin_mode?(user) - return user.is_a?(User) && Gitlab::CurrentSettings.admin_mode if Feature.disabled?(:admin_mode_for_api) - - return false unless Gitlab::CurrentSettings.admin_mode - return false unless user.is_a?(User) + return false unless user.is_a?(User) && Gitlab::CurrentSettings.admin_mode Gitlab::Session.with_session(current_request.session) { Gitlab::Auth::CurrentUserMode.new(user).admin_mode? } || Gitlab::Auth::RequestAuthenticator.new(current_request).valid_access_token?(scopes: [:admin_mode]) diff --git a/lib/api/badges.rb b/lib/api/badges.rb index 020ba53b9ee..84c9f780a53 100644 --- a/lib/api/badges.rb +++ b/lib/api/badges.rb @@ -8,7 +8,7 @@ module API helpers ::API::Helpers::BadgesHelpers - feature_category :projects + feature_category :groups_and_projects helpers do def find_source_if_admin(source_type) diff --git a/lib/api/ci/helpers/runner.rb b/lib/api/ci/helpers/runner.rb index 7ca8b2df3dd..94c1942a244 100644 --- a/lib/api/ci/helpers/runner.rb +++ b/lib/api/ci/helpers/runner.rb @@ -146,6 +146,12 @@ module API # noop: overridden in EE end + def check_if_backoff_required! + return unless Gitlab::Database::Migrations::RunnerBackoff::Communicator.backoff_runner? + + too_many_requests!('Executing database migrations. Please retry later.', retry_after: 1.minute) + end + private def get_runner_config_from_request diff --git a/lib/api/ci/pipelines.rb b/lib/api/ci/pipelines.rb index 6416de6d2a9..809a9bd781b 100644 --- a/lib/api/ci/pipelines.rb +++ b/lib/api/ci/pipelines.rb @@ -329,7 +329,8 @@ module API post ':id/pipelines/:pipeline_id/cancel', urgency: :low, feature_category: :continuous_integration do authorize! :update_pipeline, pipeline - pipeline.cancel_running + # TODO: inconsistent behavior: when pipeline is not cancelable we should return an error + ::Ci::CancelPipelineService.new(pipeline: pipeline, current_user: current_user).execute status 200 present pipeline.reset, with: Entities::Ci::Pipeline diff --git a/lib/api/ci/runner.rb b/lib/api/ci/runner.rb index 0247ce301e2..25ac1780a36 100644 --- a/lib/api/ci/runner.rb +++ b/lib/api/ci/runner.rb @@ -7,6 +7,8 @@ module API content_type :txt, 'text/plain' + before { check_if_backoff_required! } + resource :runners do desc 'Register a new runner' do detail "Register a new runner for the instance" diff --git a/lib/api/ci/secure_files.rb b/lib/api/ci/secure_files.rb index 41faaf80c82..02f625f2130 100644 --- a/lib/api/ci/secure_files.rb +++ b/lib/api/ci/secure_files.rb @@ -6,6 +6,7 @@ module API include PaginationParams before do + check_api_enabled! authenticate! authorize! :read_secure_files, user_project end @@ -64,7 +65,7 @@ module API resource do before do - read_only_feature_flag_enabled? + check_read_only_feature_flag_enabled! authorize! :admin_secure_files, user_project end @@ -81,7 +82,7 @@ module API route_setting :authentication, basic_auth_personal_access_token: true, job_token_allowed: true post ':id/secure_files' do secure_file = user_project.secure_files.new( - name: Gitlab::Utils.check_path_traversal!(params[:name]) + name: Gitlab::PathTraversal.check_path_traversal!(params[:name]) ) secure_file.file = params[:file] @@ -112,7 +113,11 @@ module API end helpers do - def read_only_feature_flag_enabled? + def check_api_enabled! + forbidden! unless Gitlab.config.ci_secure_files.enabled + end + + def check_read_only_feature_flag_enabled! service_unavailable! if Feature.enabled?(:ci_secure_files_read_only, user_project, type: :ops) end end diff --git a/lib/api/concerns/packages/nuget/private_endpoints.rb b/lib/api/concerns/packages/nuget/private_endpoints.rb new file mode 100644 index 00000000000..20c02f0a285 --- /dev/null +++ b/lib/api/concerns/packages/nuget/private_endpoints.rb @@ -0,0 +1,153 @@ +# frozen_string_literal: true + +# +# NuGet Package Manager Client API +# +# These API endpoints are not consumed directly by users, so there is no documentation for the +# individual endpoints. They are called by the NuGet package manager client when users run commands +# like `nuget install` or `nuget push`. The usage of the GitLab NuGet registry is documented here: +# https://docs.gitlab.com/ee/user/packages/nuget_repository/ +# +# Technical debt: https://gitlab.com/gitlab-org/gitlab/issues/35798 +module API + module Concerns + module Packages + module Nuget + module PrivateEndpoints + extend ActiveSupport::Concern + + POSITIVE_INTEGER_REGEX = %r{\A[1-9]\d*\z} + NON_NEGATIVE_INTEGER_REGEX = %r{\A(0|[1-9]\d*)\z} + + included do + helpers do + def find_packages(package_name) + packages = package_finder(package_name).execute + + not_found!('Packages') unless packages.exists? + + packages + end + + def find_package(package_name, package_version) + package = package_finder(package_name, package_version).execute + .first + + not_found!('Package') unless package + + package + end + + def package_finder(package_name, package_version = nil) + ::Packages::Nuget::PackageFinder.new( + current_user, + project_or_group, + package_name: package_name, + package_version: package_version + ) + end + + def search_packages(_search_term, search_options) + ::Packages::Nuget::SearchService + .new(current_user, project_or_group, params[:q], search_options) + .execute + end + end + + # https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource + params do + requires :package_name, type: String, desc: 'The NuGet package name', + regexp: API::NO_SLASH_URL_PART_REGEX, documentation: { example: 'MyNuGetPkg' } + end + namespace '/metadata/*package_name' do + after_validation do + authorize_packages_access!(project_or_group, required_permission) + end + + desc 'The NuGet Metadata Service - Package name level' do + detail 'This feature was introduced in GitLab 12.8' + success code: 200, model: ::API::Entities::Nuget::PackagesMetadata + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[nuget_packages] + end + get 'index', format: :json, urgency: :low do + present ::Packages::Nuget::PackagesMetadataPresenter.new(find_packages(params[:package_name])), + with: ::API::Entities::Nuget::PackagesMetadata + end + + desc 'The NuGet Metadata Service - Package name and version level' do + detail 'This feature was introduced in GitLab 12.8' + success code: 200, model: ::API::Entities::Nuget::PackageMetadata + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[nuget_packages] + end + params do + requires :package_version, type: String, desc: 'The NuGet package version', + regexp: API::NO_SLASH_URL_PART_REGEX, documentation: { example: '1.0.0' } + end + get '*package_version', format: :json, urgency: :low do + present ::Packages::Nuget::PackageMetadataPresenter.new( + find_package(params[:package_name], + params[:package_version]) + ), + with: ::API::Entities::Nuget::PackageMetadata + end + end + + # https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource + params do + optional :q, type: String, desc: 'The search term', documentation: { example: 'MyNuGet' } + optional :skip, type: Integer, desc: 'The number of results to skip', default: 0, + regexp: NON_NEGATIVE_INTEGER_REGEX, documentation: { example: 1 } + optional :take, type: Integer, desc: 'The number of results to return', + default: Kaminari.config.default_per_page, regexp: POSITIVE_INTEGER_REGEX, documentation: { example: 1 } + optional :prerelease, type: ::Grape::API::Boolean, desc: 'Include prerelease versions', default: true + end + namespace '/query' do + after_validation do + authorize_packages_access!(project_or_group, required_permission) + end + + desc 'The NuGet Search Service' do + detail 'This feature was introduced in GitLab 12.8' + success code: 200, model: ::API::Entities::Nuget::SearchResults + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[nuget_packages] + end + get format: :json, urgency: :low do + search_options = { + include_prerelease_versions: params[:prerelease], + per_page: params[:take], + padding: params[:skip] + } + + results = search_packages(params[:q], search_options) + + track_package_event( + 'search_package', + :nuget, + **snowplow_gitlab_standard_context.merge(category: 'API::NugetPackages') + ) + + present ::Packages::Nuget::SearchResultsPresenter.new(results), + with: ::API::Entities::Nuget::SearchResults + end + end + end + end + end + end + end +end diff --git a/lib/api/concerns/packages/nuget/public_endpoints.rb b/lib/api/concerns/packages/nuget/public_endpoints.rb new file mode 100644 index 00000000000..37b503212d9 --- /dev/null +++ b/lib/api/concerns/packages/nuget/public_endpoints.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +# NuGet Package Manager Client API + +# These API endpoints are not consumed directly by users, so there is no documentation for the +# individual endpoints. They are called by the NuGet package manager client when users run commands +# like `nuget install` or `nuget push`. The usage of the GitLab NuGet registry is documented here: +# https://docs.gitlab.com/ee/user/packages/nuget_repository/ + +module API + module Concerns + module Packages + module Nuget + module PublicEndpoints + extend ActiveSupport::Concern + + included do + # https://docs.microsoft.com/en-us/nuget/api/service-index + desc 'The NuGet Service Index' do + detail 'This feature was introduced in GitLab 12.6' + success code: 200, model: ::API::Entities::Nuget::ServiceIndex + failure [ + { code: 404, message: 'Not Found' } + ] + tags %w[nuget_packages] + end + get 'index', format: :json, urgency: :default do + track_package_event( + 'cli_metadata', + :nuget, + **snowplow_gitlab_standard_context_without_auth.merge(category: 'API::NugetPackages') + ) + + present ::Packages::Nuget::ServiceIndexPresenter.new(project_or_group_without_auth), + with: ::API::Entities::Nuget::ServiceIndex + end + end + end + end + end + end +end diff --git a/lib/api/concerns/packages/nuget_endpoints.rb b/lib/api/concerns/packages/nuget_endpoints.rb deleted file mode 100644 index 5f32f0544f4..00000000000 --- a/lib/api/concerns/packages/nuget_endpoints.rb +++ /dev/null @@ -1,159 +0,0 @@ -# frozen_string_literal: true -# -# NuGet Package Manager Client API -# -# These API endpoints are not consumed directly by users, so there is no documentation for the -# individual endpoints. They are called by the NuGet package manager client when users run commands -# like `nuget install` or `nuget push`. The usage of the GitLab NuGet registry is documented here: -# https://docs.gitlab.com/ee/user/packages/nuget_repository/ -# -# Technical debt: https://gitlab.com/gitlab-org/gitlab/issues/35798 -module API - module Concerns - module Packages - module NugetEndpoints - extend ActiveSupport::Concern - - POSITIVE_INTEGER_REGEX = %r{\A[1-9]\d*\z}.freeze - NON_NEGATIVE_INTEGER_REGEX = %r{\A(0|[1-9]\d*)\z}.freeze - - included do - helpers do - def find_packages(package_name) - packages = package_finder(package_name).execute - - not_found!('Packages') unless packages.exists? - - packages - end - - def find_package(package_name, package_version) - package = package_finder(package_name, package_version).execute - .first - - not_found!('Package') unless package - - package - end - - def package_finder(package_name, package_version = nil) - ::Packages::Nuget::PackageFinder.new( - current_user, - project_or_group, - package_name: package_name, - package_version: package_version - ) - end - - def search_packages(search_term, search_options) - ::Packages::Nuget::SearchService - .new(current_user, project_or_group, params[:q], search_options) - .execute - end - end - - # https://docs.microsoft.com/en-us/nuget/api/service-index - desc 'The NuGet Service Index' do - detail 'This feature was introduced in GitLab 12.6' - success code: 200, model: ::API::Entities::Nuget::ServiceIndex - failure [ - { code: 401, message: 'Unauthorized' }, - { code: 403, message: 'Forbidden' }, - { code: 404, message: 'Not Found' } - ] - tags %w[nuget_packages] - end - get 'index', format: :json, urgency: :default do - authorize_packages_access!(project_or_group, required_permission) - - track_package_event('cli_metadata', :nuget, **snowplow_gitlab_standard_context.merge(category: 'API::NugetPackages')) - - present ::Packages::Nuget::ServiceIndexPresenter.new(project_or_group), - with: ::API::Entities::Nuget::ServiceIndex - end - - # https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource - params do - requires :package_name, type: String, desc: 'The NuGet package name', regexp: API::NO_SLASH_URL_PART_REGEX, documentation: { example: 'MyNuGetPkg' } - end - namespace '/metadata/*package_name' do - after_validation do - authorize_packages_access!(project_or_group, required_permission) - end - - desc 'The NuGet Metadata Service - Package name level' do - detail 'This feature was introduced in GitLab 12.8' - success code: 200, model: ::API::Entities::Nuget::PackagesMetadata - failure [ - { code: 401, message: 'Unauthorized' }, - { code: 403, message: 'Forbidden' }, - { code: 404, message: 'Not Found' } - ] - tags %w[nuget_packages] - end - get 'index', format: :json, urgency: :low do - present ::Packages::Nuget::PackagesMetadataPresenter.new(find_packages(params[:package_name])), - with: ::API::Entities::Nuget::PackagesMetadata - end - - desc 'The NuGet Metadata Service - Package name and version level' do - detail 'This feature was introduced in GitLab 12.8' - success code: 200, model: ::API::Entities::Nuget::PackageMetadata - failure [ - { code: 401, message: 'Unauthorized' }, - { code: 403, message: 'Forbidden' }, - { code: 404, message: 'Not Found' } - ] - tags %w[nuget_packages] - end - params do - requires :package_version, type: String, desc: 'The NuGet package version', regexp: API::NO_SLASH_URL_PART_REGEX, documentation: { example: '1.0.0' } - end - get '*package_version', format: :json, urgency: :low do - present ::Packages::Nuget::PackageMetadataPresenter.new(find_package(params[:package_name], params[:package_version])), - with: ::API::Entities::Nuget::PackageMetadata - end - end - - # https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource - params do - optional :q, type: String, desc: 'The search term', documentation: { example: 'MyNuGet' } - optional :skip, type: Integer, desc: 'The number of results to skip', default: 0, regexp: NON_NEGATIVE_INTEGER_REGEX, documentation: { example: 1 } - optional :take, type: Integer, desc: 'The number of results to return', default: Kaminari.config.default_per_page, regexp: POSITIVE_INTEGER_REGEX, documentation: { example: 1 } - optional :prerelease, type: ::Grape::API::Boolean, desc: 'Include prerelease versions', default: true - end - namespace '/query' do - after_validation do - authorize_packages_access!(project_or_group, required_permission) - end - - desc 'The NuGet Search Service' do - detail 'This feature was introduced in GitLab 12.8' - success code: 200, model: ::API::Entities::Nuget::SearchResults - failure [ - { code: 401, message: 'Unauthorized' }, - { code: 403, message: 'Forbidden' }, - { code: 404, message: 'Not Found' } - ] - tags %w[nuget_packages] - end - get format: :json, urgency: :low do - search_options = { - include_prerelease_versions: params[:prerelease], - per_page: params[:take], - padding: params[:skip] - } - - results = search_packages(params[:q], search_options) - - track_package_event('search_package', :nuget, **snowplow_gitlab_standard_context.merge(category: 'API::NugetPackages')) - - present ::Packages::Nuget::SearchResultsPresenter.new(results), - with: ::API::Entities::Nuget::SearchResults - end - end - end - end - end - end -end diff --git a/lib/api/debian_project_packages.rb b/lib/api/debian_project_packages.rb index 4f78ac926d8..e1531847b87 100644 --- a/lib/api/debian_project_packages.rb +++ b/lib/api/debian_project_packages.rb @@ -19,6 +19,10 @@ module API def project_or_group authorized_user_project(action: :read_package) end + + def end_of_new_upload? + params[:distribution].present? || params[:file_name].end_with?('.changes') + end end after_validation do @@ -97,7 +101,7 @@ module API component: params['component'] } - package = if params[:distribution].present? + package = if end_of_new_upload? ::Packages::CreateTemporaryPackageService.new( project_or_group, current_user, declared_params.merge(build: current_authenticated_job) ).execute(:debian, name: ::Packages::Debian::TEMPORARY_PACKAGE_NAME) diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb index 634d6052b99..f8379392531 100644 --- a/lib/api/deploy_keys.rb +++ b/lib/api/deploy_keys.rb @@ -41,8 +41,10 @@ module API authenticated_as_admin! deploy_keys = params[:public] ? DeployKey.are_public : DeployKey.all + deploy_keys = deploy_keys.including_projects_with_write_access.including_projects_with_readonly_access - present paginate(deploy_keys.including_projects_with_write_access), with: Entities::DeployKey, include_projects_with_write_access: true + present paginate(deploy_keys), + with: Entities::DeployKey, include_projects_with_write_access: true, include_projects_with_readonly_access: true end params do diff --git a/lib/api/discussions.rb b/lib/api/discussions.rb index 768ffac41ce..45466a1894c 100644 --- a/lib/api/discussions.rb +++ b/lib/api/discussions.rb @@ -122,7 +122,7 @@ module API note = create_note(noteable, opts) - if note.valid? + if note.persisted? present note.discussion, with: Entities::Discussion else bad_request!("Note #{note.errors.messages}") @@ -175,7 +175,7 @@ module API } note = create_note(noteable, opts) - if note.valid? + if note.persisted? present note, with: Entities::Note else bad_request!("Note #{note.errors.messages}") diff --git a/lib/api/entities/deploy_key.rb b/lib/api/entities/deploy_key.rb index 1bcd06f2c88..0e82d9abb63 100644 --- a/lib/api/entities/deploy_key.rb +++ b/lib/api/entities/deploy_key.rb @@ -14,6 +14,7 @@ module API documentation: { type: 'string', example: 'SHA256:Jrs3LD1Ji30xNLtTVf9NDCj7kkBgPBb2pjvTZ3HfIgU' } expose :projects_with_write_access, using: Entities::ProjectIdentity, if: -> (_, options) { options[:include_projects_with_write_access] } + expose :projects_with_readonly_access, using: Entities::ProjectIdentity, if: -> (_, options) { options[:include_projects_with_readonly_access] } end end end diff --git a/lib/api/entities/dictionary/table.rb b/lib/api/entities/dictionary/table.rb new file mode 100644 index 00000000000..8d4e3fb959d --- /dev/null +++ b/lib/api/entities/dictionary/table.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module API + module Entities + module Dictionary + class Table < Grape::Entity + expose :table_name, documentation: { type: :string, example: 'users' } + expose :feature_categories, documentation: { type: :array, example: ['database'] } + end + end + end +end diff --git a/lib/api/entities/draft_note.rb b/lib/api/entities/draft_note.rb index 70b32bac502..13852513615 100644 --- a/lib/api/entities/draft_note.rb +++ b/lib/api/entities/draft_note.rb @@ -38,7 +38,7 @@ module API } } } do |note| - note.position.to_h + note.position.to_h.except(:ignore_whitespace_change) end end end diff --git a/lib/api/entities/error_tracking.rb b/lib/api/entities/error_tracking.rb index 5e3b983c58c..180293a444d 100644 --- a/lib/api/entities/error_tracking.rb +++ b/lib/api/entities/error_tracking.rb @@ -21,7 +21,7 @@ module API expose :id, documentation: { type: 'integer', example: 1 } expose :active, documentation: { type: 'boolean' } expose :public_key, documentation: { type: 'string', example: 'glet_aa77551d849c083f76d0bc545ed053a3' } - expose :sentry_dsn, documentation: { type: 'string', example: 'https://glet_aa77551d849c083f76d0bc545ed053a3@gitlab.example.com/api/v4/error_tracking/collector/5' } + expose :sentry_dsn, documentation: { type: 'string', example: 'https://glet_aa77551d849c083f76d0bc545ed053a3@example.com/errortracking/api/v1/projects/5' } end end end diff --git a/lib/api/entities/merge_request_basic.rb b/lib/api/entities/merge_request_basic.rb index f796aeba17f..adff7f87cd3 100644 --- a/lib/api/entities/merge_request_basic.rb +++ b/lib/api/entities/merge_request_basic.rb @@ -68,6 +68,7 @@ module API expose :discussion_locked expose :should_remove_source_branch?, as: :should_remove_source_branch expose :force_remove_source_branch?, as: :force_remove_source_branch + expose :prepared_at with_options if: -> (merge_request, _) { merge_request.for_fork? } do expose :allow_collaboration diff --git a/lib/api/entities/namespace.rb b/lib/api/entities/namespace.rb index 15bc7d158c4..5e0630e0f7f 100644 --- a/lib/api/entities/namespace.rb +++ b/lib/api/entities/namespace.rb @@ -10,6 +10,14 @@ module API def expose_members_count_with_descendants?(namespace, opts) namespace.kind == 'group' && Ability.allowed?(opts[:current_user], :admin_group, namespace) end + + expose :root_repository_size, documentation: { type: 'integer', example: 123 }, if: -> (namespace, opts) { expose_root_repository_size?(namespace, opts) } do |namespace, _| + namespace.root_storage_statistics&.repository_size + end + + def expose_root_repository_size?(namespace, opts) + namespace.kind == 'group' && Ability.allowed?(opts[:current_user], :admin_group, namespace) + end end end end diff --git a/lib/api/entities/note.rb b/lib/api/entities/note.rb index cac4a8280e3..6ed5ca43fbb 100644 --- a/lib/api/entities/note.rb +++ b/lib/api/entities/note.rb @@ -18,7 +18,7 @@ module API expose :commit_id, if: ->(note, options) { note.noteable_type == "MergeRequest" && note.is_a?(DiffNote) } expose :position, if: ->(note, options) { note.is_a?(DiffNote) } do |note| - note.position.to_h + note.position.to_h.except(:ignore_whitespace_change) end expose :resolvable?, as: :resolvable diff --git a/lib/api/entities/nuget/metadatum.rb b/lib/api/entities/nuget/metadatum.rb index 256b916cb64..c316dfce740 100644 --- a/lib/api/entities/nuget/metadatum.rb +++ b/lib/api/entities/nuget/metadatum.rb @@ -4,6 +4,12 @@ module API module Entities module Nuget class Metadatum < Grape::Entity + expose :authors, documentation: { type: 'string', example: 'Authors' } do |metadatum| + metadatum[:authors] || '' + end + expose :description, as: :summary, documentation: { type: 'string', example: 'Description' } do |metadatum| + metadatum[:description] || '' + end expose :project_url, as: :projectUrl, expose_nil: false, documentation: { type: 'string', example: 'http://sandbox.com/project' } expose :license_url, as: :licenseUrl, expose_nil: false, documentation: { type: 'string', example: 'http://sandbox.com/license' } expose :icon_url, as: :iconUrl, expose_nil: false, documentation: { type: 'string', example: 'http://sandbox.com/icon' } diff --git a/lib/api/entities/nuget/package_metadata_catalog_entry.rb b/lib/api/entities/nuget/package_metadata_catalog_entry.rb index ce328c5a5ca..b6e768e5083 100644 --- a/lib/api/entities/nuget/package_metadata_catalog_entry.rb +++ b/lib/api/entities/nuget/package_metadata_catalog_entry.rb @@ -5,16 +5,15 @@ module API module Nuget class PackageMetadataCatalogEntry < Grape::Entity expose :json_url, as: :@id, documentation: { type: 'string', example: 'https://gitlab.example.com/api/v4/projects/1/packages/nuget/metadata/MyNuGetPkg/1.3.0.17.json' } - expose :authors, documentation: { type: 'string', example: 'Author' } expose :dependency_groups, as: :dependencyGroups, using: ::API::Entities::Nuget::DependencyGroup, documentation: { is_array: true, type: 'API::Entities::Nuget::DependencyGroup' } expose :package_name, as: :id, documentation: { type: 'string', example: 'MyNuGetPkg' } expose :package_version, as: :version, documentation: { type: 'string', example: '1.3.0.17' } expose :tags, documentation: { type: 'string', example: 'tag#1 tag#2' } expose :archive_url, as: :packageContent, documentation: { type: 'string', example: 'https://gitlab.example.com/api/v4/projects/1/packages/nuget/download/MyNuGetPkg/1.3.0.17/helloworld.1.3.0.17.nupkg' } - expose :summary, documentation: { type: 'string', example: 'Summary' } expose :metadatum, using: ::API::Entities::Nuget::Metadatum, merge: true, documentation: { type: 'API::Entities::Nuget::Metadatum' } + expose :published, documentation: { type: 'string', example: '2023-05-08T17:23:25Z' } end end end diff --git a/lib/api/entities/nuget/search_result.rb b/lib/api/entities/nuget/search_result.rb index bb3698de30b..303efa7718e 100644 --- a/lib/api/entities/nuget/search_result.rb +++ b/lib/api/entities/nuget/search_result.rb @@ -5,10 +5,8 @@ module API module Nuget class SearchResult < Grape::Entity expose :type, as: :@type, documentation: { type: 'string', example: 'Package' } - expose :authors, documentation: { type: 'string', example: 'Author' } expose :name, as: :id, documentation: { type: 'string', example: 'MyNuGetPkg' } expose :name, as: :title, documentation: { type: 'string', example: 'MyNuGetPkg' } - expose :summary, documentation: { type: 'string', example: 'Summary' } expose :total_downloads, as: :totalDownloads, documentation: { type: 'integer', example: 1 } expose :verified, documentation: { type: 'boolean' } expose :version, documentation: { type: 'string', example: '1.3.0.17' } diff --git a/lib/api/entities/package.rb b/lib/api/entities/package.rb index ab6cc0fcb0a..5831fe68a5d 100644 --- a/lib/api/entities/package.rb +++ b/lib/api/entities/package.rb @@ -43,7 +43,7 @@ module API end expose :tags - expose :pipeline, if: ->(package) { package.original_build_info }, using: Package::Pipeline + expose :pipeline, if: ->(package) { package.last_build_info }, using: Package::Pipeline expose :pipelines, if: ->(package) { package.pipelines.present? }, using: Package::Pipeline expose :versions, using: ::API::Entities::PackageVersion, unless: ->(_, opts) { opts[:collection] } diff --git a/lib/api/entities/package_version.rb b/lib/api/entities/package_version.rb index 82522d3f423..417d8755144 100644 --- a/lib/api/entities/package_version.rb +++ b/lib/api/entities/package_version.rb @@ -8,7 +8,7 @@ module API expose :created_at expose :tags - expose :pipeline, if: ->(package) { package.original_build_info }, using: Package::Pipeline + expose :pipeline, if: ->(package) { package.last_build_info }, using: Package::Pipeline end end end diff --git a/lib/api/entities/plan_limit.rb b/lib/api/entities/plan_limit.rb index b5cff2bb73c..753c595d65f 100644 --- a/lib/api/entities/plan_limit.rb +++ b/lib/api/entities/plan_limit.rb @@ -11,9 +11,11 @@ module API expose :ci_registered_group_runners, documentation: { type: 'integer', example: 1000 } expose :ci_registered_project_runners, documentation: { type: 'integer', example: 1000 } expose :conan_max_file_size, documentation: { type: 'integer', example: 3221225472 } + expose :enforcement_limit, documentation: { type: 'integer', example: 15000 } expose :generic_packages_max_file_size, documentation: { type: 'integer', example: 5368709120 } expose :helm_max_file_size, documentation: { type: 'integer', example: 5242880 } expose :maven_max_file_size, documentation: { type: 'integer', example: 3221225472 } + expose :notification_limit, documentation: { type: 'integer', example: 15000 } expose :npm_max_file_size, documentation: { type: 'integer', example: 524288000 } expose :nuget_max_file_size, documentation: { type: 'integer', example: 524288000 } expose :pipeline_hierarchy_size, documentation: { type: 'integer', example: 1000 } diff --git a/lib/api/entities/project_integration_basic.rb b/lib/api/entities/project_integration_basic.rb index b7c56d7cca1..d7e111b990e 100644 --- a/lib/api/entities/project_integration_basic.rb +++ b/lib/api/entities/project_integration_basic.rb @@ -15,9 +15,11 @@ module API expose :push_events, documentation: { type: 'boolean' } expose :issues_events, documentation: { type: 'boolean' } expose :incident_events, documentation: { type: 'boolean' } + expose :alert_events, documentation: { type: 'boolean' } expose :confidential_issues_events, documentation: { type: 'boolean' } expose :merge_requests_events, documentation: { type: 'boolean' } expose :tag_push_events, documentation: { type: 'boolean' } + expose :deployment_events, documentation: { type: 'boolean' } expose :note_events, documentation: { type: 'boolean' } expose :confidential_note_events, documentation: { type: 'boolean' } expose :pipeline_events, documentation: { type: 'boolean' } diff --git a/lib/api/entities/project_scope_link.rb b/lib/api/entities/project_scope_link.rb new file mode 100644 index 00000000000..8a5466a0418 --- /dev/null +++ b/lib/api/entities/project_scope_link.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module API + module Entities + class ProjectScopeLink < Grape::Entity + expose :source_project_id, documentation: { type: 'integer' } + expose :target_project_id, documentation: { type: 'integer' } + end + end +end diff --git a/lib/api/error_tracking/collector.rb b/lib/api/error_tracking/collector.rb deleted file mode 100644 index e10125e02c6..00000000000 --- a/lib/api/error_tracking/collector.rb +++ /dev/null @@ -1,156 +0,0 @@ -# frozen_string_literal: true - -module API - # This API is responsible for collecting error tracking information - # from sentry client. It allows us to use GitLab as an alternative to - # sentry backend. For more details see https://gitlab.com/gitlab-org/gitlab/-/issues/329596. - class ErrorTracking::Collector < ::API::Base - feature_category :error_tracking - urgency :low - - content_type :envelope, 'application/x-sentry-envelope' - content_type :json, 'application/json' - content_type :txt, 'text/plain' - default_format :envelope - - rescue_from Gitlab::ErrorTracking::ErrorRepository::DatabaseError do |e| - render_api_error!(e.message, 400) - end - - before do - not_found!('Project') unless project - not_found! unless feature_enabled? - not_found! unless active_client_key? - end - - helpers do - def project - @project ||= find_project(params[:id]) - end - - def feature_enabled? - Feature.enabled?(:integrated_error_tracking, project) && - project.error_tracking_setting&.integrated_enabled? - end - - def find_client_key(public_key) - return unless public_key.present? - - project.error_tracking_client_keys.active.find_by_public_key(public_key) - end - - def active_client_key? - public_key = extract_public_key - - find_client_key(public_key) - end - - def extract_public_key - # Some SDK send public_key as a param. In this case we don't need to parse headers. - return params[:sentry_key] if params[:sentry_key].present? - - begin - ::ErrorTracking::Collector::SentryAuthParser.parse(request)[:public_key] - rescue StandardError - bad_request!('Failed to parse sentry request') - end - end - - def validate_payload(payload) - unless ::ErrorTracking::Collector::PayloadValidator.new.valid?(payload) - bad_request!('Unsupported sentry payload') - end - end - end - - desc 'Submit error tracking event to the project as envelope' do - detail 'This feature was introduced in GitLab 14.1.' - end - params do - requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' - end - post 'error_tracking/collector/api/:id/envelope' do - # There is a reason why we have such uncommon path. - # We depend on a client side error tracking software which - # modifies URL for its own reasons. - # - # When we give user a URL like this - # HOST/api/v4/error_tracking/collector/123 - # - # Then error tracking software will convert it like this: - # HOST/api/v4/error_tracking/collector/api/123/envelope/ - - begin - parsed_request = ::ErrorTracking::Collector::SentryRequestParser.parse(request) - rescue StandardError - bad_request!('Failed to parse sentry request') - end - - type = parsed_request[:request_type] - - # Sentry sends 2 requests on each exception: transaction and event. - # Everything else is not a desired behavior. - unless type == 'transaction' || type == 'event' - render_api_error!('400 Bad Request', 400) - - break - end - - # We don't have use for transaction request yet, - # so we record only event one. - if type == 'event' - validate_payload(parsed_request[:event]) - - ::ErrorTracking::CollectErrorService - .new(project, nil, event: parsed_request[:event]) - .execute - end - - # Collector should never return any information back. - # Because DSN and public key are designed for public use, - # it is safe only for submission of new events. - # - # Some clients sdk require status 200 OK to work correctly. - # See https://gitlab.com/gitlab-org/gitlab/-/issues/343531. - status 200 - end - - desc 'Submit error tracking event to the project' do - detail 'This feature was introduced in GitLab 14.1.' - end - params do - requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' - end - post 'error_tracking/collector/api/:id/store' do - # There is a reason why we have such uncommon path. - # We depend on a client side error tracking software which - # modifies URL for its own reasons. - # - # When we give user a URL like this - # HOST/api/v4/error_tracking/collector/123 - # - # Then error tracking software will convert it like this: - # HOST/api/v4/error_tracking/collector/api/123/store/ - - begin - parsed_body = Gitlab::Json.parse(request.body.read) - rescue StandardError - bad_request!('Failed to parse sentry request') - end - - validate_payload(parsed_body) - - ::ErrorTracking::CollectErrorService - .new(project, nil, event: parsed_body) - .execute - - # Collector should never return any information back. - # Because DSN and public key are designed for public use, - # it is safe only for submission of new events. - # - # Some clients sdk require status 200 OK to work correctly. - # See https://gitlab.com/gitlab-org/gitlab/-/issues/343531. - status 200 - end - end -end diff --git a/lib/api/group_avatar.rb b/lib/api/group_avatar.rb index 0820011fd89..eeec23f27ab 100644 --- a/lib/api/group_avatar.rb +++ b/lib/api/group_avatar.rb @@ -4,7 +4,7 @@ module API class GroupAvatar < ::API::Base helpers Helpers::GroupsHelpers - feature_category :subgroups + feature_category :groups_and_projects params do requires :id, type: String, desc: 'The ID of the group' diff --git a/lib/api/groups.rb b/lib/api/groups.rb index e13b661b357..1a2314d41f0 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -9,7 +9,7 @@ module API helpers Helpers::GroupsHelpers - feature_category :subgroups, ['/groups/:id/custom_attributes', '/groups/:id/custom_attributes/:key'] + feature_category :groups_and_projects, ['/groups/:id/custom_attributes', '/groups/:id/custom_attributes/:key'] helpers do params :statistics_params do @@ -198,7 +198,7 @@ module API use :group_list_params use :with_custom_attributes end - get feature_category: :subgroups do + get feature_category: :groups_and_projects do groups = find_groups(declared_params(include_missing: false), params[:id]) present_groups_with_pagination_strategies params, groups end @@ -214,7 +214,7 @@ module API use :optional_params end - post feature_category: :subgroups, urgency: :low do + post feature_category: :groups_and_projects, urgency: :low do parent_group = find_group!(params[:parent_id]) if params[:parent_id].present? if parent_group authorize! :create_subgroup, parent_group @@ -248,7 +248,7 @@ module API use :optional_update_params use :optional_update_params_ee end - put ':id', feature_category: :subgroups, urgency: :low do + put ':id', feature_category: :groups_and_projects, urgency: :low do group = find_group!(params[:id]) group.preload_shared_group_links @@ -272,7 +272,7 @@ module API optional :with_projects, type: Boolean, default: true, desc: 'Omit project details' end # TODO: Set higher urgency after resolving https://gitlab.com/gitlab-org/gitlab/-/issues/357841 - get ":id", feature_category: :subgroups, urgency: :low do + get ":id", feature_category: :groups_and_projects, urgency: :low do group = find_group!(params[:id]) group.preload_shared_group_links @@ -282,7 +282,7 @@ module API desc 'Remove a group.' do tags %w[groups] end - delete ":id", feature_category: :subgroups, urgency: :low do + delete ":id", feature_category: :groups_and_projects, urgency: :low do group = find_group!(params[:id]) authorize! :admin_group, group check_subscription! group @@ -320,7 +320,7 @@ module API use :optional_projects_params end # TODO: Set higher urgency after resolving https://gitlab.com/gitlab-org/gitlab/-/issues/211498 - get ":id/projects", feature_category: :subgroups, urgency: :low do + get ":id/projects", feature_category: :groups_and_projects, urgency: :low do finder_options = { only_owned: !params[:with_shared], include_subgroups: params[:include_subgroups], @@ -356,7 +356,7 @@ module API use :pagination use :with_custom_attributes end - get ":id/projects/shared", feature_category: :subgroups do + get ":id/projects/shared", feature_category: :groups_and_projects do projects = find_group_projects(params, { only_shared: true }) present_projects(params, projects) @@ -371,7 +371,7 @@ module API use :group_list_params use :with_custom_attributes end - get ":id/subgroups", feature_category: :subgroups, urgency: :low do + get ":id/subgroups", feature_category: :groups_and_projects, urgency: :low do groups = find_groups(declared_params(include_missing: false), params[:id]) present_groups params, groups end @@ -385,7 +385,7 @@ module API use :group_list_params use :with_custom_attributes end - get ":id/descendant_groups", feature_category: :subgroups, urgency: :low do + get ":id/descendant_groups", feature_category: :groups_and_projects, urgency: :low do finder_params = declared_params(include_missing: false).merge(include_parent_descendants: true) groups = find_groups(finder_params, params[:id]) present_groups params, groups @@ -398,7 +398,7 @@ module API params do requires :project_id, type: String, desc: 'The ID or path of the project' end - post ":id/projects/:project_id", requirements: { project_id: /.+/ }, feature_category: :projects do + post ":id/projects/:project_id", requirements: { project_id: /.+/ }, feature_category: :groups_and_projects do authenticated_as_admin! group = find_group!(params[:id]) group.preload_shared_group_links @@ -421,7 +421,7 @@ module API optional :search, type: String, desc: 'Return list of namespaces matching the search criteria' use :pagination end - get ':id/transfer_locations', feature_category: :subgroups do + get ':id/transfer_locations', feature_category: :groups_and_projects do authorize! :admin_group, user_group args = declared_params(include_missing: false) @@ -440,7 +440,7 @@ module API desc: 'The ID of the target group to which the group needs to be transferred to.'\ 'If not provided, the source group will be promoted to a root group.' end - post ':id/transfer', feature_category: :subgroups do + post ':id/transfer', feature_category: :groups_and_projects do group = find_group!(params[:id]) authorize! :admin_group, group @@ -465,7 +465,7 @@ module API requires :group_access, type: Integer, values: Gitlab::Access.all_values, desc: 'The group access level' optional :expires_at, type: Date, desc: 'Share expiration date' end - post ":id/share", feature_category: :subgroups, urgency: :low do + post ":id/share", feature_category: :groups_and_projects, urgency: :low do shared_with_group = find_group!(params[:group_id]) group_link_create_params = { @@ -487,7 +487,7 @@ module API requires :group_id, type: Integer, desc: 'The ID of the shared group' end # rubocop: disable CodeReuse/ActiveRecord - delete ":id/share/:group_id", feature_category: :subgroups do + delete ":id/share/:group_id", feature_category: :groups_and_projects do shared_group = find_group!(params[:id]) link = shared_group.shared_with_group_links.find_by(shared_with_group_id: params[:group_id]) diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 9fa0923d914..df080c8e666 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -20,6 +20,10 @@ module API API_RESPONSE_STATUS_CODE = 'gitlab.api.response_status_code' INTEGER_ID_REGEX = /^-?\d+$/.freeze + def logger + API.logger + end + def declared_params(options = {}) options = { include_parent_namespaces: false }.merge(options) declared(params, options).to_h.symbolize_keys @@ -202,6 +206,7 @@ module API not_found!('Namespace') end + # find_namespace returns the namespace regardless of user access level on the namespace # rubocop: disable CodeReuse/ActiveRecord def find_namespace(id) if id.to_s =~ INTEGER_ID_REGEX @@ -212,6 +217,8 @@ module API end # rubocop: enable CodeReuse/ActiveRecord + # find_namespace! returns the namespace if the current user can read the given namespace + # Otherwise, returns a not_found! error def find_namespace!(id) check_namespace_access(find_namespace(id)) end @@ -486,6 +493,12 @@ module API render_api_error!('413 Request Entity Too Large', 413) end + def too_many_requests!(message = nil, retry_after: 1.minute) + header['Retry-After'] = retry_after.to_i if retry_after + + render_api_error!(message || '429 Too Many Requests', 429) + end + def not_modified! render_api_error!('304 Not Modified', 304) end diff --git a/lib/api/helpers/integrations_helpers.rb b/lib/api/helpers/integrations_helpers.rb index 4c37a2a5aba..850cc61af2c 100644 --- a/lib/api/helpers/integrations_helpers.rb +++ b/lib/api/helpers/integrations_helpers.rb @@ -67,6 +67,12 @@ module API type: String, desc: 'The name of the channel to receive incident_events notifications' }, + { + required: false, + name: :alert_channel, + type: String, + desc: 'The name of the channel to receive alert_events notifications' + }, { required: false, name: :confidential_issue_channel, @@ -85,12 +91,24 @@ module API type: String, desc: 'The name of the channel to receive note_events notifications' }, + { + required: false, + name: :confidential_note_channel, + type: String, + desc: 'The name of the channel to receive confidential_note_events notifications' + }, { required: false, name: :tag_push_channel, type: String, desc: 'The name of the channel to receive tag_push_events notifications' }, + { + required: false, + name: :deployment_channel, + type: String, + desc: 'The name of the channel to receive deployment_events notifications' + }, { required: false, name: :pipeline_channel, @@ -108,6 +126,12 @@ module API def self.chat_notification_events [ + { + required: false, + name: :commit_events, + type: Boolean, + desc: 'Enable notifications for commit_events' + }, { required: false, name: :push_events, @@ -126,6 +150,12 @@ module API type: Boolean, desc: 'Enable notifications for incident_events' }, + { + required: false, + name: :alert_events, + type: Boolean, + desc: 'Enable notifications for alert_events' + }, { required: false, name: :confidential_issues_events, @@ -156,6 +186,18 @@ module API type: Boolean, desc: 'Enable notifications for tag_push_events' }, + { + required: false, + name: :deployment_events, + type: Boolean, + desc: 'Enable notifications for deployment_events' + }, + { + required: false, + name: :job_events, + type: Boolean, + desc: 'Enable notifications for job_events' + }, { required: false, name: :pipeline_events, @@ -197,6 +239,12 @@ module API name: :app_store_private_key_file_name, type: String, desc: 'The Apple App Store Connect Private Key File Name' + }, + { + required: false, + name: :app_store_protected_refs, + type: Boolean, + desc: 'Only enable for protected refs' } ], 'asana' => [ @@ -397,8 +445,16 @@ module API name: :webhook, type: String, desc: 'Discord webhook. For example, https://discord.com/api/webhooks/…' - } - ], + }, + { + required: false, + name: :branches_to_be_notified, + type: String, + desc: 'Branches for which notifications are to be sent' + }, + chat_notification_flags, + chat_notification_events + ].flatten, 'drone-ci' => [ { required: true, @@ -839,6 +895,20 @@ module API desc: 'The issues URL' } ], + 'clickup' => [ + { + required: true, + name: :project_url, + type: String, + desc: 'The project URL' + }, + { + required: true, + name: :issues_url, + type: String, + desc: 'The issues URL' + } + ], 'slack' => [ chat_notification_settings, chat_notification_flags, @@ -898,6 +968,21 @@ module API desc: 'The password of the user' } ], + 'telegram' => [ + { + required: true, + name: :token, + type: String, + desc: 'The Telegram chat token. For example, 123456:ABC-DEF1234ghIkl-zyx57W2v1u123ew11' + }, + { + required: true, + name: :room, + type: String, + desc: 'Unique identifier for the target chat or username of the target channel (in the format @channelusername)' + }, + chat_notification_events + ].flatten, 'unify-circuit' => [ { required: true, @@ -968,6 +1053,7 @@ module API ::Integrations::Bugzilla, ::Integrations::Buildkite, ::Integrations::Campfire, + ::Integrations::Clickup, ::Integrations::Confluence, ::Integrations::CustomIssueTracker, ::Integrations::Datadog, diff --git a/lib/api/helpers/notes_helpers.rb b/lib/api/helpers/notes_helpers.rb index da499abe475..4b5335840f6 100644 --- a/lib/api/helpers/notes_helpers.rb +++ b/lib/api/helpers/notes_helpers.rb @@ -27,7 +27,7 @@ module API note = ::Notes::UpdateService.new(project, current_user, opts).execute(note) - if note.valid? + if note.errors.blank? present note, with: Entities::Note else bad_request!("Failed to save note #{note.errors.messages}") diff --git a/lib/api/helpers/packages/basic_auth_helpers.rb b/lib/api/helpers/packages/basic_auth_helpers.rb index a62bb1d4991..4f301d7038a 100644 --- a/lib/api/helpers/packages/basic_auth_helpers.rb +++ b/lib/api/helpers/packages/basic_auth_helpers.rb @@ -41,16 +41,15 @@ module API end def find_authorized_group! - strong_memoize(:authorized_group) do - group = find_group(params[:id]) + group = find_group(params[:id]) - unless group && can?(current_user, :read_group, group) - next unauthorized_or! { not_found! } - end - - group + unless group && can?(current_user, :read_group, group) + return unauthorized_or! { not_found! } end + + group end + strong_memoize_attr :find_authorized_group! def authorize!(action, subject = :global, reason = nil) return if can?(current_user, action, subject) diff --git a/lib/api/helpers/packages/conan/api_helpers.rb b/lib/api/helpers/packages/conan/api_helpers.rb index b47bfbfb5aa..3873fe98a5f 100644 --- a/lib/api/helpers/packages/conan/api_helpers.rb +++ b/lib/api/helpers/packages/conan/api_helpers.rb @@ -125,20 +125,18 @@ module API end def project - strong_memoize(:project) do - case package_scope - when :project - user_project(action: :read_package) - when :instance - full_path = ::Packages::Conan::Metadatum.full_path_from(package_username: params[:package_username]) - find_project!(full_path) - end + case package_scope + when :project + user_project(action: :read_package) + when :instance + full_path = ::Packages::Conan::Metadatum.full_path_from(package_username: params[:package_username]) + find_project!(full_path) end end + strong_memoize_attr :project def package - strong_memoize(:package) do - project.packages + project.packages .conan .with_name(params[:package_name]) .with_version(params[:package_version]) @@ -147,18 +145,17 @@ module API .order_created .not_pending_destruction .last - end end + strong_memoize_attr :package def token - strong_memoize(:token) do - token = nil - token = ::Gitlab::ConanToken.from_personal_access_token(find_personal_access_token.user_id, access_token_from_request) if find_personal_access_token - token = ::Gitlab::ConanToken.from_deploy_token(deploy_token_from_request) if deploy_token_from_request - token = ::Gitlab::ConanToken.from_job(find_job_from_token) if find_job_from_token - token - end + token = nil + token = ::Gitlab::ConanToken.from_personal_access_token(find_personal_access_token.user_id, access_token_from_request) if find_personal_access_token + token = ::Gitlab::ConanToken.from_deploy_token(deploy_token_from_request) if deploy_token_from_request + token = ::Gitlab::ConanToken.from_job(find_job_from_token) if find_job_from_token + token end + strong_memoize_attr :token def download_package_file(file_type) authorize_read_package!(project) @@ -227,17 +224,15 @@ module API # We override this method from auth_finders because we need to # extract the token from the Conan JWT which is specific to the Conan API def find_personal_access_token - strong_memoize(:find_personal_access_token) do - PersonalAccessToken.find_by_token(access_token_from_request) - end + PersonalAccessToken.find_by_token(access_token_from_request) end + strong_memoize_attr :find_personal_access_token def access_token_from_request - strong_memoize(:access_token_from_request) do - find_personal_access_token_from_conan_jwt || - find_password_from_basic_auth - end + find_personal_access_token_from_conan_jwt || + find_password_from_basic_auth end + strong_memoize_attr :access_token_from_request def find_password_from_basic_auth return unless route_authentication_setting[:basic_auth_personal_access_token] diff --git a/lib/api/helpers/packages/npm.rb b/lib/api/helpers/packages/npm.rb index 4eb6c39b7dc..be7f57fda0c 100644 --- a/lib/api/helpers/packages/npm.rb +++ b/lib/api/helpers/packages/npm.rb @@ -11,27 +11,22 @@ module API package_name: API::NO_SLASH_URL_PART_REGEX }.freeze - def endpoint_scope - params[:id].present? ? :project : :instance - end - def project - strong_memoize(:project) do - case endpoint_scope - when :project - user_project(action: :read_package) - when :instance - # Simulate the same behavior as #user_project by re-using #find_project! - # but take care if the project_id is nil as #find_project! is not designed - # to handle it. - project_id = project_id_or_nil - - not_found!('Project') unless project_id - - find_project!(project_id) - end + case endpoint_scope + when :project + user_project(action: :read_package) + when :instance, :group + # Simulate the same behavior as #user_project by re-using #find_project! + # but take care if the project_id is nil as #find_project! is not designed + # to handle it. + project_id = project_id_or_nil + + not_found!('Project') unless project_id + + find_project!(project_id) end end + strong_memoize_attr :project def finder_for_endpoint_scope(package_name) case endpoint_scope @@ -39,49 +34,57 @@ module API ::Packages::Npm::PackageFinder.new(package_name, project: project_or_nil) when :instance ::Packages::Npm::PackageFinder.new(package_name, namespace: top_namespace_from(package_name)) + when :group + ::Packages::Npm::PackageFinder.new(package_name, namespace: group) end end def project_or_nil # mainly used by the metadata endpoint where we need to get a project # and return nil if not found (no errors should be raised) - strong_memoize(:project_or_nil) do - next unless project_id_or_nil + return unless project_id_or_nil - find_project(project_id_or_nil) - end + find_project(project_id_or_nil) end + strong_memoize_attr :project_or_nil def project_id_or_nil - strong_memoize(:project_id_or_nil) do - case endpoint_scope - when :project - params[:id] - when :instance - package_name = params[:package_name] - - namespace = - if Feature.enabled?(:npm_allow_packages_in_multiple_projects) - top_namespace_from(package_name) - else - namespace_path = ::Packages::Npm.scope_of(package_name) - next unless namespace_path - - Namespace.top_most.by_path(namespace_path) - end - - next unless namespace - - finder = ::Packages::Npm::PackageFinder.new( - package_name, - namespace: namespace, - last_of_each_version: false - ) - - finder.last&.project_id - end + case endpoint_scope + when :project + params[:id] + when :group + finder = ::Packages::Npm::PackageFinder.new( + params[:package_name], + namespace: group, + last_of_each_version: false + ) + + finder.last&.project_id + when :instance + package_name = params[:package_name] + + namespace = + if Feature.enabled?(:npm_allow_packages_in_multiple_projects) + top_namespace_from(package_name) + else + namespace_path = ::Packages::Npm.scope_of(package_name) + return unless namespace_path + + Namespace.top_most.by_path(namespace_path) + end + + return unless namespace + + finder = ::Packages::Npm::PackageFinder.new( + package_name, + namespace: namespace, + last_of_each_version: false + ) + + finder.last&.project_id end end + strong_memoize_attr :project_id_or_nil private @@ -91,6 +94,13 @@ module API Namespace.top_most.by_path(namespace_path) end + + def group + group = find_group(params[:id]) + not_found!('Group') unless can?(current_user, :read_group, group) + group + end + strong_memoize_attr :group end end end diff --git a/lib/api/helpers/packages_helpers.rb b/lib/api/helpers/packages_helpers.rb index be2b73e2d48..f3b3a299204 100644 --- a/lib/api/helpers/packages_helpers.rb +++ b/lib/api/helpers/packages_helpers.rb @@ -4,6 +4,7 @@ module API module Helpers module PackagesHelpers extend ::Gitlab::Utils::Override + include ::Gitlab::Utils::StrongMemoize MAX_PACKAGE_FILE_SIZE = 50.megabytes.freeze ALLOWED_REQUIRED_PERMISSIONS = %i[read_package read_group].freeze @@ -71,19 +72,18 @@ module API # This function is similar to the `find_project!` function, but it considers the `read_package` ability. def user_project_with_read_package - strong_memoize(:user_project_with_read_package) do - project = find_project(params[:id]) + project = find_project(params[:id]) - next forbidden! unless authorized_project_scope?(project) + return forbidden! unless authorized_project_scope?(project) - next project if can?(current_user, :read_package, project&.packages_policy_subject) - # guest users can have :read_project but not :read_package - next forbidden! if can?(current_user, :read_project, project) - next unauthorized! if authenticate_non_public? + return project if can?(current_user, :read_package, project&.packages_policy_subject) + # guest users can have :read_project but not :read_package + return forbidden! if can?(current_user, :read_project, project) + return unauthorized! if authenticate_non_public? - not_found!('Project') - end + not_found!('Project') end + strong_memoize_attr :user_project_with_read_package def track_package_event(action, scope, **args) service = ::Packages::CreateEventService.new(nil, current_user, event_name: action, scope: scope) diff --git a/lib/api/internal/error_tracking.rb b/lib/api/internal/error_tracking.rb index 1680ac8afb5..e5047ba3e54 100644 --- a/lib/api/internal/error_tracking.rb +++ b/lib/api/internal/error_tracking.rb @@ -43,7 +43,7 @@ module API project = Project.find(project_id) enabled = error_tracking_enabled? && - Feature.enabled?(:use_click_house_database_for_error_tracking, project) && + Feature.enabled?(:gitlab_error_tracking, project) && ::ErrorTracking::ClientKey.enabled_key_for(project_id, public_key).exists? status 200 diff --git a/lib/api/internal/kubernetes.rb b/lib/api/internal/kubernetes.rb index d340e097700..5592207c4b5 100644 --- a/lib/api/internal/kubernetes.rb +++ b/lib/api/internal/kubernetes.rb @@ -69,7 +69,7 @@ module API end def increment_count_events - events = params[:counters]&.slice(:gitops_sync, :k8s_api_proxy_request) + events = params[:counters]&.slice(:gitops_sync, :k8s_api_proxy_request, :flux_git_push_notifications_total) Gitlab::UsageDataCounters::KubernetesAgentCounter.increment_event_counts(events) end @@ -121,6 +121,18 @@ module API default_branch: project.default_branch_or_main } end + + desc 'Verify agent access to a project' do + detail 'Verifies if the agent (owning the token) is authorized to access the given project' + end + route_setting :authentication, cluster_agent_token_allowed: true + get '/verify_project_access', feature_category: :deployment_management, urgency: :low do + project = find_project(params[:id]) + + not_found! unless agent_has_access_to_project?(project) + + status 204 + end end namespace 'kubernetes/agent_configuration' do @@ -190,6 +202,7 @@ module API optional :counters, type: Hash do optional :gitops_sync, type: Integer, desc: 'The count to increment the gitops_sync metric by' optional :k8s_api_proxy_request, type: Integer, desc: 'The count to increment the k8s_api_proxy_request metric by' + optional :flux_git_push_notifications_total, type: Integer, desc: 'The count to increment the flux_git_push_notifications_total metrics by' end optional :unique_counters, type: Hash do diff --git a/lib/api/issues.rb b/lib/api/issues.rb index d033913aa71..a0f7c5c9b21 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -271,11 +271,9 @@ module API issue_params = convert_parameters_from_legacy_format(issue_params) begin - spam_params = ::Spam::SpamParams.new_from_request(request: request) result = ::Issues::CreateService.new(container: user_project, current_user: current_user, - params: issue_params, - spam_params: spam_params).execute + params: issue_params).execute if result.success? present result[:issue], with: Entities::Issue, current_user: current_user, project: user_project @@ -318,11 +316,10 @@ module API update_params = convert_parameters_from_legacy_format(update_params) - spam_params = ::Spam::SpamParams.new_from_request(request: request) issue = ::Issues::UpdateService.new(container: user_project, current_user: current_user, params: update_params, - spam_params: spam_params).execute(issue) + perform_spam_check: true).execute(issue) if issue.valid? present issue, with: Entities::Issue, current_user: current_user, project: user_project diff --git a/lib/api/markdown.rb b/lib/api/markdown.rb index f348e20cc0b..5ef60ab0b94 100644 --- a/lib/api/markdown.rb +++ b/lib/api/markdown.rb @@ -2,6 +2,11 @@ module API class Markdown < ::API::Base + include APIGuard + + # Although this API endpoint responds to POST requests, it is a read-only operation + allow_access_with_scope :read_api + before { authenticate! if Feature.enabled?(:authenticate_markdown_api, type: :ops) } feature_category :team_planning diff --git a/lib/api/maven_packages.rb b/lib/api/maven_packages.rb index e075a917fa9..241cd93f380 100644 --- a/lib/api/maven_packages.rb +++ b/lib/api/maven_packages.rb @@ -60,16 +60,6 @@ module API if stored_sha256 == expected_sha256 no_content! else - # Track sha1 conflicts. - # See https://gitlab.com/gitlab-org/gitlab/-/issues/367356 - Gitlab::ErrorTracking.log_exception( - ArgumentError.new, - message: 'maven package file sha1 conflict', - stored_sha1: package_file.file_sha1, - received_sha256: uploaded_file.sha256, - sha256_hexdigest_of_stored_sha1: stored_sha256 - ) - conflict! end end diff --git a/lib/api/members.rb b/lib/api/members.rb index 9321b7ad8d5..337706f36e1 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -11,8 +11,8 @@ module API helpers ::API::Helpers::MembersHelpers { - "group" => :subgroups, - "project" => :projects + "group" => :groups_and_projects, + "project" => :groups_and_projects }.each do |source_type, feature_category| params do requires :id, type: String, desc: "The #{source_type} ID" diff --git a/lib/api/ml/mlflow/entrypoint.rb b/lib/api/ml/mlflow/entrypoint.rb index 880b1efeb5a..048234eccd1 100644 --- a/lib/api/ml/mlflow/entrypoint.rb +++ b/lib/api/ml/mlflow/entrypoint.rb @@ -26,7 +26,7 @@ module API authenticate! - not_found! unless Feature.enabled?(:ml_experiment_tracking, user_project) + not_found! unless can?(current_user, :read_model_experiments, user_project) end rescue_from ActiveRecord::ActiveRecordError do |e| diff --git a/lib/api/ml_model_packages.rb b/lib/api/ml_model_packages.rb new file mode 100644 index 00000000000..fec72b03ffd --- /dev/null +++ b/lib/api/ml_model_packages.rb @@ -0,0 +1,111 @@ +# frozen_string_literal: true + +module API + class MlModelPackages < ::API::Base + include APIGuard + include ::API::Helpers::Authentication + + ML_MODEL_PACKAGES_REQUIREMENTS = { + package_name: API::NO_SLASH_URL_PART_REGEX, + file_name: API::NO_SLASH_URL_PART_REGEX + }.freeze + + ALLOWED_STATUSES = %w[default hidden].freeze + + feature_category :mlops + urgency :low + + after_validation do + require_packages_enabled! + authenticate_non_get! + + not_found! unless can?(current_user, :read_model_registry, user_project) + end + + authenticate_with do |accept| + accept.token_types(:personal_access_token, :deploy_token, :job_token) + .sent_through(:http_token) + end + + helpers do + include ::API::Helpers::PackagesHelpers + include ::API::Helpers::Packages::BasicAuthHelpers + + def project + authorized_user_project + end + + def max_file_size_exceeded? + project.actual_limits.exceeded?(:ml_model_max_file_size, params[:file].size) + end + end + + params do + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' + end + + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + namespace ':id/packages/ml_models' do + params do + requires :package_name, type: String, desc: 'Package name', regexp: Gitlab::Regex.ml_model_name_regex, + file_path: true + requires :package_version, type: String, desc: 'Package version', + regexp: Gitlab::Regex.ml_model_version_regex + requires :file_name, type: String, desc: 'Package file name', + regexp: Gitlab::Regex.ml_model_file_name_regex, file_path: true + optional :status, type: String, values: ALLOWED_STATUSES, desc: 'Package status' + end + namespace ':package_name/*package_version/:file_name', requirements: ML_MODEL_PACKAGES_REQUIREMENTS do + desc 'Workhorse authorize model package file' do + detail 'Introduced in GitLab 16.1' + success code: 200 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[ml_model_registry] + end + put 'authorize' do + authorize_workhorse!(subject: project, maximum_size: project.actual_limits.ml_model_max_file_size) + end + + desc 'Workhorse upload model package file' do + detail 'Introduced in GitLab 16.1' + success code: 201 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[ml_model_registry] + end + params do + requires :file, + type: ::API::Validations::Types::WorkhorseFile, + desc: 'The package file to be published (generated by Multipart middleware)', + documentation: { type: 'file' } + end + put do + authorize_upload!(project) + + bad_request!('File is too large') if max_file_size_exceeded? + + create_package_file_params = declared(params).merge(build: current_authenticated_job) + package_file = ::Packages::MlModel::CreatePackageFileService + .new(project, current_user, create_package_file_params) + .execute + + bad_request!('Package creation failed') unless package_file + + created! + rescue ObjectStorage::RemoteStoreError => e + Gitlab::ErrorTracking.track_exception(e, extra: { file_name: params[:file_name], project_id: project.id }) + + forbidden! + end + end + end + end + end +end diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb index c971f73ccbb..750dc7fc2a1 100644 --- a/lib/api/namespaces.rb +++ b/lib/api/namespaces.rb @@ -38,7 +38,7 @@ module API use :pagination use :optional_list_params_ee end - get feature_category: :subgroups, urgency: :low do + get feature_category: :groups_and_projects, urgency: :low do owned_only = params[:owned_only] == true namespaces = current_user.admin ? Namespace.all : current_user.namespaces(owned_only: owned_only) @@ -66,7 +66,7 @@ module API params do requires :id, types: [String, Integer], desc: 'ID or URL-encoded path of the namespace' end - get ':id', requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS, feature_category: :subgroups, urgency: :low do + get ':id', requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS, feature_category: :groups_and_projects, urgency: :low do user_namespace = find_namespace!(params[:id]) present user_namespace, with: Entities::Namespace, current_user: current_user @@ -84,7 +84,7 @@ module API requires :id, type: String, desc: "Namespace’s path" optional :parent_id, type: Integer, desc: 'The ID of the parent namespace. If no ID is specified, only top-level namespaces are considered.' end - get ':id/exists', requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS, feature_category: :subgroups, urgency: :low do + get ':id/exists', requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS, feature_category: :groups_and_projects, urgency: :low do check_rate_limit!(:namespace_exists, scope: current_user) namespace_path = params[:id] diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 8ce875cdc03..70b4a3735e3 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -98,7 +98,7 @@ module API if note.errors.attribute_names == [:commands_only, :command_names] status 202 present note, with: Entities::NoteCommands - elsif note.valid? + elsif note.persisted? present note, with: Entities.const_get(note.class.name, false) else note.errors.delete(:commands_only) if note.errors.has_key?(:commands) diff --git a/lib/api/npm_group_packages.rb b/lib/api/npm_group_packages.rb new file mode 100644 index 00000000000..1aa3135b186 --- /dev/null +++ b/lib/api/npm_group_packages.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module API + class NpmGroupPackages < ::API::Base + helpers ::API::Helpers::Packages::Npm + + feature_category :package_registry + urgency :low + + helpers do + def endpoint_scope + :group + end + end + + params do + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the group' + end + resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + namespace ':id/-/packages/npm' do + include ::API::Concerns::Packages::NpmEndpoints + end + end + end +end diff --git a/lib/api/npm_instance_packages.rb b/lib/api/npm_instance_packages.rb index e387dd65e41..8215296b617 100644 --- a/lib/api/npm_instance_packages.rb +++ b/lib/api/npm_instance_packages.rb @@ -10,6 +10,12 @@ module API render_api_error!(e.message, 400) end + helpers do + def endpoint_scope + :instance + end + end + namespace 'packages/npm' do include ::API::Concerns::Packages::NpmEndpoints end diff --git a/lib/api/npm_project_packages.rb b/lib/api/npm_project_packages.rb index 171a061bf97..61409909b06 100644 --- a/lib/api/npm_project_packages.rb +++ b/lib/api/npm_project_packages.rb @@ -10,6 +10,12 @@ module API render_api_error!(e.message, 400) end + helpers do + def endpoint_scope + :project + end + end + params do requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end diff --git a/lib/api/nuget_group_packages.rb b/lib/api/nuget_group_packages.rb index 2afcb915b06..229032f7a5a 100644 --- a/lib/api/nuget_group_packages.rb +++ b/lib/api/nuget_group_packages.rb @@ -17,11 +17,6 @@ module API default_format :json - authenticate_with do |accept| - accept.token_types(:personal_access_token_with_username, :deploy_token_with_username, :job_token_with_username) - .sent_through(:http_basic_auth) - end - rescue_from ArgumentError do |e| render_api_error!(e.message, 400) end @@ -31,10 +26,17 @@ module API end helpers do + include ::Gitlab::Utils::StrongMemoize + def project_or_group find_authorized_group! end + def project_or_group_without_auth + find_group(params[:id]).presence || not_found! + end + strong_memoize_attr :project_or_group_without_auth + def require_authenticated! unauthorized! unless current_user end @@ -43,23 +45,38 @@ module API { namespace: find_authorized_group! } end + def snowplow_gitlab_standard_context_without_auth + { namespace: project_or_group_without_auth } + end + def required_permission :read_group end end params do - requires :id, types: [Integer, String], desc: 'The group ID or full group path.', regexp: ::API::Concerns::Packages::NugetEndpoints::POSITIVE_INTEGER_REGEX + requires :id, types: [Integer, String], desc: 'The group ID or full group path.', regexp: ::API::Concerns::Packages::Nuget::PrivateEndpoints::POSITIVE_INTEGER_REGEX end resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - namespace ':id/-/packages/nuget' do - after_validation do - # This API can't be accessed anonymously - require_authenticated! + namespace ':id/-/packages' do + namespace '/nuget' do + include ::API::Concerns::Packages::Nuget::PublicEndpoints + end + + authenticate_with do |accept| + accept.token_types(:personal_access_token_with_username, :deploy_token_with_username, :job_token_with_username) + .sent_through(:http_basic_auth) end - include ::API::Concerns::Packages::NugetEndpoints + namespace '/nuget' do + after_validation do + # This API can't be accessed anonymously + require_authenticated! + end + + include ::API::Concerns::Packages::Nuget::PrivateEndpoints + end end end end diff --git a/lib/api/nuget_project_packages.rb b/lib/api/nuget_project_packages.rb index cd16aaf6b5f..2716d6f0b64 100644 --- a/lib/api/nuget_project_packages.rb +++ b/lib/api/nuget_project_packages.rb @@ -17,14 +17,10 @@ module API PACKAGE_FILENAME = 'package.nupkg' SYMBOL_PACKAGE_FILENAME = 'package.snupkg' + API_KEY_HEADER = 'X-Nuget-Apikey' default_format :json - authenticate_with do |accept| - accept.token_types(:personal_access_token_with_username, :deploy_token_with_username, :job_token_with_username) - .sent_through(:http_basic_auth) - end - rescue_from ArgumentError do |e| render_api_error!(e.message, 400) end @@ -34,6 +30,8 @@ module API end helpers do + include ::Gitlab::Utils::StrongMemoize + params :file_params do requires :package, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)', documentation: { type: 'file' } end @@ -42,10 +40,19 @@ module API authorized_user_project(action: :read_package) end + def project_or_group_without_auth + find_project(params[:id]).presence || not_found! + end + strong_memoize_attr :project_or_group_without_auth + def snowplow_gitlab_standard_context { project: project_or_group, namespace: project_or_group.namespace } end + def snowplow_gitlab_standard_context_without_auth + { project: project_or_group_without_auth, namespace: project_or_group_without_auth.namespace } + end + def authorize_nuget_upload project = project_or_group authorize_workhorse!( @@ -97,115 +104,127 @@ module API end params do - requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project', regexp: ::API::Concerns::Packages::NugetEndpoints::POSITIVE_INTEGER_REGEX + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project', regexp: ::API::Concerns::Packages::Nuget::PrivateEndpoints::POSITIVE_INTEGER_REGEX end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - namespace ':id/packages/nuget' do - include ::API::Concerns::Packages::NugetEndpoints - - # https://docs.microsoft.com/en-us/nuget/api/package-publish-resource - desc 'The NuGet Package Publish endpoint' do - detail 'This feature was introduced in GitLab 12.6' - success code: 201 - failure [ - { code: 400, message: 'Bad Request' }, - { code: 401, message: 'Unauthorized' }, - { code: 403, message: 'Forbidden' }, - { code: 404, message: 'Not Found' } - ] - tags %w[nuget_packages] + namespace ':id/packages' do + namespace '/nuget' do + include ::API::Concerns::Packages::Nuget::PublicEndpoints end - params do - use :file_params + authenticate_with do |accept| + accept.token_types(:personal_access_token_with_username, :deploy_token_with_username, :job_token_with_username) + .sent_through(:http_basic_auth) end - put urgency: :low do - upload_nuget_package_file do |package| - track_package_event( - 'push_package', - :nuget, - category: 'API::NugetPackages', - project: package.project, - namespace: package.project.namespace - ) - end - rescue ObjectStorage::RemoteStoreError => e - Gitlab::ErrorTracking.track_exception(e, extra: { file_name: params[:file_name], project_id: project_or_group.id }) - forbidden! - end + namespace '/nuget' do + include ::API::Concerns::Packages::Nuget::PrivateEndpoints - desc 'The NuGet Package Authorize endpoint' do - detail 'This feature was introduced in GitLab 14.1' - success code: 200 - failure [ - { code: 401, message: 'Unauthorized' }, - { code: 403, message: 'Forbidden' }, - { code: 404, message: 'Not Found' } - ] - tags %w[nuget_packages] - end - put 'authorize', urgency: :low do - authorize_nuget_upload - end - - # https://docs.microsoft.com/en-us/nuget/api/symbol-package-publish-resource - desc 'The NuGet Symbol Package Publish endpoint' do - detail 'This feature was introduced in GitLab 14.1' - success code: 201 - failure [ - { code: 400, message: 'Bad Request' }, - { code: 401, message: 'Unauthorized' }, - { code: 403, message: 'Forbidden' }, - { code: 404, message: 'Not Found' } - ] - tags %w[nuget_packages] - end - params do - use :file_params - end - put 'symbolpackage', urgency: :low do - upload_nuget_package_file(symbol_package: true) do |package| - track_package_event( - 'push_symbol_package', - :nuget, - category: 'API::NugetPackages', - project: package.project, - namespace: package.project.namespace - ) + # https://docs.microsoft.com/en-us/nuget/api/package-base-address-resource + params do + requires :package_name, type: String, desc: 'The NuGet package name', regexp: API::NO_SLASH_URL_PART_REGEX, documentation: { example: 'mynugetpkg.1.3.0.17.nupkg' } + end + namespace '/download/*package_name' do + after_validation do + authorize_read_package!(project_or_group) + end + + desc 'The NuGet Content Service - index request' do + detail 'This feature was introduced in GitLab 12.8' + success code: 200, model: ::API::Entities::Nuget::PackagesVersions + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[nuget_packages] + end + get 'index', format: :json, urgency: :low do + present ::Packages::Nuget::PackagesVersionsPresenter.new(find_packages(params[:package_name])), + with: ::API::Entities::Nuget::PackagesVersions + end + + desc 'The NuGet Content Service - content request' do + detail 'This feature was introduced in GitLab 12.8' + success code: 200 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[nuget_packages] + end + params do + requires :package_version, type: String, desc: 'The NuGet package version', regexp: API::NO_SLASH_URL_PART_REGEX, documentation: { example: '1.3.0.17' } + requires :package_filename, type: String, desc: 'The NuGet package filename', regexp: API::NO_SLASH_URL_PART_REGEX, documentation: { example: 'mynugetpkg.1.3.0.17.nupkg' } + end + get '*package_version/*package_filename', format: [:nupkg, :snupkg], urgency: :low do + filename = "#{params[:package_filename]}.#{params[:format]}" + package_file = ::Packages::PackageFileFinder.new(find_package(params[:package_name], params[:package_version]), filename, with_file_name_like: true) + .execute + + not_found!('Package') unless package_file + + track_package_event( + params[:format] == 'snupkg' ? 'pull_symbol_package' : 'pull_package', + :nuget, + category: 'API::NugetPackages', + project: package_file.project, + namespace: package_file.project.namespace + ) + + # nuget and dotnet don't support 302 Moved status codes, supports_direct_download has to be set to false + present_package_file!(package_file, supports_direct_download: false) + end end - rescue ObjectStorage::RemoteStoreError => e - Gitlab::ErrorTracking.track_exception(e, extra: { file_name: params[:file_name], project_id: project_or_group.id }) - - forbidden! end - desc 'The NuGet Symbol Package Authorize endpoint' do - detail 'This feature was introduced in GitLab 14.1' - success code: 200 - failure [ - { code: 401, message: 'Unauthorized' }, - { code: 403, message: 'Forbidden' }, - { code: 404, message: 'Not Found' } - ] - tags %w[nuget_packages] - end - put 'symbolpackage/authorize', urgency: :low do - authorize_nuget_upload + # To support an additional authentication option for download endpoints, + # we redefine the `authenticate_with` method by combining the previous + # authentication option with the new one. + authenticate_with do |accept| + accept.token_types(:personal_access_token_with_username, :deploy_token_with_username, :job_token_with_username) + .sent_through(:http_basic_auth) + accept.token_types(:personal_access_token, :deploy_token, :job_token) + .sent_through(http_header: API_KEY_HEADER) end - # https://docs.microsoft.com/en-us/nuget/api/package-base-address-resource - params do - requires :package_name, type: String, desc: 'The NuGet package name', regexp: API::NO_SLASH_URL_PART_REGEX, documentation: { example: 'mynugetpkg.1.3.0.17.nupkg' } - end - namespace '/download/*package_name' do - after_validation do - authorize_read_package!(project_or_group) + namespace '/nuget' do + # https://docs.microsoft.com/en-us/nuget/api/package-publish-resource + desc 'The NuGet Package Publish endpoint' do + detail 'This feature was introduced in GitLab 12.6' + success code: 201 + failure [ + { code: 400, message: 'Bad Request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[nuget_packages] + end + + params do + use :file_params + end + put urgency: :low do + upload_nuget_package_file do |package| + track_package_event( + 'push_package', + :nuget, + category: 'API::NugetPackages', + project: package.project, + namespace: package.project.namespace + ) + end + rescue ObjectStorage::RemoteStoreError => e + Gitlab::ErrorTracking.track_exception(e, extra: { file_name: params[:file_name], project_id: project_or_group.id }) + + forbidden! end - desc 'The NuGet Content Service - index request' do - detail 'This feature was introduced in GitLab 12.8' - success code: 200, model: ::API::Entities::Nuget::PackagesVersions + desc 'The NuGet Package Authorize endpoint' do + detail 'This feature was introduced in GitLab 14.1' + success code: 200 failure [ { code: 401, message: 'Unauthorized' }, { code: 403, message: 'Forbidden' }, @@ -213,15 +232,16 @@ module API ] tags %w[nuget_packages] end - get 'index', format: :json, urgency: :low do - present ::Packages::Nuget::PackagesVersionsPresenter.new(find_packages(params[:package_name])), - with: ::API::Entities::Nuget::PackagesVersions + put 'authorize', urgency: :low do + authorize_nuget_upload end - desc 'The NuGet Content Service - content request' do - detail 'This feature was introduced in GitLab 12.8' - success code: 200 + # https://docs.microsoft.com/en-us/nuget/api/symbol-package-publish-resource + desc 'The NuGet Symbol Package Publish endpoint' do + detail 'This feature was introduced in GitLab 14.1' + success code: 201 failure [ + { code: 400, message: 'Bad Request' }, { code: 401, message: 'Unauthorized' }, { code: 403, message: 'Forbidden' }, { code: 404, message: 'Not Found' } @@ -229,26 +249,36 @@ module API tags %w[nuget_packages] end params do - requires :package_version, type: String, desc: 'The NuGet package version', regexp: API::NO_SLASH_URL_PART_REGEX, documentation: { example: '1.3.0.17' } - requires :package_filename, type: String, desc: 'The NuGet package filename', regexp: API::NO_SLASH_URL_PART_REGEX, documentation: { example: 'mynugetpkg.1.3.0.17.nupkg' } + use :file_params + end + put 'symbolpackage', urgency: :low do + upload_nuget_package_file(symbol_package: true) do |package| + track_package_event( + 'push_symbol_package', + :nuget, + category: 'API::NugetPackages', + project: package.project, + namespace: package.project.namespace + ) + end + rescue ObjectStorage::RemoteStoreError => e + Gitlab::ErrorTracking.track_exception(e, extra: { file_name: params[:file_name], project_id: project_or_group.id }) + + forbidden! + end + + desc 'The NuGet Symbol Package Authorize endpoint' do + detail 'This feature was introduced in GitLab 14.1' + success code: 200 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[nuget_packages] end - get '*package_version/*package_filename', format: [:nupkg, :snupkg], urgency: :low do - filename = "#{params[:package_filename]}.#{params[:format]}" - package_file = ::Packages::PackageFileFinder.new(find_package(params[:package_name], params[:package_version]), filename, with_file_name_like: true) - .execute - - not_found!('Package') unless package_file - - track_package_event( - params[:format] == 'snupkg' ? 'pull_symbol_package' : 'pull_package', - :nuget, - category: 'API::NugetPackages', - project: package_file.project, - namespace: package_file.project.namespace - ) - - # nuget and dotnet don't support 302 Moved status codes, supports_direct_download has to be set to false - present_package_file!(package_file, supports_direct_download: false) + put 'symbolpackage/authorize', urgency: :low do + authorize_nuget_upload end end end diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index ced8ecec883..db97f4988e1 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -9,7 +9,7 @@ module API before { authenticate! } before { authorize_admin_project } - feature_category :integrations + feature_category :webhooks helpers ::API::Helpers::WebHooksHelpers diff --git a/lib/api/project_job_token_scope.rb b/lib/api/project_job_token_scope.rb index 7fd288491ef..79710bffeaf 100644 --- a/lib/api/project_job_token_scope.rb +++ b/lib/api/project_job_token_scope.rb @@ -2,6 +2,8 @@ module API class ProjectJobTokenScope < ::API::Base + include PaginationParams + before { authenticate! } feature_category :secrets_management @@ -22,6 +24,134 @@ module API present user_project, with: Entities::ProjectJobTokenScope end + + desc 'Patch CI_JOB_TOKEN access settings.' do + failure [ + { code: 400, message: 'Bad Request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + success code: 204 + tags %w[projects_job_token_scope] + end + params do + requires :enabled, + type: Boolean, + as: :ci_inbound_job_token_scope_enabled, + allow_blank: false, + desc: "Indicates CI/CD job tokens generated in other projects have restricted access to this project." + end + + patch ':id/job_token_scope' do + authorize_admin_project + + job_token_scope_params = declared_params(include_missing: false) + result = ::Projects::UpdateService.new(user_project, current_user, job_token_scope_params).execute + + break bad_request!(result[:message]) if result[:status] == :error + + no_content! + end + + desc 'Fetch project inbound allowlist for CI_JOB_TOKEN access settings.' do + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + success status: 200, model: Entities::BasicProjectDetails + tags %w[projects_job_token_scope] + end + params do + use :pagination + end + get ':id/job_token_scope/allowlist' do + authorize_admin_project + + inbound_projects = ::Ci::JobToken::Scope.new(user_project).inbound_projects + + present paginate(inbound_projects), with: Entities::BasicProjectDetails + end + + desc 'Add target project to allowlist.' do + failure [ + { code: 400, message: 'Bad Request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' }, + { code: 422, message: 'Unprocessable entity' } + ] + success status: 201, model: Entities::BasicProjectDetails + tags %w[projects_job_token_scope] + end + params do + requires :id, + allow_blank: false, + desc: 'ID of user project', + documentation: { example: 1 }, + type: Integer + + requires :target_project_id, + allow_blank: false, + desc: 'ID of target project', + documentation: { example: 2 }, + type: Integer + end + post ':id/job_token_scope/allowlist' do + authorize_admin_project + + target_project_id = declared_params(include_missing: false).fetch(:target_project_id) + target_project = Project.find_by_id(target_project_id) + break not_found!("target_project_id not found") if target_project.blank? + + result = ::Ci::JobTokenScope::AddProjectService + .new(user_project, current_user) + .execute(target_project, direction: :inbound) + + break bad_request!(result[:message]) if result.error? + + present result.payload[:project_link], with: Entities::ProjectScopeLink + end + + desc 'Delete project from allowlist.' do + failure [ + { code: 400, message: 'Bad Request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + success code: 204 + tags %w[projects_job_token_scope] + end + params do + requires :id, + allow_blank: false, + desc: 'ID of user project', + documentation: { example: 1 }, + type: Integer + + requires :target_project_id, + allow_blank: false, + desc: 'ID of the project to be removed from the allowlist', + documentation: { example: 2 }, + type: Integer + end + delete ':id/job_token_scope/allowlist/:target_project_id' do + target_project = find_project!(params[:target_project_id]) + + result = ::Ci::JobTokenScope::RemoveProjectService + .new(user_project, current_user) + .execute(target_project, :inbound) + + if result.success? + no_content! + elsif result.reason == :insufficient_permissions + forbidden!(result.message) + else + bad_request!(result.message) + end + end end end end diff --git a/lib/api/project_packages.rb b/lib/api/project_packages.rb index 158ba7465f4..43bd15931ef 100644 --- a/lib/api/project_packages.rb +++ b/lib/api/project_packages.rb @@ -2,8 +2,11 @@ module API class ProjectPackages < ::API::Base + include Gitlab::Utils::StrongMemoize include PaginationParams + PIPELINE_COLUMNS = %i[id iid project_id sha ref status source created_at updated_at user_id].freeze + before do authorize_packages_access!(user_project) end @@ -12,6 +15,13 @@ module API urgency :low helpers ::API::Helpers::PackagesHelpers + helpers do + def package + strong_memoize(:package) do # rubocop:disable Gitlab/StrongMemoizeAttr + ::Packages::PackageFinder.new(user_project, declared_params[:package_id]).execute + end + end + end params do requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' @@ -66,14 +76,45 @@ module API end route_setting :authentication, job_token_allowed: true get ':id/packages/:package_id' do - package = ::Packages::PackageFinder - .new(user_project, params[:package_id]).execute - render_api_error!('Package not found', 404) unless package.default? present package, with: ::API::Entities::Package, user: current_user, namespace: user_project.namespace end + desc 'Get the pipelines for a single project package' do + detail 'This feature was introduced in GitLab 16.1' + success code: 200, model: ::API::Entities::Package::Pipeline + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[project_packages] + end + params do + use :pagination + requires :package_id, type: Integer, desc: 'The ID of a package' + optional :cursor, type: String, desc: 'Cursor for obtaining the next set of records' + # Overrides the original definition to add the `values: 1..20` restriction + optional :per_page, type: Integer, default: 20, + desc: 'Number of items per page', documentation: { example: 20 }, + values: 1..20 + end + route_setting :authentication, job_token_allowed: true + get ':id/packages/:package_id/pipelines' do + not_found!('Package not found') unless package.default? + + params[:pagination] = 'keyset' # keyset is the only available pagination + pipelines = paginate_with_strategies( + package.build_infos.without_empty_pipelines, + paginator_params: { per_page: declared_params[:per_page], cursor: declared_params[:cursor] } + ) do |results| + ::Ci::Pipeline.id_in(results.map(&:pipeline_id)).select(PIPELINE_COLUMNS).order_id_desc + end + + present pipelines, with: ::API::Entities::Package::Pipeline, user: current_user + end + desc 'Delete a project package' do detail 'This feature was introduced in GitLab 11.9' success code: 204 @@ -90,9 +131,6 @@ module API delete ':id/packages/:package_id' do authorize_destroy_package!(user_project) - package = ::Packages::PackageFinder - .new(user_project, params[:package_id]).execute - destroy_conditionally!(package) do |package| ::Packages::MarkPackageForDestructionService.new(container: package, current_user: current_user).execute end diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb index 7ef722301ca..a503b941593 100644 --- a/lib/api/project_snippets.rb +++ b/lib/api/project_snippets.rb @@ -90,9 +90,7 @@ module API authorize! :create_snippet, user_project snippet_params = process_create_params(declared_params(include_missing: false)) - - spam_params = ::Spam::SpamParams.new_from_request(request: request) - service_response = ::Snippets::CreateService.new(project: user_project, current_user: current_user, params: snippet_params, spam_params: spam_params).execute + service_response = ::Snippets::CreateService.new(project: user_project, current_user: current_user, params: snippet_params).execute snippet = service_response.payload[:snippet] if service_response.success? @@ -138,9 +136,7 @@ module API validate_params_for_multiple_files(snippet) snippet_params = process_update_params(declared_params(include_missing: false)) - - spam_params = ::Spam::SpamParams.new_from_request(request: request) - service_response = ::Snippets::UpdateService.new(project: user_project, current_user: current_user, params: snippet_params, spam_params: spam_params).execute(snippet) + service_response = ::Snippets::UpdateService.new(project: user_project, current_user: current_user, params: snippet_params, perform_spam_check: true).execute(snippet) snippet = service_response.payload[:snippet] if service_response.success? diff --git a/lib/api/project_templates.rb b/lib/api/project_templates.rb index 2360a7e6b2a..49e2e4d8a91 100644 --- a/lib/api/project_templates.rb +++ b/lib/api/project_templates.rb @@ -4,7 +4,7 @@ module API class ProjectTemplates < ::API::Base include PaginationParams - TEMPLATE_TYPES = %w[dockerfiles gitignores gitlab_ci_ymls licenses metrics_dashboard_ymls issues merge_requests].freeze + TEMPLATE_TYPES = %w[dockerfiles gitignores gitlab_ci_ymls licenses issues merge_requests].freeze # The regex is needed to ensure a period (e.g. agpl-3.0) # isn't confused with a format type. We also need to allow encoded # values (e.g. C%2B%2B for C++), so allow % and + as well. @@ -16,7 +16,7 @@ module API params do requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' - requires :type, type: String, values: TEMPLATE_TYPES, desc: 'The type (dockerfiles|gitignores|gitlab_ci_ymls|licenses|metrics_dashboard_ymls|issues|merge_requests) of the template' + requires :type, type: String, values: TEMPLATE_TYPES, desc: 'The type (dockerfiles|gitignores|gitlab_ci_ymls|licenses|issues|merge_requests) of the template' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get a list of templates available to this project' do @@ -32,8 +32,6 @@ module API use :pagination end get ':id/templates/:type' do - bad_request! if params[:type] == 'metrics_dashboard_ymls' && Feature.enabled?(:remove_monitor_metrics) - templates = TemplateFinder.all_template_names(user_project, params[:type]).values.flatten present paginate(::Kaminari.paginate_array(templates)), with: Entities::TemplatesList @@ -62,8 +60,6 @@ module API end get ':id/templates/:type/:name', requirements: TEMPLATE_NAMES_ENDPOINT_REQUIREMENTS do - bad_request! if params[:type] == 'metrics_dashboard_ymls' && Feature.enabled?(:remove_monitor_metrics) - begin template = TemplateFinder.build( params[:type], diff --git a/lib/api/projects.rb b/lib/api/projects.rb index d6863e4eba4..7ec9f72e0b2 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -9,7 +9,7 @@ module API before { authenticate_non_get! } - feature_category :projects, %w[ + feature_category :groups_and_projects, %w[ /projects/:id/custom_attributes /projects/:id/custom_attributes/:key /projects/:id/share_locations @@ -20,8 +20,6 @@ module API helpers do # EE::API::Projects would override this method def apply_filters(projects) - projects = projects.with_issues_available_for_user(current_user) if params[:with_issues_enabled] - projects = projects.with_merge_requests_enabled if params[:with_merge_requests_enabled] projects = projects.with_statistics if params[:statistics] projects = projects.joins(:statistics) if params[:order_by].include?('project_statistics') # rubocop: disable CodeReuse/ActiveRecord projects = projects.created_by(current_user).imported.with_import_state if params[:imported] @@ -244,7 +242,7 @@ module API use :statistics_params use :with_custom_attributes end - get ":user_id/projects", feature_category: :projects, urgency: :low do + get ":user_id/projects", feature_category: :groups_and_projects, urgency: :low do user = find_user(params[:user_id]) not_found!('User') unless user @@ -264,7 +262,7 @@ module API use :collection_params use :statistics_params end - get ":user_id/starred_projects", feature_category: :projects, urgency: :low do + get ":user_id/starred_projects", feature_category: :groups_and_projects, urgency: :low do user = find_user(params[:user_id]) not_found!('User') unless user @@ -290,7 +288,7 @@ module API use :with_custom_attributes end # TODO: Set higher urgency https://gitlab.com/gitlab-org/gitlab/-/issues/211495 - get feature_category: :projects, urgency: :low do + get feature_category: :groups_and_projects, urgency: :low do validate_projects_api_rate_limit_for_unauthenticated_users! validate_updated_at_order_and_filter! @@ -359,7 +357,7 @@ module API use :create_params end # rubocop: disable CodeReuse/ActiveRecord - post "user/:user_id", feature_category: :projects do + post "user/:user_id", feature_category: :groups_and_projects do Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/issues/21139') authenticated_as_admin! user = User.find_by(id: params.delete(:user_id)) @@ -416,7 +414,7 @@ module API desc: 'Include project license data' end # TODO: Set higher urgency https://gitlab.com/gitlab-org/gitlab/-/issues/357622 - get ":id", feature_category: :projects, urgency: :low do + get ":id", feature_category: :groups_and_projects, urgency: :low do options = { with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails, current_user: current_user, @@ -527,7 +525,7 @@ module API at_least_one_of(*Helpers::ProjectsHelpers.update_params_at_least_one_of) end - put ':id', feature_category: :projects do + put ':id', feature_category: :groups_and_projects do authorize_admin_project attrs = declared_params(include_missing: false) authorize! :rename_project, user_project if attrs[:name].present? @@ -559,7 +557,7 @@ module API ] tags %w[projects] end - post ':id/archive', feature_category: :projects do + post ':id/archive', feature_category: :groups_and_projects do authorize!(:archive_project, user_project) ::Projects::UpdateService.new(user_project, current_user, archived: true).execute @@ -574,7 +572,7 @@ module API ] tags %w[projects] end - post ':id/unarchive', feature_category: :projects, urgency: :default do + post ':id/unarchive', feature_category: :groups_and_projects, urgency: :default do authorize!(:archive_project, user_project) ::Projects::UpdateService.new(user_project, current_user, archived: false).execute @@ -590,7 +588,7 @@ module API ] tags %w[projects] end - post ':id/star', feature_category: :projects do + post ':id/star', feature_category: :groups_and_projects do if current_user.starred?(user_project) not_modified! else @@ -609,7 +607,7 @@ module API ] tags %w[projects] end - post ':id/unstar', feature_category: :projects do + post ':id/unstar', feature_category: :groups_and_projects do if current_user.starred?(user_project) current_user.toggle_star(user_project) user_project.reset @@ -633,7 +631,7 @@ module API optional :search, type: String, desc: 'Return list of users matching the search criteria', documentation: { example: 'user' } use :pagination end - get ':id/starrers', feature_category: :projects do + get ':id/starrers', feature_category: :groups_and_projects do starrers = UsersStarProjectsFinder.new(user_project, params, current_user: current_user).execute present paginate(starrers), with: Entities::UserStarsProject @@ -661,7 +659,7 @@ module API ] tags %w[projects] end - delete ":id", feature_category: :projects do + delete ":id", feature_category: :groups_and_projects do authorize! :remove_project, user_project delete_project(user_project) @@ -729,7 +727,7 @@ module API requires :group_access, type: Integer, values: Gitlab::Access.values, as: :link_group_access, desc: 'The group access level' optional :expires_at, type: Date, desc: 'Share expiration date' end - post ":id/share", feature_category: :projects do + post ":id/share", feature_category: :groups_and_projects do authorize! :admin_project, user_project shared_with_group = Group.find_by_id(params[:group_id]) @@ -759,7 +757,7 @@ module API requires :group_id, type: Integer, desc: 'The ID of the group' end # rubocop: disable CodeReuse/ActiveRecord - delete ":id/share/:group_id", feature_category: :projects do + delete ":id/share/:group_id", feature_category: :groups_and_projects do authorize! :admin_project, user_project link = user_project.project_group_links.find_by(group_id: params[:group_id]) @@ -783,7 +781,7 @@ module API params do requires :project_id, type: Integer, desc: 'The ID of the source project to import the members from.' end - post ":id/import_project_members/:project_id", feature_category: :projects do + post ":id/import_project_members/:project_id", feature_category: :groups_and_projects do ::Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/355916') authorize! :admin_project, user_project @@ -947,7 +945,7 @@ module API params do requires :namespace, type: String, desc: 'The ID or path of the new namespace', documentation: { example: 'gitlab' } end - put ":id/transfer", feature_category: :projects do + put ":id/transfer", feature_category: :groups_and_projects do authorize! :change_namespace, user_project namespace = find_namespace!(params[:namespace]) @@ -972,13 +970,13 @@ module API optional :search, type: String, desc: 'Return list of namespaces matching the search criteria', documentation: { example: 'search' } use :pagination end - get ":id/transfer_locations", feature_category: :projects do + get ":id/transfer_locations", feature_category: :groups_and_projects do authorize! :change_namespace, user_project args = declared_params(include_missing: false) args[:permission_scope] = :transfer_projects groups = ::Groups::UserGroupsFinder.new(current_user, current_user, args).execute - groups = groups.with_route + groups = groups.excluding_groups(user_project.group).with_route present_groups(groups) end diff --git a/lib/api/release/links.rb b/lib/api/release/links.rb index 311fcf9aba1..d0234439057 100644 --- a/lib/api/release/links.rb +++ b/lib/api/release/links.rb @@ -20,7 +20,7 @@ module API end resource 'projects/:id', requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do params do - requires :tag_name, type: String, desc: 'The tag associated with the release', as: :tag + requires :tag_name, type: String, desc: 'The tag associated with the release' end resource 'releases/:tag_name', requirements: RELEASE_ENDPOINT_REQUIREMENTS do resource :assets do @@ -56,7 +56,8 @@ module API params do requires :name, type: String, desc: 'The name of the link. Link names must be unique in the release' requires :url, type: String, desc: 'The URL of the link. Link URLs must be unique in the release.' - optional :direct_asset_path, type: String, desc: 'Optional path for a direct asset link', as: :filepath + optional :direct_asset_path, type: String, desc: 'Optional path for a direct asset link' + optional :filepath, type: String, desc: 'Deprecated: optional path for a direct asset link' optional :link_type, type: String, values: %w[other runbook image package], @@ -110,7 +111,8 @@ module API params do optional :name, type: String, desc: 'The name of the link' optional :url, type: String, desc: 'The URL of the link' - optional :direct_asset_path, type: String, desc: 'Optional path for a direct asset link', as: :filepath + optional :direct_asset_path, type: String, desc: 'Optional path for a direct asset link' + optional :filepath, type: String, desc: 'Deprecated: optional path for a direct asset link' optional :link_type, type: String, values: %w[other runbook image package], @@ -164,11 +166,11 @@ module API helpers do def release - @release ||= user_project.releases.find_by_tag!(params[:tag]) + @release ||= user_project.releases.find_by_tag!(declared_params(include_parent_namespaces: true)[:tag_name]) end def link - @link ||= release.links.find(params[:link_id]) + @link ||= release.links.find(declared_params(include_parent_namespaces: true)[:link_id]) end end end diff --git a/lib/api/releases.rb b/lib/api/releases.rb index 0b31a3e0309..5d056ade3da 100644 --- a/lib/api/releases.rb +++ b/lib/api/releases.rb @@ -109,7 +109,7 @@ module API cache_context: -> (_) { "user:{#{current_user&.id}}" }, expires_in: 5.minutes, current_user: current_user, - include_html_description: params[:include_html_description] + include_html_description: declared_params[:include_html_description] end desc 'Get a release by a tag name' do @@ -135,7 +135,7 @@ module API not_found! unless release - present release, with: Entities::Release, current_user: current_user, include_html_description: params[:include_html_description] + present release, with: Entities::Release, current_user: current_user, include_html_description: declared_params[:include_html_description] end desc 'Download a project release asset file' do @@ -162,8 +162,8 @@ module API not_found! unless release - link = release.links.find_by_filepath!("/#{params[:filepath]}") - + filepath = declared_params(include_missing: false)[:filepath] + link = release.links.find_by_filepath!("/#{filepath}") not_found! unless link redirect link.url @@ -196,7 +196,7 @@ module API redirect_url = api_v4_projects_releases_path(id: user_project.id, tag_name: latest_release.tag) # Include the additional suffix_path if present - redirect_url += "/#{params[:suffix_path]}" if params[:suffix_path].present? + redirect_url += "/#{declared_params[:suffix_path]}" if declared_params[:suffix_path].present? # Include any query parameter except `order_by` since we have plans to extend it in the future. # See https://gitlab.com/gitlab-org/gitlab/-/issues/352945 for reference. @@ -238,7 +238,8 @@ module API optional :links, type: Array do requires :name, type: String, desc: 'The name of the link. Link names must be unique within the release' requires :url, type: String, desc: 'The URL of the link. Link URLs must be unique within the release' - optional :direct_asset_path, type: String, desc: 'Optional path for a direct asset link', as: :filepath + optional :direct_asset_path, type: String, desc: 'Optional path for a direct asset link' + optional :filepath, type: String, desc: 'Deprecated: optional path for a direct asset link' optional :link_type, type: String, desc: 'The type of the link: `other`, `runbook`, `image`, `package`. Defaults to `other`' end end @@ -392,7 +393,7 @@ module API end def release - @release ||= user_project.releases.find_by_tag(params[:tag]) + @release ||= user_project.releases.find_by_tag(declared_params[:tag]) end def find_latest_release diff --git a/lib/api/resource_access_tokens.rb b/lib/api/resource_access_tokens.rb index b98ed5ec9ff..1ad5bc8d421 100644 --- a/lib/api/resource_access_tokens.rb +++ b/lib/api/resource_access_tokens.rb @@ -92,17 +92,30 @@ module API success Entities::ResourceAccessTokenWithToken end params do - requires :id, type: String, desc: "The #{source_type} ID", documentation: { example: 2 } - requires :name, type: String, desc: "Resource access token name", documentation: { example: 'test' } - requires :scopes, type: Array[String], values: ::Gitlab::Auth.resource_bot_scopes.map(&:to_s), - desc: "The permissions of the token", - documentation: { example: %w[api read_repository] } - optional :access_level, type: Integer, - values: ALLOWED_RESOURCE_ACCESS_LEVELS.values, - default: Gitlab::Access::MAINTAINER, - desc: "The access level of the token in the #{source_type}", - documentation: { example: 40 } - optional :expires_at, type: Date, desc: "The expiration date of the token", documentation: { example: '"2021-01-31' } + requires :id, + type: String, + desc: "The #{source_type} ID", + documentation: { example: 2 } + requires :name, + type: String, + desc: "Resource access token name", + documentation: { example: 'test' } + requires :scopes, + type: Array[String], + values: ::Gitlab::Auth.resource_bot_scopes.map(&:to_s), + desc: "The permissions of the token", + documentation: { example: %w[api read_repository] } + requires :expires_at, + type: Date, + desc: "The expiration date of the token", + default: PersonalAccessToken::MAX_PERSONAL_ACCESS_TOKEN_LIFETIME_IN_DAYS.days.from_now, + documentation: { example: '"2021-01-31' } + optional :access_level, + type: Integer, + values: ALLOWED_RESOURCE_ACCESS_LEVELS.values, + default: Gitlab::Access::MAINTAINER, + desc: "The access level of the token in the #{source_type}", + documentation: { example: 40 } end post ':id/access_tokens' do resource = find_source(source_type, params[:id]) diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 7d6e2ee4d4c..5d20444cb54 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -125,11 +125,15 @@ module API given plantuml_enabled: ->(val) { val } do requires :plantuml_url, type: String, desc: 'The PlantUML server URL' end + optional :diagramsnet_enabled, type: Boolean, desc: 'Enable Diagrams.net' + given diagramsnet_enabled: ->(val) { val } do + requires :diagramsnet_url, type: String, desc: 'The Diagrams.net server URL' + end optional :polling_interval_multiplier, type: BigDecimal, desc: 'Interval multiplier used by endpoints that perform polling. Set to 0 to disable polling.' optional :project_export_enabled, type: Boolean, desc: 'Enable project export' optional :prometheus_metrics_enabled, type: Boolean, desc: 'Enable Prometheus metrics' - optional :push_event_hooks_limit, type: Integer, desc: "Number of changes (branches or tags) in a single push to determine whether webhooks and services will be fired or not. Webhooks and services won't be submitted if it surpasses that value." - optional :push_event_activities_limit, type: Integer, desc: 'Number of changes (branches or tags) in a single push to determine whether individual push events or bulk push event will be created. Bulk push event will be created if it surpasses that value.' + optional :push_event_hooks_limit, type: Integer, desc: "Maximum number of changes (branches or tags) in a single push above which webhooks and integrations are not triggered. Setting to `0` does not disable throttling." + optional :push_event_activities_limit, type: Integer, desc: 'Maximum number of changes (branches or tags) in a single push above which a bulk push event is created. Setting to `0` does not disable throttling.' optional :recaptcha_enabled, type: Boolean, desc: 'Helps prevent bots from creating accounts' given recaptcha_enabled: ->(val) { val } do requires :recaptcha_site_key, type: String, desc: 'Generate site key at http://www.google.com/recaptcha' @@ -182,6 +186,7 @@ module API optional :issues_create_limit, type: Integer, desc: "Maximum number of issue creation requests allowed per minute per user. Set to 0 for unlimited requests per minute." optional :raw_blob_request_limit, type: Integer, desc: "Maximum number of requests per minute for each raw path. Set to 0 for unlimited requests per minute." optional :wiki_page_max_content_bytes, type: Integer, desc: "Maximum wiki page content size in bytes" + optional :wiki_asciidoc_allow_uri_includes, type: Boolean, desc: "Allow URI includes for AsciiDoc wiki pages" optional :require_admin_approval_after_user_signup, type: Boolean, desc: 'Require explicit admin approval for new signups' optional :whats_new_variant, type: String, values: ApplicationSetting.whats_new_variants.keys, desc: "What's new variant, possible values: `all_tiers`, `current_tier`, and `disabled`." optional :floc_enabled, type: Grape::API::Boolean, desc: 'Enable FloC (Federated Learning of Cohorts)' diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb index 104848206a3..77872e7d13c 100644 --- a/lib/api/snippets.rb +++ b/lib/api/snippets.rb @@ -113,9 +113,7 @@ module API authorize! :create_snippet attrs = process_create_params(declared_params(include_missing: false)) - - spam_params = ::Spam::SpamParams.new_from_request(request: request) - service_response = ::Snippets::CreateService.new(project: nil, current_user: current_user, params: attrs, spam_params: spam_params).execute + service_response = ::Snippets::CreateService.new(project: nil, current_user: current_user, params: attrs).execute snippet = service_response.payload[:snippet] if service_response.success? @@ -162,9 +160,7 @@ module API validate_params_for_multiple_files(snippet) attrs = process_update_params(declared_params(include_missing: false)) - - spam_params = ::Spam::SpamParams.new_from_request(request: request) - service_response = ::Snippets::UpdateService.new(project: nil, current_user: current_user, params: attrs, spam_params: spam_params).execute(snippet) + service_response = ::Snippets::UpdateService.new(project: nil, current_user: current_user, params: attrs, perform_spam_check: true).execute(snippet) snippet = service_response.payload[:snippet] diff --git a/lib/api/system_hooks.rb b/lib/api/system_hooks.rb index f2019d785a0..3473b1922d6 100644 --- a/lib/api/system_hooks.rb +++ b/lib/api/system_hooks.rb @@ -6,7 +6,7 @@ module API system_hooks_tags = %w[system_hooks] - feature_category :integrations + feature_category :webhooks before do authenticate! diff --git a/lib/api/topics.rb b/lib/api/topics.rb index b16b40244d4..c3a753a5cac 100644 --- a/lib/api/topics.rb +++ b/lib/api/topics.rb @@ -4,7 +4,7 @@ module API class Topics < ::API::Base include PaginationParams - feature_category :projects + feature_category :groups_and_projects desc 'Get topics' do detail 'This feature was introduced in GitLab 14.5.' diff --git a/lib/api/users.rb b/lib/api/users.rb index 3d9af536c3c..ff36a4cfe95 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -99,6 +99,7 @@ module API optional :exclude_internal, as: :non_internal, type: Boolean, default: false, desc: 'Filters only non internal users' optional :without_project_bots, type: Boolean, default: false, desc: 'Filters users without project bots' optional :admins, type: Boolean, default: false, desc: 'Filters only admin users' + optional :two_factor, type: String, desc: 'Filter users by Two-factor authentication.' all_or_none_of :extern_uid, :provider use :sort_params @@ -108,10 +109,12 @@ module API end # rubocop: disable CodeReuse/ActiveRecord get feature_category: :user_profile, urgency: :low do - authenticated_as_admin! if params[:extern_uid].present? && params[:provider].present? + index_params = declared_params(include_missing: false) + + authenticated_as_admin! if index_params[:extern_uid].present? && index_params[:provider].present? unless current_user&.can_read_all_resources? - params.except!(:created_after, :created_before, :order_by, :sort, :two_factor, :without_projects) + index_params.except!(:created_after, :created_before, :order_by, :sort, :two_factor, :without_projects) end authorized = can?(current_user, :read_users_list) @@ -121,11 +124,11 @@ module API # a list of all the users on the GitLab instance. `UsersFinder` performs # an exact match on the `username` parameter, so we are guaranteed to # get either 0 or 1 `users` here. - authorized &&= params[:username].present? if current_user.blank? + authorized &&= index_params[:username].present? if current_user.blank? forbidden!("Not authorized to access /api/v4/users") unless authorized - users = UsersFinder.new(current_user, params).execute + users = UsersFinder.new(current_user, index_params).execute users = reorder_users(users) entity = current_user&.can_read_all_resources? ? Entities::UserWithAdmin : Entities::UserBasic @@ -707,9 +710,13 @@ module API user = User.find_by(id: params[:id]) not_found!('User') unless user - forbidden!('A blocked user must be unblocked to be activated') if user.blocked? - user.activate + result = ::Users::ActivateService.new(current_user).execute(user) + if result[:status] == :success + true + else + render_api_error!(result[:message], result[:reason] || :bad_request) + end end desc 'Approve a pending user. Available only for admins.' @@ -1239,7 +1246,7 @@ module API params do optional :view_diffs_file_by_file, type: Boolean, desc: 'Flag indicating the user sees only one file diff per page' optional :show_whitespace_in_diffs, type: Boolean, desc: 'Flag indicating the user sees whitespace changes in diffs' - optional :pass_user_identities_to_ci_jwt, type: Boolean, desc: 'Flag indicating the user passes their external identities as CI information' + optional :pass_user_identities_to_ci_jwt, type: Boolean, desc: 'Flag indicating the user passes their external identities to a CI job as part of a JSON web token.' at_least_one_of :view_diffs_file_by_file, :show_whitespace_in_diffs, :pass_user_identities_to_ci_jwt end put "preferences", feature_category: :user_profile, urgency: :high do diff --git a/lib/api/v3/github.rb b/lib/api/v3/github.rb index 7d8c37cd39b..7348ed612fc 100644 --- a/lib/api/v3/github.rb +++ b/lib/api/v3/github.rb @@ -29,10 +29,9 @@ module API feature_category :integrations before do - reversible_end_of_life! - authorize_jira_user_agent!(request) authenticate! + reversible_end_of_life! end helpers do @@ -50,6 +49,13 @@ module API # TODO Make the breaking change irreversible https://gitlab.com/gitlab-org/gitlab/-/issues/408148. def reversible_end_of_life! not_found! unless Feature.enabled?(:jira_dvcs_end_of_life_amnesty) + + Gitlab::IntegrationsLogger.info( + user_id: current_user&.id, + namespace: params[:namespace], + project: params[:project], + message: 'Deprecated Jira DVCS endpoint request' + ) end def authorize_jira_user_agent!(request) diff --git a/lib/api/validations/validators/absence.rb b/lib/api/validations/validators/absence.rb index 7858ce7140b..49bf7f4a1c2 100644 --- a/lib/api/validations/validators/absence.rb +++ b/lib/api/validations/validators/absence.rb @@ -3,7 +3,7 @@ module API module Validations module Validators - class Absence < Grape::Validations::Base + class Absence < Grape::Validations::Validators::Base def validate_param!(attr_name, params) return if params.respond_to?(:key?) && !params.key?(attr_name) diff --git a/lib/api/validations/validators/array_none_any.rb b/lib/api/validations/validators/array_none_any.rb index 8c064eefbf2..cc86bd1a535 100644 --- a/lib/api/validations/validators/array_none_any.rb +++ b/lib/api/validations/validators/array_none_any.rb @@ -3,7 +3,7 @@ module API module Validations module Validators - class ArrayNoneAny < Grape::Validations::Base + class ArrayNoneAny < Grape::Validations::Validators::Base def validate_param!(attr_name, params) value = params[attr_name] diff --git a/lib/api/validations/validators/bulk_imports.rb b/lib/api/validations/validators/bulk_imports.rb index bff3424a0ac..f8ad5ed6d14 100644 --- a/lib/api/validations/validators/bulk_imports.rb +++ b/lib/api/validations/validators/bulk_imports.rb @@ -4,7 +4,7 @@ module API module Validations module Validators module BulkImports - class DestinationSlugPath < Grape::Validations::Base + class DestinationSlugPath < Grape::Validations::Validators::Base def validate_param!(attr_name, params) if Feature.disabled?(:restrict_special_characters_in_namespace_path) return if params[attr_name] =~ Gitlab::Regex.group_path_regex @@ -29,7 +29,7 @@ module API end end - class DestinationNamespacePath < Grape::Validations::Base + class DestinationNamespacePath < Grape::Validations::Validators::Base def validate_param!(attr_name, params) return if params[attr_name].blank? @@ -42,7 +42,7 @@ module API end end - class SourceFullPath < Grape::Validations::Base + class SourceFullPath < Grape::Validations::Validators::Base def validate_param!(attr_name, params) return if params[attr_name] =~ Gitlab::Regex.bulk_import_source_full_path_regex diff --git a/lib/api/validations/validators/check_assignees_count.rb b/lib/api/validations/validators/check_assignees_count.rb index 15f48c09a4f..7d90f030f1a 100644 --- a/lib/api/validations/validators/check_assignees_count.rb +++ b/lib/api/validations/validators/check_assignees_count.rb @@ -3,7 +3,7 @@ module API module Validations module Validators - class CheckAssigneesCount < Grape::Validations::Base + class CheckAssigneesCount < Grape::Validations::Validators::Base def self.coerce lambda do |value| case value diff --git a/lib/api/validations/validators/email_or_email_list.rb b/lib/api/validations/validators/email_or_email_list.rb index 715a29c613d..8aa7c3bc32c 100644 --- a/lib/api/validations/validators/email_or_email_list.rb +++ b/lib/api/validations/validators/email_or_email_list.rb @@ -3,7 +3,7 @@ module API module Validations module Validators - class EmailOrEmailList < Grape::Validations::Base + class EmailOrEmailList < Grape::Validations::Validators::Base def validate_param!(attr_name, params) value = params[attr_name] diff --git a/lib/api/validations/validators/file_path.rb b/lib/api/validations/validators/file_path.rb index 268ddc29d4e..7b32e1d71f8 100644 --- a/lib/api/validations/validators/file_path.rb +++ b/lib/api/validations/validators/file_path.rb @@ -3,12 +3,12 @@ module API module Validations module Validators - class FilePath < Grape::Validations::Base + class FilePath < Grape::Validations::Validators::Base def validate_param!(attr_name, params) options = @option.is_a?(Hash) ? @option : {} path_allowlist = options.fetch(:allowlist, []) path = params[attr_name] - Gitlab::Utils.check_allowed_absolute_path_and_path_traversal!(path, path_allowlist) + Gitlab::PathTraversal.check_allowed_absolute_path_and_path_traversal!(path, path_allowlist) rescue StandardError raise Grape::Exceptions::Validation.new( params: [@scope.full_name(attr_name)], diff --git a/lib/api/validations/validators/git_ref.rb b/lib/api/validations/validators/git_ref.rb index dcb1db6ca33..711c272ab4e 100644 --- a/lib/api/validations/validators/git_ref.rb +++ b/lib/api/validations/validators/git_ref.rb @@ -3,7 +3,7 @@ module API module Validations module Validators - class GitRef < Grape::Validations::Base + class GitRef < Grape::Validations::Validators::Base # There are few checks that a Git reference should pass through to be valid reference. # The link contains some rules that have been added to this validator. # https://mirrors.edge.kernel.org/pub/software/scm/git/docs/git-check-ref-format.html diff --git a/lib/api/validations/validators/git_sha.rb b/lib/api/validations/validators/git_sha.rb index 665d1878b4c..830137a0197 100644 --- a/lib/api/validations/validators/git_sha.rb +++ b/lib/api/validations/validators/git_sha.rb @@ -3,7 +3,7 @@ module API module Validations module Validators - class GitSha < Grape::Validations::Base + class GitSha < Grape::Validations::Validators::Base def validate_param!(attr_name, params) sha = params[attr_name] diff --git a/lib/api/validations/validators/integer_or_custom_value.rb b/lib/api/validations/validators/integer_or_custom_value.rb index d2352495948..ff2df858a36 100644 --- a/lib/api/validations/validators/integer_or_custom_value.rb +++ b/lib/api/validations/validators/integer_or_custom_value.rb @@ -3,7 +3,7 @@ module API module Validations module Validators - class IntegerOrCustomValue < Grape::Validations::Base + class IntegerOrCustomValue < Grape::Validations::Validators::Base def initialize(attrs, options, required, scope, **opts) @custom_values = extract_custom_values(options) super @@ -15,7 +15,7 @@ module API return if value.is_a?(Integer) return if @custom_values.map(&:downcase).include?(value.to_s.downcase) - valid_options = Gitlab::Utils.to_exclusive_sentence(['an integer'] + @custom_values) + valid_options = Gitlab::Sentence.to_exclusive_sentence(['an integer'] + @custom_values) raise Grape::Exceptions::Validation.new( params: [@scope.full_name(attr_name)], message: "should be #{valid_options}, however got #{value}" diff --git a/lib/api/validations/validators/limit.rb b/lib/api/validations/validators/limit.rb index 7e11f1d77cc..781bebac716 100644 --- a/lib/api/validations/validators/limit.rb +++ b/lib/api/validations/validators/limit.rb @@ -3,7 +3,7 @@ module API module Validations module Validators - class Limit < Grape::Validations::Base + class Limit < Grape::Validations::Validators::Base def validate_param!(attr_name, params) value = params[attr_name] diff --git a/lib/api/validations/validators/project_portable.rb b/lib/api/validations/validators/project_portable.rb index 3a7ea5ea71e..50ab32b6b7a 100644 --- a/lib/api/validations/validators/project_portable.rb +++ b/lib/api/validations/validators/project_portable.rb @@ -3,7 +3,7 @@ module API module Validations module Validators - class ProjectPortable < Grape::Validations::Base + class ProjectPortable < Grape::Validations::Validators::Base def validate_param!(attr_name, params) portable = params[attr_name] diff --git a/lib/api/validations/validators/untrusted_regexp.rb b/lib/api/validations/validators/untrusted_regexp.rb index 3ddea2bd9de..c5560be2e16 100644 --- a/lib/api/validations/validators/untrusted_regexp.rb +++ b/lib/api/validations/validators/untrusted_regexp.rb @@ -3,7 +3,7 @@ module API module Validations module Validators - class UntrustedRegexp < Grape::Validations::Base + class UntrustedRegexp < Grape::Validations::Validators::Base def validate_param!(attr_name, params) value = params[attr_name] return unless value diff --git a/lib/atlassian/jira_issue_key_extractor.rb b/lib/atlassian/jira_issue_key_extractor.rb index 881ba4544b2..17fa40e5676 100644 --- a/lib/atlassian/jira_issue_key_extractor.rb +++ b/lib/atlassian/jira_issue_key_extractor.rb @@ -12,7 +12,9 @@ module Atlassian end def issue_keys - @text.scan(@match_regex).flatten.uniq + return @text.scan(@match_regex).flatten.uniq if @match_regex.is_a?(Regexp) + + @match_regex.scan(@text).flatten.uniq end end end diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index b5e1634004a..d56f852b23c 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -12,7 +12,8 @@ module Backup LIST_ENVS = { skipped: 'SKIP', repositories_storages: 'REPOSITORIES_STORAGES', - repositories_paths: 'REPOSITORIES_PATHS' + repositories_paths: 'REPOSITORIES_PATHS', + skip_repositories_paths: 'SKIP_REPOSITORIES_PATHS' }.freeze YAML_PERMITTED_CLASSES = [ @@ -176,6 +177,11 @@ module Backup human_name: _('packages'), destination_path: 'packages.tar.gz', task: build_files_task(Settings.packages.storage_path, excludes: ['tmp']) + ), + 'ci_secure_files' => TaskDefinition.new( + human_name: _('ci secure files'), + destination_path: 'ci_secure_files.tar.gz', + task: build_files_task(Settings.ci_secure_files.storage_path, excludes: ['tmp']) ) }.freeze end @@ -194,7 +200,8 @@ module Backup Repositories.new(progress, strategy: strategy, storages: list_env(:repositories_storages), - paths: list_env(:repositories_paths) + paths: list_env(:repositories_paths), + skip_paths: list_env(:skip_repositories_paths) ) end @@ -278,7 +285,8 @@ module Backup installation_type: Gitlab::INSTALLATION_TYPE, skipped: ENV['SKIP'], repositories_storages: ENV['REPOSITORIES_STORAGES'], - repositories_paths: ENV['REPOSITORIES_PATHS'] + repositories_paths: ENV['REPOSITORIES_PATHS'], + skip_repositories_paths: ENV['SKIP_REPOSITORIES_PATHS'] } end @@ -292,7 +300,8 @@ module Backup installation_type: Gitlab::INSTALLATION_TYPE, skipped: list_env(:skipped).join(','), repositories_storages: list_env(:repositories_storages).join(','), - repositories_paths: list_env(:repositories_paths).join(',') + repositories_paths: list_env(:repositories_paths).join(','), + skip_repositories_paths: list_env(:skip_repositories_paths).join(',') ) end diff --git a/lib/backup/repositories.rb b/lib/backup/repositories.rb index 218df3fcb6c..56726665d14 100644 --- a/lib/backup/repositories.rb +++ b/lib/backup/repositories.rb @@ -10,13 +10,15 @@ module Backup # @param [IO] progress IO interface to output progress # @param [Object] :strategy Fetches backups from gitaly # @param [Array] :storages Filter by specified storage names. Empty means all storages. - # @param [Array] :paths Filter by specified project paths. Empty means all projects, groups and snippets. - def initialize(progress, strategy:, storages: [], paths: []) + # @param [Array] :paths Filter by specified project paths. Empty means all projects, groups, and snippets. + # @param [Array] :skip_paths Skip specified project paths. Empty means all projects, groups, and snippets. + def initialize(progress, strategy:, storages: [], paths: [], skip_paths: []) super(progress) @strategy = strategy @storages = storages @paths = paths + @skip_paths = skip_paths end override :dump @@ -42,7 +44,7 @@ module Backup private - attr_reader :strategy, :storages, :paths + attr_reader :strategy, :storages, :paths, :skip_paths def remove_all_repositories return if paths.present? @@ -84,6 +86,7 @@ module Backup ) end + scope = scope.and(skipped_path_relation) if skip_paths.any? scope end @@ -98,9 +101,20 @@ module Backup ) end + if skip_paths.any? + scope = scope.where(project: skipped_path_relation) + scope = scope.or(Snippet.where(project: nil)) if !paths.any? && !storages.any? + end + scope end + def skipped_path_relation + Project.where.not(id: Project.where_full_path_in(skip_paths).or( + Project.where(namespace_id: Namespace.where_full_path_in(skip_paths).self_and_descendants) + )) + end + def restore_object_pools PoolRepository.includes(:source_project).find_each do |pool| progress.puts " - Object pool #{pool.disk_path}..." diff --git a/lib/banzai/filter/autolink_filter.rb b/lib/banzai/filter/autolink_filter.rb index a86c1bb2892..336d60055e2 100644 --- a/lib/banzai/filter/autolink_filter.rb +++ b/lib/banzai/filter/autolink_filter.rb @@ -40,7 +40,7 @@ module Banzai IGNORE_PARENTS = %w(a code kbd pre script style).to_set # The XPath query to use for finding text nodes to parse. - TEXT_QUERY = %Q(descendant-or-self::text()[ + TEXT_QUERY = %(descendant-or-self::text()[ not(#{IGNORE_PARENTS.map { |p| "ancestor::#{p}" }.join(' or ')}) and contains(., '://') ]) diff --git a/lib/banzai/filter/dollar_math_post_filter.rb b/lib/banzai/filter/dollar_math_post_filter.rb index 76f69a66e8d..b3c230131e4 100644 --- a/lib/banzai/filter/dollar_math_post_filter.rb +++ b/lib/banzai/filter/dollar_math_post_filter.rb @@ -20,13 +20,13 @@ module Banzai DOLLAR_INLINE_UNTRUSTED = '(?P\$(?P(?:\S[^$\n]*?\S|[^$\s]))\$)(?:[^\d]|$)' DOLLAR_INLINE_UNTRUSTED_REGEX = - Gitlab::UntrustedRegexp.new(DOLLAR_INLINE_UNTRUSTED, multiline: false) + Gitlab::UntrustedRegexp.new(DOLLAR_INLINE_UNTRUSTED, multiline: false).freeze # Corresponds to the "$$...$$" syntax DOLLAR_DISPLAY_INLINE_UNTRUSTED = '(?P\$\$\ *(?P[^$\n]+?)\ *\$\$)' DOLLAR_DISPLAY_INLINE_UNTRUSTED_REGEX = - Gitlab::UntrustedRegexp.new(DOLLAR_DISPLAY_INLINE_UNTRUSTED, multiline: false) + Gitlab::UntrustedRegexp.new(DOLLAR_DISPLAY_INLINE_UNTRUSTED, multiline: false).freeze # Order dependent. Handle the `$$` syntax before the `$` syntax DOLLAR_MATH_PIPELINE = [ diff --git a/lib/banzai/filter/inline_alert_metrics_filter.rb b/lib/banzai/filter/inline_alert_metrics_filter.rb deleted file mode 100644 index a6140d1ac81..00000000000 --- a/lib/banzai/filter/inline_alert_metrics_filter.rb +++ /dev/null @@ -1,47 +0,0 @@ -# frozen_string_literal: true - -module Banzai - module Filter - # HTML filter that inserts a placeholder element for each - # reference to an alert dashboard. - class InlineAlertMetricsFilter < ::Banzai::Filter::InlineEmbedsFilter - include ::Gitlab::Routing - # Search params for selecting alert metrics links. A few - # simple checks is enough to boost performance without - # the cost of doing a full regex match. - def xpath_search - "descendant-or-self::a[contains(@href,'metrics_dashboard') and \ - contains(@href,'prometheus/alerts') and \ - starts-with(@href, '#{gitlab_domain}')]" - end - - # Regular expression matching alert dashboard urls - def link_pattern - ::Gitlab::Metrics::Dashboard::Url.alert_regex - end - - private - - # Endpoint FE should hit to collect the appropriate - # chart information - def metrics_dashboard_url(params) - metrics_dashboard_namespace_project_prometheus_alert_url( - params['namespace'], - params['project'], - params['alert'], - embedded: true, - format: :json, - **query_params(params['url']) - ) - end - - # Parses query params out from full url string into hash. - # - # Ex) 'https:////metrics_dashboard?title=Title&group=Group' - # --> { title: 'Title', group: 'Group' } - def query_params(url) - ::Gitlab::Metrics::Dashboard::Url.parse_query(url) - end - end - end -end diff --git a/lib/banzai/filter/inline_cluster_metrics_filter.rb b/lib/banzai/filter/inline_cluster_metrics_filter.rb deleted file mode 100644 index a696d3a6f9c..00000000000 --- a/lib/banzai/filter/inline_cluster_metrics_filter.rb +++ /dev/null @@ -1,40 +0,0 @@ -# frozen_string_literal: true - -module Banzai - module Filter - class InlineClusterMetricsFilter < ::Banzai::Filter::InlineEmbedsFilter - def embed_params(node) - url = node['href'] - @query_params = query_params(url) - return unless [:group, :title, :y_label].all? do |param| - @query_params.include?(param) - end - - link_pattern.match(url) { |m| m.named_captures }.symbolize_keys - end - - def xpath_search - "descendant-or-self::a[contains(@href,'clusters') and \ - starts-with(@href, '#{gitlab_domain}')]" - end - - def link_pattern - ::Gitlab::Metrics::Dashboard::Url.clusters_regex - end - - def metrics_dashboard_url(params) - ::Gitlab::Routing.url_helpers.metrics_dashboard_namespace_project_cluster_url( - params[:namespace], - params[:project], - params[:cluster_id], - # Only Project clusters are supported for now - # admin and group cluster types may be supported in the future - cluster_type: :project, - embedded: true, - format: :json, - **@query_params - ) - end - end - end -end diff --git a/lib/banzai/filter/inline_diff_filter.rb b/lib/banzai/filter/inline_diff_filter.rb index 2a43540934c..fc77984f135 100644 --- a/lib/banzai/filter/inline_diff_filter.rb +++ b/lib/banzai/filter/inline_diff_filter.rb @@ -8,11 +8,11 @@ module Banzai INLINE_DIFF_DELETION_UNTRUSTED = '(?:\[\-(.*?)\-\]|\{\-(.*?)\-\})' INLINE_DIFF_DELETION_UNTRUSTED_REGEX = - Gitlab::UntrustedRegexp.new(INLINE_DIFF_DELETION_UNTRUSTED, multiline: false) + Gitlab::UntrustedRegexp.new(INLINE_DIFF_DELETION_UNTRUSTED, multiline: false).freeze INLINE_DIFF_ADDITION_UNTRUSTED = '(?:\[\+(.*?)\+\]|\{\+(.*?)\+\})' INLINE_DIFF_ADDITION_UNTRUSTED_REGEX = - Gitlab::UntrustedRegexp.new(INLINE_DIFF_ADDITION_UNTRUSTED, multiline: false) + Gitlab::UntrustedRegexp.new(INLINE_DIFF_ADDITION_UNTRUSTED, multiline: false).freeze def call doc.xpath('descendant-or-self::text()').each do |node| diff --git a/lib/banzai/filter/inline_embeds_filter.rb b/lib/banzai/filter/inline_embeds_filter.rb deleted file mode 100644 index a16166123f8..00000000000 --- a/lib/banzai/filter/inline_embeds_filter.rb +++ /dev/null @@ -1,93 +0,0 @@ -# frozen_string_literal: true - -module Banzai - module Filter - # HTML filter that inserts a node for each occurrence of - # a given link format. To transform references to DB - # resources in place, prefer to inherit from AbstractReferenceFilter. - class InlineEmbedsFilter < HTML::Pipeline::Filter - # Find every relevant link, create a new node based on - # the link, and insert this node after any html content - # surrounding the link. - def call - return doc if Feature.enabled?(:remove_monitor_metrics) - - doc.xpath(xpath_search).each do |node| - next unless element = element_to_embed(node) - - # We want this to follow any surrounding content. For example, - # if a link is inline in a paragraph. - node.parent.children.last.add_next_sibling(element) - end - - doc - end - - # Child class must provide the metrics_dashboard_url. - # - # Return a Nokogiri::XML::Element to embed in the - # markdown which provides a url to the metric_dashboard endpoint where - # data can be requested through a prometheus proxy. InlineMetricsRedactorFilter - # is responsible for permissions to see this div (and relies on the class 'js-render-metrics' ). - def create_element(params) - doc.document.create_element( - 'div', - class: 'js-render-metrics', - 'data-dashboard-url': metrics_dashboard_url(params) - ) - end - - # Implement in child class unless overriding #embed_params - # - # Returns the regex pattern used to filter - # to only matching urls. - def link_pattern - end - - # Returns the xpath query string used to select nodes - # from the html document on which the embed is based. - # - # Override to select nodes other than links. - def xpath_search - 'descendant-or-self::a[@href]' - end - - # Creates a new element based on the parameters - # obtained from the target link - def element_to_embed(node) - return unless params = embed_params(node) - - create_element(params) - end - - # Returns a hash of named parameters based on the - # provided regex with string keys. - # - # Override to select nodes other than links. - def embed_params(node) - url = node['href'] - - link_pattern.match(url) { |m| m.named_captures } - end - - # Parses query params out from full url string into hash. - # - # Ex) 'https://///metrics?title=Title&group=Group' - # --> { title: 'Title', group: 'Group' } - def query_params(url) - Gitlab::Metrics::Dashboard::Url.parse_query(url) - end - - # Implement in child class. - # - # Provides a full url to request the relevant panels of metric data. - def metrics_dashboard_url - raise NotImplementedError - end - - def gitlab_domain - ::Gitlab.config.gitlab.url - end - end - end -end diff --git a/lib/banzai/filter/inline_grafana_metrics_filter.rb b/lib/banzai/filter/inline_grafana_metrics_filter.rb deleted file mode 100644 index 07bde9858e8..00000000000 --- a/lib/banzai/filter/inline_grafana_metrics_filter.rb +++ /dev/null @@ -1,79 +0,0 @@ -# frozen_string_literal: true - -module Banzai - module Filter - # HTML filter that inserts a placeholder element for each - # reference to a grafana dashboard. - class InlineGrafanaMetricsFilter < Banzai::Filter::InlineEmbedsFilter - # Placeholder element for the frontend to use as an - # injection point for charts. - def create_element(params) - begin_loading_dashboard(params[:url]) - - super - end - - # @return [Hash] with keys :grafana_url, :start, and :end - def embed_params(node) - query_params = Gitlab::Metrics::Dashboard::Url.parse_query(node['href']) - - time_window = Grafana::TimeWindow.new(query_params[:from], query_params[:to]) - url = url_with_window(node['href'], query_params, time_window.in_milliseconds) - - { grafana_url: url }.merge(time_window.formatted) - end - - # Selects any links with an href contains the configured - # grafana domain for the project - def xpath_search - return unless grafana_url.present? - - %(descendant-or-self::a[starts-with(@href, '#{grafana_url}')]) - end - - private - - def project - context[:project] - end - - def grafana_url - project&.grafana_integration&.grafana_url - end - - def metrics_dashboard_url(params) - Gitlab::Routing.url_helpers.project_grafana_api_metrics_dashboard_url( - project, - embedded: true, - **params - ) - end - - # If the provided url is missing time window parameters, - # this inserts the default window into the url, allowing - # the embed service to correctly format prometheus - # queries during embed processing. - # - # @param url [String] - # @param query_params [Hash] - # @param time_window_params [Hash] - # @return [String] - def url_with_window(url, query_params, time_window_params) - uri = URI(url) - uri.query = time_window_params.merge(query_params).to_query - - uri.to_s - end - - # Fetches a dashboard and caches the result for the - # FE to fetch quickly while rendering charts - def begin_loading_dashboard(url) - ::Gitlab::Metrics::Dashboard::Finder.find( - project, - embedded: true, - grafana_url: url - ) - end - end - end -end diff --git a/lib/banzai/filter/inline_metrics_filter.rb b/lib/banzai/filter/inline_metrics_filter.rb deleted file mode 100644 index 2872ad7b632..00000000000 --- a/lib/banzai/filter/inline_metrics_filter.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: true - -module Banzai - module Filter - # HTML filter that inserts a placeholder element for each - # reference to a metrics dashboard. - class InlineMetricsFilter < Banzai::Filter::InlineEmbedsFilter - # Search params for selecting metrics links. A few - # simple checks is enough to boost performance without - # the cost of doing a full regex match. - def xpath_search - "descendant-or-self::a[contains(@href,'metrics') and \ - starts-with(@href, '#{gitlab_domain}')]" - end - - # Regular expression matching metrics urls - def link_pattern - Gitlab::Metrics::Dashboard::Url.metrics_regex - end - - private - - # Endpoint FE should hit to collect the appropriate - # chart information - def metrics_dashboard_url(params) - Gitlab::Metrics::Dashboard::Url.build_dashboard_url( - params['namespace'], - params['project'], - params['environment'], - embedded: true, - **query_params(params['url']).except(:environment) - ) - end - end - end -end diff --git a/lib/banzai/filter/inline_metrics_redactor_filter.rb b/lib/banzai/filter/inline_metrics_redactor_filter.rb deleted file mode 100644 index b256815ae84..00000000000 --- a/lib/banzai/filter/inline_metrics_redactor_filter.rb +++ /dev/null @@ -1,154 +0,0 @@ -# frozen_string_literal: true - -module Banzai - module Filter - # HTML filter that removes embeded elements that the current user does - # not have permission to view. - class InlineMetricsRedactorFilter < HTML::Pipeline::Filter - include Gitlab::Utils::StrongMemoize - - METRICS_CSS_CLASS = '.js-render-metrics' - XPATH = Gitlab::Utils::Nokogiri.css_to_xpath(METRICS_CSS_CLASS).freeze - EMBED_LIMIT = 100 - - Route = Struct.new(:regex, :permission) - Embed = Struct.new(:project_path, :permission) - - # Finds all embeds based on the css class the FE - # uses to identify the embedded content, removing - # only unnecessary nodes. - def call - nodes.each do |node| - embed = embeds_by_node[node] - user_has_access = user_access_by_embed[embed] - - node.remove unless user_has_access - end - - doc - end - - private - - def user - context[:current_user] - end - - # Returns all nodes which the FE will identify as - # a metrics embed placeholder element - # - # Removes any nodes beyond the first 100 - # - # @return [Nokogiri::XML::NodeSet] - def nodes - strong_memoize(:nodes) do - nodes = doc.xpath(XPATH) - nodes.drop(EMBED_LIMIT).each(&:remove) - - nodes - end - end - - # Maps a node to key properties of an embed. - # Memoized so we only need to run the regex to get - # the project full path from the url once per node. - # - # @return [Hash] - def embeds_by_node - strong_memoize(:embeds_by_node) do - nodes.each_with_object({}) do |node, embeds| - embed = Embed.new - url = node.attribute('data-dashboard-url').to_s - - permissions_by_route.each do |route| - set_path_and_permission(embed, url, route.regex, route.permission) unless embed.permission - end - - embeds[node] = embed if embed.permission - end - end - end - - def permissions_by_route - [ - Route.new( - ::Gitlab::Metrics::Dashboard::Url.metrics_regex, - :read_environment - ), - Route.new( - ::Gitlab::Metrics::Dashboard::Url.grafana_regex, - :read_project - ), - Route.new( - ::Gitlab::Metrics::Dashboard::Url.clusters_regex, - :read_cluster - ), - Route.new( - ::Gitlab::Metrics::Dashboard::Url.alert_regex, - :read_prometheus_alerts - ) - ] - end - - # Attempts to determine the path and permission attributes - # of a url based on expected dashboard url formats and - # sets the attributes on an Embed object - # - # @param embed [Embed] - # @param url [String] - # @param regex [RegExp] - # @param permission [Symbol] - def set_path_and_permission(embed, url, regex, permission) - return unless path = regex.match(url) do |m| - "#{$~[:namespace]}/#{$~[:project]}" - end - - embed.project_path = path - embed.permission = permission - end - - # Returns a mapping representing whether the current user - # has permission to view the embed for the project. - # Determined in a batch - # - # @return [Hash] - def user_access_by_embed - strong_memoize(:user_access_by_embed) do - unique_embeds.each_with_object({}) do |embed, access| - project = projects_by_path[embed.project_path] - - access[embed] = Ability.allowed?(user, embed.permission, project) - end - end - end - - # Returns a unique list of embeds - # - # @return [Array] - def unique_embeds - embeds_by_node.values.uniq - end - - # Maps a project's full path to a Project object. - # Contains all of the Projects referenced in the - # metrics placeholder elements of the current document - # - # @return [Hash] - def projects_by_path - strong_memoize(:projects_by_path) do - Project.eager_load(:route, namespace: [:route]) - .where_full_path_in(unique_project_paths) - .index_by(&:full_path) - end - end - - # Returns a list of the full_paths of every project which - # has an embed in the doc - # - # @return [Array] - def unique_project_paths - embeds_by_node.values.map(&:project_path).uniq - end - end - end -end diff --git a/lib/banzai/filter/references/reference_filter.rb b/lib/banzai/filter/references/reference_filter.rb index 37734f6a45a..a687ae2882e 100644 --- a/lib/banzai/filter/references/reference_filter.rb +++ b/lib/banzai/filter/references/reference_filter.rb @@ -143,7 +143,7 @@ module Banzai attributes.delete(:original) if context[:no_original_data] attributes.map do |key, value| - %Q(data-#{key.to_s.dasherize}="#{escape_once(value)}") + %(data-#{key.to_s.dasherize}="#{escape_once(value)}") end .join(' ') .prepend(reference_type_attribute) @@ -251,7 +251,7 @@ module Banzai end def query - @query ||= %Q{descendant-or-self::text()[not(#{ignore_ancestor_query})] + @query ||= %{descendant-or-self::text()[not(#{ignore_ancestor_query})] | descendant-or-self::a[ not(contains(concat(" ", @class, " "), " gfm ")) and not(@href = "") ]} diff --git a/lib/banzai/filter/references/user_reference_filter.rb b/lib/banzai/filter/references/user_reference_filter.rb index 5983036a8e5..d6b6fdb7149 100644 --- a/lib/banzai/filter/references/user_reference_filter.rb +++ b/lib/banzai/filter/references/user_reference_filter.rb @@ -45,7 +45,7 @@ module Banzai # have `gfm` and `gfm-project_member` class names attached for styling. def object_link_filter(text, pattern, link_content: nil, link_reference: false) references_in(text, pattern) do |match, username| - if username == 'all' && !skip_project_check? + if Feature.disabled?(:disable_all_mention) && username == 'all' && !skip_project_check? link_to_all(link_content: link_content) else cached_call(:banzai_url_for_object, match, path: [User, username.downcase]) do diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index b119c2ffccf..c2cad237d6f 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -34,7 +34,7 @@ module Banzai # Allow section elements with data-footnotes attribute allowlist[:elements].push('section') allowlist[:attributes]['section'] = %w(data-footnotes) - allowlist[:attributes]['a'].push('data-footnote-ref', 'data-footnote-backref') + allowlist[:attributes]['a'].push('data-footnote-ref', 'data-footnote-backref', 'data-footnote-backref-idx') allowlist end diff --git a/lib/banzai/filter/spaced_link_filter.rb b/lib/banzai/filter/spaced_link_filter.rb index f8d03fd6e50..d370a585271 100644 --- a/lib/banzai/filter/spaced_link_filter.rb +++ b/lib/banzai/filter/spaced_link_filter.rb @@ -42,7 +42,7 @@ module Banzai IGNORE_PARENTS = %w(a code kbd pre script style).to_set # The XPath query to use for finding text nodes to parse. - TEXT_QUERY = %Q(descendant-or-self::text()[ + TEXT_QUERY = %(descendant-or-self::text()[ not(#{IGNORE_PARENTS.map { |p| "ancestor::#{p}" }.join(' or ')}) and contains(., ']\(') ]) diff --git a/lib/banzai/filter/table_of_contents_filter.rb b/lib/banzai/filter/table_of_contents_filter.rb index d76009d08e1..de3e978b320 100644 --- a/lib/banzai/filter/table_of_contents_filter.rb +++ b/lib/banzai/filter/table_of_contents_filter.rb @@ -55,7 +55,7 @@ module Banzai def anchor_tag(href) escaped_href = CGI.escape(href) # account for non-ASCII characters - %Q{} + %{} end def push_toc(children, root: false) @@ -69,7 +69,7 @@ module Banzai end def push_anchor(header_node) - result[:toc] << %Q{
  • #{header_node.text}} + result[:toc] << %{
  • #{header_node.text}} push_toc(header_node.children) result[:toc] << '
    • ' end diff --git a/lib/banzai/issuable_extractor.rb b/lib/banzai/issuable_extractor.rb index 6428f71eb8f..c0eb159d517 100644 --- a/lib/banzai/issuable_extractor.rb +++ b/lib/banzai/issuable_extractor.rb @@ -48,7 +48,7 @@ module Banzai end def query - %Q( + %( descendant-or-self::a[contains(concat(" ", @class, " "), " gfm ")] [#{reference_types.join(' or ')}] ) diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 53f938c044f..1d6269c704d 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -26,7 +26,6 @@ module Banzai Filter::AudioLinkFilter, Filter::ImageLazyLoadFilter, Filter::ImageLinkFilter, - *metrics_filters, Filter::TableOfContentsFilter, Filter::TableOfContentsTagFilter, Filter::AutolinkFilter, @@ -44,14 +43,6 @@ module Banzai ] end - def self.metrics_filters - [ - Filter::InlineMetricsFilter, - Filter::InlineGrafanaMetricsFilter, - Filter::InlineClusterMetricsFilter - ] - end - def self.reference_filters [ Filter::References::UserReferenceFilter, diff --git a/lib/banzai/pipeline/post_process_pipeline.rb b/lib/banzai/pipeline/post_process_pipeline.rb index f8035698b9b..86c8d98494d 100644 --- a/lib/banzai/pipeline/post_process_pipeline.rb +++ b/lib/banzai/pipeline/post_process_pipeline.rb @@ -15,7 +15,6 @@ module Banzai def self.internal_link_filters [ Filter::ReferenceRedactorFilter, - Filter::InlineMetricsRedactorFilter, # UploadLinkFilter must come before RepositoryLinkFilter to # prevent unnecessary Gitaly calls from being made. Filter::UploadLinkFilter, diff --git a/lib/bitbucket/representation/pull_request.rb b/lib/bitbucket/representation/pull_request.rb index 8d9de2dbc7d..75183b1df95 100644 --- a/lib/bitbucket/representation/pull_request.rb +++ b/lib/bitbucket/representation/pull_request.rb @@ -4,7 +4,7 @@ module Bitbucket module Representation class PullRequest < Representation::Base def author - raw.fetch('author', {}).fetch('nickname', nil) + raw.dig('author', 'nickname') end def description @@ -39,19 +39,19 @@ module Bitbucket end def source_branch_name - source_branch.fetch('branch', {}).fetch('name', nil) + source_branch.dig('branch', 'name') end def source_branch_sha - source_branch.fetch('commit', {}).fetch('hash', nil) + source_branch.dig('commit', 'hash') end def target_branch_name - target_branch.fetch('branch', {}).fetch('name', nil) + target_branch.dig('branch', 'name') end def target_branch_sha - target_branch.fetch('commit', {}).fetch('hash', nil) + target_branch.dig('commit', 'hash') end private diff --git a/lib/bulk_imports/clients/http.rb b/lib/bulk_imports/clients/http.rb index 616ab8754b4..c9ed75e663e 100644 --- a/lib/bulk_imports/clients/http.rb +++ b/lib/bulk_imports/clients/http.rb @@ -158,15 +158,13 @@ module BulkImports { timeout: SIDEKIQ_REQUEST_TIMEOUT } if Gitlab::Runtime.sidekiq? end + # @raise [BulkImports::NetworkError] when unsuccessful def with_error_handling response = yield return response if response.success? raise ::BulkImports::NetworkError.new("Unsuccessful response #{response.code} from #{response.request.path.path}. Body: #{response.parsed_response}", response: response) - - rescue Gitlab::HTTP::BlockedUrlError => e - raise e rescue *Gitlab::HTTP::HTTP_ERRORS => e raise ::BulkImports::NetworkError, e end diff --git a/lib/bulk_imports/common/pipelines/lfs_objects_pipeline.rb b/lib/bulk_imports/common/pipelines/lfs_objects_pipeline.rb index 2e6a29f4738..68bd64dc2ff 100644 --- a/lib/bulk_imports/common/pipelines/lfs_objects_pipeline.rb +++ b/lib/bulk_imports/common/pipelines/lfs_objects_pipeline.rb @@ -18,8 +18,8 @@ module BulkImports # rubocop: disable CodeReuse/ActiveRecord def load(_context, file_path) - Gitlab::Utils.check_path_traversal!(file_path) - Gitlab::Utils.check_allowed_absolute_path!(file_path, [Dir.tmpdir]) + Gitlab::PathTraversal.check_path_traversal!(file_path) + Gitlab::PathTraversal.check_allowed_absolute_path!(file_path, [Dir.tmpdir]) return if tar_filepath?(file_path) return if lfs_json_filepath?(file_path) diff --git a/lib/bulk_imports/common/pipelines/members_pipeline.rb b/lib/bulk_imports/common/pipelines/members_pipeline.rb index f35eb5ccf5e..548b191dc25 100644 --- a/lib/bulk_imports/common/pipelines/members_pipeline.rb +++ b/lib/bulk_imports/common/pipelines/members_pipeline.rb @@ -7,7 +7,7 @@ module BulkImports include Pipeline transformer Common::Transformers::ProhibitedAttributesTransformer - transformer BulkImports::Groups::Transformers::MemberAttributesTransformer + transformer Common::Transformers::MemberAttributesTransformer def extract(context) graphql_extractor.extract(context) diff --git a/lib/bulk_imports/common/pipelines/uploads_pipeline.rb b/lib/bulk_imports/common/pipelines/uploads_pipeline.rb index a1b338aeb9f..06132791ea6 100644 --- a/lib/bulk_imports/common/pipelines/uploads_pipeline.rb +++ b/lib/bulk_imports/common/pipelines/uploads_pipeline.rb @@ -22,7 +22,7 @@ module BulkImports def load(context, file_path) # Validate that the path is OK to load - Gitlab::Utils.check_allowed_absolute_path_and_path_traversal!(file_path, [Dir.tmpdir]) + Gitlab::PathTraversal.check_allowed_absolute_path_and_path_traversal!(file_path, [Dir.tmpdir]) return if File.directory?(file_path) return if File.lstat(file_path).symlink? diff --git a/lib/bulk_imports/common/transformers/member_attributes_transformer.rb b/lib/bulk_imports/common/transformers/member_attributes_transformer.rb new file mode 100644 index 00000000000..382e6a51a73 --- /dev/null +++ b/lib/bulk_imports/common/transformers/member_attributes_transformer.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module BulkImports + module Common + module Transformers + class MemberAttributesTransformer + def transform(context, data) + user = find_user(data&.dig('user', 'public_email')) + access_level = data&.dig('access_level', 'integer_value') + + return unless data + return unless user + return unless valid_access_level?(access_level) + + cache_source_user_id(data, user, context) + + { + user_id: user.id, + access_level: access_level, + created_at: data['created_at'], + updated_at: data['updated_at'], + expires_at: data['expires_at'], + created_by_id: context.current_user.id + } + end + + private + + def find_user(email) + return unless email + + User.find_by_any_email(email, confirmed: true) + end + + def valid_access_level?(access_level) + Gitlab::Access.options_with_owner.value?(access_level) + end + + def cache_source_user_id(data, user, context) + gid = data&.dig('user', 'user_gid') + + return unless gid + + source_user_id = GlobalID.parse(gid).model_id + + ::BulkImports::UsersMapper.new(context: context).cache_source_user_id(source_user_id, user.id) + end + end + end + end +end diff --git a/lib/bulk_imports/file_downloads/validations.rb b/lib/bulk_imports/file_downloads/validations.rb index ae94267a6e8..b852a50c888 100644 --- a/lib/bulk_imports/file_downloads/validations.rb +++ b/lib/bulk_imports/file_downloads/validations.rb @@ -22,7 +22,7 @@ module BulkImports private def validate_filepath - Gitlab::Utils.check_path_traversal!(filepath) + Gitlab::PathTraversal.check_path_traversal!(filepath) end def validate_content_type diff --git a/lib/bulk_imports/groups/transformers/member_attributes_transformer.rb b/lib/bulk_imports/groups/transformers/member_attributes_transformer.rb deleted file mode 100644 index da50a19ee62..00000000000 --- a/lib/bulk_imports/groups/transformers/member_attributes_transformer.rb +++ /dev/null @@ -1,51 +0,0 @@ -# frozen_string_literal: true - -module BulkImports - module Groups - module Transformers - class MemberAttributesTransformer - def transform(context, data) - user = find_user(data&.dig('user', 'public_email')) - access_level = data&.dig('access_level', 'integer_value') - - return unless data - return unless user - return unless valid_access_level?(access_level) - - cache_source_user_id(data, user, context) - - { - user_id: user.id, - access_level: access_level, - created_at: data['created_at'], - updated_at: data['updated_at'], - expires_at: data['expires_at'], - created_by_id: context.current_user.id - } - end - - private - - def find_user(email) - return unless email - - User.find_by_any_email(email, confirmed: true) - end - - def valid_access_level?(access_level) - Gitlab::Access.options_with_owner.value?(access_level) - end - - def cache_source_user_id(data, user, context) - gid = data&.dig('user', 'user_gid') - - return unless gid - - source_user_id = GlobalID.parse(gid).model_id - - ::BulkImports::UsersMapper.new(context: context).cache_source_user_id(source_user_id, user.id) - end - end - end - end -end diff --git a/lib/bulk_imports/projects/pipelines/design_bundle_pipeline.rb b/lib/bulk_imports/projects/pipelines/design_bundle_pipeline.rb index 2d5231b0541..373cd2bd75a 100644 --- a/lib/bulk_imports/projects/pipelines/design_bundle_pipeline.rb +++ b/lib/bulk_imports/projects/pipelines/design_bundle_pipeline.rb @@ -20,8 +20,8 @@ module BulkImports end def load(_context, bundle_path) - Gitlab::Utils.check_path_traversal!(bundle_path) - Gitlab::Utils.check_allowed_absolute_path!(bundle_path, [Dir.tmpdir]) + Gitlab::PathTraversal.check_path_traversal!(bundle_path) + Gitlab::PathTraversal.check_allowed_absolute_path!(bundle_path, [Dir.tmpdir]) return unless portable.lfs_enabled? return unless File.exist?(bundle_path) diff --git a/lib/bulk_imports/projects/pipelines/repository_bundle_pipeline.rb b/lib/bulk_imports/projects/pipelines/repository_bundle_pipeline.rb index 9a3c582642f..f19d8931f4a 100644 --- a/lib/bulk_imports/projects/pipelines/repository_bundle_pipeline.rb +++ b/lib/bulk_imports/projects/pipelines/repository_bundle_pipeline.rb @@ -21,8 +21,8 @@ module BulkImports end def load(_context, bundle_path) - Gitlab::Utils.check_path_traversal!(bundle_path) - Gitlab::Utils.check_allowed_absolute_path!(bundle_path, [Dir.tmpdir]) + Gitlab::PathTraversal.check_path_traversal!(bundle_path) + Gitlab::PathTraversal.check_allowed_absolute_path!(bundle_path, [Dir.tmpdir]) return unless File.exist?(bundle_path) return if File.directory?(bundle_path) diff --git a/lib/error_tracking/collector/payload_validator.rb b/lib/error_tracking/collector/payload_validator.rb deleted file mode 100644 index aae19a3635a..00000000000 --- a/lib/error_tracking/collector/payload_validator.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module ErrorTracking - module Collector - class PayloadValidator - PAYLOAD_SCHEMA_PATH = Rails.root.join('app', 'validators', 'json_schemas', 'error_tracking_event_payload.json').to_s - - def valid?(payload) - JSONSchemer.schema(Pathname.new(PAYLOAD_SCHEMA_PATH)).valid?(payload) - end - end - end -end diff --git a/lib/error_tracking/collector/sentry_auth_parser.rb b/lib/error_tracking/collector/sentry_auth_parser.rb deleted file mode 100644 index 4945b8f73e1..00000000000 --- a/lib/error_tracking/collector/sentry_auth_parser.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -module ErrorTracking - module Collector - class SentryAuthParser - def self.parse(request) - # Sentry client sends auth in X-Sentry-Auth header - # - # Example of content: - # "Sentry sentry_version=7, sentry_client=sentry-ruby/4.5.1, sentry_timestamp=1623923398, - # sentry_key=afadk312..., sentry_secret=123456asd32131..." - auth = request.headers['X-Sentry-Auth'] - - # Sentry DSN contains key and secret. - # The key is required while secret is optional. - # We are going to use only the key since secret is deprecated. - public_key = auth[/sentry_key=(\w+)/, 1] - - { - public_key: public_key - } - end - end - end -end diff --git a/lib/error_tracking/collector/sentry_request_parser.rb b/lib/error_tracking/collector/sentry_request_parser.rb deleted file mode 100644 index ae632ebd518..00000000000 --- a/lib/error_tracking/collector/sentry_request_parser.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -module ErrorTracking - module Collector - class SentryRequestParser - def self.parse(request) - body = request.body.read - - # Request body contains 3 json objects merged together in one StringIO. - # We need to separate and parse them into array of hash objects. - json_objects = [] - parser = Yajl::Parser.new - - parser.parse(body) do |json_object| - json_objects << json_object - end - - # The request contains 3 objects: sentry metadata, type data and event data. - # We need only last two. Type to decide what to do with the request. - # And event data as it contains all information about the exception. - _, type, event = json_objects - - { - request_type: type['type'], - event: event - } - end - end - end -end diff --git a/lib/error_tracking/stacktrace_builder.rb b/lib/error_tracking/stacktrace_builder.rb index 024587e8683..a2d7091a62a 100644 --- a/lib/error_tracking/stacktrace_builder.rb +++ b/lib/error_tracking/stacktrace_builder.rb @@ -19,6 +19,7 @@ module ErrorTracking 'lineNo' => entry['lineno'], 'context' => build_stacktrace_context(entry), 'filename' => entry['filename'], + 'abs_path' => entry['abs_path'], 'function' => entry['function'], 'colNo' => 0 # we don't support colNo yet. } diff --git a/lib/extracts_ref.rb b/lib/extracts_ref.rb index 5f73b474956..49ec564eb8d 100644 --- a/lib/extracts_ref.rb +++ b/lib/extracts_ref.rb @@ -7,6 +7,28 @@ module ExtractsRef InvalidPathError = Class.new(StandardError) BRANCH_REF_TYPE = 'heads' TAG_REF_TYPE = 'tags' + REF_TYPES = [BRANCH_REF_TYPE, TAG_REF_TYPE].freeze + + def self.ref_type(type) + return unless REF_TYPES.include?(type) + + type + end + + def self.qualify_ref(ref, type) + validated_type = ref_type(type) + return ref unless validated_type + + %(refs/#{validated_type}/#{ref}) + end + + def self.unqualify_ref(ref, type) + validated_type = ref_type(type) + return ref unless validated_type + + ref.sub(%r{^refs/#{validated_type}/}, '') + end + # Given a string containing both a Git tree-ish, such as a branch or tag, and # a filesystem path joined by forward slashes, attempts to separate the two. # @@ -60,7 +82,6 @@ module ExtractsRef # # If the :id parameter appears to be requesting a specific response format, # that will be handled as well. - # # rubocop:disable Gitlab/ModuleWithInstanceVariables def assign_ref_vars @id, @ref, @path = extract_ref_path @@ -70,7 +91,7 @@ module ExtractsRef return unless @ref.present? @commit = if ref_type - @fully_qualified_ref = %(refs/#{ref_type}/#{@ref}) + @fully_qualified_ref = ExtractsRef.qualify_ref(@ref, ref_type) @repo.commit(@fully_qualified_ref) else @repo.commit(@ref) @@ -90,9 +111,7 @@ module ExtractsRef end def ref_type - return unless params[:ref_type].present? - - params[:ref_type] == TAG_REF_TYPE ? TAG_REF_TYPE : BRANCH_REF_TYPE + ExtractsRef.ref_type(params[:ref_type]) end private @@ -156,6 +175,7 @@ module ExtractsRef raise NotImplementedError end + # deprecated in favor of ExtractsRef::RequestedRef def ambiguous_ref?(project, ref) return false unless ref return true if project.repository.ambiguous_ref?(ref) diff --git a/lib/feature/shared.rb b/lib/feature/shared.rb index 6af24451322..d801070ff1a 100644 --- a/lib/feature/shared.rb +++ b/lib/feature/shared.rb @@ -54,6 +54,17 @@ module Feature example: <<-EOS experiment(:my_experiment, project: project, actor: current_user) { ...variant code... } EOS + }, + worker: { + description: "Feature flags for controlling Sidekiq workers behavior (e.g. deferring jobs)", + optional: true, + rollout_issue: false, + ee_only: false, + default_enabled: false, + example: '<<-EOS + Feature.enabled?(:"defer_sidekiq_jobs:AuthorizedProjectsWorker", type: :worker, + default_enabled_if_undefined: false) + EOS' } }.freeze diff --git a/lib/generators/gitlab/analytics/internal_events_generator.rb b/lib/generators/gitlab/analytics/internal_events_generator.rb new file mode 100644 index 00000000000..a85cdd352d5 --- /dev/null +++ b/lib/generators/gitlab/analytics/internal_events_generator.rb @@ -0,0 +1,278 @@ +# frozen_string_literal: true + +require 'rails/generators' + +module Gitlab + module Analytics + class InternalEventsGenerator < Rails::Generators::Base + TIME_FRAME_DIRS = { + '7d' => 'counts_7d', + '28d' => 'counts_28d' + }.freeze + + TIME_FRAMES_DEFAULT = TIME_FRAME_DIRS.keys.tap do |time_frame_defaults| + time_frame_defaults.class_eval do + def to_s + join(", ") + end + end + end.freeze + + ALLOWED_TIERS = %w[free premium ultimate].dup.tap do |tiers_default| + tiers_default.class_eval do + def to_s + join(", ") + end + end + end.freeze + + NEGATIVE_ANSWERS = %w[no n].freeze + POSITIVE_ANSWERS = %w[yes y].freeze + TOP_LEVEL_DIR = 'config' + TOP_LEVEL_DIR_EE = 'ee' + DESCRIPTION_MIN_LENGTH = 50 + KNOWN_EVENTS_PATH = 'lib/gitlab/usage_data_counters/known_events/common.yml' + KNOWN_EVENTS_PATH_EE = 'ee/lib/ee/gitlab/usage_data_counters/known_events/common.yml' + + DESCRIPTION_INQUIRY = %( + Please describe in at least #{DESCRIPTION_MIN_LENGTH} characters + what %{entity} %{entity_type} represents, + consider mentioning: %{considerations}. + Your answer will be processed by a full-text search tool and help others find and reuse this %{entity_type}. + ).freeze + + source_root File.expand_path('../../../../generator_templates/gitlab_internal_events', __dir__) + + desc 'Generates metric definitions, event definition yml files and known events entries' + + class_option :skip_namespace, + hide: true + class_option :skip_collision_check, + hide: true + class_option :time_frames, + optional: true, + default: TIME_FRAMES_DEFAULT, + type: :array, + banner: TIME_FRAMES_DEFAULT, + desc: "Indicates the metrics time frames. Please select one or more from: #{TIME_FRAMES_DEFAULT}" + class_option :tiers, + optional: true, + default: ALLOWED_TIERS, + type: :array, + banner: ALLOWED_TIERS, + desc: "Indicates the metric's GitLab subscription tiers. Please select one or more from: #{ALLOWED_TIERS}" + class_option :group, + type: :string, + optional: false, + desc: 'Name of group that added this metric' + class_option :stage, + type: :string, + optional: false, + desc: 'Name of stage that added this metric' + class_option :section, + type: :string, + optional: false, + desc: 'Name of section that added this metric' + class_option :mr, + type: :string, + optional: false, + desc: 'Merge Request that adds this metric' + class_option :event, + type: :string, + optional: false, + desc: 'Name of the event that this metric counts' + class_option :unique_on, + type: :string, + optional: false, + desc: 'Name of the event property that this metric counts' + + def create_metric_file + validate! + + template "event_definition.yml", + event_file_path(event), + ask_description(event, "event", "what the event is supposed to track, where, and when") + + time_frames.each do |time_frame| + template "metric_definition.yml", + metric_file_path(time_frame), + key_path(time_frame), + time_frame, + ask_description( + key_path(time_frame), + "metric", + "events, and event attributes in the description" + ) + end + + # ToDo: Delete during https://gitlab.com/groups/gitlab-org/-/epics/9542 cleanup + append_file known_events_file_name, known_event_entry + end + + private + + def known_event_entry + <<~YML + - name: #{event} + YML + end + + def event_identifiers + return unless include_default_event_properties? + + "\n- project\n- user\n- namespace" + end + + def include_default_event_properties? + question = <<~DESC + By convention all events automatically include the following properties: + * environment: string, + * source: string (eg: ruby, javascript) + * user_id: number + * project_id: number + * namespace_id: number + * plan: string (eg: free, premium, ultimate) + Would you like to add default properties to the event? Y(es)/N(o) + DESC + + answer = Gitlab::TaskHelpers.prompt(question, POSITIVE_ANSWERS + NEGATIVE_ANSWERS) + POSITIVE_ANSWERS.include?(answer) + end + + def event_file_path(event) + path = File.join(TOP_LEVEL_DIR, 'events', "#{event}.yml") + path = File.join(TOP_LEVEL_DIR_EE, path) unless free? + path + end + + def event + options[:event] + end + + def ask_description(entity, type, considerations) + say("") + desc = ask(format(DESCRIPTION_INQUIRY, entity: entity, entity_type: type, considerations: considerations)) + + while desc.length < DESCRIPTION_MIN_LENGTH + error_msg = <<~ERROR + Provided description is too short: #{desc.length} of required #{DESCRIPTION_MIN_LENGTH} characters + ERROR + + say(set_color(error_msg, :red)) + + desc = ask("Please provide description that is #{DESCRIPTION_MIN_LENGTH} characters long.\n") + end + desc + end + + def distributions + dist = "\n" + dist += "- ce\n" if free? + + "#{dist}- ee" + end + + def tiers + "\n- #{options[:tiers].join("\n- ")}" + end + + def milestone + Gitlab::VERSION.match('(\d+\.\d+)').captures.first + end + + def class_name + 'RedisHLLMetric' + end + + def key_path(time_frame) + "count_distinct_#{options[:unique_on]}_from_#{event}_#{time_frame}" + end + + def metric_file_path(time_frame) + path = File.join(TOP_LEVEL_DIR, 'metrics', TIME_FRAME_DIRS[time_frame], "#{key_path(time_frame)}.yml") + path = File.join(TOP_LEVEL_DIR_EE, path) unless free? + path + end + + def known_events_file_name + (free? ? KNOWN_EVENTS_PATH : KNOWN_EVENTS_PATH_EE) + end + + def validate! + raise "Required file: #{known_events_file_name} does not exists." unless File.exist?(known_events_file_name) + raise "An event '#{event}' already exists" if event_exists? + + validate_tiers! + + %i[unique_on event mr section stage group].each do |option| + raise "The option: --#{option} is missing" unless options.key? option + end + + time_frames.each do |time_frame| + validate_time_frame!(time_frame) + validate_key_path!(time_frame) + end + end + + def validate_time_frame!(time_frame) + return if TIME_FRAME_DIRS.key?(time_frame) + + raise "Invalid time frame: #{time_frame}, allowed options are: #{TIME_FRAMES_DEFAULT}" + end + + def validate_tiers! + wrong_tiers = options[:tiers] - ALLOWED_TIERS + unless wrong_tiers.empty? + raise "Tiers option included not allowed values: #{wrong_tiers}. Only allowed values are: #{ALLOWED_TIERS}" + end + + return unless options[:tiers].empty? + + raise "At least one tier must be present. Please set --tiers option" + end + + def validate_key_path!(time_frame) + return unless metric_definition_exists?(time_frame) + + raise "Metric definition with key path '#{key_path(time_frame)}' already exists" + end + + def event_exists? + return true if ::Gitlab::UsageDataCounters::HLLRedisCounter.known_event?(event) + + existing_events_from_definitions.include?(event) + end + + def existing_events_from_definitions + events_glob_path = File.join(TOP_LEVEL_DIR, 'events', "*.yml") + ee_events_glob_path = File.join(TOP_LEVEL_DIR_EE, events_glob_path) + + [ee_events_glob_path, events_glob_path].flat_map do |glob_path| + Dir.glob(glob_path).map do |path| + YAML.safe_load(File.read(path))["action"] + end + end + end + + def free? + options[:tiers].include? "free" + end + + def time_frames + options[:time_frames] + end + + def directory + @directory ||= TIME_FRAME_DIRS.find { |d| d.match?(input_dir) } + end + + def metric_definitions + @definitions ||= Gitlab::Usage::MetricDefinition.definitions(skip_validation: true) + end + + def metric_definition_exists?(time_frame) + metric_definitions[key_path(time_frame)].present? + end + end + end +end diff --git a/lib/generators/gitlab/usage_metric/templates/instrumentation_class_spec.rb.template b/lib/generators/gitlab/usage_metric/templates/instrumentation_class_spec.rb.template index f8bd502ab77..915b91e43da 100644 --- a/lib/generators/gitlab/usage_metric/templates/instrumentation_class_spec.rb.template +++ b/lib/generators/gitlab/usage_metric/templates/instrumentation_class_spec.rb.template @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Usage::Metrics::Instrumentations::<%= class_name %>Metric do +RSpec.describe Gitlab::Usage::Metrics::Instrumentations::<%= class_name %>Metric, feature_category: :service_ping do let(:expected_value) { 1 } it_behaves_like 'a correct instrumented metric value', { time_frame: 'all', data_source: 'database' } diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb index bafda11170a..f1777e059ed 100644 --- a/lib/gitlab/access.rb +++ b/lib/gitlab/access.rb @@ -23,6 +23,7 @@ module Gitlab PROTECTION_DEV_CAN_PUSH = 1 PROTECTION_FULL = 2 PROTECTION_DEV_CAN_MERGE = 3 + PROTECTION_DEV_CAN_INITIAL_PUSH = 4 # Default project creation level NO_ONE_PROJECT_ACCESS = 0 @@ -95,6 +96,11 @@ module Gitlab label: s_('DefaultBranchProtection|Fully protected'), help_text: s_('DefaultBranchProtection|Developers cannot push new commits, but maintainers can. No one can force push.'), value: PROTECTION_FULL + }, + { + label: s_('DefaultBranchProtection|Fully protected after initial push'), + help_text: s_('DefaultBranchProtection|Developers can push the initial commit to a repository, but none afterward. Maintainers can always push. No one can force push.'), + value: PROTECTION_DEV_CAN_INITIAL_PUSH } ] end diff --git a/lib/gitlab/access/branch_protection.rb b/lib/gitlab/access/branch_protection.rb index 339a99eb068..6ac8de407b0 100644 --- a/lib/gitlab/access/branch_protection.rb +++ b/lib/gitlab/access/branch_protection.rb @@ -34,6 +34,10 @@ module Gitlab level == PROTECTION_DEV_CAN_PUSH end + def developer_can_initial_push? + level == PROTECTION_DEV_CAN_INITIAL_PUSH + end + def developer_can_merge? level == PROTECTION_DEV_CAN_MERGE end diff --git a/lib/gitlab/alert_management/payload/base.rb b/lib/gitlab/alert_management/payload/base.rb index 01dcb95eab5..5b136431ce7 100644 --- a/lib/gitlab/alert_management/payload/base.rb +++ b/lib/gitlab/alert_management/payload/base.rb @@ -181,7 +181,6 @@ module Gitlab end end - # Overriden in EE::Gitlab::AlertManagement::Payload::Generic def value_for_paths(paths) target_path = paths.find { |path| payload&.dig(*path) } diff --git a/lib/gitlab/alert_management/payload/prometheus.rb b/lib/gitlab/alert_management/payload/prometheus.rb index 4c36ebbf3aa..76f3da8366b 100644 --- a/lib/gitlab/alert_management/payload/prometheus.rb +++ b/lib/gitlab/alert_management/payload/prometheus.rb @@ -94,6 +94,10 @@ module Gitlab project && title && starts_at_raw end + def source + integration&.name || monitoring_tool + end + private override :severity_mapping @@ -131,3 +135,5 @@ module Gitlab end end end + +Gitlab::AlertManagement::Payload::Prometheus.prepend_mod diff --git a/lib/gitlab/analytics/date_filler.rb b/lib/gitlab/analytics/date_filler.rb index aa3db9f3635..33ebe269f26 100644 --- a/lib/gitlab/analytics/date_filler.rb +++ b/lib/gitlab/analytics/date_filler.rb @@ -32,7 +32,7 @@ module Gitlab # End date of the range # # **period** - # Specifies the period in wich the dates should be generated. Options: + # Specifies the period in which the dates should be generated. Options: # # - :day, generate date-value pair for each day in the given period # - :week, generate date-value pair for each week (beginning of the week date) diff --git a/lib/gitlab/api_authentication/token_locator.rb b/lib/gitlab/api_authentication/token_locator.rb index df342905d2e..5656ea0d120 100644 --- a/lib/gitlab/api_authentication/token_locator.rb +++ b/lib/gitlab/api_authentication/token_locator.rb @@ -8,22 +8,23 @@ module Gitlab include ActiveModel::Validations include ActionController::HttpAuthentication::Basic + VALID_LOCATIONS = %i[ + http_basic_auth + http_token + http_bearer_token + http_deploy_token_header + http_job_token_header + http_private_token_header + http_header + token_param + ].freeze + attr_reader :location - validates :location, inclusion: { - in: %i[ - http_basic_auth - http_token - http_bearer_token - http_deploy_token_header - http_job_token_header - http_private_token_header - token_param - ] - } + validates :location, inclusion: { in: VALID_LOCATIONS } def initialize(location) - @location = location + @location = extract_location(location) validate! end @@ -41,6 +42,8 @@ module Gitlab extract_from_http_job_token_header request when :http_private_token_header extract_from_http_private_token_header request + when :http_header + extract_from_http_header request when :token_param extract_from_token_param request end @@ -48,6 +51,16 @@ module Gitlab private + def extract_location(location) + case location + when Symbol + location + when Hash + result, @token_identifier = location.detect { |k, _v| VALID_LOCATIONS.include?(k) } + result + end + end + def extract_from_http_basic_auth(request) username, password = user_name_and_password(request) return unless username.present? && password.present? @@ -96,6 +109,13 @@ module Gitlab UsernameAndPassword.new(nil, password) end + + def extract_from_http_header(request) + password = request.headers[@token_identifier] + return unless password.present? + + UsernameAndPassword.new(nil, password) + end end end end diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index a8e74cbd7e6..8d7712951e1 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -45,7 +45,7 @@ module Gitlab auto_rollback_deployment: { threshold: 1, interval: 3.minutes }, search_rate_limit: { threshold: -> { application_settings.search_rate_limit }, interval: 1.minute }, search_rate_limit_unauthenticated: { threshold: -> { application_settings.search_rate_limit_unauthenticated }, interval: 1.minute }, - gitlab_shell_operation: { threshold: 600, interval: 1.minute }, + gitlab_shell_operation: { threshold: application_settings.gitlab_shell_operation_limit, interval: 1.minute }, pipelines_create: { threshold: -> { application_settings.pipeline_limit_per_project_user_sha }, interval: 1.minute }, temporary_email_failure: { threshold: 300, interval: 1.day }, permanent_email_failure: { threshold: 5, interval: 1.day }, diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index d55f2bc8ac9..955cb14594f 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -70,7 +70,8 @@ module Gitlab .merge({ # Define the Kroki server URL from the settings. # This attribute cannot be overridden from the AsciiDoc document. - 'kroki-server-url' => Gitlab::CurrentSettings.kroki_url + 'kroki-server-url' => Gitlab::CurrentSettings.kroki_url, + 'allow-uri-read' => Gitlab::CurrentSettings.wiki_asciidoc_allow_uri_includes }), extensions: extensions } diff --git a/lib/gitlab/asciidoc/include_processor.rb b/lib/gitlab/asciidoc/include_processor.rb index 6c4ecc04cdc..ae83dbedf04 100644 --- a/lib/gitlab/asciidoc/include_processor.rb +++ b/lib/gitlab/asciidoc/include_processor.rb @@ -9,6 +9,8 @@ module Gitlab class IncludeProcessor < Asciidoctor::IncludeExt::IncludeProcessor extend ::Gitlab::Utils::Override + NoData = Class.new(StandardError) + def initialize(context) super(logger: Gitlab::AppLogger) @@ -16,6 +18,7 @@ module Gitlab @repository = context[:repository] || context[:project].try(:repository) @max_includes = context[:max_includes].to_i @included = [] + @included_content = {} # Note: Asciidoctor calls #freeze on extensions, so we can't set new # instance variables after initialization. @@ -31,9 +34,10 @@ module Gitlab doc = reader.document max_include_depth = doc.attributes.fetch('max-include-depth').to_i + allow_uri_read = doc.attributes.fetch('allow-uri-read', false) return false if max_include_depth < 1 - return false if target_http?(target) + return false if target_http?(target) && !allow_uri_read return false if included.size >= max_includes true @@ -42,6 +46,7 @@ module Gitlab override :resolve_target_path def resolve_target_path(target, reader) return unless repository.try(:exists?) + return target if target_http?(target) base_path = reader.include_stack.empty? ? requested_path : reader.file path = resolve_relative_path(target, base_path) @@ -51,12 +56,15 @@ module Gitlab override :read_lines def read_lines(filename, selector) - blob = read_blob(ref, filename) + content = read_content(filename) + raise NoData, filename if content.nil? + + included << filename if selector - blob.data.each_line.select.with_index(1, &selector) + content.each_line.select.with_index(1, &selector) else - blob.data + content.lines end end @@ -67,7 +75,17 @@ module Gitlab private - attr_reader :context, :repository, :cache, :max_includes, :included + attr_reader :context, :repository, :cache, :max_includes, :included, :included_content + + def read_content(filename) + return included_content[filename] if included_content.key?(filename) + + included_content[filename] = if target_http?(filename) + read_uri(filename) + else + read_blob(ref, filename) + end + end # Gets a Blob at a path for a specific revision. # This method will check that the Blob exists and contains readable text. @@ -75,16 +93,22 @@ module Gitlab # revision - The String SHA1. # path - The String file path. # - # Returns a Blob + # Returns a string containing the blob content def read_blob(ref, filename) blob = repository&.blob_at(ref, filename) - raise 'Blob not found' unless blob - raise 'File is not readable' unless blob.readable_text? + raise NoData, 'Blob not found' unless blob + raise NoData, 'File is not readable' unless blob.readable_text? - included << filename + blob.data + end + + def read_uri(uri) + r = Gitlab::HTTP.get(uri) + + raise NoData, uri unless r.success? - blob + r.body end # Resolves the given relative path of file in repository into canonical diff --git a/lib/gitlab/audit/auditor.rb b/lib/gitlab/audit/auditor.rb index e3d2b394404..a59237fbb1f 100644 --- a/lib/gitlab/audit/auditor.rb +++ b/lib/gitlab/audit/auditor.rb @@ -77,12 +77,12 @@ module Gitlab @authentication_provider = @context[:authentication_provider] # TODO: Remove this code once we close https://gitlab.com/gitlab-org/gitlab/-/issues/367870 - return unless @is_audit_event_yaml_defined + return if @is_audit_event_yaml_defined - # rubocop:disable Gitlab/RailsLogger - Rails.logger.warn('WARNING: Logging audit events without an event type definition will be deprecated soon.') - Rails.logger.warn('See https://docs.gitlab.com/ee/development/audit_event_guide/#event-type-definitions') - # rubocop:enable Gitlab/RailsLogger + message = 'Logging audit events without an event type definition will be deprecated soon ' \ + '(https://docs.gitlab.com/ee/development/audit_event_guide/#event-type-definitions)' + + Gitlab::AppLogger.warn(message: message, event_type: @name) end def single_audit diff --git a/lib/gitlab/audit/type/definition.rb b/lib/gitlab/audit/type/definition.rb index 81c88a3a0ae..772023616b8 100644 --- a/lib/gitlab/audit/type/definition.rb +++ b/lib/gitlab/audit/type/definition.rb @@ -13,6 +13,10 @@ module Gitlab validate :validate_schema validate :validate_file_name + def self.declarative_policy_class + 'AuditEvents::DefinitionPolicy' + end + InvalidAuditEventTypeError = Class.new(StandardError) AUDIT_EVENT_TYPE_SCHEMA_PATH = Rails.root.join('config', 'audit_events', 'types', 'type_schema.json') @@ -78,6 +82,12 @@ module Gitlab definitions.keys.map(&:to_s) end + def names_with_category + definitions.map do |event_name, value| + { event_name: event_name, feature_category: value.attributes[:feature_category] } + end + end + def defined?(key) get(key).present? end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 9268fdd8519..83d94d168a0 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -401,7 +401,7 @@ module Gitlab scopes = non_admin_available_scopes if resource.admin? # rubocop: disable Cop/UserAdmin - scopes += Feature.enabled?(:admin_mode_for_api) ? ADMIN_SCOPES : [SUDO_SCOPE] + scopes += ADMIN_SCOPES end scopes diff --git a/lib/gitlab/auth/o_auth/auth_hash.rb b/lib/gitlab/auth/o_auth/auth_hash.rb index d1eede65f0c..cce08750296 100644 --- a/lib/gitlab/auth/o_auth/auth_hash.rb +++ b/lib/gitlab/auth/o_auth/auth_hash.rb @@ -78,7 +78,7 @@ module Gitlab def get_from_auth_hash_or_info(key) if auth_hash.key?(key) coerce_utf8(auth_hash[key]) - elsif auth_hash.key?(:extra) && auth_hash.extra.key?(:raw_info) && !auth_hash.extra.raw_info[key].nil? + elsif auth_hash.key?(:extra) && auth_hash.extra.key?(:raw_info) && !auth_hash.extra.raw_info[key].blank? coerce_utf8(auth_hash.extra.raw_info[key]) else get_info(key) diff --git a/lib/gitlab/auth/saml/auth_hash.rb b/lib/gitlab/auth/saml/auth_hash.rb index a2b0dfd5c66..592d88264e9 100644 --- a/lib/gitlab/auth/saml/auth_hash.rb +++ b/lib/gitlab/auth/saml/auth_hash.rb @@ -5,7 +5,7 @@ module Gitlab module Saml class AuthHash < Gitlab::Auth::OAuth::AuthHash def groups - Array.wrap(get_raw(Gitlab::Auth::Saml::Config.groups)) + Array.wrap(get_raw(Gitlab::Auth::Saml::Config.new(auth_hash.provider).groups)) end def authn_context diff --git a/lib/gitlab/auth/saml/config.rb b/lib/gitlab/auth/saml/config.rb index 815130aeee2..7524d8b9f85 100644 --- a/lib/gitlab/auth/saml/config.rb +++ b/lib/gitlab/auth/saml/config.rb @@ -8,26 +8,32 @@ module Gitlab def enabled? ::AuthHelper.saml_providers.any? end + end - def options - Gitlab::Auth::OAuth::Provider.config_for('saml') - end + DEFAULT_PROVIDER_NAME = 'saml' - def upstream_two_factor_authn_contexts - options.args[:upstream_two_factor_authn_contexts] - end + def initialize(provider = DEFAULT_PROVIDER_NAME) + @provider = provider + end - def groups - options[:groups_attribute] - end + def options + Gitlab::Auth::OAuth::Provider.config_for(@provider) + end - def external_groups - options[:external_groups] - end + def upstream_two_factor_authn_contexts + options.args[:upstream_two_factor_authn_contexts] + end - def admin_groups - options[:admin_groups] - end + def groups + options[:groups_attribute] + end + + def external_groups + options[:external_groups] + end + + def admin_groups + options[:admin_groups] end end end diff --git a/lib/gitlab/auth/saml/user.rb b/lib/gitlab/auth/saml/user.rb index d14da41deb6..6f72f185c8d 100644 --- a/lib/gitlab/auth/saml/user.rb +++ b/lib/gitlab/auth/saml/user.rb @@ -43,7 +43,7 @@ module Gitlab protected def saml_config - Gitlab::Auth::Saml::Config + Gitlab::Auth::Saml::Config.new(auth_hash.provider) end def auto_link_saml_user? diff --git a/lib/gitlab/authorized_keys.rb b/lib/gitlab/authorized_keys.rb index e7eba65bea8..3e529a0d2f3 100644 --- a/lib/gitlab/authorized_keys.rb +++ b/lib/gitlab/authorized_keys.rb @@ -149,7 +149,7 @@ module Gitlab raise KeyError, "Invalid public_key: #{key.inspect}" end - %Q(command="#{command(id)}",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty #{strip(key)}) + %(command="#{command(id)}",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty #{strip(key)}) end def command(id) diff --git a/lib/gitlab/avatar_cache.rb b/lib/gitlab/avatar_cache.rb index ed00a279299..f4dcd6f7910 100644 --- a/lib/gitlab/avatar_cache.rb +++ b/lib/gitlab/avatar_cache.rb @@ -7,7 +7,7 @@ module Gitlab # immediate cache expiry of all avatar caches. # # @return [Integer] - VERSION = 1 + VERSION = 2 # @return [Symbol] BASE_KEY = :avatar_cache @@ -65,10 +65,8 @@ module Gitlab keys = emails.map { |email| email_key(email) } Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - if ::Feature.enabled?(:use_pipeline_over_multikey) - redis.pipelined do |pipeline| - keys.each { |key| pipeline.unlink(key) } - end.sum + if Gitlab::Redis::ClusterUtil.cluster?(redis) + Gitlab::Redis::ClusterUtil.batch_unlink(keys, redis) else redis.unlink(*keys) end diff --git a/lib/gitlab/background_migration/backfill_ci_queuing_tables.rb b/lib/gitlab/background_migration/backfill_ci_queuing_tables.rb deleted file mode 100644 index 63112b52584..00000000000 --- a/lib/gitlab/background_migration/backfill_ci_queuing_tables.rb +++ /dev/null @@ -1,153 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Ensure queuing entries are present even if admins skip upgrades. - class BackfillCiQueuingTables - class Namespace < ActiveRecord::Base # rubocop:disable Style/Documentation - self.table_name = 'namespaces' - self.inheritance_column = :_type_disabled - end - - class Project < ActiveRecord::Base # rubocop:disable Style/Documentation - self.table_name = 'projects' - - belongs_to :namespace - has_one :ci_cd_settings, class_name: 'Gitlab::BackgroundMigration::BackfillCiQueuingTables::ProjectCiCdSetting' - - def group_runners_enabled? - return false unless ci_cd_settings - - ci_cd_settings.group_runners_enabled? - end - end - - class ProjectCiCdSetting < ActiveRecord::Base # rubocop:disable Style/Documentation - self.table_name = 'project_ci_cd_settings' - end - - class Taggings < ActiveRecord::Base # rubocop:disable Style/Documentation - self.table_name = 'taggings' - end - - module Ci - class Build < ActiveRecord::Base # rubocop:disable Style/Documentation - include EachBatch - - self.table_name = 'ci_builds' - self.inheritance_column = :_type_disabled - - belongs_to :project - - scope :pending, -> do - where(status: :pending, type: 'Ci::Build', runner_id: nil) - end - - def self.each_batch(of: 1000, column: :id, order: { runner_id: :asc, id: :asc }, order_hint: nil) - start = except(:select).select(column).reorder(order) - start = start.take - return unless start - - start_id = start[column] - arel_table = self.arel_table - - 1.step do |index| - start_cond = arel_table[column].gteq(start_id) - stop = except(:select).select(column).where(start_cond).reorder(order) - stop = stop.offset(of).limit(1).take - relation = where(start_cond) - - if stop - stop_id = stop[column] - start_id = stop_id - stop_cond = arel_table[column].lt(stop_id) - relation = relation.where(stop_cond) - end - - # Any ORDER BYs are useless for this relation and can lead to less - # efficient UPDATE queries, hence we get rid of it. - relation = relation.except(:order) - - # Using unscoped is necessary to prevent leaking the current scope used by - # ActiveRecord to chain `each_batch` method. - unscoped { yield relation, index } - - break unless stop - end - end - - def tags_ids - BackfillCiQueuingTables::Taggings - .where(taggable_id: id, taggable_type: 'CommitStatus') - .pluck(:tag_id) - end - end - - class PendingBuild < ActiveRecord::Base # rubocop:disable Style/Documentation - self.table_name = 'ci_pending_builds' - - class << self - def upsert_from_build!(build) - entry = self.new(args_from_build(build)) - - self.upsert( - entry.attributes.compact, - returning: %w[build_id], - unique_by: :build_id) - end - - def args_from_build(build) - project = build.project - - { - build_id: build.id, - project_id: build.project_id, - protected: build.protected?, - namespace_id: project.namespace_id, - tag_ids: build.tags_ids, - instance_runners_enabled: project.shared_runners_enabled?, - namespace_traversal_ids: namespace_traversal_ids(project) - } - end - - def namespace_traversal_ids(project) - if project.group_runners_enabled? - project.namespace.traversal_ids - else - [] - end - end - end - end - end - - BATCH_SIZE = 100 - - def perform(start_id, end_id) - scope = BackfillCiQueuingTables::Ci::Build.pending.where(id: start_id..end_id) - pending_builds_query = BackfillCiQueuingTables::Ci::PendingBuild - .where('ci_builds.id = ci_pending_builds.build_id') - .select(1) - - scope.each_batch(of: BATCH_SIZE) do |builds| - builds = builds.where('NOT EXISTS (?)', pending_builds_query) - builds = builds.includes(:project, project: [:namespace, :ci_cd_settings]) - - builds.each do |build| - BackfillCiQueuingTables::Ci::PendingBuild.upsert_from_build!(build) - end - end - - mark_job_as_succeeded(start_id, end_id) - end - - private - - def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - self.class.name.demodulize, - arguments) - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_code_suggestions_namespace_settings.rb b/lib/gitlab/background_migration/backfill_code_suggestions_namespace_settings.rb new file mode 100644 index 00000000000..2d3bb4bbafa --- /dev/null +++ b/lib/gitlab/background_migration/backfill_code_suggestions_namespace_settings.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class sets default `code_suggestions` values on the namespace_settings table. + # For group namespace, set this to enabled. + # For user namespace, set this to disabled. + class BackfillCodeSuggestionsNamespaceSettings < BatchedMigrationJob + feature_category :code_suggestions + operation_name :update_all + + TYPE_VALUE_PAIRS = [ + { type: 'Group', value: true }, + { type: 'User', value: false } + ].freeze + + NAMESPACES_JOIN = <<~SQL + INNER JOIN namespaces + ON namespaces.id = namespace_settings.namespace_id + SQL + + def perform + TYPE_VALUE_PAIRS.each do |pair| + each_sub_batch do |sub_batch| + sub_batch.joins(NAMESPACES_JOIN) + .where(namespaces: { type: pair[:type] }) + .update_all(code_suggestions: pair[:value]) + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_group_features.rb b/lib/gitlab/background_migration/backfill_group_features.rb deleted file mode 100644 index c45dcad5b2d..00000000000 --- a/lib/gitlab/background_migration/backfill_group_features.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Backfill group_features for an array of groups - class BackfillGroupFeatures < ::Gitlab::BackgroundMigration::BatchedMigrationJob - job_arguments :batch_size - operation_name :upsert_group_features - feature_category :database - - def perform - each_sub_batch( - batching_arguments: { order_hint: :type }, - batching_scope: ->(relation) { relation.where(type: 'Group') } - ) do |sub_batch| - upsert_group_features(sub_batch) - end - end - - private - - def upsert_group_features(relation) - connection.execute( - <<~SQL - INSERT INTO group_features (group_id, created_at, updated_at) - SELECT namespaces.id as group_id, now(), now() - FROM namespaces - WHERE namespaces.type = 'Group' AND namespaces.id IN(#{relation.select(:id).limit(batch_size).to_sql}) - ON CONFLICT (group_id) DO NOTHING; - SQL - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route.rb b/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route.rb deleted file mode 100644 index 0585924cb7b..00000000000 --- a/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Backfills the `routes.namespace_id` column, by copying source_id value - # (for groups and user namespaces source_id == namespace_id) - class BackfillNamespaceIdForNamespaceRoute - include Gitlab::Database::DynamicModelHelpers - - def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms) - parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id) - - parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| - batch_metrics.time_operation(:update_all) do - sub_batch.update_all('namespace_id=source_id') - end - - pause_ms = [0, pause_ms].max - sleep(pause_ms * 0.001) - end - end - - def batch_metrics - @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new - end - - private - - def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) - define_batchable_model(source_table, connection: ApplicationRecord.connection) - .joins('inner join namespaces on routes.source_id = namespaces.id') - .where(source_key_column => start_id..stop_id) - .where(namespace_id: nil) - .where(source_type: 'Namespace') - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_resource_link_events.rb b/lib/gitlab/background_migration/backfill_resource_link_events.rb new file mode 100644 index 00000000000..a2499e90e1f --- /dev/null +++ b/lib/gitlab/background_migration/backfill_resource_link_events.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfills resource_link_events from system_note_metadata and notes records + class BackfillResourceLinkEvents < BatchedMigrationJob + operation_name :backfill_resource_link_events + feature_category :team_planning + + # AR model for resource_link_events inlined + class ResourceLinkEvent < ApplicationRecord + self.table_name = 'resource_link_events' + + enum action: { + add: 1, + remove: 2 + } + end + + scope_to ->(relation) { relation.where("action='relate_to_parent' OR action='unrelate_from_parent'") } + + def perform + each_sub_batch do |sub_batch| + values_subquery = resource_link_event_values_query(sub_batch.select(:id).to_sql) + + connection.execute(<<~SQL) + INSERT INTO resource_link_events (action, issue_id, child_work_item_id, user_id, created_at, system_note_metadata_id) + #{values_subquery} + ON CONFLICT (system_note_metadata_id) DO NOTHING; + SQL + end + end + + def resource_link_event_values_query(ids_subquery) + <<~SQL + SELECT + CASE WHEN system_note_metadata.action='relate_to_parent' THEN #{ResourceLinkEvent.actions[:add]} + ELSE #{ResourceLinkEvent.actions[:remove]} + END AS action, + parent_issues.id AS issue_id, + notes.noteable_id AS child_work_item_id, + notes.author_id AS user_id, + system_note_metadata.created_at AS created_at, + system_note_metadata.id AS system_note_metadata_id + FROM system_note_metadata + INNER JOIN notes ON system_note_metadata.note_id = notes.id + INNER JOIN issues as work_items ON work_items.id = notes.noteable_id, + LATERAL ( + -- This lateral join searches for the id of the parent issue. + -- + -- When a child work item is added to its parent, + -- "relate_to_parent" is recorded as `system_note_metadata.action` + -- and a note records to which parent the child work item is added e.g, "added #1 (iid) as parent". + -- + -- Based on the iid of the parent extracted from the note and using the child work item's project id, + -- we can find out the id of the parent issue. + SELECT issues.id + FROM issues + WHERE + issues.project_id = work_items.project_id + AND issues.iid = CASE WHEN system_note_metadata.action='relate_to_parent' THEN substring(notes.note from 'added #(\\d+) as parent')::bigint + ELSE substring(notes.note from 'removed parent \\S+ #(\\d+)')::bigint + END + ) parent_issues + WHERE + system_note_metadata.id IN (#{ids_subquery}) + SQL + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_root_storage_statistics_fork_storage_sizes.rb b/lib/gitlab/background_migration/backfill_root_storage_statistics_fork_storage_sizes.rb new file mode 100644 index 00000000000..23c510720c0 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_root_storage_statistics_fork_storage_sizes.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfill the following columns on the namespace_root_storage_statistics table: + # - public_forks_storage_size + # - internal_forks_storage_size + # - private_forks_storage_size + class BackfillRootStorageStatisticsForkStorageSizes < BatchedMigrationJob + operation_name :backfill_root_storage_statistics_fork_sizes + feature_category :consumables_cost_management + + VISIBILITY_LEVELS_TO_STORAGE_SIZE_COLUMNS = { + 0 => :private_forks_storage_size, + 10 => :internal_forks_storage_size, + 20 => :public_forks_storage_size + }.freeze + + def perform + each_sub_batch do |sub_batch| + sub_batch.each do |root_storage_statistics| + next if has_fork_data?(root_storage_statistics) + + namespace_id = root_storage_statistics.namespace_id + + namespace_type = execute("SELECT type FROM namespaces WHERE id = #{namespace_id}").first&.fetch('type') + + next if namespace_type.nil? + + sql = if user_namespace?(namespace_type) + user_namespace_sql(namespace_id) + else + group_namespace_sql(namespace_id) + end + + stats = execute(sql) + .map { |h| { h['projects_visibility_level'] => h['sum_project_statistics_storage_size'] } } + .reduce({}) { |memo, h| memo.merge(h) } + .transform_keys { |k| VISIBILITY_LEVELS_TO_STORAGE_SIZE_COLUMNS[k] } + + root_storage_statistics.update!(stats) + end + end + end + + def has_fork_data?(root_storage_statistics) + root_storage_statistics.public_forks_storage_size != 0 || + root_storage_statistics.internal_forks_storage_size != 0 || + root_storage_statistics.private_forks_storage_size != 0 + end + + def user_namespace?(type) + type.nil? || type == 'User' || !(type == 'Group' || type == 'Project') + end + + def execute(sql) + ::ApplicationRecord.connection.execute(sql) + end + + def user_namespace_sql(namespace_id) + <<~SQL + SELECT + SUM("project_statistics"."storage_size") AS sum_project_statistics_storage_size, + "projects"."visibility_level" AS projects_visibility_level + FROM + "projects" + INNER JOIN "project_statistics" ON "project_statistics"."project_id" = "projects"."id" + INNER JOIN "fork_network_members" ON "fork_network_members"."project_id" = "projects"."id" + INNER JOIN "fork_networks" ON "fork_networks"."id" = "fork_network_members"."fork_network_id" + WHERE + "projects"."namespace_id" = #{namespace_id} + AND (fork_networks.root_project_id != projects.id) + GROUP BY "projects"."visibility_level" + SQL + end + + def group_namespace_sql(namespace_id) + <<~SQL + SELECT + SUM("project_statistics"."storage_size") AS sum_project_statistics_storage_size, + "projects"."visibility_level" AS projects_visibility_level + FROM + "projects" + INNER JOIN "project_statistics" ON "project_statistics"."project_id" = "projects"."id" + INNER JOIN "fork_network_members" ON "fork_network_members"."project_id" = "projects"."id" + INNER JOIN "fork_networks" ON "fork_networks"."id" = "fork_network_members"."fork_network_id" + WHERE + "projects"."namespace_id" IN ( + SELECT namespaces.traversal_ids[array_length(namespaces.traversal_ids, 1)] AS id + FROM "namespaces" + WHERE "namespaces"."type" = 'Group' AND (traversal_ids @> ('{#{namespace_id}}')) + ) + AND (fork_networks.root_project_id != projects.id) + GROUP BY "projects"."visibility_level" + SQL + end + end + end +end diff --git a/lib/gitlab/background_migration/cleanup_draft_data_from_faulty_regex.rb b/lib/gitlab/background_migration/cleanup_draft_data_from_faulty_regex.rb deleted file mode 100644 index b703faf6a6c..00000000000 --- a/lib/gitlab/background_migration/cleanup_draft_data_from_faulty_regex.rb +++ /dev/null @@ -1,48 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Cleanup draft column data inserted by a faulty regex - # - class CleanupDraftDataFromFaultyRegex - # Migration only version of MergeRequest table - ## - class MergeRequest < ActiveRecord::Base - LEAKY_REGEXP_STR = "^\\[draft\\]|\\(draft\\)|draft:|draft|\\[WIP\\]|WIP:|WIP" - CORRECTED_REGEXP_STR = "^(\\[draft\\]|\\(draft\\)|draft:|draft|\\[WIP\\]|WIP:|WIP)" - - include EachBatch - - self.table_name = 'merge_requests' - - def self.eligible - where(state_id: 1) - .where(draft: true) - .where("title ~* ?", LEAKY_REGEXP_STR) - .where("title !~* ?", CORRECTED_REGEXP_STR) - end - end - - def perform(start_id, end_id) - eligible_mrs = MergeRequest.eligible.where(id: start_id..end_id).pluck(:id) - - return if eligible_mrs.empty? - - eligible_mrs.each_slice(10) do |slice| - MergeRequest.where(id: slice).update_all(draft: false) - end - - mark_job_as_succeeded(start_id, end_id) - end - - private - - def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - 'CleanupDraftDataFromFaultyRegex', - arguments - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/encrypt_integration_properties.rb b/lib/gitlab/background_migration/encrypt_integration_properties.rb deleted file mode 100644 index 28c28ae48eb..00000000000 --- a/lib/gitlab/background_migration/encrypt_integration_properties.rb +++ /dev/null @@ -1,84 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Migrates the integration.properties column from plaintext to encrypted text. - class EncryptIntegrationProperties - # The Integration model, with just the relevant bits. - class Integration < ActiveRecord::Base - include EachBatch - - ALGORITHM = 'aes-256-gcm' - - self.table_name = 'integrations' - self.inheritance_column = :_type_disabled - - scope :with_properties, -> { where.not(properties: nil) } - scope :not_already_encrypted, -> { where(encrypted_properties: nil) } - scope :for_batch, ->(range) { where(id: range) } - - attr_encrypted :encrypted_properties_tmp, - attribute: :encrypted_properties, - mode: :per_attribute_iv, - key: ::Settings.attr_encrypted_db_key_base_32, - algorithm: ALGORITHM, - marshal: true, - marshaler: ::Gitlab::Json, - encode: false, - encode_iv: false - - # See 'Integration#reencrypt_properties' - def encrypt_properties - data = ::Gitlab::Json.parse(properties) - iv = generate_iv(ALGORITHM) - ep = self.class.attr_encrypt(:encrypted_properties_tmp, data, { iv: iv }) - - [ep, iv] - end - end - - def perform(start_id, stop_id) - batch_query = Integration.with_properties.not_already_encrypted.for_batch(start_id..stop_id) - encrypt_batch(batch_query) - mark_job_as_succeeded(start_id, stop_id) - end - - private - - def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - self.class.name.demodulize, - arguments - ) - end - - # represent binary string as a PSQL binary literal: - # https://www.postgresql.org/docs/9.4/datatype-binary.html - def bytea(value) - "'\\x#{value.unpack1('H*')}'::bytea" - end - - def encrypt_batch(batch_query) - values = batch_query.select(:id, :properties).map do |record| - encrypted_properties, encrypted_properties_iv = record.encrypt_properties - "(#{record.id}, #{bytea(encrypted_properties)}, #{bytea(encrypted_properties_iv)})" - end - - return if values.empty? - - Integration.connection.execute(<<~SQL.squish) - WITH cte(cte_id, cte_encrypted_properties, cte_encrypted_properties_iv) - AS #{::Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( - SELECT * - FROM (VALUES #{values.join(',')}) AS t (id, encrypted_properties, encrypted_properties_iv) - ) - UPDATE #{Integration.table_name} - SET encrypted_properties = cte_encrypted_properties - , encrypted_properties_iv = cte_encrypted_properties_iv - FROM cte - WHERE cte_id = id - SQL - end - end - end -end diff --git a/lib/gitlab/background_migration/encrypt_static_object_token.rb b/lib/gitlab/background_migration/encrypt_static_object_token.rb deleted file mode 100644 index 961dea028c9..00000000000 --- a/lib/gitlab/background_migration/encrypt_static_object_token.rb +++ /dev/null @@ -1,70 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Populates "static_object_token_encrypted" field with encrypted versions - # of values from "static_object_token" field - class EncryptStaticObjectToken - # rubocop:disable Style/Documentation - class User < ActiveRecord::Base - include ::EachBatch - self.table_name = 'users' - scope :with_static_object_token, -> { where.not(static_object_token: nil) } - scope :without_static_object_token_encrypted, -> { where(static_object_token_encrypted: nil) } - end - # rubocop:enable Style/Documentation - - BATCH_SIZE = 100 - - def perform(start_id, end_id) - ranged_query = User - .where(id: start_id..end_id) - .with_static_object_token - .without_static_object_token_encrypted - - ranged_query.each_batch(of: BATCH_SIZE) do |sub_batch| - first, last = sub_batch.pick(Arel.sql('min(id), max(id)')) - - batch_query = User.unscoped - .where(id: first..last) - .with_static_object_token - .without_static_object_token_encrypted - - user_tokens = batch_query.pluck(:id, :static_object_token) - - user_encrypted_tokens = user_tokens.map do |(id, plaintext_token)| - next if plaintext_token.blank? - - [id, Gitlab::CryptoHelper.aes256_gcm_encrypt(plaintext_token)] - end - - encrypted_tokens_sql = user_encrypted_tokens.compact.map { |(id, token)| "(#{id}, '#{token}')" }.join(',') - - next unless user_encrypted_tokens.present? - - User.connection.execute(<<~SQL) - WITH cte(cte_id, cte_token) AS #{::Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( - SELECT * - FROM (VALUES #{encrypted_tokens_sql}) AS t (id, token) - ) - UPDATE #{User.table_name} - SET static_object_token_encrypted = cte_token - FROM cte - WHERE cte_id = id - SQL - end - - mark_job_as_succeeded(start_id, end_id) - end - - private - - def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - self.class.name.demodulize, - arguments - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/fix_duplicate_project_name_and_path.rb b/lib/gitlab/background_migration/fix_duplicate_project_name_and_path.rb deleted file mode 100644 index 3772430d0b7..00000000000 --- a/lib/gitlab/background_migration/fix_duplicate_project_name_and_path.rb +++ /dev/null @@ -1,82 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Fix project name duplicates and backfill missing project namespace ids - class FixDuplicateProjectNameAndPath - SUB_BATCH_SIZE = 10 - # isolated project active record - class Project < ActiveRecord::Base - include ::EachBatch - - self.table_name = 'projects' - - scope :without_project_namespace, -> { where(project_namespace_id: nil) } - scope :id_in, ->(ids) { where(id: ids) } - end - - def perform(start_id, end_id) - @project_ids = fetch_project_ids(start_id, end_id) - backfill_project_namespaces_service = init_backfill_service(project_ids) - backfill_project_namespaces_service.cleanup_gin_index('projects') - - project_ids.each_slice(SUB_BATCH_SIZE) do |ids| - ApplicationRecord.connection.execute(update_projects_name_and_path_sql(ids)) - end - - backfill_project_namespaces_service.backfill_project_namespaces - - mark_job_as_succeeded(start_id, end_id) - end - - private - - attr_accessor :project_ids - - def fetch_project_ids(start_id, end_id) - Project.without_project_namespace.where(id: start_id..end_id) - end - - def init_backfill_service(project_ids) - service = Gitlab::BackgroundMigration::ProjectNamespaces::BackfillProjectNamespaces.new - service.project_ids = project_ids - service.sub_batch_size = SUB_BATCH_SIZE - - service - end - - def update_projects_name_and_path_sql(project_ids) - <<~SQL - WITH cte (project_id, path_from_route ) AS ( - #{path_from_route_sql(project_ids).to_sql} - ) - UPDATE - projects - SET - name = concat(projects.name, '-', id), - path = CASE - WHEN projects.path <> cte.path_from_route THEN path_from_route - ELSE projects.path - END - FROM - cte - WHERE - projects.id = cte.project_id; - SQL - end - - def path_from_route_sql(project_ids) - Project.without_project_namespace.id_in(project_ids) - .joins("INNER JOIN routes ON routes.source_id = projects.id AND routes.source_type = 'Project'") - .select("projects.id, SUBSTRING(routes.path FROM '[^/]+(?=/$|$)') AS path_from_route") - end - - def mark_job_as_succeeded(*arguments) - ::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - 'FixDuplicateProjectNameAndPath', - arguments - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/fix_incorrect_max_seats_used.rb b/lib/gitlab/background_migration/fix_incorrect_max_seats_used.rb deleted file mode 100644 index 2c09b8c0b24..00000000000 --- a/lib/gitlab/background_migration/fix_incorrect_max_seats_used.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # rubocop: disable Style/Documentation - class FixIncorrectMaxSeatsUsed - def perform(batch = nil) - end - end - end -end - -Gitlab::BackgroundMigration::FixIncorrectMaxSeatsUsed.prepend_mod_with('Gitlab::BackgroundMigration::FixIncorrectMaxSeatsUsed') diff --git a/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb b/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb deleted file mode 100644 index db3f98bc2ba..00000000000 --- a/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb +++ /dev/null @@ -1,124 +0,0 @@ -# frozen_string_literal: true - -require 'parser/ruby27' - -module Gitlab - module BackgroundMigration - # This migration fixes raw_metadata entries which have incorrectly been passed a Ruby Hash instead of JSON data. - class FixVulnerabilityOccurrencesWithHashesAsRawMetadata - CLUSTER_IMAGE_SCANNING_REPORT_TYPE = 7 - GENERIC_REPORT_TYPE = 99 - - # Type error is used to handle unexpected types when parsing stringified hashes. - class TypeError < ::StandardError - attr_reader :message, :type - - def initialize(message, type) - @message = message - @type = type - end - end - - # Migration model namespace isolated from application code. - class Finding < ActiveRecord::Base - include EachBatch - - self.table_name = 'vulnerability_occurrences' - - scope :by_api_report_types, -> { where(report_type: [CLUSTER_IMAGE_SCANNING_REPORT_TYPE, GENERIC_REPORT_TYPE]) } - end - - def perform(start_id, end_id) - Finding.by_api_report_types.where(id: start_id..end_id).each do |finding| - next if valid_json?(finding.raw_metadata) - - metadata = hash_from_s(finding.raw_metadata) - - finding.update(raw_metadata: metadata.to_json) if metadata - end - mark_job_as_succeeded(start_id, end_id) - end - - def hash_from_s(str_hash) - ast = Parser::Ruby27.parse(str_hash) - - unless ast.type == :hash - ::Gitlab::AppLogger.error(message: "expected raw_metadata to be a hash", type: ast.type) - return - end - - parse_hash(ast) - rescue Parser::SyntaxError => e - ::Gitlab::AppLogger.error(message: "error parsing raw_metadata", error: e.message) - nil - rescue TypeError => e - ::Gitlab::AppLogger.error(message: "error parsing raw_metadata", error: e.message, type: e.type) - nil - end - - private - - def mark_job_as_succeeded(*arguments) - ::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - 'FixVulnerabilityOccurrencesWithHashesAsRawMetadata', - arguments - ) - end - - def valid_json?(metadata) - Oj.load(metadata) - true - rescue Oj::ParseError, EncodingError, JSON::ParserError, JSON::GeneratorError, Encoding::UndefinedConversionError - false - end - - def parse_hash(hash) - out = {} - hash.children.each do |node| - unless node.type == :pair - raise TypeError.new("expected child of hash to be a `pair`", node.type) - end - - key, value = node.children - - key = parse_key(key) - value = parse_value(value) - - out[key] = value - end - - out - end - - def parse_key(key) - case key.type - when :sym, :str, :int - key.children.first - else - raise TypeError.new("expected key to be either symbol, string, or integer", key.type) - end - end - - def parse_value(value) - case value.type - when :sym, :str, :int - value.children.first - # rubocop:disable Lint/BooleanSymbol - when :true - true - when :false - false - # rubocop:enable Lint/BooleanSymbol - when :nil - nil - when :array - value.children.map { |c| parse_value(c) } - when :hash - parse_hash(value) - else - raise TypeError.new("value of a pair was an unexpected type", value.type) - end - end - end - end -end diff --git a/lib/gitlab/background_migration/mark_duplicate_npm_packages_for_destruction.rb b/lib/gitlab/background_migration/mark_duplicate_npm_packages_for_destruction.rb new file mode 100644 index 00000000000..68cc650d130 --- /dev/null +++ b/lib/gitlab/background_migration/mark_duplicate_npm_packages_for_destruction.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # It seeks duplicate npm packages and mark them for destruction + class MarkDuplicateNpmPackagesForDestruction < BatchedMigrationJob + NPM_PACKAGE_TYPE = 2 + PENDING_DESTRUCTION_STATUS = 4 + + operation_name :update_all + feature_category :package_registry + + # Temporary class to link AR model to the `packages_packages` table + class Package < ::ApplicationRecord + include EachBatch + + self.table_name = 'packages_packages' + end + + def perform + distinct_each_batch do |batch| + project_ids = batch.pluck(:project_id) + + subquery = Package + .where(project_id: project_ids, package_type: NPM_PACKAGE_TYPE) + .where.not(status: PENDING_DESTRUCTION_STATUS) + .select('project_id, name, version, MAX(id) AS max_id') + .group(:project_id, :name, :version) + .having('COUNT(*) > 1') + + join_query = <<~SQL.squish + INNER JOIN (#{subquery.to_sql}) AS duplicates + ON packages_packages.project_id = duplicates.project_id + AND packages_packages.name = duplicates.name + AND packages_packages.version = duplicates.version + SQL + + Package + .joins(join_query) + .where.not('packages_packages.id = duplicates.max_id') + .each_batch do |batch_to_update| + batch_to_update.update_all(status: PENDING_DESTRUCTION_STATUS) + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/merge_topics_with_same_name.rb b/lib/gitlab/background_migration/merge_topics_with_same_name.rb deleted file mode 100644 index 07231098a5f..00000000000 --- a/lib/gitlab/background_migration/merge_topics_with_same_name.rb +++ /dev/null @@ -1,76 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # The class to merge project topics with the same case insensitive name - class MergeTopicsWithSameName - # Temporary AR model for topics - class Topic < ActiveRecord::Base - self.table_name = 'topics' - end - - # Temporary AR model for project topic assignment - class ProjectTopic < ActiveRecord::Base - self.table_name = 'project_topics' - end - - def perform(topic_names) - topic_names.each do |topic_name| - topics = Topic.where('LOWER(name) = ?', topic_name) - .order(total_projects_count: :desc, non_private_projects_count: :desc, id: :asc) - .to_a - topic_to_keep = topics.shift - merge_topics(topic_to_keep, topics) if topics.any? - end - end - - private - - def merge_topics(topic_to_keep, topics_to_remove) - description = topic_to_keep.description - - topics_to_remove.each do |topic| - description ||= topic.description if topic.description.present? - process_avatar(topic_to_keep, topic) if topic.avatar.present? - - ProjectTopic.transaction do - ProjectTopic.where(topic_id: topic.id) - .where.not(project_id: ProjectTopic.where(topic_id: topic_to_keep).select(:project_id)) - .update_all(topic_id: topic_to_keep.id) - ProjectTopic.where(topic_id: topic.id).delete_all - end - end - - Topic.where(id: topics_to_remove).delete_all - - topic_to_keep.update( - description: description, - total_projects_count: total_projects_count(topic_to_keep.id), - non_private_projects_count: non_private_projects_count(topic_to_keep.id) - ) - end - - # We intentionally use application code here because we need to copy/remove avatar files - def process_avatar(topic_to_keep, topic_to_remove) - topic_to_remove = ::Projects::Topic.find(topic_to_remove.id) - topic_to_keep = ::Projects::Topic.find(topic_to_keep.id) - unless topic_to_keep.avatar.present? - topic_to_keep.avatar = topic_to_remove.avatar - topic_to_keep.save! - end - - topic_to_remove.remove_avatar! - topic_to_remove.save! - end - - def total_projects_count(topic_id) - ProjectTopic.where(topic_id: topic_id).count - end - - def non_private_projects_count(topic_id) - ProjectTopic.joins('INNER JOIN projects ON project_topics.project_id = projects.id') - .where(project_topics: { topic_id: topic_id }).where('projects.visibility_level in (10, 20)').count - end - end - end -end diff --git a/lib/gitlab/background_migration/migrate_personal_namespace_project_maintainer_to_owner.rb b/lib/gitlab/background_migration/migrate_personal_namespace_project_maintainer_to_owner.rb deleted file mode 100644 index 49eff6e2771..00000000000 --- a/lib/gitlab/background_migration/migrate_personal_namespace_project_maintainer_to_owner.rb +++ /dev/null @@ -1,45 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Migrates personal namespace project `maintainer` memberships (for the associated user only) to OWNER - # Does not create any missing records, simply migrates existing ones - class MigratePersonalNamespaceProjectMaintainerToOwner - include Gitlab::Database::DynamicModelHelpers - - def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms) - parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id) - - parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| - batch_metrics.time_operation(:update_all) do - sub_batch.update_all('access_level = 50') - end - - pause_ms = 0 if pause_ms < 0 - sleep(pause_ms * 0.001) - end - end - - def batch_metrics - @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new - end - - private - - def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) - # members of projects within their own personal namespace - - # rubocop: disable CodeReuse/ActiveRecord - define_batchable_model(:members, connection: ApplicationRecord.connection) - .where(source_key_column => start_id..stop_id) - .joins("INNER JOIN projects ON members.source_id = projects.id") - .joins("INNER JOIN namespaces ON projects.namespace_id = namespaces.id") - .where(type: 'ProjectMember') - .where("namespaces.type = 'User'") - .where('members.access_level < 50') - .where('namespaces.owner_id = members.user_id') - end - end - # rubocop: enable CodeReuse/ActiveRecord - end -end diff --git a/lib/gitlab/background_migration/migrate_shimo_confluence_integration_category.rb b/lib/gitlab/background_migration/migrate_shimo_confluence_integration_category.rb deleted file mode 100644 index d7d24960a41..00000000000 --- a/lib/gitlab/background_migration/migrate_shimo_confluence_integration_category.rb +++ /dev/null @@ -1,27 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # The class to migrate category of integrations to third_party_wiki for confluence and shimo - class MigrateShimoConfluenceIntegrationCategory - include Gitlab::Database::DynamicModelHelpers - - def perform(start_id, end_id) - define_batchable_model('integrations', connection: ApplicationRecord.connection) - .where(id: start_id..end_id, type_new: %w[Integrations::Confluence Integrations::Shimo]) - .update_all(category: 'third_party_wiki') - - mark_job_as_succeeded(start_id, end_id) - end - - private - - def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - self.class.name.demodulize, - arguments - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects.rb b/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects.rb index 592ef3220ff..74f5bc3f725 100644 --- a/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects.rb +++ b/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects.rb @@ -12,7 +12,7 @@ module Gitlab end operation_name :update_all - feature_category :projects + feature_category :groups_and_projects def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds.rb b/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds.rb deleted file mode 100644 index 13b66b2e02e..00000000000 --- a/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds.rb +++ /dev/null @@ -1,44 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # A job to nullify orphan runner_id on ci_builds table - class NullifyOrphanRunnerIdOnCiBuilds - include Gitlab::Database::DynamicModelHelpers - - def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms) - pause_ms = 0 if pause_ms < 0 - - batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id) - batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| - batch_metrics.time_operation(:update_all) do - filtered_sub_batch(sub_batch).update_all(runner_id: nil) - end - - sleep(pause_ms * 0.001) - end - end - - def batch_metrics - @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new - end - - private - - def connection - ::Ci::ApplicationRecord.connection - end - - def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) - define_batchable_model(source_table, connection: connection) - .where(source_key_column => start_id..stop_id) - end - - def filtered_sub_batch(sub_batch) - sub_batch - .joins('LEFT OUTER JOIN ci_runners ON ci_runners.id = ci_builds.runner_id') - .where('ci_builds.runner_id IS NOT NULL AND ci_runners.id IS NULL') - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_container_repository_migration_plan.rb b/lib/gitlab/background_migration/populate_container_repository_migration_plan.rb deleted file mode 100644 index a9611e9814c..00000000000 --- a/lib/gitlab/background_migration/populate_container_repository_migration_plan.rb +++ /dev/null @@ -1,51 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # The class to populates the migration_plan column of container_repositories - # with the current plan of the namespaces that owns the container_repository - # - # The plan can be NULL, in which case no UPDATE - # will be executed. - class PopulateContainerRepositoryMigrationPlan - def perform(start_id, end_id) - (start_id..end_id).each do |id| - execute(<<~SQL) - WITH selected_plan AS ( - SELECT "plans"."name" - FROM "container_repositories" - INNER JOIN "projects" ON "projects"."id" = "container_repositories"."project_id" - INNER JOIN "namespaces" ON "namespaces"."id" = "projects"."namespace_id" - INNER JOIN "gitlab_subscriptions" ON "gitlab_subscriptions"."namespace_id" = "namespaces"."traversal_ids"[1] - INNER JOIN "plans" ON "plans"."id" = "gitlab_subscriptions"."hosted_plan_id" - WHERE "container_repositories"."id" = #{id} - ) - UPDATE container_repositories - SET migration_plan = selected_plan.name - FROM selected_plan - WHERE container_repositories.id = #{id}; - SQL - end - - mark_job_as_succeeded(start_id, end_id) - end - - private - - def connection - @connection ||= ApplicationRecord.connection - end - - def execute(sql) - connection.execute(sql) - end - - def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - self.class.name.demodulize, - arguments - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_namespace_statistics.rb b/lib/gitlab/background_migration/populate_namespace_statistics.rb deleted file mode 100644 index 97927ef48c2..00000000000 --- a/lib/gitlab/background_migration/populate_namespace_statistics.rb +++ /dev/null @@ -1,47 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This class creates/updates those namespace statistics - # that haven't been created nor initialized. - # It also updates the related namespace statistics - class PopulateNamespaceStatistics - def perform(group_ids, statistics) - # Updating group statistics might involve calling Gitaly. - # For example, when calculating `wiki_size`, we will need - # to perform the request to check if the repo exists and - # also the repository size. - # - # The `allow_n_plus_1_calls` method is only intended for - # dev and test. It won't be raised in prod. - ::Gitlab::GitalyClient.allow_n_plus_1_calls do - relation(group_ids).each do |group| - upsert_namespace_statistics(group, statistics) - end - end - end - - private - - def upsert_namespace_statistics(group, statistics) - response = ::Groups::UpdateStatisticsService.new(group, statistics: statistics).execute - - error_message("#{response.message} group: #{group.id}") if response.error? - end - - def logger - @logger ||= ::Gitlab::BackgroundMigration::Logger.build - end - - def error_message(message) - logger.error(message: "Namespace Statistics Migration: #{message}") - end - - def relation(group_ids) - Group.includes(:namespace_statistics).where(id: group_ids) - end - end - end -end - -Gitlab::BackgroundMigration::PopulateNamespaceStatistics.prepend_mod_with('Gitlab::BackgroundMigration::PopulateNamespaceStatistics') diff --git a/lib/gitlab/background_migration/populate_test_reports_issue_id.rb b/lib/gitlab/background_migration/populate_test_reports_issue_id.rb deleted file mode 100644 index 301efd0c943..00000000000 --- a/lib/gitlab/background_migration/populate_test_reports_issue_id.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true -# rubocop: disable Style/Documentation - -module Gitlab - module BackgroundMigration - class PopulateTestReportsIssueId - def perform(start_id, stop_id) - # NO OP - end - end - end -end - -Gitlab::BackgroundMigration::PopulateTestReportsIssueId.prepend_mod diff --git a/lib/gitlab/background_migration/populate_topics_non_private_projects_count.rb b/lib/gitlab/background_migration/populate_topics_non_private_projects_count.rb deleted file mode 100644 index 1f2b55004e4..00000000000 --- a/lib/gitlab/background_migration/populate_topics_non_private_projects_count.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # The class to populates the non private projects counter of topics - class PopulateTopicsNonPrivateProjectsCount - SUB_BATCH_SIZE = 100 - - # Temporary AR model for topics - class Topic < ActiveRecord::Base - include EachBatch - - self.table_name = 'topics' - end - - def perform(start_id, stop_id) - Topic.where(id: start_id..stop_id).each_batch(of: SUB_BATCH_SIZE) do |batch| - ApplicationRecord.connection.execute(<<~SQL) - WITH batched_relation AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (#{batch.select(:id).limit(SUB_BATCH_SIZE).to_sql}) - UPDATE topics - SET non_private_projects_count = ( - SELECT COUNT(*) - FROM project_topics - INNER JOIN projects - ON project_topics.project_id = projects.id - WHERE project_topics.topic_id = batched_relation.id - AND projects.visibility_level > 0 - ) - FROM batched_relation - WHERE topics.id = batched_relation.id - SQL - end - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_vulnerability_reads.rb b/lib/gitlab/background_migration/populate_vulnerability_reads.rb deleted file mode 100644 index 656c62d9ee5..00000000000 --- a/lib/gitlab/background_migration/populate_vulnerability_reads.rb +++ /dev/null @@ -1,84 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # rubocop:disable Style/Documentation - class PopulateVulnerabilityReads - include Gitlab::Database::DynamicModelHelpers - - PAUSE_SECONDS = 0.1 - - def perform(start_id, end_id, sub_batch_size) - vulnerability_model.where(id: start_id..end_id).each_batch(of: sub_batch_size) do |sub_batch| - first, last = sub_batch.pick(Arel.sql('min(id), max(id)')) - connection.execute(insert_query(first, last)) - - sleep PAUSE_SECONDS - end - - mark_job_as_succeeded(start_id, end_id, sub_batch_size) - end - - private - - def vulnerability_model - define_batchable_model('vulnerabilities', connection: connection) - end - - def connection - ApplicationRecord.connection - end - - def insert_query(start_id, end_id) - <<~SQL - INSERT INTO vulnerability_reads ( - vulnerability_id, - project_id, - scanner_id, - report_type, - severity, - state, - has_issues, - resolved_on_default_branch, - uuid, - location_image - ) - SELECT - vulnerabilities.id, - vulnerabilities.project_id, - vulnerability_scanners.id, - vulnerabilities.report_type, - vulnerabilities.severity, - vulnerabilities.state, - CASE - WHEN - vulnerability_issue_links.vulnerability_id IS NOT NULL - THEN - true - ELSE - false - END - has_issues, - vulnerabilities.resolved_on_default_branch, - vulnerability_occurrences.uuid::uuid, - vulnerability_occurrences.location ->> 'image' - FROM - vulnerabilities - INNER JOIN vulnerability_occurrences ON vulnerability_occurrences.vulnerability_id = vulnerabilities.id - INNER JOIN vulnerability_scanners ON vulnerability_scanners.id = vulnerability_occurrences.scanner_id - LEFT JOIN vulnerability_issue_links ON vulnerability_issue_links.vulnerability_id = vulnerabilities.id - WHERE vulnerabilities.id BETWEEN #{start_id} AND #{end_id} - ON CONFLICT(vulnerability_id) DO NOTHING; - SQL - end - - def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - self.class.name.demodulize, - arguments - ) - end - end - # rubocop:enable Style/Documentation - end -end diff --git a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb deleted file mode 100644 index 9a42d035285..00000000000 --- a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb +++ /dev/null @@ -1,218 +0,0 @@ -# frozen_string_literal: true - -# rubocop: disable Style/Documentation -class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid # rubocop:disable Metrics/ClassLength - # rubocop: disable Gitlab/NamespacedClass - class VulnerabilitiesIdentifier < ActiveRecord::Base - self.table_name = "vulnerability_identifiers" - has_many :primary_findings, class_name: 'VulnerabilitiesFinding', inverse_of: :primary_identifier, foreign_key: 'primary_identifier_id' - end - - class VulnerabilitiesFinding < ActiveRecord::Base - include EachBatch - include ShaAttribute - - self.table_name = "vulnerability_occurrences" - - has_many :signatures, foreign_key: 'finding_id', class_name: 'VulnerabilityFindingSignature', inverse_of: :finding - belongs_to :primary_identifier, class_name: 'VulnerabilitiesIdentifier', inverse_of: :primary_findings, foreign_key: 'primary_identifier_id' - - REPORT_TYPES = { - sast: 0, - dependency_scanning: 1, - container_scanning: 2, - dast: 3, - secret_detection: 4, - coverage_fuzzing: 5, - api_fuzzing: 6, - cluster_image_scanning: 7, - generic: 99 - }.with_indifferent_access.freeze - enum report_type: REPORT_TYPES - - sha_attribute :fingerprint - sha_attribute :location_fingerprint - end - - class VulnerabilityFindingSignature < ActiveRecord::Base - include ShaAttribute - - self.table_name = 'vulnerability_finding_signatures' - belongs_to :finding, foreign_key: 'finding_id', inverse_of: :signatures, class_name: 'VulnerabilitiesFinding' - - sha_attribute :signature_sha - end - - class VulnerabilitiesFindingPipeline < ActiveRecord::Base - include EachBatch - self.table_name = "vulnerability_occurrence_pipelines" - end - - class Vulnerability < ActiveRecord::Base - include EachBatch - self.table_name = "vulnerabilities" - end - - class CalculateFindingUUID - FINDING_NAMESPACES_IDS = { - development: "a143e9e2-41b3-47bc-9a19-081d089229f4", - test: "a143e9e2-41b3-47bc-9a19-081d089229f4", - staging: "a6930898-a1b2-4365-ab18-12aa474d9b26", - production: "58dc0f06-936c-43b3-93bb-71693f1b6570" - }.freeze - - NAMESPACE_REGEX = /(\h{8})-(\h{4})-(\h{4})-(\h{4})-(\h{4})(\h{8})/.freeze - PACK_PATTERN = "NnnnnN" - - def self.call(value) - Digest::UUID.uuid_v5(namespace_id, value) - end - - def self.namespace_id - namespace_uuid = FINDING_NAMESPACES_IDS.fetch(Rails.env.to_sym) - # Digest::UUID is broken when using an UUID in namespace_id - # https://github.com/rails/rails/issues/37681#issue-520718028 - namespace_uuid.scan(NAMESPACE_REGEX).flatten.map { |s| s.to_i(16) }.pack(PACK_PATTERN) - end - end - # rubocop: enable Gitlab/NamespacedClass - - # rubocop: disable Metrics/AbcSize,Metrics/MethodLength,Metrics/BlockLength - def perform(start_id, end_id) - log_info('Migration started', start_id: start_id, end_id: end_id) - - VulnerabilitiesFinding - .joins(:primary_identifier) - .includes(:signatures) - .select(:id, :report_type, :primary_identifier_id, :fingerprint, :location_fingerprint, :project_id, :created_at, :vulnerability_id, :uuid) - .where(id: start_id..end_id) - .each_batch(of: 50) do |relation| - duplicates = find_duplicates(relation) - remove_findings(ids: duplicates) if duplicates.present? - - to_update = relation.reject { |finding| duplicates.include?(finding.id) } - - begin - known_uuids = Set.new - to_be_deleted = [] - - mappings = to_update.each_with_object({}) do |finding, hash| - uuid = calculate_uuid_v5_for_finding(finding) - - if known_uuids.add?(uuid) - hash[finding] = { uuid: uuid } - else - to_be_deleted << finding.id - end - end - - # It is technically still possible to have duplicate uuids - # if the data integrity is broken somehow and the primary identifiers of - # the findings are pointing to different projects with the same fingerprint values. - if to_be_deleted.present? - log_info('Conflicting UUIDs found within the batch', finding_ids: to_be_deleted) - - remove_findings(ids: to_be_deleted) - end - - ::Gitlab::Database::BulkUpdate.execute(%i[uuid], mappings) if mappings.present? - - log_info('Recalculation is done', finding_ids: mappings.keys.pluck(:id)) - rescue ActiveRecord::RecordNotUnique => error - log_info('RecordNotUnique error received') - - match_data = /\(uuid\)=\((?\S{36})\)/.match(error.message) - - # This exception returns the **correct** UUIDv5 which probably comes from a later record - # and it's the one we can drop in the easiest way before retrying the UPDATE query - if match_data - uuid = match_data[:uuid] - log_info('Conflicting UUID found', uuid: uuid) - - id = VulnerabilitiesFinding.find_by(uuid: uuid)&.id - remove_findings(ids: id) if id - retry - else - log_error('Couldnt find conflicting uuid') - - Gitlab::ErrorTracking.track_and_raise_exception(error) - end - end - end - - mark_job_as_succeeded(start_id, end_id) - rescue StandardError => error - log_error('An exception happened') - - Gitlab::ErrorTracking.track_and_raise_exception(error) - end - # rubocop: disable Metrics/AbcSize,Metrics/MethodLength,Metrics/BlockLength - - private - - def find_duplicates(relation) - to_exclude = [] - relation.flat_map do |record| - # Assuming we're scanning id 31 and the duplicate is id 40 - # first we'd process 31 and add 40 to the list of ids to remove - # then we would process record 40 and add 31 to the list of removals - # so we would drop both records - to_exclude << record.id - - VulnerabilitiesFinding.where( - report_type: record.report_type, - location_fingerprint: record.location_fingerprint, - primary_identifier_id: record.primary_identifier_id, - project_id: record.project_id - ).where.not(id: to_exclude).pluck(:id) - end - end - - def remove_findings(ids:) - ids = Array(ids) - log_info('Removing Findings and associated records', ids: ids) - - vulnerability_ids = VulnerabilitiesFinding.where(id: ids).pluck(:vulnerability_id).uniq.compact - - VulnerabilitiesFindingPipeline.where(occurrence_id: ids).each_batch { |batch| batch.delete_all } - Vulnerability.where(id: vulnerability_ids).each_batch { |batch| batch.delete_all } - VulnerabilitiesFinding.where(id: ids).delete_all - end - - def calculate_uuid_v5_for_finding(vulnerability_finding) - return unless vulnerability_finding - - signatures = vulnerability_finding.signatures.sort_by { |signature| signature.algorithm_type_before_type_cast } - location_fingerprint = signatures.last&.signature_sha || vulnerability_finding.location_fingerprint - - uuid_v5_name_components = { - report_type: vulnerability_finding.report_type, - primary_identifier_fingerprint: vulnerability_finding.fingerprint, - location_fingerprint: location_fingerprint, - project_id: vulnerability_finding.project_id - } - - name = uuid_v5_name_components.values.join('-') - - CalculateFindingUUID.call(name) - end - - def log_info(message, **extra) - logger.info(migrator: 'RecalculateVulnerabilitiesOccurrencesUuid', message: message, **extra) - end - - def log_error(message, **extra) - logger.error(migrator: 'RecalculateVulnerabilitiesOccurrencesUuid', message: message, **extra) - end - - def logger - @logger ||= Gitlab::BackgroundMigration::Logger.build - end - - def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - 'RecalculateVulnerabilitiesOccurrencesUuid', - arguments - ) - end -end diff --git a/lib/gitlab/background_migration/recalculate_vulnerability_finding_signatures_for_findings.rb b/lib/gitlab/background_migration/recalculate_vulnerability_finding_signatures_for_findings.rb deleted file mode 100644 index 20200a1d508..00000000000 --- a/lib/gitlab/background_migration/recalculate_vulnerability_finding_signatures_for_findings.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # rubocop: disable Style/Documentation - class RecalculateVulnerabilityFindingSignaturesForFindings - def perform(start_id, stop_id) - end - end - end -end - -Gitlab::BackgroundMigration::RecalculateVulnerabilityFindingSignaturesForFindings.prepend_mod diff --git a/lib/gitlab/background_migration/remove_all_trace_expiration_dates.rb b/lib/gitlab/background_migration/remove_all_trace_expiration_dates.rb deleted file mode 100644 index d47aa76f24b..00000000000 --- a/lib/gitlab/background_migration/remove_all_trace_expiration_dates.rb +++ /dev/null @@ -1,53 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Removing expire_at timestamps that shouldn't have - # been written to traces on gitlab.com. - class RemoveAllTraceExpirationDates - include Gitlab::Database::MigrationHelpers - - BATCH_SIZE = 1_000 - - # Stubbed class to connect to the CI database - # connects_to has to be called in abstract classes. - class MultiDbAdaptableClass < ActiveRecord::Base - self.abstract_class = true - - if Gitlab::Database.has_config?(:ci) - connects_to database: { writing: :ci, reading: :ci } - end - end - - # Stubbed class to access the ci_job_artifacts table - class JobArtifact < MultiDbAdaptableClass - include EachBatch - - self.table_name = 'ci_job_artifacts' - - TARGET_TIMESTAMPS = [ - Date.new(2021, 04, 22).midnight.utc, - Date.new(2021, 05, 22).midnight.utc, - Date.new(2021, 06, 22).midnight.utc, - Date.new(2022, 01, 22).midnight.utc, - Date.new(2022, 02, 22).midnight.utc, - Date.new(2022, 03, 22).midnight.utc, - Date.new(2022, 04, 22).midnight.utc - ].freeze - - scope :traces, -> { where(file_type: 3) } - scope :between, -> (start_id, end_id) { where(id: start_id..end_id) } - scope :in_targeted_timestamps, -> { where(expire_at: TARGET_TIMESTAMPS) } - end - - def perform(start_id, end_id) - return unless Gitlab.com? - - JobArtifact.traces - .between(start_id, end_id) - .in_targeted_timestamps - .each_batch(of: BATCH_SIZE) { |batch| batch.update_all(expire_at: nil) } - end - end - end -end diff --git a/lib/gitlab/background_migration/remove_invalid_deploy_access_level_groups.rb b/lib/gitlab/background_migration/remove_invalid_deploy_access_level_groups.rb new file mode 100644 index 00000000000..0a107a136b0 --- /dev/null +++ b/lib/gitlab/background_migration/remove_invalid_deploy_access_level_groups.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class removes invalid `protected_environment_deploy_access_levels.group_id` records. + class RemoveInvalidDeployAccessLevelGroups < BatchedMigrationJob + operation_name :remove_invalid_deploy_access_level_groups + feature_category :database + + scope_to ->(relation) do + relation.joins('INNER JOIN namespaces ON namespaces.id = protected_environment_deploy_access_levels.group_id') + .where.not(protected_environment_deploy_access_levels: { group_id: nil }) + .where("namespaces.type = 'User'") + end + + def perform + each_sub_batch(&:delete_all) + end + end + end +end diff --git a/lib/gitlab/background_migration/remove_project_group_link_with_missing_groups.rb b/lib/gitlab/background_migration/remove_project_group_link_with_missing_groups.rb index 879e52c96bf..713131edd30 100644 --- a/lib/gitlab/background_migration/remove_project_group_link_with_missing_groups.rb +++ b/lib/gitlab/background_migration/remove_project_group_link_with_missing_groups.rb @@ -7,7 +7,7 @@ module Gitlab class RemoveProjectGroupLinkWithMissingGroups < Gitlab::BackgroundMigration::BatchedMigrationJob scope_to ->(relation) { relation } operation_name :delete_all - feature_category :subgroups + feature_category :groups_and_projects def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/remove_vulnerability_finding_links.rb b/lib/gitlab/background_migration/remove_vulnerability_finding_links.rb deleted file mode 100644 index 4acef9029f9..00000000000 --- a/lib/gitlab/background_migration/remove_vulnerability_finding_links.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Remove vulnerability finding link records - # The records will be repopulated from the `raw_metadata` - # column of `vulnerability_occurrences` once the unique - # index is in place. - class RemoveVulnerabilityFindingLinks - include Gitlab::Database::DynamicModelHelpers - - def perform(start_id, stop_id) - define_batchable_model('vulnerability_finding_links', connection: ApplicationRecord.connection) - .where(id: start_id..stop_id) - .delete_all - end - end - end -end diff --git a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values_on_projects.rb b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values_on_projects.rb deleted file mode 100644 index 190e2fc22fb..00000000000 --- a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values_on_projects.rb +++ /dev/null @@ -1,40 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # A job to nullify duplicate runners_token_encrypted values in projects table in batches - class ResetDuplicateCiRunnersTokenEncryptedValuesOnProjects - class Project < ActiveRecord::Base # rubocop:disable Style/Documentation - include EachBatch - - self.table_name = 'projects' - - scope :base_query, -> { where.not(runners_token_encrypted: nil) } - end - - def perform(start_id, end_id) - # Reset duplicate runner tokens that would prevent creating an unique index. - batch_records = Project.base_query.where(id: start_id..end_id) - - duplicate_tokens = Project.base_query - .where(runners_token_encrypted: batch_records.select(:runners_token_encrypted).distinct) - .group(:runners_token_encrypted) - .having('COUNT(*) > 1') - .pluck(:runners_token_encrypted) - - batch_records.where(runners_token_encrypted: duplicate_tokens).update_all(runners_token_encrypted: nil) if duplicate_tokens.any? - - mark_job_as_succeeded(start_id, end_id) - end - - private - - def mark_job_as_succeeded(*arguments) - ::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - self.class.name.demodulize, - arguments - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values_on_projects.rb b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values_on_projects.rb deleted file mode 100644 index b58eefa0ab3..00000000000 --- a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values_on_projects.rb +++ /dev/null @@ -1,40 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # A job to nullify duplicate ci_runners_token values in projects table in batches - class ResetDuplicateCiRunnersTokenValuesOnProjects - class Project < ActiveRecord::Base # rubocop:disable Style/Documentation - include EachBatch - - self.table_name = 'projects' - - scope :base_query, -> { where.not(runners_token: nil) } - end - - def perform(start_id, end_id) - # Reset duplicate runner tokens that would prevent creating an unique index. - batch_records = Project.base_query.where(id: start_id..end_id) - - duplicate_tokens = Project.base_query - .where(runners_token: batch_records.select(:runners_token).distinct) - .group(:runners_token) - .having('COUNT(*) > 1') - .pluck(:runners_token) - - batch_records.where(runners_token: duplicate_tokens).update_all(runners_token: nil) if duplicate_tokens.any? - - mark_job_as_succeeded(start_id, end_id) - end - - private - - def mark_job_as_succeeded(*arguments) - ::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - self.class.name.demodulize, - arguments - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/update_timelogs_null_spent_at.rb b/lib/gitlab/background_migration/update_timelogs_null_spent_at.rb deleted file mode 100644 index b61f2ee7f4c..00000000000 --- a/lib/gitlab/background_migration/update_timelogs_null_spent_at.rb +++ /dev/null @@ -1,39 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Class to populate spent_at for timelogs - class UpdateTimelogsNullSpentAt - include Gitlab::Database::DynamicModelHelpers - - BATCH_SIZE = 100 - - def perform(start_id, stop_id) - define_batchable_model('timelogs', connection: connection) - .where(spent_at: nil, id: start_id..stop_id) - .each_batch(of: 100) do |subbatch| - batch_start, batch_end = subbatch.pick('min(id), max(id)') - - update_timelogs(batch_start, batch_end) - end - end - - def update_timelogs(batch_start, batch_stop) - execute(<<~SQL) - UPDATE timelogs - SET spent_at = created_at - WHERE spent_at IS NULL - AND timelogs.id BETWEEN #{batch_start} AND #{batch_stop}; - SQL - end - - def connection - @connection ||= ApplicationRecord.connection - end - - def execute(sql) - connection.execute(sql) - end - end - end -end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 592e75b1430..e785ce558db 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -10,6 +10,8 @@ module Gitlab attr_reader :project, :client, :errors, :users + ALREADY_IMPORTED_CACHE_KEY = 'bitbucket_cloud-importer/already-imported/%{project}/%{collection}' + def initialize(project) @project = project @client = Bitbucket::Client.new(project.import_data.credentials) @@ -31,6 +33,18 @@ module Gitlab private + def already_imported?(collection, iid) + Gitlab::Cache::Import::Caching.set_includes?(cache_key(collection), iid) + end + + def mark_as_imported(collection, iid) + Gitlab::Cache::Import::Caching.set_add(cache_key(collection), iid) + end + + def cache_key(collection) + format(ALREADY_IMPORTED_CACHE_KEY, project: project.id, collection: collection) + end + def handle_errors return unless errors.any? @@ -97,6 +111,8 @@ module Gitlab issue_type_id = ::WorkItems::Type.default_issue_type.id client.issues(repo).each_with_index do |issue, index| + next if already_imported?(:issues, issue.iid) + # If a user creates an issue while the import is in progress, this can lead to an import failure. # The workaround is to allocate IIDs before starting the importer. allocate_issues_internal_id!(project, client) if index == 0 @@ -127,6 +143,8 @@ module Gitlab updated_at: issue.updated_at ) + mark_as_imported(:issues, issue.iid) + metrics.issues_counter.increment gitlab_issue.labels << @labels[label_name] @@ -179,6 +197,8 @@ module Gitlab pull_requests = client.pull_requests(repo) pull_requests.each do |pull_request| + next if already_imported?(:pull_requests, pull_request.iid) + import_pull_request(pull_request) end end @@ -209,6 +229,8 @@ module Gitlab updated_at: pull_request.updated_at ) + mark_as_imported(:pull_requests, pull_request.iid) + metrics.merge_requests_counter.increment import_pull_request_comments(pull_request, merge_request) if merge_request.persisted? diff --git a/lib/gitlab/bitbucket_server_import/importer.rb b/lib/gitlab/bitbucket_server_import/importer.rb index 6b163cd1b2d..f3253027d57 100644 --- a/lib/gitlab/bitbucket_server_import/importer.rb +++ b/lib/gitlab/bitbucket_server_import/importer.rb @@ -442,10 +442,9 @@ module Gitlab end def uid(rep_object) - # We want this explicit to only be username on the FF - # Otherwise, match email. - # There should be no default fall-through on username. Fall-through to import user - if Feature.enabled?(:bitbucket_server_user_mapping_by_username) + # We want this to only match either username or email depending on the flag state. + # There should be no fall-through. + if Feature.enabled?(:bitbucket_server_user_mapping_by_username, type: :ops) find_user_id(by: :username, value: rep_object.author_username) else find_user_id(by: :email, value: rep_object.author_email) diff --git a/lib/gitlab/bitbucket_server_import/importers/repository_importer.rb b/lib/gitlab/bitbucket_server_import/importers/repository_importer.rb index cd09ac40e9f..e7a9adf2beb 100644 --- a/lib/gitlab/bitbucket_server_import/importers/repository_importer.rb +++ b/lib/gitlab/bitbucket_server_import/importers/repository_importer.rb @@ -17,6 +17,8 @@ module Gitlab project.repository.import_repository(project.import_url) project.repository.fetch_as_mirror(project.import_url, refmap: refmap) + validate_repository_size! + update_clone_time end @@ -48,7 +50,13 @@ module Gitlab def update_clone_time project.touch(:last_repository_updated_at) end + + def validate_repository_size! + # Defined in EE + end end end end end + +Gitlab::BitbucketServerImport::Importers::RepositoryImporter.prepend_mod diff --git a/lib/gitlab/bitbucket_server_import/user_finder.rb b/lib/gitlab/bitbucket_server_import/user_finder.rb index f96454eb2cc..68bd2d4851a 100644 --- a/lib/gitlab/bitbucket_server_import/user_finder.rb +++ b/lib/gitlab/bitbucket_server_import/user_finder.rb @@ -24,7 +24,7 @@ module Gitlab def uid(object) # We want this to only match either username or email depending on the flag state. # There should be no fall-through. - if Feature.enabled?(:bitbucket_server_user_mapping_by_username) + if Feature.enabled?(:bitbucket_server_user_mapping_by_username, type: :ops) find_user_id(by: :username, value: object.is_a?(Hash) ? object[:author_username] : object.author_username) else find_user_id(by: :email, value: object.is_a?(Hash) ? object[:author_email] : object.author_email) diff --git a/lib/gitlab/cache/import/caching.rb b/lib/gitlab/cache/import/caching.rb index 7fec6584ba3..8f2df29c320 100644 --- a/lib/gitlab/cache/import/caching.rb +++ b/lib/gitlab/cache/import/caching.rb @@ -162,13 +162,13 @@ module Gitlab def self.write_multiple(mapping, key_prefix: nil, timeout: TIMEOUT) with_redis do |redis| Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - redis.pipelined do |multi| + Gitlab::Redis::CrossSlot::Pipeline.new(redis).pipelined do |pipeline| mapping.each do |raw_key, value| key = cache_key_for("#{key_prefix}#{raw_key}") validate_redis_value!(value) - multi.set(key, value, ex: timeout) + pipeline.set(key, value, ex: timeout) end end end diff --git a/lib/gitlab/cache/json_cache.rb b/lib/gitlab/cache/json_cache.rb new file mode 100644 index 00000000000..7450c7e540b --- /dev/null +++ b/lib/gitlab/cache/json_cache.rb @@ -0,0 +1,123 @@ +# frozen_string_literal: true + +module Gitlab + module Cache + class JsonCache + STRATEGY_KEY_COMPONENTS = { + revision: Gitlab.revision, + version: [Gitlab::VERSION, Rails.version] + }.freeze + + def initialize(options = {}) + @backend = options.fetch(:backend, Rails.cache) + @namespace = options.fetch(:namespace, nil) + @cache_key_strategy = options.fetch(:cache_key_strategy, :revision) + end + + def active? + if backend.respond_to?(:active?) + backend.active? + else + true + end + end + + def expire(key) + backend.delete(cache_key(key)) + end + + def read(key, klass = nil) + value = read_raw(key) + value = parse_value(value, klass) unless value.nil? + value + end + + def write(key, value, options = nil) + write_raw(key, value, options) + end + + def fetch(key, options = {}) + klass = options.delete(:as) + value = read(key, klass) + + return value unless value.nil? + + value = yield + + write(key, value, options) + + value + end + + private + + attr_reader :backend, :namespace, :cache_key_strategy + + def cache_key(key) + expanded_cache_key(key).compact.join(':').freeze + end + + def write_raw(_key, _value, _options) + raise NoMethodError + end + + def expanded_cache_key(_key) + raise NoMethodError + end + + def read_raw(_key) + raise NoMethodError + end + + def parse_value(value, klass) + case value + when Hash then parse_entry(value, klass) + when Array then parse_entries(value, klass) + else + value + end + end + + def parse_entry(raw, klass) + return unless valid_entry?(raw, klass) + return klass.new(raw) unless klass.ancestors.include?(ActiveRecord::Base) + + # When the cached value is a persisted instance of ActiveRecord::Base in + # some cases a relation can return an empty collection because scope.none! + # is being applied on ActiveRecord::Associations::CollectionAssociation#scope + # when the new_record? method incorrectly returns false. + # + # See https://gitlab.com/gitlab-org/gitlab/issues/9903#note_145329964 + klass.allocate.init_with(encode_for(klass, raw)) + end + + def encode_for(klass, raw) + # We have models that leave out some fields from the JSON export for + # security reasons, e.g. models that include the CacheMarkdownField. + # The ActiveRecord::AttributeSet we build from raw does know about + # these columns so we need manually set them. + missing_attributes = (klass.columns.map(&:name) - raw.keys) + missing_attributes.each { |column| raw[column] = nil } + + coder = {} + klass.new(raw).encode_with(coder) + coder["new_record"] = new_record?(raw, klass) + coder + end + + def new_record?(raw, klass) + raw.fetch(klass.primary_key, nil).blank? + end + + def valid_entry?(raw, klass) + return false unless klass && raw.is_a?(Hash) + + (raw.keys - klass.attribute_names).empty? + end + + def parse_entries(values, klass) + values.filter_map { |value| parse_entry(value, klass) } + end + end + end +end diff --git a/lib/gitlab/cache/json_caches/json_keyed.rb b/lib/gitlab/cache/json_caches/json_keyed.rb new file mode 100644 index 00000000000..701a49c23de --- /dev/null +++ b/lib/gitlab/cache/json_caches/json_keyed.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Cache + module JsonCaches + class JsonKeyed < JsonCache + private + + def expanded_cache_key(key) + [namespace, key] + end + + def write_raw(key, value, options = nil) + raw_value = {} + + begin + read_value = backend.read(cache_key(key)) + read_value = Gitlab::Json.parse(read_value.to_s) unless read_value.nil? + raw_value = read_value if read_value.is_a?(Hash) + rescue JSON::ParserError + end + + raw_value[strategy_key_component] = value + backend.write(cache_key(key), raw_value.to_json, options) + end + + def read_raw(key) + value = backend.read(cache_key(key)) + value = Gitlab::Json.parse(value.to_s) unless value.nil? + value[strategy_key_component] if value.is_a?(Hash) + rescue JSON::ParserError + nil + end + + def strategy_key_component + Array.wrap(STRATEGY_KEY_COMPONENTS.fetch(cache_key_strategy)).compact.join(':').freeze + end + end + end + end +end diff --git a/lib/gitlab/cache/json_caches/redis_keyed.rb b/lib/gitlab/cache/json_caches/redis_keyed.rb new file mode 100644 index 00000000000..92709adef63 --- /dev/null +++ b/lib/gitlab/cache/json_caches/redis_keyed.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Cache + module JsonCaches + class RedisKeyed < JsonCache + private + + def expanded_cache_key(key) + [namespace, key, *strategy_key_component] + end + + def write_raw(key, value, options) + backend.write(cache_key(key), value.to_json, options) + end + + def read_raw(key) + value = backend.read(cache_key(key)) + value = Gitlab::Json.parse(value.to_s) unless value.nil? + value + rescue JSON::ParserError + nil + end + + def strategy_key_component + STRATEGY_KEY_COMPONENTS.fetch(cache_key_strategy) + end + end + end + end +end diff --git a/lib/gitlab/checks/branch_check.rb b/lib/gitlab/checks/branch_check.rb index fa7c4972c91..8be1e1716ec 100644 --- a/lib/gitlab/checks/branch_check.rb +++ b/lib/gitlab/checks/branch_check.rb @@ -13,7 +13,8 @@ module Gitlab create_protected_branch: 'You are not allowed to create protected branches on this project.', invalid_commit_create_protected_branch: 'You can only use an existing protected branch ref as the basis of a new protected branch.', non_web_create_protected_branch: 'You can only create protected branches using the web interface and API.', - prohibited_hex_branch_name: 'You cannot create a branch with a 40-character hexadecimal branch name.' + prohibited_hex_branch_name: 'You cannot create a branch with a 40-character hexadecimal branch name.', + invalid_branch_name: 'You cannot create a branch with an invalid name.' }.freeze LOG_MESSAGES = { @@ -45,6 +46,10 @@ module Gitlab if branch_name =~ %r{\A\h{40}(/|\z)} raise GitAccess::ForbiddenError, ERROR_MESSAGES[:prohibited_hex_branch_name] end + + unless Gitlab::GitRefValidator.validate(branch_name) + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:invalid_branch_name] + end end def protected_branch_checks diff --git a/lib/gitlab/checks/diff_check.rb b/lib/gitlab/checks/diff_check.rb index 083c2448a0a..1186b532baf 100644 --- a/lib/gitlab/checks/diff_check.rb +++ b/lib/gitlab/checks/diff_check.rb @@ -18,7 +18,10 @@ module Gitlab return unless should_run_validations? return if commits.empty? - paths = project.repository.find_changed_paths(commits.map(&:sha)) + paths = project.repository.find_changed_paths( + commits.map(&:sha), merge_commit_diff_mode: :all_parents + ) + paths.each do |path| validate_path(path) end diff --git a/lib/gitlab/ci/badge/release/latest_release.rb b/lib/gitlab/ci/badge/release/latest_release.rb index 8d84a54787b..8f247006f1a 100644 --- a/lib/gitlab/ci/badge/release/latest_release.rb +++ b/lib/gitlab/ci/badge/release/latest_release.rb @@ -19,7 +19,8 @@ module Gitlab::Ci @release = ::ReleasesFinder.new( project, current_user, - order_by: opts[:order_by]).execute.first + order_by_for_latest: opts[:order_by], + latest: true).execute.first end def entity diff --git a/lib/gitlab/ci/build/rules.rb b/lib/gitlab/ci/build/rules.rb index bc7aad1b186..17b9f30db33 100644 --- a/lib/gitlab/ci/build/rules.rb +++ b/lib/gitlab/ci/build/rules.rb @@ -6,14 +6,15 @@ module Gitlab class Rules include ::Gitlab::Utils::StrongMemoize - Result = Struct.new(:when, :start_in, :allow_failure, :variables, :needs, :errors) do + Result = Struct.new(:when, :start_in, :allow_failure, :variables, :needs, :errors, keyword_init: true) do def build_attributes + needs_job = needs&.dig(:job) { when: self.when, options: { start_in: start_in }.compact, allow_failure: allow_failure, - scheduling_type: (:dag if needs), - needs_attributes: needs&.[](:job) + scheduling_type: (:dag if needs_job.present?), + needs_attributes: needs_job }.compact end @@ -29,20 +30,25 @@ module Gitlab def evaluate(pipeline, context) if @rule_list.nil? - Result.new(@default_when) + Result.new(when: @default_when) elsif matched_rule = match_rule(pipeline, context) - Result.new( - matched_rule.attributes[:when] || @default_when, - matched_rule.attributes[:start_in], - matched_rule.attributes[:allow_failure], - matched_rule.attributes[:variables], - (matched_rule.attributes[:needs] if Feature.enabled?(:introduce_rules_with_needs, pipeline.project)) + result = Result.new( + when: matched_rule.attributes[:when] || @default_when, + start_in: matched_rule.attributes[:start_in], + allow_failure: matched_rule.attributes[:allow_failure], + variables: matched_rule.attributes[:variables] ) + + if Feature.enabled?(:introduce_rules_with_needs, pipeline.project) + result.needs = matched_rule.attributes[:needs] + end + + result else - Result.new('never') + Result.new(when: 'never') end rescue Rule::Clause::ParseError => e - Result.new('never', nil, nil, nil, nil, [e.message]) + Result.new(when: 'never', errors: [e.message]) end private diff --git a/lib/gitlab/ci/config/entry/cache.rb b/lib/gitlab/ci/config/entry/cache.rb index b3ff74c14da..b3862b3f186 100644 --- a/lib/gitlab/ci/config/entry/cache.rb +++ b/lib/gitlab/ci/config/entry/cache.rb @@ -10,7 +10,7 @@ module Gitlab include ::Gitlab::Config::Entry::Attributable ALLOWED_KEYS = %i[key untracked paths when policy unprotect fallback_keys].freeze - ALLOWED_POLICY = %w[pull-push push pull].freeze + ALLOWED_POLICY = /pull-push|push|pull|\$\w{1,255}*/ DEFAULT_POLICY = 'pull-push' ALLOWED_WHEN = %w[on_success on_failure always].freeze DEFAULT_WHEN = 'on_success' @@ -18,9 +18,9 @@ module Gitlab validations do validates :config, type: Hash, allowed_keys: ALLOWED_KEYS - validates :policy, type: String, allow_blank: true, inclusion: { - in: ALLOWED_POLICY, - message: "should be one of: #{ALLOWED_POLICY.join(', ')}" + validates :policy, type: String, allow_blank: true, format: { + with: ALLOWED_POLICY, + message: "should be a variable or one of: pull-push, push, pull" } with_options allow_nil: true do diff --git a/lib/gitlab/ci/config/entry/include/rules/rule.rb b/lib/gitlab/ci/config/entry/include/rules/rule.rb index fa99a7204d6..9cdbd8cd037 100644 --- a/lib/gitlab/ci/config/entry/include/rules/rule.rb +++ b/lib/gitlab/ci/config/entry/include/rules/rule.rb @@ -9,10 +9,14 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[if exists].freeze + ALLOWED_KEYS = %i[if exists when].freeze + ALLOWED_WHEN = %w[never always].freeze - attributes :if, :exists + attributes :if, :exists, :when + # Include rules are validated before Entry validations. This is because + # the include files are expanded before `compose!` runs in Ci::Config. + # The actual validation logic is in lib/gitlab/ci/config/external/rules.rb. validations do validates :config, presence: true validates :config, type: { with: Hash } diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb index 6b635cdf33b..61d95c8d4e6 100644 --- a/lib/gitlab/ci/config/external/file/base.rb +++ b/lib/gitlab/ci/config/external/file/base.rb @@ -130,8 +130,7 @@ module Gitlab strong_memoize_attr :content_hash def interpolator - External::Interpolator - .new(content_result, content_inputs, context) + Yaml::Interpolator.new(content_result, content_inputs, context) end strong_memoize_attr :interpolator diff --git a/lib/gitlab/ci/config/external/file/project.rb b/lib/gitlab/ci/config/external/file/project.rb index 16a6bc8a692..de726b57053 100644 --- a/lib/gitlab/ci/config/external/file/project.rb +++ b/lib/gitlab/ci/config/external/file/project.rb @@ -68,8 +68,6 @@ module Gitlab private def project - return legacy_project if ::Feature.disabled?(:ci_batch_project_includes_context, context.project) - # Although we use `where_full_path_in`, this BatchLoader does not reduce the number of queries to 1. # That's because we use it in the `can_access_local_content?` and `sha` BatchLoaders # as the `for` parameter. And this loads the project immediately. @@ -83,10 +81,6 @@ module Gitlab end def can_access_local_content? - if ::Feature.disabled?(:ci_batch_project_includes_context, context.project) - return legacy_can_access_local_content? - end - return if project.nil? # We are force-loading the project with the `itself` method @@ -103,7 +97,6 @@ module Gitlab end def sha - return legacy_sha if ::Feature.disabled?(:ci_batch_project_includes_context, context.project) return if project.nil? # with `itself`, we are force-loading the project @@ -128,26 +121,6 @@ module Gitlab end end - def legacy_project - strong_memoize(:legacy_project) do - ::Project.find_by_full_path(project_name) - end - end - - def legacy_can_access_local_content? - strong_memoize(:legacy_can_access_local_content) do - context.logger.instrument(:config_file_project_validate_access) do - Ability.allowed?(context.user, :download_code, project) - end - end - end - - def legacy_sha - strong_memoize(:legacy_sha) do - project.commit(ref_name).try(:sha) - end - end - override :expand_context_attrs def expand_context_attrs { diff --git a/lib/gitlab/ci/config/external/interpolator.rb b/lib/gitlab/ci/config/external/interpolator.rb deleted file mode 100644 index f8af77fb246..00000000000 --- a/lib/gitlab/ci/config/external/interpolator.rb +++ /dev/null @@ -1,127 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - class Config - module External - ## - # Config::External::Interpolation perform includable file interpolation, and surfaces all possible interpolation - # errors. It is designed to provide an external file's validation context too. - # - class Interpolator - include ::Gitlab::Utils::StrongMemoize - - attr_reader :config, :args, :ctx, :errors - - def initialize(config, args, ctx = nil) - @config = config - @args = args.to_h - @ctx = ctx - @errors = [] - - validate! - end - - def valid? - @errors.none? - end - - def ready? - ## - # Interpolation is ready when it has been either interrupted by an error or finished with a result. - # - @result || @errors.any? - end - - def interpolate? - enabled? && has_header? && valid? - end - - def has_header? - config.has_header? && config.header.present? - end - - def to_hash - @result.to_h - end - - def error_message - # Interpolator can have multiple error messages, like: ["interpolation interrupted by errors", "unknown - # interpolation key: `abc`"] ? - # - # We are joining them together into a single one, because only one error can be surfaced when an external - # file gets included and is invalid. The limit to three error messages combined is more than required. - # - @errors.first(3).join(', ') - end - - ## - # TODO Add `instrument.logger` instrumentation blocks: - # https://gitlab.com/gitlab-org/gitlab/-/issues/396722 - # - def interpolate! - return {} unless valid? - return @result ||= content.to_h unless interpolate? - - return @errors.concat(header.errors) unless header.valid? - return @errors.concat(inputs.errors) unless inputs.valid? - return @errors.concat(context.errors) unless context.valid? - return @errors.concat(template.errors) unless template.valid? - - if ctx&.user - ::Gitlab::UsageDataCounters::HLLRedisCounter.track_event('ci_interpolation_users', values: ctx.user.id) - end - - @result ||= template.interpolated.to_h.deep_symbolize_keys - end - strong_memoize_attr :interpolate! - - private - - def validate! - return errors.push('content does not have a valid YAML syntax') unless config.valid? - - return unless has_header? && !enabled? - - errors.push('can not evaluate included file because interpolation is disabled') - end - - def enabled? - return false if ctx.nil? - - ::Feature.enabled?(:ci_includable_files_interpolation, ctx.project) - end - - def header - @entry ||= Ci::Config::Header::Root.new(config.header).tap do |header| - header.key = 'header' - - header.compose! - end - end - - def content - @content ||= config.content - end - - def spec - @spec ||= header.inputs_value - end - - def inputs - @inputs ||= Ci::Input::Inputs.new(spec, args) - end - - def context - @context ||= Ci::Interpolation::Context.new({ inputs: inputs.to_hash }) - end - - def template - @template ||= ::Gitlab::Ci::Interpolation::Template - .new(content, context) - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/external/mapper/verifier.rb b/lib/gitlab/ci/config/external/mapper/verifier.rb index 3472f2c581a..95975e4661b 100644 --- a/lib/gitlab/ci/config/external/mapper/verifier.rb +++ b/lib/gitlab/ci/config/external/mapper/verifier.rb @@ -11,10 +11,6 @@ module Gitlab # rubocop: disable Metrics/CyclomaticComplexity def process_without_instrumentation(files) - if ::Feature.disabled?(:ci_batch_project_includes_context, context.project) - return legacy_process_without_instrumentation(files) - end - files.each do |file| # When running a pipeline, some Ci::ProjectConfig sources prepend the config content with an # "internal" `include`. We use this condition to exclude that `include` from the included file set. @@ -45,30 +41,6 @@ module Gitlab end # rubocop: enable Metrics/CyclomaticComplexity - def legacy_process_without_instrumentation(files) - files.each do |file| - # When running a pipeline, some Ci::ProjectConfig sources prepend the config content with an - # "internal" `include`. We use this condition to exclude that `include` from the included file set. - context.expandset << file unless context.internal_include? - verify_max_includes! - - verify_execution_time! - - file.validate_location! - file.validate_context! if file.valid? - file.content if file.valid? - end - - # We do not combine the loops because we need to load the content of all files before continuing - # to call `BatchLoader` for all locations. - files.each do |file| # rubocop:disable Style/CombinableLoops - verify_execution_time! - - file.validate_content! if file.valid? - file.load_and_validate_expanded_hash! if file.valid? - end - end - def verify_max_includes! return if context.expandset.count <= context.max_includes diff --git a/lib/gitlab/ci/config/external/rules.rb b/lib/gitlab/ci/config/external/rules.rb index 95470537de3..134306332e6 100644 --- a/lib/gitlab/ci/config/external/rules.rb +++ b/lib/gitlab/ci/config/external/rules.rb @@ -6,6 +6,7 @@ module Gitlab module External class Rules ALLOWED_KEYS = Entry::Include::Rules::Rule::ALLOWED_KEYS + ALLOWED_WHEN = Entry::Include::Rules::Rule::ALLOWED_WHEN InvalidIncludeRulesError = Class.new(Mapper::Error) @@ -16,7 +17,17 @@ module Gitlab end def evaluate(context) - Result.new(@rule_list.nil? || match_rule(context)) + if Feature.enabled?(:ci_support_include_rules_when_never, context.project) + if @rule_list.nil? + Result.new('always') + elsif matched_rule = match_rule(context) + Result.new(matched_rule.attributes[:when]) + else + Result.new('never') + end + else + LegacyResult.new(@rule_list.nil? || match_rule(context)) + end end private @@ -29,13 +40,23 @@ module Gitlab return unless rule_hashes.is_a?(Array) rule_hashes.each do |rule_hash| - next if (rule_hash.keys - ALLOWED_KEYS).empty? + next if (rule_hash.keys - ALLOWED_KEYS).empty? && valid_when?(rule_hash) raise InvalidIncludeRulesError, "invalid include rule: #{rule_hash}" end end - Result = Struct.new(:result) do + def valid_when?(rule_hash) + rule_hash[:when].nil? || rule_hash[:when].in?(ALLOWED_WHEN) + end + + Result = Struct.new(:when) do + def pass? + self.when != 'never' + end + end + + LegacyResult = Struct.new(:result) do def pass? !!result end diff --git a/lib/gitlab/ci/config/yaml.rb b/lib/gitlab/ci/config/yaml.rb index 729e7e3ac05..f74ef95a832 100644 --- a/lib/gitlab/ci/config/yaml.rb +++ b/lib/gitlab/ci/config/yaml.rb @@ -4,51 +4,6 @@ module Gitlab module Ci class Config module Yaml - AVAILABLE_TAGS = [Config::Yaml::Tags::Reference].freeze - MAX_DOCUMENTS = 2 - - class Loader - def initialize(content, project: nil) - @content = content - @project = project - end - - def load! - ensure_custom_tags - - if project.present? && ::Feature.enabled?(:ci_multi_doc_yaml, project) - ::Gitlab::Config::Loader::MultiDocYaml.new( - content, - max_documents: MAX_DOCUMENTS, - additional_permitted_classes: AVAILABLE_TAGS, - reject_empty: true - ).load! - else - ::Gitlab::Config::Loader::Yaml - .new(content, additional_permitted_classes: AVAILABLE_TAGS) - .load! - end - end - - def to_result - Yaml::Result.new(config: load!, error: nil) - rescue ::Gitlab::Config::Loader::FormatError => e - Yaml::Result.new(error: e) - end - - private - - attr_reader :content, :project - - def ensure_custom_tags - @ensure_custom_tags ||= begin - AVAILABLE_TAGS.each { |klass| Psych.add_tag(klass.tag, klass) } - - true - end - end - end - class << self def load!(content, project: nil) Loader.new(content, project: project).to_result.then do |result| diff --git a/lib/gitlab/ci/config/yaml/interpolator.rb b/lib/gitlab/ci/config/yaml/interpolator.rb new file mode 100644 index 00000000000..4ae191dfedf --- /dev/null +++ b/lib/gitlab/ci/config/yaml/interpolator.rb @@ -0,0 +1,127 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Yaml + ## + # Config::Yaml::Interpolation performs includable file interpolation, and surfaces all possible interpolation + # errors. It is designed to provide an external file's validation context too. + # + class Interpolator + include ::Gitlab::Utils::StrongMemoize + + attr_reader :config, :args, :ctx, :errors + + def initialize(config, args, ctx = nil) + @config = config + @args = args.to_h + @ctx = ctx + @errors = [] + + validate! + end + + def valid? + @errors.none? + end + + def ready? + ## + # Interpolation is ready when it has been either interrupted by an error or finished with a result. + # + @result || @errors.any? + end + + def interpolate? + enabled? && has_header? && valid? + end + + def has_header? + config.has_header? && config.header.present? + end + + def to_hash + @result.to_h + end + + def error_message + # Interpolator can have multiple error messages, like: ["interpolation interrupted by errors", "unknown + # interpolation key: `abc`"] ? + # + # We are joining them together into a single one, because only one error can be surfaced when an external + # file gets included and is invalid. The limit to three error messages combined is more than required. + # + @errors.first(3).join(', ') + end + + ## + # TODO Add `instrument.logger` instrumentation blocks: + # https://gitlab.com/gitlab-org/gitlab/-/issues/396722 + # + def interpolate! + return {} unless valid? + return @result ||= content.to_h unless interpolate? + + return @errors.concat(header.errors) unless header.valid? + return @errors.concat(inputs.errors) unless inputs.valid? + return @errors.concat(context.errors) unless context.valid? + return @errors.concat(template.errors) unless template.valid? + + if ctx&.user + ::Gitlab::UsageDataCounters::HLLRedisCounter.track_event('ci_interpolation_users', values: ctx.user.id) + end + + @result ||= template.interpolated.to_h.deep_symbolize_keys + end + strong_memoize_attr :interpolate! + + private + + def validate! + return errors.push('content does not have a valid YAML syntax') unless config.valid? + + return unless has_header? && !enabled? + + errors.push('can not evaluate included file because interpolation is disabled') + end + + def enabled? + return false if ctx.nil? + + ::Feature.enabled?(:ci_includable_files_interpolation, ctx.project) + end + + def header + @entry ||= Ci::Config::Header::Root.new(config.header).tap do |header| + header.key = 'header' + + header.compose! + end + end + + def content + @content ||= config.content + end + + def spec + @spec ||= header.inputs_value + end + + def inputs + @inputs ||= Ci::Input::Inputs.new(spec, args) + end + + def context + @context ||= Ci::Interpolation::Context.new({ inputs: inputs.to_hash }) + end + + def template + @template ||= ::Gitlab::Ci::Interpolation::Template + .new(content, context) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/yaml/loader.rb b/lib/gitlab/ci/config/yaml/loader.rb new file mode 100644 index 00000000000..924a1f2e46b --- /dev/null +++ b/lib/gitlab/ci/config/yaml/loader.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Yaml + class Loader + AVAILABLE_TAGS = [Config::Yaml::Tags::Reference].freeze + MAX_DOCUMENTS = 2 + + def initialize(content, project: nil) + @content = content + @project = project + end + + def to_result + Yaml::Result.new(config: load!, error: nil) + rescue ::Gitlab::Config::Loader::FormatError => e + Yaml::Result.new(error: e) + end + + private + + attr_reader :content, :project + + def ensure_custom_tags + @ensure_custom_tags ||= begin + AVAILABLE_TAGS.each { |klass| Psych.add_tag(klass.tag, klass) } + + true + end + end + + def load! + ensure_custom_tags + + ::Gitlab::Config::Loader::MultiDocYaml.new( + content, + max_documents: MAX_DOCUMENTS, + additional_permitted_classes: AVAILABLE_TAGS, + reject_empty: true + ).load! + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/yaml/result.rb b/lib/gitlab/ci/config/yaml/result.rb index 33f9a454106..6b53adc3a57 100644 --- a/lib/gitlab/ci/config/yaml/result.rb +++ b/lib/gitlab/ci/config/yaml/result.rb @@ -31,7 +31,7 @@ module Gitlab def content return @config.last if has_header? - @config.first + @config.first || {} end end end diff --git a/lib/gitlab/ci/decompressed_gzip_size_validator.rb b/lib/gitlab/ci/decompressed_gzip_size_validator.rb index a92f3007671..9b7b5f0dd66 100644 --- a/lib/gitlab/ci/decompressed_gzip_size_validator.rb +++ b/lib/gitlab/ci/decompressed_gzip_size_validator.rb @@ -63,7 +63,7 @@ module Gitlab end def validate_archive_path - Gitlab::Utils.check_path_traversal!(archive_path) + Gitlab::PathTraversal.check_path_traversal!(archive_path) raise(ServiceError, 'Archive path is a symlink') if File.lstat(archive_path).symlink? raise(ServiceError, 'Archive path is not a file') unless File.file?(archive_path) diff --git a/lib/gitlab/ci/jwt_v2.rb b/lib/gitlab/ci/jwt_v2.rb index aff30455d09..9e71a9e8e91 100644 --- a/lib/gitlab/ci/jwt_v2.rb +++ b/lib/gitlab/ci/jwt_v2.rb @@ -42,11 +42,36 @@ module Gitlab end def custom_claims - super.merge( + additional_claims = { runner_id: runner&.id, runner_environment: runner_environment, sha: pipeline.sha + } + + if Feature.enabled?(:ci_jwt_v2_ref_uri_claim, pipeline.project) + additional_claims[:ci_config_ref_uri] = ci_config_ref_uri + end + + super.merge(additional_claims) + end + + def ci_config_ref_uri + project_config = Gitlab::Ci::ProjectConfig.new( + project: project, + sha: pipeline.sha, + pipeline_source: pipeline.source&.to_sym, + pipeline_source_bridge: pipeline.source_bridge ) + + return unless project_config&.source == :repository_source + + "#{project_config.url}@#{pipeline.source_ref_path}" + + # Errors are rescued to mitigate risk. This can be removed if no errors are observed. + # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/117923#note_1387660746 for context. + rescue StandardError => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, pipeline_id: pipeline.id) + nil end def runner_environment diff --git a/lib/gitlab/ci/parsers/security/common.rb b/lib/gitlab/ci/parsers/security/common.rb index 447136df81f..21408beb8cb 100644 --- a/lib/gitlab/ci/parsers/security/common.rb +++ b/lib/gitlab/ci/parsers/security/common.rb @@ -279,7 +279,6 @@ module Gitlab end def finding_name(data, identifiers, location) - return data['message'] if data['message'].present? return data['name'] if data['name'].present? identifier = identifiers.find(&:cve?) || identifiers.find(&:cwe?) || identifiers.first diff --git a/lib/gitlab/ci/project_config.rb b/lib/gitlab/ci/project_config.rb index 00b2ad58428..ffff2da9d7a 100644 --- a/lib/gitlab/ci/project_config.rb +++ b/lib/gitlab/ci/project_config.rb @@ -25,7 +25,7 @@ module Gitlab @config = find_config(project, sha, custom_content, pipeline_source, pipeline_source_bridge) end - delegate :content, :source, to: :@config, allow_nil: true + delegate :content, :source, :url, to: :@config, allow_nil: true delegate :internal_include_prepended?, to: :@config def exists? diff --git a/lib/gitlab/ci/project_config/repository.rb b/lib/gitlab/ci/project_config/repository.rb index 272425fd546..7dfd528fd6f 100644 --- a/lib/gitlab/ci/project_config/repository.rb +++ b/lib/gitlab/ci/project_config/repository.rb @@ -20,6 +20,11 @@ module Gitlab :repository_source end + override :url + def url + File.join(Settings.build_ci_component_fqdn, project.full_path, '//', ci_config_path) + end + private def file_in_repository? diff --git a/lib/gitlab/ci/project_config/source.rb b/lib/gitlab/ci/project_config/source.rb index 9a4a6394fa1..68853ca8296 100644 --- a/lib/gitlab/ci/project_config/source.rb +++ b/lib/gitlab/ci/project_config/source.rb @@ -5,6 +5,7 @@ module Gitlab class ProjectConfig class Source include Gitlab::Utils::StrongMemoize + extend ::Gitlab::Utils::Override def initialize(project, sha, custom_content, pipeline_source, pipeline_source_bridge) @project = project @@ -33,6 +34,10 @@ module Gitlab raise NotImplementedError end + def url + nil + end + private attr_reader :project, :sha, :custom_content, :pipeline_source, :pipeline_source_bridge diff --git a/lib/gitlab/ci/reports/security/finding.rb b/lib/gitlab/ci/reports/security/finding.rb index bf48c7d0bb7..d439149158a 100644 --- a/lib/gitlab/ci/reports/security/finding.rb +++ b/lib/gitlab/ci/reports/security/finding.rb @@ -82,7 +82,6 @@ module Gitlab details signatures description - message cve solution ].index_with do |key| @@ -174,10 +173,6 @@ module Gitlab original_data['description'] end - def message - original_data['message'] - end - def solution original_data['solution'] end diff --git a/lib/gitlab/ci/runner_instructions.rb b/lib/gitlab/ci/runner_instructions.rb index 1bf015a0aa0..3040acc1eb4 100644 --- a/lib/gitlab/ci/runner_instructions.rb +++ b/lib/gitlab/ci/runner_instructions.rb @@ -112,3 +112,5 @@ module Gitlab end end end + +::Gitlab::Ci::RunnerInstructions.prepend_mod diff --git a/lib/gitlab/ci/secure_files/migration_helper.rb b/lib/gitlab/ci/secure_files/migration_helper.rb new file mode 100644 index 00000000000..13796f2476e --- /dev/null +++ b/lib/gitlab/ci/secure_files/migration_helper.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module SecureFiles + class MigrationHelper + class << self + def migrate_to_remote_storage(&block) + migrate_in_batches( + ::Ci::SecureFile.with_files_stored_locally, + ::Ci::SecureFileUploader::Store::REMOTE, + &block + ) + end + + private + + def batch_size + ENV.fetch('MIGRATION_BATCH_SIZE', 10).to_i + end + + def migrate_in_batches(files, store, &block) + files.find_each(batch_size: batch_size) do |file| # rubocop:disable CodeReuse/ActiveRecord + file.file.migrate!(store) + + yield file if block + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/waiting_for_approval.rb b/lib/gitlab/ci/status/build/waiting_for_approval.rb index ac3f5838d26..c5fb32034ce 100644 --- a/lib/gitlab/ci/status/build/waiting_for_approval.rb +++ b/lib/gitlab/ci/status/build/waiting_for_approval.rb @@ -5,44 +5,14 @@ module Gitlab module Status module Build class WaitingForApproval < Status::Extended - def illustration - { - image: 'illustrations/manual_action.svg', - size: 'svg-394', - title: _('Waiting for approval'), - content: _("This job deploys to the protected environment \"%{environment}\" which requires approvals.") % { environment: subject.deployment&.environment&.name } - } - end - - def has_action? - true - end - - def action_icon - nil - end - - def action_title - nil - end - - def action_button_title - _('Go to environments page to approve or reject') - end - - def action_path - project_environments_path(subject.project) - end - - def action_method - :get - end - + ## Extended in EE def self.matches?(build, user) - build.waiting_for_deployment_approval? + false end end end end end end + +Gitlab::Ci::Status::Build::WaitingForApproval.prepend_mod_with('Gitlab::Ci::Status::Build::WaitingForApproval') diff --git a/lib/gitlab/ci/status/scheduled.rb b/lib/gitlab/ci/status/scheduled.rb index e9068c326cf..8526becfef9 100644 --- a/lib/gitlab/ci/status/scheduled.rb +++ b/lib/gitlab/ci/status/scheduled.rb @@ -5,11 +5,11 @@ module Gitlab module Status class Scheduled < Status::Core def text - s_('CiStatusText|delayed') + s_('CiStatusText|scheduled') end def label - s_('CiStatusLabel|delayed') + s_('CiStatusLabel|scheduled') end def icon diff --git a/lib/gitlab/ci/status/success_warning.rb b/lib/gitlab/ci/status/success_warning.rb index 47623ad945f..84a0e52f518 100644 --- a/lib/gitlab/ci/status/success_warning.rb +++ b/lib/gitlab/ci/status/success_warning.rb @@ -9,7 +9,7 @@ module Gitlab # class SuccessWarning < Status::Extended def text - s_('CiStatusText|passed') + s_('CiStatusText|warning') end def label diff --git a/lib/gitlab/ci/templates/Android.gitlab-ci.yml b/lib/gitlab/ci/templates/Android.gitlab-ci.yml index b8a4c59c233..95cdf9b9953 100644 --- a/lib/gitlab/ci/templates/Android.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Android.gitlab-ci.yml @@ -6,53 +6,51 @@ # Read more about this script on this blog post https://about.gitlab.com/2018/10/24/setting-up-gitlab-ci-for-android-projects/, by Jason Lenny # If you are interested in using Android with FastLane for publishing take a look at the Android-Fastlane template. -image: openjdk:8-jdk +image: eclipse-temurin:17-jdk-jammy variables: # ANDROID_COMPILE_SDK is the version of Android you're compiling with. # It should match compileSdkVersion. - ANDROID_COMPILE_SDK: "29" + ANDROID_COMPILE_SDK: "33" # ANDROID_BUILD_TOOLS is the version of the Android build tools you are using. # It should match buildToolsVersion. - ANDROID_BUILD_TOOLS: "29.0.3" + ANDROID_BUILD_TOOLS: "33.0.2" # It's what version of the command line tools we're going to download from the official site. # Official Site-> https://developer.android.com/studio/index.html # There, look down below at the cli tools only, sdk tools package is of format: # commandlinetools-os_type-ANDROID_SDK_TOOLS_latest.zip # when the script was last modified for latest compileSdkVersion, it was which is written down below - ANDROID_SDK_TOOLS: "6514223" + ANDROID_SDK_TOOLS: "9477386" # Packages installation before running script before_script: - apt-get --quiet update --yes - - apt-get --quiet install --yes wget tar unzip lib32stdc++6 lib32z1 + - apt-get --quiet install --yes wget unzip # Setup path as android_home for moving/exporting the downloaded sdk into it - - export ANDROID_HOME="${PWD}/android-home" + - export ANDROID_HOME="${PWD}/android-sdk-root" # Create a new directory at specified location - install -d $ANDROID_HOME # Here we are installing androidSDK tools from official source, # (the key thing here is the url from where you are downloading these sdk tool for command line, so please do note this url pattern there and here as well) # after that unzipping those tools and # then running a series of SDK manager commands to install necessary android SDK packages that'll allow the app to build - - wget --output-document=$ANDROID_HOME/cmdline-tools.zip https://dl.google.com/android/repository/commandlinetools-linux-${ANDROID_SDK_TOOLS}_latest.zip - # move to the archive at ANDROID_HOME - - pushd $ANDROID_HOME - - unzip -d cmdline-tools cmdline-tools.zip - - popd - - export PATH=$PATH:${ANDROID_HOME}/cmdline-tools/tools/bin/ + - wget --no-verbose --output-document=$ANDROID_HOME/cmdline-tools.zip https://dl.google.com/android/repository/commandlinetools-linux-${ANDROID_SDK_TOOLS}_latest.zip + - unzip -q -d "$ANDROID_HOME/cmdline-tools" "$ANDROID_HOME/cmdline-tools.zip" + - mv -T "$ANDROID_HOME/cmdline-tools/cmdline-tools" "$ANDROID_HOME/cmdline-tools/tools" + - export PATH=$PATH:$ANDROID_HOME/cmdline-tools/latest/bin:$ANDROID_HOME/cmdline-tools/tools/bin # Nothing fancy here, just checking sdkManager version - sdkmanager --version # use yes to accept all licenses - - yes | sdkmanager --sdk_root=${ANDROID_HOME} --licenses || true - - sdkmanager --sdk_root=${ANDROID_HOME} "platforms;android-${ANDROID_COMPILE_SDK}" - - sdkmanager --sdk_root=${ANDROID_HOME} "platform-tools" - - sdkmanager --sdk_root=${ANDROID_HOME} "build-tools;${ANDROID_BUILD_TOOLS}" + - yes | sdkmanager --licenses > /dev/null || true + - sdkmanager "platforms;android-${ANDROID_COMPILE_SDK}" + - sdkmanager "platform-tools" + - sdkmanager "build-tools;${ANDROID_BUILD_TOOLS}" # Not necessary, but just for surity - chmod +x ./gradlew @@ -64,6 +62,11 @@ lintDebug: stage: build script: - ./gradlew -Pci --console=plain :app:lintDebug -PbuildDir=lint + artifacts: + paths: + - app/lint/reports/lint-results-debug.html + expose_as: "lint-report" + when: always # Make Project assembleDebug: @@ -77,6 +80,7 @@ assembleDebug: # Run all tests, if any fails, interrupt the pipeline(fail it) debugTests: + needs: [lintDebug, assembleDebug] interruptible: true stage: test script: diff --git a/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml b/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml index 7f81755348c..3d053d4d78c 100644 --- a/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml @@ -8,9 +8,9 @@ code_quality: stage: test - image: "cirrusci/flutter:1.22.5" + image: "ghcr.io/cirruslabs/flutter:3.10.3" before_script: - - pub global activate dart_code_metrics + - flutter pub global activate dart_code_metrics - export PATH="$PATH:$HOME/.pub-cache/bin" script: - metrics lib -r codeclimate > gl-code-quality-report.json @@ -20,9 +20,9 @@ code_quality: test: stage: test - image: "cirrusci/flutter:1.22.5" + image: "ghcr.io/cirruslabs/flutter:3.10.3" before_script: - - pub global activate junitreport + - flutter pub global activate junitreport - export PATH="$PATH:$HOME/.pub-cache/bin" script: - flutter test --machine --coverage | tojunit -o report.xml diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index 7a4c65f8c5b..49d3c270bac 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.32.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.34.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml index 7a4c65f8c5b..49d3c270bac 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.32.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.34.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/CF-Provision.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/CF-Provision.gitlab-ci.yml index 6e8cf15204a..de3f688bdb6 100644 --- a/lib/gitlab/ci/templates/Jobs/CF-Provision.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/CF-Provision.gitlab-ci.yml @@ -7,7 +7,7 @@ cloud_formation: script: - gl-cloudformation create-stack rules: - - if: '($AUTO_DEVOPS_PLATFORM_TARGET != "EC2") || ($AUTO_DEVOPS_PLATFORM_TARGET != "ECS")' + - if: '($AUTO_DEVOPS_PLATFORM_TARGET != "EC2") && ($AUTO_DEVOPS_PLATFORM_TARGET != "ECS")' when: never - if: '$CI_KUBERNETES_ACTIVE || $KUBECONFIG' when: never diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index b2ab6704e35..7c9aa82b1ae 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -8,7 +8,7 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE_TAG: "0.94.0" + CODE_QUALITY_IMAGE_TAG: "0.96.0" CODE_QUALITY_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/gitlab-org/ci-cd/codequality:$CODE_QUALITY_IMAGE_TAG" needs: [] script: diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index 4ee5fa74df9..f4a13d61ba2 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.48.2' + DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.50.0' .dast-auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 622b44d78ad..c1a3daa7f5b 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.48.2' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.50.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index 2954ddf8a35..a3c7c6baf02 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.48.2' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.50.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Kaniko.gitlab-ci.yml b/lib/gitlab/ci/templates/Kaniko.gitlab-ci.yml index d46ac97ad1b..d7a6104082d 100644 --- a/lib/gitlab/ci/templates/Kaniko.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Kaniko.gitlab-ci.yml @@ -32,11 +32,11 @@ kaniko-build: VERSION="latest" elif [ -n "$CI_COMMIT_TAG" ];then NOSLASH=$(echo "$CI_COMMIT_TAG" | tr -s / - ) - SANITIZED="${NOSLASH//[^a-zA-Z0-9\-\.]/}" + SANITIZED="${NOSLASH//[^a-zA-Z0-9.-]/}" VERSION="$SANITIZED" else \ NOSLASH=$(echo "$CI_COMMIT_REF_NAME" | tr -s / - ) - SANITIZED="${NOSLASH//[^a-zA-Z0-9\-]/}" + SANITIZED="${NOSLASH//[^a-zA-Z0-9-]/}" VERSION="branch-$SANITIZED" fi export IMAGE_TAG=$CI_REGISTRY_IMAGE:$VERSION diff --git a/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml index 51d2273d41d..eb1c920e11b 100644 --- a/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml @@ -3,8 +3,9 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml -# Full project: https://gitlab.com/pages/brunch -image: node:4.2.2 +default: + # Full project: https://gitlab.com/pages/brunch + image: node:4.2.2 pages: cache: diff --git a/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml index e577a489c55..95eab65e629 100644 --- a/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml @@ -3,8 +3,9 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml -# Full project: https://gitlab.com/pages/doxygen -image: alpine +default: + # Full project: https://gitlab.com/pages/doxygen + image: alpine pages: script: diff --git a/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml index 88ed73b41e7..c58db066d61 100644 --- a/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml @@ -3,13 +3,14 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml -image: node:latest +default: + image: node:latest -# This folder is cached between builds -# https://docs.gitlab.com/ee/ci/yaml/index.html#cache -cache: - paths: - - node_modules/ + # This folder is cached between builds + # https://docs.gitlab.com/ee/ci/yaml/index.html#cache + cache: + paths: + - node_modules/ pages: script: diff --git a/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml index aa86ad2a6ad..313cb99a841 100644 --- a/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml @@ -3,8 +3,9 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml -# Full project: https://gitlab.com/pages/harp -image: node:4.2.2 +default: + # Full project: https://gitlab.com/pages/harp + image: node:4.2.2 pages: cache: diff --git a/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml index b1617e9239c..f27228f2e3c 100644 --- a/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml @@ -3,8 +3,9 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml -# Full project: https://gitlab.com/pages/hexo -image: node:10.15.3 +default: + # Full project: https://gitlab.com/pages/hexo + image: node:10.15.3 pages: script: diff --git a/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml index d6f6e94526e..6a364959db2 100644 --- a/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml @@ -6,7 +6,8 @@ --- # All available Hugo versions are listed here: # https://gitlab.com/pages/hugo/container_registry -image: "${CI_TEMPLATE_REGISTRY_HOST}/pages/hugo:latest" +default: + image: "${CI_TEMPLATE_REGISTRY_HOST}/pages/hugo:latest" variables: GIT_SUBMODULE_STRATEGY: recursive @@ -14,9 +15,8 @@ variables: test: script: - hugo - except: - variables: - - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + rules: + - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH pages: script: @@ -24,7 +24,6 @@ pages: artifacts: paths: - public - only: - variables: - - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH environment: production diff --git a/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml index fba4afca9ed..c3ee4d62359 100644 --- a/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml @@ -3,12 +3,13 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml -# Full project: https://gitlab.com/pages/hyde -image: python:2.7 +default: + # Full project: https://gitlab.com/pages/hyde + image: python:2.7 -cache: - paths: - - vendor/ + cache: + paths: + - vendor/ test: stage: test diff --git a/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml index 57e3ced4dc2..ba8eb81ca22 100644 --- a/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml @@ -13,20 +13,21 @@ # # HowTo at: https://jorge.aguilera.gitlab.io/howtojbake/ -image: java:8 +default: + image: java:8 + + # We use SDKMan as tool for managing versions + before_script: + - apt-get update -qq && apt-get install -y -qq unzip zip + - curl -sSL https://get.sdkman.io | bash + - echo sdkman_auto_answer=true > /root/.sdkman/etc/config + - source /root/.sdkman/bin/sdkman-init.sh + - sdk install jbake $JBAKE_VERSION < /dev/null + - sdk use jbake $JBAKE_VERSION variables: JBAKE_VERSION: 2.5.1 -# We use SDKMan as tool for managing versions -before_script: - - apt-get update -qq && apt-get install -y -qq unzip zip - - curl -sSL https://get.sdkman.io | bash - - echo sdkman_auto_answer=true > /root/.sdkman/etc/config - - source /root/.sdkman/bin/sdkman-init.sh - - sdk install jbake $JBAKE_VERSION < /dev/null - - sdk use jbake $JBAKE_VERSION - # This build job produced the output directory of your site pages: environment: production diff --git a/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml index 8b07454af24..812a08c33fb 100644 --- a/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml @@ -3,18 +3,19 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml -# Template project: https://gitlab.com/pages/jekyll -# Docs: https://docs.gitlab.com/ee/pages/ -image: ruby:2.6 +default: + # Template project: https://gitlab.com/pages/jekyll + # Docs: https://docs.gitlab.com/ee/pages/ + image: ruby:2.6 + + before_script: + - gem install bundler + - bundle install variables: JEKYLL_ENV: production LC_ALL: C.UTF-8 -before_script: - - gem install bundler - - bundle install - test: stage: test script: diff --git a/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml index ad083fcc5db..291c03c0002 100644 --- a/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml @@ -7,29 +7,30 @@ # # Full project: https://github.com/tightenco/jigsaw -image: php:7.2 +default: + image: php:7.2 -# These folders are cached between builds -cache: - paths: - - vendor/ - - node_modules/ + # These folders are cached between builds + cache: + paths: + - vendor/ + - node_modules/ -before_script: - # Update packages - - apt-get update -yqq - # Install dependencies - - apt-get install -yqq gnupg zlib1g-dev libpng-dev - # Install Node 8 - - curl -sL https://deb.nodesource.com/setup_8.x | bash - - - apt-get install -yqq nodejs - # Install php extensions - - docker-php-ext-install zip - # Install Composer and project dependencies - - curl -sS https://getcomposer.org/installer | php - - php composer.phar install - # Install Node dependencies - - npm install + before_script: + # Update packages + - apt-get update -yqq + # Install dependencies + - apt-get install -yqq gnupg zlib1g-dev libpng-dev + # Install Node 8 + - curl -sL https://deb.nodesource.com/setup_8.x | bash - + - apt-get install -yqq nodejs + # Install php extensions + - docker-php-ext-install zip + # Install Composer and project dependencies + - curl -sS https://getcomposer.org/installer | php + - php composer.phar install + # Install Node dependencies + - npm install pages: script: diff --git a/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml index e86337ae23c..e83cf30e999 100644 --- a/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml @@ -3,8 +3,9 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml -# Full project: https://gitlab.com/pages/hyde -image: python:2.7 +default: + # Full project: https://gitlab.com/pages/hyde + image: python:2.7 pages: script: diff --git a/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml index a49e95b62c8..bbbffa7c682 100644 --- a/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml @@ -3,8 +3,9 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml -# Full project: https://gitlab.com/pages/metalsmith -image: node:4.2.2 +default: + # Full project: https://gitlab.com/pages/metalsmith + image: node:4.2.2 pages: cache: diff --git a/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml index d8f036ab4ed..f0a7f88eaf2 100644 --- a/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml @@ -3,12 +3,13 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml -# Full project: https://gitlab.com/pages/middleman -image: ruby:2.6 +default: + # Full project: https://gitlab.com/pages/middleman + image: ruby:2.6 -cache: - paths: - - vendor + cache: + paths: + - vendor test: script: diff --git a/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml index b0511abd109..3160948ec35 100644 --- a/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml @@ -3,8 +3,9 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml -# Full project: https://gitlab.com/pages/nanoc -image: ruby:2.6 +default: + # Full project: https://gitlab.com/pages/nanoc + image: ruby:2.6 pages: script: diff --git a/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml index c89050eede7..395842c4cd1 100644 --- a/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml @@ -3,8 +3,9 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml -# Full project: https://gitlab.com/pages/octopress -image: ruby:2.6 +default: + # Full project: https://gitlab.com/pages/octopress + image: ruby:2.6 pages: script: diff --git a/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml index 3721344b21e..7921a71a89a 100644 --- a/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml @@ -3,8 +3,9 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml -# Full project: https://gitlab.com/pages/pelican -image: python:2.7-alpine +default: + # Full project: https://gitlab.com/pages/pelican + image: python:2.7-alpine pages: script: diff --git a/lib/gitlab/ci/templates/Pages/SwaggerUI.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/SwaggerUI.gitlab-ci.yml index 00efcfa1b32..d75ecc3da01 100644 --- a/lib/gitlab/ci/templates/Pages/SwaggerUI.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/SwaggerUI.gitlab-ci.yml @@ -3,7 +3,13 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/SwaggerUI.gitlab-ci.yml -image: node:10-alpine +default: + image: node:10-alpine + + # These folders are cached between builds + cache: + paths: + - ./node_modules # specify the location of the Open API Specification files within your project # and the filename of the specification that you would like to display by default @@ -11,11 +17,6 @@ variables: DOCS_FOLDER: "api-docs" SPEC_TO_DISPLAY: "my-project_specification_0.0.1.json" -# These folders are cached between builds -cache: - paths: - - ./node_modules - # publishes all files from the $DOCS_FOLDER together with the static version of SwaggerUI # sets the specification file named in $SPEC_TO_DISPLAY to be displayed by default pages: diff --git a/lib/gitlab/ci/templates/Pages/Zola.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Zola.gitlab-ci.yml new file mode 100644 index 00000000000..c2c890846b9 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Zola.gitlab-ci.yml @@ -0,0 +1,30 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Zola.gitlab-ci.yml + +# Prefer to copy-paste this template instead of include it to ensure forward compatibility + +--- +# From: https://www.getzola.org/documentation/deployment/gitlab-pages/ +# Source template is slightly modified to be self-contained + +pages: + image: alpine:latest + variables: + # This variable will ensure that the CI runner pulls in your theme from the submodule + GIT_SUBMODULE_STRATEGY: recursive + before_script: + # Install the zola package from the alpine community repositories + - apk add --update-cache --repository http://dl-cdn.alpinelinux.org/alpine/edge/community/ zola + script: + # Execute zola build + - zola build --base-url "$CI_PAGES_URL" + artifacts: + paths: + # Path of our artifacts + - public + # This config will only publish changes that are pushed on the default branch + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + environment: production diff --git a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml index 792bd7f666b..f10011ab23b 100644 --- a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml @@ -26,11 +26,12 @@ variables: # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" + DAST_IMAGE_SUFFIX: "" dast: stage: dast image: - name: "$SECURE_ANALYZERS_PREFIX/dast:$DAST_VERSION" + name: "$SECURE_ANALYZERS_PREFIX/dast:$DAST_VERSION$DAST_IMAGE_SUFFIX" variables: GIT_STRATEGY: none allow_failure: true @@ -56,6 +57,11 @@ dast: - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME && $REVIEW_DISABLED == '1' when: never + - if: $CI_COMMIT_BRANCH && + $CI_GITLAB_FIPS_MODE == "true" && + $GITLAB_FEATURES =~ /\bdast\b/ + variables: + DAST_IMAGE_SUFFIX: "-fips" - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdast\b/ after_script: diff --git a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml index d1d1c4d7e52..989f9caf601 100644 --- a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml @@ -26,11 +26,12 @@ variables: # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" + DAST_IMAGE_SUFFIX: "" dast: stage: dast image: - name: "$SECURE_ANALYZERS_PREFIX/dast:$DAST_VERSION" + name: "$SECURE_ANALYZERS_PREFIX/dast:$DAST_VERSION$DAST_IMAGE_SUFFIX" variables: GIT_STRATEGY: none allow_failure: true @@ -59,6 +60,12 @@ dast: $REVIEW_DISABLED == '1' when: never + # Add the job to merge request pipelines if there's an open merge request. (FIPS) + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $CI_GITLAB_FIPS_MODE == "true" && + $GITLAB_FEATURES =~ /\bdast\b/ + variables: + DAST_IMAGE_SUFFIX: "-fips" # Add the job to merge request pipelines if there's an open merge request. - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $GITLAB_FEATURES =~ /\bdast\b/ @@ -67,6 +74,12 @@ dast: - if: $CI_OPEN_MERGE_REQUESTS when: never + # Add the job to branch pipelines. (FIPS) + - if: $CI_COMMIT_BRANCH && + $CI_GITLAB_FIPS_MODE == "true" && + $GITLAB_FEATURES =~ /\bdast\b/ + variables: + DAST_IMAGE_SUFFIX: "-fips" # Add the job to branch pipelines. - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdast\b/ diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml index 88fe55a44ab..793030d302a 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml @@ -9,18 +9,19 @@ # There is a more opinionated template which we suggest the users to abide, # which is the lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml -image: - name: "$CI_TEMPLATE_REGISTRY_HOST/gitlab-org/terraform-images/stable:latest" +default: + image: + name: "$CI_TEMPLATE_REGISTRY_HOST/gitlab-org/terraform-images/stable:latest" + + cache: + key: "${TF_ROOT}" + paths: + - ${TF_ROOT}/.terraform/ variables: TF_ROOT: ${CI_PROJECT_DIR} # The relative path to the root directory of the Terraform project TF_STATE_NAME: default # The name of the state file used by the GitLab Managed Terraform state backend -cache: - key: "${TF_ROOT}" - paths: - - ${TF_ROOT}/.terraform/ - .terraform:fmt: &terraform_fmt stage: validate script: diff --git a/lib/gitlab/ci/variables/builder.rb b/lib/gitlab/ci/variables/builder.rb index 86e54fdfcdf..cae3a966bc6 100644 --- a/lib/gitlab/ci/variables/builder.rb +++ b/lib/gitlab/ci/variables/builder.rb @@ -139,14 +139,6 @@ module Gitlab # Set environment name here so we can access it when evaluating the job's rules variables.append(key: 'CI_ENVIRONMENT_NAME', value: job.environment) if job.environment - - if Feature.disabled?(:ci_remove_legacy_predefined_variables, project) - # legacy variables - variables.append(key: 'CI_BUILD_NAME', value: job.name) - variables.append(key: 'CI_BUILD_STAGE', value: job.stage_name) - variables.append(key: 'CI_BUILD_TRIGGERED', value: 'true') if job.trigger_request - variables.append(key: 'CI_BUILD_MANUAL', value: 'true') if job.action? - end end end diff --git a/lib/gitlab/ci/variables/builder/pipeline.rb b/lib/gitlab/ci/variables/builder/pipeline.rb index 1e7a18d70b0..c3b0cb856ba 100644 --- a/lib/gitlab/ci/variables/builder/pipeline.rb +++ b/lib/gitlab/ci/variables/builder/pipeline.rb @@ -40,7 +40,7 @@ module Gitlab attr_reader :pipeline - def predefined_commit_variables # rubocop:disable Metrics/AbcSize - Remove this rubocop:disable when FF `ci_remove_legacy_predefined_variables` is removed. + def predefined_commit_variables Gitlab::Ci::Variables::Collection.new.tap do |variables| next variables unless pipeline.sha.present? @@ -56,24 +56,10 @@ module Gitlab variables.append(key: 'CI_COMMIT_REF_PROTECTED', value: (!!pipeline.protected_ref?).to_s) variables.append(key: 'CI_COMMIT_TIMESTAMP', value: pipeline.git_commit_timestamp.to_s) variables.append(key: 'CI_COMMIT_AUTHOR', value: pipeline.git_author_full_text.to_s) - - if Feature.disabled?(:ci_remove_legacy_predefined_variables, pipeline.project) - variables.concat(legacy_predefined_commit_variables) - end end end strong_memoize_attr :predefined_commit_variables - def legacy_predefined_commit_variables - Gitlab::Ci::Variables::Collection.new.tap do |variables| - variables.append(key: 'CI_BUILD_REF', value: pipeline.sha) - variables.append(key: 'CI_BUILD_BEFORE_SHA', value: pipeline.before_sha) - variables.append(key: 'CI_BUILD_REF_NAME', value: pipeline.source_ref) - variables.append(key: 'CI_BUILD_REF_SLUG', value: pipeline.source_ref_slug) - end - end - strong_memoize_attr :legacy_predefined_commit_variables - def predefined_commit_tag_variables Gitlab::Ci::Variables::Collection.new.tap do |variables| git_tag = pipeline.project.repository.find_tag(pipeline.ref) @@ -82,21 +68,10 @@ module Gitlab variables.append(key: 'CI_COMMIT_TAG', value: pipeline.ref) variables.append(key: 'CI_COMMIT_TAG_MESSAGE', value: git_tag.message) - - if Feature.disabled?(:ci_remove_legacy_predefined_variables, pipeline.project) - variables.concat(legacy_predefined_commit_tag_variables) - end end end strong_memoize_attr :predefined_commit_tag_variables - def legacy_predefined_commit_tag_variables - Gitlab::Ci::Variables::Collection.new.tap do |variables| - variables.append(key: 'CI_BUILD_TAG', value: pipeline.ref) - end - end - strong_memoize_attr :legacy_predefined_commit_tag_variables - def predefined_merge_request_variables Gitlab::Ci::Variables::Collection.new.tap do |variables| variables.append(key: 'CI_MERGE_REQUEST_EVENT_TYPE', value: pipeline.merge_request_event_type.to_s) diff --git a/lib/gitlab/cluster/puma_worker_killer_initializer.rb b/lib/gitlab/cluster/puma_worker_killer_initializer.rb deleted file mode 100644 index 957faf797b5..00000000000 --- a/lib/gitlab/cluster/puma_worker_killer_initializer.rb +++ /dev/null @@ -1,51 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Cluster - class PumaWorkerKillerInitializer - def self.start( - puma_options, - puma_per_worker_max_memory_mb: 1200, - puma_master_max_memory_mb: 950, - additional_puma_dev_max_memory_mb: 200) - - # We are replacing PWK with Watchdog by using backward compatible RssMemoryLimit monitor by default. - # https://gitlab.com/groups/gitlab-org/-/epics/9119 - return if Gitlab::Utils.to_boolean(ENV.fetch('GITLAB_MEMORY_WATCHDOG_ENABLED', true)) - - require 'puma_worker_killer' - - PumaWorkerKiller.config do |config| - worker_count = puma_options[:workers] || 1 - # The Puma Worker Killer checks the total memory used by the cluster, - # i.e. both primary and worker processes. - # https://github.com/schneems/puma_worker_killer/blob/v0.1.0/lib/puma_worker_killer/puma_memory.rb#L57 - # - # Additional memory is added when running in `development` - config.ram = puma_master_max_memory_mb + - (worker_count * puma_per_worker_max_memory_mb) + - (Rails.env.development? ? (1 + worker_count) * additional_puma_dev_max_memory_mb : 0) - - config.frequency = 20 # seconds - - # We just want to limit to a fixed maximum, unrelated to the total amount - # of available RAM. - config.percent_usage = 0.98 - - # Ideally we'll never hit the maximum amount of memory. Restart the workers - # regularly rather than rely on OOM behavior for periodic restarting. - config.rolling_restart_frequency = 43200 # 12 hours in seconds. - - # Spread the rolling restarts out over 1 hour to avoid too many simultaneous - # process startups. - config.rolling_restart_splay_seconds = 0.0..3600.0 # 0 to 1 hour in seconds. - - observer = Gitlab::Cluster::PumaWorkerKillerObserver.new - config.pre_term = observer.callback - end - - PumaWorkerKiller.start - end - end - end -end diff --git a/lib/gitlab/cluster/puma_worker_killer_observer.rb b/lib/gitlab/cluster/puma_worker_killer_observer.rb deleted file mode 100644 index f53051c32ff..00000000000 --- a/lib/gitlab/cluster/puma_worker_killer_observer.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Cluster - class PumaWorkerKillerObserver - def initialize - @counter = Gitlab::Metrics.counter(:puma_killer_terminations_total, 'Number of workers terminated by PumaWorkerKiller') - end - - # returns the Proc to be used as the observer callback block - def callback - method(:log_termination) - end - - private - - def log_termination(worker) - @counter.increment - end - end - end -end diff --git a/lib/gitlab/counters/buffered_counter.rb b/lib/gitlab/counters/buffered_counter.rb index 3e232c78e45..258ada864c8 100644 --- a/lib/gitlab/counters/buffered_counter.rb +++ b/lib/gitlab/counters/buffered_counter.rb @@ -78,11 +78,7 @@ module Gitlab def increment(increment) result = redis_state do |redis| - if Feature.enabled?(:project_statistics_bulk_increment, type: :development) - redis.eval(LUA_INCREMENT_WITH_DEDUPLICATION_SCRIPT, **increment_args(increment)).to_i - else - redis.incrby(key, increment.amount) - end + redis.eval(LUA_INCREMENT_WITH_DEDUPLICATION_SCRIPT, **increment_args(increment)).to_i end FlushCounterIncrementsWorker.perform_in(WORKER_DELAY, counter_record.class.name, counter_record.id, attribute) @@ -94,11 +90,7 @@ module Gitlab result = redis_state do |redis| redis.pipelined do |pipeline| increments.each do |increment| - if Feature.enabled?(:project_statistics_bulk_increment, type: :development) - pipeline.eval(LUA_INCREMENT_WITH_DEDUPLICATION_SCRIPT, **increment_args(increment)) - else - pipeline.incrby(key, increment.amount) - end + pipeline.eval(LUA_INCREMENT_WITH_DEDUPLICATION_SCRIPT, **increment_args(increment)) end end end diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index ecb0cc20a64..2fa0b40df14 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -65,6 +65,7 @@ module Gitlab { id: pipeline.id, iid: pipeline.iid, + name: pipeline.name, ref: pipeline.source_ref, tag: pipeline.tag, sha: pipeline.sha, @@ -77,7 +78,8 @@ module Gitlab finished_at: pipeline.finished_at, duration: pipeline.duration, queued_duration: pipeline.queued_duration, - variables: pipeline.variables.map(&:hook_attrs) + variables: pipeline.variables.map(&:hook_attrs), + url: Gitlab::Routing.url_helpers.project_pipeline_url(pipeline.project, pipeline) } end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 4197c87f51f..da9ebf4ab0f 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -2,8 +2,6 @@ module Gitlab module Database - DATABASE_NAMES = %w[main ci main_clusterwide].freeze - MAIN_DATABASE_NAME = 'main' CI_DATABASE_NAME = 'ci' DEFAULT_POOL_HEADROOM = 10 @@ -56,67 +54,78 @@ module Gitlab MODE_SINGLE_DATABASE_CI_CONNECTION = "single-database-ci-connection" MODE_MULTIPLE_DATABASES = "multiple-databases" + def self.all_database_connection_files + Dir.glob(Rails.root.join("db/database_connections/*.yaml")) + end + + def self.all_gitlab_schema_files + Dir.glob(Rails.root.join("db/gitlab_schemas/*.yaml")) + end + + def self.all_database_connections + @all_database_connections ||= + all_database_connection_files + .map { |file| DatabaseConnectionInfo.load_file(file) } + .sort_by(&:order) + .index_by(&:name) + .with_indifferent_access.freeze + end + + def self.all_database_names + all_database_connections.keys.map(&:to_s) + end + + def self.all_gitlab_schemas + @all_gitlab_schemas ||= + all_gitlab_schema_files + .map { |file| GitlabSchemaInfo.load_file(file) } + .index_by(&:name) + .with_indifferent_access.freeze + end + def self.database_base_models - @database_base_models ||= { - # Note that we use ActiveRecord::Base here and not ApplicationRecord. - # This is deliberate, as we also use these classes to apply load - # balancing to, and the load balancer must be enabled for _all_ models - # that inherit from ActiveRecord::Base; not just our own models that - # inherit from ApplicationRecord. - main: ::ActiveRecord::Base, - main_clusterwide: ::MainClusterwide::ApplicationRecord.connection_class? ? ::MainClusterwide::ApplicationRecord : nil, - ci: ::Ci::ApplicationRecord.connection_class? ? ::Ci::ApplicationRecord : nil - }.compact.with_indifferent_access.freeze + # Note that we use ActiveRecord::Base here and not ApplicationRecord. + # This is deliberate, as we also use these classes to apply load + # balancing to, and the load balancer must be enabled for _all_ models + # that inherit from ActiveRecord::Base; not just our own models that + # inherit from ApplicationRecord. + @database_base_models ||= + all_database_connections + .transform_values(&:connection_class) + .compact.with_indifferent_access.freeze end # This returns a list of databases that contains all the gitlab_shared schema - # tables. We can't reuse database_base_models because Geo does not support - # the gitlab_shared tables yet. + # tables. def self.database_base_models_with_gitlab_shared - @database_base_models_with_gitlab_shared ||= { - # Note that we use ActiveRecord::Base here and not ApplicationRecord. - # This is deliberate, as we also use these classes to apply load - # balancing to, and the load balancer must be enabled for _all_ models - # that inher from ActiveRecord::Base; not just our own models that - # inherit from ApplicationRecord. - main: ::ActiveRecord::Base, - main_clusterwide: ::MainClusterwide::ApplicationRecord.connection_class? ? ::MainClusterwide::ApplicationRecord : nil, - ci: ::Ci::ApplicationRecord.connection_class? ? ::Ci::ApplicationRecord : nil - }.compact.with_indifferent_access.freeze + @database_base_models_with_gitlab_shared ||= + all_database_connections + .select { |_, db| db.has_gitlab_shared? } + .transform_values(&:connection_class) + .compact.with_indifferent_access.freeze end # This returns a list of databases whose connection supports database load - # balancing. We can't reuse the database_base_models method because the Geo - # database does not support load balancing yet. - # - # TODO: https://gitlab.com/gitlab-org/geo-team/discussions/-/issues/5032 + # balancing. We can't reuse the database_base_models since not all connections + # do support load balancing. def self.database_base_models_using_load_balancing - @database_base_models_using_load_balancing ||= { - # Note that we use ActiveRecord::Base here and not ApplicationRecord. - # This is deliberate, as we also use these classes to apply load - # balancing to, and the load balancer must be enabled for _all_ models - # that inher from ActiveRecord::Base; not just our own models that - # inherit from ApplicationRecord. - main: ::ActiveRecord::Base, - main_clusterwide: ::MainClusterwide::ApplicationRecord.connection_class? ? ::MainClusterwide::ApplicationRecord : nil, - ci: ::Ci::ApplicationRecord.connection_class? ? ::Ci::ApplicationRecord : nil - }.compact.with_indifferent_access.freeze + @database_base_models_using_load_balancing ||= + all_database_connections + .select { |_, db| db.uses_load_balancing? } + .transform_values(&:connection_class) + .compact.with_indifferent_access.freeze end # This returns a list of base models with connection associated for a given gitlab_schema def self.schemas_to_base_models - @schemas_to_base_models ||= { - gitlab_main: [self.database_base_models.fetch(:main)], - gitlab_ci: [self.database_base_models[:ci] || self.database_base_models.fetch(:main)], # use CI or fallback to main - gitlab_shared: database_base_models_with_gitlab_shared.values, # all models - gitlab_internal: database_base_models.values, # all models - gitlab_pm: [self.database_base_models.fetch(:main)], # package metadata models - gitlab_main_clusterwide: [self.database_base_models[:main_clusterwide] || self.database_base_models.fetch(:main)] - }.with_indifferent_access.freeze - end - - def self.all_database_names - DATABASE_NAMES + @schemas_to_base_models ||= + all_gitlab_schemas.transform_values do |schema| + all_database_connections + .values + .select { |db| db.gitlab_schemas.include?(schema.name) } + .filter_map { |db| db.connection_class_or_fallback(all_database_connections) } + .uniq + end.compact.with_indifferent_access.freeze end # We configure the database connection pool size automatically based on the @@ -255,8 +264,16 @@ module Gitlab end end - def self.db_config_names - ::ActiveRecord::Base.configurations.configs_for(env_name: Rails.env).map(&:name) - ['geo'] + def self.db_config_names(with_schema:) + db_config_names = ::ActiveRecord::Base.configurations + .configs_for(env_name: Rails.env).map(&:name) + return db_config_names unless with_schema + + schema_models = schemas_to_base_models.fetch(with_schema) + db_config_names.select do |db_config_name| + db_info = all_database_connections.fetch(db_config_name) + schema_models.include?(db_info.connection_class) + end end # This returns all matching schemas that a given connection can use diff --git a/lib/gitlab/database/async_indexes/index_base.rb b/lib/gitlab/database/async_indexes/index_base.rb index bde75e12295..93f3ba88345 100644 --- a/lib/gitlab/database/async_indexes/index_base.rb +++ b/lib/gitlab/database/async_indexes/index_base.rb @@ -81,7 +81,8 @@ module Gitlab { table_name: async_index.table_name, index_name: async_index.name, - class: self.class.name.to_s + class: self.class.name.to_s, + connection_name: database_config_name } end end diff --git a/lib/gitlab/database/async_indexes/postgres_async_index.rb b/lib/gitlab/database/async_indexes/postgres_async_index.rb index 9f5f39613ed..a3c600a4519 100644 --- a/lib/gitlab/database/async_indexes/postgres_async_index.rb +++ b/lib/gitlab/database/async_indexes/postgres_async_index.rb @@ -8,13 +8,17 @@ module Gitlab self.table_name = 'postgres_async_indexes' + # schema_name + . + table_name + MAX_TABLE_NAME_LENGTH = (Gitlab::Database::MigrationHelpers::MAX_IDENTIFIER_NAME_LENGTH * 2) + 1 MAX_IDENTIFIER_LENGTH = Gitlab::Database::MigrationHelpers::MAX_IDENTIFIER_NAME_LENGTH MAX_DEFINITION_LENGTH = 2048 validates :name, presence: true, length: { maximum: MAX_IDENTIFIER_LENGTH } - validates :table_name, presence: true, length: { maximum: MAX_IDENTIFIER_LENGTH } + validates :table_name, presence: true, length: { maximum: MAX_TABLE_NAME_LENGTH } validates :definition, presence: true, length: { maximum: MAX_DEFINITION_LENGTH } + validate :ensure_correct_schema_and_table_name + scope :to_create, -> { where("definition ILIKE 'CREATE%'") } scope :to_drop, -> { where("definition ILIKE 'DROP%'") } scope :ordered, -> { order(attempts: :asc, id: :asc) } @@ -22,6 +26,24 @@ module Gitlab def to_s definition end + + private + + def ensure_correct_schema_and_table_name + return unless table_name + + schema, table, *rest = table_name.split('.') + + too_long = (table.nil? && schema.length > MAX_DEFINITION_LENGTH) || # no schema given + # both schema and table given + (schema.length > MAX_IDENTIFIER_LENGTH || (table && table.length > MAX_IDENTIFIER_LENGTH)) + + if too_long + errors.add(:table_name, :too_long) + elsif rest.any? + errors.add(:table_name, :invalid) + end + end end end end diff --git a/lib/gitlab/database/background_migration/batched_job.rb b/lib/gitlab/database/background_migration/batched_job.rb index 523ab2a9f27..458b099924b 100644 --- a/lib/gitlab/database/background_migration/batched_job.rb +++ b/lib/gitlab/database/background_migration/batched_job.rb @@ -139,7 +139,7 @@ module Gitlab new_batch_size = batch_size / 2 - break update!(attempts: 0) if new_batch_size < 1 + next update!(attempts: 0) if new_batch_size < 1 batching_strategy = batched_migration.batch_class.new(connection: self.class.connection) next_batch_bounds = batching_strategy.next_batch( diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb index a883996a5c5..83beee091f1 100644 --- a/lib/gitlab/database/background_migration/batched_migration.rb +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -79,6 +79,10 @@ module Gitlab transition any => :finalizing end + before_transition any => :finished do |migration| + migration.finished_at = Time.current if migration.respond_to?(:finished_at) + end + before_transition any => :active do |migration| migration.started_at = Time.current if migration.respond_to?(:started_at) end @@ -92,10 +96,6 @@ module Gitlab for_configuration(gitlab_schema, job_class_name, table_name, column_name, job_arguments).first end - def self.active_migration(connection:) - active_migrations_distinct_on_table(connection: connection, limit: 1).first - end - def self.find_executable(id, connection:) for_gitlab_schema(Gitlab::Database.gitlab_schemas_for_connection(connection)) .executable.find_by_id(id) @@ -220,7 +220,12 @@ module Gitlab end def health_context - HealthStatus::Context.new(connection, [table_name], gitlab_schema.to_sym) + @health_context ||= Gitlab::Database::HealthStatus::Context.new( + self, + connection, + [table_name], + gitlab_schema.to_sym + ) end def hold!(until_time: 10.minutes.from_now) diff --git a/lib/gitlab/database/background_migration/batched_migration_runner.rb b/lib/gitlab/database/background_migration/batched_migration_runner.rb index 7224ff2b517..e3e8754c758 100644 --- a/lib/gitlab/database/background_migration/batched_migration_runner.rb +++ b/lib/gitlab/database/background_migration/batched_migration_runner.rb @@ -144,7 +144,7 @@ module Gitlab end def adjust_migration(active_migration) - signals = HealthStatus.evaluate(active_migration) + signals = Gitlab::Database::HealthStatus.evaluate(active_migration.health_context) if signals.any?(&:stop?) active_migration.hold! diff --git a/lib/gitlab/database/background_migration/health_status.rb b/lib/gitlab/database/background_migration/health_status.rb deleted file mode 100644 index 96905fd0bc5..00000000000 --- a/lib/gitlab/database/background_migration/health_status.rb +++ /dev/null @@ -1,45 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module BackgroundMigration - module HealthStatus - DEFAULT_INIDICATORS = [ - Indicators::AutovacuumActiveOnTable, - Indicators::WriteAheadLog, - Indicators::PatroniApdex - ].freeze - - # Rather than passing along the migration, we use a more explicitly defined context - Context = Struct.new(:connection, :tables, :gitlab_schema) - - def self.evaluate(migration, indicators = DEFAULT_INIDICATORS) - indicators.map do |indicator| - signal = begin - indicator.new(migration.health_context).evaluate - rescue StandardError => e - Gitlab::ErrorTracking.track_exception(e, migration_id: migration.id, - job_class_name: migration.job_class_name) - - Signals::Unknown.new(indicator, reason: "unexpected error: #{e.message} (#{e.class})") - end - - log_signal(signal, migration) if signal.log_info? - - signal - end - end - - def self.log_signal(signal, migration) - Gitlab::BackgroundMigration::Logger.info( - migration_id: migration.id, - health_status_indicator: signal.indicator_class.to_s, - indicator_signal: signal.short_name, - signal_reason: signal.reason, - message: "#{migration} signaled: #{signal}" - ) - end - end - end - end -end diff --git a/lib/gitlab/database/background_migration/health_status/indicators.rb b/lib/gitlab/database/background_migration/health_status/indicators.rb deleted file mode 100644 index 69503e5b61f..00000000000 --- a/lib/gitlab/database/background_migration/health_status/indicators.rb +++ /dev/null @@ -1,12 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module BackgroundMigration - module HealthStatus - module Indicators - end - end - end - end -end diff --git a/lib/gitlab/database/background_migration/health_status/indicators/autovacuum_active_on_table.rb b/lib/gitlab/database/background_migration/health_status/indicators/autovacuum_active_on_table.rb deleted file mode 100644 index 48e12609a13..00000000000 --- a/lib/gitlab/database/background_migration/health_status/indicators/autovacuum_active_on_table.rb +++ /dev/null @@ -1,41 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module BackgroundMigration - module HealthStatus - module Indicators - class AutovacuumActiveOnTable - def initialize(context) - @context = context - end - - def evaluate - return Signals::NotAvailable.new(self.class, reason: 'indicator disabled') unless enabled? - - autovacuum_active_on = active_autovacuums_for(context.tables) - - if autovacuum_active_on.empty? - Signals::Normal.new(self.class, reason: 'no autovacuum running on any relevant tables') - else - Signals::Stop.new(self.class, reason: "autovacuum running on: #{autovacuum_active_on.join(', ')}") - end - end - - private - - attr_reader :context - - def enabled? - Feature.enabled?(:batched_migrations_health_status_autovacuum, type: :ops) - end - - def active_autovacuums_for(tables) - Gitlab::Database::PostgresAutovacuumActivity.for_tables(tables) - end - end - end - end - end - end -end diff --git a/lib/gitlab/database/background_migration/health_status/indicators/patroni_apdex.rb b/lib/gitlab/database/background_migration/health_status/indicators/patroni_apdex.rb deleted file mode 100644 index 0dd6dd5c2a4..00000000000 --- a/lib/gitlab/database/background_migration/health_status/indicators/patroni_apdex.rb +++ /dev/null @@ -1,90 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module BackgroundMigration - module HealthStatus - module Indicators - class PatroniApdex - include Gitlab::Utils::StrongMemoize - - def initialize(context) - @context = context - end - - def evaluate - return Signals::NotAvailable.new(self.class, reason: 'indicator disabled') unless enabled? - - connection_error_message = fetch_connection_error_message - return unknown_signal(connection_error_message) if connection_error_message.present? - - apdex_sli = fetch_sli(apdex_sli_query) - return unknown_signal('Patroni service apdex can not be calculated') unless apdex_sli.present? - - if apdex_sli.to_f > apdex_slo.to_f - Signals::Normal.new(self.class, reason: 'Patroni service apdex is above SLO') - else - Signals::Stop.new(self.class, reason: 'Patroni service apdex is below SLO') - end - end - - private - - attr_reader :context - - def enabled? - Feature.enabled?(:batched_migrations_health_status_patroni_apdex, type: :ops) - end - - def unknown_signal(reason) - Signals::Unknown.new(self.class, reason: reason) - end - - def fetch_connection_error_message - return 'Patroni Apdex Settings not configured' unless database_apdex_settings.present? - return 'Prometheus client is not ready' unless client.ready? - return 'Apdex SLI query is not configured' unless apdex_sli_query - return 'Apdex SLO is not configured' unless apdex_slo - end - - def client - @client ||= Gitlab::PrometheusClient.new( - database_apdex_settings[:prometheus_api_url], - allow_local_requests: true, - verify: true - ) - end - - def database_apdex_settings - @database_apdex_settings ||= Gitlab::CurrentSettings.database_apdex_settings&.with_indifferent_access - end - - def apdex_sli_query - { - gitlab_main: database_apdex_settings[:apdex_sli_query][:main], - gitlab_ci: database_apdex_settings[:apdex_sli_query][:ci] - }.fetch(context.gitlab_schema.to_sym) - end - strong_memoize_attr :apdex_sli_query - - def apdex_slo - { - gitlab_main: database_apdex_settings[:apdex_slo][:main], - gitlab_ci: database_apdex_settings[:apdex_slo][:ci] - }.fetch(context.gitlab_schema.to_sym) - end - strong_memoize_attr :apdex_slo - - def fetch_sli(query) - response = client.query(query) - metric = response&.first || {} - value = metric.fetch('value', []) - - Array.wrap(value).second - end - end - end - end - end - end -end diff --git a/lib/gitlab/database/background_migration/health_status/indicators/write_ahead_log.rb b/lib/gitlab/database/background_migration/health_status/indicators/write_ahead_log.rb deleted file mode 100644 index d2fb0a8b751..00000000000 --- a/lib/gitlab/database/background_migration/health_status/indicators/write_ahead_log.rb +++ /dev/null @@ -1,74 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module BackgroundMigration - module HealthStatus - module Indicators - class WriteAheadLog - include Gitlab::Utils::StrongMemoize - - LIMIT = 42 - PENDING_WAL_COUNT_SQL = <<~SQL - WITH - current_wal_file AS ( - SELECT pg_walfile_name(pg_current_wal_insert_lsn()) AS pg_walfile_name - ), - current_wal AS ( - SELECT - ('x' || substring(pg_walfile_name, 9, 8))::bit(32)::int AS log, - ('x' || substring(pg_walfile_name, 17, 8))::bit(32)::int AS seg, - pg_walfile_name - FROM current_wal_file - ), - archive_wal AS ( - SELECT - ('x' || substring(last_archived_wal, 9, 8))::bit(32)::int AS log, - ('x' || substring(last_archived_wal, 17, 8))::bit(32)::int AS seg, - last_archived_wal - FROM pg_stat_archiver - ) - SELECT ((current_wal.log - archive_wal.log) * 256) + (current_wal.seg - archive_wal.seg) AS pending_wal_count - FROM current_wal, archive_wal - SQL - - def initialize(context) - @connection = context.connection - end - - def evaluate - return Signals::NotAvailable.new(self.class, reason: 'indicator disabled') unless enabled? - - unless pending_wal_count - return Signals::NotAvailable.new(self.class, reason: 'WAL archive queue can not be calculated') - end - - if pending_wal_count > LIMIT - Signals::Stop.new(self.class, reason: "WAL archive queue is too big") - else - Signals::Normal.new(self.class, reason: 'WAL archive queue is within limit') - end - end - - private - - attr_reader :connection - - def enabled? - Feature.enabled?(:batched_migrations_health_status_wal, type: :ops) - end - - # Returns number of WAL segments pending archival - def pending_wal_count - strong_memoize(:pending_wal_count) do - Gitlab::Database::LoadBalancing::Session.current.use_primary do - connection.execute(PENDING_WAL_COUNT_SQL).to_a.first&.fetch('pending_wal_count') - end - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/database/background_migration/health_status/signals.rb b/lib/gitlab/database/background_migration/health_status/signals.rb deleted file mode 100644 index 534c4330cf2..00000000000 --- a/lib/gitlab/database/background_migration/health_status/signals.rb +++ /dev/null @@ -1,71 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module BackgroundMigration - module HealthStatus - module Signals - # Base class for a signal - class Base - attr_reader :indicator_class, :reason - - def initialize(indicator_class, reason:) - @indicator_class = indicator_class - @reason = reason - end - - def to_s - "#{short_name} (indicator: #{indicator_class}; reason: #{reason})" - end - - # :nocov: - def log_info? - false - end - - def stop? - false - end - # :nocov: - - def short_name - self.class.name.demodulize - end - end - - # A Signals::Stop is an indication to put a migration on hold or stop it entirely: - # In general, we want to slow down or pause the migration. - class Stop < Base - # :nocov: - def log_info? - true - end - - def stop? - true - end - # :nocov: - end - - # A Signals::Normal indicates normal system state: We carry on with the migration - # and may even attempt to optimize its throughput etc. - class Normal < Base; end - - # When given an Signals::Unknown, something unexpected happened while - # we evaluated system indicators. - class Unknown < Base - # :nocov: - def log_info? - true - end - # :nocov: - end - - # No signal could be determined, e.g. because the indicator - # was disabled. - class NotAvailable < Base; end - end - end - end - end -end diff --git a/lib/gitlab/database/convert_feature_category_to_group_label.rb b/lib/gitlab/database/convert_feature_category_to_group_label.rb new file mode 100644 index 00000000000..5a4599312ba --- /dev/null +++ b/lib/gitlab/database/convert_feature_category_to_group_label.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class ConvertFeatureCategoryToGroupLabel + STAGES_URL = 'https://gitlab.com/gitlab-com/www-gitlab-com/-/raw/master/data/stages.yml' + + def initialize(feature_category) + @feature_category = feature_category + end + + def execute + feature_categories_map[feature_category] + end + + private + + attr_reader :feature_category + + def stages + response = Gitlab::HTTP.get(STAGES_URL) + + YAML.safe_load(response) if response.success? + end + + def feature_categories_map + stages['stages'].each_with_object({}) do |(_, stage), result| + stage['groups'].each do |group_name, group| + group['categories'].each do |category| + result[category] = "group::#{group_name.sub('_', ' ')}" + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/database_connection_info.rb b/lib/gitlab/database/database_connection_info.rb new file mode 100644 index 00000000000..57ecbcd64ae --- /dev/null +++ b/lib/gitlab/database/database_connection_info.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module Gitlab + module Database + DatabaseConnectionInfo = Struct.new( + :name, + :description, + :gitlab_schemas, + :klass, + :fallback_database, + :db_dir, + :uses_load_balancing, + :file_path, + keyword_init: true + ) do + include Gitlab::Utils::StrongMemoize + + def initialize(*) + super + self.name = name.to_sym + self.gitlab_schemas = gitlab_schemas.map(&:to_sym) + self.klass = klass.constantize + self.fallback_database = fallback_database&.to_sym + self.db_dir = Rails.root.join(db_dir || 'db') + end + + def self.load_file(yaml_file) + content = YAML.load_file(yaml_file) + new(**content.deep_symbolize_keys.merge(file_path: yaml_file)) + end + + def active_record_base? + klass == ActiveRecord::Base + end + private :active_record_base? + + strong_memoize_attr def connection_class + klass.connection_class || active_record_base? ? klass : nil + end + + strong_memoize_attr def order + # Retain order of configurations as they are defined in `config/database.yml` + ActiveRecord::Base.configurations + .configs_for(env_name: Rails.env) + .map(&:name) + .index(name.to_s) || 1_000 + end + + def connection_class_or_fallback(all_databases) + if connection_class + connection_class + elsif fallback_database + all_databases.fetch(fallback_database) + .connection_class_or_fallback(all_databases) + end + end + + def has_gitlab_shared? + gitlab_schemas.include?(:gitlab_shared) + end + + def uses_load_balancing? + !!uses_load_balancing + end + + def db_docs_dir + db_dir.join('docs') + end + end + end +end diff --git a/lib/gitlab/database/each_database.rb b/lib/gitlab/database/each_database.rb index 02f008abf85..b1af62e4875 100644 --- a/lib/gitlab/database/each_database.rb +++ b/lib/gitlab/database/each_database.rb @@ -18,6 +18,7 @@ module Gitlab end end end + alias_method :each_db_connection, :each_database_connection def each_model_connection(models, only_on: nil, &blk) selected_databases = Array.wrap(only_on).map(&:to_sym) diff --git a/lib/gitlab/database/gitlab_schema.rb b/lib/gitlab/database/gitlab_schema.rb index 4394c089b22..9b58284b389 100644 --- a/lib/gitlab/database/gitlab_schema.rb +++ b/lib/gitlab/database/gitlab_schema.rb @@ -3,13 +3,15 @@ # This module gathers information about table to schema mapping # to understand table affinity # -# Each table / view needs to have assigned gitlab_schema. Names supported today: +# Each table / view needs to have assigned gitlab_schema. For example: # # - gitlab_shared - defines a set of tables that are found on all databases (data accessed is dependent on connection) # - gitlab_main / gitlab_ci - defines a set of tables that can only exist on a given application database # - gitlab_geo - defines a set of tables that can only exist on the geo database # - gitlab_internal - defines all internal tables of Rails and PostgreSQL # +# All supported GitLab schemas can be viewed in `db/gitlab_schemas/` and `ee/db/gitlab_schemas/` +# # Tables for the purpose of tests should be prefixed with `_test_my_table_name` module Gitlab @@ -17,8 +19,6 @@ module Gitlab module GitlabSchema UnknownSchemaError = Class.new(StandardError) - DICTIONARY_PATH = 'db/docs/' - def self.table_schemas!(tables) tables.map { |table| table_schema!(table) }.to_set end @@ -67,44 +67,50 @@ module Gitlab # All `pg_` tables are marked as `internal` return :gitlab_internal if table_name.start_with?('pg_') - - # Sometimes the name of an index can be interpreted as a table's name. - # For eg, if we execute "ALTER INDEX my_index..", my_index is interpreted as a table name. - # In such cases, we should return the schema of the database table actually - # holding that index. - index_name = table_name - derive_schema_from_index(index_name) end # rubocop:enable Metrics/CyclomaticComplexity - def self.dictionary_path_globs - [Rails.root.join(DICTIONARY_PATH, '*.yml')] + def self.table_schema!(name) + # rubocop:disable Gitlab/DocUrl + self.table_schema(name) || raise( + UnknownSchemaError, + "Could not find gitlab schema for table #{name}: Any new or deleted tables must be added to the database dictionary " \ + "See https://docs.gitlab.com/ee/development/database/database_dictionary.html" + ) + # rubocop:enable Gitlab/DocUrl end - def self.view_path_globs - [Rails.root.join(DICTIONARY_PATH, 'views', '*.yml')] + private_class_method def self.cross_access_allowed?(type, table_schemas) + table_schemas.any? do |schema| + extra_schemas = table_schemas - [schema] + extra_schemas -= Gitlab::Database.all_gitlab_schemas[schema]&.public_send(type) || [] # rubocop:disable GitlabSecurity/PublicSend + extra_schemas.empty? + end end - def self.deleted_views_path_globs - [Rails.root.join(DICTIONARY_PATH, 'deleted_views', '*.yml')] + def self.cross_joins_allowed?(table_schemas) + table_schemas.empty? || self.cross_access_allowed?(:allow_cross_joins, table_schemas) end - def self.deleted_tables_path_globs - [Rails.root.join(DICTIONARY_PATH, 'deleted_tables', '*.yml')] + def self.cross_transactions_allowed?(table_schemas) + table_schemas.empty? || self.cross_access_allowed?(:allow_cross_transactions, table_schemas) end - def self.views_and_tables_to_schema - @views_and_tables_to_schema ||= self.tables_to_schema.merge(self.views_to_schema) + def self.cross_foreign_key_allowed?(table_schemas) + self.cross_access_allowed?(:allow_cross_foreign_keys, table_schemas) end - def self.table_schema!(name) - # rubocop:disable Gitlab/DocUrl - self.table_schema(name) || raise( - UnknownSchemaError, - "Could not find gitlab schema for table #{name}: Any new or deleted tables must be added to the database dictionary " \ - "See https://docs.gitlab.com/ee/development/database/database_dictionary.html" - ) - # rubocop:enable Gitlab/DocUrl + def self.dictionary_paths + Gitlab::Database.all_database_connections + .values.map(&:db_docs_dir).uniq + end + + def self.dictionary_path_globs(scope) + self.dictionary_paths.map { |path| Rails.root.join(path, scope, '*.yml') } + end + + def self.views_and_tables_to_schema + @views_and_tables_to_schema ||= self.tables_to_schema.merge(self.views_to_schema) end def self.deleted_views_and_tables_to_schema @@ -112,36 +118,27 @@ module Gitlab end def self.deleted_tables_to_schema - @deleted_tables_to_schema ||= self.build_dictionary(self.deleted_tables_path_globs) + @deleted_tables_to_schema ||= self.build_dictionary('deleted_tables').to_h end def self.deleted_views_to_schema - @deleted_views_to_schema ||= self.build_dictionary(self.deleted_views_path_globs) + @deleted_views_to_schema ||= self.build_dictionary('deleted_views').to_h end def self.tables_to_schema - @tables_to_schema ||= self.build_dictionary(self.dictionary_path_globs) + @tables_to_schema ||= self.build_dictionary('').to_h end def self.views_to_schema - @views_to_schema ||= self.build_dictionary(self.view_path_globs) + @views_to_schema ||= self.build_dictionary('views').to_h end def self.schema_names @schema_names ||= self.views_and_tables_to_schema.values.to_set end - private_class_method def self.derive_schema_from_index(index_name) - index = Gitlab::Database::PostgresIndex.find_by(name: index_name, - schema: ApplicationRecord.connection.current_schema) - - return unless index - - table_schema(index.tablename) - end - - private_class_method def self.build_dictionary(path_globs) - Dir.glob(path_globs).each_with_object({}) do |file_path, dic| + def self.build_dictionary(scope) + Dir.glob(dictionary_path_globs(scope)).map do |file_path| data = YAML.load_file(file_path) key_name = data['table_name'] || data['view_name'] @@ -156,7 +153,7 @@ module Gitlab end # rubocop:enable Gitlab/DocUrl - dic[key_name] = data['gitlab_schema'].to_sym + [key_name, data['gitlab_schema'].to_sym] end end end diff --git a/lib/gitlab/database/gitlab_schema_info.rb b/lib/gitlab/database/gitlab_schema_info.rb new file mode 100644 index 00000000000..34b89cb9006 --- /dev/null +++ b/lib/gitlab/database/gitlab_schema_info.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Database + GitlabSchemaInfo = Struct.new( + :name, + :description, + :allow_cross_joins, + :allow_cross_transactions, + :allow_cross_foreign_keys, + :file_path, + keyword_init: true + ) do + def initialize(*) + super + self.name = name.to_sym + self.allow_cross_joins = allow_cross_joins&.map(&:to_sym)&.freeze + self.allow_cross_transactions = allow_cross_transactions&.map(&:to_sym)&.freeze + self.allow_cross_foreign_keys = allow_cross_foreign_keys&.map(&:to_sym)&.freeze + end + + def self.load_file(yaml_file) + content = YAML.load_file(yaml_file) + new(**content.deep_symbolize_keys.merge(file_path: yaml_file)) + end + end + end +end diff --git a/lib/gitlab/database/health_status.rb b/lib/gitlab/database/health_status.rb new file mode 100644 index 00000000000..69bb8a70afd --- /dev/null +++ b/lib/gitlab/database/health_status.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module HealthStatus + DEFAULT_INIDICATORS = [ + Indicators::AutovacuumActiveOnTable, + Indicators::WriteAheadLog, + Indicators::PatroniApdex + ].freeze + + class << self + def evaluate(context, indicators = DEFAULT_INIDICATORS) + indicators.map do |indicator| + signal = begin + indicator.new(context).evaluate + rescue StandardError => e + Gitlab::ErrorTracking.track_exception(e, **context.status_checker_info) + + Signals::Unknown.new(indicator, reason: "unexpected error: #{e.message} (#{e.class})") + end + + log_signal(signal, context) if signal.log_info? + + signal + end + end + + def log_signal(signal, context) + Gitlab::Database::HealthStatus::Logger.info(**context.status_checker_info.merge( + health_status_indicator: signal.indicator_class.to_s, + indicator_signal: signal.short_name, + signal_reason: signal.reason, + message: "#{context.status_checker} signaled: #{signal}" + )) + end + end + end + end +end diff --git a/lib/gitlab/database/health_status/context.rb b/lib/gitlab/database/health_status/context.rb new file mode 100644 index 00000000000..717257a84ad --- /dev/null +++ b/lib/gitlab/database/health_status/context.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module HealthStatus + class Context + attr_reader :status_checker, :connection, :tables, :gitlab_schema + + # status_checker: the caller object which checks for database health status + # eg: BackgroundMigration::BatchedMigration or DeferJobs::DatabaseHealthStatusChecker + def initialize(status_checker, connection, tables, gitlab_schema) + @status_checker = status_checker + @connection = connection + @tables = tables + @gitlab_schema = gitlab_schema + end + + def status_checker_info + { + status_checker_id: status_checker.id, + status_checker_type: status_checker.class.name, + job_class_name: status_checker.job_class_name + } + end + end + end + end +end diff --git a/lib/gitlab/database/health_status/indicators.rb b/lib/gitlab/database/health_status/indicators.rb new file mode 100644 index 00000000000..a149c36aae4 --- /dev/null +++ b/lib/gitlab/database/health_status/indicators.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module HealthStatus + module Indicators + end + end + end +end diff --git a/lib/gitlab/database/health_status/indicators/autovacuum_active_on_table.rb b/lib/gitlab/database/health_status/indicators/autovacuum_active_on_table.rb new file mode 100644 index 00000000000..6bf2bbf0c70 --- /dev/null +++ b/lib/gitlab/database/health_status/indicators/autovacuum_active_on_table.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module HealthStatus + module Indicators + class AutovacuumActiveOnTable + def initialize(context) + @tables = context.tables + end + + def evaluate + return Signals::NotAvailable.new(self.class, reason: 'indicator disabled') unless enabled? + + autovacuum_active_on = active_autovacuums_for(tables) + + if autovacuum_active_on.empty? + Signals::Normal.new(self.class, reason: 'no autovacuum running on any relevant tables') + else + Signals::Stop.new(self.class, reason: "autovacuum running on: #{autovacuum_active_on.join(', ')}") + end + end + + private + + attr_reader :tables + + def enabled? + Feature.enabled?(:batched_migrations_health_status_autovacuum, type: :ops) + end + + def active_autovacuums_for(tables) + Gitlab::Database::PostgresAutovacuumActivity.for_tables(tables) + end + end + end + end + end +end diff --git a/lib/gitlab/database/health_status/indicators/patroni_apdex.rb b/lib/gitlab/database/health_status/indicators/patroni_apdex.rb new file mode 100644 index 00000000000..680c86cf7b2 --- /dev/null +++ b/lib/gitlab/database/health_status/indicators/patroni_apdex.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module HealthStatus + module Indicators + class PatroniApdex + include Gitlab::Utils::StrongMemoize + + def initialize(context) + @gitlab_schema = context.gitlab_schema.to_sym + end + + def evaluate + return Signals::NotAvailable.new(self.class, reason: 'indicator disabled') unless enabled? + + connection_error_message = fetch_connection_error_message + return unknown_signal(connection_error_message) if connection_error_message.present? + + apdex_sli = fetch_sli(apdex_sli_query) + return unknown_signal('Patroni service apdex can not be calculated') unless apdex_sli.present? + + if apdex_sli.to_f > apdex_slo.to_f + Signals::Normal.new(self.class, reason: 'Patroni service apdex is above SLO') + else + Signals::Stop.new(self.class, reason: 'Patroni service apdex is below SLO') + end + end + + private + + attr_reader :gitlab_schema + + def enabled? + Feature.enabled?(:batched_migrations_health_status_patroni_apdex, type: :ops) + end + + def unknown_signal(reason) + Signals::Unknown.new(self.class, reason: reason) + end + + def fetch_connection_error_message + return 'Patroni Apdex Settings not configured' unless database_apdex_settings.present? + return 'Prometheus client is not ready' unless client.ready? + return 'Apdex SLI query is not configured' unless apdex_sli_query + return 'Apdex SLO is not configured' unless apdex_slo + end + + def client + @client ||= Gitlab::PrometheusClient.new( + database_apdex_settings[:prometheus_api_url], + allow_local_requests: true, + verify: true + ) + end + + def database_apdex_settings + @database_apdex_settings ||= Gitlab::CurrentSettings.database_apdex_settings&.with_indifferent_access + end + + def apdex_sli_query + { + gitlab_main: database_apdex_settings[:apdex_sli_query][:main], + gitlab_ci: database_apdex_settings[:apdex_sli_query][:ci] + }.fetch(gitlab_schema) + end + strong_memoize_attr :apdex_sli_query + + def apdex_slo + { + gitlab_main: database_apdex_settings[:apdex_slo][:main], + gitlab_ci: database_apdex_settings[:apdex_slo][:ci] + }.fetch(gitlab_schema) + end + strong_memoize_attr :apdex_slo + + def fetch_sli(query) + response = client.query(query) + metric = response&.first || {} + value = metric.fetch('value', []) + + Array.wrap(value).second + end + end + end + end + end +end diff --git a/lib/gitlab/database/health_status/indicators/write_ahead_log.rb b/lib/gitlab/database/health_status/indicators/write_ahead_log.rb new file mode 100644 index 00000000000..1614b17df48 --- /dev/null +++ b/lib/gitlab/database/health_status/indicators/write_ahead_log.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module HealthStatus + module Indicators + class WriteAheadLog + include Gitlab::Utils::StrongMemoize + + LIMIT = 42 + PENDING_WAL_COUNT_SQL = <<~SQL + WITH + current_wal_file AS ( + SELECT pg_walfile_name(pg_current_wal_insert_lsn()) AS pg_walfile_name + ), + current_wal AS ( + SELECT + ('x' || substring(pg_walfile_name, 9, 8))::bit(32)::int AS log, + ('x' || substring(pg_walfile_name, 17, 8))::bit(32)::int AS seg, + pg_walfile_name + FROM current_wal_file + ), + archive_wal AS ( + SELECT + ('x' || substring(last_archived_wal, 9, 8))::bit(32)::int AS log, + ('x' || substring(last_archived_wal, 17, 8))::bit(32)::int AS seg, + last_archived_wal + FROM pg_stat_archiver + ) + SELECT ((current_wal.log - archive_wal.log) * 256) + (current_wal.seg - archive_wal.seg) AS pending_wal_count + FROM current_wal, archive_wal + SQL + + def initialize(context) + @connection = context.connection + end + + def evaluate + return Signals::NotAvailable.new(self.class, reason: 'indicator disabled') unless enabled? + + unless pending_wal_count + return Signals::NotAvailable.new(self.class, reason: 'WAL archive queue can not be calculated') + end + + if pending_wal_count > LIMIT + Signals::Stop.new(self.class, reason: "WAL archive queue is too big") + else + Signals::Normal.new(self.class, reason: 'WAL archive queue is within limit') + end + end + + private + + attr_reader :connection + + def enabled? + Feature.enabled?(:batched_migrations_health_status_wal, type: :ops) + end + + # Returns number of WAL segments pending archival + def pending_wal_count + Gitlab::Database::LoadBalancing::Session.current.use_primary do + connection.execute(PENDING_WAL_COUNT_SQL).to_a.first&.fetch('pending_wal_count') + end + end + strong_memoize_attr :pending_wal_count + end + end + end + end +end diff --git a/lib/gitlab/database/health_status/logger.rb b/lib/gitlab/database/health_status/logger.rb new file mode 100644 index 00000000000..820c1aeb695 --- /dev/null +++ b/lib/gitlab/database/health_status/logger.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module HealthStatus + class Logger < ::Gitlab::JsonLogger + exclude_context! + + def self.file_name_noext + 'database_health_status' + end + end + end + end +end diff --git a/lib/gitlab/database/health_status/signals.rb b/lib/gitlab/database/health_status/signals.rb new file mode 100644 index 00000000000..e1bcdcae9c7 --- /dev/null +++ b/lib/gitlab/database/health_status/signals.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module HealthStatus + module Signals + # Base class for a signal + class Base + attr_reader :indicator_class, :reason + + def initialize(indicator_class, reason:) + @indicator_class = indicator_class + @reason = reason + end + + def to_s + "#{short_name} (indicator: #{indicator_class}; reason: #{reason})" + end + + def log_info? + false + end + + def stop? + false + end + + def short_name + self.class.name.demodulize + end + end + + # A Signals::Stop is an indication to put a migration on hold or stop it entirely: + # In general, we want to slow down or pause the migration. + class Stop < Base + def log_info? + true + end + + def stop? + true + end + end + + # A Signals::Normal indicates normal system state: We carry on with the migration + # and may even attempt to optimize its throughput etc. + class Normal < Base; end + + # When given an Signals::Unknown, something unexpected happened while + # we evaluated system indicators. + class Unknown < Base + def log_info? + true + end + end + + # No signal could be determined, e.g. because the indicator + # was disabled. + class NotAvailable < Base; end + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/host.rb b/lib/gitlab/database/load_balancing/host.rb index bdbb80d6f31..f8ed5fcd4cc 100644 --- a/lib/gitlab/database/load_balancing/host.rb +++ b/lib/gitlab/database/load_balancing/host.rb @@ -16,6 +16,43 @@ module Gitlab PG::Error ].freeze + # This query checks that the current user has permissions before we try and query logical replication status. We + # also only allow >= PG14 because these views are only accessible to superuser before PG14 even if the + # has_table_privilege says otherwise. + CAN_TRACK_LOGICAL_LSN_QUERY = <<~SQL.squish.freeze + SELECT + has_table_privilege('pg_replication_origin_status', 'select') + AND + has_function_privilege('pg_show_replication_origin_status()', 'execute') + AND current_setting('server_version_num', true)::int >= 140000 + AS allowed + SQL + + # The following is necessary to handle a mix of logical and physical replicas. We assume that if they have + # pg_replication_origin_status then they are a logical replica. In a logical replica we need to use + # `remote_lsn` rather than `pg_last_wal_replay_lsn` in order for our LSN to be comparable to the source + # cluster. This logic would be broken if we have 2 logical subscriptions or if we have a logical subscription + # in the source primary cluster. Read more at https://gitlab.com/gitlab-org/gitlab/-/merge_requests/121621 + LATEST_LSN_WITH_LOGICAL_QUERY = <<~SQL.squish.freeze + CASE + WHEN (SELECT TRUE FROM pg_replication_origin_status) THEN + (SELECT remote_lsn FROM pg_replication_origin_status) + WHEN pg_is_in_recovery() THEN + pg_last_wal_replay_lsn() + ELSE + pg_current_wal_insert_lsn() + END + SQL + + LATEST_LSN_WITHOUT_LOGICAL_QUERY = <<~SQL.squish.freeze + CASE + WHEN pg_is_in_recovery() THEN + pg_last_wal_replay_lsn() + ELSE + pg_current_wal_insert_lsn() + END + SQL + # host - The address of the database. # load_balancer - The LoadBalancer that manages this Host. def initialize(host, load_balancer, port: nil) @@ -30,6 +67,7 @@ module Gitlab @online = true @last_checked_at = Time.zone.now + # Randomly somewhere in between interval and 2*interval we'll refresh the status of the host interval = load_balancer.configuration.replica_check_interval @intervals = (interval..(interval * 2)).step(0.5).to_a end @@ -91,6 +129,7 @@ module Gitlab end def refresh_status + @latest_lsn_query = nil # Periodically clear the cached @latest_lsn_query value in case permissions change @online = replica_is_up_to_date? @last_checked_at = Time.zone.now end @@ -142,11 +181,11 @@ module Gitlab # primary. # # This method will return nil if no lag size could be calculated. - def replication_lag_size - location = connection.quote(primary_write_location) + def replication_lag_size(location = primary_write_location) + location = connection.quote(location) + row = query_and_release(<<-SQL.squish) - SELECT pg_wal_lsn_diff(#{location}, pg_last_wal_replay_lsn())::float - AS diff + SELECT pg_wal_lsn_diff(#{location}, (#{latest_lsn_query}))::float AS diff SQL row['diff'].to_i if row.any? @@ -173,22 +212,8 @@ module Gitlab # # location - The transaction write location as reported by a primary. def caught_up?(location) - string = connection.quote(location) - - # In case the host is a primary pg_last_wal_replay_lsn/pg_last_xlog_replay_location() returns - # NULL. The recovery check ensures we treat the host as up-to-date in - # such a case. - query = <<-SQL.squish - SELECT NOT pg_is_in_recovery() - OR pg_wal_lsn_diff(pg_last_wal_replay_lsn(), #{string}) >= 0 - AS result - SQL - - row = query_and_release(query) - - ::Gitlab::Utils.to_boolean(row['result']) - rescue *CONNECTION_ERRORS - false + lag = replication_lag_size(location) + lag.present? && lag.to_i <= 0 end def query_and_release(sql) @@ -198,6 +223,22 @@ module Gitlab ensure release_connection end + + private + + def can_track_logical_lsn? + row = query_and_release(CAN_TRACK_LOGICAL_LSN_QUERY) + + ::Gitlab::Utils.to_boolean(row['allowed']) + rescue *CONNECTION_ERRORS + false + end + + # The LATEST_LSN_WITH_LOGICAL query requires permissions that may not be present in self-managed configurations. + # We fallback gracefully to the query that does not correctly handle logical replicas for such configurations. + def latest_lsn_query + @latest_lsn_query ||= can_track_logical_lsn? ? LATEST_LSN_WITH_LOGICAL_QUERY : LATEST_LSN_WITHOUT_LOGICAL_QUERY + end end end end diff --git a/lib/gitlab/database/lock_writes_manager.rb b/lib/gitlab/database/lock_writes_manager.rb index 43e71e6bda2..8ddd871f93c 100644 --- a/lib/gitlab/database/lock_writes_manager.rb +++ b/lib/gitlab/database/lock_writes_manager.rb @@ -51,10 +51,15 @@ module Gitlab execute_sql_statement(sql_statement) - result_hash(action: 'locked') + result_hash(action: dry_run ? 'needs_lock' : 'locked') end def unlock_writes + unless table_locked_for_writes? + logger&.info "Skipping unlock_writes, because #{table_name} is already unlocked for writes" + return result_hash(action: 'skipped') + end + logger&.info "Database: '#{database_name}', Table: '#{table_name}': Allow Writes".color(:green) sql_statement = <<~SQL DROP TRIGGER IF EXISTS #{write_trigger_name} ON #{table_name}; @@ -62,7 +67,7 @@ module Gitlab execute_sql_statement(sql_statement) - result_hash(action: 'unlocked') + result_hash(action: dry_run ? 'needs_unlock' : 'unlocked') end private diff --git a/lib/gitlab/database/migration_helpers/wraparound_autovacuum.rb b/lib/gitlab/database/migration_helpers/wraparound_autovacuum.rb new file mode 100644 index 00000000000..555efb58606 --- /dev/null +++ b/lib/gitlab/database/migration_helpers/wraparound_autovacuum.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module MigrationHelpers + module WraparoundAutovacuum + # This is used for partitioning CI tables because the autovacuum for wraparound + # prevention can take many hours to complete on some of the tables and this in + # turn blocks the post deployment migrations pipeline. + # Intended workflow for this helper: + # 1. Introduce a migration that is guarded with this helper + # 2. Check that the migration was successfully executed on .com + # 3. Introduce the migration again for self-managed. + # + def can_execute_on?(*tables) + return false unless Gitlab.com? || Gitlab.dev_or_test_env? + + if wraparound_prevention_on_tables?(tables) + Gitlab::AppLogger.info(message: "Wraparound prevention vacuum detected", class: self.class) + say "Wraparound prevention vacuum detected, skipping migration" + return false + end + + true + end + + def wraparound_prevention_on_tables?(tables) + Gitlab::Database::PostgresAutovacuumActivity.reset_column_information + + Gitlab::Database::PostgresAutovacuumActivity + .wraparound_prevention + .for_tables(tables) + .any? + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/constraints_helpers.rb b/lib/gitlab/database/migrations/constraints_helpers.rb index 5aafc9f1444..1771f1f0fa7 100644 --- a/lib/gitlab/database/migrations/constraints_helpers.rb +++ b/lib/gitlab/database/migrations/constraints_helpers.rb @@ -262,6 +262,16 @@ module Gitlab SQL end + def switch_constraint_names(table_name, constraint_a, constraint_b) + validate_not_in_transaction!(:switch_constraint_names) + + with_lock_retries do + rename_constraint(table_name, constraint_a, :temp_name_for_renaming) + rename_constraint(table_name, constraint_b, constraint_a) + rename_constraint(table_name, :temp_name_for_renaming, constraint_b) + end + end + def validate_check_constraint_name!(constraint_name) return unless constraint_name.to_s.length > MAX_IDENTIFIER_NAME_LENGTH diff --git a/lib/gitlab/database/partitioning.rb b/lib/gitlab/database/partitioning.rb index 7222f148b3f..9895a68ec8d 100644 --- a/lib/gitlab/database/partitioning.rb +++ b/lib/gitlab/database/partitioning.rb @@ -40,7 +40,7 @@ module Gitlab next if model < ::Gitlab::Database::SharedModel && !(model < TableWithoutModel) model_connection_name = model.connection_db_config.name - Gitlab::Database::EachDatabase.each_database_connection do |connection, connection_name| + Gitlab::Database::EachDatabase.each_db_connection(include_shared: false) do |connection, connection_name| if connection_name != model_connection_name PartitionManager.new(model, connection: connection).sync_partitions end @@ -60,6 +60,8 @@ module Gitlab end def drop_detached_partitions + return unless Feature.enabled?(:partition_manager_sync_partitions, type: :ops) + Gitlab::AppLogger.info(message: 'Dropping detached postgres partitions') Gitlab::Database::EachDatabase.each_database_connection do diff --git a/lib/gitlab/database/partitioning/list/convert_table.rb b/lib/gitlab/database/partitioning/list/convert_table.rb index d4fb150d956..9889d01be76 100644 --- a/lib/gitlab/database/partitioning/list/convert_table.rb +++ b/lib/gitlab/database/partitioning/list/convert_table.rb @@ -11,12 +11,11 @@ module Gitlab PARTITIONING_CONSTRAINT_NAME = 'partitioning_constraint' - attr_reader :partitioning_column, :table_name, :parent_table_name, :zero_partition_value, - :locking_configuration + attr_reader :partitioning_column, :table_name, :parent_table_name, :zero_partition_value def initialize( migration_context:, table_name:, parent_table_name:, partitioning_column:, - zero_partition_value:, lock_tables: []) + zero_partition_value:) @migration_context = migration_context @connection = migration_context.connection @@ -24,7 +23,6 @@ module Gitlab @parent_table_name = parent_table_name @partitioning_column = partitioning_column @zero_partition_value = zero_partition_value - @locking_configuration = LockingConfiguration.new(migration_context, table_locking_order: lock_tables) end def prepare_for_partitioning(async: false) @@ -38,22 +36,24 @@ module Gitlab end def partition - assert_existing_constraints_partitionable - assert_partitioning_constraint_present - - create_parent_table - attach_foreign_keys_to_parent - locking_sql = locking_configuration.locking_statement_for(tables_that_will_lock_during_partitioning) - - locking_configuration.with_lock_retries do - # Loose FKs trigger will exclusively lock the table and it might - # not follow the locking order needed to attach the partition. - migration_context.execute(locking_sql) if locking_sql.present? - - redefine_loose_foreign_key_triggers do - migration_context.execute(sql_to_convert_table) + # If already partitioned, the table is no longer partitionable. Thus we skip checks leading up + # to partitioning if the partitioning transaction has already succeeded. + unless already_partitioned? + assert_existing_constraints_partitionable + assert_partitioning_constraint_present + + create_parent_table + + migration_context.with_lock_retries do + redefine_loose_foreign_key_triggers do + migration_context.execute(sql_to_convert_table) + end end end + + # Attaching foreign keys handles cases where one or more foreign keys already exists, so it doesn't + # need a check similar to the rest of this method + attach_foreign_keys_to_parent end def revert_partitioning @@ -194,6 +194,8 @@ module Gitlab # At this point no other connection knows about the parent table. # Thus the only contended lock in the following transaction is on fk.to_table. # So a deadlock is impossible. + # (We also take a share update exclusive lock against the recently attached child table, + # but that only blocks vacuum and other schema modifications, not reads or writes) # If we're rerunning this migration after a failure to acquire a lock, the foreign key might already exist # Don't try to recreate it in that case @@ -299,16 +301,8 @@ module Gitlab end end - def tables_that_will_lock_during_partitioning - # Locks are taken against the table + all tables that reference it by foreign key - # postgres_foreign_keys.referenced_table_name gives the table name that we need here directly, but that - # column did not exist yet during the migration 20221021145820_create_routing_table_for_builds_metadata_v2 - # To ensure compatibility with that migration if it is run with this code, use referenced_table_identifier - # here. - referenced_tables = Gitlab::Database::PostgresForeignKey - .by_constrained_table_identifier(table_identifier) - .map { |fk| table_name_for_identifier(fk.referenced_table_identifier) } - referenced_tables + [table_name] + def already_partitioned? + Gitlab::Database::PostgresPartition.for_identifier(table_identifier).exists? end end end diff --git a/lib/gitlab/database/partitioning/sliding_list_strategy.rb b/lib/gitlab/database/partitioning/sliding_list_strategy.rb index 5bb34a86d43..8f8afdfc551 100644 --- a/lib/gitlab/database/partitioning/sliding_list_strategy.rb +++ b/lib/gitlab/database/partitioning/sliding_list_strategy.rb @@ -77,6 +77,19 @@ module Gitlab end def validate_and_fix + unless model.connection_db_config.name == + Gitlab::Database.db_config_name(Gitlab::Database::SharedModel.connection) + + Gitlab::AppLogger.warn( + message: 'Skipping fixing column default because connections mismatch', + event: :partition_manager_validate_and_fix_connection_mismatch, + model_connection_name: Gitlab::Database.db_config_name(model.connection), + shared_connection_name: Gitlab::Database.db_config_name(Gitlab::Database::SharedModel.connection) + ) + + return + end + return if no_partitions_exist? old_default_value = current_default_value diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb index 61e95dbe1a4..1ce0a44e37f 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb @@ -287,8 +287,7 @@ module Gitlab table_name: table_name, parent_table_name: parent_table_name, partitioning_column: partitioning_column, - zero_partition_value: initial_partitioning_value, - lock_tables: lock_tables + zero_partition_value: initial_partitioning_value ).partition end diff --git a/lib/gitlab/database/postgres_autovacuum_activity.rb b/lib/gitlab/database/postgres_autovacuum_activity.rb index a4dc199c259..6a80c631abb 100644 --- a/lib/gitlab/database/postgres_autovacuum_activity.rb +++ b/lib/gitlab/database/postgres_autovacuum_activity.rb @@ -6,9 +6,14 @@ module Gitlab self.table_name = 'postgres_autovacuum_activity' self.primary_key = 'table_identifier' + scope :wraparound_prevention, -> { where(wraparound_prevention: true) } + def self.for_tables(tables) Gitlab::Database::LoadBalancing::Session.current.use_primary do - where('schema = current_schema()').where(table: tables) + # calling `.to_a` here to execute the query in the primary's scope + # and to avoid having the scope chained and re-executed + # + where('schema = current_schema()').where(table: tables).to_a end end diff --git a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb index d15a0eaa44c..b5a45eed3a4 100644 --- a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb +++ b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb @@ -5,6 +5,7 @@ module Gitlab module QueryAnalyzers class PreventCrossDatabaseModification < Database::QueryAnalyzers::Base CrossDatabaseModificationAcrossUnsupportedTablesError = Class.new(QueryAnalyzerError) + QUERY_LIMIT = 10 # This method will allow cross database modifications within the block # Example: @@ -42,7 +43,8 @@ module Gitlab context.merge!({ transaction_depth_by_db: Hash.new { |h, k| h[k] = 0 }, modified_tables_by_db: Hash.new { |h, k| h[k] = Set.new }, - ignored_tables: [] + ignored_tables: [], + queries: [] }) end @@ -71,6 +73,7 @@ module Gitlab context[:transaction_depth_by_db][database] -= 1 if context[:transaction_depth_by_db][database] == 0 context[:modified_tables_by_db][database].clear + clear_queries # Attempt to troubleshoot https://gitlab.com/gitlab-org/gitlab/-/issues/351531 ::CrossDatabaseModification::TransactionStackTrackRecord.log_gitlab_transactions_stack(action: :end_of_transaction) @@ -104,22 +107,25 @@ module Gitlab # databases return if tables == ['schema_migrations'] + add_to_queries(sql) context[:modified_tables_by_db][database].merge(tables) all_tables = context[:modified_tables_by_db].values.flat_map(&:to_a) schemas = ::Gitlab::Database::GitlabSchema.table_schemas!(all_tables) - schemas += ApplicationRecord.gitlab_transactions_stack - if schemas.many? - message = "Cross-database data modification of '#{schemas.to_a.join(", ")}' were detected within " \ - "a transaction modifying the '#{all_tables.to_a.join(", ")}' tables. " \ - "Please refer to https://docs.gitlab.com/ee/development/database/multiple_databases.html#removing-cross-database-transactions for details on how to resolve this exception." + unless ::Gitlab::Database::GitlabSchema.cross_transactions_allowed?(schemas) + messages = [] + messages << "Cross-database data modification of '#{schemas.to_a.join(", ")}' were detected within " \ + "a transaction modifying the '#{all_tables.to_a.join(", ")}' tables. " + messages << "Please refer to https://docs.gitlab.com/ee/development/database/multiple_databases.html#removing-cross-database-transactions " \ + "for details on how to resolve this exception." + messages += cleaned_queries - raise CrossDatabaseModificationAcrossUnsupportedTablesError, message + raise CrossDatabaseModificationAcrossUnsupportedTablesError, messages.join("\n\n") end rescue CrossDatabaseModificationAcrossUnsupportedTablesError => e ::Gitlab::ErrorTracking.track_exception(e, { gitlab_schemas: schemas, tables: all_tables, query: parsed.sql }) - raise if raise_exception? + raise if dev_or_test_env? end # rubocop:enable Metrics/AbcSize @@ -159,12 +165,28 @@ module Gitlab end end - # We only raise in tests for now otherwise some features will be broken - # in development. For now we've mostly only added allowlist based on - # spec names. Until we have allowed all the violations inline we don't - # want to raise in development. - def self.raise_exception? - Rails.env.test? + def self.dev_or_test_env? + Gitlab.dev_or_test_env? + end + + def self.clear_queries + return unless dev_or_test_env? + + context[:queries].clear + end + + def self.add_to_queries(sql) + return unless dev_or_test_env? + + context[:queries].push(sql) + end + + def self.cleaned_queries + return [] unless dev_or_test_env? + + context[:queries].last(QUERY_LIMIT).each_with_index.map do |sql, i| + "#{i}: #{sql}" + end end def self.in_transaction? diff --git a/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb b/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb index f3e0fc26946..4e1ab700542 100644 --- a/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb +++ b/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb @@ -19,7 +19,8 @@ module Gitlab gitlab_internal: nil, # Pods specific changes - gitlab_main_clusterwide: :gitlab_main + gitlab_main_clusterwide: :gitlab_main, + gitlab_main_cell: :gitlab_main }.freeze class << self @@ -61,6 +62,7 @@ module Gitlab def restrict_to_ddl_only(parsed) tables = self.dml_tables(parsed) schemas = self.dml_schemas(tables) + schemas = self.map_schemas(schemas) if schemas.any? self.raise_dml_not_allowed_error("Modifying of '#{tables}' (#{schemas.to_a}) with '#{parsed.sql}'") @@ -78,8 +80,10 @@ module Gitlab tables = self.dml_tables(parsed) schemas = self.dml_schemas(tables) + schemas = self.map_schemas(schemas) + allowed_schemas = self.map_schemas(self.allowed_gitlab_schemas) - if (schemas - self.allowed_gitlab_schemas).any? + if (schemas - allowed_schemas).any? raise DMLAccessDeniedError, \ "Select/DML queries (SELECT/UPDATE/DELETE) do access '#{tables}' (#{schemas.to_a}) " \ "which is outside of list of allowed schemas: '#{self.allowed_gitlab_schemas}'. " \ @@ -100,15 +104,19 @@ module Gitlab end def dml_schemas(tables) - extra_schemas = ::Gitlab::Database::GitlabSchema.table_schemas!(tables) + ::Gitlab::Database::GitlabSchema.table_schemas!(tables) + end + + def map_schemas(schemas) + schemas = schemas.to_set - SCHEMA_MAPPING.each do |schema, mapped_schema| - next unless extra_schemas.delete?(schema) + SCHEMA_MAPPING.each do |in_schema, mapped_schema| + next unless schemas.delete?(in_schema) - extra_schemas.add(mapped_schema) if mapped_schema + schemas.add(mapped_schema) if mapped_schema end - extra_schemas + schemas end def raise_dml_not_allowed_error(message) diff --git a/lib/gitlab/database/schema_validation/adapters/column_structure_sql_adapter.rb b/lib/gitlab/database/schema_validation/adapters/column_structure_sql_adapter.rb index 420195d89dd..20814b098c1 100644 --- a/lib/gitlab/database/schema_validation/adapters/column_structure_sql_adapter.rb +++ b/lib/gitlab/database/schema_validation/adapters/column_structure_sql_adapter.rb @@ -104,7 +104,7 @@ module Gitlab when :func_call "#{parse_node(node.func_call.funcname.first)}()" when :a_const - parse_node(node.a_const.val) + parse_a_const(node.a_const) when :type_cast value = parse_node(node.type_cast.arg) type = type(node.type_cast.type_name) @@ -112,10 +112,21 @@ module Gitlab [MAPPINGS.fetch(value, "'#{value}'"), separator].compact.join('') else - node.to_h[node.node].values.last + get_value_from_key(node, key: node.node) end end + def parse_a_const(a_const) + return unless a_const + + type = a_const.val + get_value_from_key(a_const, key: type) + end + + def get_value_from_key(node, key:) + node.to_h[key].values.last + end + def partition_keys return [] unless partitioning_stmt diff --git a/lib/gitlab/database/schema_validation/adapters/foreign_key_database_adapter.rb b/lib/gitlab/database/schema_validation/adapters/foreign_key_database_adapter.rb new file mode 100644 index 00000000000..3b45f5c77ca --- /dev/null +++ b/lib/gitlab/database/schema_validation/adapters/foreign_key_database_adapter.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + module Adapters + class ForeignKeyDatabaseAdapter + def initialize(query_result) + @query_result = query_result + end + + def name + "#{query_result['schema']}.#{query_result['foreign_key_name']}" + end + + def table_name + query_result['table_name'] + end + + def statement + query_result['foreign_key_definition'] + end + + private + + attr_reader :query_result + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/adapters/foreign_key_structure_sql_adapter.rb b/lib/gitlab/database/schema_validation/adapters/foreign_key_structure_sql_adapter.rb new file mode 100644 index 00000000000..e4c1e1adab3 --- /dev/null +++ b/lib/gitlab/database/schema_validation/adapters/foreign_key_structure_sql_adapter.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + module Adapters + class ForeignKeyStructureSqlAdapter + STATEMENT_REGEX = /\bREFERENCES\s\K\S+\K\s\(/ + EXTRACT_REGEX = /\bFOREIGN KEY.*/ + + def initialize(parsed_stmt) + @parsed_stmt = parsed_stmt + end + + def name + "#{schema_name}.#{foreign_key_name}" + end + + def table_name + parsed_stmt.relation.relname + end + + # PgQuery parses FK statements with an extra space in the referenced table column. + # This extra space needs to be removed. + # + # @example REFERENCES ci_pipelines (id) => REFERENCES ci_pipelines(id) + def statement + deparse_stmt[EXTRACT_REGEX].gsub(STATEMENT_REGEX, '(') + end + + private + + attr_reader :parsed_stmt + + def schema_name + parsed_stmt.relation.schemaname + end + + def foreign_key_name + parsed_stmt.cmds.first.alter_table_cmd.def.constraint.conname + end + + def deparse_stmt + PgQuery.deparse_stmt(parsed_stmt) + end + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/database.rb b/lib/gitlab/database/schema_validation/database.rb index 7a0e348a27b..858bf618f44 100644 --- a/lib/gitlab/database/schema_validation/database.rb +++ b/lib/gitlab/database/schema_validation/database.rb @@ -18,6 +18,10 @@ module Gitlab trigger_map[trigger_name] end + def fetch_foreign_key_by_name(foreign_key_name) + foreign_key_map[foreign_key_name] + end + def fetch_table_by_name(table_name) table_map[table_name] end @@ -30,6 +34,10 @@ module Gitlab trigger_map[trigger_name].present? end + def foreign_key_exists?(foreign_key_name) + fetch_foreign_key_by_name(foreign_key_name).present? + end + def table_exists?(table_name) fetch_table_by_name(table_name).present? end @@ -42,6 +50,10 @@ module Gitlab trigger_map.values end + def foreign_keys + foreign_key_map.values + end + def tables table_map.values end @@ -68,6 +80,14 @@ module Gitlab end end + def foreign_key_map + @foreign_key_map ||= fetch_fks.each_with_object({}) do |stmt, result| + adapter = Adapters::ForeignKeyDatabaseAdapter.new(stmt) + + result[adapter.name] = SchemaObjects::ForeignKey.new(adapter) + end + end + def table_map @table_map ||= fetch_tables.transform_values! do |stmt| columns = stmt.map { |column| SchemaObjects::Column.new(Adapters::ColumnDatabaseAdapter.new(column)) } @@ -122,6 +142,24 @@ module Gitlab connection.exec_query(sql, nil, schemas).group_by { |row| row['table_name'] } end + + def fetch_fks + sql = <<~SQL + SELECT + pg_namespace.nspname::text AS schema, + pg_class.relname::text AS table_name, + pg_constraint.conname AS foreign_key_name, + pg_get_constraintdef(pg_constraint.oid) AS foreign_key_definition + FROM pg_constraint + INNER JOIN pg_class ON pg_constraint.conrelid = pg_class.oid + INNER JOIN pg_namespace ON pg_class.relnamespace = pg_namespace.oid + WHERE contype = 'f' + AND pg_namespace.nspname = $1 + AND pg_constraint.conparentid = 0 + SQL + + connection.exec_query(sql, nil, [connection.current_schema]) + end end end end diff --git a/lib/gitlab/database/schema_validation/schema_inconsistency.rb b/lib/gitlab/database/schema_validation/schema_inconsistency.rb index 6f50603e784..9f39db5b4c0 100644 --- a/lib/gitlab/database/schema_validation/schema_inconsistency.rb +++ b/lib/gitlab/database/schema_validation/schema_inconsistency.rb @@ -8,7 +8,9 @@ module Gitlab belongs_to :issue - validates :object_name, :valitador_name, :table_name, presence: true + validates :object_name, :valitador_name, :table_name, :diff, presence: true + + scope :with_open_issues, -> { joins(:issue).where('issue.state_id': Issue.available_states[:opened]) } end end end diff --git a/lib/gitlab/database/schema_validation/schema_objects/foreign_key.rb b/lib/gitlab/database/schema_validation/schema_objects/foreign_key.rb new file mode 100644 index 00000000000..b616b1a72b7 --- /dev/null +++ b/lib/gitlab/database/schema_validation/schema_objects/foreign_key.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + module SchemaObjects + class ForeignKey + def initialize(adapter) + @adapter = adapter + end + + # Foreign key name should include the schema, as the same name could be used across different schemas + # + # @example public.foreign_key_name + def name + @name ||= adapter.name + end + + def table_name + @table_name ||= adapter.table_name + end + + def statement + @statement ||= adapter.statement + end + + private + + attr_reader :adapter + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/structure_sql.rb b/lib/gitlab/database/schema_validation/structure_sql.rb index 299db1c2a7e..4d6fa17f0fc 100644 --- a/lib/gitlab/database/schema_validation/structure_sql.rb +++ b/lib/gitlab/database/schema_validation/structure_sql.rb @@ -19,6 +19,10 @@ module Gitlab triggers.find { |trigger| trigger.name == trigger_name }.present? end + def foreign_key_exists?(foreign_key_name) + foreign_keys.find { |fk| fk.name == foreign_key_name }.present? + end + def fetch_table_by_name(table_name) tables.find { |table| table.name == table_name } end @@ -35,6 +39,14 @@ module Gitlab @triggers ||= map_with_default_schema(trigger_statements, SchemaObjects::Trigger) end + def foreign_keys + @foreign_keys ||= foreign_key_statements.map do |stmt| + stmt.relation.schemaname = schema_name if stmt.relation.schemaname == '' + + SchemaObjects::ForeignKey.new(Adapters::ForeignKeyStructureSqlAdapter.new(stmt)) + end + end + def tables @tables ||= table_statements.map do |stmt| table_name = stmt.relation.relname @@ -65,6 +77,33 @@ module Gitlab statements.filter_map { |s| s.stmt.create_stmt } end + def foreign_key_statements + constraint_statements(:CONSTR_FOREIGN) + end + + # Filter constraint statement nodes + # + # @param constraint_type [Symbol] node type. One of CONSTR_PRIMARY, CONSTR_CHECK, CONSTR_EXCLUSION, + # CONSTR_UNIQUE or CONSTR_FOREIGN. + def constraint_statements(constraint_type) + alter_table_statements(:AT_AddConstraint).filter do |stmt| + stmt.cmds.first.alter_table_cmd.def.constraint.contype == constraint_type + end + end + + # Filter alter table statement nodes + # + # @param subtype [Symbol] node subtype +AT_AttachPartition+, +AT_ColumnDefault+ or +AT_AddConstraint+ + def alter_table_statements(subtype) + statements.filter_map do |statement| + node = statement.stmt.alter_table_stmt + + next unless node + + node if node.cmds.first.alter_table_cmd.subtype == subtype + end + end + def statements @statements ||= parsed_structure_file.tree.stmts end diff --git a/lib/gitlab/database/schema_validation/track_inconsistency.rb b/lib/gitlab/database/schema_validation/track_inconsistency.rb index 47c3492971c..6e167653d32 100644 --- a/lib/gitlab/database/schema_validation/track_inconsistency.rb +++ b/lib/gitlab/database/schema_validation/track_inconsistency.rb @@ -4,6 +4,8 @@ module Gitlab module Database module SchemaValidation class TrackInconsistency + COLUMN_TEXT_LIMIT = 6144 + def initialize(inconsistency, project, user) @inconsistency = inconsistency @project = project @@ -12,10 +14,13 @@ module Gitlab def execute return unless Gitlab.com? - return if inconsistency_record.present? + return refresh_issue if inconsistency_record.present? - result = ::Issues::CreateService.new(container: project, current_user: user, params: params, - spam_params: nil).execute + result = ::Issues::CreateService.new( + container: project, + current_user: user, + params: params, + perform_spam_check: false).execute track_inconsistency(result[:issue]) if result.success? end @@ -25,20 +30,21 @@ module Gitlab attr_reader :inconsistency, :project, :user def track_inconsistency(issue) - schema_inconsistency_model.create( + schema_inconsistency_model.create!( issue: issue, object_name: inconsistency.object_name, table_name: inconsistency.table_name, - valitador_name: inconsistency.type + valitador_name: inconsistency.type, + diff: inconsistency_diff ) end def params { title: issue_title, - description: issue_description, + description: description, issue_type: 'issue', - labels: %w[database database-inconsistency-report] + labels: default_labels + group_labels } end @@ -46,7 +52,7 @@ module Gitlab "New schema inconsistency: #{inconsistency.object_name}" end - def issue_description + def description <<~MSG We have detected a new schema inconsistency. @@ -81,12 +87,46 @@ module Gitlab MSG end + def group_labels + dictionary = YAML.safe_load(File.read(table_file_path)) + + dictionary['feature_categories'].to_a.filter_map do |feature_category| + Gitlab::Database::ConvertFeatureCategoryToGroupLabel.new(feature_category).execute + end + rescue Errno::ENOENT + [] + end + + def default_labels + %w[database database-inconsistency-report type::maintenance severity::4] + end + + def table_file_path + Rails.root.join(Gitlab::Database::GitlabSchema.dictionary_paths.first, "#{inconsistency.table_name}.yml") + end + def schema_inconsistency_model Gitlab::Database::SchemaValidation::SchemaInconsistency end + def refresh_issue + return if inconsistency_diff == inconsistency_record.diff # Nothing to refresh + + note = ::Notes::CreateService.new( + inconsistency_record.issue.project, + user, + { noteable_type: 'Issue', noteable: inconsistency_record.issue, note: description } + ).execute + + inconsistency_record.update!(diff: inconsistency_diff) if note.persisted? + end + + def inconsistency_diff + @inconsistency_diff ||= inconsistency.diff.to_s.first(COLUMN_TEXT_LIMIT) + end + def inconsistency_record - schema_inconsistency_model.find_by( + @inconsistency_record ||= schema_inconsistency_model.with_open_issues.find_by( object_name: inconsistency.object_name, table_name: inconsistency.table_name, valitador_name: inconsistency.type diff --git a/lib/gitlab/database/schema_validation/validators/base_validator.rb b/lib/gitlab/database/schema_validation/validators/base_validator.rb index 58e0bf5292b..ee322e50a2c 100644 --- a/lib/gitlab/database/schema_validation/validators/base_validator.rb +++ b/lib/gitlab/database/schema_validation/validators/base_validator.rb @@ -18,13 +18,16 @@ module Gitlab ExtraTableColumns, ExtraIndexes, ExtraTriggers, + ExtraForeignKeys, MissingTables, MissingTableColumns, MissingIndexes, MissingTriggers, + MissingForeignKeys, DifferentDefinitionTables, DifferentDefinitionIndexes, - DifferentDefinitionTriggers + DifferentDefinitionTriggers, + DifferentDefinitionForeignKeys ] end diff --git a/lib/gitlab/database/schema_validation/validators/different_definition_foreign_keys.rb b/lib/gitlab/database/schema_validation/validators/different_definition_foreign_keys.rb new file mode 100644 index 00000000000..8969fa76cd8 --- /dev/null +++ b/lib/gitlab/database/schema_validation/validators/different_definition_foreign_keys.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + module Validators + class DifferentDefinitionForeignKeys < BaseValidator + ERROR_MESSAGE = "The %s foreign key has a different statement between structure.sql and database" + + def execute + structure_sql.foreign_keys.filter_map do |structure_sql_fk| + database_fk = database.fetch_foreign_key_by_name(structure_sql_fk.name) + + next if database_fk.nil? + next if database_fk.statement == structure_sql_fk.statement + + build_inconsistency(self.class, structure_sql_fk, database_fk) + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/validators/extra_foreign_keys.rb b/lib/gitlab/database/schema_validation/validators/extra_foreign_keys.rb new file mode 100644 index 00000000000..887e86c7bfd --- /dev/null +++ b/lib/gitlab/database/schema_validation/validators/extra_foreign_keys.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + module Validators + class ExtraForeignKeys < BaseValidator + ERROR_MESSAGE = "The foreign key %s is present in the database, but not in the structure.sql file" + + def execute + database.foreign_keys.filter_map do |database_fk| + next if structure_sql.foreign_key_exists?(database_fk.name) + + build_inconsistency(self.class, nil, database_fk) + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/validators/missing_foreign_keys.rb b/lib/gitlab/database/schema_validation/validators/missing_foreign_keys.rb new file mode 100644 index 00000000000..b20f8474426 --- /dev/null +++ b/lib/gitlab/database/schema_validation/validators/missing_foreign_keys.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + module Validators + class MissingForeignKeys < BaseValidator + ERROR_MESSAGE = "The foreign key %s is missing from the database" + + def execute + structure_sql.foreign_keys.filter_map do |structure_sql_fk| + next if database.foreign_key_exists?(structure_sql_fk.name) + + build_inconsistency(self.class, structure_sql_fk, nil) + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/tables_locker.rb b/lib/gitlab/database/tables_locker.rb index 0b0d46f4b0e..02e0da022f9 100644 --- a/lib/gitlab/database/tables_locker.rb +++ b/lib/gitlab/database/tables_locker.rb @@ -5,10 +5,11 @@ module Gitlab class TablesLocker GITLAB_SCHEMAS_TO_IGNORE = %i[gitlab_embedding gitlab_geo].freeze - def initialize(logger: nil, dry_run: false) + def initialize(logger: nil, dry_run: false, include_partitions: true) @logger = logger @dry_run = dry_run @result = [] + @include_partitions = include_partitions end def unlock_writes @@ -50,6 +51,7 @@ module Gitlab # Unlocks the writes on the table and its partitions def unlock_writes_on_table(table_name, connection, database_name) @result << lock_writes_manager(table_name, connection, database_name).unlock_writes + return unless @include_partitions table_attached_partitions(table_name, connection) do |postgres_partition| @result << lock_writes_manager(postgres_partition.identifier, connection, database_name).unlock_writes @@ -59,6 +61,7 @@ module Gitlab # It locks the writes on the table and its partitions def lock_writes_on_table(table_name, connection, database_name) @result << lock_writes_manager(table_name, connection, database_name).lock_writes + return unless @include_partitions table_attached_partitions(table_name, connection) do |postgres_partition| @result << lock_writes_manager(postgres_partition.identifier, connection, database_name).lock_writes @@ -67,6 +70,7 @@ module Gitlab def tables_to_lock(connection, &block) Gitlab::Database::GitlabSchema.tables_to_schema.each(&block) + return unless @include_partitions Gitlab::Database::SharedModel.using_connection(connection) do Postgresql::DetachedPartition.find_each do |detached_partition| diff --git a/lib/gitlab/database/tables_truncate.rb b/lib/gitlab/database/tables_truncate.rb index a6430d1758b..7ae1981fa2b 100644 --- a/lib/gitlab/database/tables_truncate.rb +++ b/lib/gitlab/database/tables_truncate.rb @@ -3,7 +3,7 @@ module Gitlab module Database class TablesTruncate - GITLAB_SCHEMAS_TO_IGNORE = %i[gitlab_geo].freeze + GITLAB_SCHEMAS_TO_IGNORE = %i[gitlab_geo gitlab_embedding].freeze def initialize(database_name:, min_batch_size:, logger: nil, until_table: nil, dry_run: false) @database_name = database_name diff --git a/lib/gitlab/database_importers/common_metrics.rb b/lib/gitlab/database_importers/common_metrics.rb deleted file mode 100644 index f964ae8a275..00000000000 --- a/lib/gitlab/database_importers/common_metrics.rb +++ /dev/null @@ -1,8 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module DatabaseImporters - module CommonMetrics - end - end -end diff --git a/lib/gitlab/database_importers/common_metrics/importer.rb b/lib/gitlab/database_importers/common_metrics/importer.rb deleted file mode 100644 index 6c61e05674e..00000000000 --- a/lib/gitlab/database_importers/common_metrics/importer.rb +++ /dev/null @@ -1,78 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module DatabaseImporters - module CommonMetrics - class Importer - MissingQueryId = Class.new(StandardError) - - attr_reader :content - - def initialize(filename = 'common_metrics.yml') - @content = YAML.load_file(Rails.root.join('config', 'prometheus', filename)) - end - - def execute - CommonMetrics::PrometheusMetric.reset_column_information - - process_content do |id, attributes| - find_or_build_metric!(id) - .update!(**attributes) - end - end - - private - - def process_content(&blk) - content['panel_groups'].map do |group| - process_group(group, &blk) - end - end - - def process_group(group, &blk) - attributes = { - group: find_group_title_key(group['group']) - } - - group['panels'].map do |panel| - process_panel(panel, attributes, &blk) - end - end - - def process_panel(panel, attributes, &blk) - attributes = attributes.merge( - title: panel['title'], - y_label: panel['y_label']) - - panel['metrics'].map do |metric_details| - process_metric_details(metric_details, attributes, &blk) - end - end - - def process_metric_details(metric_details, attributes, &blk) - attributes = attributes.merge( - legend: metric_details['label'], - query: metric_details['query_range'], - unit: metric_details['unit']) - - yield(metric_details['id'], attributes) - end - - def find_or_build_metric!(id) - raise MissingQueryId unless id - - CommonMetrics::PrometheusMetric.common.find_by(identifier: id) || - CommonMetrics::PrometheusMetric.new(common: true, identifier: id) - end - - def find_group_title_key(title) - CommonMetrics::PrometheusMetricEnums.groups[find_group_title(title)] - end - - def find_group_title(title) - CommonMetrics::PrometheusMetricEnums.group_titles.invert[title] - end - end - end - end -end diff --git a/lib/gitlab/database_importers/common_metrics/prometheus_metric.rb b/lib/gitlab/database_importers/common_metrics/prometheus_metric.rb deleted file mode 100644 index b4a392cbea9..00000000000 --- a/lib/gitlab/database_importers/common_metrics/prometheus_metric.rb +++ /dev/null @@ -1,12 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module DatabaseImporters - module CommonMetrics - class PrometheusMetric < ApplicationRecord - enum group: PrometheusMetricEnums.groups - scope :common, -> { where(common: true) } - end - end - end -end diff --git a/lib/gitlab/database_importers/common_metrics/prometheus_metric_enums.rb b/lib/gitlab/database_importers/common_metrics/prometheus_metric_enums.rb deleted file mode 100644 index 8a5f53be20f..00000000000 --- a/lib/gitlab/database_importers/common_metrics/prometheus_metric_enums.rb +++ /dev/null @@ -1,45 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module DatabaseImporters - module CommonMetrics - module PrometheusMetricEnums - def self.groups - { - # built-in groups - nginx_ingress_vts: -1, - ha_proxy: -2, - aws_elb: -3, - nginx: -4, - kubernetes: -5, - nginx_ingress: -6, - - # custom groups - business: 0, - response: 1, - system: 2, - custom: 3, - - cluster_health: -100 - } - end - - def self.group_titles - { - business: _('Business metrics (Custom)'), - response: _('Response metrics (Custom)'), - system: _('System metrics (Custom)'), - nginx_ingress_vts: _('Response metrics (NGINX Ingress VTS)'), - nginx_ingress: _('Response metrics (NGINX Ingress)'), - ha_proxy: _('Response metrics (HA Proxy)'), - aws_elb: _('Response metrics (AWS ELB)'), - nginx: _('Response metrics (NGINX)'), - kubernetes: _('System metrics (Kubernetes)'), - cluster_health: _('Cluster Health'), - custom: _('Custom metrics') - } - end - end - end - end -end diff --git a/lib/gitlab/database_importers/default_organization_importer.rb b/lib/gitlab/database_importers/default_organization_importer.rb new file mode 100644 index 00000000000..147c0d19b01 --- /dev/null +++ b/lib/gitlab/database_importers/default_organization_importer.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module DatabaseImporters + module DefaultOrganizationImporter + def self.create_default_organization + return if Organizations::Organization.default_organization + + # When adding or changing attributes, consider changing the factory for Organization model as well + # spec/factories/organizations/organizations.rb + Organizations::Organization.create!( + id: Organizations::Organization::DEFAULT_ORGANIZATION_ID, + name: 'Default', + path: 'default' + ) + end + end + end +end diff --git a/lib/gitlab/dependency_linker/requirements_txt_linker.rb b/lib/gitlab/dependency_linker/requirements_txt_linker.rb index f630c13b760..dc654964e0b 100644 --- a/lib/gitlab/dependency_linker/requirements_txt_linker.rb +++ b/lib/gitlab/dependency_linker/requirements_txt_linker.rb @@ -9,7 +9,7 @@ module Gitlab def link_dependencies link_regex(/^(?(?![a-z+]+:)[^#.-][^ ><=~!;\[]+)/) do |name| - "https://pypi.python.org/pypi/#{name}" + "https://pypi.org/project/#{name}/" end link_regex(%r{^(?https?://[^ ]+)}, &:itself) diff --git a/lib/gitlab/devise_failure.rb b/lib/gitlab/devise_failure.rb index ffd057e1d33..2c48ad3b46b 100644 --- a/lib/gitlab/devise_failure.rb +++ b/lib/gitlab/devise_failure.rb @@ -7,6 +7,14 @@ module Gitlab def http_auth? request_format && super end + + def respond + if warden_options[:reason] == :too_many_requests + self.status = 403 + else + super + end + end end end diff --git a/lib/gitlab/diff/formatters/base_formatter.rb b/lib/gitlab/diff/formatters/base_formatter.rb index 19fc028594c..807ce3682ab 100644 --- a/lib/gitlab/diff/formatters/base_formatter.rb +++ b/lib/gitlab/diff/formatters/base_formatter.rb @@ -9,6 +9,7 @@ module Gitlab attr_reader :base_sha attr_reader :start_sha attr_reader :head_sha + attr_reader :ignore_whitespace_change def initialize(attrs) if diff_file = attrs[:diff_file] diff --git a/lib/gitlab/diff/formatters/file_formatter.rb b/lib/gitlab/diff/formatters/file_formatter.rb new file mode 100644 index 00000000000..37b9ad85ef8 --- /dev/null +++ b/lib/gitlab/diff/formatters/file_formatter.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Diff + module Formatters + class FileFormatter < BaseFormatter + def initialize(attrs) + @ignore_whitespace_change = false + + super(attrs) + end + + def key + @key ||= super.push(new_path, old_path) + end + + def position_type + "file" + end + + def complete? + [new_path, old_path].all?(&:present?) + end + + def ==(other) + other.is_a?(self.class) && + old_path == other.old_path && + new_path == other.new_path + end + end + end + end +end diff --git a/lib/gitlab/diff/formatters/image_formatter.rb b/lib/gitlab/diff/formatters/image_formatter.rb index d0c13dee1aa..f0d25885387 100644 --- a/lib/gitlab/diff/formatters/image_formatter.rb +++ b/lib/gitlab/diff/formatters/image_formatter.rb @@ -14,6 +14,7 @@ module Gitlab @y = attrs[:y] @width = attrs[:width] @height = attrs[:height] + @ignore_whitespace_change = false super(attrs) end diff --git a/lib/gitlab/diff/formatters/text_formatter.rb b/lib/gitlab/diff/formatters/text_formatter.rb index 9ea9bdfdf15..bd083720165 100644 --- a/lib/gitlab/diff/formatters/text_formatter.rb +++ b/lib/gitlab/diff/formatters/text_formatter.rb @@ -12,6 +12,7 @@ module Gitlab @old_line = attrs[:old_line] @new_line = attrs[:new_line] @line_range = attrs[:line_range] + @ignore_whitespace_change = !!attrs[:ignore_whitespace_change] super(attrs) end @@ -25,7 +26,8 @@ module Gitlab end def to_h - super.merge(old_line: old_line, new_line: new_line, line_range: line_range) + super.merge(old_line: old_line, new_line: new_line, line_range: line_range, + ignore_whitespace_change: ignore_whitespace_change) end def line_age diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb index 40b6ae2f14e..feee4bcc7f9 100644 --- a/lib/gitlab/diff/position.rb +++ b/lib/gitlab/diff/position.rb @@ -19,7 +19,8 @@ module Gitlab :x, :y, :line_range, - :position_type, to: :formatter + :position_type, + :ignore_whitespace_change, to: :formatter # A position can belong to a text line or to an image coordinate # it depends of the position_type argument. @@ -69,11 +70,11 @@ module Gitlab end def to_json(opts = nil) - Gitlab::Json.generate(formatter.to_h, opts) + Gitlab::Json.generate(to_h.except(:ignore_whitespace_change), opts) end def as_json(opts = nil) - to_h.as_json(opts) + to_h.except(:ignore_whitespace_change).as_json(opts) end def type @@ -134,7 +135,7 @@ module Gitlab end def diff_options - { paths: paths, expanded: true, include_stats: false } + { paths: paths, expanded: true, include_stats: false, ignore_whitespace_change: ignore_whitespace_change } end def diff_line(repository) @@ -149,6 +150,10 @@ module Gitlab @file_hash ||= Digest::SHA1.hexdigest(file_path) end + def on_file? + position_type == 'file' + end + def on_image? position_type == 'image' end @@ -184,6 +189,8 @@ module Gitlab case type when 'image' Gitlab::Diff::Formatters::ImageFormatter + when 'file' + Gitlab::Diff::Formatters::FileFormatter else Gitlab::Diff::Formatters::TextFormatter end diff --git a/lib/gitlab/diff/position_tracer.rb b/lib/gitlab/diff/position_tracer.rb index 1c21c35fa60..a8c0108fa34 100644 --- a/lib/gitlab/diff/position_tracer.rb +++ b/lib/gitlab/diff/position_tracer.rb @@ -21,9 +21,9 @@ module Gitlab return unless old_diff_refs&.complete? && new_diff_refs&.complete? return unless old_position.diff_refs == old_diff_refs - strategy = old_position.on_text? ? LineStrategy : ImageStrategy + @ignore_whitespace_change = old_position.ignore_whitespace_change - strategy.new(self).trace(old_position) + strategy(old_position).new(self).trace(old_position) end def ac_diffs @@ -48,9 +48,19 @@ module Gitlab private + def strategy(old_position) + if old_position.on_text? + LineStrategy + elsif old_position.on_file? + FileStrategy + else + ImageStrategy + end + end + def compare(start_sha, head_sha, straight: false) compare = CompareService.new(project, head_sha).execute(project, start_sha, straight: straight) - compare.diffs(paths: paths, expanded: true) + compare.diffs(paths: paths, expanded: true, ignore_whitespace_change: @ignore_whitespace_change) end end end diff --git a/lib/gitlab/diff/position_tracer/file_strategy.rb b/lib/gitlab/diff/position_tracer/file_strategy.rb new file mode 100644 index 00000000000..171d78bf46f --- /dev/null +++ b/lib/gitlab/diff/position_tracer/file_strategy.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module Diff + class PositionTracer + class FileStrategy < BaseStrategy + def trace(position) + a_path = position.old_path + b_path = position.new_path + + # If file exists in B->D (e.g. updated, renamed, removed), let the + # note become outdated. + bd_diff = bd_diffs.diff_file_with_old_path(b_path) + + return { position: new_position(position, bd_diff), outdated: true } if bd_diff + + # If file still exists in the new diff, update the position. + cd_diff = cd_diffs.diff_file_with_new_path(b_path) + + return { position: new_position(position, cd_diff), outdated: false } if cd_diff + + # If file exists in A->C (e.g. rebased and same changes were present + # in target branch), let the note become outdated. + ac_diff = ac_diffs.diff_file_with_old_path(a_path) + + return { position: new_position(position, ac_diff), outdated: true } if ac_diff + + # If ever there's a case that the file no longer exists in any diff, + # don't set a change position and let the note become outdated. + # + # This should never happen given the file should exist in one of the + # diffs above. + { outdated: true } + end + + private + + def new_position(position, diff_file) + Position.new( + diff_file: diff_file, + position_type: position.position_type + ) + end + end + end + end +end diff --git a/lib/gitlab/diff/position_tracer/image_strategy.rb b/lib/gitlab/diff/position_tracer/image_strategy.rb index aac52b536f7..172eba4acd3 100644 --- a/lib/gitlab/diff/position_tracer/image_strategy.rb +++ b/lib/gitlab/diff/position_tracer/image_strategy.rb @@ -3,36 +3,7 @@ module Gitlab module Diff class PositionTracer - class ImageStrategy < BaseStrategy - def trace(position) - a_path = position.old_path - b_path = position.new_path - - # If file exists in B->D (e.g. updated, renamed, removed), let the - # note become outdated. - bd_diff = bd_diffs.diff_file_with_old_path(b_path) - - return { position: new_position(position, bd_diff), outdated: true } if bd_diff - - # If file still exists in the new diff, update the position. - cd_diff = cd_diffs.diff_file_with_new_path(b_path) - - return { position: new_position(position, cd_diff), outdated: false } if cd_diff - - # If file exists in A->C (e.g. rebased and same changes were present - # in target branch), let the note become outdated. - ac_diff = ac_diffs.diff_file_with_old_path(a_path) - - return { position: new_position(position, ac_diff), outdated: true } if ac_diff - - # If ever there's a case that the file no longer exists in any diff, - # don't set a change position and let the note become outdated. - # - # This should never happen given the file should exist in one of the - # diffs above. - { outdated: true } - end - + class ImageStrategy < FileStrategy private def new_position(position, diff_file) diff --git a/lib/gitlab/diff/position_tracer/line_strategy.rb b/lib/gitlab/diff/position_tracer/line_strategy.rb index d7a7e3f5425..0de9aa22008 100644 --- a/lib/gitlab/diff/position_tracer/line_strategy.rb +++ b/lib/gitlab/diff/position_tracer/line_strategy.rb @@ -62,6 +62,8 @@ module Gitlab # The line number as of D can be found by using the LineMapper on diff C->D # and providing the line number as of C. + @ignore_whitespace_change = position.ignore_whitespace_change + if position.added? trace_added_line(position) elsif position.removed? @@ -189,7 +191,8 @@ module Gitlab diff_file: diff_file, old_line: old_line, new_line: new_line, - line_range: line_range + line_range: line_range, + ignore_whitespace_change: @ignore_whitespace_change }.compact Position.new(**params) diff --git a/lib/gitlab/discussions_diff/highlight_cache.rb b/lib/gitlab/discussions_diff/highlight_cache.rb index df93e6e91b4..18ff7c28e17 100644 --- a/lib/gitlab/discussions_diff/highlight_cache.rb +++ b/lib/gitlab/discussions_diff/highlight_cache.rb @@ -16,7 +16,7 @@ module Gitlab def write_multiple(mapping) with_redis do |redis| Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - redis.pipelined do |pipelined| + Gitlab::Redis::CrossSlot::Pipeline.new(redis).pipelined do |pipelined| mapping.each do |raw_key, value| key = cache_key_for(raw_key) @@ -41,8 +41,8 @@ module Gitlab content = with_redis do |redis| Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - if ::Feature.enabled?(:use_pipeline_over_multikey) - redis.pipelined do |pipeline| + if Gitlab::Redis::ClusterUtil.cluster?(redis) + Gitlab::Redis::CrossSlot::Pipeline.new(redis).pipelined do |pipeline| keys.each { |key| pipeline.get(key) } end else @@ -72,10 +72,8 @@ module Gitlab with_redis do |redis| Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - if ::Feature.enabled?(:use_pipeline_over_multikey) - redis.pipelined do |pipeline| - keys.each { |key| pipeline.del(key) } - end.sum + if Gitlab::Redis::ClusterUtil.cluster?(redis) + Gitlab::Redis::ClusterUtil.batch_unlink(keys, redis) else redis.del(keys) end diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb index c325112b673..869bcc6e2be 100644 --- a/lib/gitlab/email/handler/create_issue_handler.rb +++ b/lib/gitlab/email/handler/create_issue_handler.rb @@ -68,7 +68,7 @@ module Gitlab title: mail.subject, description: message_including_reply_or_only_quotes }, - spam_params: nil + perform_spam_check: false ).execute end diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb index 076ba42daac..215ba77db13 100644 --- a/lib/gitlab/email/handler/service_desk_handler.rb +++ b/lib/gitlab/email/handler/service_desk_handler.rb @@ -103,7 +103,7 @@ module Gitlab cc: mail.cc } }, - spam_params: nil + perform_spam_check: false ).execute raise InvalidIssueError if result.error? diff --git a/lib/gitlab/email/hook/silent_mode_interceptor.rb b/lib/gitlab/email/hook/silent_mode_interceptor.rb index 56f94119472..774d4ac1f45 100644 --- a/lib/gitlab/email/hook/silent_mode_interceptor.rb +++ b/lib/gitlab/email/hook/silent_mode_interceptor.rb @@ -5,19 +5,17 @@ module Gitlab module Hook class SilentModeInterceptor def self.delivering_email(message) - if Gitlab::CurrentSettings.silent_mode_enabled? + if ::Gitlab::SilentMode.enabled? message.perform_deliveries = false - Gitlab::AppJsonLogger.info( + ::Gitlab::SilentMode.log_info( message: "SilentModeInterceptor prevented sending mail", - mail_subject: message.subject, - silent_mode_enabled: true + mail_subject: message.subject ) else - Gitlab::AppJsonLogger.debug( + ::Gitlab::SilentMode.log_debug( message: "SilentModeInterceptor did nothing", - mail_subject: message.subject, - silent_mode_enabled: false + mail_subject: message.subject ) end end diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb index c2d645138d7..e7462b711f1 100644 --- a/lib/gitlab/email/reply_parser.rb +++ b/lib/gitlab/email/reply_parser.rb @@ -80,9 +80,7 @@ module Gitlab # # Plain email. # ``` - # So, we had to force its part to corresponding encoding before able - # to convert it to UTF-8 - force_utf8(object.body.decoded.force_encoding(object.charset.gsub(/utf8/i, "UTF-8"))) + object.body.decoded.force_encoding(object.charset.gsub(/utf8/i, "UTF-8")).encode("UTF-8").to_s else object.body.to_s end diff --git a/lib/gitlab/error_tracking/error_repository.rb b/lib/gitlab/error_tracking/error_repository.rb index fd2467add20..3871305c9c5 100644 --- a/lib/gitlab/error_tracking/error_repository.rb +++ b/lib/gitlab/error_tracking/error_repository.rb @@ -16,7 +16,7 @@ module Gitlab # @return [self] def self.build(project) strategy = - if Feature.enabled?(:use_click_house_database_for_error_tracking, project) + if Feature.enabled?(:gitlab_error_tracking, project) OpenApiStrategy.new(project) else ActiveRecordStrategy.new(project) diff --git a/lib/gitlab/error_tracking/error_repository/open_api_strategy.rb b/lib/gitlab/error_tracking/error_repository/open_api_strategy.rb index e168fa10630..398ddebd355 100644 --- a/lib/gitlab/error_tracking/error_repository/open_api_strategy.rb +++ b/lib/gitlab/error_tracking/error_repository/open_api_strategy.rb @@ -127,21 +127,22 @@ module Gitlab def to_sentry_error(error) Gitlab::ErrorTracking::Error.new( id: error.fingerprint.to_s, - title: error.name, + title: "#{error.name}: #{error.description}", message: error.description, culprit: error.actor, first_seen: error.first_seen_at, last_seen: error.last_seen_at, status: error.status, count: error.event_count, - user_count: error.approximated_user_count + user_count: error.approximated_user_count, + frequency: error.stats&.frequency&.dig(:'24h') || [] ) end def to_sentry_detailed_error(error) Gitlab::ErrorTracking::DetailedError.new( id: error.fingerprint.to_s, - title: error.name, + title: "#{error.name}: #{error.description}", message: error.description, culprit: error.actor, first_seen: error.first_seen_at.to_s, @@ -155,7 +156,8 @@ module Gitlab external_base_url: external_base_url, integrated: true, first_release_version: release_from(oldest_event_for(error.fingerprint)), - last_release_version: release_from(newest_event_for(error.fingerprint)) + last_release_version: release_from(newest_event_for(error.fingerprint)), + frequency: error.stats&.frequency&.dig(:'24h') || [] ) end diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb index 7aabf699a59..786a68c86f2 100644 --- a/lib/gitlab/etag_caching/middleware.rb +++ b/lib/gitlab/etag_caching/middleware.rb @@ -57,7 +57,7 @@ module Gitlab end def weak_etag_format(value) - %Q{W/"#{value}"} + %{W/"#{value}"} end def handle_cache_hit(etag, route, request) diff --git a/lib/gitlab/etag_caching/router/rails.rb b/lib/gitlab/etag_caching/router/rails.rb index 2924370f494..5fd592c43e4 100644 --- a/lib/gitlab/etag_caching/router/rails.rb +++ b/lib/gitlab/etag_caching/router/rails.rb @@ -17,7 +17,7 @@ module Gitlab new environments].freeze RESERVED_WORDS = Gitlab::PathRegex::ILLEGAL_PROJECT_PATH_WORDS - USED_IN_ROUTES RESERVED_WORDS_REGEX = Regexp.union(*RESERVED_WORDS.map(&Regexp.method(:escape))) - RESERVED_WORDS_PREFIX = %Q(^(?!.*\/(#{RESERVED_WORDS_REGEX})\/).*) + RESERVED_WORDS_PREFIX = %(^(?!.*\/(#{RESERVED_WORDS_REGEX})\/).*) ROUTES = [ [ diff --git a/lib/gitlab/front_matter.rb b/lib/gitlab/front_matter.rb index 2a759434b81..d215a77f4d7 100644 --- a/lib/gitlab/front_matter.rb +++ b/lib/gitlab/front_matter.rb @@ -37,6 +37,6 @@ module Gitlab # rubocop:enable Style/StringConcatenation PATTERN_UNTRUSTED_REGEX = - Gitlab::UntrustedRegexp.new(PATTERN_UNTRUSTED, multiline: true) + Gitlab::UntrustedRegexp.new(PATTERN_UNTRUSTED, multiline: true).freeze end end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 80d0fd17568..ed45d3eb030 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -91,12 +91,10 @@ module Gitlab end # Default branch in the repository - def root_ref - gitaly_ref_client.default_branch_name - rescue GRPC::NotFound => e - raise NoRepository, e.message - rescue GRPC::Unknown => e - raise Gitlab::Git::CommandError, e.message + def root_ref(head_only: false) + wrapped_gitaly_errors do + gitaly_ref_client.default_branch_name(head_only: head_only) + end end def exists? @@ -520,13 +518,15 @@ module Gitlab empty_diff_stats end - def find_changed_paths(commits) + def find_changed_paths(commits, merge_commit_diff_mode: nil) processed_commits = commits.reject { |ref| ref.blank? || Gitlab::Git.blank_ref?(ref) } return [] if processed_commits.empty? wrapped_gitaly_errors do - gitaly_commit_client.find_changed_paths(processed_commits) + gitaly_commit_client.find_changed_paths( + processed_commits, merge_commit_diff_mode: merge_commit_diff_mode + ) end rescue CommandError, TypeError, NoRepository [] diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb index 37977a1dfb6..5b54ba472d9 100644 --- a/lib/gitlab/git/tag.rb +++ b/lib/gitlab/git/tag.rb @@ -76,8 +76,16 @@ module Gitlab encode! @message end - def tagger - @raw_tag.tagger + def user_name + encode! tagger.name if tagger + end + + def user_email + encode! tagger.email if tagger + end + + def date + Time.at(tagger.date.seconds).utc if tagger end def has_signature? @@ -105,6 +113,10 @@ module Gitlab private + def tagger + @raw_tag.tagger + end + def message_from_gitaly_tag return @raw_tag.message.dup if full_message_fetched_from_gitaly? diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index e437f99dab3..df3d8165ef2 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -6,7 +6,7 @@ module Gitlab include Gitlab::EncodingHelper extend Gitlab::Git::WrapsGitalyErrors - attr_accessor :id, :type, :mode, :commit_id, :submodule_url + attr_accessor :id, :type, :mode, :commit_id, :submodule_url, :ref_type attr_writer :name, :path, :flat_path class << self diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 0c67b9fa078..aa25fd3589a 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -12,6 +12,11 @@ module Gitlab 'unspecified' => Gitaly::CommitDiffRequest::WhitespaceChanges::WHITESPACE_CHANGES_UNSPECIFIED }.freeze + MERGE_COMMIT_DIFF_MODES = { + all_parents: Gitaly::FindChangedPathsRequest::MergeCommitDiffMode::MERGE_COMMIT_DIFF_MODE_ALL_PARENTS, + include_merges: Gitaly::FindChangedPathsRequest::MergeCommitDiffMode::MERGE_COMMIT_DIFF_MODE_INCLUDE_MERGES + }.freeze + TREE_ENTRIES_DEFAULT_LIMIT = 100_000 def initialize(repository) @@ -123,8 +128,10 @@ module Gitlab end def tree_entries(repository, revision, path, recursive, skip_flat_paths, pagination_params) - pagination_params ||= {} - pagination_params[:limit] ||= TREE_ENTRIES_DEFAULT_LIMIT + unless pagination_params.nil? && recursive + pagination_params ||= {} + pagination_params[:limit] ||= TREE_ENTRIES_DEFAULT_LIMIT + end request = Gitaly::GetTreeEntriesRequest.new( repository: @gitaly_repo, @@ -157,6 +164,17 @@ module Gitlab end [entries, cursor] + rescue GRPC::BadStatus => e + detailed_error = GitalyClient.decode_detailed_error(e) + + case detailed_error.try(:error) + when :path + raise Gitlab::Git::Index::IndexError, path_error_message(detailed_error.path) + when :resolve_tree + raise Gitlab::Git::Index::IndexError, e.details + else + raise e + end end def commit_count(ref, options = {}) @@ -229,11 +247,35 @@ module Gitlab response.flat_map { |rsp| rsp.stats.to_a } end - def find_changed_paths(commits) - request = Gitaly::FindChangedPathsRequest.new( - repository: @gitaly_repo, - commits: commits - ) + # When finding changed paths and passing a sha for a merge commit we can + # specify how to diff the commit. + # + # When diffing a merge commit and merge_commit_diff_mode is :all_parents + # file paths are only returned if changed in both parents (or all parents + # if diffing an octopus merge) + # + # This means if we create a merge request that includes a merge commit + # of changes already existing in the target branch, we can omit those + # changes when looking up the changed paths. + # + # e.g. + # 1. User branches from master to new branch named feature/foo_bar + # 2. User changes ./foo_bar.rb and commits change to feature/foo_bar + # 3. Another user merges a change to ./bar_baz.rb to master + # 4. User merges master into feature/foo_bar + # 5. User pushes to GitLab + # 6. GitLab checks which files have changed + # + # case merge_commit_diff_mode + # when :all_parents + # ['foo_bar.rb'] + # when :include_merges + # ['foo_bar.rb', 'bar_baz.rb'], + # else # defaults to :include_merges behavior + # ['foo_bar.rb', 'bar_baz.rb'], + # + def find_changed_paths(commits, merge_commit_diff_mode: nil) + request = find_changed_paths_request(commits, merge_commit_diff_mode) response = gitaly_client_call(@repository.storage, :diff_service, :find_changed_paths, request, timeout: GitalyClient.medium_timeout) response.flat_map do |msg| @@ -595,6 +637,37 @@ module Gitlab response.commit end + + def find_changed_paths_request(commits, merge_commit_diff_mode) + diff_mode = MERGE_COMMIT_DIFF_MODES[merge_commit_diff_mode] if Feature.enabled?(:merge_commit_diff_modes) + + if Feature.disabled?(:find_changed_paths_new_format) + return Gitaly::FindChangedPathsRequest.new(repository: @gitaly_repo, commits: commits, merge_commit_diff_mode: diff_mode) + end + + commit_requests = commits.map do |commit| + Gitaly::FindChangedPathsRequest::Request.new( + commit_request: Gitaly::FindChangedPathsRequest::Request::CommitRequest.new(commit_revision: commit) + ) + end + + Gitaly::FindChangedPathsRequest.new(repository: @gitaly_repo, requests: commit_requests, merge_commit_diff_mode: diff_mode) + end + + def path_error_message(path_error) + case path_error.error_type + when :ERROR_TYPE_EMPTY_PATH + "You must provide a file path" + when :ERROR_TYPE_RELATIVE_PATH_ESCAPES_REPOSITORY + "Path cannot include traversal syntax" + when :ERROR_TYPE_ABSOLUTE_PATH + "Only relative path is accepted" + when :ERROR_TYPE_LONG_PATH + "Path is too long" + else + "Unknown path error" + end + end end end end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 1af06cc7490..bd6cc9105d9 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -8,6 +8,8 @@ module Gitlab MAX_MSG_SIZE = 128.kilobytes.freeze + CUSTOM_HOOK_FALLBACK_MESSAGE = 'Prevented by server hooks' + def initialize(repository) @gitaly_repo = repository.gitaly_repository @repository = repository @@ -52,7 +54,7 @@ module Gitlab raise Gitlab::Git::PreReceiveError.new(fallback_message: access_check_error.error_message) when :custom_hook raise Gitlab::Git::PreReceiveError.new(custom_hook_error_message(detailed_error.custom_hook), - fallback_message: e.details) + fallback_message: CUSTOM_HOOK_FALLBACK_MESSAGE) when :reference_exists raise Gitlab::Git::Repository::TagExistsError else @@ -85,7 +87,7 @@ module Gitlab case detailed_error.try(:error) when :custom_hook raise Gitlab::Git::PreReceiveError.new(custom_hook_error_message(detailed_error.custom_hook), - fallback_message: e.details) + fallback_message: CUSTOM_HOOK_FALLBACK_MESSAGE) else if e.code == GRPC::Core::StatusCodes::FAILED_PRECONDITION raise Gitlab::Git::Repository::InvalidRef, e @@ -127,7 +129,7 @@ module Gitlab case detailed_error.try(:error) when :custom_hook raise Gitlab::Git::PreReceiveError.new(custom_hook_error_message(detailed_error.custom_hook), - fallback_message: e.details) + fallback_message: CUSTOM_HOOK_FALLBACK_MESSAGE) else raise end @@ -195,7 +197,7 @@ module Gitlab raise Gitlab::Git::PreReceiveError.new(fallback_message: access_check_error.error_message) when :custom_hook raise Gitlab::Git::PreReceiveError.new(custom_hook_error_message(detailed_error.custom_hook), - fallback_message: e.details) + fallback_message: CUSTOM_HOOK_FALLBACK_MESSAGE) when :reference_update # We simply ignore any reference update errors which are typically an # indicator of multiple RPC calls trying to update the same reference @@ -465,7 +467,7 @@ module Gitlab raise Gitlab::Git::PreReceiveError.new(fallback_message: access_check_error.error_message) when :custom_hook raise Gitlab::Git::PreReceiveError.new(custom_hook_error_message(detailed_error.custom_hook), - fallback_message: e.details) + fallback_message: CUSTOM_HOOK_FALLBACK_MESSAGE) when :index_update raise Gitlab::Git::Index::IndexError, index_error_message(detailed_error.index_update) else @@ -583,8 +585,7 @@ module Gitlab def custom_hook_error_message(custom_hook_error) # Custom hooks may return messages via either stdout or stderr which have a specific prefix. If - # that prefix is present we'll want to print the hook's output, otherwise we'll want to print the - # Gitaly error as a fallback. + # that prefix is present we'll want to print the hook's output. custom_hook_output = custom_hook_error.stderr.presence || custom_hook_error.stdout EncodingHelper.encode!(custom_hook_output) end diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index 88c79eb8954..45edfd4cbbf 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -50,8 +50,8 @@ module Gitlab consume_find_all_branches_response(response) end - def default_branch_name - request = Gitaly::FindDefaultBranchNameRequest.new(repository: @gitaly_repo) + def default_branch_name(head_only: false) + request = Gitaly::FindDefaultBranchNameRequest.new(repository: @gitaly_repo, head_only: head_only) response = gitaly_client_call(@storage, :ref_service, :find_default_branch_name, request, timeout: GitalyClient.fast_timeout) Gitlab::Git.branch_name(response.name) end diff --git a/lib/gitlab/github_gists_import/importer/gist_importer.rb b/lib/gitlab/github_gists_import/importer/gist_importer.rb index 4018f425e7c..71dfe5e2aa5 100644 --- a/lib/gitlab/github_gists_import/importer/gist_importer.rb +++ b/lib/gitlab/github_gists_import/importer/gist_importer.rb @@ -4,10 +4,13 @@ module Gitlab module GithubGistsImport module Importer class GistImporter - attr_reader :gist, :user + attr_reader :gist, :user, :snippet FileCountLimitError = Class.new(StandardError) + RepoSizeLimitError = Class.new(StandardError) + SnippetRepositoryError = Class.new(StandardError) FILE_COUNT_LIMIT_MESSAGE = 'Snippet maximum file count exceeded' + REPO_SIZE_LIMIT_MESSAGE = 'Snippet repository size exceeded' # gist - An instance of `Gitlab::GithubGistsImport::Representation::Gist`. def initialize(gist, user_id) @@ -16,12 +19,15 @@ module Gitlab end def execute - snippet = build_snippet - import_repository(snippet) if snippet.save! + validate_gist! - return ServiceResponse.success unless max_snippet_files_count_exceeded?(snippet) + @snippet = build_snippet + import_repository if snippet.save! + validate_repository! - fail_and_track(snippet) + ServiceResponse.success + rescue FileCountLimitError, RepoSizeLimitError, SnippetRepositoryError => exception + fail_and_track(snippet, exception) end private @@ -40,13 +46,13 @@ module Gitlab PersonalSnippet.new(attrs) end - def import_repository(snippet) + def import_repository resolved_address = get_resolved_address snippet.create_repository snippet.repository.fetch_as_mirror(gist.git_pull_url, forced: true, resolved_address: resolved_address) rescue StandardError - remove_snippet_and_repository(snippet) + remove_snippet_and_repository raise end @@ -61,11 +67,19 @@ module Gitlab host.present? ? validated_pull_url.host.to_s : '' end - def max_snippet_files_count_exceeded?(snippet) - snippet.all_files.size > Snippet.max_file_limit + def check_gist_files_count! + return if gist.files.count <= Snippet.max_file_limit + + raise FileCountLimitError, FILE_COUNT_LIMIT_MESSAGE end - def remove_snippet_and_repository(snippet) + def check_gist_repo_size! + return if gist.total_files_size <= Gitlab::CurrentSettings.snippet_size_limit + + raise RepoSizeLimitError, REPO_SIZE_LIMIT_MESSAGE + end + + def remove_snippet_and_repository snippet.repository.remove if snippet.repository_exists? snippet.destroy end @@ -74,10 +88,21 @@ module Gitlab Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services? end - def fail_and_track(snippet) - remove_snippet_and_repository(snippet) + def fail_and_track(snippet, exception) + remove_snippet_and_repository if snippet + + ServiceResponse.error(message: exception.message).track_exception(as: exception.class) + end + + def validate_gist! + check_gist_files_count! + check_gist_repo_size! + end + + def validate_repository! + result = Snippets::RepositoryValidationService.new(user, snippet).execute - ServiceResponse.error(message: FILE_COUNT_LIMIT_MESSAGE).track_exception(as: FileCountLimitError) + raise SnippetRepositoryError, result.message if result.error? end end end diff --git a/lib/gitlab/github_gists_import/representation/gist.rb b/lib/gitlab/github_gists_import/representation/gist.rb index 0d309a98f38..674da4f3400 100644 --- a/lib/gitlab/github_gists_import/representation/gist.rb +++ b/lib/gitlab/github_gists_import/representation/gist.rb @@ -65,6 +65,10 @@ module Gitlab def github_identifiers { id: id } end + + def total_files_size + files.values.sum { |f| f[:size].to_i } + end end end end diff --git a/lib/gitlab/github_import/importer/repository_importer.rb b/lib/gitlab/github_import/importer/repository_importer.rb index 2654812b64a..d37942aa8a3 100644 --- a/lib/gitlab/github_import/importer/repository_importer.rb +++ b/lib/gitlab/github_import/importer/repository_importer.rb @@ -54,6 +54,8 @@ module Gitlab project.change_head(default_branch) if default_branch + validate_repository_size! + # The initial fetch can bring in lots of loose refs and objects. # Running a `git gc` will make importing pull requests faster. Repositories::HousekeepingService.new(project, :gc).execute @@ -89,7 +91,13 @@ module Gitlab strong_memoize_attr def client_repository client.repository(project.import_source) end + + def validate_repository_size! + # Defined in EE + end end end end end + +Gitlab::GithubImport::Importer::RepositoryImporter.prepend_mod diff --git a/lib/gitlab/github_import/representation/diff_note.rb b/lib/gitlab/github_import/representation/diff_note.rb index 0408b34bb02..191e15962a6 100644 --- a/lib/gitlab/github_import/representation/diff_note.rb +++ b/lib/gitlab/github_import/representation/diff_note.rb @@ -12,7 +12,7 @@ module Gitlab expose_attribute :noteable_id, :commit_id, :file_path, :diff_hunk, :author, :created_at, :updated_at, :original_commit_id, :note_id, :end_line, :start_line, - :side, :in_reply_to_id, :discussion_id + :side, :in_reply_to_id, :discussion_id, :subject_type # Builds a diff note from a GitHub API response. # @@ -43,7 +43,8 @@ module Gitlab start_line: note[:start_line], side: note[:side], in_reply_to_id: note[:in_reply_to_id], - discussion_id: DiffNotes::DiscussionId.new(note).find_or_generate + discussion_id: DiffNotes::DiscussionId.new(note).find_or_generate, + subject_type: note[:subject_type] } new(hash) @@ -84,8 +85,14 @@ module Gitlab end def line_code - diff_line = Gitlab::Diff::Parser.new.parse(diff_hunk.lines).to_a.last + # on the GitHub side it is possible to leave a comment on a file + # or on a line. When the comment is left on a file there is no + # diff hunk, but LegacyDiffNote requires line_code to be always present + # and DiffFile requires it for text files + # so it is set as the first line for any type of file (image, binary, text) + return Gitlab::Git.diff_line_code(file_path, 1, 1) if on_file? + diff_line = Gitlab::Diff::Parser.new.parse(diff_hunk.lines).to_a.last Gitlab::Git.diff_line_code(file_path, diff_line.new_pos, diff_line.old_pos) end @@ -141,6 +148,10 @@ module Gitlab def addition? side == 'RIGHT' end + + def on_file? + subject_type == 'file' + end end end end diff --git a/lib/gitlab/gl_repository.rb b/lib/gitlab/gl_repository.rb index 268d5d3e564..9161a1a138f 100644 --- a/lib/gitlab/gl_repository.rb +++ b/lib/gitlab/gl_repository.rb @@ -34,7 +34,7 @@ module Gitlab DESIGN = ::Gitlab::GlRepository::RepoType.new( name: :design, access_checker_class: ::Gitlab::GitAccessDesign, - repository_resolver: -> (project) { project.design_management_repository.repository }, + repository_resolver: -> (project) { project.find_or_create_design_management_repository.repository }, suffix: :design, container_class: DesignManagement::Repository ).freeze diff --git a/lib/gitlab/gl_repository/identifier.rb b/lib/gitlab/gl_repository/identifier.rb index f521a14ea19..787e80fb763 100644 --- a/lib/gitlab/gl_repository/identifier.rb +++ b/lib/gitlab/gl_repository/identifier.rb @@ -23,7 +23,7 @@ module Gitlab return identifier if identifier&.valid? - raise InvalidIdentifier, %Q(Invalid GL Repository "#{gl_repository}") + raise InvalidIdentifier, %(Invalid GL Repository "#{gl_repository}") end # The older 2-segment format, where the container is implied. diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 904a2ccc79b..9eeea7336b5 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -50,8 +50,12 @@ module Gitlab gon.jh = Gitlab.jh? gon.dot_com = Gitlab.com? gon.uf_error_prefix = ::Gitlab::Utils::ErrorMessage::UF_ERROR_PREFIX + gon.pat_prefix = Gitlab::CurrentSettings.current_application_settings.personal_access_token_prefix + + gon.diagramsnet_url = Gitlab::CurrentSettings.diagramsnet_url if Gitlab::CurrentSettings.diagramsnet_enabled if current_user + gon.version = Gitlab::VERSION # publish version only for logged in users gon.current_user_id = current_user.id gon.current_username = current_user.username gon.current_user_fullname = current_user.name @@ -66,10 +70,11 @@ module Gitlab push_frontend_feature_flag(:security_auto_fix) push_frontend_feature_flag(:source_editor_toolbar) push_frontend_feature_flag(:vscode_web_ide, current_user) - push_frontend_feature_flag(:super_sidebar_peek, current_user) push_frontend_feature_flag(:unbatch_graphql_queries, current_user) + push_frontend_feature_flag(:command_palette, current_user) # To be removed with https://gitlab.com/gitlab-org/gitlab/-/issues/399248 push_frontend_feature_flag(:remove_monitor_metrics) + push_frontend_feature_flag(:gitlab_duo, current_user) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/graphql/generic_tracing.rb b/lib/gitlab/graphql/generic_tracing.rb index d3de9c714f4..dc3f6574631 100644 --- a/lib/gitlab/graphql/generic_tracing.rb +++ b/lib/gitlab/graphql/generic_tracing.rb @@ -39,11 +39,15 @@ module Gitlab ensure duration = Gitlab::Metrics::System.monotonic_time - start - graphql_duration_seconds.observe(tags, duration) + graphql_duration_seconds.observe(tags, duration) unless deactivated? end private + def deactivated? + Feature.enabled?(:graphql_generic_tracing_metrics_deactivate) + end + def with_labkit_tracing(tags, &block) return yield unless Labkit::Tracing.enabled? diff --git a/lib/gitlab/hotlinking_detector.rb b/lib/gitlab/hotlinking_detector.rb index b5000777010..e81983cd014 100644 --- a/lib/gitlab/hotlinking_detector.rb +++ b/lib/gitlab/hotlinking_detector.rb @@ -25,6 +25,11 @@ module Gitlab return true if INVALID_FORMATS.include?(request_accepts.first) false + + rescue ActionDispatch::Http::MimeNegotiation::InvalidType, Mime::Type::InvalidMimeType + # Malformed requests with invalid MIME types prevent the checks from + # being executed correctly, so we should intercept those requests. + true end private diff --git a/lib/gitlab/http.rb b/lib/gitlab/http.rb index c6cd5fbfced..8b19611e5c0 100644 --- a/lib/gitlab/http.rb +++ b/lib/gitlab/http.rb @@ -10,6 +10,7 @@ module Gitlab RedirectionTooDeep = Class.new(StandardError) ReadTotalTimeout = Class.new(Net::ReadTimeout) HeaderReadTimeout = Class.new(Net::ReadTimeout) + SilentModeBlockedError = Class.new(StandardError) HTTP_TIMEOUT_ERRORS = [ Net::OpenTimeout, Net::ReadTimeout, Net::WriteTimeout, Gitlab::HTTP::ReadTotalTimeout @@ -28,6 +29,13 @@ module Gitlab }.freeze DEFAULT_READ_TOTAL_TIMEOUT = 30.seconds + SILENT_MODE_ALLOWED_METHODS = [ + Net::HTTP::Get, + Net::HTTP::Head, + Net::HTTP::Options, + Net::HTTP::Trace + ].freeze + include HTTParty # rubocop:disable Gitlab/HTTParty class << self @@ -37,6 +45,8 @@ module Gitlab connection_adapter HTTPConnectionAdapter def self.perform_request(http_method, path, options, &block) + raise_if_blocked_by_silent_mode(http_method) + log_info = options.delete(:extra_log_info) options_with_timeouts = if !options.has_key?(:timeout) @@ -76,5 +86,20 @@ module Gitlab rescue *HTTP_ERRORS nil end + + def self.raise_if_blocked_by_silent_mode(http_method) + return unless blocked_by_silent_mode?(http_method) + + ::Gitlab::SilentMode.log_info( + message: 'Outbound HTTP request blocked', + outbound_http_request_method: http_method.to_s + ) + + raise SilentModeBlockedError, 'only get, head, options, and trace methods are allowed in silent mode' + end + + def self.blocked_by_silent_mode?(http_method) + ::Gitlab::SilentMode.enabled? && SILENT_MODE_ALLOWED_METHODS.exclude?(http_method) + end end end diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index fa85d839927..180ccf21264 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -45,27 +45,27 @@ module Gitlab 'bg' => 0, 'cs_CZ' => 0, 'da_DK' => 31, - 'de' => 15, + 'de' => 97, 'en' => 100, 'eo' => 0, - 'es' => 31, + 'es' => 30, 'fil_PH' => 0, 'fr' => 98, 'gl_ES' => 0, 'id_ID' => 0, 'it' => 1, - 'ja' => 77, + 'ja' => 98, 'ko' => 18, - 'nb_NO' => 23, + 'nb_NO' => 22, 'nl_NL' => 0, 'pl_PL' => 3, 'pt_BR' => 56, - 'ro_RO' => 84, - 'ru' => 24, + 'ro_RO' => 82, + 'ru' => 23, 'si_LK' => 10, 'tr_TR' => 9, - 'uk' => 54, - 'zh_CN' => 99, + 'uk' => 53, + 'zh_CN' => 98, 'zh_HK' => 1, 'zh_TW' => 99 }.freeze diff --git a/lib/gitlab/import_export/decompressed_archive_size_validator.rb b/lib/gitlab/import_export/decompressed_archive_size_validator.rb index 564008e7a73..104c9e6c456 100644 --- a/lib/gitlab/import_export/decompressed_archive_size_validator.rb +++ b/lib/gitlab/import_export/decompressed_archive_size_validator.rb @@ -85,7 +85,7 @@ module Gitlab end def validate_archive_path - Gitlab::Utils.check_path_traversal!(@archive_path) + Gitlab::PathTraversal.check_path_traversal!(@archive_path) raise(ServiceError, 'Archive path is a symlink') if File.lstat(@archive_path).symlink? raise(ServiceError, 'Archive path is not a file') unless File.file?(@archive_path) diff --git a/lib/gitlab/import_export/group/import_export.yml b/lib/gitlab/import_export/group/import_export.yml index e30414265be..c2a1a1f8575 100644 --- a/lib/gitlab/import_export/group/import_export.yml +++ b/lib/gitlab/import_export/group/import_export.yml @@ -56,7 +56,6 @@ excluded_attributes: - :runners_token - :runners_token_encrypted - :saml_discovery_token - - :visibility_level - :trial_ends_on - :shared_runners_minute_limit - :extra_shared_runners_minutes_limit diff --git a/lib/gitlab/import_export/group/tree_restorer.rb b/lib/gitlab/import_export/group/tree_restorer.rb index 19d707aaca5..0b92942eb8a 100644 --- a/lib/gitlab/import_export/group/tree_restorer.rb +++ b/lib/gitlab/import_export/group/tree_restorer.rb @@ -65,6 +65,15 @@ module Gitlab # with existing groups name and/or path. group_attributes.delete_attributes('name', 'path') + if @top_level_group.has_parent? + group_attributes.attributes['visibility_level'] = sub_group_visibility_level( + group_attributes.attributes['visibility_level'], + @top_level_group.parent + ) + elsif Gitlab::VisibilityLevel.restricted_level?(group_attributes.attributes['visibility_level']) + group_attributes.delete_attribute('visibility_level') + end + restore_group(@top_level_group, group_attributes) end @@ -86,6 +95,7 @@ module Gitlab parent_id = group_attributes.delete_attribute('parent_id') name = group_attributes.delete_attribute('name') path = group_attributes.delete_attribute('path') + visibility_level = group_attributes.delete_attribute('visibility_level') parent_group = @groups_mapping.fetch(parent_id) { raise(ArgumentError, 'Parent group not found') } @@ -94,7 +104,7 @@ module Gitlab name: name, path: path, parent_id: parent_group.id, - visibility_level: sub_group_visibility_level(group_attributes.attributes, parent_group) + visibility_level: sub_group_visibility_level(visibility_level, parent_group) ).execute group.validate! @@ -124,16 +134,23 @@ module Gitlab end end - def sub_group_visibility_level(group_hash, parent_group) - original_visibility_level = group_hash['visibility_level'] || Gitlab::VisibilityLevel::PRIVATE + def sub_group_visibility_level(visibility_level, parent_group) + parent_visibility_level = parent_group.visibility_level - if parent_group && parent_group.visibility_level < original_visibility_level - Gitlab::VisibilityLevel.closest_allowed_level(parent_group.visibility_level) + original_visibility_level = visibility_level || + closest_allowed_level(parent_visibility_level) + + if parent_visibility_level < original_visibility_level + closest_allowed_level(parent_visibility_level) else - original_visibility_level + closest_allowed_level(original_visibility_level) end end + def closest_allowed_level(visibility_level) + Gitlab::VisibilityLevel.closest_allowed_level(visibility_level) + end + def reader strong_memoize(:reader) do Gitlab::ImportExport::Reader.new( diff --git a/lib/gitlab/import_export/legacy_relation_tree_saver.rb b/lib/gitlab/import_export/legacy_relation_tree_saver.rb deleted file mode 100644 index cf75a2c7fa8..00000000000 --- a/lib/gitlab/import_export/legacy_relation_tree_saver.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - class LegacyRelationTreeSaver - include Gitlab::ImportExport::CommandLineUtil - - def serialize(exportable, relations_tree) - Gitlab::ImportExport::FastHashSerializer - .new(exportable, relations_tree) - .execute - end - - def save(tree, dir_path, filename) - mkdir_p(dir_path) - - tree_json = ::JSON.generate(tree) - - File.write(File.join(dir_path, filename), tree_json) - end - end - end -end diff --git a/lib/gitlab/import_export/project/exported_relations_merger.rb b/lib/gitlab/import_export/project/exported_relations_merger.rb index dda3d00d608..b5eba768e56 100644 --- a/lib/gitlab/import_export/project/exported_relations_merger.rb +++ b/lib/gitlab/import_export/project/exported_relations_merger.rb @@ -20,8 +20,8 @@ module Gitlab tar_gz_full_path = File.join(dirpath, filename) decompress_path = File.join(dirpath, relation) - Gitlab::Utils.check_path_traversal!(tar_gz_full_path) - Gitlab::Utils.check_path_traversal!(decompress_path) + Gitlab::PathTraversal.check_path_traversal!(tar_gz_full_path) + Gitlab::PathTraversal.check_path_traversal!(decompress_path) # Download tar.gz download_or_copy_upload( diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 36a3c73271b..410e918649b 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -121,7 +121,6 @@ tree: - label: - :priorities - :service_desk_setting - - :design_management_repository group_members: - :user @@ -322,6 +321,7 @@ included_attributes: - :feature_flags_access_level - :releases_access_level - :infrastructure_access_level + - :model_experiments_access_level prometheus_metrics: - :created_at - :updated_at @@ -742,6 +742,7 @@ included_attributes: - :feature_flags_access_level - :releases_access_level - :infrastructure_access_level + - :model_experiments_access_level - :auto_devops_deploy_strategy - :auto_devops_enabled - :container_registry_enabled diff --git a/lib/gitlab/import_export/project/relation_factory.rb b/lib/gitlab/import_export/project/relation_factory.rb index 5d7e3ea9ed7..8c673acdd1a 100644 --- a/lib/gitlab/import_export/project/relation_factory.rb +++ b/lib/gitlab/import_export/project/relation_factory.rb @@ -23,6 +23,7 @@ module Gitlab design: 'DesignManagement::Design', designs: 'DesignManagement::Design', design_management_repository: 'DesignManagement::Repository', + design_management_repository_state: 'Geo::DesignManagementRepositoryState', design_versions: 'DesignManagement::Version', actions: 'DesignManagement::Action', labels: :project_labels, diff --git a/lib/gitlab/import_export/recursive_merge_folders.rb b/lib/gitlab/import_export/recursive_merge_folders.rb index 982358699bd..827385d4daf 100644 --- a/lib/gitlab/import_export/recursive_merge_folders.rb +++ b/lib/gitlab/import_export/recursive_merge_folders.rb @@ -45,9 +45,9 @@ module Gitlab DEFAULT_DIR_MODE = 0o700 def self.merge(source_path, target_path) - Gitlab::Utils.check_path_traversal!(source_path) - Gitlab::Utils.check_path_traversal!(target_path) - Gitlab::Utils.check_allowed_absolute_path!(source_path, [Dir.tmpdir]) + Gitlab::PathTraversal.check_path_traversal!(source_path) + Gitlab::PathTraversal.check_path_traversal!(target_path) + Gitlab::PathTraversal.check_allowed_absolute_path!(source_path, [Dir.tmpdir]) recursive_merge(source_path, target_path) end diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index cc214d730fe..d7d262501de 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -42,7 +42,7 @@ module Gitlab def ensure_repository_does_not_exist! if repository.exists? shared.logger.info( - message: %Q{Deleting existing "#{repository.disk_path}" to re-import it.} + message: %{Deleting existing "#{repository.disk_path}" to re-import it.} ) Repositories::DestroyService.new(repository).execute diff --git a/lib/gitlab/instrumentation/redis_base.rb b/lib/gitlab/instrumentation/redis_base.rb index 00a7387afe2..70aaa59f912 100644 --- a/lib/gitlab/instrumentation/redis_base.rb +++ b/lib/gitlab/instrumentation/redis_base.rb @@ -94,9 +94,9 @@ module Gitlab raise RedisClusterValidator::CrossSlotError, "Redis command #{result[:command_name]} arguments hash to different slots. See https://docs.gitlab.com/ee/development/redis.html#multi-key-commands" end - increment_allowed_cross_slot_request_count if result[:allowed] + increment_allowed_cross_slot_request_count if result[:allowed] && !result[:valid] - result[:valid] + result[:valid] || result[:allowed] end def enable_redis_cluster_validation diff --git a/lib/gitlab/instrumentation/redis_cluster_validator.rb b/lib/gitlab/instrumentation/redis_cluster_validator.rb index 1567e54d8da..948132e6edd 100644 --- a/lib/gitlab/instrumentation/redis_cluster_validator.rb +++ b/lib/gitlab/instrumentation/redis_cluster_validator.rb @@ -193,8 +193,7 @@ module Gitlab keys = commands.map { |command| extract_keys(command) }.flatten { - # calculate key-slots only if not allowed - valid: allow_cross_slot_commands? || !has_cross_slot_keys?(keys), + valid: !has_cross_slot_keys?(keys), command_name: command_name, key_count: keys.size, allowed: allow_cross_slot_commands? @@ -211,6 +210,10 @@ module Gitlab Thread.current[:allow_cross_slot_commands] -= 1 end + def allow_cross_slot_commands? + Thread.current[:allow_cross_slot_commands].to_i > 0 + end + private def extract_keys(command) @@ -226,10 +229,6 @@ module Gitlab keys.map { |key| key_slot(key) }.uniq.many? # rubocop: disable CodeReuse/ActiveRecord end - def allow_cross_slot_commands? - Thread.current[:allow_cross_slot_commands].to_i > 0 - end - def key_slot(key) ::Redis::Cluster::KeySlotConverter.convert(extract_hash_tag(key)) end diff --git a/lib/gitlab/internal_events.rb b/lib/gitlab/internal_events.rb new file mode 100644 index 00000000000..cde83068de1 --- /dev/null +++ b/lib/gitlab/internal_events.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Gitlab + module InternalEvents + class << self + include Gitlab::Tracking::Helpers + + def track_event(event_name, **kwargs) + user_id = kwargs.delete(:user_id) + UsageDataCounters::HLLRedisCounter.track_event(event_name, values: user_id) + + project_id = kwargs.delete(:project_id) + namespace_id = kwargs.delete(:namespace_id) + + namespace = Namespace.find(namespace_id) if namespace_id + + standard_context = Tracking::StandardContext.new( + project_id: project_id, + user_id: user_id, + namespace_id: namespace&.id, + plan_name: namespace&.actual_plan_name + ).to_context + + service_ping_context = Tracking::ServicePingContext.new( + data_source: :redis_hll, + event: event_name + ).to_context + + track_struct_event(event_name, contexts: [standard_context, service_ping_context]) + end + + private + + def track_struct_event(event_name, contexts:) + category = 'InternalEventTracking' + tracker = Gitlab::Tracking.tracker + tracker.event(category, event_name, context: contexts) + rescue StandardError => error + Gitlab::ErrorTracking + .track_and_raise_for_dev_exception(error, snowplow_category: category, snowplow_action: event_name) + end + end + end +end diff --git a/lib/gitlab/json_cache.rb b/lib/gitlab/json_cache.rb deleted file mode 100644 index 0087c2accc3..00000000000 --- a/lib/gitlab/json_cache.rb +++ /dev/null @@ -1,118 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - class JsonCache - attr_reader :backend, :namespace - - STRATEGY_KEY_COMPONENTS = { - revision: Gitlab.revision, - version: [Gitlab::VERSION, Rails.version] - }.freeze - - def initialize(options = {}) - @backend = options.fetch(:backend, Rails.cache) - @namespace = options.fetch(:namespace, nil) - @cache_key_strategy = options.fetch(:cache_key_strategy, :revision) - end - - def active? - if backend.respond_to?(:active?) - backend.active? - else - true - end - end - - def cache_key(key) - expanded_cache_key = [namespace, key, *strategy_key_component].compact - expanded_cache_key.join(':').freeze - end - - def strategy_key_component - STRATEGY_KEY_COMPONENTS.fetch(@cache_key_strategy) - end - - def expire(key) - backend.delete(cache_key(key)) - end - - def read(key, klass = nil) - value = backend.read(cache_key(key)) - value = parse_value(value, klass) unless value.nil? - value - end - - def write(key, value, options = nil) - backend.write(cache_key(key), value.to_json, options) - end - - def fetch(key, options = {}, &block) - klass = options.delete(:as) - value = read(key, klass) - - return value unless value.nil? - - value = yield - - write(key, value, options) - - value - end - - private - - def parse_value(raw, klass) - value = Gitlab::Json.parse(raw.to_s) - - case value - when Hash then parse_entry(value, klass) - when Array then parse_entries(value, klass) - else - value - end - rescue JSON::ParserError - nil - end - - def parse_entry(raw, klass) - return unless valid_entry?(raw, klass) - return klass.new(raw) unless klass.ancestors.include?(ActiveRecord::Base) - - # When the cached value is a persisted instance of ActiveRecord::Base in - # some cases a relation can return an empty collection becauses scope.none! - # is being applied on ActiveRecord::Associations::CollectionAssociation#scope - # when the new_record? method incorrectly returns false. - # - # See https://gitlab.com/gitlab-org/gitlab/issues/9903#note_145329964 - klass.allocate.init_with(encode_for(klass, raw)) - end - - def encode_for(klass, raw) - # We have models that leave out some fields from the JSON export for - # security reasons, e.g. models that include the CacheMarkdownField. - # The ActiveRecord::AttributeSet we build from raw does know about - # these columns so we need manually set them. - missing_attributes = (klass.columns.map(&:name) - raw.keys) - missing_attributes.each { |column| raw[column] = nil } - - coder = {} - klass.new(raw).encode_with(coder) - coder["new_record"] = new_record?(raw, klass) - coder - end - - def new_record?(raw, klass) - raw.fetch(klass.primary_key, nil).blank? - end - - def valid_entry?(raw, klass) - return false unless klass && raw.is_a?(Hash) - - (raw.keys - klass.attribute_names).empty? - end - - def parse_entries(values, klass) - values.filter_map { |value| parse_entry(value, klass) } - end - end -end diff --git a/lib/gitlab/markdown_cache/redis/store.rb b/lib/gitlab/markdown_cache/redis/store.rb index 8cab069e1bf..f742cb82b8d 100644 --- a/lib/gitlab/markdown_cache/redis/store.rb +++ b/lib/gitlab/markdown_cache/redis/store.rb @@ -11,7 +11,7 @@ module Gitlab Gitlab::Redis::Cache.with do |r| Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - r.pipelined do |pipeline| + Gitlab::Redis::CrossSlot::Pipeline.new(r).pipelined do |pipeline| subjects.each do |subject| results[subject.cache_key] = new(subject).read(pipeline) end diff --git a/lib/gitlab/merge_requests/message_generator.rb b/lib/gitlab/merge_requests/message_generator.rb index 5113fbdcd7b..523e0e665dc 100644 --- a/lib/gitlab/merge_requests/message_generator.rb +++ b/lib/gitlab/merge_requests/message_generator.rb @@ -51,6 +51,7 @@ module Gitlab end, 'description' => ->(merge_request, _, _) { merge_request.description }, 'reference' => ->(merge_request, _, _) { merge_request.to_reference(full: true) }, + 'local_reference' => ->(merge_request, _, _) { merge_request.to_reference(full: false) }, 'first_commit' => -> (merge_request, _, _) { return unless merge_request.persisted? || merge_request.compare_commits.present? diff --git a/lib/gitlab/metrics/loose_foreign_keys_slis.rb b/lib/gitlab/metrics/loose_foreign_keys_slis.rb index 5d8245aa609..c0db709fe13 100644 --- a/lib/gitlab/metrics/loose_foreign_keys_slis.rb +++ b/lib/gitlab/metrics/loose_foreign_keys_slis.rb @@ -26,7 +26,7 @@ module Gitlab private def possible_labels - ::Gitlab::Database.db_config_names.map do |db_config_name| + ::Gitlab::Database.db_config_names(with_schema: :gitlab_shared).map do |db_config_name| { db_config_name: db_config_name, feature_category: :database diff --git a/lib/gitlab/metrics/subscribers/rails_cache.rb b/lib/gitlab/metrics/subscribers/rails_cache.rb index b4e9e85a012..e2cdd6c5358 100644 --- a/lib/gitlab/metrics/subscribers/rails_cache.rb +++ b/lib/gitlab/metrics/subscribers/rails_cache.rb @@ -13,7 +13,7 @@ module Gitlab return unless current_transaction - labels = { store: event.payload[:store].split('::').last } + labels = { store: extract_store_name(event) } current_transaction.observe(:gitlab_cache_read_multikey_count, event.payload[:key].size, labels) do buckets [10, 50, 100, 1000] docstring 'Number of keys for mget in read_multi/fetch_multi' @@ -26,11 +26,7 @@ module Gitlab return unless current_transaction return if event.payload[:super_operation] == :fetch - unless event.payload[:hit] - current_transaction.increment(:gitlab_cache_misses_total, 1) do - docstring 'Cache read miss' - end - end + track_cache_miss(event) unless event.payload[:hit] end def cache_write(event) @@ -48,23 +44,23 @@ module Gitlab def cache_fetch_hit(event) return unless current_transaction - current_transaction.increment(:gitlab_transaction_cache_read_hit_count_total, 1) + labels = { store: extract_store_name(event) } + current_transaction.increment(:gitlab_transaction_cache_read_hit_count_total, 1, labels) end def cache_generate(event) return unless current_transaction - current_transaction.increment(:gitlab_cache_misses_total, 1) do - docstring 'Cache read miss' - end + track_cache_miss(event) - current_transaction.increment(:gitlab_transaction_cache_read_miss_count_total, 1) + labels = { store: extract_store_name(event) } + current_transaction.increment(:gitlab_transaction_cache_read_miss_count_total, 1, labels) end def observe(key, event) return unless current_transaction - labels = { operation: key, store: event.payload[:store].split('::').last } + labels = { operation: key, store: extract_store_name(event) } current_transaction.increment(:gitlab_cache_operations_total, 1, labels) do docstring 'Cache operations' @@ -76,6 +72,20 @@ module Gitlab private + def track_cache_miss(event) + # avoid passing in labels to ensure metric has consistent set of labels + labels = { store: extract_store_name(event) } + + current_transaction.increment(:gitlab_cache_misses_total, 1, labels) do + docstring 'Cache read miss' + end + end + + def extract_store_name(event) + # see payload documentation in https://guides.rubyonrails.org/active_support_instrumentation.html#active-support + event.payload[:store].to_s.split('::').last + end + def current_transaction ::Gitlab::Metrics::WebTransaction.current end diff --git a/lib/gitlab/middleware/compressed_json.rb b/lib/gitlab/middleware/compressed_json.rb index cc485d8a5db..1f15f1d5857 100644 --- a/lib/gitlab/middleware/compressed_json.rb +++ b/lib/gitlab/middleware/compressed_json.rb @@ -3,7 +3,6 @@ module Gitlab module Middleware class CompressedJson - COLLECTOR_PATH = '/api/v4/error_tracking/collector' INSTANCE_PACKAGES_PATH = %r{ \A/api/v4/packages/npm/-/npm/v1/security/ (?:(?:advisories/bulk)|(?:audits/quick))\z (?# end) @@ -79,8 +78,7 @@ module Gitlab end def match_path?(env) - env['PATH_INFO'].start_with?((File.join(relative_url, COLLECTOR_PATH))) || - match_packages_path?(env) + match_packages_path?(env) end def match_packages_path?(env) diff --git a/lib/gitlab/pagination/cursor_based_keyset.rb b/lib/gitlab/pagination/cursor_based_keyset.rb index a21d0228082..ee8259cc671 100644 --- a/lib/gitlab/pagination/cursor_based_keyset.rb +++ b/lib/gitlab/pagination/cursor_based_keyset.rb @@ -6,7 +6,8 @@ module Gitlab SUPPORTED_ORDERING = { Group => { name: :asc }, AuditEvent => { id: :desc }, - ::Ci::Build => { id: :desc } + ::Ci::Build => { id: :desc }, + ::Packages::BuildInfo => { id: :desc } }.freeze # Relation types that are enforced in this list diff --git a/lib/gitlab/patch/redis_cache_store.rb b/lib/gitlab/patch/redis_cache_store.rb new file mode 100644 index 00000000000..5279c4081b2 --- /dev/null +++ b/lib/gitlab/patch/redis_cache_store.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +module Gitlab + module Patch + module RedisCacheStore + PIPELINE_BATCH_SIZE = 100 + + # We will try keep patched code explicit and matching the original signature in + # https://github.com/rails/rails/blob/v6.1.7.2/activesupport/lib/active_support/cache/redis_cache_store.rb#L361 + def read_multi_mget(*names) # rubocop:disable Style/ArgumentsForwarding + return super unless enable_rails_cache_pipeline_patch? + return super unless use_patched_mget? + + patched_read_multi_mget(*names) # rubocop:disable Style/ArgumentsForwarding + end + + # `delete_multi_entries` in Rails runs a multi-key `del` command + # patch will run pipelined single-key `del` for Redis Cluster compatibility + def delete_multi_entries(entries, **options) + return super unless enable_rails_cache_pipeline_patch? + + delete_count = 0 + redis.with do |conn| + entries.each_slice(PIPELINE_BATCH_SIZE) do |subset| + delete_count += Gitlab::Redis::CrossSlot::Pipeline.new(conn).pipelined do |pipeline| + subset.each { |entry| pipeline.del(entry) } + end.sum + end + end + delete_count + end + + # Copied from https://github.com/rails/rails/blob/v6.1.6.1/activesupport/lib/active_support/cache/redis_cache_store.rb + # re-implements `read_multi_mget` using a pipeline of `get`s rather than an `mget` + # + def patched_read_multi_mget(*names) + options = names.extract_options! + options = merged_options(options) + return {} if names == [] + + raw = options&.fetch(:raw, false) + + keys = names.map { |name| normalize_key(name, options) } + + values = failsafe(:patched_read_multi_mget, returning: {}) do + redis.with { |c| pipeline_mget(c, keys) } + end + + names.zip(values).each_with_object({}) do |(name, value), results| + if value # rubocop:disable Style/Next + entry = deserialize_entry(value, raw: raw) + unless entry.nil? || entry.expired? || entry.mismatched?(normalize_version(name, options)) + results[name] = entry.value + end + end + end + end + + def pipeline_mget(conn, keys) + keys.each_slice(PIPELINE_BATCH_SIZE).flat_map do |subset| + Gitlab::Redis::CrossSlot::Pipeline.new(conn).pipelined do |p| + subset.each { |key| p.get(key) } + end + end + end + + private + + def enable_rails_cache_pipeline_patch? + redis.with { |c| ::Gitlab::Redis::ClusterUtil.cluster?(c) } + end + + # MultiStore reads ONLY from the default store (no fallback), hence we can use `mget` + # if the default store is not a Redis::Cluster. We should do that as pipelining gets on a single redis is slow + def use_patched_mget? + redis.with do |conn| + next true unless conn.is_a?(Gitlab::Redis::MultiStore) + + ::Gitlab::Redis::ClusterUtil.cluster?(conn.default_store) + end + end + end + end +end diff --git a/lib/gitlab/patch/redis_cluster.rb b/lib/gitlab/patch/redis_cluster.rb new file mode 100644 index 00000000000..145ce35a317 --- /dev/null +++ b/lib/gitlab/patch/redis_cluster.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +# Patch to expose `find_node_key` method for cross-slot pipelining +# In redis v5.0.x, cross-slot pipelining is implemented via redis-cluster-client. +# This patch should be removed since there is no need for it. +# Gitlab::Redis::CrossSlot and its usage should be removed as well. +if Gem::Version.new(Redis::VERSION) != Gem::Version.new('4.8.0') + raise 'New version of redis detected, please remove or update this patch' +end + +module Gitlab + module Patch + module RedisCluster + # _find_node_key exposes a private function of the same name in Redis::Cluster. + # See https://github.com/redis/redis-rb/blob/v4.8.0/lib/redis/cluster.rb#L282 + def _find_node_key(command) + find_node_key(command) + end + end + end +end diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index b0804c2ff66..e112423f167 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -122,6 +122,7 @@ module Gitlab ILLEGAL_PROJECT_PATH_WORDS = PROJECT_WILDCARD_ROUTES ILLEGAL_GROUP_PATH_WORDS = (PROJECT_WILDCARD_ROUTES | GROUP_ROUTES).freeze + ILLEGAL_ORGANIZATION_PATH_WORDS = (TOP_LEVEL_ROUTES | PROJECT_WILDCARD_ROUTES | GROUP_ROUTES).freeze # The namespace regex is used in JavaScript to validate usernames in the "Register" form. However, Javascript # does not support the negative lookbehind assertion (?' types Issue - condition { can_relate_issues? } + condition { can_admin_link? } parse_params { |issues| format_params(issues) } command :relate do |target_issues| create_links(target_issues) end + desc { _("Remove link with another issue") } + explanation do |issue| + _('Removes link with %{issue_ref}.') % { issue_ref: issue.to_reference(quick_action_target) } + end + execution_message do |issue| + _('Removed link with %{issue_ref}.') % { issue_ref: issue.to_reference(quick_action_target) } + end + params '<#issue | group/project#issue | issue URL>' + types Issue + condition { can_admin_link? } + parse_params do |issue_param| + extract_references(issue_param, :issue).first + end + command :unlink do |issue| + link = IssueLink.for_issues(quick_action_target, issue).first + + if link + call_link_service(IssueLinks::DestroyService.new(link, current_user)) + else + @execution_message[:unlink] = _('No linked issue matches the provided parameter.') + end + end + private + def can_admin_link? + current_user.can?(:admin_issue_link, quick_action_target) + end + def create_links(references, type: 'relates_to') - service = IssueLinks::CreateService.new( + create_service_instance = IssueLinks::CreateService.new( quick_action_target, current_user, { issuable_references: references, link_type: type } ) - create_issue_link = proc { service.execute } + + call_link_service(create_service_instance) + end + + def call_link_service(service_instance) + execute_service = proc { service_instance.execute } if quick_action_target.persisted? - create_issue_link.call + execute_service.call else - quick_action_target.run_after_commit(&create_issue_link) + quick_action_target.run_after_commit(&execute_service) end end - def can_relate_issues? - current_user.can?(:admin_issue_link, quick_action_target) - end - def format_params(issue_references) issue_references.split(' ') end diff --git a/lib/gitlab/quick_actions/work_item_actions.rb b/lib/gitlab/quick_actions/work_item_actions.rb index fa43308c9e2..5664410f3ca 100644 --- a/lib/gitlab/quick_actions/work_item_actions.rb +++ b/lib/gitlab/quick_actions/work_item_actions.rb @@ -12,42 +12,94 @@ module Gitlab format(_("Converts work item to %{type}. Widgets not supported in new type are removed."), type: target_type) end types WorkItem - condition do - quick_action_target&.project&.work_items_mvc_2_feature_flag_enabled? - end params 'Task | Objective | Key Result | Issue' command :type do |type_name| - work_item_type = ::WorkItems::Type.find_by_name(type_name) - errors = validate_type(work_item_type) - - if errors.present? - @execution_message[:type] = errors - else - @updates[:issue_type] = work_item_type.base_type - @updates[:work_item_type] = work_item_type - @execution_message[:type] = _('Type changed successfully.') - end + @execution_message[:type] = update_type(type_name, :type) + end + + desc { _('Promote work item') } + explanation do |type_name| + format(_("Promotes work item to %{type}."), type: type_name) + end + types WorkItem + params 'issue | objective' + condition { supports_promotion? } + command :promote_to do |type_name| + @execution_message[:promote_to] = update_type(type_name, :promote_to) end end private + # rubocop:disable Gitlab/ModuleWithInstanceVariables + def update_type(type_name, command) + new_type = ::WorkItems::Type.find_by_name(type_name.titleize) + error_message = command == :type ? validate_type(new_type) : validate_promote_to(new_type) + return error_message if error_message.present? + + @updates[:issue_type] = new_type.base_type + @updates[:work_item_type] = new_type + + success_msg[command] + end + # rubocop:enable Gitlab/ModuleWithInstanceVariables + def validate_type(type) - return type_error(:not_found) unless type.present? - return type_error(:same_type) if quick_action_target.work_item_type == type - return type_error(:forbidden) unless current_user.can?(:"create_#{type.base_type}", quick_action_target) + return error_msg(:not_found) unless type.present? + return error_msg(:same_type) if quick_action_target.work_item_type == type + return error_msg(:forbidden) unless current_user.can?(:"create_#{type.base_type}", quick_action_target) nil end - def type_error(reason) + def validate_promote_to(type) + return error_msg(:not_found, action: 'promote') unless type && supports_promote_to?(type.name) + + unless current_user.can?(:"create_#{type.base_type}", quick_action_target) + return error_msg(:forbidden, action: 'promote') + end + + validate_hierarchy + end + + def validate_hierarchy + return unless current_type.task? && quick_action_target.parent_link + + error_msg(:hierarchy, action: 'promote') + end + + def current_type + quick_action_target.work_item_type + end + + def supports_promotion? + current_type.base_type.in?(promote_to_map.keys) + end + + def supports_promote_to?(type_name) + type_name == promote_to_map[current_type.base_type] + end + + def promote_to_map + { issue: 'Incident', task: 'Issue' }.with_indifferent_access + end + + def error_msg(reason, action: 'convert') message = { not_found: 'Provided type is not supported', same_type: 'Types are the same', - forbidden: 'You have insufficient permissions' + forbidden: 'You have insufficient permissions', + hierarchy: 'A task cannot be promoted when a parent issue is present' }.freeze - format(_("Failed to convert this work item: %{reason}."), { reason: message[reason] }) + format(_("Failed to %{action} this work item: %{reason}."), { action: action, reason: message[reason] }) + end + + def success_msg + { + type: _('Type changed successfully.'), + promote_to: _("Work Item promoted successfully.") + } end end end diff --git a/lib/gitlab/reactive_cache_set_cache.rb b/lib/gitlab/reactive_cache_set_cache.rb index dc13bb927e6..9f5769a5c97 100644 --- a/lib/gitlab/reactive_cache_set_cache.rb +++ b/lib/gitlab/reactive_cache_set_cache.rb @@ -11,17 +11,15 @@ module Gitlab end def clear_cache!(key) - use_pipeline = ::Feature.enabled?(:use_pipeline_over_multikey) - with do |redis| keys = read(key).map { |value| "#{cache_namespace}:#{value}" } keys << cache_key(key) Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - redis.pipelined do |pipeline| - if use_pipeline - keys.each { |key| pipeline.unlink(key) } - else + if Gitlab::Redis::ClusterUtil.cluster?(redis) + Gitlab::Redis::ClusterUtil.batch_unlink(keys, redis) + else + redis.pipelined do |pipeline| keys.each_slice(1000) { |subset| pipeline.unlink(*subset) } end end diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb index 64ca89c6bff..06bce7649bf 100644 --- a/lib/gitlab/redis.rb +++ b/lib/gitlab/redis.rb @@ -9,6 +9,7 @@ module Gitlab # config/initializers/7_redis.rb, instrumented, and used in health- & readiness checks. ALL_CLASSES = [ Gitlab::Redis::Cache, + Gitlab::Redis::ClusterCache, Gitlab::Redis::DbLoadBalancing, Gitlab::Redis::FeatureFlag, Gitlab::Redis::Queues, @@ -16,7 +17,8 @@ module Gitlab Gitlab::Redis::RepositoryCache, Gitlab::Redis::Sessions, Gitlab::Redis::SharedState, - Gitlab::Redis::TraceChunks + Gitlab::Redis::TraceChunks, + Gitlab::Redis::Chat ].freeze end end diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb index ba3af3e7a6f..60944268f91 100644 --- a/lib/gitlab/redis/cache.rb +++ b/lib/gitlab/redis/cache.rb @@ -5,19 +5,35 @@ module Gitlab class Cache < ::Gitlab::Redis::Wrapper CACHE_NAMESPACE = 'cache:gitlab' - # Full list of options: - # https://api.rubyonrails.org/classes/ActiveSupport/Cache/RedisCacheStore.html#method-c-new - def self.active_support_config - { - redis: pool, - compress: Gitlab::Utils.to_boolean(ENV.fetch('ENABLE_REDIS_CACHE_COMPRESSION', '1')), - namespace: CACHE_NAMESPACE, - expires_in: default_ttl_seconds - } - end + class << self + # Full list of options: + # https://api.rubyonrails.org/classes/ActiveSupport/Cache/RedisCacheStore.html#method-c-new + def active_support_config + { + redis: pool, + compress: Gitlab::Utils.to_boolean(ENV.fetch('ENABLE_REDIS_CACHE_COMPRESSION', '1')), + namespace: CACHE_NAMESPACE, + expires_in: default_ttl_seconds + } + end + + def default_ttl_seconds + ENV.fetch('GITLAB_RAILS_CACHE_DEFAULT_TTL_SECONDS', 8.hours).to_i + end + + # Exposes redis for Peek adapter. To be removed after ClusterCache migration. + def multistore_redis + redis + end + + private + + def redis + primary_store = ::Redis.new(Gitlab::Redis::ClusterCache.params) + secondary_store = ::Redis.new(params) - def self.default_ttl_seconds - ENV.fetch('GITLAB_RAILS_CACHE_DEFAULT_TTL_SECONDS', 8.hours).to_i + MultiStore.new(primary_store, secondary_store, store_name) + end end end end diff --git a/lib/gitlab/redis/chat.rb b/lib/gitlab/redis/chat.rb new file mode 100644 index 00000000000..6f320fa6fc6 --- /dev/null +++ b/lib/gitlab/redis/chat.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module Redis + class Chat < ::Gitlab::Redis::Wrapper + class << self + def config_fallback + Cache + end + end + end + end +end diff --git a/lib/gitlab/redis/cluster_cache.rb b/lib/gitlab/redis/cluster_cache.rb new file mode 100644 index 00000000000..15a87739c6d --- /dev/null +++ b/lib/gitlab/redis/cluster_cache.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module Redis + class ClusterCache < ::Gitlab::Redis::Wrapper + class << self + def config_fallback + Cache + end + end + end + end +end diff --git a/lib/gitlab/redis/cluster_util.rb b/lib/gitlab/redis/cluster_util.rb new file mode 100644 index 00000000000..5f1f39b5237 --- /dev/null +++ b/lib/gitlab/redis/cluster_util.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Redis + module ClusterUtil + class << self + # clusters? is used to select Redis command types, on `true`, the subsequent + # commands should be compatible with Redis Cluster. + # + # When working with MultiStore, if even 1 of 2 stores is a Redis::Cluster, + # we should err on the side of caution and return `true `, + def cluster?(obj) + if obj.is_a?(MultiStore) + cluster?(obj.primary_store) || cluster?(obj.secondary_store) + else + obj.respond_to?(:_client) && obj._client.is_a?(::Redis::Cluster) + end + end + + def batch_unlink(keys, redis) + expired_count = 0 + keys.each_slice(1000) do |subset| + expired_count += Gitlab::Redis::CrossSlot::Pipeline.new(redis).pipelined do |pipeline| + subset.each { |key| pipeline.unlink(key) } + end.sum + end + expired_count + end + end + end + end +end diff --git a/lib/gitlab/redis/cross_slot.rb b/lib/gitlab/redis/cross_slot.rb new file mode 100644 index 00000000000..e5aa6d9ce72 --- /dev/null +++ b/lib/gitlab/redis/cross_slot.rb @@ -0,0 +1,141 @@ +# frozen_string_literal: true + +module Gitlab + module Redis + module CrossSlot + class Router + attr_reader :node_mapping, :futures, :node_sequence, :cmd_queue + + delegate :respond_to_missing?, to: :@redis + + # This map contains redis-rb methods which does not map directly + # to a standard Redis command. It is used transform unsupported commands to standard commands + # to find the node key for unsupported commands. + # + # Redis::Cluster::Command only contains details of commands which the Redis Server + # returns. Hence, commands like mapped_hmget and hscan_each internally will call the + # base command, hmget and hscan respectively. + # + # See https://github.com/redis/redis-rb/blob/v4.8.0/lib/redis/cluster/command.rb + UNSUPPORTED_CMD_MAPPING = { + # Internally, redis-rb calls the supported Redis command and transforms the output. + # See https://github.com/redis/redis-rb/blob/v4.8.0/lib/redis/commands/hashes.rb#L104 + mapped_hmget: :hmget + }.freeze + + # Initializes the CrossSlot::Router + # @param {::Redis} + def initialize(redis) + @redis = redis + @node_mapping = {} + @futures = {} + @node_sequence = [] + @cmd_queue = [] + end + + # For now we intercept every redis.call and return a Gitlab-Future object. + # This method groups every commands to a node for fan-out. Commands are grouped using the first key. + # + # rubocop:disable Style/MissingRespondToMissing + def method_missing(cmd, *args, **kwargs, &blk) + # Note that we can re-map the command without affecting execution as it is + # solely for finding the node key. The original cmd will be executed. + node = @redis._client._find_node_key([UNSUPPORTED_CMD_MAPPING.fetch(cmd, cmd)] + args) + + @node_mapping[node] ||= [] + @futures[node] ||= [] + + @node_sequence << node + @node_mapping[node] << [cmd, args, kwargs || {}, blk] + f = Future.new + @futures[node] << f + @cmd_queue << [f, cmd, args, kwargs || {}, blk] + f + end + # rubocop:enable Style/MissingRespondToMissing + end + + # Wraps over redis-rb's Future in + # https://github.com/redis/redis-rb/blob/v4.8.0/lib/redis/pipeline.rb#L244 + class Future + def set(future, is_val = false) + @redis_future = future + @is_val = is_val + end + + def value + return @redis_val if @is_val + + @redis_future.value + end + end + + # Pipeline allows cross-slot pipelined to be called. The fan-out logic is implemented in + # https://github.com/redis-rb/redis-cluster-client/blob/master/lib/redis_client/cluster/pipeline.rb + # which is available in redis-rb v5.0. + # + # This file can be deprecated after redis-rb v4.8.0 is upgraded to v5.0 + class Pipeline + # Initializes the CrossSlot::Pipeline + # @param {::Redis} + def initialize(redis) + @redis = redis + end + + # pipelined is used in place of ::Redis `.pipelined` when running in a cluster context + # where cross-slot operations may happen. + def pipelined(&block) + # Directly call .pipelined and defer the pipeline execution to MultiStore. + # MultiStore could wrap over 0, 1, or 2 Redis Cluster clients, handling it here + # will not work for 2 clients since the key-slot topology can differ. + if use_cross_slot_pipelining? + router = Router.new(@redis) + yield router + execute_commands(router) + else + # use redis-rb's pipelined method + @redis.pipelined(&block) + end + end + + private + + def use_cross_slot_pipelining? + !@redis.instance_of?(::Gitlab::Redis::MultiStore) && @redis._client.instance_of?(::Redis::Cluster) + end + + def execute_commands(router) + router.node_mapping.each do |node_key, commands| + # TODO possibly use Threads to speed up but for now `n` is 3-5 which is small. + @redis.pipelined do |p| + commands.each_with_index do |command, idx| + future = router.futures[node_key][idx] + cmd, args, kwargs, blk = command + future.set(p.public_send(cmd, *args, **kwargs, &blk)) # rubocop:disable GitlabSecurity/PublicSend + end + end + end + + router.node_sequence.map do |node_key| + router.futures[node_key].shift.value + end + rescue ::Redis::CommandError => err + if err.message.start_with?('MOVED', 'ASK') + Gitlab::ErrorTracking.log_exception(err) + return execute_commands_sequentially(router) + end + + raise + end + + def execute_commands_sequentially(router) + router.cmd_queue.map do |command| + future, cmd, args, kwargs, blk = command + future.set(@redis.public_send(cmd, *args, **kwargs, &blk), true) # rubocop:disable GitlabSecurity/PublicSend + future.value + end + end + end + end + end +end diff --git a/lib/gitlab/redis/multi_store.rb b/lib/gitlab/redis/multi_store.rb index 9571e2f92e6..d36ef6b99ee 100644 --- a/lib/gitlab/redis/multi_store.rb +++ b/lib/gitlab/redis/multi_store.rb @@ -44,6 +44,7 @@ module Gitlab hscan_each mapped_hmget mget + scan scan_each scard sismember @@ -66,11 +67,14 @@ module Gitlab mapped_hmset rpush sadd + sadd? set setex setnx srem unlink + + memory ].freeze PIPELINED_COMMANDS = %i[ @@ -122,7 +126,7 @@ module Gitlab if use_primary_and_secondary_stores? pipelined_both(name, *args, **kwargs, &block) else - default_store.send(name, *args, **kwargs, &block) + send_command(default_store, name, *args, **kwargs, &block) end end end @@ -289,6 +293,16 @@ module Gitlab # rubocop:disable GitlabSecurity/PublicSend def send_command(redis_instance, command_name, *args, **kwargs, &block) + # Run wrapped pipeline for each instance individually so that the fan-out is distinct. + # If both primary and secondary are Redis Clusters, the slot-node distribution could + # be different. + # + # We ignore args and kwargs since `pipelined` does not accept arguments + # See https://github.com/redis/redis-rb/blob/v4.8.0/lib/redis.rb#L164 + if command_name.to_s == 'pipelined' && redis_instance._client.instance_of?(::Redis::Cluster) + return Gitlab::Redis::CrossSlot::Pipeline.new(redis_instance).pipelined(&block) + end + if block # Make sure that block is wrapped and executed only on the redis instance that is executing the block redis_instance.send(command_name, *args, **kwargs) do |*params| diff --git a/lib/gitlab/redis/rate_limiting.rb b/lib/gitlab/redis/rate_limiting.rb index 74b4ca12d18..30ec44b748d 100644 --- a/lib/gitlab/redis/rate_limiting.rb +++ b/lib/gitlab/redis/rate_limiting.rb @@ -3,18 +3,11 @@ module Gitlab module Redis class RateLimiting < ::Gitlab::Redis::Wrapper - # We create a subclass only for the purpose of differentiating between different stores in cache metrics - RateLimitingStore = Class.new(ActiveSupport::Cache::RedisCacheStore) - class << self # The data we store on RateLimiting used to be stored on Cache. def config_fallback Cache end - - def cache_store - @cache_store ||= RateLimitingStore.new(redis: pool, namespace: Cache::CACHE_NAMESPACE) - end end end end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index eb99805e2e8..26ca9d2547c 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -548,8 +548,8 @@ module Gitlab # https://confluence.atlassian.com/adminjiraserver073/changing-the-project-key-format-861253229.html # Avoids linking CVE IDs (https://cve.mitre.org/cve/identifiers/syntaxchange.html#new) as Jira issues. # CVE IDs use the format of CVE-YYYY-NNNNNNN - def jira_issue_key_regex - @jira_issue_key_regex ||= /(?!CVE-\d+-\d+)[A-Z][A-Z_0-9]+-\d+/ + def jira_issue_key_regex(expression_escape: '\b') + /#{expression_escape}(?!CVE-\d+-\d+)[A-Z][A-Z_0-9]+-\d+/ end def jira_issue_key_project_key_extraction_regex @@ -623,6 +623,18 @@ module Gitlab def x509_subject_key_identifier_regex @x509_subject_key_identifier_regex ||= /\A(?:\h{2}:)*\h{2}\z/.freeze end + + def ml_model_version_regex + maven_version_regex + end + + def ml_model_name_regex + package_name_regex + end + + def ml_model_file_name_regex + maven_file_name_regex + end end end diff --git a/lib/gitlab/repository_hash_cache.rb b/lib/gitlab/repository_hash_cache.rb index 1f3c084e194..fab0e9e09e8 100644 --- a/lib/gitlab/repository_hash_cache.rb +++ b/lib/gitlab/repository_hash_cache.rb @@ -40,7 +40,11 @@ module Gitlab keys = keys.map { |key| cache_key(key) } Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - redis.unlink(*keys) + if Gitlab::Redis::ClusterUtil.cluster?(redis) + Gitlab::Redis::ClusterUtil.batch_unlink(keys, redis) + else + redis.unlink(*keys) + end end end end diff --git a/lib/gitlab/resource_events/assignment_event_recorder.rb b/lib/gitlab/resource_events/assignment_event_recorder.rb index 94bd05a17ba..0f1ceeb2a66 100644 --- a/lib/gitlab/resource_events/assignment_event_recorder.rb +++ b/lib/gitlab/resource_events/assignment_event_recorder.rb @@ -11,8 +11,6 @@ module Gitlab end def record - return if Feature.disabled?(:record_issue_and_mr_assignee_events, parent.project) - case parent when Issue record_for_parent( diff --git a/lib/gitlab/search/abuse_detection.rb b/lib/gitlab/search/abuse_detection.rb index 7b5377bce88..8711d078ea9 100644 --- a/lib/gitlab/search/abuse_detection.rb +++ b/lib/gitlab/search/abuse_detection.rb @@ -8,7 +8,6 @@ module Gitlab ABUSIVE_TERM_SIZE = 100 ALLOWED_CHARS_REGEX = %r{\A[[:alnum:]_\-\/\.!]+\z}.freeze - MINIMUM_SEARCH_CHARS = 2 ALLOWED_SCOPES = %w( blobs @@ -50,7 +49,8 @@ module Gitlab exclusion: { in: STOP_WORDS, message: 'stopword only abusive search detected' }, allow_blank: true validates :query_string, - length: { minimum: MINIMUM_SEARCH_CHARS, message: 'abusive tiny search detected' }, unless: :skip_tiny_search_validation?, allow_blank: true + length: { minimum: Params::MIN_TERM_LENGTH, message: 'abusive tiny search detected' }, + unless: :skip_tiny_search_validation?, allow_blank: true validates :query_string, no_abusive_term_length: { maximum: ABUSIVE_TERM_SIZE, maximum_for_url: ABUSIVE_TERM_SIZE * 2 } diff --git a/lib/gitlab/search/params.rb b/lib/gitlab/search/params.rb index 1ae14e5e618..6eb24a92be6 100644 --- a/lib/gitlab/search/params.rb +++ b/lib/gitlab/search/params.rb @@ -7,7 +7,7 @@ module Gitlab SEARCH_CHAR_LIMIT = 4096 SEARCH_TERM_LIMIT = 64 - MIN_TERM_LENGTH = 3 + MIN_TERM_LENGTH = 2 # Generic validation validates :query_string, length: { maximum: SEARCH_CHAR_LIMIT } diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 93befc2df57..a733dca6a56 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -107,7 +107,11 @@ module Gitlab def users return User.none unless Ability.allowed?(current_user, :read_users_list) - UsersFinder.new(current_user, search: query).execute + if Feature.enabled?(:autocomplete_users_use_search_service) + UsersFinder.new(current_user, { search: query, use_minimum_char_limit: false }).execute + else + UsersFinder.new(current_user, search: query).execute + end end # highlighting is only performed by Elasticsearch backed results @@ -174,7 +178,9 @@ module Gitlab # rubocop: enable CodeReuse/ActiveRecord def projects - limit_projects.search(query) + scope = limit_projects + scope = scope.non_archived if Feature.enabled?(:search_projects_hide_archived) && !filters[:include_archived] + scope.search(query) end def issues(finder_params = {}) diff --git a/lib/gitlab/sentence.rb b/lib/gitlab/sentence.rb new file mode 100644 index 00000000000..963459e31a3 --- /dev/null +++ b/lib/gitlab/sentence.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Gitlab + module Sentence + extend self + # Wraps ActiveSupport's Array#to_sentence to convert the given array to a + # comma-separated sentence joined with localized 'or' Strings instead of 'and'. + def to_exclusive_sentence(array) + array.to_sentence(two_words_connector: _(' or '), last_word_connector: _(', or ')) + end + end +end diff --git a/lib/gitlab/set_cache.rb b/lib/gitlab/set_cache.rb index 623b254c4e0..eb73a0a3d31 100644 --- a/lib/gitlab/set_cache.rb +++ b/lib/gitlab/set_cache.rb @@ -22,10 +22,8 @@ module Gitlab keys_to_expire = keys.map { |key| cache_key(key) } Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - if ::Feature.enabled?(:use_pipeline_over_multikey) - redis.pipelined do |pipeline| - keys_to_expire.each { |key| pipeline.unlink(key) } - end.sum + if Gitlab::Redis::ClusterUtil.cluster?(redis) + Gitlab::Redis::ClusterUtil.batch_unlink(keys_to_expire, redis) else redis.unlink(*keys_to_expire) end diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb index 7ce3f6b5ccb..c4566a6dc2a 100644 --- a/lib/gitlab/sidekiq_logging/structured_logger.rb +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -76,15 +76,19 @@ module Gitlab payload['load_balancing_strategy'] = job['load_balancing_strategy'] if job['load_balancing_strategy'] payload['dedup_wal_locations'] = job['dedup_wal_locations'] if job['dedup_wal_locations'].present? - if job_exception - payload['message'] = "#{message}: fail: #{payload['duration_s']} sec" - payload['job_status'] = 'fail' - - Gitlab::ExceptionLogFormatter.format!(job_exception, payload) - else - payload['message'] = "#{message}: done: #{payload['duration_s']} sec" - payload['job_status'] = 'done' - end + job_status = if job_exception + 'fail' + elsif job['deferred'] + 'deferred' + else + 'done' + end + + payload['message'] = "#{message}: #{job_status}: #{payload['duration_s']} sec" + payload['job_status'] = job_status + payload['job_deferred_by'] = job['deferred_by'] if job['deferred'] + + Gitlab::ExceptionLogFormatter.format!(job_exception, payload) if job_exception db_duration = ActiveRecord::LogSubscriber.runtime payload['db_duration_s'] = Gitlab::Utils.ms_to_round_sec(db_duration) diff --git a/lib/gitlab/sidekiq_middleware.rb b/lib/gitlab/sidekiq_middleware.rb index b20f639ce85..ec2a6472809 100644 --- a/lib/gitlab/sidekiq_middleware.rb +++ b/lib/gitlab/sidekiq_middleware.rb @@ -7,7 +7,7 @@ module Gitlab # The result of this method should be passed to # Sidekiq's `config.server_middleware` method # eg: `config.server_middleware(&Gitlab::SidekiqMiddleware.server_configurator)` - def self.server_configurator(metrics: true, arguments_logger: true) + def self.server_configurator(metrics: true, arguments_logger: true, defer_jobs: true) lambda do |chain| # Size limiter should be placed at the top chain.add ::Gitlab::SidekiqMiddleware::SizeLimiter::Server @@ -36,10 +36,11 @@ module Gitlab chain.add ::Gitlab::SidekiqVersioning::Middleware chain.add ::Gitlab::SidekiqStatus::ServerMiddleware chain.add ::Gitlab::SidekiqMiddleware::WorkerContext::Server - # DuplicateJobs::Server should be placed at the bottom, but before the SidekiqServerMiddleware, + # DuplicateJobs::Server should be placed at the bottom, but before the SidekiqServerMiddleware, # so we can compare the latest WAL location against replica chain.add ::Gitlab::SidekiqMiddleware::DuplicateJobs::Server chain.add ::Gitlab::Database::LoadBalancing::SidekiqServerMiddleware + chain.add ::Gitlab::SidekiqMiddleware::DeferJobs if defer_jobs end end diff --git a/lib/gitlab/sidekiq_middleware/defer_jobs.rb b/lib/gitlab/sidekiq_middleware/defer_jobs.rb new file mode 100644 index 00000000000..0a12667865c --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/defer_jobs.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + class DeferJobs + DELAY = ENV.fetch("SIDEKIQ_DEFER_JOBS_DELAY", 5.minutes) + FEATURE_FLAG_PREFIX = "defer_sidekiq_jobs" + + DatabaseHealthStatusChecker = Struct.new(:id, :job_class_name) + + # There are 2 scenarios under which this middleware defers a job + # 1. defer_sidekiq_jobs_#{worker_name} FF, jobs are deferred indefinitely until this feature flag + # is turned off or when Feature.enabled? returns false by chance while using `percentage of time` value. + # 2. Gitlab::Database::HealthStatus, on evaluating the db health status if it returns any indicator + # with stop signal, the jobs will be delayed by 'x' seconds (set in worker). + def call(worker, job, _queue) + # ActiveJobs have wrapped class stored in 'wrapped' key + resolved_class = job['wrapped']&.safe_constantize || worker.class + defer_job, delay, deferred_by = defer_job_info(resolved_class, job) + + if !!defer_job + # Referred in job_logger's 'log_job_done' method to compute proper 'job_status' + job['deferred'] = true + job['deferred_by'] = deferred_by + + worker.class.perform_in(delay, *job['args']) + counter.increment({ worker: worker.class.name }) + + # This breaks the middleware chain and return + return + end + + yield + end + + private + + def defer_job_info(worker_class, job) + if defer_job_by_ff?(worker_class) + [true, DELAY, :feature_flag] + elsif defer_job_by_database_health_signal?(job, worker_class) + [true, worker_class.database_health_check_attrs[:delay_by], :database_health_check] + end + end + + def defer_job_by_ff?(worker_class) + Feature.enabled?( + :"#{FEATURE_FLAG_PREFIX}_#{worker_class.name}", + type: :worker, + default_enabled_if_undefined: false + ) + end + + def defer_job_by_database_health_signal?(job, worker_class) + unless worker_class.respond_to?(:defer_on_database_health_signal?) && + worker_class.defer_on_database_health_signal? + return false + end + + health_check_attrs = worker_class.database_health_check_attrs + job_base_model = Gitlab::Database.schemas_to_base_models[health_check_attrs[:gitlab_schema]].first + + health_context = Gitlab::Database::HealthStatus::Context.new( + DatabaseHealthStatusChecker.new(job['jid'], worker_class.name), + job_base_model.connection, + health_check_attrs[:gitlab_schema], + health_check_attrs[:tables] + ) + + Gitlab::Database::HealthStatus.evaluate(health_context).any?(&:stop?) + end + + def counter + @counter ||= Gitlab::Metrics.counter(:sidekiq_jobs_deferred_total, 'The number of jobs deferred') + end + end + end +end diff --git a/lib/gitlab/silent_mode.rb b/lib/gitlab/silent_mode.rb new file mode 100644 index 00000000000..7c7cbf8f1d9 --- /dev/null +++ b/lib/gitlab/silent_mode.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module SilentMode + def self.enabled? + Gitlab::CurrentSettings.silent_mode_enabled? + end + + def self.log_info(data) + Gitlab::AppJsonLogger.info(**add_silent_mode_log_data(data)) + end + + def self.log_debug(data) + Gitlab::AppJsonLogger.debug(**add_silent_mode_log_data(data)) + end + + def self.add_silent_mode_log_data(data) + data.merge!({ silent_mode_enabled: enabled? }) + end + end +end diff --git a/lib/gitlab/slash_commands/incident_management/incident_command.rb b/lib/gitlab/slash_commands/incident_management/incident_command.rb index 3fa08621777..1c870efe8b1 100644 --- a/lib/gitlab/slash_commands/incident_management/incident_command.rb +++ b/lib/gitlab/slash_commands/incident_management/incident_command.rb @@ -11,9 +11,13 @@ module Gitlab def collection IssuesFinder.new(current_user, project_id: project.id, issue_types: :incident).execute end + + def slack_installation + slack_workspace_id = params[:team_id] + + SlackIntegration.with_bot.find_by_team_id(slack_workspace_id) + end end end end end - -Gitlab::SlashCommands::IncidentManagement::IncidentCommand.prepend_mod diff --git a/lib/gitlab/slash_commands/incident_management/incident_new.rb b/lib/gitlab/slash_commands/incident_management/incident_new.rb index a43235bdeb6..b5c43873355 100644 --- a/lib/gitlab/slash_commands/incident_management/incident_new.rb +++ b/lib/gitlab/slash_commands/incident_management/incident_new.rb @@ -16,6 +16,14 @@ module Gitlab text == 'incident declare' end + def execute(_match) + response = ::Integrations::SlackInteractions::IncidentManagement::IncidentModalOpenedService + .new(slack_installation, current_user, params) + .execute + + presenter.present(response.message) + end + private def presenter diff --git a/lib/gitlab/slash_commands/issue_new.rb b/lib/gitlab/slash_commands/issue_new.rb index 508526ac500..dd2bf632e2c 100644 --- a/lib/gitlab/slash_commands/issue_new.rb +++ b/lib/gitlab/slash_commands/issue_new.rb @@ -37,7 +37,7 @@ module Gitlab private def create_issue(title:, description:) - ::Issues::CreateService.new(container: project, current_user: current_user, params: { title: title, description: description }, spam_params: nil).execute + ::Issues::CreateService.new(container: project, current_user: current_user, params: { title: title, description: description }, perform_spam_check: false).execute end def presenter(issue) diff --git a/lib/gitlab/spamcheck/client.rb b/lib/gitlab/spamcheck/client.rb index 222dd54b7b4..0afaf46fa9b 100644 --- a/lib/gitlab/spamcheck/client.rb +++ b/lib/gitlab/spamcheck/client.rb @@ -58,6 +58,7 @@ module Gitlab pb.title = spammable.spam_title || '' if pb.respond_to?(:title) pb.description = spammable.spam_description || '' if pb.respond_to?(:description) pb.text = spammable.spammable_text || '' if pb.respond_to?(:text) + pb.type = spammable.spammable_entity_type if pb.respond_to?(:type) pb.created_at = convert_to_pb_timestamp(spammable.created_at) if spammable.created_at pb.updated_at = convert_to_pb_timestamp(spammable.updated_at) if spammable.updated_at pb.action = ACTION_MAPPING.fetch(context.fetch(:action)) if context.has_key?(:action) diff --git a/lib/gitlab/template/finders/global_template_finder.rb b/lib/gitlab/template/finders/global_template_finder.rb index 6d2677175e6..a6ab36e6cd2 100644 --- a/lib/gitlab/template/finders/global_template_finder.rb +++ b/lib/gitlab/template/finders/global_template_finder.rb @@ -25,7 +25,7 @@ module Gitlab # The key is untrusted input, so ensure we can't be directed outside # of base_dir - Gitlab::Utils.check_path_traversal!(file_name) + Gitlab::PathTraversal.check_path_traversal!(file_name) directory = select_directory(file_name) directory ? File.join(category_directory(directory), file_name) : nil diff --git a/lib/gitlab/template/finders/repo_template_finder.rb b/lib/gitlab/template/finders/repo_template_finder.rb index 9f0ba97bcdf..8343750e04a 100644 --- a/lib/gitlab/template/finders/repo_template_finder.rb +++ b/lib/gitlab/template/finders/repo_template_finder.rb @@ -29,7 +29,7 @@ module Gitlab # The key is untrusted input, so ensure we can't be directed outside # of base_dir inside the repository - Gitlab::Utils.check_path_traversal!(file_name) + Gitlab::PathTraversal.check_path_traversal!(file_name) directory = select_directory(file_name) raise FileNotFoundError if directory.nil? diff --git a/lib/gitlab/template/metrics_dashboard_template.rb b/lib/gitlab/template/metrics_dashboard_template.rb deleted file mode 100644 index 469f97d7cb1..00000000000 --- a/lib/gitlab/template/metrics_dashboard_template.rb +++ /dev/null @@ -1,31 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Template - class MetricsDashboardTemplate < BaseTemplate - def description - "# This file is a template, and might need editing before it works on your project." - end - - class << self - def extension - '.metrics-dashboard.yml' - end - - def categories - { - "General" => '' - } - end - - def base_dir - Rails.root.join('lib/gitlab/metrics/templates') - end - - def finder(project = nil) - Gitlab::Template::Finders::GlobalTemplateFinder.new(self.base_dir, self.extension, self.categories) - end - end - end - end -end diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb index 52aee4d2d45..f127e14243c 100644 --- a/lib/gitlab/tracking.rb +++ b/lib/gitlab/tracking.rb @@ -68,6 +68,14 @@ module Gitlab false end + def tracker + @tracker ||= if snowplow_micro_enabled? + Gitlab::Tracking::Destinations::SnowplowMicro.new + else + Gitlab::Tracking::Destinations::Snowplow.new + end + end + private def track_struct_event(destination, category, action, label:, property:, value:, contexts:) # rubocop:disable Metrics/ParameterLists @@ -76,14 +84,6 @@ module Gitlab rescue StandardError => error Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error, snowplow_category: category, snowplow_action: action) end - - def tracker - @tracker ||= if snowplow_micro_enabled? - Gitlab::Tracking::Destinations::SnowplowMicro.new - else - Gitlab::Tracking::Destinations::Snowplow.new - end - end end end end diff --git a/lib/gitlab/tracking/standard_context.rb b/lib/gitlab/tracking/standard_context.rb index 62c45368410..61d6fdc6dca 100644 --- a/lib/gitlab/tracking/standard_context.rb +++ b/lib/gitlab/tracking/standard_context.rb @@ -3,7 +3,7 @@ module Gitlab module Tracking class StandardContext - GITLAB_STANDARD_SCHEMA_URL = 'iglu:com.gitlab/gitlab_standard/jsonschema/1-0-8' + GITLAB_STANDARD_SCHEMA_URL = 'iglu:com.gitlab/gitlab_standard/jsonschema/1-0-9' GITLAB_RAILS_SOURCE = 'gitlab-rails' def initialize(namespace_id: nil, plan_name: nil, project_id: nil, user_id: nil, **extra) diff --git a/lib/gitlab/usage/metrics/instrumentations/count_all_ci_builds_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_all_ci_builds_metric.rb new file mode 100644 index 00000000000..2a3cbaf5d03 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_all_ci_builds_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountAllCiBuildsMetric < DatabaseMetric + operation :count + + relation { ::Ci::Build } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_deployments_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_deployments_metric.rb new file mode 100644 index 00000000000..97813fbb5c0 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_deployments_metric.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountDeploymentsMetric < DatabaseMetric + operation :count + + start { Deployment.minimum(:id) } + finish { Deployment.maximum(:id) } + + def initialize(metric_definition) + super + + raise ArgumentError, 'Missing Deployment type' unless type + raise ArgumentError, "Invalid Deployment type: #{type}" unless type.in?(%i[all success failed]) + end + + private + + def type + options[:type].to_sym + end + + def relation + @metric_relation = case type + when :all + Deployment + when :success + Deployment.success + when :failed + Deployment.failed + end.where(time_constraints) + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_imported_projects_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_imported_projects_metric.rb index d485e8b4f72..05e29f2d885 100644 --- a/lib/gitlab/usage/metrics/instrumentations/count_imported_projects_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/count_imported_projects_metric.rb @@ -23,7 +23,7 @@ module Gitlab ::Project .select(:id) .where(Project.arel_table[:created_at].gteq(start)) # rubocop:disable UsageData/LargeTable - .order(created_at: :asc).limit(1).first&.id + .order(created_at: :asc).order(id: :asc).limit(1).first&.id end end end @@ -36,7 +36,7 @@ module Gitlab ::Project .select(:id) .where(Project.arel_table[:created_at].lteq(finish)) # rubocop:disable UsageData/LargeTable - .order(created_at: :desc).limit(1).first&.id + .order(created_at: :desc).order(id: :desc).limit(1).first&.id end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_personal_snippets_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_personal_snippets_metric.rb new file mode 100644 index 00000000000..9a34c535676 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_personal_snippets_metric.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountPersonalSnippetsMetric < DatabaseMetric + operation :count + + relation do + PersonalSnippet + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_project_snippets_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_project_snippets_metric.rb new file mode 100644 index 00000000000..af25a32592c --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_project_snippets_metric.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountProjectSnippetsMetric < DatabaseMetric + operation :count + + relation do + ProjectSnippet + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_projects_with_alerts_created_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_projects_with_alerts_created_metric.rb new file mode 100644 index 00000000000..8ae4000b802 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_projects_with_alerts_created_metric.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountProjectsWithAlertsCreatedMetric < DatabaseMetric + operation :distinct_count, column: :project_id + + relation do + ::AlertManagement::Alert + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_snippets_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_snippets_metric.rb new file mode 100644 index 00000000000..342ba802fd8 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_snippets_metric.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountSnippetsMetric < DatabaseMetric + operation :count + # Relation and operation are not used, but are included to satisfy expectations + # of other metric generation logic. + relation { Snippet } + + def value + count(project_snippet_relation) + count(personal_snippet_relation) + end + + def project_snippet_relation + ProjectSnippet.where(time_constraints) + end + + def personal_snippet_relation + PersonalSnippet.where(time_constraints) + end + + def to_sql + project_snippet_relation_sql = Gitlab::Usage::Metrics::Query.for(:count, project_snippet_relation) + personal_snippet_relation_sql = Gitlab::Usage::Metrics::Query.for(:count, personal_snippet_relation) + + "SELECT (#{project_snippet_relation_sql}) + (#{personal_snippet_relation_sql})" + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/installation_creation_date_metric.rb b/lib/gitlab/usage/metrics/instrumentations/installation_creation_date_metric.rb deleted file mode 100644 index c2ca62f9eba..00000000000 --- a/lib/gitlab/usage/metrics/instrumentations/installation_creation_date_metric.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Usage - module Metrics - module Instrumentations - class InstallationCreationDateMetric < GenericMetric - value do - User.where(id: 1).pick(:created_at) - end - end - end - end - end -end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 846bb934a3d..72168bce782 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -42,14 +42,11 @@ module Gitlab clear_memoized with_finished_at(:recording_ce_finished_at) do - usage_data_metrics + { recorded_at: recorded_at } + .merge(usage_data_metrics) end end - def license_usage_data - { recorded_at: recorded_at } - end - def recorded_at @recorded_at ||= Time.current end @@ -70,9 +67,6 @@ module Gitlab auto_devops_disabled: count(::ProjectAutoDevops.disabled), deploy_keys: count(DeployKey), # rubocop: disable UsageData/LargeTable: - deployments: deployment_count(Deployment), - successful_deployments: deployment_count(Deployment.success), - failed_deployments: deployment_count(Deployment.failed), feature_flags: count(Operations::FeatureFlag), # rubocop: enable UsageData/LargeTable: environments: count(::Environment), @@ -101,7 +95,7 @@ module Gitlab issues_using_zoom_quick_actions: distinct_count(ZoomMeeting, :issue_id), issues_with_embedded_grafana_charts_approx: grafana_embed_usage_data, issues_created_from_alerts: total_alert_issues, - incident_issues: count(::Issue.incident, start: minimum_id(Issue), finish: maximum_id(Issue)), + incident_issues: count(::Issue.with_issue_type(:incident), start: minimum_id(Issue), finish: maximum_id(Issue)), alert_bot_incident_issues: count(::Issue.authored(::User.alert_bot), start: minimum_id(Issue), finish: maximum_id(Issue)), keys: count(Key), label_lists: count(List.label), @@ -113,11 +107,10 @@ module Gitlab pages_domains: count(PagesDomain), pool_repositories: count(PoolRepository), projects: count(Project), - projects_creating_incidents: distinct_count(Issue.incident, :project_id), + projects_creating_incidents: distinct_count(Issue.with_issue_type(:incident), :project_id), projects_imported_from_github: count(Project.where(import_type: 'github')), projects_with_repositories_enabled: count(ProjectFeature.where('repository_access_level > ?', ProjectFeature::DISABLED)), projects_with_error_tracking_enabled: count(::ErrorTracking::ProjectErrorTrackingSetting.where(enabled: true)), - projects_with_alerts_created: distinct_count(::AlertManagement::Alert, :project_id), projects_with_enabled_alert_integrations: distinct_count(::AlertManagement::HttpIntegration.active, :project_id), projects_with_terraform_reports: distinct_count(::Ci::JobArtifact.of_report_type(:terraform), :project_id), projects_with_terraform_states: distinct_count(::Terraform::State, :project_id), @@ -125,8 +118,6 @@ module Gitlab protected_branches_except_default: count(ProtectedBranch.where.not(name: ['main', 'master', Gitlab::CurrentSettings.default_branch_name])), releases: count(Release), remote_mirrors: count(RemoteMirror), - personal_snippets: count(PersonalSnippet), - project_snippets: count(ProjectSnippet), suggestions: count(Suggestion), terraform_reports: count(::Ci::JobArtifact.of_report_type(:terraform)), terraform_states: count(::Terraform::State), @@ -140,9 +131,7 @@ module Gitlab integrations_usage, user_preferences_usage, service_desk_counts - ).tap do |data| - data[:snippets] = add(data[:personal_snippets], data[:project_snippets]) - end + ) } end # rubocop: enable Metrics/AbcSize @@ -150,19 +139,9 @@ module Gitlab def system_usage_data_monthly { counts_monthly: { - # rubocop: disable UsageData/LargeTable: - deployments: deployment_count(Deployment.where(monthly_time_range_db_params)), - successful_deployments: deployment_count(Deployment.success.where(monthly_time_range_db_params)), - failed_deployments: deployment_count(Deployment.failed.where(monthly_time_range_db_params)), - # rubocop: enable UsageData/LargeTable: projects: count(Project.where(monthly_time_range_db_params), start: minimum_id(Project), finish: maximum_id(Project)), - packages: count(::Packages::Package.where(monthly_time_range_db_params)), - personal_snippets: count(PersonalSnippet.where(monthly_time_range_db_params)), - project_snippets: count(ProjectSnippet.where(monthly_time_range_db_params)), - projects_with_alerts_created: distinct_count(::AlertManagement::Alert.where(monthly_time_range_db_params), :project_id) - }.tap do |data| - data[:snippets] = add(data[:personal_snippets], data[:project_snippets]) - end + packages: count(::Packages::Package.where(monthly_time_range_db_params)) + } } end # rubocop: enable CodeReuse/ActiveRecord @@ -372,7 +351,6 @@ module Gitlab package: usage_activity_by_stage_package(time_period), plan: usage_activity_by_stage_plan(time_period), release: usage_activity_by_stage_release(time_period), - secure: usage_activity_by_stage_secure(time_period), verify: usage_activity_by_stage_verify(time_period) } } @@ -450,8 +428,11 @@ module Gitlab start: minimum_id(User), finish: maximum_id(User)), projects_with_error_tracking_enabled: distinct_count(::Project.with_enabled_error_tracking.where(time_period), :creator_id), - projects_with_incidents: distinct_count(::Issue.incident.where(time_period), :project_id), - projects_with_alert_incidents: distinct_count(::Issue.incident.with_alert_management_alerts.where(time_period), :project_id), + projects_with_incidents: distinct_count(::Issue.with_issue_type(:incident).where(time_period), :project_id), + # We are making an assumption here that all alert_management_alerts are associated with an issue of type + # incident. In reality this is very close to the truth and allows more efficient queries. + # More info in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/121297#note_1416999956 + projects_with_alert_incidents: distinct_count(::AlertManagement::Alert.where(time_period).where.not(issue_id: nil), :project_id), projects_with_enabled_alert_integrations_histogram: integrations_histogram }.compact end @@ -514,13 +495,6 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord - # Currently too complicated and to get reliable counts for these stats: - # container_scanning_jobs, dast_jobs, dependency_scanning_jobs, license_management_jobs, sast_jobs, secret_detection_jobs - # Once https://gitlab.com/gitlab-org/gitlab/merge_requests/17568 is merged, this might be doable - def usage_activity_by_stage_secure(time_period) - {} - end - def with_metadata result = nil error = nil @@ -552,8 +526,7 @@ module Gitlab end def usage_data_metrics - license_usage_data - .merge(system_usage_data_license) + system_usage_data_license .merge(system_usage_data_settings) .merge(system_usage_data) .merge(system_usage_data_monthly) @@ -637,10 +610,6 @@ module Gitlab omniauth_provider_names.reject { |name| name.starts_with?('ldap') } end - def deployment_count(relation) - count relation, start: minimum_id(Deployment), finish: maximum_id(Deployment) - end - def project_imports(time_period) time_frame = metric_time_period(time_period) counters = { diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index badcda1def0..eaa4bf15fe1 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -3,18 +3,14 @@ module Gitlab module UsageDataCounters module HLLRedisCounter - DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH = 6.weeks - DEFAULT_DAILY_KEY_EXPIRY_LENGTH = 29.days + KEY_EXPIRY_LENGTH = 6.weeks REDIS_SLOT = 'hll_counters' EventError = Class.new(StandardError) UnknownEvent = Class.new(EventError) - UnknownAggregation = Class.new(EventError) - AggregationMismatch = Class.new(EventError) InvalidContext = Class.new(EventError) KNOWN_EVENTS_PATH = File.expand_path('known_events/*.yml', __dir__) - ALLOWED_AGGREGATIONS = %i(daily weekly).freeze # Track event on entity_id # Increment a Redis HLL counter for unique event_name and entity_id @@ -24,7 +20,6 @@ module Gitlab # Event example: # # - name: g_compliance_dashboard # Unique event name - # aggregation: weekly # Aggregation level, keys are stored weekly # # Usage: # @@ -63,8 +58,7 @@ module Gitlab # end_date - The end date of the time range. # context - Event context, plan level tracking. Available if set when tracking. def unique_events(event_names:, start_date:, end_date:, context: '') - count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date, context: context) do |events| - raise AggregationMismatch, events unless events_same_aggregation?(events) + count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date, context: context) do raise InvalidContext if context.present? && !context.in?(valid_context_list) end end @@ -78,9 +72,7 @@ module Gitlab end def calculate_events_union(event_names:, start_date:, end_date:) - count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date) do |events| - raise AggregationMismatch, events unless events_same_aggregation?(events) - end + count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date) end private @@ -94,12 +86,7 @@ module Gitlab return if event.blank? return unless Feature.enabled?(:redis_hll_tracking, type: :ops) - if event[:aggregation].to_sym == :daily - weekly_event = event.dup.tap { |e| e['aggregation'] = 'weekly' } - Gitlab::Redis::HLL.add(key: redis_key(weekly_event, time, context), value: values, expiry: expiry(weekly_event)) - end - - Gitlab::Redis::HLL.add(key: redis_key(event, time, context), value: values, expiry: expiry(event)) + Gitlab::Redis::HLL.add(key: redis_key(event, time, context), value: values, expiry: KEY_EXPIRY_LENGTH) rescue StandardError => e # Ignore any exceptions unless is dev or test env @@ -117,25 +104,18 @@ module Gitlab yield events if block_given? - aggregation = events.first[:aggregation] - - if Feature.disabled?(:revert_daily_hll_events_to_weekly_aggregation) - aggregation = 'weekly' - events = events.map { |e| e.merge(aggregation: 'weekly') } - end + keys = keys_for_aggregation(events: events, start_date: start_date, end_date: end_date, context: context) - keys = keys_for_aggregation(aggregation, events: events, start_date: start_date, end_date: end_date, context: context) return FALLBACK unless keys.any? redis_usage_data { Gitlab::Redis::HLL.count(keys: keys) } end - def keys_for_aggregation(aggregation, events:, start_date:, end_date:, context: '') - if aggregation.to_sym == :daily - daily_redis_keys(events: events, start_date: start_date, end_date: end_date, context: context) - else - weekly_redis_keys(events: events, start_date: start_date, end_date: end_date, context: context) - end + def keys_for_aggregation(events:, start_date:, end_date:, context: '') + end_date = end_date.end_of_week - 1.week + (start_date.to_date..end_date.to_date).map do |date| + events.map { |event| redis_key(event, date, context) } + end.flatten.uniq end def load_events(wildcard) @@ -152,15 +132,6 @@ module Gitlab known_events.map { |event| event[:name] } end - def events_same_aggregation?(events) - aggregation = events.first[:aggregation] - events.all? { |event| event[:aggregation] == aggregation } - end - - def expiry(event) - event[:aggregation].to_sym == :daily ? DEFAULT_DAILY_KEY_EXPIRY_LENGTH : DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH - end - def event_for(event_name) known_events.find { |event| event[:name] == event_name.to_s } end @@ -173,36 +144,13 @@ module Gitlab def redis_key(event, time, context = '') raise UnknownEvent, "Unknown event #{event[:name]}" unless known_events_names.include?(event[:name].to_s) - # ToDo: remove during https://gitlab.com/groups/gitlab-org/-/epics/9542 cleanup - raise UnknownAggregation, "Use :daily or :weekly aggregation" unless ALLOWED_AGGREGATIONS.include?(event[:aggregation].to_sym) - key = "{#{REDIS_SLOT}}_#{event[:name]}" - key = apply_time_aggregation(key, time, event) - key = "#{context}_#{key}" if context.present? - key - end - def apply_time_aggregation(key, time, event) - if event[:aggregation].to_sym == :daily - year_day = time.strftime('%G-%j') - "#{year_day}-#{key}" - else - year_week = time.strftime('%G-%V') - "#{key}-#{year_week}" - end - end + year_week = time.strftime('%G-%V') + key = "#{key}-#{year_week}" - def daily_redis_keys(events:, start_date:, end_date:, context: '') - (start_date.to_date..end_date.to_date).map do |date| - events.map { |event| redis_key(event, date, context) } - end.flatten - end - - def weekly_redis_keys(events:, start_date:, end_date:, context: '') - end_date = end_date.end_of_week - 1.week - (start_date.to_date..end_date.to_date).map do |date| - events.map { |event| redis_key(event, date, context) } - end.flatten.uniq + key = "#{context}_#{key}" if context.present? + key end end end diff --git a/lib/gitlab/usage_data_counters/jetbrains_bundled_plugin_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/jetbrains_bundled_plugin_activity_unique_counter.rb new file mode 100644 index 00000000000..a9e8d9bf0cb --- /dev/null +++ b/lib/gitlab/usage_data_counters/jetbrains_bundled_plugin_activity_unique_counter.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + module JetBrainsBundledPluginActivityUniqueCounter + JETBRAINS_BUNDLED_API_REQUEST_ACTION = 'i_editor_extensions_user_jetbrains_bundled_api_request' + JETBRAINS_BUNDLED_USER_AGENT_REGEX = /\AIntelliJ-GitLab-Plugin/ + + class << self + def track_api_request_when_trackable(user_agent:, user:) + user_agent&.match?(JETBRAINS_BUNDLED_USER_AGENT_REGEX) && + track_unique_action_by_user(JETBRAINS_BUNDLED_API_REQUEST_ACTION, user) + end + + private + + def track_unique_action_by_user(action, user) + return unless user + + track_unique_action(action, user.id) + end + + def track_unique_action(action, value) + Gitlab::UsageDataCounters::HLLRedisCounter.track_usage_event(action, value) + end + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml index f685f0d65d9..c3e1c34151b 100644 --- a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml +++ b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml @@ -307,3 +307,5 @@ aggregation: weekly - name: p_ci_templates_terraform_module aggregation: weekly +- name: p_ci_templates_pages_zola + aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml index db0c0653f63..bd8c79f4801 100644 --- a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml @@ -79,6 +79,8 @@ aggregation: weekly - name: i_code_review_user_jetbrains_api_request aggregation: weekly +- name: i_editor_extensions_user_jetbrains_bundled_api_request + aggregation: weekly - name: i_code_review_user_gitlab_cli_api_request aggregation: weekly - name: i_code_review_user_create_mr_from_issue diff --git a/lib/gitlab/usage_data_counters/known_events/product_analytics.yml b/lib/gitlab/usage_data_counters/known_events/product_analytics.yml index 5a791c4b3c2..c43bf9040dd 100644 --- a/lib/gitlab/usage_data_counters/known_events/product_analytics.yml +++ b/lib/gitlab/usage_data_counters/known_events/product_analytics.yml @@ -2,3 +2,7 @@ aggregation: weekly - name: project_initialized_product_analytics aggregation: weekly +- name: user_created_analytics_dashboard + aggregation: weekly +- name: user_visited_dashboard + aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/quickactions.yml b/lib/gitlab/usage_data_counters/known_events/quickactions.yml index 136d284f462..69f92ac5c0a 100644 --- a/lib/gitlab/usage_data_counters/known_events/quickactions.yml +++ b/lib/gitlab/usage_data_counters/known_events/quickactions.yml @@ -133,3 +133,7 @@ aggregation: weekly - name: i_quickactions_blocks aggregation: weekly +- name: i_quickactions_unlink + aggregation: weekly +- name: i_quickactions_promote_to + aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/workspaces.yml b/lib/gitlab/usage_data_counters/known_events/workspaces.yml new file mode 100644 index 00000000000..8a96524b167 --- /dev/null +++ b/lib/gitlab/usage_data_counters/known_events/workspaces.yml @@ -0,0 +1,5 @@ +- name: users_updating_workspaces + aggregation: weekly + +- name: users_creating_workspaces + aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb b/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb index d6e05f30a0d..ece2ffea83b 100644 --- a/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb +++ b/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb @@ -4,7 +4,7 @@ module Gitlab module UsageDataCounters class KubernetesAgentCounter < BaseCounter PREFIX = 'kubernetes_agent' - KNOWN_EVENTS = %w[gitops_sync k8s_api_proxy_request].freeze + KNOWN_EVENTS = %w[gitops_sync k8s_api_proxy_request flux_git_push_notifications_total].freeze class << self def increment_event_counts(events) diff --git a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb index fceeacb60ca..1ed2e891a1f 100644 --- a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb @@ -64,20 +64,15 @@ module Gitlab end def track_create_mr_action(user:, merge_request:) - track_unique_action_by_user(MR_USER_CREATE_ACTION, user) track_unique_action_by_merge_request(MR_CREATE_ACTION, merge_request) project = merge_request.target_project - Gitlab::Tracking.event( - name, - :create, - project: project, - namespace: project.namespace, - user: user, - property: MR_USER_CREATE_ACTION, - label: 'redis_hll_counters.code_review.i_code_review_user_create_mr_monthly', - context: [Gitlab::Tracking::ServicePingContext.new(data_source: :redis_hll, - event: MR_USER_CREATE_ACTION).to_context] + + Gitlab::InternalEvents.track_event( + MR_USER_CREATE_ACTION, + user_id: user.id, + project_id: project.id, + namespace_id: project.namespace_id ) end diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index b92e7dbb725..dc0112c14d6 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -3,34 +3,8 @@ module Gitlab module Utils extend self - PathTraversalAttackError ||= Class.new(StandardError) DoubleEncodingError ||= Class.new(StandardError) - private_class_method def logger - @logger ||= Gitlab::AppLogger - end - - # Ensure that the relative path will not traverse outside the base directory - # We url decode the path to avoid passing invalid paths forward in url encoded format. - # Also see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24223#note_284122580 - # It also checks for ALT_SEPARATOR aka '\' (forward slash) - def check_path_traversal!(path) - return unless path - - path = path.to_s if path.is_a?(Gitlab::HashedPath) - raise PathTraversalAttackError, 'Invalid path' unless path.is_a?(String) - - path = decode_path(path) - path_regex = %r{(\A(\.{1,2})\z|\A\.\.[/\\]|[/\\]\.\.\z|[/\\]\.\.[/\\]|\n)} - - if path.match?(path_regex) - logger.warn(message: "Potential path traversal attempt detected", path: "#{path}") - raise PathTraversalAttackError, 'Invalid path' - end - - path - end - def allowlisted?(absolute_path, allowlist) path = absolute_path.downcase @@ -39,20 +13,6 @@ module Gitlab end end - def check_allowed_absolute_path!(path, allowlist) - return unless Pathname.new(path).absolute? - return if allowlisted?(path, allowlist) - - raise StandardError, "path #{path} is not allowed" - end - - def check_allowed_absolute_path_and_path_traversal!(path, path_allowlist) - traversal_path = check_path_traversal!(path) - raise StandardError, "path is not a string!" unless traversal_path.is_a?(String) - - check_allowed_absolute_path!(traversal_path, path_allowlist) - end - def decode_path(encoded_path) decoded = CGI.unescape(encoded_path) if decoded != CGI.unescape(decoded) @@ -103,12 +63,6 @@ module Gitlab .gsub(/(\A-+|-+\z)/, '') end - # Wraps ActiveSupport's Array#to_sentence to convert the given array to a - # comma-separated sentence joined with localized 'or' Strings instead of 'and'. - def to_exclusive_sentence(array) - array.to_sentence(two_words_connector: _(' or '), last_word_connector: _(', or ')) - end - # Converts newlines into HTML line break elements def nlbr(str) ActionView::Base.full_sanitizer.sanitize(+str, tags: []).gsub(/\r?\n/, '
    ').html_safe diff --git a/lib/gitlab/utils/markdown.rb b/lib/gitlab/utils/markdown.rb index 5087020affe..c95398a15df 100644 --- a/lib/gitlab/utils/markdown.rb +++ b/lib/gitlab/utils/markdown.rb @@ -4,7 +4,7 @@ module Gitlab module Utils module Markdown PUNCTUATION_REGEXP = /[^\p{Word}\- ]/u.freeze - PRODUCT_SUFFIX = /\s*\**\((core|starter|premium|ultimate|free|bronze|silver|gold)(\s+(only|self|sass))?\)\**/.freeze + PRODUCT_SUFFIX = /\s*\**\((core|starter|premium|ultimate|free|bronze|silver|gold)(\s+(only|self|saas))?\)\**/.freeze def string_to_anchor(string) string diff --git a/lib/gitlab/utils/sanitize_node_link.rb b/lib/gitlab/utils/sanitize_node_link.rb index 9c34302f75e..183741ff5f5 100644 --- a/lib/gitlab/utils/sanitize_node_link.rb +++ b/lib/gitlab/utils/sanitize_node_link.rb @@ -51,7 +51,9 @@ module Gitlab begin node[attr] = node[attr].strip + uri = Addressable::URI.parse(node[attr]) + uri = uri.normalize next unless uri.scheme next if safe_protocol?(uri.scheme) diff --git a/lib/gitlab/verify/ci_secure_files.rb b/lib/gitlab/verify/ci_secure_files.rb new file mode 100644 index 00000000000..9bb7f7260c4 --- /dev/null +++ b/lib/gitlab/verify/ci_secure_files.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module Verify + class CiSecureFiles < BatchVerifier + def name + 'CI Secure Files' + end + + def describe(object) + "SecureFile: #{object.id}" + end + + private + + # rubocop: disable CodeReuse/ActiveRecord + def all_relation + ::Ci::SecureFile.all + end + # rubocop: enable CodeReuse/ActiveRecord + + def local?(secure_file) + secure_file.local? + end + + def expected_checksum(secure_file) + secure_file.checksum + end + + def actual_checksum(secure_file) + Digest::SHA256.hexdigest(secure_file.file.read) + end + + def remote_object_exists?(secure_file) + secure_file.file.file.exists? + end + end + end +end diff --git a/lib/gitlab/x509/tag.rb b/lib/gitlab/x509/tag.rb index cf24e6f62bd..b9a3b11ac34 100644 --- a/lib/gitlab/x509/tag.rb +++ b/lib/gitlab/x509/tag.rb @@ -11,7 +11,7 @@ module Gitlab strong_memoize(:signature) do super - signature = X509::Signature.new(signature_text, signed_text, @tag.tagger.email, Time.at(@tag.tagger.date.seconds)) + signature = X509::Signature.new(signature_text, signed_text, @tag.user_email, @tag.date) signature unless signature.verified_signature.nil? end end diff --git a/lib/google_api/cloud_platform/client.rb b/lib/google_api/cloud_platform/client.rb index 5eef4fd0e4e..b3346d14ada 100644 --- a/lib/google_api/cloud_platform/client.rb +++ b/lib/google_api/cloud_platform/client.rb @@ -123,6 +123,10 @@ module GoogleApi enable_service(gcp_project_id, 'servicenetworking.googleapis.com') end + def enable_vision_api(gcp_project_id) + enable_service(gcp_project_id, 'vision.googleapis.com') + end + def revoke_authorizations uri = URI(REVOKE_URL) Gitlab::HTTP.post(uri, body: { 'token' => access_token }) diff --git a/lib/google_cloud/authentication.rb b/lib/google_cloud/authentication.rb new file mode 100644 index 00000000000..68dd0bdcccb --- /dev/null +++ b/lib/google_cloud/authentication.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module GoogleCloud + class Authentication + def initialize(scope:) + @scope = scope + end + + def generate_access_token(client_email, private_key) + credentials = Google::Auth::ServiceAccountCredentials.make_creds( + json_key_io: StringIO.new({ client_email: client_email, private_key: private_key }.to_json), + scope: @scope + ) + credentials.fetch_access_token!["access_token"] + rescue StandardError => e + ::Gitlab::ErrorTracking.track_exception(e, client_email: client_email) + nil + end + end +end diff --git a/lib/google_cloud/logging_service/logger.rb b/lib/google_cloud/logging_service/logger.rb new file mode 100644 index 00000000000..2c6dd6ea732 --- /dev/null +++ b/lib/google_cloud/logging_service/logger.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module GoogleCloud + module LoggingService + class Logger + WRITE_URL = "https://logging.googleapis.com/v2/entries:write" + SCOPE = "https://www.googleapis.com/auth/logging.write" + + def initialize + @auth = GoogleCloud::Authentication.new(scope: SCOPE) + end + + def log(client_email, private_key, payload) + access_token = @auth.generate_access_token(client_email, private_key) + + return unless access_token + + headers = build_headers(access_token) + + post(WRITE_URL, body: payload, headers: headers) + end + + private + + def build_headers(access_token) + { 'Authorization' => "Bearer #{access_token}", 'Content-Type' => 'application/json' } + end + + def post(url, body:, headers:) + Gitlab::HTTP.post( + url, + body: body, + headers: headers + ) + rescue URI::InvalidURIError => e + Gitlab::ErrorTracking.log_exception(e) + rescue *Gitlab::HTTP::HTTP_ERRORS + end + end + end +end diff --git a/lib/grafana/validator.rb b/lib/grafana/validator.rb index 760263f7ec9..616bb6aaaf0 100644 --- a/lib/grafana/validator.rb +++ b/lib/grafana/validator.rb @@ -42,7 +42,6 @@ module Grafana private - # See defaults in Banzai::Filter::InlineGrafanaMetricsFilter. def validate_query_params! return if [:from, :to].all? { |param| query_params.include?(param) } diff --git a/lib/kramdown/parser/atlassian_document_format.rb b/lib/kramdown/parser/atlassian_document_format.rb index d27697a59a6..5a481042b15 100644 --- a/lib/kramdown/parser/atlassian_document_format.rb +++ b/lib/kramdown/parser/atlassian_document_format.rb @@ -219,8 +219,8 @@ module Kramdown # opportunity to replace it later. Mention name can have # spaces, so double quote it mention_text = ast_node.dig('attrs', 'text')&.gsub('@', '') - mention_text = %Q("#{mention_text}") if mention_text&.include?(' ') - mention_text = %Q(@adf-mention:#{mention_text}) + mention_text = %("#{mention_text}") if mention_text&.include?(' ') + mention_text = %(@adf-mention:#{mention_text}) add_text(mention_text, element, :text) end diff --git a/lib/object_storage/fog_helpers.rb b/lib/object_storage/fog_helpers.rb new file mode 100644 index 00000000000..1db75ea24b9 --- /dev/null +++ b/lib/object_storage/fog_helpers.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module ObjectStorage + module FogHelpers + include ::Gitlab::Utils::StrongMemoize + + def available? + object_store.enabled + end + + private + + def delete_object(key) + return unless available? + + connection.delete_object(bucket_name, object_key(key)) + + # So far, only GoogleCloudStorage raises an exception when the file is not found. + # Other providers support idempotent requests and does not raise an error + # when the file is missing. + rescue ::Google::Apis::ClientError => e + Gitlab::ErrorTracking.log_exception(e) + end + + def storage_location_identifier + raise NotImplementedError, "#{self} does not implement #{__method__}" + end + + def object_store + ObjectStorage::Config::LOCATIONS.fetch(storage_location_identifier).object_store + end + + def bucket_name + object_store.remote_directory + end + + def object_key(key) + # We allow administrators to create "sub buckets" by setting a prefix. + # This makes it possible to deploy GitLab with only one object storage + # bucket. This mirrors the implementation in app/uploaders/object_storage.rb. + File.join([object_store.bucket_prefix, key].compact) + end + + def connection + return unless available? + + ::Fog::Storage.new(object_store.connection.to_hash.deep_symbolize_keys) + end + strong_memoize_attr :connection + end +end diff --git a/lib/object_storage/pending_direct_upload.rb b/lib/object_storage/pending_direct_upload.rb index 3e84bc4ebc9..3a930e0e0af 100644 --- a/lib/object_storage/pending_direct_upload.rb +++ b/lib/object_storage/pending_direct_upload.rb @@ -2,31 +2,97 @@ module ObjectStorage class PendingDirectUpload + include ObjectStorage::FogHelpers + KEY = 'pending_direct_uploads' + MAX_UPLOAD_DURATION = 3.hours.freeze - def self.prepare(location_identifier, path) - ::Gitlab::Redis::SharedState.with do |redis| + def self.prepare(location_identifier, object_storage_path) + with_redis do |redis| # We need to store the location_identifier together with the timestamp to properly delete # this object if ever this upload gets stale. The location identifier will be used # by the clean up worker to properly generate the storage options through ObjectStorage::Config.for_location - redis.hset(KEY, key(location_identifier, path), Time.current.utc.to_i) + key = redis_key(location_identifier, object_storage_path) + redis.hset(KEY, key, Time.current.utc.to_i) + log_event(:prepared, key) + end + end + + def self.exists?(location_identifier, object_storage_path) + with_redis do |redis| + redis.hexists(KEY, redis_key(location_identifier, object_storage_path)) + end + end + + def self.complete(location_identifier, object_storage_path) + with_redis do |redis| + key = redis_key(location_identifier, object_storage_path) + redis.hdel(KEY, key) + log_event(:completed, key) end end - def self.exists?(location_identifier, path) - ::Gitlab::Redis::SharedState.with do |redis| - redis.hexists(KEY, key(location_identifier, path)) + def self.redis_key(location_identifier, object_storage_path) + [location_identifier, object_storage_path].join(':') + end + + def self.count + with_redis do |redis| + redis.hlen(KEY) end end - def self.complete(location_identifier, path) - ::Gitlab::Redis::SharedState.with do |redis| - redis.hdel(KEY, key(location_identifier, path)) + def self.each + with_redis do |redis| + redis.hscan_each(KEY) do |entry| + redis_key, timestamp = entry + storage_location_identifier, object_storage_path = redis_key.split(':') + + object = new( + redis_key: redis_key, + storage_location_identifier: storage_location_identifier, + object_storage_path: object_storage_path, + timestamp: timestamp + ) + + yield(object) + end end end - def self.key(location_identifier, path) - [location_identifier, path].join(':') + def self.with_redis(&block) + Gitlab::Redis::SharedState.with(&block) # rubocop:disable CodeReuse/ActiveRecord end + + def self.log_event(event, redis_key) + Gitlab::AppLogger.info( + message: "Pending direct upload #{event}", + redis_key: redis_key + ) + end + + def initialize(redis_key:, storage_location_identifier:, object_storage_path:, timestamp:) + @redis_key = redis_key + @storage_location_identifier = storage_location_identifier.to_sym + @object_storage_path = object_storage_path + @timestamp = timestamp.to_i + end + + def stale? + timestamp < MAX_UPLOAD_DURATION.ago.utc.to_i + end + + def delete + delete_object(object_storage_path) + + self.class.with_redis do |redis| + redis.hdel(self.class::KEY, redis_key) + self.class.log_event(:deleted, redis_key) + end + end + + private + + attr_reader :redis_key, :storage_location_identifier, :object_storage_path, :timestamp end end diff --git a/lib/product_analytics/settings.rb b/lib/product_analytics/settings.rb index 5d52965f5be..1c5f25c36b9 100644 --- a/lib/product_analytics/settings.rb +++ b/lib/product_analytics/settings.rb @@ -2,9 +2,16 @@ module ProductAnalytics class Settings - CONFIG_KEYS = (%w[jitsu_host jitsu_project_xid jitsu_administrator_email jitsu_administrator_password] + - %w[product_analytics_data_collector_host product_analytics_clickhouse_connection_string] + - %w[cube_api_base_url cube_api_key]).freeze + BASE_CONFIG_KEYS = %w[product_analytics_data_collector_host cube_api_base_url cube_api_key].freeze + + JITSU_CONFIG_KEYS = (%w[jitsu_host jitsu_project_xid jitsu_administrator_email jitsu_administrator_password] + + %w[product_analytics_clickhouse_connection_string] + BASE_CONFIG_KEYS).freeze + + SNOWPLOW_CONFIG_KEYS = (%w[product_analytics_configurator_connection_string] + + BASE_CONFIG_KEYS).freeze + + ALL_CONFIG_KEYS = (ProductAnalytics::Settings::BASE_CONFIG_KEYS + ProductAnalytics::Settings::JITSU_CONFIG_KEYS + + ProductAnalytics::Settings::SNOWPLOW_CONFIG_KEYS).freeze def initialize(project:) @project = project @@ -14,25 +21,39 @@ module ProductAnalytics ::Gitlab::CurrentSettings.product_analytics_enabled? && configured? end - # rubocop:disable GitlabSecurity/PublicSend def configured? - CONFIG_KEYS.all? do |key| - @project.project_setting.public_send(key).present? || - ::Gitlab::CurrentSettings.public_send(key).present? + return configured_snowplow? if Feature.enabled?(:product_analytics_snowplow_support, @project) + + JITSU_CONFIG_KEYS.all? do |key| + get_setting_value(key).present? + end + end + + def configured_snowplow? + SNOWPLOW_CONFIG_KEYS.all? do |key| + get_setting_value(key).present? end end - CONFIG_KEYS.each do |key| + ALL_CONFIG_KEYS.each do |key| define_method key.to_sym do - @project.project_setting.public_send(key).presence || ::Gitlab::CurrentSettings.public_send(key) + get_setting_value(key) end end - # rubocop:enable GitlabSecurity/PublicSend class << self def for_project(project) ProductAnalytics::Settings.new(project: project) end end + + private + + # rubocop:disable GitlabSecurity/PublicSend + def get_setting_value(key) + @project.project_setting.public_send(key).presence || + ::Gitlab::CurrentSettings.public_send(key) + end + # rubocop:enable GitlabSecurity/PublicSend end end diff --git a/lib/quality/seeders/issues.rb b/lib/quality/seeders/issues.rb index cac034767f6..fb3d78bc8d2 100644 --- a/lib/quality/seeders/issues.rb +++ b/lib/quality/seeders/issues.rb @@ -31,7 +31,7 @@ module Quality } params[:closed_at] = params[:created_at] + rand(35).days if params[:state] == 'closed' - create_result = ::Issues::CreateService.new(container: project, current_user: team.sample, params: params, spam_params: nil).execute_without_rate_limiting + create_result = ::Issues::CreateService.new(container: project, current_user: team.sample, params: params, perform_spam_check: false).execute_without_rate_limiting if create_result.success? created_issues_count += 1 diff --git a/lib/sidebars/groups/menus/packages_registries_menu.rb b/lib/sidebars/groups/menus/packages_registries_menu.rb index 68c8e9675a7..bd1cca6473a 100644 --- a/lib/sidebars/groups/menus/packages_registries_menu.rb +++ b/lib/sidebars/groups/menus/packages_registries_menu.rb @@ -36,7 +36,7 @@ module Sidebars ::Sidebars::MenuItem.new( title: _('Package Registry'), link: group_packages_path(context.group), - super_sidebar_parent: ::Sidebars::Groups::SuperSidebarMenus::OperationsMenu, + super_sidebar_parent: ::Sidebars::Groups::SuperSidebarMenus::DeployMenu, active_routes: { controller: 'groups/packages' }, item_id: :packages_registry ) diff --git a/lib/sidebars/groups/super_sidebar_menus/deploy_menu.rb b/lib/sidebars/groups/super_sidebar_menus/deploy_menu.rb new file mode 100644 index 00000000000..fe9cc5280c7 --- /dev/null +++ b/lib/sidebars/groups/super_sidebar_menus/deploy_menu.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Sidebars + module Groups + module SuperSidebarMenus + class DeployMenu < ::Sidebars::Menu + override :title + def title + s_('Navigation|Deploy') + end + + override :sprite_icon + def sprite_icon + 'deployments' + end + + override :configure_menu_items + def configure_menu_items + [ + :packages_registry + ].each { |id| add_item(::Sidebars::NilMenuItem.new(item_id: id)) } + end + end + end + end +end diff --git a/lib/sidebars/groups/super_sidebar_menus/operations_menu.rb b/lib/sidebars/groups/super_sidebar_menus/operations_menu.rb index fe17ada69e4..e716801486e 100644 --- a/lib/sidebars/groups/super_sidebar_menus/operations_menu.rb +++ b/lib/sidebars/groups/super_sidebar_menus/operations_menu.rb @@ -11,14 +11,13 @@ module Sidebars override :sprite_icon def sprite_icon - 'deployments' + 'cloud-pod' end override :configure_menu_items def configure_menu_items [ :dependency_proxy, - :packages_registry, :container_registry, :group_kubernetes_clusters ].each { |id| add_item(::Sidebars::NilMenuItem.new(item_id: id)) } diff --git a/lib/sidebars/groups/super_sidebar_menus/secure_menu.rb b/lib/sidebars/groups/super_sidebar_menus/secure_menu.rb index c79e7e379d0..042e2381744 100644 --- a/lib/sidebars/groups/super_sidebar_menus/secure_menu.rb +++ b/lib/sidebars/groups/super_sidebar_menus/secure_menu.rb @@ -19,6 +19,7 @@ module Sidebars [ :security_dashboard, :vulnerability_report, + :dependency_list, :audit_events, :compliance, :scan_policies diff --git a/lib/sidebars/groups/super_sidebar_panel.rb b/lib/sidebars/groups/super_sidebar_panel.rb index 03af904d99d..1f79dff398e 100644 --- a/lib/sidebars/groups/super_sidebar_panel.rb +++ b/lib/sidebars/groups/super_sidebar_panel.rb @@ -18,6 +18,7 @@ module Sidebars add_menu(Sidebars::Groups::SuperSidebarMenus::CodeMenu.new(context)) add_menu(Sidebars::Groups::SuperSidebarMenus::BuildMenu.new(context)) add_menu(Sidebars::Groups::SuperSidebarMenus::SecureMenu.new(context)) + add_menu(Sidebars::Groups::SuperSidebarMenus::DeployMenu.new(context)) add_menu(Sidebars::Groups::SuperSidebarMenus::OperationsMenu.new(context)) add_menu(Sidebars::Groups::SuperSidebarMenus::MonitorMenu.new(context)) add_menu(Sidebars::Groups::SuperSidebarMenus::AnalyzeMenu.new(context)) diff --git a/lib/sidebars/projects/menus/deployments_menu.rb b/lib/sidebars/projects/menus/deployments_menu.rb index 19612fcee85..0a411f075b7 100644 --- a/lib/sidebars/projects/menus/deployments_menu.rb +++ b/lib/sidebars/projects/menus/deployments_menu.rb @@ -49,7 +49,7 @@ module Sidebars ::Sidebars::MenuItem.new( title: s_('FeatureFlags|Feature flags'), link: project_feature_flags_path(context.project), - super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::BuildMenu, + super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::DeployMenu, active_routes: { controller: :feature_flags }, container_html_options: { class: 'shortcuts-feature-flags' }, item_id: :feature_flags @@ -64,7 +64,7 @@ module Sidebars ::Sidebars::MenuItem.new( title: _('Environments'), link: project_environments_path(context.project), - super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::BuildMenu, + super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::OperationsMenu, active_routes: { controller: :environments }, container_html_options: { class: 'shortcuts-environments' }, item_id: :environments @@ -80,7 +80,7 @@ module Sidebars ::Sidebars::MenuItem.new( title: _('Releases'), link: project_releases_path(context.project), - super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::BuildMenu, + super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::DeployMenu, item_id: :releases, active_routes: { controller: :releases }, container_html_options: { class: 'shortcuts-deployments-releases' } diff --git a/lib/sidebars/projects/menus/monitor_menu.rb b/lib/sidebars/projects/menus/monitor_menu.rb index f1fc9f70ef8..a74448d0bdc 100644 --- a/lib/sidebars/projects/menus/monitor_menu.rb +++ b/lib/sidebars/projects/menus/monitor_menu.rb @@ -8,7 +8,6 @@ module Sidebars def configure_menu_items return false unless feature_enabled? - add_item(metrics_dashboard_menu_item) add_item(error_tracking_menu_item) add_item(alert_management_menu_item) add_item(incidents_menu_item) @@ -49,23 +48,6 @@ module Sidebars context.project.feature_available?(:monitor, context.current_user) end - def metrics_dashboard_menu_item - return ::Sidebars::NilMenuItem.new(item_id: :metrics) if Feature.enabled?(:remove_monitor_metrics) - - unless can?(context.current_user, :metrics_dashboard, context.project) - return ::Sidebars::NilMenuItem.new(item_id: :metrics) - end - - ::Sidebars::MenuItem.new( - title: _('Metrics'), - link: project_metrics_dashboard_path(context.project), - super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::MonitorMenu, - active_routes: { path: 'metrics_dashboard#show' }, - container_html_options: { class: 'shortcuts-metrics' }, - item_id: :metrics - ) - end - def error_tracking_menu_item unless can?(context.current_user, :read_sentry_issue, context.project) return ::Sidebars::NilMenuItem.new(item_id: :error_tracking) diff --git a/lib/sidebars/projects/menus/packages_registries_menu.rb b/lib/sidebars/projects/menus/packages_registries_menu.rb index 31a1aa56ab5..f41b7ce1a73 100644 --- a/lib/sidebars/projects/menus/packages_registries_menu.rb +++ b/lib/sidebars/projects/menus/packages_registries_menu.rb @@ -39,7 +39,7 @@ module Sidebars ::Sidebars::MenuItem.new( title: _('Package Registry'), link: project_packages_path(context.project), - super_sidebar_parent: Sidebars::Projects::SuperSidebarMenus::OperationsMenu, + super_sidebar_parent: Sidebars::Projects::SuperSidebarMenus::DeployMenu, active_routes: { controller: :packages }, item_id: :packages_registry, container_html_options: { class: 'shortcuts-container-registry' } @@ -54,7 +54,7 @@ module Sidebars ::Sidebars::MenuItem.new( title: _('Container Registry'), link: project_container_registry_index_path(context.project), - super_sidebar_parent: Sidebars::Projects::SuperSidebarMenus::OperationsMenu, + super_sidebar_parent: Sidebars::Projects::SuperSidebarMenus::DeployMenu, active_routes: { controller: 'projects/registry/repositories' }, item_id: :container_registry ) @@ -91,14 +91,14 @@ module Sidebars end def model_experiments_menu_item - if Feature.disabled?(:ml_experiment_tracking, context.project) + unless can?(context.current_user, :read_model_experiments, context.project) return ::Sidebars::NilMenuItem.new(item_id: :model_experiments) end ::Sidebars::MenuItem.new( title: _('Model experiments'), link: project_ml_experiments_path(context.project), - super_sidebar_parent: Sidebars::Projects::SuperSidebarMenus::AnalyzeMenu, + super_sidebar_parent: Sidebars::Projects::SuperSidebarMenus::DeployMenu, active_routes: { controller: %w[projects/ml/experiments projects/ml/candidates] }, item_id: :model_experiments ) diff --git a/lib/sidebars/projects/super_sidebar_menus/analyze_menu.rb b/lib/sidebars/projects/super_sidebar_menus/analyze_menu.rb index 58b231a269c..2c5dc8a08e7 100644 --- a/lib/sidebars/projects/super_sidebar_menus/analyze_menu.rb +++ b/lib/sidebars/projects/super_sidebar_menus/analyze_menu.rb @@ -25,8 +25,7 @@ module Sidebars :code_review, :merge_request_analytics, :issues, - :insights, - :model_experiments + :insights ].each { |id| add_item(::Sidebars::NilMenuItem.new(item_id: id)) } end end diff --git a/lib/sidebars/projects/super_sidebar_menus/build_menu.rb b/lib/sidebars/projects/super_sidebar_menus/build_menu.rb index 30603e1deeb..119ddf28873 100644 --- a/lib/sidebars/projects/super_sidebar_menus/build_menu.rb +++ b/lib/sidebars/projects/super_sidebar_menus/build_menu.rb @@ -20,10 +20,7 @@ module Sidebars :pipelines, :jobs, :pipelines_editor, - :releases, - :environments, :pipeline_schedules, - :feature_flags, :test_cases, :artifacts ].each { |id| add_item(::Sidebars::NilMenuItem.new(item_id: id)) } diff --git a/lib/sidebars/projects/super_sidebar_menus/deploy_menu.rb b/lib/sidebars/projects/super_sidebar_menus/deploy_menu.rb new file mode 100644 index 00000000000..49aa6a23a0e --- /dev/null +++ b/lib/sidebars/projects/super_sidebar_menus/deploy_menu.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + module SuperSidebarMenus + class DeployMenu < ::Sidebars::Menu + override :title + def title + s_('Navigation|Deploy') + end + + override :sprite_icon + def sprite_icon + 'deployments' + end + + override :configure_menu_items + def configure_menu_items + [ + :releases, + :feature_flags, + :packages_registry, + :container_registry, + :model_experiments + ].each { |id| add_item(::Sidebars::NilMenuItem.new(item_id: id)) } + end + end + end + end +end diff --git a/lib/sidebars/projects/super_sidebar_menus/monitor_menu.rb b/lib/sidebars/projects/super_sidebar_menus/monitor_menu.rb index fb56f6f3792..6e64ac01ffa 100644 --- a/lib/sidebars/projects/super_sidebar_menus/monitor_menu.rb +++ b/lib/sidebars/projects/super_sidebar_menus/monitor_menu.rb @@ -17,7 +17,6 @@ module Sidebars override :configure_menu_items def configure_menu_items [ - :metrics, :error_tracking, :alert_management, :incidents, diff --git a/lib/sidebars/projects/super_sidebar_menus/operations_menu.rb b/lib/sidebars/projects/super_sidebar_menus/operations_menu.rb index 64cf4aee9c5..85d60c0bad3 100644 --- a/lib/sidebars/projects/super_sidebar_menus/operations_menu.rb +++ b/lib/sidebars/projects/super_sidebar_menus/operations_menu.rb @@ -11,14 +11,13 @@ module Sidebars override :sprite_icon def sprite_icon - 'deployments' + 'cloud-pod' end override :configure_menu_items def configure_menu_items [ - :packages_registry, - :container_registry, + :environments, :kubernetes, :terraform_states, :infrastructure_registry, diff --git a/lib/sidebars/projects/super_sidebar_panel.rb b/lib/sidebars/projects/super_sidebar_panel.rb index 640666fd968..e9de4c2d92c 100644 --- a/lib/sidebars/projects/super_sidebar_panel.rb +++ b/lib/sidebars/projects/super_sidebar_panel.rb @@ -18,6 +18,7 @@ module Sidebars add_menu(Sidebars::Projects::SuperSidebarMenus::CodeMenu.new(context)) add_menu(Sidebars::Projects::SuperSidebarMenus::BuildMenu.new(context)) add_menu(Sidebars::Projects::SuperSidebarMenus::SecureMenu.new(context)) + add_menu(Sidebars::Projects::SuperSidebarMenus::DeployMenu.new(context)) add_menu(Sidebars::Projects::SuperSidebarMenus::OperationsMenu.new(context)) add_menu(Sidebars::Projects::SuperSidebarMenus::MonitorMenu.new(context)) add_menu(Sidebars::Projects::SuperSidebarMenus::AnalyzeMenu.new(context)) diff --git a/lib/sidebars/user_profile/panel.rb b/lib/sidebars/user_profile/panel.rb index 9a595fdf64c..1852ef928f4 100644 --- a/lib/sidebars/user_profile/panel.rb +++ b/lib/sidebars/user_profile/panel.rb @@ -4,6 +4,9 @@ module Sidebars module UserProfile class Panel < ::Sidebars::Panel include UsersHelper + include Gitlab::Allowable + + delegate :current_user, to: :@context override :configure_menus def configure_menus diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake index 13365b9ec07..8c7e429ef24 100644 --- a/lib/tasks/cache.rake +++ b/lib/tasks/cache.rake @@ -7,24 +7,26 @@ namespace :cache do desc "GitLab | Cache | Clear redis cache" task redis: :environment do - Gitlab::Redis::Cache.with do |redis| - cache_key_pattern = %W[#{Gitlab::Redis::Cache::CACHE_NAMESPACE}* - projects/*/pipeline_status] + [Gitlab::Redis::Cache, Gitlab::Redis::RepositoryCache].each do |redis_instance| + redis_instance.with do |redis| + cache_key_pattern = %W[#{Gitlab::Redis::Cache::CACHE_NAMESPACE}* + projects/*/pipeline_status] - cache_key_pattern.each do |match| - cursor = REDIS_SCAN_START_STOP - loop do - cursor, keys = redis.scan( - cursor, - match: match, - count: REDIS_CLEAR_BATCH_SIZE - ) + cache_key_pattern.each do |match| + cursor = REDIS_SCAN_START_STOP + loop do + cursor, keys = redis.scan( + cursor, + match: match, + count: REDIS_CLEAR_BATCH_SIZE + ) - Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - redis.del(*keys) if keys.any? - end + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + Gitlab::Redis::ClusterUtil.batch_unlink(keys, redis) if keys.any? + end - break if cursor == REDIS_SCAN_START_STOP + break if cursor == REDIS_SCAN_START_STOP + end end end end diff --git a/lib/tasks/frontend.rake b/lib/tasks/frontend.rake index e768c42736d..a6754f13089 100644 --- a/lib/tasks/frontend.rake +++ b/lib/tasks/frontend.rake @@ -18,30 +18,12 @@ unless Rails.env.production? t.rspec_opts = '--format documentation' end - desc 'GitLab | Frontend | Generate fixtures for JavaScript integration tests' - RSpec::Core::RakeTask.new(:mock_server_rspec_fixtures) do |t, args| - require 'yaml' - - base_path = Pathname.new('spec/frontend_integration/fixture_generators.yml') - ee_path = Pathname.new('ee') + base_path - - fixtures = YAML.safe_load(base_path.read) - fixtures.concat(Array(YAML.safe_load(ee_path.read))) if Gitlab.ee? && ee_path.exist? - - t.pattern = fixtures.join(',') - ENV['NO_KNAPSACK'] = 'true' - t.rspec_opts = '--format documentation' - end - desc 'GitLab | Frontend | Run JavaScript tests' task tests: ['yarn:check'] do sh "yarn test" do |ok, res| abort('rake frontend:tests failed') unless ok end end - - desc 'GitLab | Frontend | Shortcut for generating all fixtures used by MirajeJS mock server' - task mock_server_fixtures: ['frontend:mock_server_rspec_fixtures', 'gitlab:graphql:schema:dump'] end desc 'GitLab | Frontend | Shortcut for frontend:fixtures and frontend:tests' diff --git a/lib/tasks/gitlab/assets.rake b/lib/tasks/gitlab/assets.rake index 2522488f579..b8a6e701876 100644 --- a/lib/tasks/gitlab/assets.rake +++ b/lib/tasks/gitlab/assets.rake @@ -15,6 +15,7 @@ module Tasks yarn.lock babel.config.js config/webpack.config.js + .nvmrc ].freeze # Ruby gems might emit assets which have an impact on compilation # or have a direct impact on asset compilation (e.g. scss) and therefore diff --git a/lib/tasks/gitlab/background_migrations.rake b/lib/tasks/gitlab/background_migrations.rake index eca51c345d1..a4e14af22bd 100644 --- a/lib/tasks/gitlab/background_migrations.rake +++ b/lib/tasks/gitlab/background_migrations.rake @@ -6,7 +6,7 @@ namespace :gitlab do namespace :background_migrations do desc 'Synchronously finish executing a batched background migration' task :finalize, [:job_class_name, :table_name, :column_name, :job_arguments] => :environment do |_, args| - if Gitlab::Database.db_config_names.size > 1 + if Gitlab::Database.db_config_names(with_schema: :gitlab_shared).size > 1 puts "Please specify the database".color(:red) exit 1 end diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 22e1d903c8d..4143200ece4 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -206,6 +206,16 @@ namespace :gitlab do Tasks::Gitlab::Backup.restore_task('packages') end end + + namespace :ci_secure_files do + task create: :gitlab_environment do + Tasks::Gitlab::Backup.create_task('ci_secure_files') + end + + task restore: :gitlab_environment do + Tasks::Gitlab::Backup.restore_task('ci_secure_files') + end + end end # namespace end: backup end diff --git a/lib/tasks/gitlab/ci_secure_files/check.rake b/lib/tasks/gitlab/ci_secure_files/check.rake new file mode 100644 index 00000000000..2c538a9365c --- /dev/null +++ b/lib/tasks/gitlab/ci_secure_files/check.rake @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +namespace :gitlab do + namespace :ci_secure_files do + desc 'GitLab | CI Secure Files | Check integrity of uploaded Secure Files' + task check: :environment do + Gitlab::Verify::RakeTask.run!(Gitlab::Verify::CiSecureFiles) + end + end +end diff --git a/lib/tasks/gitlab/ci_secure_files/migrate.rake b/lib/tasks/gitlab/ci_secure_files/migrate.rake new file mode 100644 index 00000000000..8de1b7da6be --- /dev/null +++ b/lib/tasks/gitlab/ci_secure_files/migrate.rake @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +desc "GitLab | CI Secure Files | Migrate Secure Files to remote storage" +namespace :gitlab do + namespace :ci_secure_files do + task migrate: :environment do + require 'logger' + + logger = Logger.new($stdout) + logger.info('Starting transfer of Secure Files to object storage') + + begin + Gitlab::Ci::SecureFiles::MigrationHelper.migrate_to_remote_storage do |file| + message = "Transferred Secure File ID #{file.id} (#{file.name}) to object storage" + + logger.info(message) + end + rescue StandardError => e + logger.error("Failed to migrate: #{e.message}") + end + end + end +end diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 34ccce3ba2f..026cb39a92f 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -473,7 +473,7 @@ namespace :gitlab do Gitlab::Database::SchemaValidation::TrackInconsistency.new( inconsistency, Project.find_by_full_path(gitlab_url), - User.support_bot + User.automation_bot ).execute puts inconsistency.inspect @@ -482,18 +482,12 @@ namespace :gitlab do end namespace :dictionary do - DB_DOCS_PATH = Rails.root.join('db', 'docs') - desc 'Generate database docs yaml' task generate: :environment do next if Gitlab.jh? - FileUtils.mkdir_p(DB_DOCS_PATH) - - if Gitlab.ee? - Gitlab::Database::EE_DATABASES_NAME_TO_DIR.each do |_, ee_db_dir| - FileUtils.mkdir_p(Rails.root.join(ee_db_dir, 'docs')) - end + Gitlab::Database.all_database_connections.values.map(&:db_docs_dir).each do |db_dir| + FileUtils.mkdir_p(db_dir) end Rails.application.eager_load! @@ -502,7 +496,6 @@ namespace :gitlab do milestone = version.release.segments.first(2).join('.') classes = {} - ignored_tables = %w[p_ci_builds] Gitlab::Database.database_base_models.each do |_, model_class| tables = model_class.connection.tables @@ -521,11 +514,10 @@ namespace :gitlab do .reject { |c| c.name =~ /^(?:EE::)?Gitlab::(?:BackgroundMigration|DatabaseImporters)::/ } .reject { |c| c.name =~ /^HABTM_/ } .reject { |c| c < Gitlab::Database::Migration[1.0]::MigrationRecord } - .each { |c| classes[c.table_name] << c.name if classes.has_key?(c.table_name) } + .each { |c| classes[c.table_name] << c.name if classes.has_key?(c.table_name) && c.name.present? } sources.each do |source_name| next if source_name.start_with?('_test_') # Ignore test tables - next if ignored_tables.include?(source_name) database = model_class.connection_db_config.name file = dictionary_file_path(source_name, views, database) @@ -574,12 +566,7 @@ namespace :gitlab do def dictionary_file_path(source_name, views, database) sub_directory = views.include?(source_name) ? 'views' : '' - path = if Gitlab.ee? && Gitlab::Database::EE_DATABASES_NAME_TO_DIR.key?(database.to_s) - Rails.root.join(Gitlab::Database::EE_DATABASES_NAME_TO_DIR[database.to_s], 'docs') - else - DB_DOCS_PATH - end - + path = Gitlab::Database.all_database_connections.fetch(database).db_docs_dir File.join(path, sub_directory, "#{source_name}.yml") end diff --git a/lib/tasks/gitlab/packages/events.rake b/lib/tasks/gitlab/packages/events.rake index 1234ba039a3..b5dfd163dba 100644 --- a/lib/tasks/gitlab/packages/events.rake +++ b/lib/tasks/gitlab/packages/events.rake @@ -45,10 +45,7 @@ namespace :gitlab do events = event_pairs.each_with_object([]) do |(event_type, event_scope), events| Packages::Event::ORIGINATOR_TYPES.excluding(:guest).each do |originator_type| events_definition = Packages::Event.unique_counters_for(event_scope, event_type, originator_type).map do |event_name| - { - "name" => event_name, - "aggregation" => "weekly" - } + { "name" => event_name } end events.concat(events_definition) diff --git a/lib/tasks/gitlab/tw/codeowners.rake b/lib/tasks/gitlab/tw/codeowners.rake index b4b34581f43..afe2c564247 100644 --- a/lib/tasks/gitlab/tw/codeowners.rake +++ b/lib/tasks/gitlab/tw/codeowners.rake @@ -48,18 +48,18 @@ namespace :tw do CodeOwnerRule.new('Fuzz Testing', '@rdickenson'), CodeOwnerRule.new('Geo', '@axil'), CodeOwnerRule.new('Gitaly', '@eread'), - CodeOwnerRule.new('GitLab Dedicated', '@drcatherinepope'), + # CodeOwnerRule.new('GitLab Dedicated', ''), CodeOwnerRule.new('Global Search', '@ashrafkhamis'), CodeOwnerRule.new('Import and Integrate', '@eread @ashrafkhamis'), CodeOwnerRule.new('Infrastructure', '@sselhorn'), # CodeOwnerRule.new('Knowledge', ''), # CodeOwnerRule.new('MLOps', '') - CodeOwnerRule.new('Observability', '@drcatherinepope'), + # CodeOwnerRule.new('Observability', ''), CodeOwnerRule.new('Optimize', '@lciutacu'), CodeOwnerRule.new('Organization', '@lciutacu'), - CodeOwnerRule.new('Package Registry', '@marcel.amirault'), + CodeOwnerRule.new('Package Registry', '@phillipwells'), CodeOwnerRule.new('Pipeline Authoring', '@marcel.amirault'), - CodeOwnerRule.new('Pipeline Execution', '@drcatherinepope'), + CodeOwnerRule.new('Pipeline Execution', '@marcel.amirault'), CodeOwnerRule.new('Pipeline Security', '@marcel.amirault'), CodeOwnerRule.new('Product Analytics', '@lciutacu'), CodeOwnerRule.new('Product Planning', '@msedlakjakubowski'), @@ -71,7 +71,7 @@ namespace :tw do CodeOwnerRule.new('Runner', '@fneill'), CodeOwnerRule.new('Runner SaaS', '@fneill'), CodeOwnerRule.new('Security Policies', '@rdickenson'), - CodeOwnerRule.new('Source Code', '@aqualls'), + CodeOwnerRule.new('Source Code', '@aqualls @msedlakjakubowski'), CodeOwnerRule.new('Static Analysis', '@rdickenson'), CodeOwnerRule.new('Style Guide', '@sselhorn'), CodeOwnerRule.new('Tenant Scale', '@lciutacu'), diff --git a/lib/tasks/gitlab/usage_data.rake b/lib/tasks/gitlab/usage_data.rake index fcbec4b0dba..f5bf1a266e5 100644 --- a/lib/tasks/gitlab/usage_data.rake +++ b/lib/tasks/gitlab/usage_data.rake @@ -85,12 +85,13 @@ namespace :gitlab do end end + # rubocop:disable Gitlab/NoCodeCoverageComment + # :nocov: remove in https://gitlab.com/gitlab-org/gitlab/-/issues/299453 def ci_template_event(event_name) - { - 'name' => event_name, - 'aggregation' => 'weekly' - } + { 'name' => event_name } end + # :nocov: + # rubocop:enable Gitlab/NoCodeCoverageComment def implicit_auto_devops_event(expanded_template_name) event_name = Gitlab::UsageDataCounters::CiTemplateUniqueCounter.ci_template_event_name(expanded_template_name, :auto_devops_source) diff --git a/lib/tasks/tanuki_emoji.rake b/lib/tasks/tanuki_emoji.rake index b02d7a532c4..de2ae656952 100644 --- a/lib/tasks/tanuki_emoji.rake +++ b/lib/tasks/tanuki_emoji.rake @@ -157,9 +157,9 @@ namespace :tanuki_emoji do # SpriteFactory's SCSS is a bit too verbose for our purposes here, so # let's simplify it - system(%Q(sed -i '' "s/width: #{SIZE}px; height: #{SIZE}px; background: image-url('emoji.png')/background-position:/" #{style_path})) - system(%Q(sed -i '' "s/ no-repeat//" #{style_path})) - system(%Q(sed -i '' "s/ 0px/ 0/g" #{style_path})) + system(%(sed -i '' "s/width: #{SIZE}px; height: #{SIZE}px; background: image-url('emoji.png')/background-position:/" #{style_path})) + system(%(sed -i '' "s/ no-repeat//" #{style_path})) + system(%(sed -i '' "s/ 0px/ 0/g" #{style_path})) # Append a generic rule that applies to all Emojis File.open(style_path, 'a') do |f| diff --git a/lib/tasks/tokens.rake b/lib/tasks/tokens.rake index 81e24f4f7b6..c974aebf503 100644 --- a/lib/tasks/tokens.rake +++ b/lib/tasks/tokens.rake @@ -31,6 +31,11 @@ class TmpUser < ActiveRecord::Base # rubocop:disable Rails/ApplicationRecord self.table_name = 'users' - add_authentication_token_field :incoming_email_token, token_generator: -> { SecureRandom.hex.to_i(16).to_s(36) } - add_authentication_token_field :feed_token + add_authentication_token_field :incoming_email_token, + token_generator: -> { User.generate_incoming_mail_token } + add_authentication_token_field :feed_token, format_with_prefix: :prefix_for_feed_token + + def prefix_for_feed_token + User::FEED_TOKEN_PREFIX + end end -- cgit v1.2.3