diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-08-04 15:11:22 +0300 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-08-04 15:11:22 +0300 |
| commit | 39e07b68051f93f95d11b8607dfaa8ae87458bb7 (patch) | |
| tree | da662669a0061282555dfe646367c11614eeb1ec | |
| parent | 13f31ab91aeb5233a7b64f2740fee246294161fc (diff) | |
Add latest changes from gitlab-org/gitlab@master
92 files changed, 2365 insertions, 1700 deletions
diff --git a/.rubocop.yml b/.rubocop.yml index e2cbda196c3..baac21de331 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -598,15 +598,15 @@ RSpec/HaveGitlabHttpStatus: RSpec/ContextWording: Prefixes: - - when - - with - - without - - for - - and - - on - - in - - as - - if + - 'when' + - 'with' + - 'without' + - 'for' + - 'and' + - 'on' + - 'in' + - 'as' + - 'if' Style/MultilineWhenThen: Enabled: false diff --git a/.rubocop_todo/rspec/context_wording.yml b/.rubocop_todo/rspec/context_wording.yml index 5a889b5866d..f29e7bb49ae 100644 --- a/.rubocop_todo/rspec/context_wording.yml +++ b/.rubocop_todo/rspec/context_wording.yml @@ -21,7 +21,6 @@ RSpec/ContextWording: - 'ee/spec/controllers/ee/root_controller_spec.rb' - 'ee/spec/controllers/ee/search_controller_spec.rb' - 'ee/spec/controllers/ee/sent_notifications_controller_spec.rb' - - 'ee/spec/controllers/ee/sessions_controller_spec.rb' - 'ee/spec/controllers/groups/analytics/cycle_analytics_controller_spec.rb' - 'ee/spec/controllers/groups/audit_events_controller_spec.rb' - 'ee/spec/controllers/groups/billings_controller_spec.rb' @@ -44,8 +43,6 @@ RSpec/ContextWording: - 'ee/spec/controllers/oauth/geo_auth_controller_spec.rb' - 'ee/spec/controllers/operations_controller_spec.rb' - 'ee/spec/controllers/profiles_controller_spec.rb' - - 'ee/spec/controllers/projects/approver_groups_controller_spec.rb' - - 'ee/spec/controllers/projects/approvers_controller_spec.rb' - 'ee/spec/controllers/projects/audit_events_controller_spec.rb' - 'ee/spec/controllers/projects/boards_controller_spec.rb' - 'ee/spec/controllers/projects/environments_controller_spec.rb' @@ -67,7 +64,6 @@ RSpec/ContextWording: - 'ee/spec/controllers/projects/settings/repository_controller_spec.rb' - 'ee/spec/controllers/projects/vulnerability_feedback_controller_spec.rb' - 'ee/spec/controllers/projects_controller_spec.rb' - - 'ee/spec/controllers/registrations/company_controller_spec.rb' - 'ee/spec/controllers/registrations/groups_projects_controller_spec.rb' - 'ee/spec/controllers/registrations/welcome_controller_spec.rb' - 'ee/spec/controllers/repositories/git_http_controller_spec.rb' @@ -85,11 +81,9 @@ RSpec/ContextWording: - 'ee/spec/elastic_integration/global_search_spec.rb' - 'ee/spec/features/admin/admin_audit_logs_spec.rb' - 'ee/spec/features/admin/admin_credentials_inventory_spec.rb' - - 'ee/spec/features/admin/admin_dashboard_spec.rb' - 'ee/spec/features/admin/admin_dev_ops_reports_spec.rb' - 'ee/spec/features/admin/admin_settings_spec.rb' - 'ee/spec/features/admin/geo/admin_geo_nodes_spec.rb' - - 'ee/spec/features/admin/geo/admin_geo_sidebar_spec.rb' - 'ee/spec/features/admin/licenses/admin_adds_license_spec.rb' - 'ee/spec/features/analytics/code_analytics_spec.rb' - 'ee/spec/features/billings/billing_plans_spec.rb' @@ -113,16 +107,12 @@ RSpec/ContextWording: - 'ee/spec/features/epics/gfm_autocomplete_spec.rb' - 'ee/spec/features/epics/referencing_epics_spec.rb' - 'ee/spec/features/epics/update_epic_spec.rb' - - 'ee/spec/features/epics/user_uses_quick_actions_spec.rb' - 'ee/spec/features/geo_node_spec.rb' - 'ee/spec/features/gitlab_subscriptions/seat_count_alert_spec.rb' - - 'ee/spec/features/google_analytics_datalayer_spec.rb' - - 'ee/spec/features/groups/active_tabs_spec.rb' - 'ee/spec/features/groups/analytics/cycle_analytics/charts_spec.rb' - 'ee/spec/features/groups/analytics/cycle_analytics/filters_and_data_spec.rb' - 'ee/spec/features/groups/analytics/cycle_analytics/multiple_value_streams_spec.rb' - 'ee/spec/features/groups/audit_events_spec.rb' - - 'ee/spec/features/groups/group_overview_spec.rb' - 'ee/spec/features/groups/group_roadmap_spec.rb' - 'ee/spec/features/groups/group_settings_spec.rb' - 'ee/spec/features/groups/groups_security_credentials_spec.rb' @@ -142,7 +132,6 @@ RSpec/ContextWording: - 'ee/spec/features/groups_spec.rb' - 'ee/spec/features/ide/user_commits_changes_spec.rb' - 'ee/spec/features/ide/user_opens_ide_spec.rb' - - 'ee/spec/features/integrations/jira/jira_issues_list_spec.rb' - 'ee/spec/features/issues/epic_in_issue_sidebar_spec.rb' - 'ee/spec/features/issues/filtered_search/filter_issues_by_iteration_spec.rb' - 'ee/spec/features/issues/form_spec.rb' @@ -255,7 +244,6 @@ RSpec/ContextWording: - 'ee/spec/graphql/mutations/boards/epic_boards/destroy_spec.rb' - 'ee/spec/graphql/mutations/boards/epic_boards/epic_move_list_spec.rb' - 'ee/spec/graphql/mutations/boards/epic_boards/update_spec.rb' - - 'ee/spec/graphql/mutations/boards/epic_lists/update_spec.rb' - 'ee/spec/graphql/mutations/boards/epics/create_spec.rb' - 'ee/spec/graphql/mutations/compliance_management/frameworks/create_spec.rb' - 'ee/spec/graphql/mutations/compliance_management/frameworks/destroy_spec.rb' @@ -289,7 +277,6 @@ RSpec/ContextWording: - 'ee/spec/graphql/resolvers/path_locks_resolver_spec.rb' - 'ee/spec/graphql/resolvers/requirements_management/requirements_resolver_spec.rb' - 'ee/spec/graphql/resolvers/security_report_summary_resolver_spec.rb' - - 'ee/spec/graphql/types/audit_events/exterrnal_audit_event_destination_type_spec.rb' - 'ee/spec/graphql/types/global_id_type_spec.rb' - 'ee/spec/graphql/types/incident_management/escalation_rule_input_type_spec.rb' - 'ee/spec/graphql/types/issue_type_spec.rb' @@ -383,7 +370,6 @@ RSpec/ContextWording: - 'ee/spec/lib/gitlab/analytics/cycle_analytics/data_collector_spec.rb' - 'ee/spec/lib/gitlab/analytics/cycle_analytics/request_params_spec.rb' - 'ee/spec/lib/gitlab/auth/group_saml/gma_membership_enforcer_spec.rb' - - 'ee/spec/lib/gitlab/auth/group_saml/group_lookup_spec.rb' - 'ee/spec/lib/gitlab/auth/group_saml/identity_linker_spec.rb' - 'ee/spec/lib/gitlab/auth/group_saml/sso_enforcer_spec.rb' - 'ee/spec/lib/gitlab/auth/group_saml/token_actor_spec.rb' @@ -395,17 +381,12 @@ RSpec/ContextWording: - 'ee/spec/lib/gitlab/auth/smartcard/certificate_spec.rb' - 'ee/spec/lib/gitlab/auth/smartcard/ldap_certificate_spec.rb' - 'ee/spec/lib/gitlab/auth/smartcard/san_extension_spec.rb' - - 'ee/spec/lib/gitlab/background_migration/remove_all_trace_expiration_dates_spec.rb' - 'ee/spec/lib/gitlab/checks/diff_check_spec.rb' - 'ee/spec/lib/gitlab/ci/minutes/cost_factor_spec.rb' - 'ee/spec/lib/gitlab/ci/minutes/runners_availability_spec.rb' - 'ee/spec/lib/gitlab/ci/pipeline/chain/create_cross_database_associations_spec.rb' - 'ee/spec/lib/gitlab/ci/reports/dependency_list/dependency_spec.rb' - - 'ee/spec/lib/gitlab/ci/templates/Jobs/browser_performance_testing_gitlab_ci_yaml_spec.rb' - 'ee/spec/lib/gitlab/ci/templates/Jobs/dast_default_branch_gitlab_ci_yaml_spec.rb' - - 'ee/spec/lib/gitlab/ci/templates/Jobs/load_performance_testing_gitlab_ci_yaml_spec.rb' - - 'ee/spec/lib/gitlab/ci/templates/Verify/browser_performance_testing_gitlab_ci_yaml_spec.rb' - - 'ee/spec/lib/gitlab/ci/templates/Verify/load_performance_testing_gitlab_ci_yaml_spec.rb' - 'ee/spec/lib/gitlab/ci/templates/api_security_gitlab_ci_yaml_spec.rb' - 'ee/spec/lib/gitlab/ci/templates/api_security_latest_gitlab_ci_yaml_spec.rb' - 'ee/spec/lib/gitlab/ci/templates/container_scanning_gitlab_ci_yaml_spec.rb' @@ -450,7 +431,6 @@ RSpec/ContextWording: - 'ee/spec/lib/gitlab/geo/replication/blob_downloader_spec.rb' - 'ee/spec/lib/gitlab/geo/replicator_spec.rb' - 'ee/spec/lib/gitlab/geo/signed_data_spec.rb' - - 'ee/spec/lib/gitlab/geo_spec.rb' - 'ee/spec/lib/gitlab/git_access_spec.rb' - 'ee/spec/lib/gitlab/git_access_wiki_spec.rb' - 'ee/spec/lib/gitlab/gl_repository/identifier_spec.rb' @@ -468,7 +448,6 @@ RSpec/ContextWording: - 'ee/spec/lib/gitlab/slash_commands/presenters/issue_show_spec.rb' - 'ee/spec/lib/gitlab/status_page/filter/image_filter_spec.rb' - 'ee/spec/lib/gitlab/status_page/storage/s3_client_spec.rb' - - 'ee/spec/lib/gitlab/status_page/storage/s3_multipart_upload_spec.rb' - 'ee/spec/lib/gitlab/tracking/snowplow_schema_validation_spec.rb' - 'ee/spec/lib/gitlab/tracking/standard_context_spec.rb' - 'ee/spec/lib/gitlab/usage/metrics/instrumentations/advanced_search/build_type_metric_spec.rb' @@ -484,7 +463,6 @@ RSpec/ContextWording: - 'ee/spec/mailers/ee/emails/issues_spec.rb' - 'ee/spec/mailers/notify_spec.rb' - 'ee/spec/migrations/schedule_requirements_migration_spec.rb' - - 'ee/spec/migrations/schedule_trace_expiry_removal_spec.rb' - 'ee/spec/models/alert_management/alert_payload_field_spec.rb' - 'ee/spec/models/allowed_email_domain_spec.rb' - 'ee/spec/models/analytics/cycle_analytics/group_stage_spec.rb' @@ -496,11 +474,9 @@ RSpec/ContextWording: - 'ee/spec/models/audit_events/external_audit_event_destination_spec.rb' - 'ee/spec/models/board_spec.rb' - 'ee/spec/models/boards/epic_board_position_spec.rb' - - 'ee/spec/models/broadcast_message_spec.rb' - 'ee/spec/models/ci/build_spec.rb' - 'ee/spec/models/ci/minutes/namespace_monthly_usage_spec.rb' - 'ee/spec/models/ci/minutes/project_monthly_usage_spec.rb' - - 'ee/spec/models/ci/pipeline_spec.rb' - 'ee/spec/models/ci/sources/project_spec.rb' - 'ee/spec/models/ci/subscriptions/project_spec.rb' - 'ee/spec/models/concerns/approval_rule_like_spec.rb' @@ -553,7 +529,6 @@ RSpec/ContextWording: - 'ee/spec/models/gitlab_subscriptions/features_spec.rb' - 'ee/spec/models/group_member_spec.rb' - 'ee/spec/models/group_wiki_repository_spec.rb' - - 'ee/spec/models/group_wiki_spec.rb' - 'ee/spec/models/incident_management/escalation_rule_spec.rb' - 'ee/spec/models/incident_management/oncall_rotation_spec.rb' - 'ee/spec/models/incident_management/pending_escalations/alert_spec.rb' @@ -582,16 +557,13 @@ RSpec/ContextWording: - 'ee/spec/models/protected_environments/approval_summary_spec.rb' - 'ee/spec/models/push_rule_spec.rb' - 'ee/spec/models/release_highlight_spec.rb' - - 'ee/spec/models/repository_spec.rb' - 'ee/spec/models/requirements_management/test_report_spec.rb' - 'ee/spec/models/saml_group_link_spec.rb' - 'ee/spec/models/saml_provider_spec.rb' - 'ee/spec/models/status_page/project_setting_spec.rb' - - 'ee/spec/models/uploads/local_spec.rb' - 'ee/spec/models/vulnerabilities/feedback_spec.rb' - 'ee/spec/models/vulnerabilities/finding_pipeline_spec.rb' - 'ee/spec/models/vulnerabilities/finding_spec.rb' - - 'ee/spec/models/vulnerabilities/read_spec.rb' - 'ee/spec/models/vulnerabilities/statistic_spec.rb' - 'ee/spec/policies/app_sec/fuzzing/coverage/corpus_policy_spec.rb' - 'ee/spec/policies/compliance_management/framework_policy_spec.rb' @@ -609,7 +581,6 @@ RSpec/ContextWording: - 'ee/spec/policies/project_snippet_policy_spec.rb' - 'ee/spec/policies/protected_branch_policy_spec.rb' - 'ee/spec/policies/saml_provider_policy_spec.rb' - - 'ee/spec/policies/user_policy_spec.rb' - 'ee/spec/presenters/approval_rule_presenter_spec.rb' - 'ee/spec/presenters/audit_event_presenter_spec.rb' - 'ee/spec/presenters/ci/build_runner_presenter_spec.rb' @@ -617,12 +588,10 @@ RSpec/ContextWording: - 'ee/spec/presenters/group_clusterable_presenter_spec.rb' - 'ee/spec/presenters/merge_request_approver_presenter_spec.rb' - 'ee/spec/presenters/subscription_presenter_spec.rb' - - 'ee/spec/replicators/geo/pipeline_replicator_spec.rb' - 'ee/spec/requests/admin/credentials_controller_spec.rb' - 'ee/spec/requests/admin/geo/replicables_controller_spec.rb' - 'ee/spec/requests/api/analytics/group_activity_analytics_spec.rb' - 'ee/spec/requests/api/audit_events_spec.rb' - - 'ee/spec/requests/api/award_emoji_spec.rb' - 'ee/spec/requests/api/boards_spec.rb' - 'ee/spec/requests/api/ci/jobs_spec.rb' - 'ee/spec/requests/api/ci/pipelines_spec.rb' @@ -734,14 +703,12 @@ RSpec/ContextWording: - 'ee/spec/serializers/ee/merge_request_poll_cached_widget_entity_spec.rb' - 'ee/spec/serializers/ee/user_serializer_spec.rb' - 'ee/spec/serializers/environment_entity_spec.rb' - - 'ee/spec/serializers/epic_note_entity_spec.rb' - 'ee/spec/serializers/issue_serializer_spec.rb' - 'ee/spec/serializers/member_entity_spec.rb' - 'ee/spec/serializers/member_user_entity_spec.rb' - 'ee/spec/serializers/merge_request_widget_entity_spec.rb' - 'ee/spec/serializers/project_mirror_entity_spec.rb' - 'ee/spec/serializers/vulnerabilities/finding_entity_spec.rb' - - 'ee/spec/services/alert_management/process_prometheus_alert_service_spec.rb' - 'ee/spec/services/analytics/cycle_analytics/stages/delete_service_spec.rb' - 'ee/spec/services/analytics/cycle_analytics/value_streams/update_service_spec.rb' - 'ee/spec/services/app_sec/dast/builds/associate_service_spec.rb' @@ -754,7 +721,6 @@ RSpec/ContextWording: - 'ee/spec/services/approval_rules/update_service_spec.rb' - 'ee/spec/services/audit_event_service_spec.rb' - 'ee/spec/services/audit_events/export_csv_service_spec.rb' - - 'ee/spec/services/audit_events/register_runner_audit_event_service_spec.rb' - 'ee/spec/services/boards/create_service_spec.rb' - 'ee/spec/services/boards/epic_boards/create_service_spec.rb' - 'ee/spec/services/boards/epics/move_service_spec.rb' @@ -765,8 +731,6 @@ RSpec/ContextWording: - 'ee/spec/services/ci/pipeline_trigger_service_spec.rb' - 'ee/spec/services/ci/register_job_service_spec.rb' - 'ee/spec/services/ci/retry_job_service_spec.rb' - - 'ee/spec/services/ci/runners/stale_group_runners_prune_service_spec.rb' - - 'ee/spec/services/ci/runners/unregister_runner_service_spec.rb' - 'ee/spec/services/ci/sync_reports_to_approval_rules_service_spec.rb' - 'ee/spec/services/ci_cd/setup_project_spec.rb' - 'ee/spec/services/compliance_management/frameworks/create_service_spec.rb' @@ -786,7 +750,6 @@ RSpec/ContextWording: - 'ee/spec/services/ee/integrations/test/project_service_spec.rb' - 'ee/spec/services/ee/ip_restrictions/update_service_spec.rb' - 'ee/spec/services/ee/issuable/bulk_update_service_spec.rb' - - 'ee/spec/services/ee/issuable/common_system_notes_service_spec.rb' - 'ee/spec/services/ee/issues/clone_service_spec.rb' - 'ee/spec/services/ee/issues/close_service_spec.rb' - 'ee/spec/services/ee/issues/create_service_spec.rb' @@ -828,7 +791,6 @@ RSpec/ContextWording: - 'ee/spec/services/epic_issues/update_service_spec.rb' - 'ee/spec/services/epics/create_service_spec.rb' - 'ee/spec/services/epics/epic_links/list_service_spec.rb' - - 'ee/spec/services/epics/issue_promote_service_spec.rb' - 'ee/spec/services/epics/related_epic_links/create_service_spec.rb' - 'ee/spec/services/epics/related_epic_links/destroy_service_spec.rb' - 'ee/spec/services/epics/update_dates_service_spec.rb' @@ -891,7 +853,6 @@ RSpec/ContextWording: - 'ee/spec/services/merge_requests/reload_merge_head_diff_service_spec.rb' - 'ee/spec/services/merge_requests/update_blocks_service_spec.rb' - 'ee/spec/services/milestones/update_service_spec.rb' - - 'ee/spec/services/namespaces/in_product_marketing_emails_service_spec.rb' - 'ee/spec/services/personal_access_tokens/revoke_invalid_tokens_spec.rb' - 'ee/spec/services/personal_access_tokens/rotation_verifier_service_spec.rb' - 'ee/spec/services/projects/alerting/notify_service_spec.rb' @@ -902,7 +863,6 @@ RSpec/ContextWording: - 'ee/spec/services/projects/group_links/create_service_spec.rb' - 'ee/spec/services/projects/group_links/destroy_service_spec.rb' - 'ee/spec/services/projects/group_links/update_service_spec.rb' - - 'ee/spec/services/projects/hashed_storage/migrate_attachments_service_spec.rb' - 'ee/spec/services/projects/import_export/export_service_spec.rb' - 'ee/spec/services/projects/mark_for_deletion_service_spec.rb' - 'ee/spec/services/projects/operations/update_service_spec.rb' @@ -970,7 +930,6 @@ RSpec/ContextWording: - 'ee/spec/support/shared_examples/features/password_complexity_shared_examples.rb' - 'ee/spec/support/shared_examples/features/protected_branches_access_control_shared_examples.rb' - 'ee/spec/support/shared_examples/features/sidebar_shared_examples.rb' - - 'ee/spec/support/shared_examples/features/ultimate_trial_callout_shared_examples.rb' - 'ee/spec/support/shared_examples/graphql/mutations/epics/permission_check_shared_examples.rb' - 'ee/spec/support/shared_examples/lib/gitlab/git_access_shared_examples.rb' - 'ee/spec/support/shared_examples/lib/gitlab/middleware/allowlisted_admin_geo_requests_shared_examples.rb' @@ -990,8 +949,6 @@ RSpec/ContextWording: - 'ee/spec/support/shared_examples/models/protected_environments/authorizable_examples.rb' - 'ee/spec/support/shared_examples/quick_actions/issue/status_page_quick_actions_shared_examples.rb' - 'ee/spec/support/shared_examples/requests/credentials_inventory_shared_examples.rb' - - 'ee/spec/support/shared_examples/services/alert_management/alert_processing/alert_firing_shared_examples.rb' - - 'ee/spec/support/shared_examples/services/alert_management/alert_processing/alert_recovery_shared_examples.rb' - 'ee/spec/support/shared_examples/services/base_sync_service_shared_examples.rb' - 'ee/spec/support/shared_examples/services/boards/base_service_shared_examples.rb' - 'ee/spec/support/shared_examples/services/merge_merge_requests_shared_examples.rb' @@ -1040,7 +997,6 @@ RSpec/ContextWording: - 'ee/spec/workers/elastic_association_indexer_worker_spec.rb' - 'ee/spec/workers/elastic_index_bulk_cron_worker_spec.rb' - 'ee/spec/workers/elastic_indexing_control_worker_spec.rb' - - 'ee/spec/workers/geo/create_repository_updated_event_worker_spec.rb' - 'ee/spec/workers/geo/prune_event_log_worker_spec.rb' - 'ee/spec/workers/geo/repository_shard_sync_worker_spec.rb' - 'ee/spec/workers/geo/repository_sync_worker_spec.rb' @@ -1073,15 +1029,11 @@ RSpec/ContextWording: - 'qa/qa/specs/features/browser_ui/1_manage/project/invite_group_to_project_spec.rb' - 'qa/qa/specs/features/browser_ui/1_manage/user/user_access_termination_spec.rb' - 'qa/qa/specs/features/browser_ui/2_plan/milestone/assign_milestone_spec.rb' - - 'qa/qa/specs/features/browser_ui/3_create/design_management/add_design_content_spec.rb' - - 'qa/qa/specs/features/browser_ui/3_create/design_management/archive_design_content_spec.rb' - - 'qa/qa/specs/features/browser_ui/3_create/design_management/modify_design_content_spec.rb' - 'qa/qa/specs/features/browser_ui/3_create/merge_request/suggestions/batch_suggestion_spec.rb' - 'qa/qa/specs/features/browser_ui/3_create/merge_request/suggestions/custom_commit_suggestion_spec.rb' - 'qa/qa/specs/features/browser_ui/3_create/repository/file/create_file_via_web_spec.rb' - 'qa/qa/specs/features/browser_ui/3_create/repository/file/delete_file_via_web_spec.rb' - 'qa/qa/specs/features/browser_ui/3_create/repository/file/edit_file_via_web_spec.rb' - - 'qa/qa/specs/features/browser_ui/3_create/repository/license_detecton_spec.rb' - 'qa/qa/specs/features/browser_ui/3_create/repository/ssh_key_support_spec.rb' - 'qa/qa/specs/features/browser_ui/3_create/web_ide/server_hooks_custom_error_message_spec.rb' - 'qa/qa/specs/features/browser_ui/3_create/wiki/content_editor_spec.rb' @@ -1109,14 +1061,12 @@ RSpec/ContextWording: - 'qa/qa/specs/features/ee/browser_ui/12_geo/wiki_ssh_push_spec.rb' - 'qa/qa/specs/features/ee/browser_ui/13_secure/enable_scanning_from_configuration_spec.rb' - 'qa/qa/specs/features/ee/browser_ui/13_secure/license_compliance_spec.rb' - - 'qa/qa/specs/features/ee/browser_ui/13_secure/security_reports_spec.rb' - 'qa/qa/specs/features/ee/browser_ui/1_manage/group/group_audit_logs_1_spec.rb' - 'qa/qa/specs/features/ee/browser_ui/1_manage/group/group_audit_logs_2_spec.rb' - 'qa/qa/specs/features/ee/browser_ui/1_manage/group/group_ldap_sync_spec.rb' - 'qa/qa/specs/features/ee/browser_ui/1_manage/insights/default_insights_spec.rb' - 'qa/qa/specs/features/ee/browser_ui/1_manage/instance/instance_audit_logs_spec.rb' - 'qa/qa/specs/features/ee/browser_ui/1_manage/project/project_audit_logs_spec.rb' - - 'qa/qa/specs/features/ee/browser_ui/1_manage/project/project_templates_spec.rb' - 'qa/qa/specs/features/ee/browser_ui/2_plan/epic/epics_management_spec.rb' - 'qa/qa/specs/features/ee/browser_ui/2_plan/issue_boards/project_issue_boards_spec.rb' - 'qa/qa/specs/features/ee/browser_ui/3_create/repository/push_rules_spec.rb' @@ -1178,7 +1128,6 @@ RSpec/ContextWording: - 'spec/controllers/groups/settings/ci_cd_controller_spec.rb' - 'spec/controllers/groups/settings/integrations_controller_spec.rb' - 'spec/controllers/groups/settings/repository_controller_spec.rb' - - 'spec/controllers/groups/uploads_controller_spec.rb' - 'spec/controllers/groups_controller_spec.rb' - 'spec/controllers/help_controller_spec.rb' - 'spec/controllers/import/bitbucket_controller_spec.rb' @@ -1197,7 +1146,6 @@ RSpec/ContextWording: - 'spec/controllers/profiles/emails_controller_spec.rb' - 'spec/controllers/profiles/notifications_controller_spec.rb' - 'spec/controllers/profiles/personal_access_tokens_controller_spec.rb' - - 'spec/controllers/profiles/preferences_controller_spec.rb' - 'spec/controllers/projects/alerting/notifications_controller_spec.rb' - 'spec/controllers/projects/artifacts_controller_spec.rb' - 'spec/controllers/projects/badges_controller_spec.rb' @@ -1252,7 +1200,6 @@ RSpec/ContextWording: - 'spec/controllers/projects/tags_controller_spec.rb' - 'spec/controllers/projects/todos_controller_spec.rb' - 'spec/controllers/projects/tree_controller_spec.rb' - - 'spec/controllers/projects/uploads_controller_spec.rb' - 'spec/controllers/projects/web_ide_terminals_controller_spec.rb' - 'spec/controllers/projects_controller_spec.rb' - 'spec/controllers/registrations/welcome_controller_spec.rb' @@ -1276,7 +1223,6 @@ RSpec/ContextWording: - 'spec/features/admin/admin_hooks_spec.rb' - 'spec/features/admin/admin_jobs_spec.rb' - 'spec/features/admin/admin_mode/login_spec.rb' - - 'spec/features/admin/admin_mode/logout_spec.rb' - 'spec/features/admin/admin_mode_spec.rb' - 'spec/features/admin/admin_settings_spec.rb' - 'spec/features/admin/dashboard_spec.rb' @@ -1299,7 +1245,6 @@ RSpec/ContextWording: - 'spec/features/clusters/cluster_health_dashboard_spec.rb' - 'spec/features/commits_spec.rb' - 'spec/features/dashboard/activity_spec.rb' - - 'spec/features/dashboard/datetime_on_tooltips_spec.rb' - 'spec/features/dashboard/groups_list_spec.rb' - 'spec/features/dashboard/issues_filter_spec.rb' - 'spec/features/dashboard/label_filter_spec.rb' @@ -1365,8 +1310,6 @@ RSpec/ContextWording: - 'spec/features/issues/user_creates_issue_spec.rb' - 'spec/features/issues/user_edits_issue_spec.rb' - 'spec/features/issues/user_interacts_with_awards_spec.rb' - - 'spec/features/issues/user_scrolls_to_deeplinked_note_spec.rb' - - 'spec/features/issues/user_sorts_issue_comments_spec.rb' - 'spec/features/issues/user_toggles_subscription_spec.rb' - 'spec/features/issues/user_uses_quick_actions_spec.rb' - 'spec/features/issues/user_views_issues_spec.rb' @@ -1491,10 +1434,8 @@ RSpec/ContextWording: - 'spec/features/projects/show/user_sees_setup_shortcut_buttons_spec.rb' - 'spec/features/projects/snippets/user_views_snippets_spec.rb' - 'spec/features/projects/sourcegraph_csp_spec.rb' - - 'spec/features/projects/tags/user_edits_tags_spec.rb' - 'spec/features/projects/tags/user_views_tags_spec.rb' - 'spec/features/projects/tree/tree_show_spec.rb' - - 'spec/features/projects/user_sees_sidebar_spec.rb' - 'spec/features/projects/user_sorts_projects_spec.rb' - 'spec/features/projects/user_uses_shortcuts_spec.rb' - 'spec/features/projects_spec.rb' @@ -1512,7 +1453,6 @@ RSpec/ContextWording: - 'spec/features/user_can_display_performance_bar_spec.rb' - 'spec/features/user_opens_link_to_comment_spec.rb' - 'spec/features/users/login_spec.rb' - - 'spec/features/users/logout_spec.rb' - 'spec/features/users/overview_spec.rb' - 'spec/features/users/show_spec.rb' - 'spec/features/users/snippets_spec.rb' @@ -1601,7 +1541,6 @@ RSpec/ContextWording: - 'spec/graphql/mutations/alert_management/prometheus_integration/reset_token_spec.rb' - 'spec/graphql/mutations/alert_management/prometheus_integration/update_spec.rb' - 'spec/graphql/mutations/alert_management/update_alert_status_spec.rb' - - 'spec/graphql/mutations/boards/lists/update_spec.rb' - 'spec/graphql/mutations/clusters/agent_tokens/revoke_spec.rb' - 'spec/graphql/mutations/clusters/agents/create_spec.rb' - 'spec/graphql/mutations/custom_emoji/destroy_spec.rb' @@ -1652,7 +1591,6 @@ RSpec/ContextWording: - 'spec/graphql/resolvers/snippets/blobs_resolver_spec.rb' - 'spec/graphql/resolvers/snippets_resolver_spec.rb' - 'spec/graphql/resolvers/terraform/states_resolver_spec.rb' - - 'spec/graphql/resolvers/timelog_resolver_spec.rb' - 'spec/graphql/resolvers/user_resolver_spec.rb' - 'spec/graphql/resolvers/users/group_count_resolver_spec.rb' - 'spec/graphql/resolvers/users/participants_resolver_spec.rb' @@ -1711,7 +1649,6 @@ RSpec/ContextWording: - 'spec/helpers/todos_helper_spec.rb' - 'spec/helpers/tree_helper_spec.rb' - 'spec/helpers/users/callouts_helper_spec.rb' - - 'spec/helpers/users/group_callouts_helper_spec.rb' - 'spec/helpers/users_helper_spec.rb' - 'spec/helpers/visibility_level_helper_spec.rb' - 'spec/helpers/web_hooks/web_hooks_helper_spec.rb' @@ -1883,7 +1820,6 @@ RSpec/ContextWording: - 'spec/lib/gitlab/chat/responder_spec.rb' - 'spec/lib/gitlab/checks/branch_check_spec.rb' - 'spec/lib/gitlab/checks/lfs_integrity_spec.rb' - - 'spec/lib/gitlab/checks/matching_merge_request_spec.rb' - 'spec/lib/gitlab/checks/push_file_count_check_spec.rb' - 'spec/lib/gitlab/checks/snippet_check_spec.rb' - 'spec/lib/gitlab/checks/tag_check_spec.rb' @@ -1944,15 +1880,11 @@ RSpec/ContextWording: - 'spec/lib/gitlab/ci/status/composite_spec.rb' - 'spec/lib/gitlab/ci/status/factory_spec.rb' - 'spec/lib/gitlab/ci/templates/5_minute_production_app_ci_yaml_spec.rb' - - 'spec/lib/gitlab/ci/templates/Jobs/build_gitlab_ci_yaml_spec.rb' - 'spec/lib/gitlab/ci/templates/Jobs/code_quality_gitlab_ci_yaml_spec.rb' - - 'spec/lib/gitlab/ci/templates/Jobs/deploy_gitlab_ci_yaml_spec.rb' - 'spec/lib/gitlab/ci/templates/Jobs/sast_iac_gitlab_ci_yaml_spec.rb' - 'spec/lib/gitlab/ci/templates/Jobs/sast_iac_latest_gitlab_ci_yaml_spec.rb' - 'spec/lib/gitlab/ci/templates/Jobs/test_gitlab_ci_yaml_spec.rb' - - 'spec/lib/gitlab/ci/templates/Verify/load_performance_testing_gitlab_ci_yaml_spec.rb' - 'spec/lib/gitlab/ci/templates/auto_devops_gitlab_ci_yaml_spec.rb' - - 'spec/lib/gitlab/ci/templates/npm_spec.rb' - 'spec/lib/gitlab/ci/templates/templates_spec.rb' - 'spec/lib/gitlab/ci/templates/terraform_gitlab_ci_yaml_spec.rb' - 'spec/lib/gitlab/ci/templates/terraform_latest_gitlab_ci_yaml_spec.rb' @@ -1975,7 +1907,6 @@ RSpec/ContextWording: - 'spec/lib/gitlab/config/entry/composable_array_spec.rb' - 'spec/lib/gitlab/config/entry/composable_hash_spec.rb' - 'spec/lib/gitlab/config_checker/puma_rugged_checker_spec.rb' - - 'spec/lib/gitlab/console_spec.rb' - 'spec/lib/gitlab/consul/internal_spec.rb' - 'spec/lib/gitlab/content_security_policy/config_loader_spec.rb' - 'spec/lib/gitlab/cycle_analytics/permissions_spec.rb' @@ -1986,7 +1917,6 @@ RSpec/ContextWording: - 'spec/lib/gitlab/database/background_migration/batch_optimizer_spec.rb' - 'spec/lib/gitlab/database/background_migration/batched_migration_runner_spec.rb' - 'spec/lib/gitlab/database/background_migration/batched_migration_spec.rb' - - 'spec/lib/gitlab/database/background_migration/health_status_spec.rb' - 'spec/lib/gitlab/database/batch_count_spec.rb' - 'spec/lib/gitlab/database/bulk_update_spec.rb' - 'spec/lib/gitlab/database/count/reltuples_count_strategy_spec.rb' @@ -2022,7 +1952,6 @@ RSpec/ContextWording: - 'spec/lib/gitlab/database/query_analyzers/gitlab_schemas_validate_connection_spec.rb' - 'spec/lib/gitlab/database/query_analyzers/prevent_cross_database_modification_spec.rb' - 'spec/lib/gitlab/database/query_analyzers/restrict_allowed_schemas_spec.rb' - - 'spec/lib/gitlab/database/reflection_spec.rb' - 'spec/lib/gitlab/database/reindexing/coordinator_spec.rb' - 'spec/lib/gitlab/database/reindexing/grafana_notifier_spec.rb' - 'spec/lib/gitlab/database/reindexing/reindex_action_spec.rb' @@ -2116,7 +2045,6 @@ RSpec/ContextWording: - 'spec/lib/gitlab/graphql/loaders/issuable_loader_spec.rb' - 'spec/lib/gitlab/graphql/markdown_field_spec.rb' - 'spec/lib/gitlab/graphql/pagination/externally_paginated_array_connection_spec.rb' - - 'spec/lib/gitlab/graphql/pagination/keyset/connection_spec.rb' - 'spec/lib/gitlab/graphql/queries_spec.rb' - 'spec/lib/gitlab/graphql_logger_spec.rb' - 'spec/lib/gitlab/harbor/client_spec.rb' @@ -2179,7 +2107,6 @@ RSpec/ContextWording: - 'spec/lib/gitlab/kubernetes/default_namespace_spec.rb' - 'spec/lib/gitlab/kubernetes/helm/api_spec.rb' - 'spec/lib/gitlab/kubernetes/helm/v2/certificate_spec.rb' - - 'spec/lib/gitlab/kubernetes/helm/v2/init_command_spec.rb' - 'spec/lib/gitlab/kubernetes/kube_client_spec.rb' - 'spec/lib/gitlab/legacy_github_import/client_spec.rb' - 'spec/lib/gitlab/lfs/client_spec.rb' @@ -2187,13 +2114,11 @@ RSpec/ContextWording: - 'spec/lib/gitlab/lograge/custom_options_spec.rb' - 'spec/lib/gitlab/mail_room/authenticator_spec.rb' - 'spec/lib/gitlab/mail_room/mail_room_spec.rb' - - 'spec/lib/gitlab/mailgun/webhook_processors/failure_logger_spec.rb' - 'spec/lib/gitlab/manifest_import/manifest_spec.rb' - 'spec/lib/gitlab/markdown_cache/active_record/extension_spec.rb' - 'spec/lib/gitlab/memory/reports_daemon_spec.rb' - 'spec/lib/gitlab/memory/watchdog_spec.rb' - 'spec/lib/gitlab/merge_requests/commit_message_generator_spec.rb' - - 'spec/lib/gitlab/metrics/boot_time_tracker_spec.rb' - 'spec/lib/gitlab/metrics/dashboard/cache_spec.rb' - 'spec/lib/gitlab/metrics/dashboard/importer_spec.rb' - 'spec/lib/gitlab/metrics/dashboard/importers/prometheus_metrics_spec.rb' @@ -2379,8 +2304,6 @@ RSpec/ContextWording: - 'spec/mailers/emails/service_desk_spec.rb' - 'spec/mailers/notify_spec.rb' - 'spec/migrations/20210812013042_remove_duplicate_project_authorizations_spec.rb' - - 'spec/migrations/20220520040416_schedule_set_legacy_open_source_license_available_for_non_public_projects_spec.rb' - - 'spec/migrations/20220627090231_schedule_disable_legacy_open_source_license_for_inactive_public_projects_spec.rb' - 'spec/migrations/finalize_traversal_ids_background_migrations_spec.rb' - 'spec/migrations/rename_services_to_integrations_spec.rb' - 'spec/models/ability_spec.rb' @@ -2393,7 +2316,6 @@ RSpec/ContextWording: - 'spec/models/application_setting_spec.rb' - 'spec/models/approval_spec.rb' - 'spec/models/audit_event_spec.rb' - - 'spec/models/authentication_event_spec.rb' - 'spec/models/award_emoji_spec.rb' - 'spec/models/aws/role_spec.rb' - 'spec/models/badge_spec.rb' @@ -2501,7 +2423,6 @@ RSpec/ContextWording: - 'spec/models/identity_spec.rb' - 'spec/models/import_export_upload_spec.rb' - 'spec/models/import_failure_spec.rb' - - 'spec/models/incident_management/timeline_event_spec.rb' - 'spec/models/integration_spec.rb' - 'spec/models/integrations/asana_spec.rb' - 'spec/models/integrations/bamboo_spec.rb' @@ -2620,7 +2541,6 @@ RSpec/ContextWording: - 'spec/policies/issuable_policy_spec.rb' - 'spec/policies/issue_policy_spec.rb' - 'spec/policies/metrics/dashboard/annotation_policy_spec.rb' - - 'spec/policies/namespaces/project_namespace_policy_spec.rb' - 'spec/policies/namespaces/user_namespace_policy_spec.rb' - 'spec/policies/personal_access_token_policy_spec.rb' - 'spec/policies/personal_snippet_policy_spec.rb' @@ -2658,7 +2578,6 @@ RSpec/ContextWording: - 'spec/requests/api/appearance_spec.rb' - 'spec/requests/api/applications_spec.rb' - 'spec/requests/api/avatar_spec.rb' - - 'spec/requests/api/award_emoji_spec.rb' - 'spec/requests/api/badges_spec.rb' - 'spec/requests/api/branches_spec.rb' - 'spec/requests/api/bulk_imports_spec.rb' @@ -2828,7 +2747,6 @@ RSpec/ContextWording: - 'spec/requests/groups/clusters/integrations_controller_spec.rb' - 'spec/requests/groups/crm/contacts_controller_spec.rb' - 'spec/requests/groups/crm/organizations_controller_spec.rb' - - 'spec/requests/groups/email_campaigns_controller_spec.rb' - 'spec/requests/groups/milestones_controller_spec.rb' - 'spec/requests/groups/settings/access_tokens_controller_spec.rb' - 'spec/requests/groups_controller_spec.rb' @@ -2878,7 +2796,6 @@ RSpec/ContextWording: - 'spec/rubocop/cop/graphql/old_types_spec.rb' - 'spec/rubocop/cop/lint/last_keyword_argument_spec.rb' - 'spec/rubocop/cop/migration/add_index_spec.rb' - - 'spec/rubocop/cop/migration/add_limit_to_text_columns_spec.rb' - 'spec/rubocop/cop/migration/background_migration_record_spec.rb' - 'spec/rubocop/cop/migration/create_table_with_foreign_keys_spec.rb' - 'spec/rubocop/cop/migration/migration_record_spec.rb' @@ -3133,7 +3050,6 @@ RSpec/ContextWording: - 'spec/services/packages/composer/create_package_service_spec.rb' - 'spec/services/packages/conan/create_package_file_service_spec.rb' - 'spec/services/packages/conan/create_package_service_spec.rb' - - 'spec/services/packages/create_event_service_spec.rb' - 'spec/services/packages/create_package_file_service_spec.rb' - 'spec/services/packages/debian/create_distribution_service_spec.rb' - 'spec/services/packages/debian/create_package_file_service_spec.rb' @@ -3147,14 +3063,12 @@ RSpec/ContextWording: - 'spec/services/packages/maven/metadata/create_versions_xml_service_spec.rb' - 'spec/services/packages/maven/metadata/sync_service_spec.rb' - 'spec/services/packages/npm/create_package_service_spec.rb' - - 'spec/services/packages/npm/create_tag_service_spec.rb' - 'spec/services/packages/nuget/metadata_extraction_service_spec.rb' - 'spec/services/packages/nuget/search_service_spec.rb' - 'spec/services/packages/nuget/update_package_from_metadata_service_spec.rb' - 'spec/services/packages/rubygems/dependency_resolver_service_spec.rb' - 'spec/services/packages/rubygems/process_gem_service_spec.rb' - 'spec/services/packages/terraform_module/create_package_service_spec.rb' - - 'spec/services/packages/update_tags_service_spec.rb' - 'spec/services/pages/zip_directory_service_spec.rb' - 'spec/services/personal_access_tokens/create_service_spec.rb' - 'spec/services/personal_access_tokens/revoke_service_spec.rb' @@ -3198,7 +3112,6 @@ RSpec/ContextWording: - 'spec/services/quick_actions/interpret_service_spec.rb' - 'spec/services/releases/create_service_spec.rb' - 'spec/services/releases/update_service_spec.rb' - - 'spec/services/repositories/destroy_service_spec.rb' - 'spec/services/resource_access_tokens/create_service_spec.rb' - 'spec/services/search/global_service_spec.rb' - 'spec/services/search/group_service_spec.rb' @@ -3217,7 +3130,6 @@ RSpec/ContextWording: - 'spec/services/submodules/update_service_spec.rb' - 'spec/services/suggestions/apply_service_spec.rb' - 'spec/services/suggestions/create_service_spec.rb' - - 'spec/services/system_notes/commit_service_spec.rb' - 'spec/services/system_notes/design_management_service_spec.rb' - 'spec/services/system_notes/issuables_service_spec.rb' - 'spec/services/system_notes/merge_requests_service_spec.rb' @@ -3338,7 +3250,6 @@ RSpec/ContextWording: - 'spec/support/shared_examples/controllers/hotlink_interceptor_shared_examples.rb' - 'spec/support/shared_examples/controllers/issuables_list_metadata_shared_examples.rb' - 'spec/support/shared_examples/controllers/snippet_blob_shared_examples.rb' - - 'spec/support/shared_examples/controllers/uploads_actions_shared_examples.rb' - 'spec/support/shared_examples/controllers/wiki_actions_shared_examples.rb' - 'spec/support/shared_examples/csp.rb' - 'spec/support/shared_examples/features/access_tokens_shared_examples.rb' @@ -3384,7 +3295,6 @@ RSpec/ContextWording: - 'spec/support/shared_examples/lib/gitlab/sidekiq_middleware/strategy_shared_examples.rb' - 'spec/support/shared_examples/models/application_setting_shared_examples.rb' - 'spec/support/shared_examples/models/chat_integration_shared_examples.rb' - - 'spec/support/shared_examples/models/cluster_application_helm_cert_shared_examples.rb' - 'spec/support/shared_examples/models/cluster_application_status_shared_examples.rb' - 'spec/support/shared_examples/models/cluster_application_version_shared_examples.rb' - 'spec/support/shared_examples/models/clusters/prometheus_client_shared.rb' @@ -3392,13 +3302,11 @@ RSpec/ContextWording: - 'spec/support/shared_examples/models/concerns/composite_id_shared_examples.rb' - 'spec/support/shared_examples/models/concerns/incident_management/escalatable_shared_examples.rb' - 'spec/support/shared_examples/models/concerns/integrations/slack_mattermost_notifier_shared_examples.rb' - - 'spec/support/shared_examples/models/concerns/issuable_shared_examples.rb' - 'spec/support/shared_examples/models/concerns/repository_storage_movable_shared_examples.rb' - 'spec/support/shared_examples/models/concerns/timebox_shared_examples.rb' - 'spec/support/shared_examples/models/cycle_analytics_stage_shared_examples.rb' - 'spec/support/shared_examples/models/diff_positionable_note_shared_examples.rb' - 'spec/support/shared_examples/models/integrations/base_slash_commands_shared_examples.rb' - - 'spec/support/shared_examples/models/members_notifications_shared_example.rb' - 'spec/support/shared_examples/models/mentionable_shared_examples.rb' - 'spec/support/shared_examples/models/packages/debian/component_file_shared_example.rb' - 'spec/support/shared_examples/models/project_latest_successful_build_for_shared_examples.rb' @@ -3466,7 +3374,6 @@ RSpec/ContextWording: - 'spec/support/shared_examples/workers/batched_background_migration_worker_shared_examples.rb' - 'spec/support/shared_examples/workers/concerns/git_garbage_collect_methods_shared_examples.rb' - 'spec/support/shared_examples/workers/project_export_shared_examples.rb' - - 'spec/support_specs/database/multiple_databases_spec.rb' - 'spec/support_specs/helpers/migrations_helpers_spec.rb' - 'spec/support_specs/helpers/stub_feature_flags_spec.rb' - 'spec/support_specs/helpers/stub_method_calls_spec.rb' @@ -3529,7 +3436,6 @@ RSpec/ContextWording: - 'spec/views/layouts/_header_search.html.haml_spec.rb' - 'spec/views/layouts/application.html.haml_spec.rb' - 'spec/views/layouts/header/_new_dropdown.haml_spec.rb' - - 'spec/views/layouts/nav/sidebar/_admin.html.haml_spec.rb' - 'spec/views/layouts/nav/sidebar/_project.html.haml_spec.rb' - 'spec/views/notify/changed_milestone_email.html.haml_spec.rb' - 'spec/views/profiles/keys/_key.html.haml_spec.rb' diff --git a/app/assets/javascripts/design_management/components/design_notes/design_reply_form.vue b/app/assets/javascripts/design_management/components/design_notes/design_reply_form.vue index 818299e36bd..1b6458668f5 100644 --- a/app/assets/javascripts/design_management/components/design_notes/design_reply_form.vue +++ b/app/assets/javascripts/design_management/components/design_notes/design_reply_form.vue @@ -90,7 +90,6 @@ export default { <form class="new-note common-note-form" @submit.prevent> <markdown-field :markdown-preview-path="markdownPreviewPath" - :can-attach-file="false" :enable-autocomplete="true" :textarea-value="value" :markdown-docs-path="markdownDocsPath" diff --git a/app/assets/javascripts/editor/schema/ci.json b/app/assets/javascripts/editor/schema/ci.json index 74bd8ff3b15..846475bc1a2 100644 --- a/app/assets/javascripts/editor/schema/ci.json +++ b/app/assets/javascripts/editor/schema/ci.json @@ -606,11 +606,33 @@ "markdownDescription": "Expression to evaluate whether additional attributes should be provided to the job. [Learn More](https://docs.gitlab.com/ee/ci/yaml/#rulesif)." }, "changes": { - "type": "array", "markdownDescription": "Additional attributes will be provided to job if any of the provided paths matches a modified file. [Learn More](https://docs.gitlab.com/ee/ci/yaml/#ruleschanges).", - "items": { - "type": "string" - } + "anyOf": [ + { + "type": "object", + "additionalProperties": false, + "required": ["paths"], + "properties": { + "paths": { + "type": "array", + "description": "List of file paths.", + "items": { + "type": "string" + } + }, + "compare_to": { + "type": "string", + "description": "Ref for comparing changes." + } + } + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] }, "exists": { "type": "array", @@ -623,11 +645,11 @@ "markdownDescription": "Defines environment variables for specific jobs. Job level property overrides global variables. If a job sets `variables: {}`, all global variables are turned off. [Learn More](https://docs.gitlab.com/ee/ci/yaml/#rulesvariables).", "anyOf": [ { - "type": "object", + "type": "object", "additionalProperties": { "type": ["string", "integer", "array"] } - }, + }, { "type": "array", "items": { diff --git a/app/assets/javascripts/frequent_items/components/frequent_items_list_item.vue b/app/assets/javascripts/frequent_items/components/frequent_items_list_item.vue index 6b1676eca8a..9fb69a3cae3 100644 --- a/app/assets/javascripts/frequent_items/components/frequent_items_list_item.vue +++ b/app/assets/javascripts/frequent_items/components/frequent_items_list_item.vue @@ -75,6 +75,7 @@ export default { <project-avatar class="gl-float-left gl-mr-3" :project-avatar-url="avatarUrl" + :project-id="itemId" :project-name="itemName" aria-hidden="true" /> diff --git a/app/assets/javascripts/invite_members/components/user_limit_notification.vue b/app/assets/javascripts/invite_members/components/user_limit_notification.vue index ae5c3c11386..6c9b1f8e6d0 100644 --- a/app/assets/javascripts/invite_members/components/user_limit_notification.vue +++ b/app/assets/javascripts/invite_members/components/user_limit_notification.vue @@ -10,7 +10,6 @@ import { CLOSE_TO_LIMIT_MESSAGE, CLOSE_TO_LIMIT_MESSAGE_PERSONAL_NAMESPACE, DANGER_ALERT_TITLE_PERSONAL_NAMESPACE, - WARNING_ALERT_TITLE_PERSONAL_NAMESPACE, } from '../constants'; export default { @@ -46,13 +45,6 @@ export default { return this.usersLimitDataset.purchasePath; }, warningAlertTitle() { - if (this.usersLimitDataset.userNamespace) { - return sprintf(WARNING_ALERT_TITLE_PERSONAL_NAMESPACE, { - count: this.freeUsersLimit - this.membersCount, - members: this.pluralMembers(this.freeUsersLimit - this.membersCount), - }); - } - return sprintf(WARNING_ALERT_TITLE, { count: this.freeUsersLimit - this.membersCount, members: this.pluralMembers(this.freeUsersLimit - this.membersCount), diff --git a/app/assets/javascripts/invite_members/constants.js b/app/assets/javascripts/invite_members/constants.js index f5187a3ef0c..6a0be3fc75b 100644 --- a/app/assets/javascripts/invite_members/constants.js +++ b/app/assets/javascripts/invite_members/constants.js @@ -138,9 +138,6 @@ export const ON_SUBMIT_TRACK_LABEL = 'manage_members_clicked'; export const WARNING_ALERT_TITLE = s__( 'InviteMembersModal|You only have space for %{count} more %{members} in %{name}', ); -export const WARNING_ALERT_TITLE_PERSONAL_NAMESPACE = s__( - 'InviteMembersModal|You only have space for %{count} more %{members} in your personal projects', -); export const DANGER_ALERT_TITLE = s__( "InviteMembersModal|You've reached your %{count} %{members} limit for %{name}", ); @@ -155,12 +152,12 @@ export const REACHED_LIMIT_MESSAGE = s__( export const REACHED_LIMIT_UPGRADE_SUGGESTION_MESSAGE = REACHED_LIMIT_MESSAGE.concat( s__( - 'InviteMembersModal| To get more members and access to additional paid features, an owner of this namespace can start a trial or upgrade to a paid tier.', + 'InviteMembersModal| To get more members and access to additional paid features, an owner of the group can start a trial or upgrade to a paid tier.', ), ); export const CLOSE_TO_LIMIT_MESSAGE = s__( - 'InviteMembersModal|To get more members an owner of this namespace can %{trialLinkStart}start a trial%{trialLinkEnd} or %{upgradeLinkStart}upgrade%{upgradeLinkEnd} to a paid tier.', + 'InviteMembersModal|To get more members an owner of the group can %{trialLinkStart}start a trial%{trialLinkEnd} or %{upgradeLinkStart}upgrade%{upgradeLinkEnd} to a paid tier.', ); export const CLOSE_TO_LIMIT_MESSAGE_PERSONAL_NAMESPACE = s__( 'InviteMembersModal|To make more space, you can remove members who no longer need access.', diff --git a/app/assets/javascripts/invite_members/init_invite_members_modal.js b/app/assets/javascripts/invite_members/init_invite_members_modal.js index a4be3f205a3..6e2c0ecb5bb 100644 --- a/app/assets/javascripts/invite_members/init_invite_members_modal.js +++ b/app/assets/javascripts/invite_members/init_invite_members_modal.js @@ -20,6 +20,8 @@ export default (function initInviteMembersModal() { return false; } + const usersLimitDataset = JSON.parse(el.dataset.usersLimitDataset || '{}'); + inviteMembersModal = new Vue({ el, name: 'InviteMembersModalRoot', @@ -38,9 +40,10 @@ export default (function initInviteMembersModal() { projects: JSON.parse(el.dataset.projects || '[]'), usersFilter: el.dataset.usersFilter, filterId: parseInt(el.dataset.filterId, 10), - usersLimitDataset: convertObjectPropsToCamelCase( - JSON.parse(el.dataset.usersLimitDataset || '{}'), - ), + usersLimitDataset: convertObjectPropsToCamelCase({ + ...usersLimitDataset, + user_namespace: parseBoolean(usersLimitDataset.user_namespace), + }), }, }), }); diff --git a/app/assets/javascripts/vue_shared/components/markdown/header.vue b/app/assets/javascripts/vue_shared/components/markdown/header.vue index 480d412f220..0f2340b8006 100644 --- a/app/assets/javascripts/vue_shared/components/markdown/header.vue +++ b/app/assets/javascripts/vue_shared/components/markdown/header.vue @@ -203,6 +203,44 @@ export default { :class="{ 'gl-display-none!': previewMarkdown }" class="md-header-toolbar gl-ml-auto gl-pb-3 gl-justify-content-center" > + <template v-if="canSuggest"> + <toolbar-button + ref="suggestButton" + :tag="mdSuggestion" + :prepend="true" + :button-title="__('Insert suggestion')" + :cursor-offset="4" + :tag-content="lineContent" + icon="doc-code" + data-qa-selector="suggestion_button" + class="js-suggestion-btn" + @click="handleSuggestDismissed" + /> + <gl-popover + v-if="suggestPopoverVisible" + :target="$refs.suggestButton.$el" + :css-classes="['diff-suggest-popover']" + placement="bottom" + :show="suggestPopoverVisible" + > + <strong>{{ __('New! Suggest changes directly') }}</strong> + <p class="mb-2"> + {{ + __( + 'Suggest code changes which can be immediately applied in one click. Try it out!', + ) + }} + </p> + <gl-button + variant="confirm" + category="primary" + size="small" + @click="handleSuggestDismissed" + > + {{ __('Got it') }} + </gl-button> + </gl-popover> + </template> <toolbar-button tag="**" :button-title=" @@ -245,44 +283,6 @@ export default { icon="quote" @click="handleQuote" /> - <template v-if="canSuggest"> - <toolbar-button - ref="suggestButton" - :tag="mdSuggestion" - :prepend="true" - :button-title="__('Insert suggestion')" - :cursor-offset="4" - :tag-content="lineContent" - icon="doc-code" - data-qa-selector="suggestion_button" - class="js-suggestion-btn" - @click="handleSuggestDismissed" - /> - <gl-popover - v-if="suggestPopoverVisible" - :target="$refs.suggestButton.$el" - :css-classes="['diff-suggest-popover']" - placement="bottom" - :show="suggestPopoverVisible" - > - <strong>{{ __('New! Suggest changes directly') }}</strong> - <p class="mb-2"> - {{ - __( - 'Suggest code changes which can be immediately applied in one click. Try it out!', - ) - }} - </p> - <gl-button - variant="confirm" - category="primary" - size="small" - @click="handleSuggestDismissed" - > - {{ __('Got it') }} - </gl-button> - </gl-popover> - </template> <toolbar-button tag="`" tag-block="```" :button-title="__('Insert code')" icon="code" /> <toolbar-button tag="[{text}](url)" diff --git a/app/assets/javascripts/vue_shared/components/project_avatar.vue b/app/assets/javascripts/vue_shared/components/project_avatar.vue index 402e75962d2..6ac96d38b9f 100644 --- a/app/assets/javascripts/vue_shared/components/project_avatar.vue +++ b/app/assets/javascripts/vue_shared/components/project_avatar.vue @@ -7,6 +7,11 @@ export default { GlAvatar, }, props: { + projectId: { + type: Number, + default: 0, + required: false, + }, projectName: { type: String, required: true, @@ -39,6 +44,7 @@ export default { <template> <gl-avatar :shape="$options.AVATAR_SHAPE_OPTION_RECT" + :entity-id="projectId" :entity-name="projectName" :src="projectAvatarUrl" :alt="avatarAlt" diff --git a/app/assets/stylesheets/pages/notes.scss b/app/assets/stylesheets/pages/notes.scss index 4d0cf30a3b2..db07f16dfd0 100644 --- a/app/assets/stylesheets/pages/notes.scss +++ b/app/assets/stylesheets/pages/notes.scss @@ -4,7 +4,7 @@ $system-note-svg-size: 16px; @mixin vertical-line($left) { &::before { content: ''; - border-left: 2px solid $gray-50; + border-left: 2px solid $gray-10; position: absolute; top: 0; bottom: 0; @@ -29,7 +29,7 @@ $system-note-svg-size: 16px; .issuable-discussion { .main-notes-list { - @include vertical-line(36px); + @include vertical-line(35px); } } @@ -300,17 +300,17 @@ $system-note-svg-size: 16px; .timeline-icon { display: flex; align-items: center; - background-color: $white; + background-color: $gray-10; width: $system-note-icon-size; height: $system-note-icon-size; - border: 1px solid $border-color; + border: 1px solid $gray-10; border-radius: $system-note-icon-size; margin: -6px 20px 0 0; svg { width: $system-note-svg-size; height: $system-note-svg-size; - fill: $gray-darkest; + fill: $gray-400; display: block; margin: 0 auto; } diff --git a/app/graphql/mutations/ci/runner/bulk_delete.rb b/app/graphql/mutations/ci/runner/bulk_delete.rb new file mode 100644 index 00000000000..09aeee37317 --- /dev/null +++ b/app/graphql/mutations/ci/runner/bulk_delete.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Mutations + module Ci + module Runner + class BulkDelete < BaseMutation + graphql_name 'BulkRunnerDelete' + + RunnerID = ::Types::GlobalIDType[::Ci::Runner] + + argument :ids, [RunnerID], + required: false, + description: 'IDs of the runners to delete.' + + field :deleted_count, + ::GraphQL::Types::Int, + null: true, + description: 'Number of records effectively deleted. ' \ + 'Only present if operation was performed synchronously.' + + field :deleted_ids, + [RunnerID], + null: true, + description: 'IDs of records effectively deleted. ' \ + 'Only present if operation was performed synchronously.' + + def resolve(**runner_attrs) + raise_resource_not_available_error! unless Ability.allowed?(current_user, :delete_runners) + + if ids = runner_attrs[:ids] + runners = find_all_runners_by_ids(model_ids_of(ids)) + + result = ::Ci::Runners::BulkDeleteRunnersService.new(runners: runners).execute + result.slice(:deleted_count, :deleted_ids).merge(errors: []) + else + { errors: [] } + end + end + + private + + def model_ids_of(ids) + ids.map do |gid| + gid.model_id.to_i + end.compact + end + + def find_all_runners_by_ids(ids) + return ::Ci::Runner.none if ids.blank? + + ::Ci::Runner.id_in(ids) + end + end + end + end +end diff --git a/app/graphql/types/mutation_type.rb b/app/graphql/types/mutation_type.rb index dc9eb369dc8..ba595a11911 100644 --- a/app/graphql/types/mutation_type.rb +++ b/app/graphql/types/mutation_type.rb @@ -130,6 +130,7 @@ module Types mount_mutation Mutations::Ci::JobTokenScope::RemoveProject mount_mutation Mutations::Ci::Runner::Update mount_mutation Mutations::Ci::Runner::Delete + mount_mutation Mutations::Ci::Runner::BulkDelete, alpha: { milestone: '15.3' } mount_mutation Mutations::Ci::RunnersRegistrationToken::Reset mount_mutation Mutations::Namespace::PackageSettings::Update mount_mutation Mutations::Groups::Update diff --git a/app/helpers/groups/group_members_helper.rb b/app/helpers/groups/group_members_helper.rb index 2021961772a..6a013a6c864 100644 --- a/app/helpers/groups/group_members_helper.rb +++ b/app/helpers/groups/group_members_helper.rb @@ -21,10 +21,11 @@ module Groups::GroupMembersHelper end def group_member_header_subtext(group) - html_escape(_('You can invite a new member to ' \ - '%{strong_start}%{group_name}%{strong_end}.')) % { group_name: group.name, - strong_start: '<strong>'.html_safe, - strong_end: '</strong>'.html_safe } + html_escape(_("You're viewing members of %{strong_start}%{group_name}%{strong_end}.").html_safe) % { + group_name: group.name, + strong_start: '<strong>'.html_safe, + strong_end: '</strong>'.html_safe + } end private diff --git a/app/views/shared/issuable/_sidebar.html.haml b/app/views/shared/issuable/_sidebar.html.haml index 55f5dce8b37..321311e1d96 100644 --- a/app/views/shared/issuable/_sidebar.html.haml +++ b/app/views/shared/issuable/_sidebar.html.haml @@ -29,6 +29,9 @@ .block.assignee.qa-assignee-block{ class: "#{'gl-mt-3' if !signed_in && moved_sidebar_enabled}" } = render "shared/issuable/sidebar_assignees", issuable_sidebar: issuable_sidebar, assignees: assignees, signed_in: signed_in + - if issuable_sidebar[:supports_severity] + #js-severity + - if reviewers .block.reviewer.qa-reviewer-block = render "shared/issuable/sidebar_reviewers", issuable_sidebar: issuable_sidebar, reviewers: reviewers, signed_in: signed_in @@ -67,9 +70,6 @@ = _('Time tracking') = gl_loading_icon(inline: true) - - if issuable_sidebar[:supports_severity] - #js-severity - - if issuable_sidebar.dig(:features_available, :health_status) .js-sidebar-status-entry-point{ data: sidebar_status_data(issuable_sidebar, @project) } diff --git a/db/post_migrate/20220721031446_schedule_disable_legacy_open_source_license_for_one_member_no_repo_projects.rb b/db/post_migrate/20220721031446_schedule_disable_legacy_open_source_license_for_one_member_no_repo_projects.rb new file mode 100644 index 00000000000..7665d49b1d9 --- /dev/null +++ b/db/post_migrate/20220721031446_schedule_disable_legacy_open_source_license_for_one_member_no_repo_projects.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +class ScheduleDisableLegacyOpenSourceLicenseForOneMemberNoRepoProjects < Gitlab::Database::Migration[2.0] + MIGRATION = 'DisableLegacyOpenSourceLicenseForOneMemberNoRepoProjects' + INTERVAL = 2.minutes + BATCH_SIZE = 5_000 + MAX_BATCH_SIZE = 10_000 + SUB_BATCH_SIZE = 500 + + disable_ddl_transaction! + + restrict_gitlab_migration gitlab_schema: :gitlab_main + + def up + return unless Gitlab.com? + + queue_batched_background_migration( + MIGRATION, + :projects, + :id, + job_interval: INTERVAL, + batch_size: BATCH_SIZE, + max_batch_size: MAX_BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + return unless Gitlab.com? + + delete_batched_background_migration(MIGRATION, :projects, :id, []) + end +end diff --git a/db/schema_migrations/20220721031446 b/db/schema_migrations/20220721031446 new file mode 100644 index 00000000000..cb58abdd70c --- /dev/null +++ b/db/schema_migrations/20220721031446 @@ -0,0 +1 @@ +fb37a812240cd314227b112f1c5f379fece783fddcf922ceafbf2c968c72ab30
\ No newline at end of file diff --git a/doc/administration/geo/replication/troubleshooting.md b/doc/administration/geo/replication/troubleshooting.md index b610247a091..427381dd769 100644 --- a/doc/administration/geo/replication/troubleshooting.md +++ b/doc/administration/geo/replication/troubleshooting.md @@ -837,7 +837,7 @@ to transfer each affected repository from the primary to the secondary site. The following are possible error messages that might be encountered during failover or when promoting a secondary to a primary node with strategies to resolve them. -### Message: ActiveRecord::RecordInvalid: Validation failed: Name has already been taken +### Message: `ActiveRecord::RecordInvalid: Validation failed: Name has already been taken` When [promoting a **secondary** site](../disaster_recovery/index.md#step-3-promoting-a-secondary-site), you might encounter the following error message: @@ -873,7 +873,7 @@ or `gitlab-ctl promote-to-primary-node`, either: bug](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22021) was fixed. -### Message: ActiveRecord::RecordInvalid: Validation failed: Enabled Geo primary node cannot be disabled +### Message: `ActiveRecord::RecordInvalid: Validation failed: Enabled Geo primary node cannot be disabled` If you disabled a secondary node, either with the [replication pause task](../index.md#pausing-and-resuming-replication) (GitLab 13.2) or by using the user interface (GitLab 13.1 and earlier), you must first @@ -1146,7 +1146,7 @@ Geo::TrackingBase::SecondaryNotConfigured: Geo secondary database is not configu On a Geo primary site this error can be ignored. -This happens because GitLab is attempting to display registries from the [Geo tracking database](../../../administration/geo/#geo-tracking-database) which doesn't exist on the primary site (only the original projects exist on the primary; no replicated projects are present, therefore no tracking database exists). +This happens because GitLab is attempting to display registries from the [Geo tracking database](../../../administration/geo/index.md#geo-tracking-database) which doesn't exist on the primary site (only the original projects exist on the primary; no replicated projects are present, therefore no tracking database exists). ## Fixing client errors diff --git a/doc/administration/gitaly/troubleshooting.md b/doc/administration/gitaly/troubleshooting.md index fc69aadd236..9286967c227 100644 --- a/doc/administration/gitaly/troubleshooting.md +++ b/doc/administration/gitaly/troubleshooting.md @@ -444,15 +444,15 @@ If you receive an error, check `/var/log/gitlab/gitlab-rails/production.log`. Here are common errors and potential causes: - 500 response code - - **ActionView::Template::Error (7:permission denied)** + - `ActionView::Template::Error (7:permission denied)` - `praefect['auth_token']` and `gitlab_rails['gitaly_token']` do not match on the GitLab server. - - **Unable to save project. Error: 7:permission denied** + - `Unable to save project. Error: 7:permission denied` - Secret token in `praefect['storage_nodes']` on GitLab server does not match the value in `gitaly['auth_token']` on one or more Gitaly servers. - 503 response code - - **GRPC::Unavailable (14:failed to connect to all addresses)** + - `GRPC::Unavailable (14:failed to connect to all addresses)` - GitLab was unable to reach Praefect. - - **GRPC::Unavailable (14:all SubCons are in TransientFailure...)** + - `GRPC::Unavailable (14:all SubCons are in TransientFailure...)` - Praefect cannot reach one or more of its child Gitaly nodes. Try running the Praefect connection checker to diagnose. diff --git a/doc/administration/packages/container_registry.md b/doc/administration/packages/container_registry.md index c5ece72b764..cc3d3abe2cf 100644 --- a/doc/administration/packages/container_registry.md +++ b/doc/administration/packages/container_registry.md @@ -739,7 +739,7 @@ auth: Without these entries, the registry logins cannot authenticate with GitLab. GitLab also remains unaware of -[nested image names](../../user/packages/container_registry/#image-naming-convention) +[nested image names](../../user/packages/container_registry/index.md#image-naming-convention) under the project hierarchy, like `registry.example.com/group/project/image-name:tag` or `registry.example.com/group/project/my/image-name:tag`, and only recognizes diff --git a/doc/api/discussions.md b/doc/api/discussions.md index c30c00cef67..c168c3ae983 100644 --- a/doc/api/discussions.md +++ b/doc/api/discussions.md @@ -170,7 +170,7 @@ curl --request POST --header "PRIVATE-TOKEN: <your_access_token>"\ ### Add note to existing issue thread -Adds a new note to the thread. This can also [create a thread from a single comment](../user/discussions/#create-a-thread-by-replying-to-a-standard-comment). +Adds a new note to the thread. This can also [create a thread from a single comment](../user/discussions/index.md#create-a-thread-by-replying-to-a-standard-comment). **WARNING** Notes can be added to other items than comments, such as system notes, making them threads. @@ -599,7 +599,7 @@ curl --request POST --header "PRIVATE-TOKEN: <your_access_token>"\ ### Add note to existing epic thread Adds a new note to the thread. This can also -[create a thread from a single comment](../user/discussions/#create-a-thread-by-replying-to-a-standard-comment). +[create a thread from a single comment](../user/discussions/index.md#create-a-thread-by-replying-to-a-standard-comment). ```plaintext POST /groups/:id/epics/:epic_id/discussions/:discussion_id/notes @@ -1021,7 +1021,7 @@ curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>"\ ### Add note to existing merge request thread Adds a new note to the thread. This can also -[create a thread from a single comment](../user/discussions/#create-a-thread-by-replying-to-a-standard-comment). +[create a thread from a single comment](../user/discussions/index.md#create-a-thread-by-replying-to-a-standard-comment). ```plaintext POST /projects/:id/merge_requests/:merge_request_iid/discussions/:discussion_id/notes diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index 565e4b0be6c..500daa65d63 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -951,6 +951,30 @@ Input type: `BulkEnableDevopsAdoptionNamespacesInput` | <a id="mutationbulkenabledevopsadoptionnamespacesenablednamespaces"></a>`enabledNamespaces` | [`[DevopsAdoptionEnabledNamespace!]`](#devopsadoptionenablednamespace) | Enabled namespaces after mutation. | | <a id="mutationbulkenabledevopsadoptionnamespaceserrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +### `Mutation.bulkRunnerDelete` + +WARNING: +**Introduced** in 15.3. +This feature is in Alpha. It can be changed or removed at any time. + +Input type: `BulkRunnerDeleteInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationbulkrunnerdeleteclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationbulkrunnerdeleteids"></a>`ids` | [`[CiRunnerID!]`](#cirunnerid) | IDs of the runners to delete. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationbulkrunnerdeleteclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationbulkrunnerdeletedeletedcount"></a>`deletedCount` | [`Int`](#int) | Number of records effectively deleted. Only present if operation was performed synchronously. | +| <a id="mutationbulkrunnerdeletedeletedids"></a>`deletedIds` | [`[CiRunnerID!]`](#cirunnerid) | IDs of records effectively deleted. Only present if operation was performed synchronously. | +| <a id="mutationbulkrunnerdeleteerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | + ### `Mutation.ciCdSettingsUpdate` WARNING: diff --git a/doc/api/merge_request_approvals.md b/doc/api/merge_request_approvals.md index 890d7084ec9..1d99c323946 100644 --- a/doc/api/merge_request_approvals.md +++ b/doc/api/merge_request_approvals.md @@ -80,6 +80,7 @@ POST /projects/:id/approvals > - Moved to GitLab Premium in 13.9. > - `protected_branches` property was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/460) in GitLab 12.7. > - Pagination support [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31011) in GitLab 15.3 [with a flag](../administration/feature_flags.md) named `approval_rules_pagination`. Enabled by default. +> - `applies_to_all_protected_branches` property was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/335316) in GitLab 15.3. You can request information about a project's approval rules using the following endpoint: @@ -148,6 +149,7 @@ Use the `page` and `per_page` [pagination](index.md#offset-based-pagination) par "ldap_access": null } ], + "applies_to_all_protected_branches": false, "protected_branches": [ { "id": 1, @@ -180,7 +182,8 @@ Use the `page` and `per_page` [pagination](index.md#offset-based-pagination) par ### Get a single project-level rule -> Introduced in GitLab 13.7. +> - Introduced in GitLab 13.7. +> - `applies_to_all_protected_branches` property was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/335316) in GitLab 15.3. You can request information about a single project approval rules using the following endpoint: @@ -247,6 +250,7 @@ GET /projects/:id/approval_rules/:approval_rule_id "ldap_access": null } ], + "applies_to_all_protected_branches": false, "protected_branches": [ { "id": 1, @@ -281,6 +285,7 @@ GET /projects/:id/approval_rules/:approval_rule_id > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11877) in GitLab 12.3. > - Moved to GitLab Premium in 13.9. > - [Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/357300) the Vulnerability-Check feature in GitLab 15.0. +> - `applies_to_all_protected_branches` property was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/335316) in GitLab 15.3. You can create project approval rules using the following endpoint: @@ -290,16 +295,17 @@ POST /projects/:id/approval_rules **Parameters:** -| Attribute | Type | Required | Description | -|------------------------|---------|----------|------------------------------------------------------------------| -| `id` | integer or string | yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding) | -| `name` | string | yes | The name of the approval rule | -| `approvals_required` | integer | yes | The number of required approvals for this rule | -| `rule_type` | string | no | The type of rule. `any_approver` is a pre-configured default rule with `approvals_required` at `0`. Other rules are `regular`. -| `user_ids` | Array | no | The ids of users as approvers | -| `group_ids` | Array | no | The ids of groups as approvers | -| `protected_branch_ids` | Array | no | The IDs of protected branches to scope the rule by. To identify the ID, [use the API](protected_branches.md#list-protected-branches). | -| `report_type` | string | no | The report type required when the rule type is `report_approver`. The supported report types are: `license_scanning` and `code_coverage`.| +| Attribute | Type | Required | Description | +|-------------------------------------|-------------------|----------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `id` | integer or string | yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding) | +| `name` | string | yes | The name of the approval rule | +| `report_type` | string | no | The report type required when the rule type is `report_approver`. The supported report types are: `license_scanning` and `code_coverage`. | +| `approvals_required` | integer | yes | The number of required approvals for this rule | +| `rule_type` | string | no | The type of rule. `any_approver` is a pre-configured default rule with `approvals_required` at `0`. Other rules are `regular`. | +| `user_ids` | Array | no | The ids of users as approvers | +| `group_ids` | Array | no | The ids of groups as approvers | +| `protected_branch_ids` | Array | no | The IDs of protected branches to scope the rule by. To identify the ID, [use the API](protected_branches.md#list-protected-branches). | +| `applies_to_all_protected_branches` | boolean | no | Whether the rule is applied to all protected branches. If set to `true`, the value of `protected_branch_ids` is ignored. Default is `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/335316) in GitLab 15.3. | ```json { @@ -353,6 +359,7 @@ POST /projects/:id/approval_rules "ldap_access": null } ], + "applies_to_all_protected_branches": false, "protected_branches": [ { "id": 1, @@ -404,6 +411,7 @@ curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" \ > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11877) in GitLab 12.3. > - Moved to GitLab Premium in 13.9. > - [Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/357300) the Vulnerability-Check feature in GitLab 15.0. +> - `applies_to_all_protected_branches` property was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/335316) in GitLab 15.3. You can update project approval rules using the following endpoint: @@ -415,15 +423,16 @@ PUT /projects/:id/approval_rules/:approval_rule_id **Parameters:** -| Attribute | Type | Required | Description | -|------------------------|---------|----------|------------------------------------------------------------------| -| `id` | integer or string | yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding) | -| `approval_rule_id` | integer | yes | The ID of a approval rule | -| `name` | string | yes | The name of the approval rule | -| `approvals_required` | integer | yes | The number of required approvals for this rule | -| `user_ids` | Array | no | The ids of users as approvers | -| `group_ids` | Array | no | The ids of groups as approvers | -| `protected_branch_ids` | Array | no | The IDs of protected branches to scope the rule by. To identify the ID, [use the API](protected_branches.md#list-protected-branches). | +| Attribute | Type | Required | Description | +|-------------------------------------|-------------------|----------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `id` | integer or string | yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding) | +| `approval_rule_id` | integer | yes | The ID of a approval rule | +| `name` | string | yes | The name of the approval rule | +| `approvals_required` | integer | yes | The number of required approvals for this rule | +| `user_ids` | Array | no | The ids of users as approvers | +| `group_ids` | Array | no | The ids of groups as approvers | +| `protected_branch_ids` | Array | no | The IDs of protected branches to scope the rule by. To identify the ID, [use the API](protected_branches.md#list-protected-branches). | +| `applies_to_all_protected_branches` | boolean | no | Whether the rule is applied to all protected branches. If set to `true`, the value of `protected_branch_ids` is ignored. Default is `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/335316) in GitLab 15.3. | ```json { @@ -477,6 +486,7 @@ PUT /projects/:id/approval_rules/:approval_rule_id "ldap_access": null } ], + "applies_to_all_protected_branches": false, "protected_branches": [ { "id": 1, @@ -519,10 +529,10 @@ DELETE /projects/:id/approval_rules/:approval_rule_id **Parameters:** -| Attribute | Type | Required | Description | -|----------------------|---------|----------|-----------------------------------------------------------| -| `id` | integer or string | yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding) | -| `approval_rule_id` | integer | yes | The ID of a approval rule +| Attribute | Type | Required | Description | +|--------------------|-------------------|----------|------------------------------------------------------------------------------| +| `id` | integer or string | yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding) | +| `approval_rule_id` | integer | yes | The ID of a approval rule | ## Merge request-level MR approvals @@ -541,10 +551,10 @@ GET /projects/:id/merge_requests/:merge_request_iid/approvals **Parameters:** -| Attribute | Type | Required | Description | -|---------------------|---------|----------|---------------------| +| Attribute | Type | Required | Description | +|---------------------|-------------------|----------|------------------------------------------------------------------------------| | `id` | integer or string | yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding) | -| `merge_request_iid` | integer | yes | The IID of MR | +| `merge_request_iid` | integer | yes | The IID of MR | ```json { @@ -587,11 +597,11 @@ POST /projects/:id/merge_requests/:merge_request_iid/approvals **Parameters:** -| Attribute | Type | Required | Description | -|----------------------|---------|----------|--------------------------------------------| -| `id` | integer or string | yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding) | -| `merge_request_iid` | integer | yes | The IID of MR | -| `approvals_required` | integer | yes | Approvals required before MR can be merged. Deprecated in 12.0 in favor of Approval Rules API. | +| Attribute | Type | Required | Description | +|----------------------|-------------------|----------|------------------------------------------------------------------------------------------------| +| `id` | integer or string | yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding) | +| `merge_request_iid` | integer | yes | The IID of MR | +| `approvals_required` | integer | yes | Approvals required before MR can be merged. Deprecated in 12.0 in favor of Approval Rules API. | ```json { @@ -629,10 +639,10 @@ This includes additional information about the users who have already approved **Parameters:** -| Attribute | Type | Required | Description | -|----------------------|---------|----------|---------------------| -| `id` | integer or string | yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding) | -| `merge_request_iid` | integer | yes | The IID of MR | +| Attribute | Type | Required | Description | +|---------------------|-------------------|----------|------------------------------------------------------------------------------| +| `id` | integer or string | yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding) | +| `merge_request_iid` | integer | yes | The IID of MR | ```json { diff --git a/doc/api/personal_access_tokens.md b/doc/api/personal_access_tokens.md index 46a4c674560..620b5c2ed0b 100644 --- a/doc/api/personal_access_tokens.md +++ b/doc/api/personal_access_tokens.md @@ -91,7 +91,12 @@ curl --request GET --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab ### Responses -- `401: Unauthorized` if the user doesn't have access to the token they're requesting the ID or if the token with matching ID doesn't exist. +> `404` HTTP status code [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/93650) in GitLab 15.3. + +- `401: Unauthorized` if either: + - The user doesn't have access to the token with the specified ID. + - The token with the specified ID doesn't exist. +- `404: Not Found` if the user is an administrator but the token with the specified ID doesn't exist. ## Revoke a personal access token diff --git a/doc/ci/yaml/index.md b/doc/ci/yaml/index.md index 409877f159b..cdf034d077a 100644 --- a/doc/ci/yaml/index.md +++ b/doc/ci/yaml/index.md @@ -3247,6 +3247,7 @@ branch or merge request pipelines. **Possible inputs**: - An array of file paths. In GitLab 13.6 and later, [file paths can include variables](../jobs/job_control.md#variables-in-ruleschanges). +- Alternatively, the array of file paths can be in [`rules:changes:paths`](#ruleschangespaths). **Example of `rules:changes`**: @@ -3265,6 +3266,8 @@ docker build: - If `Dockerfile` has changed, add the job to the pipeline as a manual job, and the pipeline continues running even if the job is not triggered (`allow_failure: true`). - If `Dockerfile` has not changed, do not add job to any pipeline (same as `when: never`). +- [`rules:changes:paths`](#ruleschangespaths) is the same as `rules:changes` without + any subkeys. **Additional details**: @@ -3280,18 +3283,29 @@ docker build: > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/90171) in GitLab 15.2. -`rules:changes:paths` is an alias for `rules:changes`. +Use `rules:changes` to specify that a job only be added to a pipeline when specific +files are changed, and use `rules:changes:paths` to specify the files. + +`rules:changes:paths` is the same as using [`rules:changes`](#ruleschanges) without +any subkeys. All additional details and related topics are the same. **Keyword type**: Job keyword. You can use it only as part of a job. **Possible inputs**: -- An array of file paths. +- An array of file paths. In GitLab 13.6 and later, [file paths can include variables](../jobs/job_control.md#variables-in-ruleschanges). **Example of `rules:changes:paths`**: ```yaml -docker build: +docker-build-1: + script: docker build -t my-image:$CI_COMMIT_REF_SLUG . + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + changes: + - Dockerfile + +docker-build-2: script: docker build -t my-image:$CI_COMMIT_REF_SLUG . rules: - if: $CI_PIPELINE_SOURCE == "merge_request_event" @@ -3300,8 +3314,7 @@ docker build: - Dockerfile ``` -In this example, the `docker build` job is only included when the `Dockerfile` has changed -and the pipeline source is a merge request event. +In this example, both jobs have the same behavior. ##### `rules:changes:compare_to` diff --git a/doc/development/chatops_on_gitlabcom.md b/doc/development/chatops_on_gitlabcom.md index 2065021c61b..7309b92c702 100644 --- a/doc/development/chatops_on_gitlabcom.md +++ b/doc/development/chatops_on_gitlabcom.md @@ -59,5 +59,5 @@ To request access to ChatOps on GitLab.com: ## See also - [ChatOps Usage](../ci/chatops/index.md) -- [Understanding EXPLAIN plans](understanding_explain_plans.md) +- [Understanding EXPLAIN plans](database/understanding_explain_plans.md) - [Feature Groups](feature_flags/index.md#feature-groups) diff --git a/doc/development/database/database_reviewer_guidelines.md b/doc/development/database/database_reviewer_guidelines.md index a34ea52454f..64a570ed43d 100644 --- a/doc/development/database/database_reviewer_guidelines.md +++ b/doc/development/database/database_reviewer_guidelines.md @@ -53,14 +53,14 @@ that require a more in-depth discussion between the database reviewers and maint - [Database Office Hours Agenda](https://docs.google.com/document/d/1wgfmVL30F8SdMg-9yY6Y8djPSxWNvKmhR5XmsvYX1EI/edit). - <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> [YouTube playlist with past recordings](https://www.youtube.com/playlist?list=PL05JrBw4t0Kp-kqXeiF7fF7cFYaKtdqXM). -You should also join the [#database-lab](../understanding_explain_plans.md#database-lab-engine) +You should also join the [#database-lab](understanding_explain_plans.md#database-lab-engine) Slack channel and get familiar with how to use Joe, the Slackbot that provides developers with their own clone of the production database. Understanding and efficiently using `EXPLAIN` plans is at the core of the database review process. The following guides provide a quick introduction and links to follow on more advanced topics: -- Guide on [understanding EXPLAIN plans](../understanding_explain_plans.md). +- Guide on [understanding EXPLAIN plans](understanding_explain_plans.md). - [Explaining the unexplainable series in `depesz`](https://www.depesz.com/tag/unexplainable/). We also have licensed access to The Art of PostgreSQL available, if you are interested in getting access please check out the diff --git a/doc/development/database/index.md b/doc/development/database/index.md index b427f54ff3c..1b8dead942c 100644 --- a/doc/development/database/index.md +++ b/doc/development/database/index.md @@ -16,7 +16,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w ## Tooling -- [Understanding EXPLAIN plans](../understanding_explain_plans.md) +- [Understanding EXPLAIN plans](understanding_explain_plans.md) - [explain.depesz.com](https://explain.depesz.com/) or [explain.dalibo.com](https://explain.dalibo.com/) for visualizing the output of `EXPLAIN` - [pgFormatter](https://sqlformat.darold.net/) a PostgreSQL SQL syntax beautifier - [db:check-migrations job](dbcheck-migrations-job.md) @@ -30,7 +30,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w - [Testing Rails migrations](../testing_guide/testing_migrations_guide.md) guide - [Post deployment migrations](post_deployment_migrations.md) - [Background migrations](background_migrations.md) -- [Swapping tables](../swapping_tables.md) +- [Swapping tables](swapping_tables.md) - [Deleting migrations](deleting_migrations.md) - [Partitioning tables](table_partitioning.md) @@ -47,13 +47,13 @@ info: To determine the technical writer assigned to the Stage/Group associated w - [`NOT NULL` constraints](not_null_constraints.md) - [Strings and the Text data type](strings_and_the_text_data_type.md) - [Single table inheritance](../single_table_inheritance.md) -- [Polymorphic associations](../polymorphic_associations.md) +- [Polymorphic associations](polymorphic_associations.md) - [Serializing data](../serializing_data.md) - [Hash indexes](../hash_indexes.md) - [Storing SHA1 hashes as binary](../sha1_as_binary.md) - [Iterating tables in batches](../iterating_tables_in_batches.md) - [Insert into tables in batches](../insert_into_tables_in_batches.md) -- [Ordering table columns](../ordering_table_columns.md) +- [Ordering table columns](ordering_table_columns.md) - [Verifying database capabilities](../verifying_database_capabilities.md) - [Database Debugging and Troubleshooting](../database_debugging.md) - [Query Count Limits](../query_count_limits.md) diff --git a/doc/development/database/ordering_table_columns.md b/doc/development/database/ordering_table_columns.md new file mode 100644 index 00000000000..7cd3d4fb208 --- /dev/null +++ b/doc/development/database/ordering_table_columns.md @@ -0,0 +1,152 @@ +--- +stage: Data Stores +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Ordering Table Columns in PostgreSQL + +For GitLab we require that columns of new tables are ordered to use the +least amount of space. An easy way of doing this is to order them based on the +type size in descending order with variable sizes (`text`, `varchar`, arrays, +`json`, `jsonb`, and so on) at the end. + +Similar to C structures the space of a table is influenced by the order of +columns. This is because the size of columns is aligned depending on the type of +the following column. Let's consider an example: + +- `id` (integer, 4 bytes) +- `name` (text, variable) +- `user_id` (integer, 4 bytes) + +The first column is a 4-byte integer. The next is text of variable length. The +`text` data type requires 1-word alignment, and on 64-bit platform, 1 word is 8 +bytes. To meet the alignment requirements, four zeros are to be added right +after the first column, so `id` occupies 4 bytes, then 4 bytes of alignment +padding, and only next `name` is being stored. Therefore, in this case, 8 bytes +are spent for storing a 4-byte integer. + +The space between rows is also subject to alignment padding. The `user_id` +column takes only 4 bytes, and on 64-bit platform, 4 zeroes are added for +alignment padding, to allow storing the next row beginning with the "clear" word. + +As a result, the actual size of each column would be (omitting variable length +data and 24-byte tuple header): 8 bytes, variable, 8 bytes. This means that +each row requires at least 16 bytes for the two 4-byte integers. If a table +has a few rows this is not an issue. However, once you start storing millions of +rows you can save space by using a different order. For the above example, the +ideal column order would be the following: + +- `id` (integer, 4 bytes) +- `user_id` (integer, 4 bytes) +- `name` (text, variable) + +or + +- `name` (text, variable) +- `id` (integer, 4 bytes) +- `user_id` (integer, 4 bytes) + +In these examples, the `id` and `user_id` columns are packed together, which +means we only need 8 bytes to store _both_ of them. This in turn means each row +requires 8 bytes less space. + +Since Ruby on Rails 5.1, the default data type for IDs is `bigint`, which uses 8 bytes. +We are using `integer` in the examples to showcase a more realistic reordering scenario. + +## Type Sizes + +While the [PostgreSQL documentation](https://www.postgresql.org/docs/current/datatype.html) contains plenty +of information we list the sizes of common types here so it's easier to +look them up. Here "word" refers to the word size, which is 4 bytes for a 32 +bits platform and 8 bytes for a 64 bits platform. + +| Type | Size | Alignment needed | +|:-----------------|:-------------------------------------|:-----------| +| `smallint` | 2 bytes | 1 word | +| `integer` | 4 bytes | 1 word | +| `bigint` | 8 bytes | 8 bytes | +| `real` | 4 bytes | 1 word | +| `double precision` | 8 bytes | 8 bytes | +| `boolean` | 1 byte | not needed | +| `text` / `string` | variable, 1 byte plus the data | 1 word | +| `bytea` | variable, 1 or 4 bytes plus the data | 1 word | +| `timestamp` | 8 bytes | 8 bytes | +| `timestamptz` | 8 bytes | 8 bytes | +| `date` | 4 bytes | 1 word | + +A "variable" size means the actual size depends on the value being stored. If +PostgreSQL determines this can be embedded directly into a row it may do so, but +for very large values it stores the data externally and store a pointer (of +1 word in size) in the column. Because of this variable sized columns should +always be at the end of a table. + +## Real Example + +Let's use the `events` table as an example, which currently has the following +layout: + +| Column | Type | Size | +|:--------------|:----------------------------|:---------| +| `id` | integer | 4 bytes | +| `target_type` | character varying | variable | +| `target_id` | integer | 4 bytes | +| `title` | character varying | variable | +| `data` | text | variable | +| `project_id` | integer | 4 bytes | +| `created_at` | timestamp without time zone | 8 bytes | +| `updated_at` | timestamp without time zone | 8 bytes | +| `action` | integer | 4 bytes | +| `author_id` | integer | 4 bytes | + +After adding padding to align the columns this would translate to columns being +divided into fixed size chunks as follows: + +| Chunk Size | Columns | +|:-----------|:----------------------| +| 8 bytes | `id` | +| variable | `target_type` | +| 8 bytes | `target_id` | +| variable | `title` | +| variable | `data` | +| 8 bytes | `project_id` | +| 8 bytes | `created_at` | +| 8 bytes | `updated_at` | +| 8 bytes | `action`, `author_id` | + +This means that excluding the variable sized data and tuple header, we need at +least 8 * 6 = 48 bytes per row. + +We can optimise this by using the following column order instead: + +| Column | Type | Size | +|:--------------|:----------------------------|:---------| +| `created_at` | timestamp without time zone | 8 bytes | +| `updated_at` | timestamp without time zone | 8 bytes | +| `id` | integer | 4 bytes | +| `target_id` | integer | 4 bytes | +| `project_id` | integer | 4 bytes | +| `action` | integer | 4 bytes | +| `author_id` | integer | 4 bytes | +| `target_type` | character varying | variable | +| `title` | character varying | variable | +| `data` | text | variable | + +This would produce the following chunks: + +| Chunk Size | Columns | +|:-----------|:-----------------------| +| 8 bytes | `created_at` | +| 8 bytes | `updated_at` | +| 8 bytes | `id`, `target_id` | +| 8 bytes | `project_id`, `action` | +| 8 bytes | `author_id` | +| variable | `target_type` | +| variable | `title` | +| variable | `data` | + +Here we only need 40 bytes per row excluding the variable sized data and 24-byte +tuple header. 8 bytes being saved may not sound like much, but for tables as +large as the `events` table it does begin to matter. For example, when storing +80 000 000 rows this translates to a space saving of at least 610 MB, all by +just changing the order of a few columns. diff --git a/doc/development/database/pagination_guidelines.md b/doc/development/database/pagination_guidelines.md index 1641708ce01..fe2e3b46939 100644 --- a/doc/development/database/pagination_guidelines.md +++ b/doc/development/database/pagination_guidelines.md @@ -192,7 +192,7 @@ The query execution plan shows that this query is efficient, the database only r (6 rows) ``` -See the [Understanding EXPLAIN plans](../understanding_explain_plans.md) to find more information about reading execution plans. +See the [Understanding EXPLAIN plans](understanding_explain_plans.md) to find more information about reading execution plans. Let's visit the 50_000th page: diff --git a/doc/development/database/pagination_performance_guidelines.md b/doc/development/database/pagination_performance_guidelines.md index b5040e499e4..0fef246f133 100644 --- a/doc/development/database/pagination_performance_guidelines.md +++ b/doc/development/database/pagination_performance_guidelines.md @@ -12,11 +12,11 @@ The following document gives a few ideas for improving the pagination (sorting) When ordering the columns it's advised to order by distinct columns only. Consider the following example: -|`id`|`created_at`| -|-|-| -|1|2021-01-04 14:13:43| -|2|2021-01-05 19:03:12| -|3|2021-01-05 19:03:12| +| `id` | `created_at` | +|------|-----------------------| +| `1` | `2021-01-04 14:13:43` | +| `2` | `2021-01-05 19:03:12` | +| `3` | `2021-01-05 19:03:12` | If we order by `created_at`, the result would likely depend on how the records are located on the disk. diff --git a/doc/development/database/polymorphic_associations.md b/doc/development/database/polymorphic_associations.md new file mode 100644 index 00000000000..ac4dc7720a5 --- /dev/null +++ b/doc/development/database/polymorphic_associations.md @@ -0,0 +1,152 @@ +--- +stage: Data Stores +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Polymorphic Associations + +**Summary:** always use separate tables instead of polymorphic associations. + +Rails makes it possible to define so called "polymorphic associations". This +usually works by adding two columns to a table: a target type column, and a +target ID. For example, at the time of writing we have such a setup for +`members` with the following columns: + +- `source_type`: a string defining the model to use, can be either `Project` or + `Namespace`. +- `source_id`: the ID of the row to retrieve based on `source_type`. For + example, when `source_type` is `Project` then `source_id` contains a + project ID. + +While such a setup may appear to be useful, it comes with many drawbacks; enough +that you should avoid this at all costs. + +## Space Wasted + +Because this setup relies on string values to determine the model to use, it +wastes a lot of space. For example, for `Project` and `Namespace` the +maximum size is 9 bytes, plus 1 extra byte for every string when using +PostgreSQL. While this may only be 10 bytes per row, given enough tables and +rows using such a setup we can end up wasting quite a bit of disk space and +memory (for any indexes). + +## Indexes + +Because our associations are broken up into two columns this may result in +requiring composite indexes for queries to be performed efficiently. While +composite indexes are not wrong at all, they can be tricky to set up as the +ordering of columns in these indexes is important to ensure optimal performance. + +## Consistency + +One really big problem with polymorphic associations is being unable to enforce +data consistency on the database level using foreign keys. For consistency to be +enforced on the database level one would have to write their own foreign key +logic to support polymorphic associations. + +Enforcing consistency on the database level is absolutely crucial for +maintaining a healthy environment, and thus is another reason to avoid +polymorphic associations. + +## Query Overhead + +When using polymorphic associations you always need to filter using both +columns. For example, you may end up writing a query like this: + +```sql +SELECT * +FROM members +WHERE source_type = 'Project' +AND source_id = 13083; +``` + +Here PostgreSQL can perform the query quite efficiently if both columns are +indexed. As the query gets more complex, it may not be able to use these +indexes effectively. + +## Mixed Responsibilities + +Similar to functions and classes, a table should have a single responsibility: +storing data with a certain set of pre-defined columns. When using polymorphic +associations, you are storing different types of data (possibly with +different columns set) in the same table. + +## The Solution + +Fortunately, there is a solution to these problems: use a +separate table for every type you would otherwise store in the same table. Using +a separate table allows you to use everything a database may provide to ensure +consistency and query data efficiently, without any additional application logic +being necessary. + +Let's say you have a `members` table storing both approved and pending members, +for both projects and groups, and the pending state is determined by the column +`requested_at` being set or not. Schema wise such a setup can lead to various +columns only being set for certain rows, wasting space. It's also possible that +certain indexes are only set for certain rows, again wasting space. Finally, +querying such a table requires less than ideal queries. For example: + +```sql +SELECT * +FROM members +WHERE requested_at IS NULL +AND source_type = 'GroupMember' +AND source_id = 4 +``` + +Instead such a table should be broken up into separate tables. For example, you +may end up with 4 tables in this case: + +- project_members +- group_members +- pending_project_members +- pending_group_members + +This makes querying data trivial. For example, to get the members of a group +you'd run: + +```sql +SELECT * +FROM group_members +WHERE group_id = 4 +``` + +To get all the pending members of a group in turn you'd run: + +```sql +SELECT * +FROM pending_group_members +WHERE group_id = 4 +``` + +If you want to get both you can use a `UNION`, though you need to be explicit +about what columns you want to `SELECT` as otherwise the result set uses the +columns of the first query. For example: + +```sql +SELECT id, 'Group' AS target_type, group_id AS target_id +FROM group_members + +UNION ALL + +SELECT id, 'Project' AS target_type, project_id AS target_id +FROM project_members +``` + +The above example is perhaps a bit silly, but it shows that there's nothing +stopping you from merging the data together and presenting it on the same page. +Selecting columns explicitly can also speed up queries as the database has to do +less work to get the data (compared to selecting all columns, even ones you're +not using). + +Our schema also becomes easier. No longer do we need to both store and index the +`source_type` column, we can define foreign keys easily, and we don't need to +filter rows using the `IS NULL` condition. + +To summarize: using separate tables allows us to use foreign keys effectively, +create indexes only where necessary, conserve space, query data more +efficiently, and scale these tables more easily (for example, by storing them on +separate disks). A nice side effect of this is that code can also become easier, +as a single model isn't responsible for handling different kinds of +data. diff --git a/doc/development/database/swapping_tables.md b/doc/development/database/swapping_tables.md new file mode 100644 index 00000000000..efb481ccf35 --- /dev/null +++ b/doc/development/database/swapping_tables.md @@ -0,0 +1,51 @@ +--- +stage: Data Stores +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Swapping Tables + +Sometimes you need to replace one table with another. For example, when +migrating data in a very large table it's often better to create a copy of the +table and insert & migrate the data into this new table in the background. + +Let's say you want to swap the table `events` with `events_for_migration`. In +this case you need to follow 3 steps: + +1. Rename `events` to `events_temporary` +1. Rename `events_for_migration` to `events` +1. Rename `events_temporary` to `events_for_migration` + +Rails allows you to do this using the `rename_table` method: + +```ruby +rename_table :events, :events_temporary +rename_table :events_for_migration, :events +rename_table :events_temporary, :events_for_migration +``` + +This does not require any downtime as long as the 3 `rename_table` calls are +executed in the _same_ database transaction. Rails by default uses database +transactions for migrations, but if it doesn't you need to start one +manually: + +```ruby +Event.transaction do + rename_table :events, :events_temporary + rename_table :events_for_migration, :events + rename_table :events_temporary, :events_for_migration +end +``` + +Once swapped you _have to_ reset the primary key of the new table. For +PostgreSQL you can use the `reset_pk_sequence!` method like so: + +```ruby +reset_pk_sequence!('events') +``` + +Failure to reset the primary keys results in newly created rows starting +with an ID value of 1. Depending on the existing data this can then lead to +duplicate key constraints from popping up, preventing users from creating new +data. diff --git a/doc/development/database/understanding_explain_plans.md b/doc/development/database/understanding_explain_plans.md new file mode 100644 index 00000000000..49babde737a --- /dev/null +++ b/doc/development/database/understanding_explain_plans.md @@ -0,0 +1,829 @@ +--- +stage: Data Stores +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Understanding EXPLAIN plans + +PostgreSQL allows you to obtain query plans using the `EXPLAIN` command. This +command can be invaluable when trying to determine how a query performs. +You can use this command directly in your SQL query, as long as the query starts +with it: + +```sql +EXPLAIN +SELECT COUNT(*) +FROM projects +WHERE visibility_level IN (0, 20); +``` + +When running this on GitLab.com, we are presented with the following output: + +```sql +Aggregate (cost=922411.76..922411.77 rows=1 width=8) + -> Seq Scan on projects (cost=0.00..908044.47 rows=5746914 width=0) + Filter: (visibility_level = ANY ('{0,20}'::integer[])) +``` + +When using _just_ `EXPLAIN`, PostgreSQL does not actually execute our query, +instead it produces an _estimated_ execution plan based on the available +statistics. This means the actual plan can differ quite a bit. Fortunately, +PostgreSQL provides us with the option to execute the query as well. To do so, +we need to use `EXPLAIN ANALYZE` instead of just `EXPLAIN`: + +```sql +EXPLAIN ANALYZE +SELECT COUNT(*) +FROM projects +WHERE visibility_level IN (0, 20); +``` + +This produces: + +```sql +Aggregate (cost=922420.60..922420.61 rows=1 width=8) (actual time=3428.535..3428.535 rows=1 loops=1) + -> Seq Scan on projects (cost=0.00..908053.18 rows=5746969 width=0) (actual time=0.041..2987.606 rows=5746940 loops=1) + Filter: (visibility_level = ANY ('{0,20}'::integer[])) + Rows Removed by Filter: 65677 +Planning time: 2.861 ms +Execution time: 3428.596 ms +``` + +As we can see this plan is quite different, and includes a lot more data. Let's +discuss this step by step. + +Because `EXPLAIN ANALYZE` executes the query, care should be taken when using a +query that writes data or might time out. If the query modifies data, +consider wrapping it in a transaction that rolls back automatically like so: + +```sql +BEGIN; +EXPLAIN ANALYZE +DELETE FROM users WHERE id = 1; +ROLLBACK; +``` + +The `EXPLAIN` command also takes additional options, such as `BUFFERS`: + +```sql +EXPLAIN (ANALYZE, BUFFERS) +SELECT COUNT(*) +FROM projects +WHERE visibility_level IN (0, 20); +``` + +This then produces: + +```sql +Aggregate (cost=922420.60..922420.61 rows=1 width=8) (actual time=3428.535..3428.535 rows=1 loops=1) + Buffers: shared hit=208846 + -> Seq Scan on projects (cost=0.00..908053.18 rows=5746969 width=0) (actual time=0.041..2987.606 rows=5746940 loops=1) + Filter: (visibility_level = ANY ('{0,20}'::integer[])) + Rows Removed by Filter: 65677 + Buffers: shared hit=208846 +Planning time: 2.861 ms +Execution time: 3428.596 ms +``` + +For more information, refer to the official +[`EXPLAIN` documentation](https://www.postgresql.org/docs/current/sql-explain.html) +and [using `EXPLAIN` guide](https://www.postgresql.org/docs/current/using-explain.html). + +## Nodes + +Every query plan consists of nodes. Nodes can be nested, and are executed from +the inside out. This means that the innermost node is executed before an outer +node. This can be best thought of as nested function calls, returning their +results as they unwind. For example, a plan starting with an `Aggregate` +followed by a `Nested Loop`, followed by an `Index Only scan` can be thought of +as the following Ruby code: + +```ruby +aggregate( + nested_loop( + index_only_scan() + index_only_scan() + ) +) +``` + +Nodes are indicated using a `->` followed by the type of node taken. For +example: + +```sql +Aggregate (cost=922411.76..922411.77 rows=1 width=8) + -> Seq Scan on projects (cost=0.00..908044.47 rows=5746914 width=0) + Filter: (visibility_level = ANY ('{0,20}'::integer[])) +``` + +Here the first node executed is `Seq scan on projects`. The `Filter:` is an +additional filter applied to the results of the node. A filter is very similar +to Ruby's `Array#select`: it takes the input rows, applies the filter, and +produces a new list of rows. After the node is done, we perform the `Aggregate` +above it. + +Nested nodes look like this: + +```sql +Aggregate (cost=176.97..176.98 rows=1 width=8) (actual time=0.252..0.252 rows=1 loops=1) + Buffers: shared hit=155 + -> Nested Loop (cost=0.86..176.75 rows=87 width=0) (actual time=0.035..0.249 rows=36 loops=1) + Buffers: shared hit=155 + -> Index Only Scan using users_pkey on users users_1 (cost=0.43..4.95 rows=87 width=4) (actual time=0.029..0.123 rows=36 loops=1) + Index Cond: (id < 100) + Heap Fetches: 0 + -> Index Only Scan using users_pkey on users (cost=0.43..1.96 rows=1 width=4) (actual time=0.003..0.003 rows=1 loops=36) + Index Cond: (id = users_1.id) + Heap Fetches: 0 +Planning time: 2.585 ms +Execution time: 0.310 ms +``` + +Here we first perform two separate "Index Only" scans, followed by performing a +"Nested Loop" on the result of these two scans. + +## Node statistics + +Each node in a plan has a set of associated statistics, such as the cost, the +number of rows produced, the number of loops performed, and more. For example: + +```sql +Seq Scan on projects (cost=0.00..908044.47 rows=5746914 width=0) +``` + +Here we can see that our cost ranges from `0.00..908044.47` (we cover this in +a moment), and we estimate (since we're using `EXPLAIN` and not `EXPLAIN +ANALYZE`) a total of 5,746,914 rows to be produced by this node. The `width` +statistics describes the estimated width of each row, in bytes. + +The `costs` field specifies how expensive a node was. The cost is measured in +arbitrary units determined by the query planner's cost parameters. What +influences the costs depends on a variety of settings, such as `seq_page_cost`, +`cpu_tuple_cost`, and various others. +The format of the costs field is as follows: + +```sql +STARTUP COST..TOTAL COST +``` + +The startup cost states how expensive it was to start the node, with the total +cost describing how expensive the entire node was. In general: the greater the +values, the more expensive the node. + +When using `EXPLAIN ANALYZE`, these statistics also include the actual time +(in milliseconds) spent, and other runtime statistics (for example, the actual number of +produced rows): + +```sql +Seq Scan on projects (cost=0.00..908053.18 rows=5746969 width=0) (actual time=0.041..2987.606 rows=5746940 loops=1) +``` + +Here we can see we estimated 5,746,969 rows to be returned, but in reality we +returned 5,746,940 rows. We can also see that _just_ this sequential scan took +2.98 seconds to run. + +Using `EXPLAIN (ANALYZE, BUFFERS)` also gives us information about the +number of rows removed by a filter, the number of buffers used, and more. For +example: + +```sql +Seq Scan on projects (cost=0.00..908053.18 rows=5746969 width=0) (actual time=0.041..2987.606 rows=5746940 loops=1) + Filter: (visibility_level = ANY ('{0,20}'::integer[])) + Rows Removed by Filter: 65677 + Buffers: shared hit=208846 +``` + +Here we can see that our filter has to remove 65,677 rows, and that we use +208,846 buffers. Each buffer in PostgreSQL is 8 KB (8192 bytes), meaning our +above node uses *1.6 GB of buffers*. That's a lot! + +Keep in mind that some statistics are per-loop averages, while others are total values: + +| Field name | Value type | +| --- | --- | +| Actual Total Time | per-loop average | +| Actual Rows | per-loop average | +| Buffers Shared Hit | total value | +| Buffers Shared Read | total value | +| Buffers Shared Dirtied | total value | +| Buffers Shared Written | total value | +| I/O Read Time | total value | +| I/O Read Write | total value | + +For example: + +```sql + -> Index Scan using users_pkey on public.users (cost=0.43..3.44 rows=1 width=1318) (actual time=0.025..0.025 rows=1 loops=888) + Index Cond: (users.id = issues.author_id) + Buffers: shared hit=3543 read=9 + I/O Timings: read=17.760 write=0.000 +``` + +Here we can see that this node used 3552 buffers (3543 + 9), returned 888 rows (`888 * 1`), and the actual duration was 22.2 milliseconds (`888 * 0.025`). +17.76 milliseconds of the total duration was spent in reading from disk, to retrieve data that was not in the cache. + +## Node types + +There are quite a few different types of nodes, so we only cover some of the +more common ones here. + +A full list of all the available nodes and their descriptions can be found in +the [PostgreSQL source file `plannodes.h`](https://gitlab.com/postgres/postgres/blob/master/src/include/nodes/plannodes.h). +pgMustard's [EXPLAIN docs](https://www.pgmustard.com/docs/explain) also offer detailed look into nodes and their fields. + +### Seq Scan + +A sequential scan over (a chunk of) a database table. This is like using +`Array#each`, but on a database table. Sequential scans can be quite slow when +retrieving lots of rows, so it's best to avoid these for large tables. + +### Index Only Scan + +A scan on an index that did not require fetching anything from the table. In +certain cases an index only scan may still fetch data from the table, in this +case the node includes a `Heap Fetches:` statistic. + +### Index Scan + +A scan on an index that required retrieving some data from the table. + +### Bitmap Index Scan and Bitmap Heap scan + +Bitmap scans fall between sequential scans and index scans. These are typically +used when we would read too much data from an index scan, but too little to +perform a sequential scan. A bitmap scan uses what is known as a [bitmap +index](https://en.wikipedia.org/wiki/Bitmap_index) to perform its work. + +The [source code of PostgreSQL](https://gitlab.com/postgres/postgres/blob/REL_11_STABLE/src/include/nodes/plannodes.h#L441) +states the following on bitmap scans: + +> Bitmap Index Scan delivers a bitmap of potential tuple locations; it does not +> access the heap itself. The bitmap is used by an ancestor Bitmap Heap Scan +> node, possibly after passing through intermediate Bitmap And and/or Bitmap Or +> nodes to combine it with the results of other Bitmap Index Scans. + +### Limit + +Applies a `LIMIT` on the input rows. + +### Sort + +Sorts the input rows as specified using an `ORDER BY` statement. + +### Nested Loop + +A nested loop executes its child nodes for every row produced by a node that +precedes it. For example: + +```sql +-> Nested Loop (cost=0.86..176.75 rows=87 width=0) (actual time=0.035..0.249 rows=36 loops=1) + Buffers: shared hit=155 + -> Index Only Scan using users_pkey on users users_1 (cost=0.43..4.95 rows=87 width=4) (actual time=0.029..0.123 rows=36 loops=1) + Index Cond: (id < 100) + Heap Fetches: 0 + -> Index Only Scan using users_pkey on users (cost=0.43..1.96 rows=1 width=4) (actual time=0.003..0.003 rows=1 loops=36) + Index Cond: (id = users_1.id) + Heap Fetches: 0 +``` + +Here the first child node (`Index Only Scan using users_pkey on users users_1`) +produces 36 rows, and is executed once (`rows=36 loops=1`). The next node +produces 1 row (`rows=1`), but is repeated 36 times (`loops=36`). This is +because the previous node produced 36 rows. + +This means that nested loops can quickly slow the query down if the various +child nodes keep producing many rows. + +## Optimising queries + +With that out of the way, let's see how we can optimise a query. Let's use the +following query as an example: + +```sql +SELECT COUNT(*) +FROM users +WHERE twitter != ''; +``` + +This query counts the number of users that have a Twitter profile set. +Let's run this using `EXPLAIN (ANALYZE, BUFFERS)`: + +```sql +EXPLAIN (ANALYZE, BUFFERS) +SELECT COUNT(*) +FROM users +WHERE twitter != ''; +``` + +This produces the following plan: + +```sql +Aggregate (cost=845110.21..845110.22 rows=1 width=8) (actual time=1271.157..1271.158 rows=1 loops=1) + Buffers: shared hit=202662 + -> Seq Scan on users (cost=0.00..844969.99 rows=56087 width=0) (actual time=0.019..1265.883 rows=51833 loops=1) + Filter: ((twitter)::text <> ''::text) + Rows Removed by Filter: 2487813 + Buffers: shared hit=202662 +Planning time: 0.390 ms +Execution time: 1271.180 ms +``` + +From this query plan we can see the following: + +1. We need to perform a sequential scan on the `users` table. +1. This sequential scan filters out 2,487,813 rows using a `Filter`. +1. We use 202,622 buffers, which equals 1.58 GB of memory. +1. It takes us 1.2 seconds to do all of this. + +Considering we are just counting users, that's quite expensive! + +Before we start making any changes, let's see if there are any existing indexes +on the `users` table that we might be able to use. We can obtain this +information by running `\d users` in a `psql` console, then scrolling down to +the `Indexes:` section: + +```sql +Indexes: + "users_pkey" PRIMARY KEY, btree (id) + "index_users_on_confirmation_token" UNIQUE, btree (confirmation_token) + "index_users_on_email" UNIQUE, btree (email) + "index_users_on_reset_password_token" UNIQUE, btree (reset_password_token) + "index_users_on_static_object_token" UNIQUE, btree (static_object_token) + "index_users_on_unlock_token" UNIQUE, btree (unlock_token) + "index_on_users_name_lower" btree (lower(name::text)) + "index_users_on_accepted_term_id" btree (accepted_term_id) + "index_users_on_admin" btree (admin) + "index_users_on_created_at" btree (created_at) + "index_users_on_email_trigram" gin (email gin_trgm_ops) + "index_users_on_feed_token" btree (feed_token) + "index_users_on_group_view" btree (group_view) + "index_users_on_incoming_email_token" btree (incoming_email_token) + "index_users_on_managing_group_id" btree (managing_group_id) + "index_users_on_name" btree (name) + "index_users_on_name_trigram" gin (name gin_trgm_ops) + "index_users_on_public_email" btree (public_email) WHERE public_email::text <> ''::text + "index_users_on_state" btree (state) + "index_users_on_state_and_user_type" btree (state, user_type) + "index_users_on_unconfirmed_email" btree (unconfirmed_email) WHERE unconfirmed_email IS NOT NULL + "index_users_on_user_type" btree (user_type) + "index_users_on_username" btree (username) + "index_users_on_username_trigram" gin (username gin_trgm_ops) + "tmp_idx_on_user_id_where_bio_is_filled" btree (id) WHERE COALESCE(bio, ''::character varying)::text IS DISTINCT FROM ''::text +``` + +Here we can see there is no index on the `twitter` column, which means +PostgreSQL has to perform a sequential scan in this case. Let's try to fix this +by adding the following index: + +```sql +CREATE INDEX CONCURRENTLY twitter_test ON users (twitter); +``` + +If we now re-run our query using `EXPLAIN (ANALYZE, BUFFERS)` we get the +following plan: + +```sql +Aggregate (cost=61002.82..61002.83 rows=1 width=8) (actual time=297.311..297.312 rows=1 loops=1) + Buffers: shared hit=51854 dirtied=19 + -> Index Only Scan using twitter_test on users (cost=0.43..60873.13 rows=51877 width=0) (actual time=279.184..293.532 rows=51833 loops=1) + Filter: ((twitter)::text <> ''::text) + Rows Removed by Filter: 2487830 + Heap Fetches: 26037 + Buffers: shared hit=51854 dirtied=19 +Planning time: 0.191 ms +Execution time: 297.334 ms +``` + +Now it takes just under 300 milliseconds to get our data, instead of 1.2 +seconds. However, we still use 51,854 buffers, which is about 400 MB of memory. +300 milliseconds is also quite slow for such a simple query. To understand why +this query is still expensive, let's take a look at the following: + +```sql +Index Only Scan using twitter_test on users (cost=0.43..60873.13 rows=51877 width=0) (actual time=279.184..293.532 rows=51833 loops=1) + Filter: ((twitter)::text <> ''::text) + Rows Removed by Filter: 2487830 +``` + +We start with an index only scan on our index, but we somehow still apply a +`Filter` that filters out 2,487,830 rows. Why is that? Well, let's look at how +we created the index: + +```sql +CREATE INDEX CONCURRENTLY twitter_test ON users (twitter); +``` + +We told PostgreSQL to index all possible values of the `twitter` column, +even empty strings. Our query in turn uses `WHERE twitter != ''`. This means +that the index does improve things, as we don't need to do a sequential scan, +but we may still encounter empty strings. This means PostgreSQL _has_ to apply a +Filter on the index results to get rid of those values. + +Fortunately, we can improve this even further using "partial indexes". Partial +indexes are indexes with a `WHERE` condition that is applied when indexing data. +For example: + +```sql +CREATE INDEX CONCURRENTLY some_index ON users (email) WHERE id < 100 +``` + +This index would only index the `email` value of rows that match `WHERE id < +100`. We can use partial indexes to change our Twitter index to the following: + +```sql +CREATE INDEX CONCURRENTLY twitter_test ON users (twitter) WHERE twitter != ''; +``` + +After being created, if we run our query again we are given the following plan: + +```sql +Aggregate (cost=1608.26..1608.27 rows=1 width=8) (actual time=19.821..19.821 rows=1 loops=1) + Buffers: shared hit=44036 + -> Index Only Scan using twitter_test on users (cost=0.41..1479.71 rows=51420 width=0) (actual time=0.023..15.514 rows=51833 loops=1) + Heap Fetches: 1208 + Buffers: shared hit=44036 +Planning time: 0.123 ms +Execution time: 19.848 ms +``` + +That's _a lot_ better! Now it only takes 20 milliseconds to get the data, and we +only use about 344 MB of buffers (instead of the original 1.58 GB). The reason +this works is that now PostgreSQL no longer needs to apply a `Filter`, as the +index only contains `twitter` values that are not empty. + +Keep in mind that you shouldn't just add partial indexes every time you want to +optimise a query. Every index has to be updated for every write, and they may +require quite a bit of space, depending on the amount of indexed data. As a +result, first check if there are any existing indexes you may be able to reuse. +If there aren't any, check if you can perhaps slightly change an existing one to +fit both the existing and new queries. Only add a new index if none of the +existing indexes can be used in any way. + +When comparing execution plans, don't take timing as the only important metric. +Good timing is the main goal of any optimization, but it can be too volatile to +be used for comparison (for example, it depends a lot on the state of cache). +When optimizing a query, we usually need to reduce the amount of data we're +dealing with. Indexes are the way to work with fewer pages (buffers) to get the +result, so, during optimization, look at the number of buffers used (read and hit), +and work on reducing these numbers. Reduced timing is the consequence of reduced +buffer numbers. [Database Lab Engine](#database-lab-engine) guarantees that the plan is structurally +identical to production (and overall number of buffers is the same as on production), +but difference in cache state and I/O speed may lead to different timings. + +## Queries that can't be optimised + +Now that we have seen how to optimise a query, let's look at another query that +we might not be able to optimise: + +```sql +EXPLAIN (ANALYZE, BUFFERS) +SELECT COUNT(*) +FROM projects +WHERE visibility_level IN (0, 20); +``` + +The output of `EXPLAIN (ANALYZE, BUFFERS)` is as follows: + +```sql +Aggregate (cost=922420.60..922420.61 rows=1 width=8) (actual time=3428.535..3428.535 rows=1 loops=1) + Buffers: shared hit=208846 + -> Seq Scan on projects (cost=0.00..908053.18 rows=5746969 width=0) (actual time=0.041..2987.606 rows=5746940 loops=1) + Filter: (visibility_level = ANY ('{0,20}'::integer[])) + Rows Removed by Filter: 65677 + Buffers: shared hit=208846 +Planning time: 2.861 ms +Execution time: 3428.596 ms +``` + +Looking at the output we see the following Filter: + +```sql +Filter: (visibility_level = ANY ('{0,20}'::integer[])) +Rows Removed by Filter: 65677 +``` + +Looking at the number of rows removed by the filter, we may be tempted to add an +index on `projects.visibility_level` to somehow turn this Sequential scan + +filter into an index-only scan. + +Unfortunately, doing so is unlikely to improve anything. Contrary to what some +might believe, an index being present _does not guarantee_ that PostgreSQL +actually uses it. For example, when doing a `SELECT * FROM projects` it is much +cheaper to just scan the entire table, instead of using an index and then +fetching data from the table. In such cases PostgreSQL may decide to not use an +index. + +Second, let's think for a moment what our query does: it gets all projects with +visibility level 0 or 20. In the above plan we can see this produces quite a lot +of rows (5,745,940), but how much is that relative to the total? Let's find out +by running the following query: + +```sql +SELECT visibility_level, count(*) AS amount +FROM projects +GROUP BY visibility_level +ORDER BY visibility_level ASC; +``` + +For GitLab.com this produces: + +```sql + visibility_level | amount +------------------+--------- + 0 | 5071325 + 10 | 65678 + 20 | 674801 +``` + +Here the total number of projects is 5,811,804, and 5,746,126 of those are of +level 0 or 20. That's 98% of the entire table! + +So no matter what we do, this query retrieves 98% of the entire table. Since +most time is spent doing exactly that, there isn't really much we can do to +improve this query, other than _not_ running it at all. + +What is important here is that while some may recommend to straight up add an +index the moment you see a sequential scan, it is _much more important_ to first +understand what your query does, how much data it retrieves, and so on. After +all, you can not optimise something you do not understand. + +### Cardinality and selectivity + +Earlier we saw that our query had to retrieve 98% of the rows in the table. +There are two terms commonly used for databases: cardinality, and selectivity. +Cardinality refers to the number of unique values in a particular column in a +table. + +Selectivity is the number of unique values produced by an operation (for example, an +index scan or filter), relative to the total number of rows. The higher the +selectivity, the more likely PostgreSQL is able to use an index. + +In the above example, there are only 3 unique values: 0, 10, and 20. This means +the cardinality is 3. The selectivity in turn is also very low: 0.0000003% (2 / +5,811,804), because our `Filter` only filters using two values (`0` and `20`). +With such a low selectivity value it's not surprising that PostgreSQL decides +using an index is not worth it, because it would produce almost no unique rows. + +## Rewriting queries + +So the above query can't really be optimised as-is, or at least not much. But +what if we slightly change the purpose of it? What if instead of retrieving all +projects with `visibility_level` 0 or 20, we retrieve those that a user +interacted with somehow? + +Fortunately, GitLab has an answer for this, and it's a table called +`user_interacted_projects`. This table has the following schema: + +```sql +Table "public.user_interacted_projects" + Column | Type | Modifiers +------------+---------+----------- + user_id | integer | not null + project_id | integer | not null +Indexes: + "index_user_interacted_projects_on_project_id_and_user_id" UNIQUE, btree (project_id, user_id) + "index_user_interacted_projects_on_user_id" btree (user_id) +Foreign-key constraints: + "fk_rails_0894651f08" FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE + "fk_rails_722ceba4f7" FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE +``` + +Let's rewrite our query to `JOIN` this table onto our projects, and get the +projects for a specific user: + +```sql +EXPLAIN ANALYZE +SELECT COUNT(*) +FROM projects +INNER JOIN user_interacted_projects ON user_interacted_projects.project_id = projects.id +WHERE projects.visibility_level IN (0, 20) +AND user_interacted_projects.user_id = 1; +``` + +What we do here is the following: + +1. Get our projects. +1. `INNER JOIN` `user_interacted_projects`, meaning we're only left with rows in + `projects` that have a corresponding row in `user_interacted_projects`. +1. Limit this to the projects with `visibility_level` of 0 or 20, and to + projects that the user with ID 1 interacted with. + +If we run this query we get the following plan: + +```sql + Aggregate (cost=871.03..871.04 rows=1 width=8) (actual time=9.763..9.763 rows=1 loops=1) + -> Nested Loop (cost=0.86..870.52 rows=203 width=0) (actual time=1.072..9.748 rows=143 loops=1) + -> Index Scan using index_user_interacted_projects_on_user_id on user_interacted_projects (cost=0.43..160.71 rows=205 width=4) (actual time=0.939..2.508 rows=145 loops=1) + Index Cond: (user_id = 1) + -> Index Scan using projects_pkey on projects (cost=0.43..3.45 rows=1 width=4) (actual time=0.049..0.050 rows=1 loops=145) + Index Cond: (id = user_interacted_projects.project_id) + Filter: (visibility_level = ANY ('{0,20}'::integer[])) + Rows Removed by Filter: 0 + Planning time: 2.614 ms + Execution time: 9.809 ms +``` + +Here it only took us just under 10 milliseconds to get the data. We can also see +we're retrieving far fewer projects: + +```sql +Index Scan using projects_pkey on projects (cost=0.43..3.45 rows=1 width=4) (actual time=0.049..0.050 rows=1 loops=145) + Index Cond: (id = user_interacted_projects.project_id) + Filter: (visibility_level = ANY ('{0,20}'::integer[])) + Rows Removed by Filter: 0 +``` + +Here we see we perform 145 loops (`loops=145`), with every loop producing 1 row +(`rows=1`). This is much less than before, and our query performs much better! + +If we look at the plan we also see our costs are very low: + +```sql +Index Scan using projects_pkey on projects (cost=0.43..3.45 rows=1 width=4) (actual time=0.049..0.050 rows=1 loops=145) +``` + +Here our cost is only 3.45, and it takes us 7.25 milliseconds to do so (0.05 * 145). +The next index scan is a bit more expensive: + +```sql +Index Scan using index_user_interacted_projects_on_user_id on user_interacted_projects (cost=0.43..160.71 rows=205 width=4) (actual time=0.939..2.508 rows=145 loops=1) +``` + +Here the cost is 160.71 (`cost=0.43..160.71`), taking about 2.5 milliseconds +(based on the output of `actual time=....`). + +The most expensive part here is the "Nested Loop" that acts upon the result of +these two index scans: + +```sql +Nested Loop (cost=0.86..870.52 rows=203 width=0) (actual time=1.072..9.748 rows=143 loops=1) +``` + +Here we had to perform 870.52 disk page fetches for 203 rows, 9.748 +milliseconds, producing 143 rows in a single loop. + +The key takeaway here is that sometimes you have to rewrite (parts of) a query +to make it better. Sometimes that means having to slightly change your feature +to accommodate for better performance. + +## What makes a bad plan + +This is a bit of a difficult question to answer, because the definition of "bad" +is relative to the problem you are trying to solve. However, some patterns are +best avoided in most cases, such as: + +- Sequential scans on large tables +- Filters that remove a lot of rows +- Performing a certain step that requires _a lot_ of + buffers (for example, an index scan for GitLab.com that requires more than 512 MB). + +As a general guideline, aim for a query that: + +1. Takes no more than 10 milliseconds. Our target time spent in SQL per request + is around 100 milliseconds, so every query should be as fast as possible. +1. Does not use an excessive number of buffers, relative to the workload. For + example, retrieving ten rows shouldn't require 1 GB of buffers. +1. Does not spend a long amount of time performing disk IO operations. The + setting `track_io_timing` must be enabled for this data to be included in the + output of `EXPLAIN ANALYZE`. +1. Applies a `LIMIT` when retrieving rows without aggregating them, such as + `SELECT * FROM users`. +1. Doesn't use a `Filter` to filter out too many rows, especially if the query + does not use a `LIMIT` to limit the number of returned rows. Filters can + usually be removed by adding a (partial) index. + +These are _guidelines_ and not hard requirements, as different needs may require +different queries. The only _rule_ is that you _must always measure_ your query +(preferably using a production-like database) using `EXPLAIN (ANALYZE, BUFFERS)` +and related tools such as: + +- [`explain.depesz.com`](https://explain.depesz.com/). +- [`explain.dalibo.com/`](https://explain.dalibo.com/). + +## Producing query plans + +There are a few ways to get the output of a query plan. Of course you +can directly run the `EXPLAIN` query in the `psql` console, or you can +follow one of the other options below. + +### Database Lab Engine + +GitLab team members can use [Database Lab Engine](https://gitlab.com/postgres-ai/database-lab), and the companion +SQL optimization tool - [Joe Bot](https://gitlab.com/postgres-ai/joe). + +Database Lab Engine provides developers with their own clone of the production database, while Joe Bot helps with exploring execution plans. + +Joe Bot is available in the [`#database-lab`](https://gitlab.slack.com/archives/CLJMDRD8C) channel on Slack, +and through its [web interface](https://console.postgres.ai/gitlab/joe-instances). + +With Joe Bot you can execute DDL statements (like creating indexes, tables, and columns) and get query plans for `SELECT`, `UPDATE`, and `DELETE` statements. + +For example, in order to test new index on a column that is not existing on production yet, you can do the following: + +Create the column: + +```sql +exec ALTER TABLE projects ADD COLUMN last_at timestamp without time zone +``` + +Create the index: + +```sql +exec CREATE INDEX index_projects_last_activity ON projects (last_activity_at) WHERE last_activity_at IS NOT NULL +``` + +Analyze the table to update its statistics: + +```sql +exec ANALYZE projects +``` + +Get the query plan: + +```sql +explain SELECT * FROM projects WHERE last_activity_at < CURRENT_DATE +``` + +Once done you can rollback your changes: + +```sql +reset +``` + +For more information about the available options, run: + +```sql +help +``` + +The web interface comes with the following execution plan visualizers included: + +- [Depesz](https://explain.depesz.com/) +- [PEV2](https://github.com/dalibo/pev2) +- [FlameGraph](https://github.com/mgartner/pg_flame) + +#### Tips & Tricks + +The database connection is now maintained during your whole session, so you can use `exec set ...` for any session variables (such as `enable_seqscan` or `work_mem`). These settings are applied to all subsequent commands until you reset them. For example you can disable parallel queries with + +```sql +exec SET max_parallel_workers_per_gather = 0 +``` + +### Rails console + +Using the [`activerecord-explain-analyze`](https://github.com/6/activerecord-explain-analyze) +you can directly generate the query plan from the Rails console: + +```ruby +pry(main)> require 'activerecord-explain-analyze' +=> true +pry(main)> Project.where('build_timeout > ?', 3600).explain(analyze: true) + Project Load (1.9ms) SELECT "projects".* FROM "projects" WHERE (build_timeout > 3600) + ↳ (pry):12 +=> EXPLAIN for: SELECT "projects".* FROM "projects" WHERE (build_timeout > 3600) +Seq Scan on public.projects (cost=0.00..2.17 rows=1 width=742) (actual time=0.040..0.041 rows=0 loops=1) + Output: id, name, path, description, created_at, updated_at, creator_id, namespace_id, ... + Filter: (projects.build_timeout > 3600) + Rows Removed by Filter: 14 + Buffers: shared hit=2 +Planning time: 0.411 ms +Execution time: 0.113 ms +``` + +### ChatOps + +[GitLab team members can also use our ChatOps solution, available in Slack using the +`/chatops` slash command](../chatops_on_gitlabcom.md). + +NOTE: +While ChatOps is still available, the recommended way to generate execution plans is to use [Database Lab Engine](#database-lab-engine). + +You can use ChatOps to get a query plan by running the following: + +```sql +/chatops run explain SELECT COUNT(*) FROM projects WHERE visibility_level IN (0, 20) +``` + +Visualising the plan using <https://explain.depesz.com/> is also supported: + +```sql +/chatops run explain --visual SELECT COUNT(*) FROM projects WHERE visibility_level IN (0, 20) +``` + +Quoting the query is not necessary. + +For more information about the available options, run: + +```sql +/chatops run explain --help +``` + +## Further reading + +A more extensive guide on understanding query plans can be found in +the [presentation](https://public.dalibo.com/exports/conferences/_archives/_2012/201211_explain/understanding_explain.pdf) +from [Dalibo.org](https://www.dalibo.com/en/). + +Depesz's blog also has a good [section](https://www.depesz.com/tag/unexplainable/) dedicated to query plans. diff --git a/doc/development/database_review.md b/doc/development/database_review.md index 37643f04db7..7f7b8189086 100644 --- a/doc/development/database_review.md +++ b/doc/development/database_review.md @@ -179,7 +179,7 @@ Include in the MR description: - [explain.depesz.com](https://explain.depesz.com) or [explain.dalibo.com](https://explain.dalibo.com): Paste both the plan and the query used in the form. - When providing query plans, make sure it hits enough data: - You can use a GitLab production replica to test your queries on a large scale, - through the `#database-lab` Slack channel or through [ChatOps](understanding_explain_plans.md#chatops). + through the `#database-lab` Slack channel or through [ChatOps](database/understanding_explain_plans.md#chatops). - Usually, the `gitlab-org` namespace (`namespace_id = 9970`) and the `gitlab-org/gitlab-foss` (`project_id = 13083`) or the `gitlab-org/gitlab` (`project_id = 278964`) projects provide enough data to serve as a good example. @@ -187,7 +187,7 @@ Include in the MR description: - If your queries belong to a new feature in GitLab.com and thus they don't return data in production: - You may analyze the query and to provide the plan from a local environment. - `#database-lab` and [postgres.ai](https://postgres.ai/) both allow updates to data (`exec UPDATE issues SET ...`) and creation of new tables and columns (`exec ALTER TABLE issues ADD COLUMN ...`). - - More information on how to find the number of actual returned records in [Understanding EXPLAIN plans](understanding_explain_plans.md) + - More information on how to find the number of actual returned records in [Understanding EXPLAIN plans](database/understanding_explain_plans.md) - For query changes, it is best to provide both the SQL queries along with the plan _before_ and _after_ the change. This helps spot differences quickly. - Include data that shows the performance improvement, preferably in @@ -200,7 +200,7 @@ Include in the MR description: #### Preparation when adding tables -- Order columns based on the [Ordering Table Columns](ordering_table_columns.md) guidelines. +- Order columns based on the [Ordering Table Columns](database/ordering_table_columns.md) guidelines. - Add foreign keys to any columns pointing to data in other tables, including [an index](migration_style_guide.md#adding-foreign-key-constraints). - Add indexes for fields that are used in statements such as `WHERE`, `ORDER BY`, `GROUP BY`, and `JOIN`s. - New tables and columns are not necessarily risky, but over time some access patterns are inherently @@ -225,7 +225,7 @@ Include in the MR description: - Consider [access patterns and data layout](database/layout_and_access_patterns.md) if new tables or columns are added. - Review migrations follow [database migration style guide](migration_style_guide.md), for example - - [Check ordering of columns](ordering_table_columns.md) + - [Check ordering of columns](database/ordering_table_columns.md) - [Check indexes are present for foreign keys](migration_style_guide.md#adding-foreign-key-constraints) - Ensure that migrations execute in a transaction or only contain concurrent index/foreign key helpers (with transactions disabled) @@ -256,7 +256,7 @@ Include in the MR description: - Check migrations are reversible and implement a `#down` method - Check new table migrations: - Are the stated access patterns and volume reasonable? Do the assumptions they're based on seem sound? Do these patterns pose risks to stability? - - Are the columns [ordered to conserve space](ordering_table_columns.md)? + - Are the columns [ordered to conserve space](database/ordering_table_columns.md)? - Are there foreign keys for references to other tables? - Check data migrations: - Establish a time estimate for execution on GitLab.com. @@ -267,10 +267,10 @@ Include in the MR description: - Check for any overly complex queries and queries the author specifically points out for review (if any) - If not present, ask the author to provide SQL queries and query plans - (for example, by using [ChatOps](understanding_explain_plans.md#chatops) or direct + (for example, by using [ChatOps](database/understanding_explain_plans.md#chatops) or direct database access) - For given queries, review parameters regarding data distribution - - [Check query plans](understanding_explain_plans.md) and suggest improvements + - [Check query plans](database/understanding_explain_plans.md) and suggest improvements to queries (changing the query, schema or adding indexes and similar) - General guideline is for queries to come in below [100ms execution time](query_performance.md#timing-guidelines-for-queries) - Avoid N+1 problems and minimize the [query count](merge_request_performance_guidelines.md#query-counts). diff --git a/doc/development/ordering_table_columns.md b/doc/development/ordering_table_columns.md index 7cd3d4fb208..b665cb0d4c7 100644 --- a/doc/development/ordering_table_columns.md +++ b/doc/development/ordering_table_columns.md @@ -1,152 +1,11 @@ --- -stage: Data Stores -group: Database -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'database/ordering_table_columns.md' +remove_date: '2022-11-04' --- -# Ordering Table Columns in PostgreSQL +This document was moved to [another location](database/ordering_table_columns.md). -For GitLab we require that columns of new tables are ordered to use the -least amount of space. An easy way of doing this is to order them based on the -type size in descending order with variable sizes (`text`, `varchar`, arrays, -`json`, `jsonb`, and so on) at the end. - -Similar to C structures the space of a table is influenced by the order of -columns. This is because the size of columns is aligned depending on the type of -the following column. Let's consider an example: - -- `id` (integer, 4 bytes) -- `name` (text, variable) -- `user_id` (integer, 4 bytes) - -The first column is a 4-byte integer. The next is text of variable length. The -`text` data type requires 1-word alignment, and on 64-bit platform, 1 word is 8 -bytes. To meet the alignment requirements, four zeros are to be added right -after the first column, so `id` occupies 4 bytes, then 4 bytes of alignment -padding, and only next `name` is being stored. Therefore, in this case, 8 bytes -are spent for storing a 4-byte integer. - -The space between rows is also subject to alignment padding. The `user_id` -column takes only 4 bytes, and on 64-bit platform, 4 zeroes are added for -alignment padding, to allow storing the next row beginning with the "clear" word. - -As a result, the actual size of each column would be (omitting variable length -data and 24-byte tuple header): 8 bytes, variable, 8 bytes. This means that -each row requires at least 16 bytes for the two 4-byte integers. If a table -has a few rows this is not an issue. However, once you start storing millions of -rows you can save space by using a different order. For the above example, the -ideal column order would be the following: - -- `id` (integer, 4 bytes) -- `user_id` (integer, 4 bytes) -- `name` (text, variable) - -or - -- `name` (text, variable) -- `id` (integer, 4 bytes) -- `user_id` (integer, 4 bytes) - -In these examples, the `id` and `user_id` columns are packed together, which -means we only need 8 bytes to store _both_ of them. This in turn means each row -requires 8 bytes less space. - -Since Ruby on Rails 5.1, the default data type for IDs is `bigint`, which uses 8 bytes. -We are using `integer` in the examples to showcase a more realistic reordering scenario. - -## Type Sizes - -While the [PostgreSQL documentation](https://www.postgresql.org/docs/current/datatype.html) contains plenty -of information we list the sizes of common types here so it's easier to -look them up. Here "word" refers to the word size, which is 4 bytes for a 32 -bits platform and 8 bytes for a 64 bits platform. - -| Type | Size | Alignment needed | -|:-----------------|:-------------------------------------|:-----------| -| `smallint` | 2 bytes | 1 word | -| `integer` | 4 bytes | 1 word | -| `bigint` | 8 bytes | 8 bytes | -| `real` | 4 bytes | 1 word | -| `double precision` | 8 bytes | 8 bytes | -| `boolean` | 1 byte | not needed | -| `text` / `string` | variable, 1 byte plus the data | 1 word | -| `bytea` | variable, 1 or 4 bytes plus the data | 1 word | -| `timestamp` | 8 bytes | 8 bytes | -| `timestamptz` | 8 bytes | 8 bytes | -| `date` | 4 bytes | 1 word | - -A "variable" size means the actual size depends on the value being stored. If -PostgreSQL determines this can be embedded directly into a row it may do so, but -for very large values it stores the data externally and store a pointer (of -1 word in size) in the column. Because of this variable sized columns should -always be at the end of a table. - -## Real Example - -Let's use the `events` table as an example, which currently has the following -layout: - -| Column | Type | Size | -|:--------------|:----------------------------|:---------| -| `id` | integer | 4 bytes | -| `target_type` | character varying | variable | -| `target_id` | integer | 4 bytes | -| `title` | character varying | variable | -| `data` | text | variable | -| `project_id` | integer | 4 bytes | -| `created_at` | timestamp without time zone | 8 bytes | -| `updated_at` | timestamp without time zone | 8 bytes | -| `action` | integer | 4 bytes | -| `author_id` | integer | 4 bytes | - -After adding padding to align the columns this would translate to columns being -divided into fixed size chunks as follows: - -| Chunk Size | Columns | -|:-----------|:----------------------| -| 8 bytes | `id` | -| variable | `target_type` | -| 8 bytes | `target_id` | -| variable | `title` | -| variable | `data` | -| 8 bytes | `project_id` | -| 8 bytes | `created_at` | -| 8 bytes | `updated_at` | -| 8 bytes | `action`, `author_id` | - -This means that excluding the variable sized data and tuple header, we need at -least 8 * 6 = 48 bytes per row. - -We can optimise this by using the following column order instead: - -| Column | Type | Size | -|:--------------|:----------------------------|:---------| -| `created_at` | timestamp without time zone | 8 bytes | -| `updated_at` | timestamp without time zone | 8 bytes | -| `id` | integer | 4 bytes | -| `target_id` | integer | 4 bytes | -| `project_id` | integer | 4 bytes | -| `action` | integer | 4 bytes | -| `author_id` | integer | 4 bytes | -| `target_type` | character varying | variable | -| `title` | character varying | variable | -| `data` | text | variable | - -This would produce the following chunks: - -| Chunk Size | Columns | -|:-----------|:-----------------------| -| 8 bytes | `created_at` | -| 8 bytes | `updated_at` | -| 8 bytes | `id`, `target_id` | -| 8 bytes | `project_id`, `action` | -| 8 bytes | `author_id` | -| variable | `target_type` | -| variable | `title` | -| variable | `data` | - -Here we only need 40 bytes per row excluding the variable sized data and 24-byte -tuple header. 8 bytes being saved may not sound like much, but for tables as -large as the `events` table it does begin to matter. For example, when storing -80 000 000 rows this translates to a space saving of at least 610 MB, all by -just changing the order of a few columns. +<!-- This redirect file can be deleted after <2022-11-04>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/polymorphic_associations.md b/doc/development/polymorphic_associations.md index bbeaab40a90..6b9158b8408 100644 --- a/doc/development/polymorphic_associations.md +++ b/doc/development/polymorphic_associations.md @@ -1,152 +1,11 @@ --- -stage: none -group: unassigned -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'database/polymorphic_associations.md' +remove_date: '2022-11-04' --- -# Polymorphic Associations +This document was moved to [another location](database/polymorphic_associations.md). -**Summary:** always use separate tables instead of polymorphic associations. - -Rails makes it possible to define so called "polymorphic associations". This -usually works by adding two columns to a table: a target type column, and a -target ID. For example, at the time of writing we have such a setup for -`members` with the following columns: - -- `source_type`: a string defining the model to use, can be either `Project` or - `Namespace`. -- `source_id`: the ID of the row to retrieve based on `source_type`. For - example, when `source_type` is `Project` then `source_id` contains a - project ID. - -While such a setup may appear to be useful, it comes with many drawbacks; enough -that you should avoid this at all costs. - -## Space Wasted - -Because this setup relies on string values to determine the model to use, it -wastes a lot of space. For example, for `Project` and `Namespace` the -maximum size is 9 bytes, plus 1 extra byte for every string when using -PostgreSQL. While this may only be 10 bytes per row, given enough tables and -rows using such a setup we can end up wasting quite a bit of disk space and -memory (for any indexes). - -## Indexes - -Because our associations are broken up into two columns this may result in -requiring composite indexes for queries to be performed efficiently. While -composite indexes are not wrong at all, they can be tricky to set up as the -ordering of columns in these indexes is important to ensure optimal performance. - -## Consistency - -One really big problem with polymorphic associations is being unable to enforce -data consistency on the database level using foreign keys. For consistency to be -enforced on the database level one would have to write their own foreign key -logic to support polymorphic associations. - -Enforcing consistency on the database level is absolutely crucial for -maintaining a healthy environment, and thus is another reason to avoid -polymorphic associations. - -## Query Overhead - -When using polymorphic associations you always need to filter using both -columns. For example, you may end up writing a query like this: - -```sql -SELECT * -FROM members -WHERE source_type = 'Project' -AND source_id = 13083; -``` - -Here PostgreSQL can perform the query quite efficiently if both columns are -indexed. As the query gets more complex, it may not be able to use these -indexes effectively. - -## Mixed Responsibilities - -Similar to functions and classes, a table should have a single responsibility: -storing data with a certain set of pre-defined columns. When using polymorphic -associations, you are storing different types of data (possibly with -different columns set) in the same table. - -## The Solution - -Fortunately, there is a solution to these problems: use a -separate table for every type you would otherwise store in the same table. Using -a separate table allows you to use everything a database may provide to ensure -consistency and query data efficiently, without any additional application logic -being necessary. - -Let's say you have a `members` table storing both approved and pending members, -for both projects and groups, and the pending state is determined by the column -`requested_at` being set or not. Schema wise such a setup can lead to various -columns only being set for certain rows, wasting space. It's also possible that -certain indexes are only set for certain rows, again wasting space. Finally, -querying such a table requires less than ideal queries. For example: - -```sql -SELECT * -FROM members -WHERE requested_at IS NULL -AND source_type = 'GroupMember' -AND source_id = 4 -``` - -Instead such a table should be broken up into separate tables. For example, you -may end up with 4 tables in this case: - -- project_members -- group_members -- pending_project_members -- pending_group_members - -This makes querying data trivial. For example, to get the members of a group -you'd run: - -```sql -SELECT * -FROM group_members -WHERE group_id = 4 -``` - -To get all the pending members of a group in turn you'd run: - -```sql -SELECT * -FROM pending_group_members -WHERE group_id = 4 -``` - -If you want to get both you can use a `UNION`, though you need to be explicit -about what columns you want to `SELECT` as otherwise the result set uses the -columns of the first query. For example: - -```sql -SELECT id, 'Group' AS target_type, group_id AS target_id -FROM group_members - -UNION ALL - -SELECT id, 'Project' AS target_type, project_id AS target_id -FROM project_members -``` - -The above example is perhaps a bit silly, but it shows that there's nothing -stopping you from merging the data together and presenting it on the same page. -Selecting columns explicitly can also speed up queries as the database has to do -less work to get the data (compared to selecting all columns, even ones you're -not using). - -Our schema also becomes easier. No longer do we need to both store and index the -`source_type` column, we can define foreign keys easily, and we don't need to -filter rows using the `IS NULL` condition. - -To summarize: using separate tables allows us to use foreign keys effectively, -create indexes only where necessary, conserve space, query data more -efficiently, and scale these tables more easily (for example, by storing them on -separate disks). A nice side effect of this is that code can also become easier, -as a single model isn't responsible for handling different kinds of -data. +<!-- This redirect file can be deleted after <2022-11-04>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/query_performance.md b/doc/development/query_performance.md index 4fe27d42c38..139dc025190 100644 --- a/doc/development/query_performance.md +++ b/doc/development/query_performance.md @@ -36,9 +36,9 @@ The first time a query is made, it is made on a "cold cache". Meaning it needs to read from disk. If you run the query again, the data can be read from the cache, or what PostgreSQL calls shared buffers. This is the "warm cache" query. -When analyzing an [`EXPLAIN` plan](understanding_explain_plans.md), you can see +When analyzing an [`EXPLAIN` plan](database/understanding_explain_plans.md), you can see the difference not only in the timing, but by looking at the output for `Buffers` -by running your explain with `EXPLAIN(analyze, buffers)`. [Database Lab](understanding_explain_plans.md#database-lab-engine) +by running your explain with `EXPLAIN(analyze, buffers)`. [Database Lab](database/understanding_explain_plans.md#database-lab-engine) automatically includes these options. If you are making a warm cache query, you see only the `shared hits`. diff --git a/doc/development/swapping_tables.md b/doc/development/swapping_tables.md index efb481ccf35..eaa6568dc36 100644 --- a/doc/development/swapping_tables.md +++ b/doc/development/swapping_tables.md @@ -1,51 +1,11 @@ --- -stage: Data Stores -group: Database -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'database/swapping_tables.md' +remove_date: '2022-11-04' --- -# Swapping Tables +This document was moved to [another location](database/swapping_tables.md). -Sometimes you need to replace one table with another. For example, when -migrating data in a very large table it's often better to create a copy of the -table and insert & migrate the data into this new table in the background. - -Let's say you want to swap the table `events` with `events_for_migration`. In -this case you need to follow 3 steps: - -1. Rename `events` to `events_temporary` -1. Rename `events_for_migration` to `events` -1. Rename `events_temporary` to `events_for_migration` - -Rails allows you to do this using the `rename_table` method: - -```ruby -rename_table :events, :events_temporary -rename_table :events_for_migration, :events -rename_table :events_temporary, :events_for_migration -``` - -This does not require any downtime as long as the 3 `rename_table` calls are -executed in the _same_ database transaction. Rails by default uses database -transactions for migrations, but if it doesn't you need to start one -manually: - -```ruby -Event.transaction do - rename_table :events, :events_temporary - rename_table :events_for_migration, :events - rename_table :events_temporary, :events_for_migration -end -``` - -Once swapped you _have to_ reset the primary key of the new table. For -PostgreSQL you can use the `reset_pk_sequence!` method like so: - -```ruby -reset_pk_sequence!('events') -``` - -Failure to reset the primary keys results in newly created rows starting -with an ID value of 1. Depending on the existing data this can then lead to -duplicate key constraints from popping up, preventing users from creating new -data. +<!-- This redirect file can be deleted after <2022-11-04>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/understanding_explain_plans.md b/doc/development/understanding_explain_plans.md index 17fcd5b3e88..72c3df11a96 100644 --- a/doc/development/understanding_explain_plans.md +++ b/doc/development/understanding_explain_plans.md @@ -1,829 +1,11 @@ --- -stage: Data Stores -group: Database -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'database/understanding_explain_plans.md' +remove_date: '2022-11-04' --- -# Understanding EXPLAIN plans +This document was moved to [another location](database/understanding_explain_plans.md). -PostgreSQL allows you to obtain query plans using the `EXPLAIN` command. This -command can be invaluable when trying to determine how a query performs. -You can use this command directly in your SQL query, as long as the query starts -with it: - -```sql -EXPLAIN -SELECT COUNT(*) -FROM projects -WHERE visibility_level IN (0, 20); -``` - -When running this on GitLab.com, we are presented with the following output: - -```sql -Aggregate (cost=922411.76..922411.77 rows=1 width=8) - -> Seq Scan on projects (cost=0.00..908044.47 rows=5746914 width=0) - Filter: (visibility_level = ANY ('{0,20}'::integer[])) -``` - -When using _just_ `EXPLAIN`, PostgreSQL does not actually execute our query, -instead it produces an _estimated_ execution plan based on the available -statistics. This means the actual plan can differ quite a bit. Fortunately, -PostgreSQL provides us with the option to execute the query as well. To do so, -we need to use `EXPLAIN ANALYZE` instead of just `EXPLAIN`: - -```sql -EXPLAIN ANALYZE -SELECT COUNT(*) -FROM projects -WHERE visibility_level IN (0, 20); -``` - -This produces: - -```sql -Aggregate (cost=922420.60..922420.61 rows=1 width=8) (actual time=3428.535..3428.535 rows=1 loops=1) - -> Seq Scan on projects (cost=0.00..908053.18 rows=5746969 width=0) (actual time=0.041..2987.606 rows=5746940 loops=1) - Filter: (visibility_level = ANY ('{0,20}'::integer[])) - Rows Removed by Filter: 65677 -Planning time: 2.861 ms -Execution time: 3428.596 ms -``` - -As we can see this plan is quite different, and includes a lot more data. Let's -discuss this step by step. - -Because `EXPLAIN ANALYZE` executes the query, care should be taken when using a -query that writes data or might time out. If the query modifies data, -consider wrapping it in a transaction that rolls back automatically like so: - -```sql -BEGIN; -EXPLAIN ANALYZE -DELETE FROM users WHERE id = 1; -ROLLBACK; -``` - -The `EXPLAIN` command also takes additional options, such as `BUFFERS`: - -```sql -EXPLAIN (ANALYZE, BUFFERS) -SELECT COUNT(*) -FROM projects -WHERE visibility_level IN (0, 20); -``` - -This then produces: - -```sql -Aggregate (cost=922420.60..922420.61 rows=1 width=8) (actual time=3428.535..3428.535 rows=1 loops=1) - Buffers: shared hit=208846 - -> Seq Scan on projects (cost=0.00..908053.18 rows=5746969 width=0) (actual time=0.041..2987.606 rows=5746940 loops=1) - Filter: (visibility_level = ANY ('{0,20}'::integer[])) - Rows Removed by Filter: 65677 - Buffers: shared hit=208846 -Planning time: 2.861 ms -Execution time: 3428.596 ms -``` - -For more information, refer to the official -[`EXPLAIN` documentation](https://www.postgresql.org/docs/current/sql-explain.html) -and [using `EXPLAIN` guide](https://www.postgresql.org/docs/current/using-explain.html). - -## Nodes - -Every query plan consists of nodes. Nodes can be nested, and are executed from -the inside out. This means that the innermost node is executed before an outer -node. This can be best thought of as nested function calls, returning their -results as they unwind. For example, a plan starting with an `Aggregate` -followed by a `Nested Loop`, followed by an `Index Only scan` can be thought of -as the following Ruby code: - -```ruby -aggregate( - nested_loop( - index_only_scan() - index_only_scan() - ) -) -``` - -Nodes are indicated using a `->` followed by the type of node taken. For -example: - -```sql -Aggregate (cost=922411.76..922411.77 rows=1 width=8) - -> Seq Scan on projects (cost=0.00..908044.47 rows=5746914 width=0) - Filter: (visibility_level = ANY ('{0,20}'::integer[])) -``` - -Here the first node executed is `Seq scan on projects`. The `Filter:` is an -additional filter applied to the results of the node. A filter is very similar -to Ruby's `Array#select`: it takes the input rows, applies the filter, and -produces a new list of rows. After the node is done, we perform the `Aggregate` -above it. - -Nested nodes look like this: - -```sql -Aggregate (cost=176.97..176.98 rows=1 width=8) (actual time=0.252..0.252 rows=1 loops=1) - Buffers: shared hit=155 - -> Nested Loop (cost=0.86..176.75 rows=87 width=0) (actual time=0.035..0.249 rows=36 loops=1) - Buffers: shared hit=155 - -> Index Only Scan using users_pkey on users users_1 (cost=0.43..4.95 rows=87 width=4) (actual time=0.029..0.123 rows=36 loops=1) - Index Cond: (id < 100) - Heap Fetches: 0 - -> Index Only Scan using users_pkey on users (cost=0.43..1.96 rows=1 width=4) (actual time=0.003..0.003 rows=1 loops=36) - Index Cond: (id = users_1.id) - Heap Fetches: 0 -Planning time: 2.585 ms -Execution time: 0.310 ms -``` - -Here we first perform two separate "Index Only" scans, followed by performing a -"Nested Loop" on the result of these two scans. - -## Node statistics - -Each node in a plan has a set of associated statistics, such as the cost, the -number of rows produced, the number of loops performed, and more. For example: - -```sql -Seq Scan on projects (cost=0.00..908044.47 rows=5746914 width=0) -``` - -Here we can see that our cost ranges from `0.00..908044.47` (we cover this in -a moment), and we estimate (since we're using `EXPLAIN` and not `EXPLAIN -ANALYZE`) a total of 5,746,914 rows to be produced by this node. The `width` -statistics describes the estimated width of each row, in bytes. - -The `costs` field specifies how expensive a node was. The cost is measured in -arbitrary units determined by the query planner's cost parameters. What -influences the costs depends on a variety of settings, such as `seq_page_cost`, -`cpu_tuple_cost`, and various others. -The format of the costs field is as follows: - -```sql -STARTUP COST..TOTAL COST -``` - -The startup cost states how expensive it was to start the node, with the total -cost describing how expensive the entire node was. In general: the greater the -values, the more expensive the node. - -When using `EXPLAIN ANALYZE`, these statistics also include the actual time -(in milliseconds) spent, and other runtime statistics (for example, the actual number of -produced rows): - -```sql -Seq Scan on projects (cost=0.00..908053.18 rows=5746969 width=0) (actual time=0.041..2987.606 rows=5746940 loops=1) -``` - -Here we can see we estimated 5,746,969 rows to be returned, but in reality we -returned 5,746,940 rows. We can also see that _just_ this sequential scan took -2.98 seconds to run. - -Using `EXPLAIN (ANALYZE, BUFFERS)` also gives us information about the -number of rows removed by a filter, the number of buffers used, and more. For -example: - -```sql -Seq Scan on projects (cost=0.00..908053.18 rows=5746969 width=0) (actual time=0.041..2987.606 rows=5746940 loops=1) - Filter: (visibility_level = ANY ('{0,20}'::integer[])) - Rows Removed by Filter: 65677 - Buffers: shared hit=208846 -``` - -Here we can see that our filter has to remove 65,677 rows, and that we use -208,846 buffers. Each buffer in PostgreSQL is 8 KB (8192 bytes), meaning our -above node uses *1.6 GB of buffers*. That's a lot! - -Keep in mind that some statistics are per-loop averages, while others are total values: - -| Field name | Value type | -| --- | --- | -| Actual Total Time | per-loop average | -| Actual Rows | per-loop average | -| Buffers Shared Hit | total value | -| Buffers Shared Read | total value | -| Buffers Shared Dirtied | total value | -| Buffers Shared Written | total value | -| I/O Read Time | total value | -| I/O Read Write | total value | - -For example: - -```sql - -> Index Scan using users_pkey on public.users (cost=0.43..3.44 rows=1 width=1318) (actual time=0.025..0.025 rows=1 loops=888) - Index Cond: (users.id = issues.author_id) - Buffers: shared hit=3543 read=9 - I/O Timings: read=17.760 write=0.000 -``` - -Here we can see that this node used 3552 buffers (3543 + 9), returned 888 rows (`888 * 1`), and the actual duration was 22.2 milliseconds (`888 * 0.025`). -17.76 milliseconds of the total duration was spent in reading from disk, to retrieve data that was not in the cache. - -## Node types - -There are quite a few different types of nodes, so we only cover some of the -more common ones here. - -A full list of all the available nodes and their descriptions can be found in -the [PostgreSQL source file `plannodes.h`](https://gitlab.com/postgres/postgres/blob/master/src/include/nodes/plannodes.h). -pgMustard's [EXPLAIN docs](https://www.pgmustard.com/docs/explain) also offer detailed look into nodes and their fields. - -### Seq Scan - -A sequential scan over (a chunk of) a database table. This is like using -`Array#each`, but on a database table. Sequential scans can be quite slow when -retrieving lots of rows, so it's best to avoid these for large tables. - -### Index Only Scan - -A scan on an index that did not require fetching anything from the table. In -certain cases an index only scan may still fetch data from the table, in this -case the node includes a `Heap Fetches:` statistic. - -### Index Scan - -A scan on an index that required retrieving some data from the table. - -### Bitmap Index Scan and Bitmap Heap scan - -Bitmap scans fall between sequential scans and index scans. These are typically -used when we would read too much data from an index scan, but too little to -perform a sequential scan. A bitmap scan uses what is known as a [bitmap -index](https://en.wikipedia.org/wiki/Bitmap_index) to perform its work. - -The [source code of PostgreSQL](https://gitlab.com/postgres/postgres/blob/REL_11_STABLE/src/include/nodes/plannodes.h#L441) -states the following on bitmap scans: - -> Bitmap Index Scan delivers a bitmap of potential tuple locations; it does not -> access the heap itself. The bitmap is used by an ancestor Bitmap Heap Scan -> node, possibly after passing through intermediate Bitmap And and/or Bitmap Or -> nodes to combine it with the results of other Bitmap Index Scans. - -### Limit - -Applies a `LIMIT` on the input rows. - -### Sort - -Sorts the input rows as specified using an `ORDER BY` statement. - -### Nested Loop - -A nested loop executes its child nodes for every row produced by a node that -precedes it. For example: - -```sql --> Nested Loop (cost=0.86..176.75 rows=87 width=0) (actual time=0.035..0.249 rows=36 loops=1) - Buffers: shared hit=155 - -> Index Only Scan using users_pkey on users users_1 (cost=0.43..4.95 rows=87 width=4) (actual time=0.029..0.123 rows=36 loops=1) - Index Cond: (id < 100) - Heap Fetches: 0 - -> Index Only Scan using users_pkey on users (cost=0.43..1.96 rows=1 width=4) (actual time=0.003..0.003 rows=1 loops=36) - Index Cond: (id = users_1.id) - Heap Fetches: 0 -``` - -Here the first child node (`Index Only Scan using users_pkey on users users_1`) -produces 36 rows, and is executed once (`rows=36 loops=1`). The next node -produces 1 row (`rows=1`), but is repeated 36 times (`loops=36`). This is -because the previous node produced 36 rows. - -This means that nested loops can quickly slow the query down if the various -child nodes keep producing many rows. - -## Optimising queries - -With that out of the way, let's see how we can optimise a query. Let's use the -following query as an example: - -```sql -SELECT COUNT(*) -FROM users -WHERE twitter != ''; -``` - -This query counts the number of users that have a Twitter profile set. -Let's run this using `EXPLAIN (ANALYZE, BUFFERS)`: - -```sql -EXPLAIN (ANALYZE, BUFFERS) -SELECT COUNT(*) -FROM users -WHERE twitter != ''; -``` - -This produces the following plan: - -```sql -Aggregate (cost=845110.21..845110.22 rows=1 width=8) (actual time=1271.157..1271.158 rows=1 loops=1) - Buffers: shared hit=202662 - -> Seq Scan on users (cost=0.00..844969.99 rows=56087 width=0) (actual time=0.019..1265.883 rows=51833 loops=1) - Filter: ((twitter)::text <> ''::text) - Rows Removed by Filter: 2487813 - Buffers: shared hit=202662 -Planning time: 0.390 ms -Execution time: 1271.180 ms -``` - -From this query plan we can see the following: - -1. We need to perform a sequential scan on the `users` table. -1. This sequential scan filters out 2,487,813 rows using a `Filter`. -1. We use 202,622 buffers, which equals 1.58 GB of memory. -1. It takes us 1.2 seconds to do all of this. - -Considering we are just counting users, that's quite expensive! - -Before we start making any changes, let's see if there are any existing indexes -on the `users` table that we might be able to use. We can obtain this -information by running `\d users` in a `psql` console, then scrolling down to -the `Indexes:` section: - -```sql -Indexes: - "users_pkey" PRIMARY KEY, btree (id) - "index_users_on_confirmation_token" UNIQUE, btree (confirmation_token) - "index_users_on_email" UNIQUE, btree (email) - "index_users_on_reset_password_token" UNIQUE, btree (reset_password_token) - "index_users_on_static_object_token" UNIQUE, btree (static_object_token) - "index_users_on_unlock_token" UNIQUE, btree (unlock_token) - "index_on_users_name_lower" btree (lower(name::text)) - "index_users_on_accepted_term_id" btree (accepted_term_id) - "index_users_on_admin" btree (admin) - "index_users_on_created_at" btree (created_at) - "index_users_on_email_trigram" gin (email gin_trgm_ops) - "index_users_on_feed_token" btree (feed_token) - "index_users_on_group_view" btree (group_view) - "index_users_on_incoming_email_token" btree (incoming_email_token) - "index_users_on_managing_group_id" btree (managing_group_id) - "index_users_on_name" btree (name) - "index_users_on_name_trigram" gin (name gin_trgm_ops) - "index_users_on_public_email" btree (public_email) WHERE public_email::text <> ''::text - "index_users_on_state" btree (state) - "index_users_on_state_and_user_type" btree (state, user_type) - "index_users_on_unconfirmed_email" btree (unconfirmed_email) WHERE unconfirmed_email IS NOT NULL - "index_users_on_user_type" btree (user_type) - "index_users_on_username" btree (username) - "index_users_on_username_trigram" gin (username gin_trgm_ops) - "tmp_idx_on_user_id_where_bio_is_filled" btree (id) WHERE COALESCE(bio, ''::character varying)::text IS DISTINCT FROM ''::text -``` - -Here we can see there is no index on the `twitter` column, which means -PostgreSQL has to perform a sequential scan in this case. Let's try to fix this -by adding the following index: - -```sql -CREATE INDEX CONCURRENTLY twitter_test ON users (twitter); -``` - -If we now re-run our query using `EXPLAIN (ANALYZE, BUFFERS)` we get the -following plan: - -```sql -Aggregate (cost=61002.82..61002.83 rows=1 width=8) (actual time=297.311..297.312 rows=1 loops=1) - Buffers: shared hit=51854 dirtied=19 - -> Index Only Scan using twitter_test on users (cost=0.43..60873.13 rows=51877 width=0) (actual time=279.184..293.532 rows=51833 loops=1) - Filter: ((twitter)::text <> ''::text) - Rows Removed by Filter: 2487830 - Heap Fetches: 26037 - Buffers: shared hit=51854 dirtied=19 -Planning time: 0.191 ms -Execution time: 297.334 ms -``` - -Now it takes just under 300 milliseconds to get our data, instead of 1.2 -seconds. However, we still use 51,854 buffers, which is about 400 MB of memory. -300 milliseconds is also quite slow for such a simple query. To understand why -this query is still expensive, let's take a look at the following: - -```sql -Index Only Scan using twitter_test on users (cost=0.43..60873.13 rows=51877 width=0) (actual time=279.184..293.532 rows=51833 loops=1) - Filter: ((twitter)::text <> ''::text) - Rows Removed by Filter: 2487830 -``` - -We start with an index only scan on our index, but we somehow still apply a -`Filter` that filters out 2,487,830 rows. Why is that? Well, let's look at how -we created the index: - -```sql -CREATE INDEX CONCURRENTLY twitter_test ON users (twitter); -``` - -We told PostgreSQL to index all possible values of the `twitter` column, -even empty strings. Our query in turn uses `WHERE twitter != ''`. This means -that the index does improve things, as we don't need to do a sequential scan, -but we may still encounter empty strings. This means PostgreSQL _has_ to apply a -Filter on the index results to get rid of those values. - -Fortunately, we can improve this even further using "partial indexes". Partial -indexes are indexes with a `WHERE` condition that is applied when indexing data. -For example: - -```sql -CREATE INDEX CONCURRENTLY some_index ON users (email) WHERE id < 100 -``` - -This index would only index the `email` value of rows that match `WHERE id < -100`. We can use partial indexes to change our Twitter index to the following: - -```sql -CREATE INDEX CONCURRENTLY twitter_test ON users (twitter) WHERE twitter != ''; -``` - -After being created, if we run our query again we are given the following plan: - -```sql -Aggregate (cost=1608.26..1608.27 rows=1 width=8) (actual time=19.821..19.821 rows=1 loops=1) - Buffers: shared hit=44036 - -> Index Only Scan using twitter_test on users (cost=0.41..1479.71 rows=51420 width=0) (actual time=0.023..15.514 rows=51833 loops=1) - Heap Fetches: 1208 - Buffers: shared hit=44036 -Planning time: 0.123 ms -Execution time: 19.848 ms -``` - -That's _a lot_ better! Now it only takes 20 milliseconds to get the data, and we -only use about 344 MB of buffers (instead of the original 1.58 GB). The reason -this works is that now PostgreSQL no longer needs to apply a `Filter`, as the -index only contains `twitter` values that are not empty. - -Keep in mind that you shouldn't just add partial indexes every time you want to -optimise a query. Every index has to be updated for every write, and they may -require quite a bit of space, depending on the amount of indexed data. As a -result, first check if there are any existing indexes you may be able to reuse. -If there aren't any, check if you can perhaps slightly change an existing one to -fit both the existing and new queries. Only add a new index if none of the -existing indexes can be used in any way. - -When comparing execution plans, don't take timing as the only important metric. -Good timing is the main goal of any optimization, but it can be too volatile to -be used for comparison (for example, it depends a lot on the state of cache). -When optimizing a query, we usually need to reduce the amount of data we're -dealing with. Indexes are the way to work with fewer pages (buffers) to get the -result, so, during optimization, look at the number of buffers used (read and hit), -and work on reducing these numbers. Reduced timing is the consequence of reduced -buffer numbers. [Database Lab Engine](#database-lab-engine) guarantees that the plan is structurally -identical to production (and overall number of buffers is the same as on production), -but difference in cache state and I/O speed may lead to different timings. - -## Queries that can't be optimised - -Now that we have seen how to optimise a query, let's look at another query that -we might not be able to optimise: - -```sql -EXPLAIN (ANALYZE, BUFFERS) -SELECT COUNT(*) -FROM projects -WHERE visibility_level IN (0, 20); -``` - -The output of `EXPLAIN (ANALYZE, BUFFERS)` is as follows: - -```sql -Aggregate (cost=922420.60..922420.61 rows=1 width=8) (actual time=3428.535..3428.535 rows=1 loops=1) - Buffers: shared hit=208846 - -> Seq Scan on projects (cost=0.00..908053.18 rows=5746969 width=0) (actual time=0.041..2987.606 rows=5746940 loops=1) - Filter: (visibility_level = ANY ('{0,20}'::integer[])) - Rows Removed by Filter: 65677 - Buffers: shared hit=208846 -Planning time: 2.861 ms -Execution time: 3428.596 ms -``` - -Looking at the output we see the following Filter: - -```sql -Filter: (visibility_level = ANY ('{0,20}'::integer[])) -Rows Removed by Filter: 65677 -``` - -Looking at the number of rows removed by the filter, we may be tempted to add an -index on `projects.visibility_level` to somehow turn this Sequential scan + -filter into an index-only scan. - -Unfortunately, doing so is unlikely to improve anything. Contrary to what some -might believe, an index being present _does not guarantee_ that PostgreSQL -actually uses it. For example, when doing a `SELECT * FROM projects` it is much -cheaper to just scan the entire table, instead of using an index and then -fetching data from the table. In such cases PostgreSQL may decide to not use an -index. - -Second, let's think for a moment what our query does: it gets all projects with -visibility level 0 or 20. In the above plan we can see this produces quite a lot -of rows (5,745,940), but how much is that relative to the total? Let's find out -by running the following query: - -```sql -SELECT visibility_level, count(*) AS amount -FROM projects -GROUP BY visibility_level -ORDER BY visibility_level ASC; -``` - -For GitLab.com this produces: - -```sql - visibility_level | amount -------------------+--------- - 0 | 5071325 - 10 | 65678 - 20 | 674801 -``` - -Here the total number of projects is 5,811,804, and 5,746,126 of those are of -level 0 or 20. That's 98% of the entire table! - -So no matter what we do, this query retrieves 98% of the entire table. Since -most time is spent doing exactly that, there isn't really much we can do to -improve this query, other than _not_ running it at all. - -What is important here is that while some may recommend to straight up add an -index the moment you see a sequential scan, it is _much more important_ to first -understand what your query does, how much data it retrieves, and so on. After -all, you can not optimise something you do not understand. - -### Cardinality and selectivity - -Earlier we saw that our query had to retrieve 98% of the rows in the table. -There are two terms commonly used for databases: cardinality, and selectivity. -Cardinality refers to the number of unique values in a particular column in a -table. - -Selectivity is the number of unique values produced by an operation (for example, an -index scan or filter), relative to the total number of rows. The higher the -selectivity, the more likely PostgreSQL is able to use an index. - -In the above example, there are only 3 unique values: 0, 10, and 20. This means -the cardinality is 3. The selectivity in turn is also very low: 0.0000003% (2 / -5,811,804), because our `Filter` only filters using two values (`0` and `20`). -With such a low selectivity value it's not surprising that PostgreSQL decides -using an index is not worth it, because it would produce almost no unique rows. - -## Rewriting queries - -So the above query can't really be optimised as-is, or at least not much. But -what if we slightly change the purpose of it? What if instead of retrieving all -projects with `visibility_level` 0 or 20, we retrieve those that a user -interacted with somehow? - -Fortunately, GitLab has an answer for this, and it's a table called -`user_interacted_projects`. This table has the following schema: - -```sql -Table "public.user_interacted_projects" - Column | Type | Modifiers -------------+---------+----------- - user_id | integer | not null - project_id | integer | not null -Indexes: - "index_user_interacted_projects_on_project_id_and_user_id" UNIQUE, btree (project_id, user_id) - "index_user_interacted_projects_on_user_id" btree (user_id) -Foreign-key constraints: - "fk_rails_0894651f08" FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE - "fk_rails_722ceba4f7" FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE -``` - -Let's rewrite our query to `JOIN` this table onto our projects, and get the -projects for a specific user: - -```sql -EXPLAIN ANALYZE -SELECT COUNT(*) -FROM projects -INNER JOIN user_interacted_projects ON user_interacted_projects.project_id = projects.id -WHERE projects.visibility_level IN (0, 20) -AND user_interacted_projects.user_id = 1; -``` - -What we do here is the following: - -1. Get our projects. -1. `INNER JOIN` `user_interacted_projects`, meaning we're only left with rows in - `projects` that have a corresponding row in `user_interacted_projects`. -1. Limit this to the projects with `visibility_level` of 0 or 20, and to - projects that the user with ID 1 interacted with. - -If we run this query we get the following plan: - -```sql - Aggregate (cost=871.03..871.04 rows=1 width=8) (actual time=9.763..9.763 rows=1 loops=1) - -> Nested Loop (cost=0.86..870.52 rows=203 width=0) (actual time=1.072..9.748 rows=143 loops=1) - -> Index Scan using index_user_interacted_projects_on_user_id on user_interacted_projects (cost=0.43..160.71 rows=205 width=4) (actual time=0.939..2.508 rows=145 loops=1) - Index Cond: (user_id = 1) - -> Index Scan using projects_pkey on projects (cost=0.43..3.45 rows=1 width=4) (actual time=0.049..0.050 rows=1 loops=145) - Index Cond: (id = user_interacted_projects.project_id) - Filter: (visibility_level = ANY ('{0,20}'::integer[])) - Rows Removed by Filter: 0 - Planning time: 2.614 ms - Execution time: 9.809 ms -``` - -Here it only took us just under 10 milliseconds to get the data. We can also see -we're retrieving far fewer projects: - -```sql -Index Scan using projects_pkey on projects (cost=0.43..3.45 rows=1 width=4) (actual time=0.049..0.050 rows=1 loops=145) - Index Cond: (id = user_interacted_projects.project_id) - Filter: (visibility_level = ANY ('{0,20}'::integer[])) - Rows Removed by Filter: 0 -``` - -Here we see we perform 145 loops (`loops=145`), with every loop producing 1 row -(`rows=1`). This is much less than before, and our query performs much better! - -If we look at the plan we also see our costs are very low: - -```sql -Index Scan using projects_pkey on projects (cost=0.43..3.45 rows=1 width=4) (actual time=0.049..0.050 rows=1 loops=145) -``` - -Here our cost is only 3.45, and it takes us 7.25 milliseconds to do so (0.05 * 145). -The next index scan is a bit more expensive: - -```sql -Index Scan using index_user_interacted_projects_on_user_id on user_interacted_projects (cost=0.43..160.71 rows=205 width=4) (actual time=0.939..2.508 rows=145 loops=1) -``` - -Here the cost is 160.71 (`cost=0.43..160.71`), taking about 2.5 milliseconds -(based on the output of `actual time=....`). - -The most expensive part here is the "Nested Loop" that acts upon the result of -these two index scans: - -```sql -Nested Loop (cost=0.86..870.52 rows=203 width=0) (actual time=1.072..9.748 rows=143 loops=1) -``` - -Here we had to perform 870.52 disk page fetches for 203 rows, 9.748 -milliseconds, producing 143 rows in a single loop. - -The key takeaway here is that sometimes you have to rewrite (parts of) a query -to make it better. Sometimes that means having to slightly change your feature -to accommodate for better performance. - -## What makes a bad plan - -This is a bit of a difficult question to answer, because the definition of "bad" -is relative to the problem you are trying to solve. However, some patterns are -best avoided in most cases, such as: - -- Sequential scans on large tables -- Filters that remove a lot of rows -- Performing a certain step that requires _a lot_ of - buffers (for example, an index scan for GitLab.com that requires more than 512 MB). - -As a general guideline, aim for a query that: - -1. Takes no more than 10 milliseconds. Our target time spent in SQL per request - is around 100 milliseconds, so every query should be as fast as possible. -1. Does not use an excessive number of buffers, relative to the workload. For - example, retrieving ten rows shouldn't require 1 GB of buffers. -1. Does not spend a long amount of time performing disk IO operations. The - setting `track_io_timing` must be enabled for this data to be included in the - output of `EXPLAIN ANALYZE`. -1. Applies a `LIMIT` when retrieving rows without aggregating them, such as - `SELECT * FROM users`. -1. Doesn't use a `Filter` to filter out too many rows, especially if the query - does not use a `LIMIT` to limit the number of returned rows. Filters can - usually be removed by adding a (partial) index. - -These are _guidelines_ and not hard requirements, as different needs may require -different queries. The only _rule_ is that you _must always measure_ your query -(preferably using a production-like database) using `EXPLAIN (ANALYZE, BUFFERS)` -and related tools such as: - -- [`explain.depesz.com`](https://explain.depesz.com/). -- [`explain.dalibo.com/`](https://explain.dalibo.com/). - -## Producing query plans - -There are a few ways to get the output of a query plan. Of course you -can directly run the `EXPLAIN` query in the `psql` console, or you can -follow one of the other options below. - -### Database Lab Engine - -GitLab team members can use [Database Lab Engine](https://gitlab.com/postgres-ai/database-lab), and the companion -SQL optimization tool - [Joe Bot](https://gitlab.com/postgres-ai/joe). - -Database Lab Engine provides developers with their own clone of the production database, while Joe Bot helps with exploring execution plans. - -Joe Bot is available in the [`#database-lab`](https://gitlab.slack.com/archives/CLJMDRD8C) channel on Slack, -and through its [web interface](https://console.postgres.ai/gitlab/joe-instances). - -With Joe Bot you can execute DDL statements (like creating indexes, tables, and columns) and get query plans for `SELECT`, `UPDATE`, and `DELETE` statements. - -For example, in order to test new index on a column that is not existing on production yet, you can do the following: - -Create the column: - -```sql -exec ALTER TABLE projects ADD COLUMN last_at timestamp without time zone -``` - -Create the index: - -```sql -exec CREATE INDEX index_projects_last_activity ON projects (last_activity_at) WHERE last_activity_at IS NOT NULL -``` - -Analyze the table to update its statistics: - -```sql -exec ANALYZE projects -``` - -Get the query plan: - -```sql -explain SELECT * FROM projects WHERE last_activity_at < CURRENT_DATE -``` - -Once done you can rollback your changes: - -```sql -reset -``` - -For more information about the available options, run: - -```sql -help -``` - -The web interface comes with the following execution plan visualizers included: - -- [Depesz](https://explain.depesz.com/) -- [PEV2](https://github.com/dalibo/pev2) -- [FlameGraph](https://github.com/mgartner/pg_flame) - -#### Tips & Tricks - -The database connection is now maintained during your whole session, so you can use `exec set ...` for any session variables (such as `enable_seqscan` or `work_mem`). These settings are applied to all subsequent commands until you reset them. For example you can disable parallel queries with - -```sql -exec SET max_parallel_workers_per_gather = 0 -``` - -### Rails console - -Using the [`activerecord-explain-analyze`](https://github.com/6/activerecord-explain-analyze) -you can directly generate the query plan from the Rails console: - -```ruby -pry(main)> require 'activerecord-explain-analyze' -=> true -pry(main)> Project.where('build_timeout > ?', 3600).explain(analyze: true) - Project Load (1.9ms) SELECT "projects".* FROM "projects" WHERE (build_timeout > 3600) - ↳ (pry):12 -=> EXPLAIN for: SELECT "projects".* FROM "projects" WHERE (build_timeout > 3600) -Seq Scan on public.projects (cost=0.00..2.17 rows=1 width=742) (actual time=0.040..0.041 rows=0 loops=1) - Output: id, name, path, description, created_at, updated_at, creator_id, namespace_id, ... - Filter: (projects.build_timeout > 3600) - Rows Removed by Filter: 14 - Buffers: shared hit=2 -Planning time: 0.411 ms -Execution time: 0.113 ms -``` - -### ChatOps - -[GitLab team members can also use our ChatOps solution, available in Slack using the -`/chatops` slash command](chatops_on_gitlabcom.md). - -NOTE: -While ChatOps is still available, the recommended way to generate execution plans is to use [Database Lab Engine](#database-lab-engine). - -You can use ChatOps to get a query plan by running the following: - -```sql -/chatops run explain SELECT COUNT(*) FROM projects WHERE visibility_level IN (0, 20) -``` - -Visualising the plan using <https://explain.depesz.com/> is also supported: - -```sql -/chatops run explain --visual SELECT COUNT(*) FROM projects WHERE visibility_level IN (0, 20) -``` - -Quoting the query is not necessary. - -For more information about the available options, run: - -```sql -/chatops run explain --help -``` - -## Further reading - -A more extensive guide on understanding query plans can be found in -the [presentation](https://public.dalibo.com/exports/conferences/_archives/_2012/201211_explain/understanding_explain.pdf) -from [Dalibo.org](https://www.dalibo.com/en/). - -Depesz's blog also has a good [section](https://www.depesz.com/tag/unexplainable/) dedicated to query plans. +<!-- This redirect file can be deleted after <2022-11-04>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/security/two_factor_authentication.md b/doc/security/two_factor_authentication.md index 48f9cadd11b..93d77a69f0e 100644 --- a/doc/security/two_factor_authentication.md +++ b/doc/security/two_factor_authentication.md @@ -129,6 +129,7 @@ sudo -u git -H bundle exec rake gitlab:two_factor:disable_for_all_users RAILS_EN > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/270554) in GitLab 13.7. > - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/299088) from GitLab Free to GitLab Premium in 13.9. > - It's deployed behind a feature flag, disabled by default. +> - Push notification support [introduced](https://gitlab.com/gitlab-org/gitlab-shell/-/issues/506) in GitLab 15.3. FLAG: On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../administration/feature_flags.md) named `two_factor_for_cli`. On GitLab.com, this feature is not available. The feature is not ready for production use. This feature flag also affects [session duration for Git Operations when 2FA is enabled](../user/admin_area/settings/account_and_limit_settings.md#customize-session-duration-for-git-operations-when-2fa-is-enabled). @@ -136,19 +137,20 @@ On self-managed GitLab, by default this feature is not available. To make it ava Two-factor authentication can be enforced for Git over SSH operations. However, we recommend using [ED25519_SK](../user/ssh.md#ed25519_sk-ssh-keys) or [ECDSA_SK](../user/ssh.md#ecdsa_sk-ssh-keys) SSH keys instead. -The one-time password (OTP) verification can be done using a command: +To perform one-time password (OTP) verification, run: ```shell ssh git@<hostname> 2fa_verify ``` -In GitLab 15.3 and later, users can authenticate by either: +Then authenticate by either: - Entering the correct OTP. -- Responding to a device push notification, if [FortiAuthenticator is enabled](../user/profile/account/two_factor_authentication.md#enable-one-time-password-using-fortiauthenticator). +- In GitLab 15.3 and later, responding to a device push notification if + [FortiAuthenticator is enabled](../user/profile/account/two_factor_authentication.md#enable-one-time-password-using-fortiauthenticator). -After the successful authentication, Git over SSH operations can be used for a session duration of -15 minutes (default) with the associated SSH key. +After successful authentication, you can perform Git over SSH operations for 15 minutes (default) with the associated +SSH key. ### Security limitation diff --git a/doc/user/project/merge_requests/approvals/img/scoped_to_protected_branch_v13_10.png b/doc/user/project/merge_requests/approvals/img/scoped_to_protected_branch_v13_10.png Binary files differdeleted file mode 100644 index a6636f0bc7f..00000000000 --- a/doc/user/project/merge_requests/approvals/img/scoped_to_protected_branch_v13_10.png +++ /dev/null diff --git a/doc/user/project/merge_requests/approvals/rules.md b/doc/user/project/merge_requests/approvals/rules.md index 70112135ba9..89d75aa196a 100644 --- a/doc/user/project/merge_requests/approvals/rules.md +++ b/doc/user/project/merge_requests/approvals/rules.md @@ -213,7 +213,8 @@ appreciated, but not required. To make an approval rule optional: ## Approvals for protected branches -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/460) in GitLab 12.8. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/460) in GitLab 12.8. +> - **All protected branches** target branch option [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/360930) in GitLab 15.3. Approval rules are often relevant only to specific branches, like your [default branch](../../repository/branches/default.md). To configure an @@ -223,10 +224,10 @@ approval rule for certain branches: 1. Go to your project and select **Settings**. 1. Expand **Merge request (MR) approvals**. 1. Select a **Target branch**: - - To protect all branches, select **All branches**. - - To select a specific branch, select it from the list: + - To apply the rule to all branches, select **All branches**. + - To apply the rule to all protected branches, select **All protected branches** (GitLab 15.3 and later). + - To apply the rule to a specific branch, select it from the list: -  1. To enable this configuration, read [Code Owner's approvals for protected branches](../../protected_branches.md#require-code-owner-approval-on-a-protected-branch). diff --git a/doc/user/project/releases/index.md b/doc/user/project/releases/index.md index b22b8716d52..e8ccb24e2c5 100644 --- a/doc/user/project/releases/index.md +++ b/doc/user/project/releases/index.md @@ -250,27 +250,6 @@ The `ADDITIONAL_CA_CERT_BUNDLE` value can also be configured as a either as a `file`, which requires the path to the certificate, or as a variable, which requires the text representation of the certificate. -## GitLab Release CLI tool - -The [GitLab Release CLI (`release-cli`)](https://gitlab.com/gitlab-org/release-cli) tool -is a command-line tool for managing releases from the command line or from a CI/CD pipeline. -You can use the release CLI to create, update, modify, and delete releases. - -When you [use a CI/CD job to create a release](#creating-a-release-by-using-a-cicd-job), -the `release` keyword entries are transformed into Bash commands and sent to the Docker -container containing the `release-cli` tool. The tool then creates the release. - -You can also call the `release-cli` tool directly from a [`script`](../../../ci/yaml/index.md#script). -For example: - -```shell -release-cli create --name "Release $CI_COMMIT_SHA" --description \ - "Created using the release-cli $EXTRA_DESCRIPTION" \ - --tag-name "v${MAJOR}.${MINOR}.${REVISION}" --ref "$CI_COMMIT_SHA" \ - --released-at "2020-07-15T08:00:00Z" --milestone "m1" --milestone "m2" --milestone "m3" \ - --assets-link "{\"name\":\"asset1\",\"url\":\"https://example.com/assets/1\",\"link_type\":\"other\"} -``` - ### Create multiple releases in a single pipeline A pipeline can have multiple `release` jobs, for example: diff --git a/doc/user/project/releases/release_cli.md b/doc/user/project/releases/release_cli.md index b55f0b0a734..9e65ab4bc01 100644 --- a/doc/user/project/releases/release_cli.md +++ b/doc/user/project/releases/release_cli.md @@ -4,16 +4,38 @@ group: Release info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Install the `release-cli` for the Shell executor **(FREE)** + +# GitLab Release CLI tool + +The [GitLab Release CLI (`release-cli`)](https://gitlab.com/gitlab-org/release-cli) tool +is a command-line tool for managing releases from the command line or from a CI/CD pipeline. +You can use the release CLI to create, update, modify, and delete releases. + +When you [use a CI/CD job to create a release](index.md#creating-a-release-by-using-a-cicd-job), +the `release` keyword entries are transformed into Bash commands and sent to the Docker +container containing the `release-cli` tool. The tool then creates the release. + +You can also call the `release-cli` tool directly from a [`script`](../../../ci/yaml/index.md#script). +For example: + +```shell +release-cli create --name "Release $CI_COMMIT_SHA" --description \ + "Created using the release-cli $EXTRA_DESCRIPTION" \ + --tag-name "v${MAJOR}.${MINOR}.${REVISION}" --ref "$CI_COMMIT_SHA" \ + --released-at "2020-07-15T08:00:00Z" --milestone "m1" --milestone "m2" --milestone "m3" \ + --assets-link "{\"name\":\"asset1\",\"url\":\"https://example.com/assets/1\",\"link_type\":\"other\"} +``` + +## Install the `release-cli` for the Shell executor **(FREE)** > - [Introduced](https://gitlab.com/gitlab-org/release-cli/-/issues/21) in GitLab 13.8. -> - [Changed](https://gitlab.com/gitlab-org/release-cli/-/merge_requests/108) in GitLab 14.2, the `release-cli` binaries are also [available in the Package Registry](https://gitlab.com/jaime/release-cli/-/packages). +> - [Changed](https://gitlab.com/gitlab-org/release-cli/-/merge_requests/108) in GitLab 14.2, the `release-cli` binaries are also [available in the Package Registry](https://gitlab.com/gitlab-org/release-cli/-/packages). When you use a runner with the Shell executor, you can download and install the `release-cli` manually for your [supported OS and architecture](https://release-cli-downloads.s3.amazonaws.com/latest/index.html). Once installed, [the `release` keyword](../../../ci/yaml/index.md#release) is available to use in your CI/CD jobs. -## Install on Unix/Linux +### Install on Unix/Linux 1. Download the binary for your system from S3, in the following example for amd64 systems: @@ -41,7 +63,7 @@ Once installed, [the `release` keyword](../../../ci/yaml/index.md#release) is av release-cli version 0.6.0 ``` -## Install on Windows PowerShell +### Install on Windows PowerShell 1. Create a folder somewhere in your system, for example `C:\GitLab\Release-CLI\bin` diff --git a/lib/api/personal_access_tokens.rb b/lib/api/personal_access_tokens.rb index f8b744bb14b..0d7d2dc6a0c 100644 --- a/lib/api/personal_access_tokens.rb +++ b/lib/api/personal_access_tokens.rb @@ -57,9 +57,14 @@ module API get ':id' do token = PersonalAccessToken.find_by_id(params[:id]) - unauthorized! unless token && Ability.allowed?(current_user, :read_user_personal_access_tokens, token.user) - - present token, with: Entities::PersonalAccessToken + allowed = Ability.allowed?(current_user, :read_user_personal_access_tokens, token&.user) + + if allowed + present token, with: Entities::PersonalAccessToken + else + # Only admins should be informed if the token doesn't exist + current_user.admin? ? not_found! : unauthorized! + end end delete 'self' do diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_one_member_no_repo_projects.rb b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_one_member_no_repo_projects.rb new file mode 100644 index 00000000000..3a9049b1f19 --- /dev/null +++ b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_one_member_no_repo_projects.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Set `project_settings.legacy_open_source_license_available` to false for public projects with 1 member and no repo + class DisableLegacyOpenSourceLicenseForOneMemberNoRepoProjects < ::Gitlab::BackgroundMigration::BatchedMigrationJob + PUBLIC = 20 + + # Migration only version of `project_settings` table + class ProjectSetting < ApplicationRecord + self.table_name = 'project_settings' + end + + def perform + each_sub_batch( + operation_name: :disable_legacy_open_source_license_for_one_member_no_repo_projects, + batching_scope: ->(relation) { relation.where(visibility_level: PUBLIC) } + ) do |sub_batch| + one_member_no_repo_projects = + sub_batch + .joins('LEFT OUTER JOIN project_statistics ON project_statistics.project_id = projects.id') + .joins('LEFT OUTER JOIN project_settings ON project_settings.project_id = projects.id') + .joins('LEFT OUTER JOIN project_authorizations ON project_authorizations.project_id = projects.id') + .where('project_statistics.repository_size' => 0, + 'project_settings.legacy_open_source_license_available' => true) + .group('projects.id') + .having('COUNT(project_authorizations.user_id) = 1') + + ProjectSetting + .where(project_id: one_member_no_repo_projects) + .update_all(legacy_open_source_license_available: false) + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers/sbom/cyclonedx.rb b/lib/gitlab/ci/parsers/sbom/cyclonedx.rb index 86516aa2a7a..0e2ca97b9cc 100644 --- a/lib/gitlab/ci/parsers/sbom/cyclonedx.rb +++ b/lib/gitlab/ci/parsers/sbom/cyclonedx.rb @@ -18,7 +18,7 @@ module Gitlab return unless valid? - parse_components + parse_report rescue JSON::ParserError => e report.add_error("Report JSON is invalid: #{e}") end @@ -54,6 +54,17 @@ module Gitlab false end + def parse_report + parse_metadata_properties + parse_components + end + + def parse_metadata_properties + properties = data.dig('metadata', 'properties') + source = CyclonedxProperties.parse_source(properties) + report.set_source(source) if source + end + def parse_components data['components']&.each do |component| next unless supported_component_type?(component['type']) diff --git a/lib/gitlab/ci/parsers/sbom/cyclonedx_properties.rb b/lib/gitlab/ci/parsers/sbom/cyclonedx_properties.rb new file mode 100644 index 00000000000..3dc73544208 --- /dev/null +++ b/lib/gitlab/ci/parsers/sbom/cyclonedx_properties.rb @@ -0,0 +1,112 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Sbom + # Parses GitLab CycloneDX metadata properties which are defined by the taxonomy at + # https://gitlab.com/gitlab-org/security-products/gitlab-cyclonedx-property-taxonomy + # + # This parser knows how to process schema version 1 and will not attempt to parse + # later versions. Each source type has it's own namespace in the property schema, + # and is also given its own parser. Properties are filtered by namespace, + # and then passed to each source parser for processing. + class CyclonedxProperties + SUPPORTED_SCHEMA_VERSION = '1' + GITLAB_PREFIX = 'gitlab:' + SOURCE_PARSERS = { + 'dependency_scanning' => ::Gitlab::Ci::Parsers::Sbom::Source::DependencyScanning + }.freeze + SUPPORTED_PROPERTIES = %w[ + meta:schema_version + dependency_scanning:category + dependency_scanning:input_file:path + dependency_scanning:source_file:path + dependency_scanning:package_manager:name + dependency_scanning:language:name + ].freeze + + def self.parse_source(...) + new(...).parse_source + end + + def initialize(properties) + @properties = properties + end + + def parse_source + return unless properties.present? + return unless supported_schema_version? + + source + end + + private + + attr_reader :properties + + def property_data + @property_data ||= properties + .each_with_object({}) { |property, data| parse_property(property, data) } + end + + def parse_property(property, data) + name = property['name'] + value = property['value'] + + # The specification permits the name or value to be absent. + return unless name.present? && value.present? + return unless name.start_with?(GITLAB_PREFIX) + + namespaced_name = name.delete_prefix(GITLAB_PREFIX) + + return unless SUPPORTED_PROPERTIES.include?(namespaced_name) + + parse_name_value_pair(namespaced_name, value, data) + end + + def parse_name_value_pair(name, value, data) + # Each namespace in the property name reflects a key in the hash. + # A property with the name `dependency_scanning:input_file:path` + # and the value `package-lock.json` should be transformed into + # this data: + # {"dependency_scanning": {"input_file": {"path": "package-lock.json"}}} + keys = name.split(':') + + # Remove last item from the keys and use it to create + # the initial object. + last = keys.pop + + # Work backwards. For each key, create a new hash wrapping the previous one. + # Using `dependency_scanning:input_file:path` as an example: + # + # 1. memo = { "path" => "package-lock.json" } (arguments given to reduce) + # 2. memo = { "input_file" => memo } + # 3. memo = { "dependency_scanning" => memo } + property = keys.reverse.reduce({ last => value }) do |memo, key| + { key => memo } + end + + data.deep_merge!(property) + end + + def schema_version + @schema_version ||= property_data.dig('meta', 'schema_version') + end + + def supported_schema_version? + schema_version == SUPPORTED_SCHEMA_VERSION + end + + def source + @source ||= property_data + .slice(*SOURCE_PARSERS.keys) + .lazy + .filter_map { |namespace, data| SOURCE_PARSERS[namespace].source(data) } + .first + end + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers/sbom/source/dependency_scanning.rb b/lib/gitlab/ci/parsers/sbom/source/dependency_scanning.rb new file mode 100644 index 00000000000..ad04b3257f9 --- /dev/null +++ b/lib/gitlab/ci/parsers/sbom/source/dependency_scanning.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Sbom + module Source + class DependencyScanning + REQUIRED_ATTRIBUTES = [ + %w[input_file path] + ].freeze + + def self.source(...) + new(...).source + end + + def initialize(data) + @data = data + end + + def source + return unless required_attributes_present? + + { + 'type' => :dependency_scanning, + 'data' => data, + 'fingerprint' => fingerprint + } + end + + private + + attr_reader :data + + def required_attributes_present? + REQUIRED_ATTRIBUTES.all? do |keys| + data.dig(*keys).present? + end + end + + def fingerprint + Digest::SHA256.hexdigest(data.to_json) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/sbom/report.rb b/lib/gitlab/ci/reports/sbom/report.rb index a7edcd20877..dc6b3153e51 100644 --- a/lib/gitlab/ci/reports/sbom/report.rb +++ b/lib/gitlab/ci/reports/sbom/report.rb @@ -5,25 +5,28 @@ module Gitlab module Reports module Sbom class Report - attr_reader :components, :sources, :errors + attr_reader :components, :source, :errors def initialize @components = [] @errors = [] - @sources = [] end def add_error(error) errors << error end - def add_source(source) - sources << Source.new(source) + def set_source(source) + self.source = Source.new(source) end def add_component(component) components << Component.new(component) end + + private + + attr_writer :source end end end diff --git a/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml b/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml index 71f38ededd9..19d7f49aac0 100644 --- a/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml @@ -26,6 +26,7 @@ variables: TF_VAR_SERVICE_DESK_EMAIL: incoming+${CI_PROJECT_PATH_SLUG}-${CI_PROJECT_ID}-issue-@incoming.gitlab.com TF_VAR_SHORT_ENVIRONMENT_NAME: ${CI_PROJECT_ID}-${CI_COMMIT_REF_SLUG} TF_VAR_SMTP_FROM: ${SMTP_FROM} + TEMPLATE_REGISTRY_HOST: 'registry.gitlab.com' cache: paths: @@ -39,7 +40,7 @@ cache: terraform_apply: stage: provision - image: registry.gitlab.com/gitlab-org/5-minute-production-app/deploy-template/stable + image: "$TEMPLATE_REGISTRY_HOST/gitlab-org/5-minute-production-app/deploy-template/stable" extends: .needs_aws_vars resource_group: terraform before_script: @@ -53,7 +54,7 @@ terraform_apply: deploy: stage: deploy - image: registry.gitlab.com/gitlab-org/5-minute-production-app/deploy-template/stable + image: "$TEMPLATE_REGISTRY_HOST/gitlab-org/5-minute-production-app/deploy-template/stable" extends: .needs_aws_vars resource_group: deploy before_script: @@ -74,7 +75,7 @@ terraform_destroy: variables: GIT_STRATEGY: none stage: destroy - image: registry.gitlab.com/gitlab-org/5-minute-production-app/deploy-template/stable + image: "$TEMPLATE_REGISTRY_HOST/gitlab-org/5-minute-production-app/deploy-template/stable" before_script: - cp /*.tf . - cp /deploy.sh . diff --git a/lib/gitlab/ci/templates/Indeni.Cloudrail.gitlab-ci.yml b/lib/gitlab/ci/templates/Indeni.Cloudrail.gitlab-ci.yml index 7f33d048c1e..a4fdd18aa40 100644 --- a/lib/gitlab/ci/templates/Indeni.Cloudrail.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Indeni.Cloudrail.gitlab-ci.yml @@ -24,6 +24,7 @@ variables: TEST_ROOT: ${CI_PROJECT_DIR}/my_folder_with_terraform_content + TEMPLATE_REGISTRY_HOST: 'registry.gitlab.com' default: before_script: @@ -31,7 +32,7 @@ default: init_and_plan: stage: build - image: registry.gitlab.com/gitlab-org/terraform-images/releases/0.13 + image: "$TEMPLATE_REGISTRY_HOST/gitlab-org/terraform-images/releases/0.13" rules: - if: $SAST_DISABLED when: never diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index 8c63019d743..11447a36045 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,9 +1,10 @@ variables: AUTO_BUILD_IMAGE_VERSION: 'v1.14.0' + TEMPLATE_REGISTRY_HOST: 'registry.gitlab.com' build: stage: build - image: 'registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:${AUTO_BUILD_IMAGE_VERSION}' + image: '${TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-build-image:${AUTO_BUILD_IMAGE_VERSION}' variables: DOCKER_TLS_CERTDIR: '' services: diff --git a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml index 8c63019d743..11447a36045 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml @@ -1,9 +1,10 @@ variables: AUTO_BUILD_IMAGE_VERSION: 'v1.14.0' + TEMPLATE_REGISTRY_HOST: 'registry.gitlab.com' build: stage: build - image: 'registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:${AUTO_BUILD_IMAGE_VERSION}' + image: '${TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-build-image:${AUTO_BUILD_IMAGE_VERSION}' variables: DOCKER_TLS_CERTDIR: '' services: diff --git a/lib/gitlab/ci/templates/Jobs/CF-Provision.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/CF-Provision.gitlab-ci.yml index 11f8376f0b4..b5efcb7bba3 100644 --- a/lib/gitlab/ci/templates/Jobs/CF-Provision.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/CF-Provision.gitlab-ci.yml @@ -1,8 +1,11 @@ +variables: + TEMPLATE_REGISTRY_HOST: 'registry.gitlab.com' + stages: - provision cloud_formation: - image: 'registry.gitlab.com/gitlab-org/cloud-deploy/aws-cloudformation:latest' + image: '${TEMPLATE_REGISTRY_HOST}/gitlab-org/cloud-deploy/aws-cloudformation:latest' stage: provision script: - gl-cloudformation create-stack diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 86e3ace84c5..dc46be4257f 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -8,7 +8,8 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.29" + TEMPLATE_REGISTRY_HOST: 'registry.gitlab.com' + CODE_QUALITY_IMAGE: "$TEMPLATE_REGISTRY_HOST/gitlab-org/ci-cd/codequality:0.85.29" needs: [] script: - export SOURCE_CODE=$PWD diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index b41e92e3a56..074f2f0f78e 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,8 +1,9 @@ variables: DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.30.0' + TEMPLATE_REGISTRY_HOST: 'registry.gitlab.com' .dast-auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" + image: "${TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" .common_rules: &common_rules - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME @@ -57,7 +58,7 @@ stop_dast_environment: when: always .ecs_image: - image: 'registry.gitlab.com/gitlab-org/cloud-deploy/aws-ecs:latest' + image: '${TEMPLATE_REGISTRY_HOST}/gitlab-org/cloud-deploy/aws-ecs:latest' .ecs_rules: &ecs_rules - if: $AUTO_DEVOPS_PLATFORM_TARGET != "ECS" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index f9c0d4333ff..2478b14348a 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,8 +1,9 @@ variables: AUTO_DEPLOY_IMAGE_VERSION: 'v2.30.0' + TEMPLATE_REGISTRY_HOST: 'registry.gitlab.com' .auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" + image: "${TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" dependencies: [] review: diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index 36f1b6981c4..37ba6c5cb47 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,8 +1,9 @@ variables: AUTO_DEPLOY_IMAGE_VERSION: 'v2.30.0' + TEMPLATE_REGISTRY_HOST: 'registry.gitlab.com' .auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" + image: "${TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" dependencies: [] review: diff --git a/lib/gitlab/ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml index ab3bc511cba..c5ae7d406ee 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml @@ -1,9 +1,12 @@ +variables: + TEMPLATE_REGISTRY_HOST: 'registry.gitlab.com' + stages: - review - production .push-and-deploy: - image: 'registry.gitlab.com/gitlab-org/cloud-deploy/aws-ec2:latest' + image: '${TEMPLATE_REGISTRY_HOST}/gitlab-org/cloud-deploy/aws-ec2:latest' script: - gl-ec2 push-to-s3 - gl-ec2 deploy-to-ec2 diff --git a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml index c2d31fd9669..4cdd54dcc2f 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml @@ -7,9 +7,11 @@ # then result in potentially breaking your future pipelines. # # More about including CI templates: https://docs.gitlab.com/ee/ci/yaml/#includetemplate +variables: + TEMPLATE_REGISTRY_HOST: 'registry.gitlab.com' .ecs_image: - image: 'registry.gitlab.com/gitlab-org/cloud-deploy/aws-ecs:latest' + image: '${TEMPLATE_REGISTRY_HOST}/gitlab-org/cloud-deploy/aws-ecs:latest' .deploy_to_ecs: extends: .ecs_image diff --git a/lib/gitlab/ci/templates/Jobs/Helm-2to3.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Helm-2to3.gitlab-ci.yml index d55c126eeb7..9940dab3989 100644 --- a/lib/gitlab/ci/templates/Jobs/Helm-2to3.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Helm-2to3.gitlab-ci.yml @@ -3,9 +3,11 @@ # # To use, set the CI variable MIGRATE_HELM_2TO3 to "true". # For more details, go to https://docs.gitlab.com/ee/topics/autodevops/upgrading_auto_deploy_dependencies.html#helm-v3 +variables: + TEMPLATE_REGISTRY_HOST: 'registry.gitlab.com' .helm-2to3-migrate: - image: registry.gitlab.com/gitlab-org/cluster-integration/helm-install-image/releases/helm-2to3-2.17.0-3.5.3-kube-1.16.15-alpine-3.12 + image: "${TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/helm-install-image/releases/helm-2to3-2.17.0-3.5.3-kube-1.16.15-alpine-3.12" # NOTE: We use the deploy stage because: # - It exists in all versions of Auto DevOps. # - It is _empty_. @@ -54,7 +56,7 @@ done .helm-2to3-cleanup: - image: registry.gitlab.com/gitlab-org/cluster-integration/helm-install-image/releases/helm-2to3-2.17.0-3.5.3-kube-1.16.15-alpine-3.12 + image: "${TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/helm-install-image/releases/helm-2to3-2.17.0-3.5.3-kube-1.16.15-alpine-3.12" stage: cleanup environment: action: prepare diff --git a/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml index cfc4a1d904a..77ebff5d5de 100644 --- a/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml @@ -6,10 +6,11 @@ --- # All available Hugo versions are listed here: # https://gitlab.com/pages/hugo/container_registry -image: registry.gitlab.com/pages/hugo:latest +image: "${TEMPLATE_REGISTRY_HOST}/pages/hugo:latest" variables: GIT_SUBMODULE_STRATEGY: recursive + TEMPLATE_REGISTRY_HOST: 'registry.gitlab.com' test: script: diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index bec269e2933..5d6c1b05976 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -22,7 +22,8 @@ # List of available variables: https://docs.gitlab.com/ee/user/application_security/container_scanning/#available-variables variables: - CS_ANALYZER_IMAGE: registry.gitlab.com/security-products/container-scanning:5 + TEMPLATE_REGISTRY_HOST: 'registry.gitlab.com' + CS_ANALYZER_IMAGE: "$TEMPLATE_REGISTRY_HOST/security-products/container-scanning:5" container_scanning: image: "$CS_ANALYZER_IMAGE$CS_IMAGE_SUFFIX" diff --git a/lib/gitlab/ci/templates/Security/DAST-Runner-Validation.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-Runner-Validation.gitlab-ci.yml index d27a08db181..62423e4134f 100644 --- a/lib/gitlab/ci/templates/Security/DAST-Runner-Validation.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-Runner-Validation.gitlab-ci.yml @@ -11,11 +11,12 @@ stages: variables: DAST_RUNNER_VALIDATION_VERSION: 1 + TEMPLATE_REGISTRY_HOST: 'registry.gitlab.com' validation: stage: dast image: - name: "registry.gitlab.com/security-products/dast-runner-validation:$DAST_RUNNER_VALIDATION_VERSION" + name: "$TEMPLATE_REGISTRY_HOST/security-products/dast-runner-validation:$DAST_RUNNER_VALIDATION_VERSION" variables: GIT_STRATEGY: none allow_failure: false diff --git a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml index edc59d0194e..ad6cc634176 100644 --- a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml @@ -247,7 +247,7 @@ dast-runner-validation: extends: .download_images variables: SECURE_BINARIES_ANALYZER_VERSION: "1" - SECURE_BINARIES_IMAGE: "registry.gitlab.com/security-products/${CI_JOB_NAME}:${SECURE_BINARIES_ANALYZER_VERSION}" + SECURE_BINARIES_IMAGE: "${TEMPLATE_REGISTRY_HOST}/security-products/${CI_JOB_NAME}:${SECURE_BINARIES_ANALYZER_VERSION}" only: variables: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && diff --git a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml index 6f9a9c5133c..ef6fd896bf5 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml @@ -9,11 +9,12 @@ # There is a more opinionated template which we suggest the users to abide, # which is the lib/gitlab/ci/templates/Terraform.gitlab-ci.yml image: - name: registry.gitlab.com/gitlab-org/terraform-images/releases/terraform:1.1.9 + name: "$TEMPLATE_REGISTRY_HOST/gitlab-org/terraform-images/releases/terraform:1.1.9" variables: TF_ROOT: ${CI_PROJECT_DIR} # The relative path to the root directory of the Terraform project TF_STATE_NAME: ${TF_STATE_NAME:-default} # The name of the state file used by the GitLab Managed Terraform state backend + TEMPLATE_REGISTRY_HOST: 'registry.gitlab.com' cache: key: "${TF_ROOT}" diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml index 9ba009a5bca..3277442ea50 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml @@ -10,11 +10,12 @@ # which is the lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml image: - name: registry.gitlab.com/gitlab-org/terraform-images/stable:latest + name: "$TEMPLATE_REGISTRY_HOST/gitlab-org/terraform-images/stable:latest" variables: TF_ROOT: ${CI_PROJECT_DIR} # The relative path to the root directory of the Terraform project TF_STATE_NAME: ${TF_STATE_NAME:-default} # The name of the state file used by the GitLab Managed Terraform state backend + TEMPLATE_REGISTRY_HOST: 'registry.gitlab.com' cache: key: "${TF_ROOT}" diff --git a/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml index 2b5e86f4066..9aa0cf94b94 100644 --- a/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml @@ -5,6 +5,9 @@ # Read more about the feature here: https://docs.gitlab.com/ee/user/project/merge_requests/accessibility_testing.html +variables: + TEMPLATE_REGISTRY_HOST: 'registry.gitlab.com' + stages: - build - test @@ -13,7 +16,7 @@ stages: a11y: stage: accessibility - image: registry.gitlab.com/gitlab-org/ci-cd/accessibility:6.2.3 + image: "$TEMPLATE_REGISTRY_HOST/gitlab-org/ci-cd/accessibility:6.2.3" script: - /gitlab-accessibility.sh "$a11y_urls" allow_failure: true diff --git a/lib/gitlab/git/remote_repository.rb b/lib/gitlab/git/remote_repository.rb deleted file mode 100644 index 0ea009930b0..00000000000 --- a/lib/gitlab/git/remote_repository.rb +++ /dev/null @@ -1,72 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Git - # - # When a Gitaly call involves two repositories instead of one we cannot - # assume that both repositories are on the same Gitaly server. In this - # case we need to make a distinction between the repository that the - # call is being made on (a Repository instance), and the "other" - # repository (a RemoteRepository instance). This is the reason why we - # have the RemoteRepository class in Gitlab::Git. - # - # When you make changes, be aware that gitaly-ruby sub-classes this - # class. - # - class RemoteRepository - attr_reader :relative_path, :gitaly_repository - - def initialize(repository) - @relative_path = repository.relative_path - @gitaly_repository = repository.gitaly_repository - - # These instance variables will not be available in gitaly-ruby, where - # we have no disk access to this repository. - @repository = repository - end - - def empty? - # We will override this implementation in gitaly-ruby because we cannot - # use '@repository' there. - # - # Caches and memoization used on the Rails side - !@repository.exists? || @repository.empty? - end - - def commit_id(revision) - # We will override this implementation in gitaly-ruby because we cannot - # use '@repository' there. - @repository.commit(revision)&.sha - end - - def branch_exists?(name) - # We will override this implementation in gitaly-ruby because we cannot - # use '@repository' there. - @repository.branch_exists?(name) - end - - # Compares self to a Gitlab::Git::Repository. This implementation uses - # 'self.gitaly_repository' so that it will also work in the - # GitalyRemoteRepository subclass defined in gitaly-ruby. - def same_repository?(other_repository) - gitaly_repository.storage_name == other_repository.storage && - gitaly_repository.relative_path == other_repository.relative_path - end - - def path - @repository.path - end - - private - - # Must return an object that responds to 'address' and 'storage'. - def gitaly_client - Gitlab::GitalyClient - end - - def storage - gitaly_repository.storage_name - end - end - end -end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 31387932cb0..9722f885206 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -885,6 +885,15 @@ msgstr "" msgid "%{openedIssues} open, %{closedIssues} closed" msgstr "" +msgid "%{over_limit_message} To get more members, an owner of the group can start a trial or upgrade to a paid tier." +msgstr "" + +msgid "%{over_limit_message} To view and manage members, check the members page for each personal project. We recommend you %{link_start}move your projects to a group%{link_end} so you can easily manage users and features." +msgstr "" + +msgid "%{over_limit_message} To view and manage members, check the members page for each project in your namespace. We recommend you %{link_start}move your projects to a group%{link_end} so you can easily manage users and features." +msgstr "" + msgid "%{percentageUsed}%% used" msgstr "" @@ -6211,6 +6220,9 @@ msgstr "" msgid "Billings|Your account has been validated" msgstr "" +msgid "Billing|%{overLimitMessage} To ensure all members (active and %{linkStart}over limit%{linkEnd}) can access the group, you can start a trial or upgrade to a paid tier." +msgstr "" + msgid "Billing|%{user} was successfully approved" msgstr "" @@ -6265,8 +6277,10 @@ msgstr "" msgid "Billing|Group invite" msgstr "" -msgid "Billing|If the group has over %{maxNamespaceSeats} members, only those occupying a seat can access the namespace. To ensure all members (active and %{linkStart}over limit%{linkEnd}) can access the namespace, you can start a trial or upgrade to a paid tier." -msgstr "" +msgid "Billing|If the group has over %d member, only those occupying a seat can access the group." +msgid_plural "Billing|If the group has over %d members, only those occupying a seat can access the group." +msgstr[0] "" +msgstr[1] "" msgid "Billing|Members who were invited via a group invitation cannot be removed. You can either remove the entire group, or ask an Owner of the invited group to remove the member." msgstr "" @@ -6301,10 +6315,12 @@ msgstr "" msgid "Billing|You can begin moving members in %{namespaceName} now. A member loses access to the group when you turn off %{strongStart}In a seat%{strongEnd}. If over 5 members have %{strongStart}In a seat%{strongEnd} enabled after October 19, 2022, we'll select the 5 members who maintain access. We'll first count members that have Owner and Maintainer roles, then the most recently active members until we reach 5 members. The remaining members will get a status of Over limit and lose access to the group." msgstr "" -msgid "Billing|Your free group is now limited to %{free_user_limit} members" -msgstr "" +msgid "Billing|Your free group is now limited to %d member" +msgid_plural "Billing|Your free group is now limited to %d members" +msgstr[0] "" +msgstr[1] "" -msgid "Billing|Your group recently changed to use the Free plan. Free groups are limited to %{free_user_limit} members and the remaining members will get a status of over-limit and lose access to the group. You can free up space for new members by removing those who no longer need access or toggling them to over-limit. To get an unlimited number of members, you can %{link_start}upgrade%{link_end} to a paid tier." +msgid "Billing|Your group recently changed to use the Free plan. %{over_limit_message} You can free up space for new members by removing those who no longer need access or toggling them to over-limit. To get an unlimited number of members, you can %{link_start}upgrade%{link_end} to a paid tier." msgstr "" msgid "Bitbucket Server Import" @@ -16765,6 +16781,11 @@ msgstr "" msgid "Free Trial of GitLab.com Ultimate" msgstr "" +msgid "Free groups are limited to %{free_user_limit} member and the remaining members will get a status of over-limit and lose access to the group." +msgid_plural "Free groups are limited to %{free_user_limit} members and the remaining members will get a status of over-limit and lose access to the group." +msgstr[0] "" +msgstr[1] "" + msgid "Freeze end" msgstr "" @@ -16789,11 +16810,15 @@ msgstr "" msgid "From %{providerTitle}" msgstr "" -msgid "From October 19, 2022, free personal namespaces and top-level groups will be limited to %{free_limit} members" -msgstr "" +msgid "From October 19, 2022, free groups will be limited to %d member" +msgid_plural "From October 19, 2022, free groups will be limited to %d members" +msgstr[0] "" +msgstr[1] "" -msgid "From October 19, 2022, you can have a maximum of %{free_limit} unique members across all of your personal projects" -msgstr "" +msgid "From October 19, 2022, you can have a maximum of %d unique member across all of your personal projects" +msgid_plural "From October 19, 2022, you can have a maximum of %d unique members across all of your personal projects" +msgstr[0] "" +msgstr[1] "" msgid "From issue creation until deploy to production" msgstr "" @@ -21495,7 +21520,7 @@ msgstr "" msgid "InviteMembersBanner|We noticed that you haven't invited anyone to this group. Invite your colleagues so you can discuss issues, collaborate on merge requests, and share your knowledge." msgstr "" -msgid "InviteMembersModal| To get more members and access to additional paid features, an owner of this namespace can start a trial or upgrade to a paid tier." +msgid "InviteMembersModal| To get more members and access to additional paid features, an owner of the group can start a trial or upgrade to a paid tier." msgstr "" msgid "InviteMembersModal|%{linkStart}Read more%{linkEnd} about role permissions" @@ -21572,7 +21597,7 @@ msgstr "" msgid "InviteMembersModal|To assign issues to a new team member, you need a project for the issues. %{linkStart}Create a project to get started.%{linkEnd}" msgstr "" -msgid "InviteMembersModal|To get more members an owner of this namespace can %{trialLinkStart}start a trial%{trialLinkEnd} or %{upgradeLinkStart}upgrade%{upgradeLinkEnd} to a paid tier." +msgid "InviteMembersModal|To get more members an owner of the group can %{trialLinkStart}start a trial%{trialLinkEnd} or %{upgradeLinkStart}upgrade%{upgradeLinkEnd} to a paid tier." msgstr "" msgid "InviteMembersModal|To make more space, you can remove members who no longer need access." @@ -21587,9 +21612,6 @@ msgstr "" msgid "InviteMembersModal|You only have space for %{count} more %{members} in %{name}" msgstr "" -msgid "InviteMembersModal|You only have space for %{count} more %{members} in your personal projects" -msgstr "" - msgid "InviteMembersModal|You're inviting a group to the %{strongStart}%{name}%{strongEnd} group." msgstr "" @@ -40718,7 +40740,7 @@ msgstr "" msgid "To learn more about this project, read %{link_to_wiki}" msgstr "" -msgid "To manage all members associated with this group and its subgroups and projects, visit the %{link_start}usage quotas page%{link_end}." +msgid "To manage seats for all members associated with this group and its subgroups and projects, visit the %{link_start}usage quotas page%{link_end}." msgstr "" msgid "To move or copy an entire GitLab project from another GitLab installation to this one, navigate to the original project's settings page, generate an export file, and upload it here." @@ -44587,8 +44609,10 @@ msgstr "" msgid "You can group test cases using labels. To learn about the future direction of this feature, visit %{linkStart}Quality Management direction page%{linkEnd}." msgstr "" -msgid "You can have a maximum of %{free_limit} unique members across all of your personal projects. To view and manage members, check the members page for each project in your namespace. We recommend you %{move_link_start}move your projects to a group%{move_link_end} so you can easily manage users and features." -msgstr "" +msgid "You can have a maximum of %{free_user_limit} unique member across all of your personal projects." +msgid_plural "You can have a maximum of %{free_user_limit} unique members across all of your personal projects." +msgstr[0] "" +msgstr[1] "" msgid "You can invite a new member to %{project_name} or invite another group." msgstr "" @@ -44596,9 +44620,6 @@ msgstr "" msgid "You can invite a new member to %{project_name}." msgstr "" -msgid "You can invite a new member to %{strong_start}%{group_name}%{strong_end}." -msgstr "" - msgid "You can invite another group to %{project_name}." msgstr "" @@ -44653,7 +44674,7 @@ msgstr "" msgid "You can view the source or %{linkStart}%{cloneIcon} clone the repository%{linkEnd}" msgstr "" -msgid "You can't add any more, but you can manage your existing members, for example, by removing inactive members and replacing them with new members. To get more members an owner of this namespace can start a trial or upgrade to a paid tier." +msgid "You can't add any more, but you can manage your existing members, for example, by removing inactive members and replacing them with new members. To get more members an owner of the group can start a trial or upgrade to a paid tier." msgstr "" msgid "You cannot %{action} %{state} users." @@ -44701,8 +44722,10 @@ msgstr "" msgid "You could not create a new trigger." msgstr "" -msgid "You currently have more than %{free_limit} members across all your personal projects. From October 19, 2022, the %{free_limit} most recently active members will remain active, and the remaining members will get a %{link_start}status of Over limit%{link_end} and lose access. To view and manage members, check the members page for each project in your namespace. We recommend you %{move_link_start}move your project to a group%{move_link_end} so you can easily manage users and features." -msgstr "" +msgid "You currently have more than %{free_user_limit} member across all your personal projects. From October 19, 2022, the %{free_user_limit} most recently active member will remain active, and the remaining members will have the %{link_start}Over limit status%{link_end} and lose access." +msgid_plural "You currently have more than %{free_user_limit} members across all your personal projects. From October 19, 2022, the %{free_user_limit} most recently active members will remain active, and the remaining members will have the %{link_start}Over limit status%{link_end} and lose access." +msgstr[0] "" +msgstr[1] "" msgid "You do not have any subscriptions yet" msgstr "" @@ -45036,6 +45059,9 @@ msgstr "" msgid "You're receiving this email because you have been mentioned on %{host}. %{unsubscribe_link_start}Unsubscribe%{unsubscribe_link_end} from this thread · %{manage_notifications_link_start}Manage all notifications%{manage_notifications_link_end} · %{help_link_start}Help%{help_link_end}" msgstr "" +msgid "You're viewing members of %{strong_start}%{group_name}%{strong_end}." +msgstr "" + msgid "You've already enabled two-factor authentication using one time password authenticators. In order to register a different device, you must first disable two-factor authentication." msgstr "" @@ -45048,9 +45074,6 @@ msgstr "" msgid "YouTube" msgstr "" -msgid "Your %{doc_link_start}namespace%{doc_link_end}, %{strong_start}%{namespace_name}%{strong_end} has more than %{free_limit} members. From October 19, 2022, it will be limited to %{free_limit}, and the remaining members will get a %{link_start}status of Over limit%{link_end} and lose access to the namespace. You can go to the Usage Quotas page to manage which %{free_limit} members will remain in your namespace. To get more members, an owner can start a trial or upgrade to a paid tier." -msgstr "" - msgid "Your %{group} membership will now expire in %{days}." msgstr "" @@ -45237,6 +45260,11 @@ msgstr "" msgid "Your first project" msgstr "" +msgid "Your group, %{strong_start}%{namespace_name}%{strong_end} has more than %{free_user_limit} member. From October 19, 2022, the %{free_user_limit} most recently active member will remain active, and the remaining members will have the %{link_start}Over limit status%{link_end} and lose access to the group. You can go to the Usage Quotas page to manage which %{free_user_limit} member will remain in your group." +msgid_plural "Your group, %{strong_start}%{namespace_name}%{strong_end} has more than %{free_user_limit} members. From October 19, 2022, the %{free_user_limit} most recently active members will remain active, and the remaining members will have the %{link_start}Over limit status%{link_end} and lose access to the group. You can go to the Usage Quotas page to manage which %{free_user_limit} members will remain in your group." +msgstr[0] "" +msgstr[1] "" + msgid "Your groups" msgstr "" diff --git a/qa/qa/specs/features/browser_ui/1_manage/user/impersonation_token_spec.rb b/qa/qa/specs/features/browser_ui/1_manage/user/impersonation_token_spec.rb index 5547ebff8bc..5bcea1ff094 100644 --- a/qa/qa/specs/features/browser_ui/1_manage/user/impersonation_token_spec.rb +++ b/qa/qa/specs/features/browser_ui/1_manage/user/impersonation_token_spec.rb @@ -17,6 +17,7 @@ module QA testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/368888' ) do impersonation_token = QA::Resource::ImpersonationToken.fabricate_via_browser_ui! do |impersonation_token| + impersonation_token.api_client = admin_api_client impersonation_token.user = user end diff --git a/spec/frontend/design_management/components/design_notes/design_reply_form_spec.js b/spec/frontend/design_management/components/design_notes/design_reply_form_spec.js index d2d1fe6b2d8..38dd2df01a3 100644 --- a/spec/frontend/design_management/components/design_notes/design_reply_form_spec.js +++ b/spec/frontend/design_management/components/design_notes/design_reply_form_spec.js @@ -42,6 +42,18 @@ describe('Design reply form component', () => { expect(findTextarea().element).toEqual(document.activeElement); }); + it('renders "Attach a file or image" button in markdown toolbar', () => { + createComponent(); + + expect(wrapper.find('[data-testid="button-attach-file"]').exists()).toBe(true); + }); + + it('renders file upload progress container', () => { + createComponent(); + + expect(wrapper.find('.comment-toolbar .uploading-container').exists()).toBe(true); + }); + it('renders button text as "Comment" when creating a comment', () => { createComponent(); diff --git a/spec/frontend/editor/schema/ci/yaml_tests/negative_tests/rules.yml b/spec/frontend/editor/schema/ci/yaml_tests/negative_tests/rules.yml new file mode 100644 index 00000000000..76560b8e7ec --- /dev/null +++ b/spec/frontend/editor/schema/ci/yaml_tests/negative_tests/rules.yml @@ -0,0 +1,11 @@ +# unnecessary ref declaration +rules: + - changes: + paths: + - README.md + compare_to: { ref: 'main' } + +# wrong path declaration +rules: + - changes: + paths: { file: 'DOCKER' } diff --git a/spec/frontend/editor/schema/ci/yaml_tests/positive_tests/rules.yml b/spec/frontend/editor/schema/ci/yaml_tests/positive_tests/rules.yml index 27a199cff13..b8d9a55e862 100644 --- a/spec/frontend/editor/schema/ci/yaml_tests/positive_tests/rules.yml +++ b/spec/frontend/editor/schema/ci/yaml_tests/positive_tests/rules.yml @@ -1,13 +1,30 @@ -# Covers https://gitlab.com/gitlab-org/gitlab/-/merge_requests/74164 +# tests for: +# workflow:rules:changes +# workflow:rules:exists +# rules:changes:path + +job_name1: + script: exit 0 + rules: + - changes: + paths: + - README.md + compare_to: main + +job_name2: + script: exit 0 + rules: + - changes: + - README.md -# test for workflow:rules:changes and workflow:rules:exists workflow: rules: + - changes: + paths: + - README.md - if: '$CI_PIPELINE_SOURCE == "schedule"' exists: - Dockerfile - changes: - - Dockerfile variables: IS_A_FEATURE: 'true' when: always diff --git a/spec/frontend/invite_members/components/user_limit_notification_spec.js b/spec/frontend/invite_members/components/user_limit_notification_spec.js index bbc17932a49..543fc28a342 100644 --- a/spec/frontend/invite_members/components/user_limit_notification_spec.js +++ b/spec/frontend/invite_members/components/user_limit_notification_spec.js @@ -9,6 +9,8 @@ import { import { freeUsersLimit, membersCount } from '../mock_data/member_modal'; +const WARNING_ALERT_TITLE = 'You only have space for 2 more members in name'; + describe('UserLimitNotification', () => { let wrapper; @@ -33,7 +35,7 @@ describe('UserLimitNotification', () => { }, ...props, }, - provide: { name: 'my group' }, + provide: { name: 'name' }, stubs: { GlSprintf }, }); }; @@ -50,7 +52,7 @@ describe('UserLimitNotification', () => { }); }); - describe('when close to limit with a personal namepace', () => { + describe('when close to limit within a personal namepace', () => { beforeEach(() => { createComponent(true, false, { membersCount: 3, userNamespace: true }); }); @@ -58,27 +60,24 @@ describe('UserLimitNotification', () => { it('renders the limit for a personal namespace', () => { const alert = findAlert(); - expect(alert.attributes('title')).toEqual( - 'You only have space for 2 more members in your personal projects', - ); + expect(alert.attributes('title')).toEqual(WARNING_ALERT_TITLE); + expect(alert.text()).toEqual( 'To make more space, you can remove members who no longer need access.', ); }); }); - describe('when close to limit', () => { + describe('when close to limit within a group', () => { it("renders user's limit notification", () => { createComponent(true, false, { membersCount: 3 }); const alert = findAlert(); - expect(alert.attributes('title')).toEqual( - 'You only have space for 2 more members in my group', - ); + expect(alert.attributes('title')).toEqual(WARNING_ALERT_TITLE); expect(alert.text()).toEqual( - 'To get more members an owner of this namespace can start a trial or upgrade to a paid tier.', + 'To get more members an owner of the group can start a trial or upgrade to a paid tier.', ); }); }); @@ -89,7 +88,7 @@ describe('UserLimitNotification', () => { const alert = findAlert(); - expect(alert.attributes('title')).toEqual("You've reached your 5 members limit for my group"); + expect(alert.attributes('title')).toEqual("You've reached your 5 members limit for name"); expect(alert.text()).toEqual(REACHED_LIMIT_UPGRADE_SUGGESTION_MESSAGE); }); diff --git a/spec/frontend/vue_shared/components/markdown/header_spec.js b/spec/frontend/vue_shared/components/markdown/header_spec.js index eccf5df82d3..79bbf1fc911 100644 --- a/spec/frontend/vue_shared/components/markdown/header_spec.js +++ b/spec/frontend/vue_shared/components/markdown/header_spec.js @@ -44,11 +44,11 @@ describe('Markdown field header component', () => { describe('markdown header buttons', () => { it('renders the buttons with the correct title', () => { const buttons = [ + 'Insert suggestion', 'Add bold text (⌘B)', 'Add italic text (⌘I)', 'Add strikethrough text (⌘⇧X)', 'Insert a quote', - 'Insert suggestion', 'Insert code', 'Add a link (⌘K)', 'Add a bullet list', diff --git a/spec/frontend/vue_shared/components/project_avatar_spec.js b/spec/frontend/vue_shared/components/project_avatar_spec.js index d55f3127a74..51750808bfd 100644 --- a/spec/frontend/vue_shared/components/project_avatar_spec.js +++ b/spec/frontend/vue_shared/components/project_avatar_spec.js @@ -42,6 +42,16 @@ describe('ProjectAvatar', () => { }); }); + describe('with `projectId` prop', () => { + it('renders GlAvatar with specified `entityId` prop', () => { + const mockProjectId = 1; + createComponent({ props: { projectId: mockProjectId } }); + + const avatar = findGlAvatar(); + expect(avatar.props('entityId')).toBe(mockProjectId); + }); + }); + describe('with `projectAvatarUrl` prop', () => { it('renders GlAvatar with specified `src` prop', () => { const mockProjectAvatarUrl = 'https://gitlab.com'; diff --git a/spec/graphql/mutations/ci/runner/bulk_delete_spec.rb b/spec/graphql/mutations/ci/runner/bulk_delete_spec.rb new file mode 100644 index 00000000000..f47f1b9869e --- /dev/null +++ b/spec/graphql/mutations/ci/runner/bulk_delete_spec.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Mutations::Ci::Runner::BulkDelete do + include GraphqlHelpers + + let_it_be(:admin_user) { create(:user, :admin) } + let_it_be(:user) { create(:user) } + + let(:current_ctx) { { current_user: user } } + + let(:mutation_params) do + {} + end + + describe '#resolve' do + subject(:response) do + sync(resolve(described_class, args: mutation_params, ctx: current_ctx)) + end + + context 'when the user cannot admin the runner' do + let(:runner) { create(:ci_runner) } + let(:mutation_params) do + { ids: [runner.to_global_id] } + end + + it 'generates an error' do + expect_graphql_error_to_be_created(Gitlab::Graphql::Errors::ResourceNotAvailable) { response } + end + end + + context 'when user can delete runners' do + let(:user) { admin_user } + let!(:runners) do + create_list(:ci_runner, 2, :instance) + end + + context 'when required arguments are missing' do + let(:mutation_params) { {} } + + context 'when admin mode is enabled', :enable_admin_mode do + it 'does not return an error' do + is_expected.to match a_hash_including(errors: []) + end + end + end + + context 'with runners specified by id' do + let(:mutation_params) do + { ids: runners.map(&:to_global_id) } + end + + context 'when admin mode is enabled', :enable_admin_mode do + it 'deletes runners', :aggregate_failures do + expect_next_instance_of( + ::Ci::Runners::BulkDeleteRunnersService, { runners: runners } + ) do |service| + expect(service).to receive(:execute).once.and_call_original + end + + expect { response }.to change { Ci::Runner.count }.by(-2) + expect(response[:errors]).to be_empty + end + + context 'when runner list is is above limit' do + before do + stub_const('::Ci::Runners::BulkDeleteRunnersService::RUNNER_LIMIT', 1) + end + + it 'only deletes up to the defined limit', :aggregate_failures do + expect { response }.to change { Ci::Runner.count } + .by(-::Ci::Runners::BulkDeleteRunnersService::RUNNER_LIMIT) + expect(response[:errors]).to be_empty + end + end + end + + context 'when admin mode is disabled', :aggregate_failures do + it 'returns error', :aggregate_failures do + expect do + expect_graphql_error_to_be_created(Gitlab::Graphql::Errors::ResourceNotAvailable) do + response + end + end.not_to change { Ci::Runner.count } + end + end + end + end + end +end diff --git a/spec/graphql/mutations/ci/runner/update_spec.rb b/spec/graphql/mutations/ci/runner/update_spec.rb index ffaa6e93d1b..b8efd4213fa 100644 --- a/spec/graphql/mutations/ci/runner/update_spec.rb +++ b/spec/graphql/mutations/ci/runner/update_spec.rb @@ -2,12 +2,11 @@ require 'spec_helper' -RSpec.describe 'Mutations::Ci::Runner::Update' do +RSpec.describe Mutations::Ci::Runner::Update do include GraphqlHelpers let_it_be(:user) { create(:user) } let_it_be(:runner) { create(:ci_runner, active: true, locked: false, run_untagged: true) } - let_it_be(:described_class) { Mutations::Ci::Runner::Update } let(:current_ctx) { { current_user: user } } let(:mutated_runner) { subject[:runner] } diff --git a/spec/helpers/groups/group_members_helper_spec.rb b/spec/helpers/groups/group_members_helper_spec.rb index 89c26c21338..0d53225bbcf 100644 --- a/spec/helpers/groups/group_members_helper_spec.rb +++ b/spec/helpers/groups/group_members_helper_spec.rb @@ -173,7 +173,7 @@ RSpec.describe Groups::GroupMembersHelper do describe '#group_member_header_subtext' do it 'contains expected text with group name' do - expect(helper.group_member_header_subtext(group)).to match("You can invite a new member to .*#{group.name}") + expect(helper.group_member_header_subtext(group)).to match("You're viewing members of .*#{group.name}") end end end diff --git a/spec/lib/gitlab/background_migration/disable_legacy_open_source_license_for_one_member_no_repo_projects_spec.rb b/spec/lib/gitlab/background_migration/disable_legacy_open_source_license_for_one_member_no_repo_projects_spec.rb new file mode 100644 index 00000000000..0dba1d7c8a2 --- /dev/null +++ b/spec/lib/gitlab/background_migration/disable_legacy_open_source_license_for_one_member_no_repo_projects_spec.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::DisableLegacyOpenSourceLicenseForOneMemberNoRepoProjects, + :migration, + schema: 20220721031446 do + let(:namespaces_table) { table(:namespaces) } + let(:projects_table) { table(:projects) } + let(:project_settings_table) { table(:project_settings) } + let(:project_statistics_table) { table(:project_statistics) } + let(:users_table) { table(:users) } + let(:project_authorizations_table) { table(:project_authorizations) } + + subject(:perform_migration) do + described_class.new(start_id: projects_table.minimum(:id), + end_id: projects_table.maximum(:id), + batch_table: :projects, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: ActiveRecord::Base.connection) + .perform + end + + it 'sets `legacy_open_source_license_available` to false only for public projects with 1 member and no repo', + :aggregate_failures do + project_with_no_repo_one_member = create_legacy_license_public_project('project-with-one-member-no-repo') + project_with_repo_one_member = create_legacy_license_public_project('project-with-repo', repo_size: 1) + project_with_no_repo_two_members = create_legacy_license_public_project('project-with-two-members', members: 2) + project_with_repo_two_members = + create_legacy_license_public_project('project-with-repo', repo_size: 1, members: 2) + + queries = ActiveRecord::QueryRecorder.new { perform_migration } + + expect(queries.count).to eq(7) + expect(migrated_attribute(project_with_no_repo_one_member)).to be_falsey + expect(migrated_attribute(project_with_repo_one_member)).to be_truthy + expect(migrated_attribute(project_with_no_repo_two_members)).to be_truthy + expect(migrated_attribute(project_with_repo_two_members)).to be_truthy + end + + def create_legacy_license_public_project(path, repo_size: 0, members: 1) + namespace = namespaces_table.create!(name: "namespace-#{path}", path: "namespace-#{path}") + project_namespace = + namespaces_table.create!(name: "-project-namespace-#{path}", path: "project-namespace-#{path}", type: 'Project') + project = projects_table + .create!( + name: path, path: path, namespace_id: namespace.id, + project_namespace_id: project_namespace.id, visibility_level: 20 + ) + + members.times do |member_id| + user = users_table.create!(email: "user#{member_id}-project-#{project.id}@gitlab.com", projects_limit: 100) + project_authorizations_table.create!(project_id: project.id, user_id: user.id, access_level: 50) + end + project_statistics_table.create!(project_id: project.id, namespace_id: namespace.id, repository_size: repo_size) + project_settings_table.create!(project_id: project.id, legacy_open_source_license_available: true) + + project + end + + def migrated_attribute(project) + project_settings_table.find(project.id).legacy_open_source_license_available + end +end diff --git a/spec/lib/gitlab/ci/parsers/sbom/cyclonedx_properties_spec.rb b/spec/lib/gitlab/ci/parsers/sbom/cyclonedx_properties_spec.rb new file mode 100644 index 00000000000..c99cfa94aa6 --- /dev/null +++ b/spec/lib/gitlab/ci/parsers/sbom/cyclonedx_properties_spec.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Parsers::Sbom::CyclonedxProperties do + subject(:parse_source) { described_class.parse_source(properties) } + + context 'when properties are nil' do + let(:properties) { nil } + + it { is_expected.to be_nil } + end + + context 'when report does not have gitlab properties' do + let(:properties) { ['name' => 'foo', 'value' => 'bar'] } + + it { is_expected.to be_nil } + end + + context 'when schema_version is missing' do + let(:properties) do + [ + { 'name' => 'gitlab:dependency_scanning:dependency_file', 'value' => 'package-lock.json' }, + { 'name' => 'gitlab:dependency_scanning:package_manager_name', 'value' => 'npm' }, + { 'name' => 'gitlab:dependency_scanning:language', 'value' => 'JavaScript' } + ] + end + + it { is_expected.to be_nil } + end + + context 'when schema version is unsupported' do + let(:properties) do + [ + { 'name' => 'gitlab:meta:schema_version', 'value' => '2' }, + { 'name' => 'gitlab:dependency_scanning:dependency_file', 'value' => 'package-lock.json' }, + { 'name' => 'gitlab:dependency_scanning:package_manager_name', 'value' => 'npm' }, + { 'name' => 'gitlab:dependency_scanning:language', 'value' => 'JavaScript' } + ] + end + + it { is_expected.to be_nil } + end + + context 'when no dependency_scanning properties are present' do + let(:properties) do + [ + { 'name' => 'gitlab:meta:schema_version', 'value' => '1' } + ] + end + + it 'does not call dependency_scanning parser' do + expect(Gitlab::Ci::Parsers::Sbom::Source::DependencyScanning).not_to receive(:parse_source) + + parse_source + end + end + + context 'when dependency_scanning properties are present' do + let(:properties) do + [ + { 'name' => 'gitlab:meta:schema_version', 'value' => '1' }, + { 'name' => 'gitlab:dependency_scanning:category', 'value' => 'development' }, + { 'name' => 'gitlab:dependency_scanning:input_file:path', 'value' => 'package-lock.json' }, + { 'name' => 'gitlab:dependency_scanning:source_file:path', 'value' => 'package.json' }, + { 'name' => 'gitlab:dependency_scanning:package_manager:name', 'value' => 'npm' }, + { 'name' => 'gitlab:dependency_scanning:language:name', 'value' => 'JavaScript' }, + { 'name' => 'gitlab:dependency_scanning:unsupported_property', 'value' => 'Should be ignored' } + ] + end + + let(:expected_input) do + { + 'category' => 'development', + 'input_file' => { 'path' => 'package-lock.json' }, + 'source_file' => { 'path' => 'package.json' }, + 'package_manager' => { 'name' => 'npm' }, + 'language' => { 'name' => 'JavaScript' } + } + end + + it 'passes only supported properties to the dependency scanning parser' do + expect(Gitlab::Ci::Parsers::Sbom::Source::DependencyScanning).to receive(:source).with(expected_input) + + parse_source + end + end +end diff --git a/spec/lib/gitlab/ci/parsers/sbom/cyclonedx_spec.rb b/spec/lib/gitlab/ci/parsers/sbom/cyclonedx_spec.rb index c58e11063b8..cb6d8b62f94 100644 --- a/spec/lib/gitlab/ci/parsers/sbom/cyclonedx_spec.rb +++ b/spec/lib/gitlab/ci/parsers/sbom/cyclonedx_spec.rb @@ -8,6 +8,7 @@ RSpec.describe Gitlab::Ci::Parsers::Sbom::Cyclonedx do let(:raw_report_data) { report_data.to_json } let(:report_valid?) { true } let(:validator_errors) { [] } + let(:properties_parser) { class_double('Gitlab::Ci::Parsers::Sbom::CyclonedxProperties') } let(:base_report_data) do { @@ -24,6 +25,9 @@ RSpec.describe Gitlab::Ci::Parsers::Sbom::Cyclonedx do allow(validator).to receive(:valid?).and_return(report_valid?) allow(validator).to receive(:errors).and_return(validator_errors) end + + allow(properties_parser).to receive(:parse_source) + stub_const('Gitlab::Ci::Parsers::Sbom::CyclonedxProperties', properties_parser) end context 'when report JSON is invalid' do @@ -107,4 +111,25 @@ RSpec.describe Gitlab::Ci::Parsers::Sbom::Cyclonedx do parse! end end + + context 'when report has metadata properties' do + let(:report_data) { base_report_data.merge({ 'metadata' => { 'properties' => properties } }) } + + let(:properties) do + [ + { 'name' => 'gitlab:meta:schema_version', 'value' => '1' }, + { 'name' => 'gitlab:dependency_scanning:category', 'value' => 'development' }, + { 'name' => 'gitlab:dependency_scanning:input_file:path', 'value' => 'package-lock.json' }, + { 'name' => 'gitlab:dependency_scanning:source_file:path', 'value' => 'package.json' }, + { 'name' => 'gitlab:dependency_scanning:package_manager:name', 'value' => 'npm' }, + { 'name' => 'gitlab:dependency_scanning:language:name', 'value' => 'JavaScript' } + ] + end + + it 'passes them to the properties parser' do + expect(properties_parser).to receive(:parse_source).with(properties) + + parse! + end + end end diff --git a/spec/lib/gitlab/ci/parsers/sbom/source/dependency_scanning_spec.rb b/spec/lib/gitlab/ci/parsers/sbom/source/dependency_scanning_spec.rb new file mode 100644 index 00000000000..30114b17cac --- /dev/null +++ b/spec/lib/gitlab/ci/parsers/sbom/source/dependency_scanning_spec.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Parsers::Sbom::Source::DependencyScanning do + subject { described_class.source(property_data) } + + context 'when all property data is present' do + let(:property_data) do + { + 'category' => 'development', + 'input_file' => { 'path' => 'package-lock.json' }, + 'source_file' => { 'path' => 'package.json' }, + 'package_manager' => { 'name' => 'npm' }, + 'language' => { 'name' => 'JavaScript' } + } + end + + it 'returns expected source data' do + is_expected.to eq({ + 'type' => :dependency_scanning, + 'data' => property_data, + 'fingerprint' => '4dbcb747e6f0fb3ed4f48d96b777f1d64acdf43e459fdfefad404e55c004a188' + }) + end + end + + context 'when required properties are missing' do + let(:property_data) do + { + 'category' => 'development', + 'source_file' => { 'path' => 'package.json' }, + 'package_manager' => { 'name' => 'npm' }, + 'language' => { 'name' => 'JavaScript' } + } + end + + it { is_expected.to be_nil } + end +end diff --git a/spec/lib/gitlab/ci/reports/sbom/report_spec.rb b/spec/lib/gitlab/ci/reports/sbom/report_spec.rb index fe1025d16bf..d7a285ab13c 100644 --- a/spec/lib/gitlab/ci/reports/sbom/report_spec.rb +++ b/spec/lib/gitlab/ci/reports/sbom/report_spec.rb @@ -14,35 +14,24 @@ RSpec.describe Gitlab::Ci::Reports::Sbom::Report do end end - describe '#add_source' do - let_it_be(:sources) do - [ - { - 'type' => :dependency_file, - 'data' => { - 'input_file' => { 'name' => 'package-lock.json' }, - 'package_manager' => { 'name' => 'npm' }, - 'language' => { 'name' => 'JavaScript' } - }, - 'fingerprint' => '4ee1623c8f3ddd152b3c1fc340b3ece3cbcf807efa2726307ea34e7d6d36a6c1' + describe '#set_source' do + let_it_be(:source) do + { + 'type' => :dependency_scanning, + 'data' => { + 'input_file' => { 'path' => 'package-lock.json' }, + 'source_file' => { 'path' => 'package.json' }, + 'package_manager' => { 'name' => 'npm' }, + 'language' => { 'name' => 'JavaScript' } }, - { - 'type' => :dependency_file, - 'data' => { - 'input_file' => { 'name' => 'go.sum' }, - 'package_manager' => { 'name' => 'go' }, - 'language' => { 'name' => 'Go' } - }, - 'fingerprint' => 'e78eee13d87248d5b7e3df21de67365a4996b3a547e033b8e8b180b24c300fd8' - } - ] + 'fingerprint' => 'c01df1dc736c1148717e053edbde56cb3a55d3e31f87cea955945b6f67c17d42' + } end - it 'stores each source with the given attributes' do - sources.each { |source| report.add_source(source) } + it 'stores the source' do + report.set_source(source) - expect(report.sources.size).to eq(2) - expect(report.sources).to all(be_a(Gitlab::Ci::Reports::Sbom::Source)) + expect(report.source).to be_a(Gitlab::Ci::Reports::Sbom::Source) end end diff --git a/spec/lib/gitlab/ci/reports/sbom/source_spec.rb b/spec/lib/gitlab/ci/reports/sbom/source_spec.rb index f84ed85b651..2d6434534a0 100644 --- a/spec/lib/gitlab/ci/reports/sbom/source_spec.rb +++ b/spec/lib/gitlab/ci/reports/sbom/source_spec.rb @@ -5,27 +5,25 @@ require 'spec_helper' RSpec.describe Gitlab::Ci::Reports::Sbom::Source do let(:attributes) do { - 'type' => :dependency_file, + 'type' => :dependency_scanning, 'data' => { - 'input_file' => { 'name' => 'package-lock.json' }, + 'category' => 'development', + 'input_file' => { 'path' => 'package-lock.json' }, + 'source_file' => { 'path' => 'package.json' }, 'package_manager' => { 'name' => 'npm' }, 'language' => { 'name' => 'JavaScript' } }, - 'fingerprint' => '4ee1623c8f3ddd152b3c1fc340b3ece3cbcf807efa2726307ea34e7d6d36a6c1' + 'fingerprint' => '4dbcb747e6f0fb3ed4f48d96b777f1d64acdf43e459fdfefad404e55c004a188' } end - subject { described_class.new(**attributes) } + subject { described_class.new(attributes) } it 'has correct attributes' do expect(subject).to have_attributes( - source_type: :dependency_file, - data: { - 'input_file' => { 'name' => 'package-lock.json' }, - 'package_manager' => { 'name' => 'npm' }, - 'language' => { 'name' => 'JavaScript' } - }, - fingerprint: '4ee1623c8f3ddd152b3c1fc340b3ece3cbcf807efa2726307ea34e7d6d36a6c1' + source_type: attributes['type'], + data: attributes['data'], + fingerprint: attributes['fingerprint'] ) end end diff --git a/spec/lib/gitlab/git/remote_repository_spec.rb b/spec/lib/gitlab/git/remote_repository_spec.rb deleted file mode 100644 index c7bc81573a6..00000000000 --- a/spec/lib/gitlab/git/remote_repository_spec.rb +++ /dev/null @@ -1,61 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Git::RemoteRepository, :seed_helper do - let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH, '', 'group/project') } - - subject { described_class.new(repository) } - - describe '#empty?' do - using RSpec::Parameterized::TableSyntax - - where(:repository, :result) do - Gitlab::Git::Repository.new('default', TEST_REPO_PATH, '', 'group/project') | false - Gitlab::Git::Repository.new('default', 'does-not-exist.git', '', 'group/project') | true - end - - with_them do - it { expect(subject.empty?).to eq(result) } - end - end - - describe '#commit_id' do - it 'returns an OID if the revision exists' do - expect(subject.commit_id('v1.0.0')).to eq('6f6d7e7ed97bb5f0054f2b1df789b39ca89b6ff9') - end - - it 'is nil when the revision does not exist' do - expect(subject.commit_id('does-not-exist')).to be_nil - end - end - - describe '#branch_exists?' do - using RSpec::Parameterized::TableSyntax - - where(:branch, :result) do - 'master' | true - 'does-not-exist' | false - end - - with_them do - it { expect(subject.branch_exists?(branch)).to eq(result) } - end - end - - describe '#same_repository?' do - using RSpec::Parameterized::TableSyntax - - where(:other_repository, :result) do - repository | true - Gitlab::Git::Repository.new(repository.storage, repository.relative_path, '', 'group/project') | true - Gitlab::Git::Repository.new('broken', TEST_REPO_PATH, '', 'group/project') | false - Gitlab::Git::Repository.new(repository.storage, 'wrong/relative-path.git', '', 'group/project') | false - Gitlab::Git::Repository.new('broken', 'wrong/relative-path.git', '', 'group/project') | false - end - - with_them do - it { expect(subject.same_repository?(other_repository)).to eq(result) } - end - end -end diff --git a/spec/migrations/20220721031446_schedule_disable_legacy_open_source_license_for_one_member_no_repo_projects_spec.rb b/spec/migrations/20220721031446_schedule_disable_legacy_open_source_license_for_one_member_no_repo_projects_spec.rb new file mode 100644 index 00000000000..b17a0215f4e --- /dev/null +++ b/spec/migrations/20220721031446_schedule_disable_legacy_open_source_license_for_one_member_no_repo_projects_spec.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe ScheduleDisableLegacyOpenSourceLicenseForOneMemberNoRepoProjects do + context 'when on gitlab.com' do + let(:migration) { described_class::MIGRATION } + + before do + allow(Gitlab).to receive(:com?).and_return(true) + end + + describe '#up' do + it 'schedules background jobs for each batch of projects' do + migrate! + + expect(migration).to( + have_scheduled_batched_migration( + table_name: :projects, + column_name: :id, + interval: described_class::INTERVAL, + batch_size: described_class::BATCH_SIZE, + max_batch_size: described_class::MAX_BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE + ) + ) + end + end + + describe '#down' do + it 'deletes all batched migration records' do + migrate! + schema_migrate_down! + + expect(migration).not_to have_scheduled_batched_migration + end + end + end + + context 'when on self-managed instance' do + let(:migration) { described_class.new } + + before do + allow(Gitlab).to receive(:com?).and_return(false) + end + + describe '#up' do + it 'does not schedule background job' do + expect(migration).not_to receive(:queue_batched_background_migration) + + migration.up + end + end + + describe '#down' do + it 'does not delete background job' do + expect(migration).not_to receive(:delete_batched_background_migration) + + migration.down + end + end + end +end diff --git a/spec/requests/api/personal_access_tokens_spec.rb b/spec/requests/api/personal_access_tokens_spec.rb index f261e7db024..8d8998cfdd6 100644 --- a/spec/requests/api/personal_access_tokens_spec.rb +++ b/spec/requests/api/personal_access_tokens_spec.rb @@ -100,7 +100,7 @@ RSpec.describe API::PersonalAccessTokens do it 'fails to return PAT because no PAT exists with this id' do get api(invalid_path, admin_user) - expect(response).to have_gitlab_http_status(:unauthorized) + expect(response).to have_gitlab_http_status(:not_found) end end diff --git a/spec/views/groups/group_members/index.html.haml_spec.rb b/spec/views/groups/group_members/index.html.haml_spec.rb index 2d7d50555d6..c7aebb94a45 100644 --- a/spec/views/groups/group_members/index.html.haml_spec.rb +++ b/spec/views/groups/group_members/index.html.haml_spec.rb @@ -21,7 +21,7 @@ RSpec.describe 'groups/group_members/index', :aggregate_failures do render expect(rendered).to have_content('Group members') - expect(rendered).to have_content('You can invite a new member') + expect(rendered).to have_content("You're viewing members") expect(rendered).to have_selector('.js-invite-group-trigger') expect(rendered).to have_selector('.js-invite-members-trigger') |
