diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-07-19 15:10:08 +0300 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-07-19 15:10:08 +0300 |
| commit | 1072f96e340ddad78e5dad6dfedc7c6e85fc53ea (patch) | |
| tree | 573525bb3525f79cbb011f39ad11c11c6efea315 | |
| parent | 8cc0a0aa965798e74826197d350472d235af2f84 (diff) | |
Add latest changes from gitlab-org/gitlab@master
64 files changed, 1030 insertions, 818 deletions
diff --git a/.gitlab/ci/reports.gitlab-ci.yml b/.gitlab/ci/reports.gitlab-ci.yml index 7671a1a8650..690a971927c 100644 --- a/.gitlab/ci/reports.gitlab-ci.yml +++ b/.gitlab/ci/reports.gitlab-ci.yml @@ -92,7 +92,7 @@ gemnasium-python-dependency_scanning: - .default-retry stage: test image: - name: registry.gitlab.com/gitlab-com/gl-security/security-research/package-hunter-cli:latest + name: registry.gitlab.com/gitlab-com/gl-security/security-research/package-hunter-cli:1.1.0 entrypoint: [""] variables: DEBUG: '*' diff --git a/.rubocop_manual_todo.yml b/.rubocop_manual_todo.yml index c417ea92cc0..61593c3b464 100644 --- a/.rubocop_manual_todo.yml +++ b/.rubocop_manual_todo.yml @@ -454,7 +454,6 @@ RSpec/AnyInstanceOf: - 'ee/spec/features/admin/admin_audit_logs_spec.rb' - 'ee/spec/features/admin/admin_reset_pipeline_minutes_spec.rb' - 'ee/spec/features/admin/admin_users_spec.rb' - - 'ee/spec/features/admin/licenses/admin_views_license_spec.rb' - 'ee/spec/features/boards/scoped_issue_board_spec.rb' - 'ee/spec/features/ci_shared_runner_warnings_spec.rb' - 'ee/spec/features/groups/group_settings_spec.rb' @@ -2621,3 +2620,397 @@ Database/MultipleDatabases: - 'spec/tasks/gitlab/db_rake_spec.rb' - 'spec/workers/analytics/usage_trends/counter_job_worker_spec.rb' - 'spec/workers/users/create_statistics_worker_spec.rb' + +# See https://gitlab.com/gitlab-org/gitlab/-/issues/335170 +Cop/WorkerDataConsistency: + Exclude: + - 'app/workers/admin_email_worker.rb' + - 'app/workers/analytics/usage_trends/count_job_trigger_worker.rb' + - 'app/workers/analytics/usage_trends/counter_job_worker.rb' + - 'app/workers/approve_blocked_pending_approval_users_worker.rb' + - 'app/workers/archive_trace_worker.rb' + - 'app/workers/authorized_keys_worker.rb' + - 'app/workers/authorized_project_update/periodic_recalculate_worker.rb' + - 'app/workers/authorized_project_update/project_create_worker.rb' + - 'app/workers/authorized_project_update/project_group_link_create_worker.rb' + - 'app/workers/authorized_project_update/project_recalculate_worker.rb' + - 'app/workers/authorized_project_update/user_refresh_from_replica_worker.rb' + - 'app/workers/authorized_projects_worker.rb' + - 'app/workers/auto_devops/disable_worker.rb' + - 'app/workers/auto_merge_process_worker.rb' + - 'app/workers/background_migration_worker.rb' + - 'app/workers/build_finished_worker.rb' + - 'app/workers/build_success_worker.rb' + - 'app/workers/bulk_import_worker.rb' + - 'app/workers/bulk_imports/entity_worker.rb' + - 'app/workers/bulk_imports/export_request_worker.rb' + - 'app/workers/bulk_imports/pipeline_worker.rb' + - 'app/workers/bulk_imports/relation_export_worker.rb' + - 'app/workers/chaos/cpu_spin_worker.rb' + - 'app/workers/chaos/db_spin_worker.rb' + - 'app/workers/chaos/kill_worker.rb' + - 'app/workers/chaos/leak_mem_worker.rb' + - 'app/workers/chaos/sleep_worker.rb' + - 'app/workers/chat_notification_worker.rb' + - 'app/workers/ci/archive_trace_worker.rb' + - 'app/workers/ci/archive_traces_cron_worker.rb' + - 'app/workers/ci/build_finished_worker.rb' + - 'app/workers/ci/build_prepare_worker.rb' + - 'app/workers/ci/build_schedule_worker.rb' + - 'app/workers/ci/build_trace_chunk_flush_worker.rb' + - 'app/workers/ci/daily_build_group_report_results_worker.rb' + - 'app/workers/ci/delete_objects_worker.rb' + - 'app/workers/ci/delete_unit_tests_worker.rb' + - 'app/workers/ci/drop_pipeline_worker.rb' + - 'app/workers/ci/initial_pipeline_process_worker.rb' + - 'app/workers/ci/merge_requests/add_todo_when_build_fails_worker.rb' + - 'app/workers/ci/pipeline_artifacts/coverage_report_worker.rb' + - 'app/workers/ci/pipeline_artifacts/create_quality_report_worker.rb' + - 'app/workers/ci/pipeline_artifacts/expire_artifacts_worker.rb' + - 'app/workers/ci/pipeline_success_unlock_artifacts_worker.rb' + - 'app/workers/ci/ref_delete_unlock_artifacts_worker.rb' + - 'app/workers/ci/resource_groups/assign_resource_from_resource_group_worker.rb' + - 'app/workers/ci/schedule_delete_objects_cron_worker.rb' + - 'app/workers/ci/test_failure_history_worker.rb' + - 'app/workers/ci_platform_metrics_update_cron_worker.rb' + - 'app/workers/cleanup_container_repository_worker.rb' + - 'app/workers/cluster_configure_istio_worker.rb' + - 'app/workers/cluster_install_app_worker.rb' + - 'app/workers/cluster_patch_app_worker.rb' + - 'app/workers/cluster_provision_worker.rb' + - 'app/workers/cluster_update_app_worker.rb' + - 'app/workers/cluster_upgrade_app_worker.rb' + - 'app/workers/cluster_wait_for_app_installation_worker.rb' + - 'app/workers/cluster_wait_for_app_update_worker.rb' + - 'app/workers/cluster_wait_for_ingress_ip_address_worker.rb' + - 'app/workers/clusters/applications/activate_service_worker.rb' + - 'app/workers/clusters/applications/check_prometheus_health_worker.rb' + - 'app/workers/clusters/applications/deactivate_service_worker.rb' + - 'app/workers/clusters/applications/uninstall_worker.rb' + - 'app/workers/clusters/applications/wait_for_uninstall_app_worker.rb' + - 'app/workers/container_expiration_policies/cleanup_container_repository_worker.rb' + - 'app/workers/container_expiration_policy_worker.rb' + - 'app/workers/create_commit_signature_worker.rb' + - 'app/workers/create_note_diff_file_worker.rb' + - 'app/workers/create_pipeline_worker.rb' + - 'app/workers/database/batched_background_migration_worker.rb' + - 'app/workers/database/partition_management_worker.rb' + - 'app/workers/delete_container_repository_worker.rb' + - 'app/workers/delete_diff_files_worker.rb' + - 'app/workers/delete_merged_branches_worker.rb' + - 'app/workers/delete_stored_files_worker.rb' + - 'app/workers/delete_user_worker.rb' + - 'app/workers/deployments/drop_older_deployments_worker.rb' + - 'app/workers/deployments/finished_worker.rb' + - 'app/workers/deployments/forward_deployment_worker.rb' + - 'app/workers/deployments/hooks_worker.rb' + - 'app/workers/deployments/link_merge_request_worker.rb' + - 'app/workers/deployments/success_worker.rb' + - 'app/workers/deployments/update_environment_worker.rb' + - 'app/workers/design_management/copy_design_collection_worker.rb' + - 'app/workers/design_management/new_version_worker.rb' + - 'app/workers/destroy_pages_deployments_worker.rb' + - 'app/workers/detect_repository_languages_worker.rb' + - 'app/workers/disallow_two_factor_for_group_worker.rb' + - 'app/workers/disallow_two_factor_for_subgroups_worker.rb' + - 'app/workers/email_receiver_worker.rb' + - 'app/workers/emails_on_push_worker.rb' + - 'app/workers/environments/auto_stop_cron_worker.rb' + - 'app/workers/environments/canary_ingress/update_worker.rb' + - 'app/workers/error_tracking_issue_link_worker.rb' + - 'app/workers/experiments/record_conversion_event_worker.rb' + - 'app/workers/expire_build_artifacts_worker.rb' + - 'app/workers/expire_build_instance_artifacts_worker.rb' + - 'app/workers/expire_job_cache_worker.rb' + - 'app/workers/export_csv_worker.rb' + - 'app/workers/file_hook_worker.rb' + - 'app/workers/flush_counter_increments_worker.rb' + - 'app/workers/gitlab/github_import/advance_stage_worker.rb' + - 'app/workers/gitlab/github_import/refresh_import_jid_worker.rb' + - 'app/workers/gitlab/github_import/stage/finish_import_worker.rb' + - 'app/workers/gitlab/github_import/stage/import_base_data_worker.rb' + - 'app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb' + - 'app/workers/gitlab/github_import/stage/import_lfs_objects_worker.rb' + - 'app/workers/gitlab/github_import/stage/import_notes_worker.rb' + - 'app/workers/gitlab/github_import/stage/import_pull_requests_merged_by_worker.rb' + - 'app/workers/gitlab/github_import/stage/import_pull_requests_reviews_worker.rb' + - 'app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb' + - 'app/workers/gitlab/github_import/stage/import_repository_worker.rb' + - 'app/workers/gitlab/jira_import/advance_stage_worker.rb' + - 'app/workers/gitlab/jira_import/import_issue_worker.rb' + - 'app/workers/gitlab/jira_import/stage/start_import_worker.rb' + - 'app/workers/gitlab/phabricator_import/import_tasks_worker.rb' + - 'app/workers/gitlab_performance_bar_stats_worker.rb' + - 'app/workers/gitlab_service_ping_worker.rb' + - 'app/workers/gitlab_shell_worker.rb' + - 'app/workers/group_destroy_worker.rb' + - 'app/workers/group_export_worker.rb' + - 'app/workers/group_import_worker.rb' + - 'app/workers/hashed_storage/migrator_worker.rb' + - 'app/workers/hashed_storage/project_migrate_worker.rb' + - 'app/workers/hashed_storage/project_rollback_worker.rb' + - 'app/workers/hashed_storage/rollbacker_worker.rb' + - 'app/workers/import_export_project_cleanup_worker.rb' + - 'app/workers/import_issues_csv_worker.rb' + - 'app/workers/incident_management/add_severity_system_note_worker.rb' + - 'app/workers/incident_management/pager_duty/process_incident_worker.rb' + - 'app/workers/incident_management/process_alert_worker_v2.rb' + - 'app/workers/invalid_gpg_signature_update_worker.rb' + - 'app/workers/irker_worker.rb' + - 'app/workers/issuable/label_links_destroy_worker.rb' + - 'app/workers/issuable_export_csv_worker.rb' + - 'app/workers/issuables/clear_groups_issue_counter_worker.rb' + - 'app/workers/issue_due_scheduler_worker.rb' + - 'app/workers/issue_placement_worker.rb' + - 'app/workers/issue_rebalancing_worker.rb' + - 'app/workers/jira_connect/forward_event_worker.rb' + - 'app/workers/jira_connect/sync_branch_worker.rb' + - 'app/workers/jira_connect/sync_builds_worker.rb' + - 'app/workers/jira_connect/sync_deployments_worker.rb' + - 'app/workers/jira_connect/sync_feature_flags_worker.rb' + - 'app/workers/jira_connect/sync_merge_request_worker.rb' + - 'app/workers/jira_connect/sync_project_worker.rb' + - 'app/workers/mail_scheduler/issue_due_worker.rb' + - 'app/workers/mail_scheduler/notification_service_worker.rb' + - 'app/workers/member_invitation_reminder_emails_worker.rb' + - 'app/workers/members_destroyer/unassign_issuables_worker.rb' + - 'app/workers/merge_request_cleanup_refs_worker.rb' + - 'app/workers/merge_request_mergeability_check_worker.rb' + - 'app/workers/merge_requests/create_pipeline_worker.rb' + - 'app/workers/merge_requests/delete_source_branch_worker.rb' + - 'app/workers/merge_requests/handle_assignees_change_worker.rb' + - 'app/workers/merge_requests/resolve_todos_worker.rb' + - 'app/workers/merge_worker.rb' + - 'app/workers/metrics/dashboard/prune_old_annotations_worker.rb' + - 'app/workers/metrics/dashboard/schedule_annotations_prune_worker.rb' + - 'app/workers/metrics/dashboard/sync_dashboards_worker.rb' + - 'app/workers/migrate_external_diffs_worker.rb' + - 'app/workers/namespaceless_project_destroy_worker.rb' + - 'app/workers/namespaces/in_product_marketing_emails_worker.rb' + - 'app/workers/namespaces/onboarding_issue_created_worker.rb' + - 'app/workers/namespaces/onboarding_pipeline_created_worker.rb' + - 'app/workers/namespaces/onboarding_progress_worker.rb' + - 'app/workers/namespaces/onboarding_user_added_worker.rb' + - 'app/workers/namespaces/prune_aggregation_schedules_worker.rb' + - 'app/workers/namespaces/root_statistics_worker.rb' + - 'app/workers/namespaces/schedule_aggregation_worker.rb' + - 'app/workers/new_issue_worker.rb' + - 'app/workers/new_merge_request_worker.rb' + - 'app/workers/new_note_worker.rb' + - 'app/workers/object_pool/create_worker.rb' + - 'app/workers/object_pool/destroy_worker.rb' + - 'app/workers/object_pool/join_worker.rb' + - 'app/workers/object_pool/schedule_join_worker.rb' + - 'app/workers/object_storage/background_move_worker.rb' + - 'app/workers/object_storage/migrate_uploads_worker.rb' + - 'app/workers/packages/composer/cache_cleanup_worker.rb' + - 'app/workers/packages/composer/cache_update_worker.rb' + - 'app/workers/packages/debian/generate_distribution_worker.rb' + - 'app/workers/packages/debian/process_changes_worker.rb' + - 'app/workers/packages/go/sync_packages_worker.rb' + - 'app/workers/packages/helm/extraction_worker.rb' + - 'app/workers/packages/maven/metadata/sync_worker.rb' + - 'app/workers/packages/nuget/extraction_worker.rb' + - 'app/workers/packages/rubygems/extraction_worker.rb' + - 'app/workers/pages_domain_removal_cron_worker.rb' + - 'app/workers/pages_domain_ssl_renewal_cron_worker.rb' + - 'app/workers/pages_domain_ssl_renewal_worker.rb' + - 'app/workers/pages_domain_verification_cron_worker.rb' + - 'app/workers/pages_domain_verification_worker.rb' + - 'app/workers/pages_remove_worker.rb' + - 'app/workers/pages_transfer_worker.rb' + - 'app/workers/pages_update_configuration_worker.rb' + - 'app/workers/pages_worker.rb' + - 'app/workers/partition_creation_worker.rb' + - 'app/workers/personal_access_tokens/expired_notification_worker.rb' + - 'app/workers/personal_access_tokens/expiring_worker.rb' + - 'app/workers/pipeline_metrics_worker.rb' + - 'app/workers/pipeline_notification_worker.rb' + - 'app/workers/pipeline_process_worker.rb' + - 'app/workers/pipeline_schedule_worker.rb' + - 'app/workers/post_receive.rb' + - 'app/workers/process_commit_worker.rb' + - 'app/workers/project_cache_worker.rb' + - 'app/workers/project_daily_statistics_worker.rb' + - 'app/workers/project_destroy_worker.rb' + - 'app/workers/project_export_worker.rb' + - 'app/workers/project_service_worker.rb' + - 'app/workers/projects/post_creation_worker.rb' + - 'app/workers/projects/schedule_bulk_repository_shard_moves_worker.rb' + - 'app/workers/prometheus/create_default_alerts_worker.rb' + - 'app/workers/propagate_integration_group_worker.rb' + - 'app/workers/propagate_integration_inherit_descendant_worker.rb' + - 'app/workers/propagate_integration_inherit_worker.rb' + - 'app/workers/propagate_integration_project_worker.rb' + - 'app/workers/propagate_integration_worker.rb' + - 'app/workers/propagate_service_template_worker.rb' + - 'app/workers/prune_old_events_worker.rb' + - 'app/workers/purge_dependency_proxy_cache_worker.rb' + - 'app/workers/rebase_worker.rb' + - 'app/workers/releases/create_evidence_worker.rb' + - 'app/workers/releases/manage_evidence_worker.rb' + - 'app/workers/remote_mirror_notification_worker.rb' + - 'app/workers/remove_expired_group_links_worker.rb' + - 'app/workers/remove_expired_members_worker.rb' + - 'app/workers/remove_unaccepted_member_invites_worker.rb' + - 'app/workers/remove_unreferenced_lfs_objects_worker.rb' + - 'app/workers/repository_archive_cache_worker.rb' + - 'app/workers/repository_check/batch_worker.rb' + - 'app/workers/repository_check/clear_worker.rb' + - 'app/workers/repository_check/dispatch_worker.rb' + - 'app/workers/repository_check/single_repository_worker.rb' + - 'app/workers/repository_cleanup_worker.rb' + - 'app/workers/repository_fork_worker.rb' + - 'app/workers/repository_import_worker.rb' + - 'app/workers/repository_remove_remote_worker.rb' + - 'app/workers/repository_update_remote_mirror_worker.rb' + - 'app/workers/requests_profiles_worker.rb' + - 'app/workers/run_pipeline_schedule_worker.rb' + - 'app/workers/schedule_merge_request_cleanup_refs_worker.rb' + - 'app/workers/schedule_migrate_external_diffs_worker.rb' + - 'app/workers/self_monitoring_project_create_worker.rb' + - 'app/workers/self_monitoring_project_delete_worker.rb' + - 'app/workers/service_desk_email_receiver_worker.rb' + - 'app/workers/snippets/schedule_bulk_repository_shard_moves_worker.rb' + - 'app/workers/ssh_keys/expired_notification_worker.rb' + - 'app/workers/ssh_keys/expiring_soon_notification_worker.rb' + - 'app/workers/stage_update_worker.rb' + - 'app/workers/stuck_ci_jobs_worker.rb' + - 'app/workers/stuck_export_jobs_worker.rb' + - 'app/workers/stuck_merge_jobs_worker.rb' + - 'app/workers/system_hook_push_worker.rb' + - 'app/workers/todos_destroyer/confidential_issue_worker.rb' + - 'app/workers/todos_destroyer/destroyed_issuable_worker.rb' + - 'app/workers/todos_destroyer/entity_leave_worker.rb' + - 'app/workers/todos_destroyer/group_private_worker.rb' + - 'app/workers/todos_destroyer/private_features_worker.rb' + - 'app/workers/todos_destroyer/project_private_worker.rb' + - 'app/workers/trending_projects_worker.rb' + - 'app/workers/update_container_registry_info_worker.rb' + - 'app/workers/update_external_pull_requests_worker.rb' + - 'app/workers/update_head_pipeline_for_merge_request_worker.rb' + - 'app/workers/update_highest_role_worker.rb' + - 'app/workers/update_merge_requests_worker.rb' + - 'app/workers/update_project_statistics_worker.rb' + - 'app/workers/upload_checksum_worker.rb' + - 'app/workers/user_status_cleanup/batch_worker.rb' + - 'app/workers/users/create_statistics_worker.rb' + - 'app/workers/users/deactivate_dormant_users_worker.rb' + - 'app/workers/wait_for_cluster_creation_worker.rb' + - 'app/workers/web_hooks/destroy_worker.rb' + - 'app/workers/web_hooks/log_execution_worker.rb' + - 'app/workers/x509_certificate_revoke_worker.rb' + - 'app/workers/x509_issuer_crl_check_worker.rb' + - 'ee/app/workers/active_user_count_threshold_worker.rb' + - 'ee/app/workers/adjourned_group_deletion_worker.rb' + - 'ee/app/workers/adjourned_project_deletion_worker.rb' + - 'ee/app/workers/adjourned_projects_deletion_cron_worker.rb' + - 'ee/app/workers/admin_emails_worker.rb' + - 'ee/app/workers/analytics/code_review_metrics_worker.rb' + - 'ee/app/workers/analytics/devops_adoption/create_all_snapshots_worker.rb' + - 'ee/app/workers/analytics/devops_adoption/create_snapshot_worker.rb' + - 'ee/app/workers/approval_rules/external_approval_rule_payload_worker.rb' + - 'ee/app/workers/ci/batch_reset_minutes_worker.rb' + - 'ee/app/workers/ci/sync_reports_to_report_approval_rules_worker.rb' + - 'ee/app/workers/clear_shared_runners_minutes_worker.rb' + - 'ee/app/workers/create_github_webhook_worker.rb' + - 'ee/app/workers/dast_site_validation_worker.rb' + - 'ee/app/workers/deployments/auto_rollback_worker.rb' + - 'ee/app/workers/dora/daily_metrics/refresh_worker.rb' + - 'ee/app/workers/dora/daily_metrics/refresh_worker.rb' + - 'ee/app/workers/elastic/migration_worker.rb' + - 'ee/app/workers/elastic_association_indexer_worker.rb' + - 'ee/app/workers/elastic_cluster_reindexing_cron_worker.rb' + - 'ee/app/workers/elastic_commit_indexer_worker.rb' + - 'ee/app/workers/elastic_delete_project_worker.rb' + - 'ee/app/workers/elastic_full_index_worker.rb' + - 'ee/app/workers/elastic_indexing_control_worker.rb' + - 'ee/app/workers/elastic_namespace_indexer_worker.rb' + - 'ee/app/workers/elastic_namespace_rollout_worker.rb' + - 'ee/app/workers/elastic_remove_expired_namespace_subscriptions_from_index_cron_worker.rb' + - 'ee/app/workers/epics/new_epic_issue_worker.rb' + - 'ee/app/workers/epics/update_epics_dates_worker.rb' + - 'ee/app/workers/geo/batch/project_registry_scheduler_worker.rb' + - 'ee/app/workers/geo/batch/project_registry_worker.rb' + - 'ee/app/workers/geo/container_repository_sync_worker.rb' + - 'ee/app/workers/geo/design_repository_sync_worker.rb' + - 'ee/app/workers/geo/destroy_worker.rb' + - 'ee/app/workers/geo/event_worker.rb' + - 'ee/app/workers/geo/file_download_worker.rb' + - 'ee/app/workers/geo/file_registry_removal_worker.rb' + - 'ee/app/workers/geo/file_removal_worker.rb' + - 'ee/app/workers/geo/hashed_storage_attachments_migration_worker.rb' + - 'ee/app/workers/geo/hashed_storage_migration_worker.rb' + - 'ee/app/workers/geo/metrics_update_worker.rb' + - 'ee/app/workers/geo/project_sync_worker.rb' + - 'ee/app/workers/geo/prune_event_log_worker.rb' + - 'ee/app/workers/geo/rename_repository_worker.rb' + - 'ee/app/workers/geo/repositories_clean_up_worker.rb' + - 'ee/app/workers/geo/repository_cleanup_worker.rb' + - 'ee/app/workers/geo/repository_verification/primary/single_worker.rb' + - 'ee/app/workers/geo/repository_verification/secondary/single_worker.rb' + - 'ee/app/workers/geo/reverification_batch_worker.rb' + - 'ee/app/workers/geo/scheduler/per_shard_scheduler_worker.rb' + - 'ee/app/workers/geo/scheduler/scheduler_worker.rb' + - 'ee/app/workers/geo/secondary/registry_consistency_worker.rb' + - 'ee/app/workers/geo/secondary_usage_data_cron_worker.rb' + - 'ee/app/workers/geo/sidekiq_cron_config_worker.rb' + - 'ee/app/workers/geo/sync_timeout_cron_worker.rb' + - 'ee/app/workers/geo/verification_batch_worker.rb' + - 'ee/app/workers/geo/verification_cron_worker.rb' + - 'ee/app/workers/geo/verification_timeout_worker.rb' + - 'ee/app/workers/geo/verification_worker.rb' + - 'ee/app/workers/geo_repository_destroy_worker.rb' + - 'ee/app/workers/group_saml_group_sync_worker.rb' + - 'ee/app/workers/groups/schedule_bulk_repository_shard_moves_worker.rb' + - 'ee/app/workers/groups/update_statistics_worker.rb' + - 'ee/app/workers/historical_data_worker.rb' + - 'ee/app/workers/import_software_licenses_worker.rb' + - 'ee/app/workers/incident_management/apply_incident_sla_exceeded_label_worker.rb' + - 'ee/app/workers/incident_management/incident_sla_exceeded_check_worker.rb' + - 'ee/app/workers/incident_management/oncall_rotations/persist_all_rotations_shifts_job.rb' + - 'ee/app/workers/incident_management/oncall_rotations/persist_shifts_job.rb' + - 'ee/app/workers/incident_management/pending_escalations/alert_check_worker.rb' + - 'ee/app/workers/incident_management/pending_escalations/alert_create_worker.rb' + - 'ee/app/workers/incident_management/pending_escalations/schedule_check_cron_worker.rb' + - 'ee/app/workers/iterations/cadences/create_iterations_worker.rb' + - 'ee/app/workers/iterations/cadences/schedule_create_iterations_worker.rb' + - 'ee/app/workers/iterations/roll_over_issues_worker.rb' + - 'ee/app/workers/iterations_update_status_worker.rb' + - 'ee/app/workers/ldap_all_groups_sync_worker.rb' + - 'ee/app/workers/ldap_group_sync_worker.rb' + - 'ee/app/workers/ldap_sync_worker.rb' + - 'ee/app/workers/merge_request_reset_approvals_worker.rb' + - 'ee/app/workers/merge_requests/sync_code_owner_approval_rules_worker.rb' + - 'ee/app/workers/merge_trains/refresh_worker.rb' + - 'ee/app/workers/network_policy_metrics_worker.rb' + - 'ee/app/workers/new_epic_worker.rb' + - 'ee/app/workers/personal_access_tokens/groups/policy_worker.rb' + - 'ee/app/workers/personal_access_tokens/instance/policy_worker.rb' + - 'ee/app/workers/project_import_schedule_worker.rb' + - 'ee/app/workers/pseudonymizer_worker.rb' + - 'ee/app/workers/refresh_license_compliance_checks_worker.rb' + - 'ee/app/workers/repository_push_audit_event_worker.rb' + - 'ee/app/workers/repository_update_mirror_worker.rb' + - 'ee/app/workers/requirements_management/import_requirements_csv_worker.rb' + - 'ee/app/workers/requirements_management/process_requirements_reports_worker.rb' + - 'ee/app/workers/scan_security_report_secrets_worker.rb' + - 'ee/app/workers/security/auto_fix_worker.rb' + - 'ee/app/workers/security/create_orchestration_policy_worker.rb' + - 'ee/app/workers/security/orchestration_policy_rule_schedule_worker.rb' + - 'ee/app/workers/security/store_scans_worker.rb' + - 'ee/app/workers/security/track_secure_scans_worker.rb' + - 'ee/app/workers/set_user_status_based_on_user_cap_setting_worker.rb' + - 'ee/app/workers/status_page/publish_worker.rb' + - 'ee/app/workers/store_security_reports_worker.rb' + - 'ee/app/workers/store_security_scans_worker.rb' + - 'ee/app/workers/sync_seat_link_request_worker.rb' + - 'ee/app/workers/sync_seat_link_worker.rb' + - 'ee/app/workers/todos_destroyer/confidential_epic_worker.rb' + - 'ee/app/workers/update_all_mirrors_worker.rb' + - 'ee/app/workers/update_max_seats_used_for_gitlab_com_subscriptions_worker.rb' + - 'ee/app/workers/vulnerabilities/historical_statistics/deletion_worker.rb' + - 'ee/app/workers/vulnerabilities/statistics/adjustment_worker.rb' + - 'ee/app/workers/vulnerabilities/statistics/schedule_worker.rb' + - 'ee/app/workers/vulnerability_exports/export_deletion_worker.rb' + - 'ee/app/workers/vulnerability_exports/export_worker.rb' @@ -472,7 +472,7 @@ end gem 'spamcheck', '~> 0.1.0' # Gitaly GRPC protocol definitions -gem 'gitaly', '~> 14.1.0.pre.rc2' +gem 'gitaly', '~> 14.1.0.pre.rc3' # KAS GRPC protocol definitions gem 'kas-grpc', '~> 0.0.2' diff --git a/Gemfile.lock b/Gemfile.lock index 1c1fc949882..e3bb0584149 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -460,7 +460,7 @@ GEM rails (>= 3.2.0) git (1.7.0) rchardet (~> 1.8) - gitaly (14.1.0.pre.rc2) + gitaly (14.1.0.pre.rc3) grpc (~> 1.0) github-markup (1.7.0) gitlab (4.16.1) @@ -1482,7 +1482,7 @@ DEPENDENCIES gettext (~> 3.3) gettext_i18n_rails (~> 1.8.0) gettext_i18n_rails_js (~> 1.3) - gitaly (~> 14.1.0.pre.rc2) + gitaly (~> 14.1.0.pre.rc3) github-markup (~> 1.7.0) gitlab-chronic (~> 0.10.5) gitlab-dangerfiles (~> 2.2.2) diff --git a/app/assets/javascripts/locale/index.js b/app/assets/javascripts/locale/index.js index 10518fa73d9..ad01da2eb17 100644 --- a/app/assets/javascripts/locale/index.js +++ b/app/assets/javascripts/locale/index.js @@ -2,7 +2,10 @@ import Jed from 'jed'; import ensureSingleLine from './ensure_single_line'; import sprintf from './sprintf'; -const languageCode = () => document.querySelector('html').getAttribute('lang') || 'en'; +const GITLAB_FALLBACK_LANGUAGE = 'en'; + +const languageCode = () => + document.querySelector('html').getAttribute('lang') || GITLAB_FALLBACK_LANGUAGE; const locale = new Jed(window.translations || {}); delete window.translations; @@ -51,12 +54,52 @@ const pgettext = (keyOrContext, key) => { }; /** + * Filters navigator languages by the set GitLab language. + * + * This allows us to decide better what a user wants as a locale, for using with the Intl browser APIs. + * If they have set their GitLab to a language, it will check whether `navigator.languages` contains matching ones. + * This function always adds `en` as a fallback in order to have date renders if all fails before it. + * + * - Example one: GitLab language is `en` and browser languages are: + * `['en-GB', 'en-US']`. This function returns `['en-GB', 'en-US', 'en']` as + * the preferred locales, the Intl APIs would try to format first as British English, + * if that isn't available US or any English. + * - Example two: GitLab language is `en` and browser languages are: + * `['de-DE', 'de']`. This function returns `['en']`, so the Intl APIs would prefer English + * formatting in order to not have German dates mixed with English GitLab UI texts. + * If the user wants for example British English formatting (24h, etc), + * they could set their browser languages to `['de-DE', 'de', 'en-GB']`. + * - Example three: GitLab language is `de` and browser languages are `['en-US', 'en']`. + * This function returns `['de', 'en']`, aligning German dates with the chosen translation of GitLab. + * + * @returns {string[]} + */ +export const getPreferredLocales = () => { + const gitlabLanguage = languageCode(); + // The GitLab language may or may not contain a country code, + // so we create the short version as well, e.g. de-AT => de + const lang = gitlabLanguage.substring(0, 2); + const locales = navigator.languages.filter((l) => l.startsWith(lang)); + if (!locales.includes(gitlabLanguage)) { + locales.push(gitlabLanguage); + } + if (!locales.includes(lang)) { + locales.push(lang); + } + if (!locales.includes(GITLAB_FALLBACK_LANGUAGE)) { + locales.push(GITLAB_FALLBACK_LANGUAGE); + } + return locales; +}; + +/** Creates an instance of Intl.DateTimeFormat for the current locale. @param formatOptions for available options, please see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/DateTimeFormat @returns {Intl.DateTimeFormat} */ -const createDateTimeFormat = (formatOptions) => Intl.DateTimeFormat(languageCode(), formatOptions); +const createDateTimeFormat = (formatOptions) => + Intl.DateTimeFormat(getPreferredLocales(), formatOptions); /** * Formats a number as a string using `toLocaleString`. diff --git a/app/assets/javascripts/pages/admin/application_settings/metrics_and_profiling/usage_statistics.js b/app/assets/javascripts/pages/admin/application_settings/metrics_and_profiling/usage_statistics.js index bab3cce39ac..bf27b1a81ff 100644 --- a/app/assets/javascripts/pages/admin/application_settings/metrics_and_profiling/usage_statistics.js +++ b/app/assets/javascripts/pages/admin/application_settings/metrics_and_profiling/usage_statistics.js @@ -1,25 +1,25 @@ import { __ } from '~/locale'; -export const HELPER_TEXT_USAGE_PING_DISABLED = __( +export const HELPER_TEXT_SERVICE_PING_DISABLED = __( 'To enable Registration Features, make sure "Enable service ping" is checked.', ); -export const HELPER_TEXT_USAGE_PING_ENABLED = __( +export const HELPER_TEXT_SERVICE_PING_ENABLED = __( 'You can enable Registration Features because Service Ping is enabled. To continue using Registration Features in the future, you will also need to register with GitLab via a new cloud licensing service.', ); function setHelperText(usagePingCheckbox) { - const helperTextId = document.getElementById('usage_ping_features_helper_text'); + const helperTextId = document.getElementById('service_ping_features_helper_text'); - const usagePingFeaturesLabel = document.getElementById('usage_ping_features_label'); + const usagePingFeaturesLabel = document.getElementById('service_ping_features_label'); const usagePingFeaturesCheckbox = document.getElementById( 'application_setting_usage_ping_features_enabled', ); helperTextId.textContent = usagePingCheckbox.checked - ? HELPER_TEXT_USAGE_PING_ENABLED - : HELPER_TEXT_USAGE_PING_DISABLED; + ? HELPER_TEXT_SERVICE_PING_ENABLED + : HELPER_TEXT_SERVICE_PING_DISABLED; usagePingFeaturesLabel.classList.toggle('gl-cursor-not-allowed', !usagePingCheckbox.checked); diff --git a/app/graphql/types/ci/pipeline_type.rb b/app/graphql/types/ci/pipeline_type.rb index 2eeddaca6ba..f4a6c18f73e 100644 --- a/app/graphql/types/ci/pipeline_type.rb +++ b/app/graphql/types/ci/pipeline_type.rb @@ -150,6 +150,9 @@ module Types description: 'A specific test suite in a pipeline test report.', resolver: Resolvers::Ci::TestSuiteResolver + field :ref, GraphQL::STRING_TYPE, null: true, + description: 'Reference to the branch from which the pipeline was triggered.' + def detailed_status object.detailed_status(current_user) end diff --git a/app/models/concerns/has_repository.rb b/app/models/concerns/has_repository.rb index 33f6904bc91..1b4c590694a 100644 --- a/app/models/concerns/has_repository.rb +++ b/app/models/concerns/has_repository.rb @@ -14,6 +14,7 @@ module HasRepository include Gitlab::Utils::StrongMemoize delegate :base_dir, :disk_path, to: :storage + delegate :change_head, to: :repository def valid_repo? repository.exists? @@ -117,4 +118,8 @@ module HasRepository def repository_size_checker raise NotImplementedError end + + def after_repository_change_head + reload_default_branch + end end diff --git a/app/models/project.rb b/app/models/project.rb index 6873c5f8236..6d99fa31c3d 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1649,18 +1649,11 @@ class Project < ApplicationRecord :visibility_level end - def change_head(branch) - if repository.branch_exists?(branch) - repository.before_change_head - repository.raw_repository.write_ref('HEAD', "refs/heads/#{branch}") - repository.copy_gitattributes(branch) - repository.after_change_head - ProjectCacheWorker.perform_async(self.id, [], [:commit_count]) - reload_default_branch - else - errors.add(:base, _("Could not change HEAD: branch '%{branch}' does not exist") % { branch: branch }) - false - end + override :after_repository_change_head + def after_repository_change_head + ProjectCacheWorker.perform_async(self.id, [], [:commit_count]) + + super end def forked_from?(other_project) diff --git a/app/models/repository.rb b/app/models/repository.rb index 1bd61fe48cb..a77aaf02e06 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -466,6 +466,7 @@ class Repository # Runs code after the HEAD of a repository is changed. def after_change_head expire_all_method_caches + container.after_repository_change_head end # Runs code after a new commit has been pushed. @@ -1142,6 +1143,18 @@ class Repository Gitlab::CurrentSettings.pick_repository_storage end + def change_head(branch) + if branch_exists?(branch) + before_change_head + raw_repository.write_ref('HEAD', "refs/heads/#{branch}") + copy_gitattributes(branch) + after_change_head + else + container.errors.add(:base, _("Could not change HEAD: branch '%{branch}' does not exist") % { branch: branch }) + false + end + end + private # TODO Genericize finder, later split this on finders by Ref or Oid diff --git a/app/policies/project_policy.rb b/app/policies/project_policy.rb index 2f01ab771dc..85547834a2e 100644 --- a/app/policies/project_policy.rb +++ b/app/policies/project_policy.rb @@ -54,6 +54,11 @@ class ProjectPolicy < BasePolicy !access_allowed_to?(:container_registry) end + desc "Container registry is enabled for everyone with access to the project" + condition(:container_registry_enabled_for_everyone_with_access, scope: :subject) do + project.container_registry_access_level == ProjectFeature::ENABLED + end + desc "Project has an external wiki" condition(:has_external_wiki, scope: :subject, score: 0) { project.has_external_wiki? } @@ -297,10 +302,13 @@ class ProjectPolicy < BasePolicy enable :guest_access enable :build_download_code - enable :build_read_container_image enable :request_access end + rule { container_registry_enabled_for_everyone_with_access & can?(:public_user_access) }.policy do + enable :build_read_container_image + end + rule { (can?(:public_user_access) | can?(:reporter_access)) & forking_allowed }.policy do enable :fork_project end diff --git a/app/views/admin/application_settings/_usage.html.haml b/app/views/admin/application_settings/_usage.html.haml index f45e6c5e8e9..ddd0abb4c34 100644 --- a/app/views/admin/application_settings/_usage.html.haml +++ b/app/views/admin/application_settings/_usage.html.haml @@ -39,14 +39,14 @@ - usage_ping_enabled = @application_setting.usage_ping_enabled? .form-check = f.check_box :usage_ping_features_enabled?, disabled: !usage_ping_enabled, class: 'form-check-input' - = f.label :usage_ping_features_enabled?, class: 'form-check-label gl-cursor-not-allowed', id: 'usage_ping_features_label' do + = f.label :usage_ping_features_enabled?, class: 'form-check-label gl-cursor-not-allowed', id: 'service_ping_features_label' do = _('Enable Registration Features') = link_to sprite_icon('question-o'), help_page_path('development/service_ping/index.md', anchor: 'registration-features-program') .form-text.text-muted - if usage_ping_enabled - %p.gl-mb-3.text-muted{ id: 'usage_ping_features_helper_text' }= _('You can enable Registration Features because Service Ping is enabled. To continue using Registration Features in the future, you will also need to register with GitLab via a new cloud licensing service.') + %p.gl-mb-3.text-muted{ id: 'service_ping_features_helper_text' }= _('You can enable Registration Features because Service Ping is enabled. To continue using Registration Features in the future, you will also need to register with GitLab via a new cloud licensing service.') - else - %p.gl-mb-3.text-muted{ id: 'usage_ping_features_helper_text' }= _('To enable Registration Features, make sure "Enable service ping" is checked.') + %p.gl-mb-3.text-muted{ id: 'service_ping_features_helper_text' }= _('To enable Registration Features, make sure "Enable service ping" is checked.') %p.gl-mb-3.text-muted= _('Registration Features include:') .form-text diff --git a/app/views/admin/dev_ops_report/_report.html.haml b/app/views/admin/dev_ops_report/_report.html.haml index c08a57fdb25..0b26548d6e6 100644 --- a/app/views/admin/dev_ops_report/_report.html.haml +++ b/app/views/admin/dev_ops_report/_report.html.haml @@ -4,6 +4,6 @@ = render 'callout' - if !service_ping_enabled - #js-devops-service-ping-disabled{ data: { is_admin: current_user&.admin.to_s, empty_state_svg_path: image_path('illustrations/convdev/convdev_no_index.svg'), enable_service_ping_path: metrics_and_profiling_admin_application_settings_path(anchor: 'js-usage-settings'), docs_link: help_page_path('development/usage_ping/index.md') } } + #js-devops-service-ping-disabled{ data: { is_admin: current_user&.admin.to_s, empty_state_svg_path: image_path('illustrations/convdev/convdev_no_index.svg'), enable_service_ping_path: metrics_and_profiling_admin_application_settings_path(anchor: 'js-usage-settings'), docs_link: help_page_path('development/service_ping/index.md') } } - else #js-devops-score{ data: { devops_score_metrics: devops_score_metrics(@metric).to_json, devops_report_docs_path: help_page_path('user/admin_area/analytics/dev_ops_report'), no_data_image_path: image_path('dev_ops_report_no_data.svg') } } diff --git a/app/workers/jira_connect/sync_branch_worker.rb b/app/workers/jira_connect/sync_branch_worker.rb index bd6023ccdd1..2723287b77b 100644 --- a/app/workers/jira_connect/sync_branch_worker.rb +++ b/app/workers/jira_connect/sync_branch_worker.rb @@ -8,7 +8,7 @@ module JiraConnect queue_namespace :jira_connect feature_category :integrations - data_consistency :delayed, feature_flag: :load_balancing_for_jira_connect_workers + data_consistency :delayed loggable_arguments 1, 2 worker_has_external_dependencies! diff --git a/app/workers/jira_connect/sync_builds_worker.rb b/app/workers/jira_connect/sync_builds_worker.rb index 4193b8ccdd7..4c4daba3314 100644 --- a/app/workers/jira_connect/sync_builds_worker.rb +++ b/app/workers/jira_connect/sync_builds_worker.rb @@ -8,7 +8,7 @@ module JiraConnect queue_namespace :jira_connect feature_category :integrations - data_consistency :delayed, feature_flag: :load_balancing_for_jira_connect_workers + data_consistency :delayed tags :exclude_from_kubernetes worker_has_external_dependencies! diff --git a/app/workers/jira_connect/sync_deployments_worker.rb b/app/workers/jira_connect/sync_deployments_worker.rb index d8d556d7102..0dc34b5999f 100644 --- a/app/workers/jira_connect/sync_deployments_worker.rb +++ b/app/workers/jira_connect/sync_deployments_worker.rb @@ -8,7 +8,7 @@ module JiraConnect queue_namespace :jira_connect feature_category :integrations - data_consistency :delayed, feature_flag: :load_balancing_for_jira_connect_workers + data_consistency :delayed tags :exclude_from_kubernetes worker_has_external_dependencies! diff --git a/app/workers/jira_connect/sync_feature_flags_worker.rb b/app/workers/jira_connect/sync_feature_flags_worker.rb index 919c76ab4c7..c484cabbe6b 100644 --- a/app/workers/jira_connect/sync_feature_flags_worker.rb +++ b/app/workers/jira_connect/sync_feature_flags_worker.rb @@ -8,7 +8,7 @@ module JiraConnect queue_namespace :jira_connect feature_category :integrations - data_consistency :delayed, feature_flag: :load_balancing_for_jira_connect_workers + data_consistency :delayed tags :exclude_from_kubernetes worker_has_external_dependencies! diff --git a/app/workers/jira_connect/sync_merge_request_worker.rb b/app/workers/jira_connect/sync_merge_request_worker.rb index f7115212792..bb0d24667e9 100644 --- a/app/workers/jira_connect/sync_merge_request_worker.rb +++ b/app/workers/jira_connect/sync_merge_request_worker.rb @@ -8,7 +8,7 @@ module JiraConnect queue_namespace :jira_connect feature_category :integrations - data_consistency :delayed, feature_flag: :load_balancing_for_jira_connect_workers + data_consistency :delayed worker_has_external_dependencies! diff --git a/app/workers/jira_connect/sync_project_worker.rb b/app/workers/jira_connect/sync_project_worker.rb index bca05b9311f..317bace89b4 100644 --- a/app/workers/jira_connect/sync_project_worker.rb +++ b/app/workers/jira_connect/sync_project_worker.rb @@ -8,7 +8,7 @@ module JiraConnect queue_namespace :jira_connect feature_category :integrations - data_consistency :delayed, feature_flag: :load_balancing_for_jira_connect_workers + data_consistency :delayed tags :exclude_from_kubernetes worker_has_external_dependencies! diff --git a/config/feature_flags/development/database_reindexing_pg12.yml b/config/feature_flags/development/list_commits.yml index cf0545bb2f9..45c2ff53d47 100644 --- a/config/feature_flags/development/database_reindexing_pg12.yml +++ b/config/feature_flags/development/list_commits.yml @@ -1,8 +1,8 @@ --- -name: database_reindexing_pg12 -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/64695 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/334372 +name: list_commits +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/65468 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/335208 milestone: '14.1' type: development -group: group::database +group: group::gitaly default_enabled: false diff --git a/config/feature_flags/development/load_balancing_for_jira_connect_workers.yml b/config/feature_flags/development/load_balancing_for_jira_connect_workers.yml deleted file mode 100644 index 03c6de7309f..00000000000 --- a/config/feature_flags/development/load_balancing_for_jira_connect_workers.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -name: load_balancing_for_jira_connect_workers -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/64715 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/335420 -milestone: '14.1' -type: development -group: group::ecosystem -default_enabled: false diff --git a/config/metrics/schema.json b/config/metrics/schema.json index 476e292e5c0..0ce5a163184 100644 --- a/config/metrics/schema.json +++ b/config/metrics/schema.json @@ -1,6 +1,6 @@ { "type": "object", - "required": ["key_path", "description", "value_type", "status", "product_group", "time_frame", "data_source", "distribution", "tier"], + "required": ["key_path", "description", "value_type", "status", "product_group", "time_frame", "data_source", "distribution", "tier", "data_category"], "properties": { "key_path": { "type": "string" diff --git a/doc/administration/raketasks/maintenance.md b/doc/administration/raketasks/maintenance.md index 2811ca1c7d7..5ddab999efe 100644 --- a/doc/administration/raketasks/maintenance.md +++ b/doc/administration/raketasks/maintenance.md @@ -330,7 +330,7 @@ migrations are completed (have an `up` status). ## Rebuild database indexes WARNING: -This is an experimental feature that isn't enabled by default. +This is an experimental feature that isn't enabled by default. It requires PostgreSQL 12 or later. Database indexes can be rebuilt regularly to reclaim space and maintain healthy levels of index bloat over time. @@ -348,7 +348,6 @@ sudo gitlab-rake gitlab:db:reindex['public.a_specific_index'] The following index types are not supported: -1. Unique and primary key indexes 1. Indexes used for constraint exclusion 1. Partitioned indexes 1. Expression indexes diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index f452639d254..ca924255154 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -11237,6 +11237,7 @@ Represents a file or directory in the project repository that has been locked. | <a id="pipelinepath"></a>`path` | [`String`](#string) | Relative path to the pipeline's page. | | <a id="pipelineproject"></a>`project` | [`Project`](#project) | Project the pipeline belongs to. | | <a id="pipelinequeuedduration"></a>`queuedDuration` | [`Duration`](#duration) | How long the pipeline was queued before starting. | +| <a id="pipelineref"></a>`ref` | [`String`](#string) | Reference to the branch from which the pipeline was triggered. | | <a id="pipelineretryable"></a>`retryable` | [`Boolean!`](#boolean) | Specifies if a pipeline can be retried. | | <a id="pipelinesecurityreportsummary"></a>`securityReportSummary` | [`SecurityReportSummary`](#securityreportsummary) | Vulnerability and scanned resource counts for each security scanner of the pipeline. | | <a id="pipelinesha"></a>`sha` | [`String!`](#string) | SHA of the pipeline's commit. | diff --git a/doc/development/sidekiq_style_guide.md b/doc/development/sidekiq_style_guide.md index 277d427ce9c..a77c9908ef4 100644 --- a/doc/development/sidekiq_style_guide.md +++ b/doc/development/sidekiq_style_guide.md @@ -484,14 +484,22 @@ under several strategies outlined below. ## Trading immediacy for reduced primary load -Not requiring immediate data consistency allows developers to decide to either: +We require Sidekiq workers to make an explicit decision around whether they need to use the +primary database node for all reads and writes, or whether reads can be served from replicas. This is +enforced by a RuboCop rule, which ensures that the `data_consistency` field is set. + +When setting this field, consider the following trade-off: - Ensure immediately consistent reads, but increase load on the primary database. - Prefer read replicas to add relief to the primary, but increase the likelihood of stale reads that have to be retried. -By default, any worker has a data consistency requirement of `:always`, so, as before, all -database operations target the primary. To allow for reads to be served from replicas instead, we -added two additional consistency modes: `:sticky` and `:delayed`. +To maintain the same behavior compared to before this field was introduced, set it to `:always`, so +database operations will only target the primary. Reasons for having to do so include workers +that mostly or exclusively perform writes, or workers that read their own writes and who might run +into data consistency issues should a stale record be read back from a replica. **Try to avoid +these scenarios, since `:always` should be considered the exception, not the rule.** + +To allow for reads to be served from replicas, we added two additional consistency modes: `:sticky` and `:delayed`. When you declare either `:sticky` or `:delayed` consistency, workers become eligible for database load-balancing. In both cases, jobs are enqueued with a short delay. @@ -508,7 +516,7 @@ they prefer read replicas and will wait for replicas to catch up: | **Data Consistency** | **Description** | |--------------|-----------------------------| -| `:always` | The job is required to use the primary database (default). It should be used for workers that primarily perform writes or that have very strict requirements around reading their writes without suffering any form of delay. | +| `:always` | The job is required to use the primary database (default). It should be used for workers that primarily perform writes or that have strict requirements around data consistency when reading their own writes. | | `:sticky` | The job prefers replicas, but switches to the primary for writes or when encountering replication lag. It should be used for jobs that require to be executed as fast as possible but can sustain a small initial queuing delay. | | `:delayed` | The job prefers replicas, but switches to the primary for writes. When encountering replication lag before the job starts, the job is retried once. If the replica is still not up to date on the next retry, it switches to the primary. It should be used for jobs where delaying execution further typically does not matter, such as cache expiration or web hooks execution. | diff --git a/doc/user/profile/preferences.md b/doc/user/profile/preferences.md index 03d8b0e564a..fad9b67eee4 100644 --- a/doc/user/profile/preferences.md +++ b/doc/user/profile/preferences.md @@ -176,7 +176,7 @@ You can select your preferred time format for the GitLab user interface: - Relative times, for example, `30 minutes ago`. - Absolute times, for example, `May 18, 2021, 3:57 PM`. -The times are formatted depending on your chosen language. +The times are formatted depending on your chosen language and browser locale. To set your time preference: diff --git a/lib/backup/database.rb b/lib/backup/database.rb index 51de31b3a87..f07fd786b4b 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -9,6 +9,8 @@ module Backup attr_reader :config, :db_file_name IGNORED_ERRORS = [ + # Ignore warnings + /WARNING:/, # Ignore the DROP errors; recent database dumps will use --if-exists with pg_dump /does not exist$/, # User may not have permissions to drop extensions or schemas diff --git a/lib/gitlab/database/postgres_index.rb b/lib/gitlab/database/postgres_index.rb index b6c57528e9f..58e4e7e7924 100644 --- a/lib/gitlab/database/postgres_index.rb +++ b/lib/gitlab/database/postgres_index.rb @@ -18,19 +18,12 @@ module Gitlab find(identifier) end - # A 'regular' index is a non-unique index, - # that does not serve an exclusion constraint and - # is defined on a table that is not partitioned. - # - # Deprecated: Switch to scope .reindexing_support - scope :regular, -> { where(unique: false, partitioned: false, exclusion: false, expression: false, type: Gitlab::Database::Reindexing::SUPPORTED_TYPES)} - - # Indexes for reindexing with PG12 + # Indexes with reindexing support scope :reindexing_support, -> { where(partitioned: false, exclusion: false, expression: false, type: Gitlab::Database::Reindexing::SUPPORTED_TYPES) } - scope :not_match, ->(regex) { where("name !~ ?", regex)} + scope :not_match, ->(regex) { where("name !~ ?", regex) } - scope :match, ->(regex) { where("name ~* ?", regex)} + scope :match, ->(regex) { where("name ~* ?", regex) } scope :not_recently_reindexed, -> do recent_actions = Reindexing::ReindexAction.recent.where('index_identifier = identifier') diff --git a/lib/gitlab/database/reindexing.rb b/lib/gitlab/database/reindexing.rb index 547783e99e2..841e04ccbd1 100644 --- a/lib/gitlab/database/reindexing.rb +++ b/lib/gitlab/database/reindexing.rb @@ -16,16 +16,9 @@ module Gitlab end def self.candidate_indexes - indexes = Gitlab::Database::PostgresIndex - .not_match("^#{ConcurrentReindex::TEMPORARY_INDEX_PREFIX}") - .not_match("^#{ConcurrentReindex::REPLACED_INDEX_PREFIX}") + Gitlab::Database::PostgresIndex .not_match("#{ReindexConcurrently::TEMPORARY_INDEX_PATTERN}$") - - if Feature.enabled?(:database_reindexing_pg12, type: :development) - indexes.reindexing_support - else - indexes.regular - end + .reindexing_support end end end diff --git a/lib/gitlab/database/reindexing/concurrent_reindex.rb b/lib/gitlab/database/reindexing/concurrent_reindex.rb deleted file mode 100644 index 7e2dd55d21b..00000000000 --- a/lib/gitlab/database/reindexing/concurrent_reindex.rb +++ /dev/null @@ -1,154 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module Reindexing - class ConcurrentReindex - include Gitlab::Utils::StrongMemoize - - ReindexError = Class.new(StandardError) - - PG_IDENTIFIER_LENGTH = 63 - TEMPORARY_INDEX_PREFIX = 'tmp_reindex_' - REPLACED_INDEX_PREFIX = 'old_reindex_' - STATEMENT_TIMEOUT = 9.hours - - # When dropping an index, we acquire a SHARE UPDATE EXCLUSIVE lock, - # which only conflicts with DDL and vacuum. We therefore execute this with a rather - # high lock timeout and a long pause in between retries. This is an alternative to - # setting a high statement timeout, which would lead to a long running query with effects - # on e.g. vacuum. - REMOVE_INDEX_RETRY_CONFIG = [[1.minute, 9.minutes]] * 30 - - attr_reader :index, :logger - - def initialize(index, logger: Gitlab::AppLogger) - @index = index - @logger = logger - end - - def perform - raise ReindexError, 'UNIQUE indexes are currently not supported' if index.unique? - raise ReindexError, 'partitioned indexes are currently not supported' if index.partitioned? - raise ReindexError, 'indexes serving an exclusion constraint are currently not supported' if index.exclusion? - raise ReindexError, 'index is a left-over temporary index from a previous reindexing run' if index.name.start_with?(TEMPORARY_INDEX_PREFIX, REPLACED_INDEX_PREFIX) - - logger.info "Starting reindex of #{index}" - - with_rebuilt_index do |replacement_index| - swap_index(replacement_index) - end - end - - private - - def with_rebuilt_index - if Gitlab::Database::PostgresIndex.find_by(schema: index.schema, name: replacement_index_name) - logger.debug("dropping dangling index from previous run (if it exists): #{replacement_index_name}") - remove_index(index.schema, replacement_index_name) - end - - create_replacement_index_statement = index.definition - .sub(/CREATE INDEX #{index.name}/, "CREATE INDEX CONCURRENTLY #{replacement_index_name}") - - logger.info("creating replacement index #{replacement_index_name}") - logger.debug("replacement index definition: #{create_replacement_index_statement}") - - set_statement_timeout do - connection.execute(create_replacement_index_statement) - end - - replacement_index = Gitlab::Database::PostgresIndex.find_by(schema: index.schema, name: replacement_index_name) - - unless replacement_index.valid_index? - message = 'replacement index was created as INVALID' - logger.error("#{message}, cleaning up") - raise ReindexError, "failed to reindex #{index}: #{message}" - end - - # Some expression indexes (aka functional indexes) - # require additional statistics. The existing statistics - # are tightly bound to the original index. We have to - # rebuild statistics for the new index before dropping - # the original one. - rebuild_statistics if index.expression? - - yield replacement_index - ensure - begin - remove_index(index.schema, replacement_index_name) - rescue StandardError => e - logger.error(e) - end - end - - def swap_index(replacement_index) - logger.info("swapping replacement index #{replacement_index} with #{index}") - - with_lock_retries do - rename_index(index.schema, index.name, replaced_index_name) - rename_index(replacement_index.schema, replacement_index.name, index.name) - rename_index(index.schema, replaced_index_name, replacement_index.name) - end - end - - def rename_index(schema, old_index_name, new_index_name) - connection.execute(<<~SQL) - ALTER INDEX #{quote_table_name(schema)}.#{quote_table_name(old_index_name)} - RENAME TO #{quote_table_name(new_index_name)} - SQL - end - - def remove_index(schema, name) - logger.info("Removing index #{schema}.#{name}") - - retries = Gitlab::Database::WithLockRetriesOutsideTransaction.new( - timing_configuration: REMOVE_INDEX_RETRY_CONFIG, - klass: self.class, - logger: logger - ) - - retries.run(raise_on_exhaustion: false) do - connection.execute(<<~SQL) - DROP INDEX CONCURRENTLY - IF EXISTS #{quote_table_name(schema)}.#{quote_table_name(name)} - SQL - end - end - - def rebuild_statistics - logger.info("rebuilding table statistics for #{index.schema}.#{index.tablename}") - - connection.execute(<<~SQL) - ANALYZE #{quote_table_name(index.schema)}.#{quote_table_name(index.tablename)} - SQL - end - - def replacement_index_name - @replacement_index_name ||= "#{TEMPORARY_INDEX_PREFIX}#{index.indexrelid}" - end - - def replaced_index_name - @replaced_index_name ||= "#{REPLACED_INDEX_PREFIX}#{index.indexrelid}" - end - - def with_lock_retries(&block) - arguments = { klass: self.class, logger: logger } - Gitlab::Database::WithLockRetries.new(**arguments).run(raise_on_exhaustion: true, &block) - end - - def set_statement_timeout - execute("SET statement_timeout TO '%ds'" % STATEMENT_TIMEOUT) - yield - ensure - execute('RESET statement_timeout') - end - - delegate :execute, :quote_table_name, to: :connection - def connection - @connection ||= ActiveRecord::Base.connection - end - end - end - end -end diff --git a/lib/gitlab/database/reindexing/coordinator.rb b/lib/gitlab/database/reindexing/coordinator.rb index e8a92c44836..13298f67ca9 100644 --- a/lib/gitlab/database/reindexing/coordinator.rb +++ b/lib/gitlab/database/reindexing/coordinator.rb @@ -41,13 +41,7 @@ module Gitlab end def perform_for(index, action) - strategy = if Feature.enabled?(:database_reindexing_pg12, type: :development) - ReindexConcurrently - else - ConcurrentReindex - end - - strategy.new(index).perform + ReindexConcurrently.new(index).perform rescue StandardError action.state = :failed diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 3d84e34297b..70d072e8082 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -354,9 +354,13 @@ module Gitlab end end - def new_commits(newrev) + def new_commits(newrevs) wrapped_gitaly_errors do - gitaly_ref_client.list_new_commits(newrev) + if Feature.enabled?(:list_commits) + gitaly_commit_client.list_commits(Array.wrap(newrevs) + %w[--not --all]) + else + Array.wrap(newrevs).flat_map { |newrev| gitaly_ref_client.list_new_commits(newrev) } + end end end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 3d24b4d53a4..b894207f0aa 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -248,6 +248,16 @@ module Gitlab consume_commits_response(response) end + def list_commits(revisions) + request = Gitaly::ListCommitsRequest.new( + repository: @gitaly_repo, + revisions: Array.wrap(revisions) + ) + + response = GitalyClient.call(@repository.storage, :commit_service, :list_commits, request, timeout: GitalyClient.medium_timeout) + consume_commits_response(response) + end + def list_commits_by_oid(oids) return [] if oids.empty? diff --git a/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb b/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb index d86f1609f14..b37eeb8bad1 100644 --- a/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb +++ b/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb @@ -99,6 +99,10 @@ module Gitlab return job_args unless compress_mode? return job_args if job_args.bytesize < @compression_threshold + # When a job was scheduled in the future, it runs through the middleware + # twice. Once on scheduling and once on queueing. No need to compress twice. + return job_args if ::Gitlab::SidekiqMiddleware::SizeLimiter::Compressor.compressed?(@job) + ::Gitlab::SidekiqMiddleware::SizeLimiter::Compressor.compress(@job, job_args) end diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 82d0c83b311..2b508b341dd 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -154,7 +154,7 @@ namespace :gitlab do Rake::Task['gitlab:db:create_dynamic_partitions'].invoke end - desc 'reindex a regular (non-unique) index without downtime to eliminate bloat' + desc 'reindex a regular index without downtime to eliminate bloat' task :reindex, [:index_name] => :environment do |_, args| unless Feature.enabled?(:database_reindexing, type: :ops) puts "This feature (database_reindexing) is currently disabled.".color(:yellow) diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 2d40344c4b3..6d5e640b8d8 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -5771,9 +5771,6 @@ msgstr "" msgid "Buy CI Minutes" msgstr "" -msgid "Buy License" -msgstr "" - msgid "Buy more Pipeline minutes" msgstr "" @@ -8511,9 +8508,6 @@ msgstr "" msgid "Consistency guarantee method" msgstr "" -msgid "Contact Sales to upgrade" -msgstr "" - msgid "Contact support" msgstr "" @@ -9759,9 +9753,6 @@ msgstr "" msgid "Custom range (UTC)" msgstr "" -msgid "Customer Portal" -msgstr "" - msgid "Customizable by an administrator." msgstr "" @@ -11758,9 +11749,6 @@ msgstr "" msgid "Download image" msgstr "" -msgid "Download license" -msgstr "" - msgid "Download raw data (.csv)" msgstr "" @@ -13377,9 +13365,6 @@ msgstr "" msgid "Expires in %{expires_at}" msgstr "" -msgid "Expires on" -msgstr "" - msgid "Expires:" msgstr "" @@ -14404,9 +14389,6 @@ msgstr "" msgid "Framework successfully deleted" msgstr "" -msgid "Free Trial" -msgstr "" - msgid "Free Trial of GitLab.com Ultimate" msgstr "" @@ -16293,9 +16275,6 @@ msgstr "" msgid "Hide host keys manual input" msgstr "" -msgid "Hide license key" -msgstr "" - msgid "Hide list" msgstr "" @@ -16550,9 +16529,6 @@ msgstr "" msgid "If the number of active users exceeds the user limit, you will be charged for the number of %{users_over_license_link} at your next license reconciliation." msgstr "" -msgid "If there is no previous license or if the previous license has expired, some GitLab functionality will be blocked until a new, valid license is uploaded." -msgstr "" - msgid "If this email was added in error, you can remove it here:" msgstr "" @@ -16592,9 +16568,6 @@ msgstr "" msgid "If you recently signed in and recognize the IP address, you may disregard this email." msgstr "" -msgid "If you remove this license, GitLab will fall back on the previous license, if any." -msgstr "" - msgid "If you want to re-enable two-factor authentication, visit %{two_factor_link}" msgstr "" @@ -17597,9 +17570,6 @@ msgstr "" msgid "Install GitLab Runner on Kubernetes" msgstr "" -msgid "Install license" -msgstr "" - msgid "Install on clusters" msgstr "" @@ -19533,18 +19503,9 @@ msgstr "" msgid "Let's talk!" msgstr "" -msgid "License" -msgstr "" - msgid "License Compliance" msgstr "" -msgid "License History" -msgstr "" - -msgid "License ID:" -msgstr "" - msgid "License file" msgstr "" @@ -19656,9 +19617,6 @@ msgstr "" msgid "Licensed Features" msgstr "" -msgid "Licensed to" -msgstr "" - msgid "Licensed to:" msgstr "" @@ -19716,27 +19674,6 @@ msgstr "" msgid "Licenses|View license details for your project" msgstr "" -msgid "License|Buy license" -msgstr "" - -msgid "License|License" -msgstr "" - -msgid "License|You can restore access to the Gold features at any time by upgrading." -msgstr "" - -msgid "License|You can start a free trial of GitLab Ultimate without any obligation or payment details." -msgstr "" - -msgid "License|You do not have a license." -msgstr "" - -msgid "License|Your License" -msgstr "" - -msgid "License|Your free trial of GitLab Ultimate expired on %{trial_ends_on}." -msgstr "" - msgid "Limit display of time tracking units to hours." msgstr "" @@ -24554,9 +24491,6 @@ msgstr "" msgid "Plain diff" msgstr "" -msgid "Plan" -msgstr "" - msgid "Plan:" msgstr "" @@ -30239,9 +30173,6 @@ msgstr "" msgid "Show latest version" msgstr "" -msgid "Show license key" -msgstr "" - msgid "Show links anyways" msgstr "" @@ -31194,9 +31125,6 @@ msgstr "" msgid "Started asynchronous removal of all repository check states." msgstr "" -msgid "Started:" -msgstr "" - msgid "Starting..." msgstr "" @@ -31419,9 +31347,6 @@ msgstr "" msgid "Step 4." msgstr "" -msgid "Still, we recommend keeping a backup saved somewhere. Otherwise, if you ever need it and have lost it, you will need to request GitLab Inc. to send it to you again." -msgstr "" - msgid "Stop Terminal" msgstr "" @@ -32519,9 +32444,6 @@ msgstr "" msgid "Text style" msgstr "" -msgid "Thank you for signing up for your free trial! You will get additional instructions in your inbox shortly." -msgstr "" - msgid "Thank you for your business." msgstr "" @@ -35403,9 +35325,6 @@ msgstr "" msgid "Upload New File" msgstr "" -msgid "Upload New License" -msgstr "" - msgid "Upload a certificate for your domain with all intermediates" msgstr "" @@ -35430,12 +35349,6 @@ msgstr "" msgid "UploadLink|click to upload" msgstr "" -msgid "Uploaded on" -msgstr "" - -msgid "Uploaded:" -msgstr "" - msgid "Uploading changes to terminal" msgstr "" @@ -36123,9 +36036,6 @@ msgstr "" msgid "Using the %{codeStart}needs%{codeEnd} keyword makes jobs run before their stage is reached. Jobs run as soon as their %{codeStart}needs%{codeEnd} relationships are met, which speeds up your pipelines." msgstr "" -msgid "Valid from" -msgstr "" - msgid "Validate" msgstr "" @@ -37701,9 +37611,6 @@ msgstr "" msgid "You didn't renew your subscription for %{strong}%{namespace_name}%{strong_close} so it was downgraded to the free plan." msgstr "" -msgid "You do not have an active license" -msgstr "" - msgid "You do not have any subscriptions yet" msgstr "" @@ -37767,9 +37674,6 @@ msgstr "" msgid "You don’t have access to Value Stream Analytics for this group" msgstr "" -msgid "You have a license that activates at a future date. Please see the License History table below." -msgstr "" - msgid "You have been granted %{access_level} access to the %{source_link} %{source_type}." msgstr "" @@ -38052,9 +37956,6 @@ msgstr "" msgid "Your Groups" msgstr "" -msgid "Your License" -msgstr "" - msgid "Your Personal Access Token was revoked" msgstr "" @@ -38208,9 +38109,6 @@ msgstr "" msgid "Your license is valid from" msgstr "" -msgid "Your license will be included in your GitLab backup and will survive upgrades, so in normal usage you should never need to re-upload your %{code_open}.gitlab-license%{code_close} file." -msgstr "" - msgid "Your membership in %{group} no longer expires." msgstr "" @@ -38301,12 +38199,6 @@ msgstr "" msgid "Your subscription will expire in %{remaining_days}." msgstr "" -msgid "Your trial license was issued and activated. Install it to enjoy GitLab Ultimate for 30 days." -msgstr "" - -msgid "Your trial license was issued!" -msgstr "" - msgid "Your username is %{username}." msgstr "" diff --git a/package.json b/package.json index b8773f4329a..dd2ac8027ef 100644 --- a/package.json +++ b/package.json @@ -215,7 +215,7 @@ "commander": "^2.18.0", "custom-jquery-matchers": "^2.1.0", "docdash": "^1.0.2", - "eslint": "7.30.0", + "eslint": "7.31.0", "eslint-import-resolver-jest": "3.0.0", "eslint-import-resolver-webpack": "0.13.1", "eslint-plugin-jasmine": "4.1.2", diff --git a/rubocop/cop/worker_data_consistency.rb b/rubocop/cop/worker_data_consistency.rb new file mode 100644 index 00000000000..e9047750f3e --- /dev/null +++ b/rubocop/cop/worker_data_consistency.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +require_relative '../code_reuse_helpers' + +module RuboCop + module Cop + # This cop checks for a call to `data_consistency` to exist in Sidekiq workers. + # + # @example + # + # # bad + # class BadWorker + # def perform + # end + # end + # + # # good + # class GoodWorker + # data_consistency :delayed + # + # def perform + # end + # end + # + class WorkerDataConsistency < RuboCop::Cop::Cop + include CodeReuseHelpers + + HELP_LINK = 'https://docs.gitlab.com/ee/development/sidekiq_style_guide.html#job-data-consistency-strategies' + + MSG = <<~MSG + Should define data_consistency expectation. + + It is encouraged for workers to use database replicas as much as possible by declaring + data_consistency to use the :delayed or :sticky modes. Mode :always will result in the + worker always hitting the primary database node for both reads and writes, which limits + scalability. + + Some guidelines: + + 1. If your worker mostly writes or reads its own writes, use mode :always. TRY TO AVOID THIS. + 2. If your worker performs mostly reads and can tolerate small delays, use mode :delayed. + 3. If your worker performs mostly reads but cannot tolerate any delays, use mode :sticky. + + See #{HELP_LINK} for a more detailed explanation of these settings. + MSG + + def_node_search :application_worker?, <<~PATTERN + `(send nil? :include (const nil? :ApplicationWorker)) + PATTERN + + def_node_search :data_consistency_defined?, <<~PATTERN + `(send nil? :data_consistency ...) + PATTERN + + def on_class(node) + return unless in_worker?(node) && application_worker?(node) + return if data_consistency_defined?(node) + + add_offense(node, location: :expression) + end + end + end +end diff --git a/scripts/verify-tff-mapping b/scripts/verify-tff-mapping index 4555a9854dd..ca73d4f5f7a 100755 --- a/scripts/verify-tff-mapping +++ b/scripts/verify-tff-mapping @@ -90,8 +90,8 @@ tests = [ { explanation: 'EE views should map to respective spec', - source: 'ee/app/views/admin/licenses/show.html.haml', - expected: ['ee/spec/views/admin/licenses/show.html.haml_spec.rb'] + source: 'ee/app/views/subscriptions/new.html.haml', + expected: ['ee/spec/views/subscriptions/new.html.haml_spec.rb'] }, { diff --git a/spec/frontend/locale/index_spec.js b/spec/frontend/locale/index_spec.js index a08be502735..220061fc64a 100644 --- a/spec/frontend/locale/index_spec.js +++ b/spec/frontend/locale/index_spec.js @@ -1,5 +1,5 @@ import { setLanguage } from 'helpers/locale_helper'; -import { createDateTimeFormat, formatNumber, languageCode } from '~/locale'; +import { createDateTimeFormat, formatNumber, languageCode, getPreferredLocales } from '~/locale'; describe('locale', () => { afterEach(() => setLanguage(null)); @@ -18,13 +18,91 @@ describe('locale', () => { }); }); + describe('getPreferredLocales', () => { + beforeEach(() => { + // Need to spy on window.navigator.languages as it is read-only + jest + .spyOn(window.navigator, 'languages', 'get') + .mockReturnValueOnce(['en-GB', 'en-US', 'de-AT']); + }); + + it('filters navigator.languages by GitLab language', () => { + setLanguage('en'); + + expect(getPreferredLocales()).toEqual(['en-GB', 'en-US', 'en']); + }); + + it('filters navigator.languages by GitLab language without locale and sets English Fallback', () => { + setLanguage('de'); + + expect(getPreferredLocales()).toEqual(['de-AT', 'de', 'en']); + }); + + it('filters navigator.languages by GitLab language with locale and sets English Fallback', () => { + setLanguage('de-DE'); + + expect(getPreferredLocales()).toEqual(['de-AT', 'de-DE', 'de', 'en']); + }); + + it('adds GitLab language if navigator.languages does not contain it', () => { + setLanguage('es-ES'); + + expect(getPreferredLocales()).toEqual(['es-ES', 'es', 'en']); + }); + }); + describe('createDateTimeFormat', () => { - beforeEach(() => setLanguage('en')); + const date = new Date(2015, 0, 3, 15, 13, 22); + const formatOptions = { dateStyle: 'long', timeStyle: 'medium' }; it('creates an instance of Intl.DateTimeFormat', () => { - const dateFormat = createDateTimeFormat({ year: 'numeric', month: 'long', day: 'numeric' }); + const dateFormat = createDateTimeFormat(formatOptions); + + expect(dateFormat).toBeInstanceOf(Intl.DateTimeFormat); + }); + + it('falls back to `en` and GitLab language is default', () => { + setLanguage(null); + jest.spyOn(window.navigator, 'languages', 'get').mockReturnValueOnce(['de-AT', 'en-GB']); + + const dateFormat = createDateTimeFormat(formatOptions); + expect(dateFormat.format(date)).toBe( + new Intl.DateTimeFormat('en-GB', formatOptions).format(date), + ); + }); + + it('falls back to `en` locale if browser languages are empty', () => { + setLanguage('en'); + jest.spyOn(window.navigator, 'languages', 'get').mockReturnValueOnce([]); + + const dateFormat = createDateTimeFormat(formatOptions); + expect(dateFormat.format(date)).toBe( + new Intl.DateTimeFormat('en', formatOptions).format(date), + ); + }); + + it('prefers `en-GB` if it is the preferred language and GitLab language is `en`', () => { + setLanguage('en'); + jest + .spyOn(window.navigator, 'languages', 'get') + .mockReturnValueOnce(['en-GB', 'en-US', 'en']); + + const dateFormat = createDateTimeFormat(formatOptions); + expect(dateFormat.format(date)).toBe( + new Intl.DateTimeFormat('en-GB', formatOptions).format(date), + ); + }); + + it('prefers `de-AT` if it is GitLab language and not part of the browser languages', () => { + setLanguage('de-AT'); + jest + .spyOn(window.navigator, 'languages', 'get') + .mockReturnValueOnce(['en-GB', 'en-US', 'en']); - expect(dateFormat.format(new Date(2015, 6, 3))).toBe('July 3, 2015'); + const dateFormat = createDateTimeFormat(formatOptions); + expect(dateFormat.format(date)).toBe( + new Intl.DateTimeFormat('de-AT', formatOptions).format(date), + ); }); }); diff --git a/spec/frontend/pages/admin/application_settings/metrics_and_profiling/usage_statistics_spec.js b/spec/frontend/pages/admin/application_settings/metrics_and_profiling/usage_statistics_spec.js index ef649e7697b..858c7b76ac8 100644 --- a/spec/frontend/pages/admin/application_settings/metrics_and_profiling/usage_statistics_spec.js +++ b/spec/frontend/pages/admin/application_settings/metrics_and_profiling/usage_statistics_spec.js @@ -1,6 +1,6 @@ import initSetHelperText, { - HELPER_TEXT_USAGE_PING_DISABLED, - HELPER_TEXT_USAGE_PING_ENABLED, + HELPER_TEXT_SERVICE_PING_DISABLED, + HELPER_TEXT_SERVICE_PING_ENABLED, } from '~/pages/admin/application_settings/metrics_and_profiling/usage_statistics'; describe('UsageStatistics', () => { @@ -17,18 +17,18 @@ describe('UsageStatistics', () => { usagePingFeaturesCheckBox = document.getElementById( 'application_setting_usage_ping_features_enabled', ); - usagePingFeaturesLabel = document.getElementById('usage_ping_features_label'); - usagePingFeaturesHelperText = document.getElementById('usage_ping_features_helper_text'); + usagePingFeaturesLabel = document.getElementById('service_ping_features_label'); + usagePingFeaturesHelperText = document.getElementById('service_ping_features_helper_text'); }); const expectEnabledUsagePingFeaturesCheckBox = () => { expect(usagePingFeaturesCheckBox.classList.contains('gl-cursor-not-allowed')).toBe(false); - expect(usagePingFeaturesHelperText.textContent).toEqual(HELPER_TEXT_USAGE_PING_ENABLED); + expect(usagePingFeaturesHelperText.textContent).toEqual(HELPER_TEXT_SERVICE_PING_ENABLED); }; const expectDisabledUsagePingFeaturesCheckBox = () => { expect(usagePingFeaturesLabel.classList.contains('gl-cursor-not-allowed')).toBe(true); - expect(usagePingFeaturesHelperText.textContent).toEqual(HELPER_TEXT_USAGE_PING_DISABLED); + expect(usagePingFeaturesHelperText.textContent).toEqual(HELPER_TEXT_SERVICE_PING_DISABLED); }; describe('Registration Features checkbox', () => { diff --git a/spec/graphql/types/ci/pipeline_type_spec.rb b/spec/graphql/types/ci/pipeline_type_spec.rb index 35d48229fa4..9ba4252bcd5 100644 --- a/spec/graphql/types/ci/pipeline_type_spec.rb +++ b/spec/graphql/types/ci/pipeline_type_spec.rb @@ -14,7 +14,7 @@ RSpec.describe Types::Ci::PipelineType do coverage created_at updated_at started_at finished_at committed_at stages user retryable cancelable jobs source_job job downstream upstream path project active user_permissions warnings commit_path uses_needs - test_report_summary test_suite + test_report_summary test_suite ref ] if Gitlab.ee? diff --git a/spec/lib/backup/database_spec.rb b/spec/lib/backup/database_spec.rb index 2bce4cab679..f57037d5652 100644 --- a/spec/lib/backup/database_spec.rb +++ b/spec/lib/backup/database_spec.rb @@ -38,7 +38,7 @@ RSpec.describe Backup::Database do context 'when the restore command prints errors' do let(:visible_error) { "This is a test error\n" } - let(:noise) { "Table projects does not exist\nmust be owner of extension pg_trgm\n" } + let(:noise) { "Table projects does not exist\nmust be owner of extension pg_trgm\nWARNING: no privileges could be revoked for public\n" } let(:cmd) { %W[#{Gem.ruby} -e $stderr.write("#{noise}#{visible_error}")] } it 'filters out noise from errors' do diff --git a/spec/lib/gitlab/database/postgres_index_spec.rb b/spec/lib/gitlab/database/postgres_index_spec.rb index da047632985..e1832219ebf 100644 --- a/spec/lib/gitlab/database/postgres_index_spec.rb +++ b/spec/lib/gitlab/database/postgres_index_spec.rb @@ -22,30 +22,6 @@ RSpec.describe Gitlab::Database::PostgresIndex do it_behaves_like 'a postgres model' - describe '.regular' do - it 'only non-unique indexes' do - expect(described_class.regular).to all(have_attributes(unique: false)) - end - - it 'only non partitioned indexes' do - expect(described_class.regular).to all(have_attributes(partitioned: false)) - end - - it 'only indexes that dont serve an exclusion constraint' do - expect(described_class.regular).to all(have_attributes(exclusion: false)) - end - - it 'only non-expression indexes' do - expect(described_class.regular).to all(have_attributes(expression: false)) - end - - it 'only btree and gist indexes' do - types = described_class.regular.map(&:type).uniq - - expect(types & %w(btree gist)).to eq(types) - end - end - describe '.reindexing_support' do it 'only non partitioned indexes' do expect(described_class.reindexing_support).to all(have_attributes(partitioned: false)) diff --git a/spec/lib/gitlab/database/reindexing/concurrent_reindex_spec.rb b/spec/lib/gitlab/database/reindexing/concurrent_reindex_spec.rb deleted file mode 100644 index d9077969003..00000000000 --- a/spec/lib/gitlab/database/reindexing/concurrent_reindex_spec.rb +++ /dev/null @@ -1,303 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Database::Reindexing::ConcurrentReindex, '#perform' do - subject { described_class.new(index, logger: logger) } - - let(:table_name) { '_test_reindex_table' } - let(:column_name) { '_test_column' } - let(:index_name) { '_test_reindex_index' } - let(:index) { instance_double(Gitlab::Database::PostgresIndex, indexrelid: 42, name: index_name, schema: 'public', tablename: table_name, partitioned?: false, unique?: false, exclusion?: false, expression?: false, definition: 'CREATE INDEX _test_reindex_index ON public._test_reindex_table USING btree (_test_column)') } - let(:logger) { double('logger', debug: nil, info: nil, error: nil ) } - let(:connection) { ActiveRecord::Base.connection } - - before do - connection.execute(<<~SQL) - CREATE TABLE #{table_name} ( - id serial NOT NULL PRIMARY KEY, - #{column_name} integer NOT NULL); - - CREATE INDEX #{index.name} ON #{table_name} (#{column_name}); - SQL - end - - context 'when the index is unique' do - before do - allow(index).to receive(:unique?).and_return(true) - end - - it 'raises an error' do - expect do - subject.perform - end.to raise_error(described_class::ReindexError, /UNIQUE indexes are currently not supported/) - end - end - - context 'when the index is partitioned' do - before do - allow(index).to receive(:partitioned?).and_return(true) - end - - it 'raises an error' do - expect do - subject.perform - end.to raise_error(described_class::ReindexError, /partitioned indexes are currently not supported/) - end - end - - context 'when the index serves an exclusion constraint' do - before do - allow(index).to receive(:exclusion?).and_return(true) - end - - it 'raises an error' do - expect do - subject.perform - end.to raise_error(described_class::ReindexError, /indexes serving an exclusion constraint are currently not supported/) - end - end - - context 'when the index is a lingering temporary index from a previous reindexing run' do - context 'with the temporary index prefix' do - let(:index_name) { 'tmp_reindex_something' } - - it 'raises an error' do - expect do - subject.perform - end.to raise_error(described_class::ReindexError, /left-over temporary index/) - end - end - - context 'with the replaced index prefix' do - let(:index_name) { 'old_reindex_something' } - - it 'raises an error' do - expect do - subject.perform - end.to raise_error(described_class::ReindexError, /left-over temporary index/) - end - end - end - - context 'replacing the original index with a rebuilt copy' do - let(:replacement_name) { 'tmp_reindex_42' } - let(:replaced_name) { 'old_reindex_42' } - - let(:create_index) { "CREATE INDEX CONCURRENTLY #{replacement_name} ON public.#{table_name} USING btree (#{column_name})" } - let(:drop_index) do - <<~SQL - DROP INDEX CONCURRENTLY - IF EXISTS "public"."#{replacement_name}" - SQL - end - - let!(:original_index) { find_index_create_statement } - - it 'integration test: executing full index replacement without mocks' do - allow(connection).to receive(:execute).and_wrap_original do |method, sql| - method.call(sql.sub(/CONCURRENTLY/, '')) - end - - subject.perform - - check_index_exists - end - - context 'mocked specs' do - before do - allow(subject).to receive(:connection).and_return(connection) - allow(connection).to receive(:execute).and_call_original - end - - it 'replaces the existing index with an identical index' do - expect(connection).to receive(:execute).with('SET statement_timeout TO \'32400s\'') - - expect_to_execute_concurrently_in_order(create_index) - - expect_next_instance_of(::Gitlab::Database::WithLockRetries) do |instance| - expect(instance).to receive(:run).with(raise_on_exhaustion: true).and_yield - end - - expect_index_rename(index.name, replaced_name) - expect_index_rename(replacement_name, index.name) - expect_index_rename(replaced_name, replacement_name) - - expect_next_instance_of(::Gitlab::Database::WithLockRetries) do |instance| - expect(instance).to receive(:run).with(raise_on_exhaustion: false).and_yield - end - - expect_to_execute_concurrently_in_order(drop_index) - - subject.perform - - check_index_exists - end - - context 'for expression indexes' do - before do - allow(index).to receive(:expression?).and_return(true) - end - - it 'rebuilds table statistics before dropping the original index' do - expect(connection).to receive(:execute).with('SET statement_timeout TO \'32400s\'') - - expect_to_execute_concurrently_in_order(create_index) - - expect_to_execute_concurrently_in_order(<<~SQL) - ANALYZE "#{index.schema}"."#{index.tablename}" - SQL - - expect_next_instance_of(::Gitlab::Database::WithLockRetries) do |instance| - expect(instance).to receive(:run).with(raise_on_exhaustion: true).and_yield - end - - expect_index_rename(index.name, replaced_name) - expect_index_rename(replacement_name, index.name) - expect_index_rename(replaced_name, replacement_name) - - expect_index_drop(drop_index) - - subject.perform - - check_index_exists - end - end - - context 'when a dangling index is left from a previous run' do - before do - connection.execute("CREATE INDEX #{replacement_name} ON #{table_name} (#{column_name})") - end - - it 'replaces the existing index with an identical index' do - expect_index_drop(drop_index) - expect_to_execute_concurrently_in_order(create_index) - - expect_next_instance_of(::Gitlab::Database::WithLockRetries) do |instance| - expect(instance).to receive(:run).with(raise_on_exhaustion: true).and_yield - end - - expect_index_rename(index.name, replaced_name) - expect_index_rename(replacement_name, index.name) - expect_index_rename(replaced_name, replacement_name) - - expect_index_drop(drop_index) - - subject.perform - - check_index_exists - end - end - - context 'when it fails to create the replacement index' do - it 'safely cleans up and signals the error' do - expect(connection).to receive(:execute).with(create_index).ordered - .and_raise(ActiveRecord::ConnectionTimeoutError, 'connect timeout') - - expect_next_instance_of(::Gitlab::Database::WithLockRetries) do |instance| - expect(instance).to receive(:run).with(raise_on_exhaustion: false).and_yield - end - - expect_to_execute_concurrently_in_order(drop_index) - - expect { subject.perform }.to raise_error(ActiveRecord::ConnectionTimeoutError, /connect timeout/) - - check_index_exists - end - end - - context 'when the replacement index is not valid' do - it 'safely cleans up and signals the error' do - replacement_index = double('replacement index', valid_index?: false) - allow(Gitlab::Database::PostgresIndex).to receive(:find_by).with(schema: 'public', name: replacement_name).and_return(nil, replacement_index) - - expect_to_execute_concurrently_in_order(create_index) - - expect_next_instance_of(::Gitlab::Database::WithLockRetries) do |instance| - expect(instance).to receive(:run).with(raise_on_exhaustion: false).and_yield - end - - expect_to_execute_concurrently_in_order(drop_index) - - expect { subject.perform }.to raise_error(described_class::ReindexError, /replacement index was created as INVALID/) - - check_index_exists - end - end - - context 'when a database error occurs while swapping the indexes' do - it 'safely cleans up and signals the error' do - replacement_index = double('replacement index', valid_index?: true) - allow(Gitlab::Database::PostgresIndex).to receive(:find_by).with(schema: 'public', name: replacement_name).and_return(nil, replacement_index) - - expect_to_execute_concurrently_in_order(create_index) - - expect_next_instance_of(::Gitlab::Database::WithLockRetries) do |instance| - expect(instance).to receive(:run).with(raise_on_exhaustion: true).and_yield - end - - expect_index_rename(index.name, replaced_name).and_raise(ActiveRecord::ConnectionTimeoutError, 'connect timeout') - - expect_index_drop(drop_index) - - expect { subject.perform }.to raise_error(ActiveRecord::ConnectionTimeoutError, /connect timeout/) - - check_index_exists - end - end - - context 'when with_lock_retries fails to acquire the lock' do - it 'safely cleans up and signals the error' do - expect_to_execute_concurrently_in_order(create_index) - - expect_next_instance_of(::Gitlab::Database::WithLockRetries) do |instance| - expect(instance).to receive(:run).with(raise_on_exhaustion: true) - .and_raise(::Gitlab::Database::WithLockRetries::AttemptsExhaustedError, 'exhausted') - end - - expect_index_drop(drop_index) - - expect { subject.perform }.to raise_error(::Gitlab::Database::WithLockRetries::AttemptsExhaustedError, /exhausted/) - - check_index_exists - end - end - end - end - - def expect_to_execute_concurrently_in_order(sql) - # Indexes cannot be created CONCURRENTLY in a transaction. Since the tests are wrapped in transactions, - # verify the original call but pass through the non-concurrent form. - expect(connection).to receive(:execute).with(sql).ordered.and_wrap_original do |method, sql| - method.call(sql.sub(/CONCURRENTLY/, '')) - end - end - - def expect_index_rename(from, to) - expect(connection).to receive(:execute).with(<<~SQL).ordered - ALTER INDEX "public"."#{from}" - RENAME TO "#{to}" - SQL - end - - def expect_index_drop(drop_index) - expect_next_instance_of(::Gitlab::Database::WithLockRetries) do |instance| - expect(instance).to receive(:run).with(raise_on_exhaustion: false).and_yield - end - - expect_to_execute_concurrently_in_order(drop_index) - end - - def find_index_create_statement - ActiveRecord::Base.connection.select_value(<<~SQL) - SELECT indexdef - FROM pg_indexes - WHERE schemaname = 'public' - AND indexname = #{ActiveRecord::Base.connection.quote(index.name)} - SQL - end - - def check_index_exists - expect(find_index_create_statement).to eq(original_index) - end -end diff --git a/spec/lib/gitlab/database/reindexing/coordinator_spec.rb b/spec/lib/gitlab/database/reindexing/coordinator_spec.rb index fd8b177c809..085fd3061ad 100644 --- a/spec/lib/gitlab/database/reindexing/coordinator_spec.rb +++ b/spec/lib/gitlab/database/reindexing/coordinator_spec.rb @@ -11,7 +11,7 @@ RSpec.describe Gitlab::Database::Reindexing::Coordinator do let(:index) { create(:postgres_index) } let(:notifier) { instance_double(Gitlab::Database::Reindexing::GrafanaNotifier, notify_start: nil, notify_end: nil) } - let(:reindexer) { instance_double(Gitlab::Database::Reindexing::ConcurrentReindex, perform: nil) } + let(:reindexer) { instance_double(Gitlab::Database::Reindexing::ReindexConcurrently, perform: nil) } let(:action) { create(:reindex_action, index: index) } let!(:lease) { stub_exclusive_lease(lease_key, uuid, timeout: lease_timeout) } @@ -19,83 +19,64 @@ RSpec.describe Gitlab::Database::Reindexing::Coordinator do let(:lease_timeout) { 1.day } let(:uuid) { 'uuid' } - shared_examples_for 'reindexing coordination' do - context 'locking' do - it 'acquires a lock while reindexing' do - expect(lease).to receive(:try_obtain).ordered.and_return(uuid) + before do + swapout_view_for_table(:postgres_indexes) - expect(reindexer).to receive(:perform).ordered + allow(Gitlab::Database::Reindexing::ReindexConcurrently).to receive(:new).with(index).and_return(reindexer) + allow(Gitlab::Database::Reindexing::ReindexAction).to receive(:create_for).with(index).and_return(action) + end - expect(Gitlab::ExclusiveLease).to receive(:cancel).ordered.with(lease_key, uuid) + context 'locking' do + it 'acquires a lock while reindexing' do + expect(lease).to receive(:try_obtain).ordered.and_return(uuid) - subject - end + expect(reindexer).to receive(:perform).ordered - it 'does not perform reindexing actions if lease is not granted' do - expect(lease).to receive(:try_obtain).ordered.and_return(false) - expect(Gitlab::Database::Reindexing::ConcurrentReindex).not_to receive(:new) + expect(Gitlab::ExclusiveLease).to receive(:cancel).ordered.with(lease_key, uuid) - subject - end + subject end - context 'notifications' do - it 'sends #notify_start before reindexing' do - expect(notifier).to receive(:notify_start).with(action).ordered - expect(reindexer).to receive(:perform).ordered - - subject - end + it 'does not perform reindexing actions if lease is not granted' do + expect(lease).to receive(:try_obtain).ordered.and_return(false) + expect(Gitlab::Database::Reindexing::ReindexConcurrently).not_to receive(:new) - it 'sends #notify_end after reindexing and updating the action is done' do - expect(action).to receive(:finish).ordered - expect(notifier).to receive(:notify_end).with(action).ordered - - subject - end + subject end + end - context 'action tracking' do - it 'calls #finish on the action' do - expect(reindexer).to receive(:perform).ordered - expect(action).to receive(:finish).ordered - - subject - end + context 'notifications' do + it 'sends #notify_start before reindexing' do + expect(notifier).to receive(:notify_start).with(action).ordered + expect(reindexer).to receive(:perform).ordered - it 'upon error, it still calls finish and raises the error' do - expect(reindexer).to receive(:perform).ordered.and_raise('something went wrong') - expect(action).to receive(:finish).ordered + subject + end - expect { subject }.to raise_error(/something went wrong/) + it 'sends #notify_end after reindexing and updating the action is done' do + expect(action).to receive(:finish).ordered + expect(notifier).to receive(:notify_end).with(action).ordered - expect(action).to be_failed - end + subject end end - context 'legacy reindexing method (< PG12) - to be removed' do - before do - stub_feature_flags(database_reindexing_pg12: false) - swapout_view_for_table(:postgres_indexes) + context 'action tracking' do + it 'calls #finish on the action' do + expect(reindexer).to receive(:perform).ordered + expect(action).to receive(:finish).ordered - allow(Gitlab::Database::Reindexing::ConcurrentReindex).to receive(:new).with(index).and_return(reindexer) - allow(Gitlab::Database::Reindexing::ReindexAction).to receive(:create_for).with(index).and_return(action) + subject end - it_behaves_like 'reindexing coordination' - end + it 'upon error, it still calls finish and raises the error' do + expect(reindexer).to receive(:perform).ordered.and_raise('something went wrong') + expect(action).to receive(:finish).ordered - context 'PG12 reindexing method' do - before do - stub_feature_flags(database_reindexing_pg12: true) - swapout_view_for_table(:postgres_indexes) + expect { subject }.to raise_error(/something went wrong/) - allow(Gitlab::Database::Reindexing::ReindexConcurrently).to receive(:new).with(index).and_return(reindexer) - allow(Gitlab::Database::Reindexing::ReindexAction).to receive(:create_for).with(index).and_return(action) + expect(action).to be_failed end - - it_behaves_like 'reindexing coordination' end end end diff --git a/spec/lib/gitlab/database/reindexing_spec.rb b/spec/lib/gitlab/database/reindexing_spec.rb index 0a3cbcef46a..8aff99544ca 100644 --- a/spec/lib/gitlab/database/reindexing_spec.rb +++ b/spec/lib/gitlab/database/reindexing_spec.rb @@ -29,30 +29,11 @@ RSpec.describe Gitlab::Database::Reindexing do describe '.candidate_indexes' do subject { described_class.candidate_indexes } - context 'with deprecated method for < PG12' do - before do - stub_feature_flags(database_reindexing_pg12: false) - end - - it 'retrieves regular indexes that are no left-overs from previous runs' do - result = double - expect(Gitlab::Database::PostgresIndex).to receive_message_chain('not_match.not_match.not_match.regular').with('^tmp_reindex_').with('^old_reindex_').with('\_ccnew[0-9]*$').with(no_args).and_return(result) - - expect(subject).to eq(result) - end - end + it 'retrieves regular indexes that are no left-overs from previous runs' do + result = double + expect(Gitlab::Database::PostgresIndex).to receive_message_chain('not_match.reindexing_support').with('\_ccnew[0-9]*$').with(no_args).and_return(result) - context 'with deprecated method for >= PG12' do - before do - stub_feature_flags(database_reindexing_pg12: true) - end - - it 'retrieves regular indexes that are no left-overs from previous runs' do - result = double - expect(Gitlab::Database::PostgresIndex).to receive_message_chain('not_match.not_match.not_match.reindexing_support').with('^tmp_reindex_').with('^old_reindex_').with('\_ccnew[0-9]*$').with(no_args).and_return(result) - - expect(subject).to eq(result) - end + expect(subject).to eq(result) end end end diff --git a/spec/lib/gitlab/git/repository_spec.rb b/spec/lib/gitlab/git/repository_spec.rb index 10dfc46c0bb..706bcdea291 100644 --- a/spec/lib/gitlab/git/repository_spec.rb +++ b/spec/lib/gitlab/git/repository_spec.rb @@ -934,6 +934,63 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do end end + describe '#new_commits' do + let(:repository) { mutable_repository } + let(:new_commit) do + author = { name: 'Test User', email: 'mail@example.com', time: Time.now } + + Rugged::Commit.create(repository_rugged, + author: author, + committer: author, + message: "Message", + parents: [], + tree: "4b825dc642cb6eb9a060e54bf8d69288fbee4904") + end + + let(:expected_commits) { 1 } + let(:revisions) { [new_commit] } + + shared_examples 'an enumeration of new commits' do + it 'enumerates commits' do + commits = repository.new_commits(revisions).to_a + + expect(commits.size).to eq(expected_commits) + commits.each do |commit| + expect(commit.id).to eq(new_commit) + expect(commit.message).to eq("Message") + end + end + end + + context 'with list_commits disabled' do + before do + stub_feature_flags(list_commits: false) + + expect_next_instance_of(Gitlab::GitalyClient::RefService) do |service| + expect(service) + .to receive(:list_new_commits) + .with(new_commit) + .and_call_original + end + end + + it_behaves_like 'an enumeration of new commits' + end + + context 'with list_commits enabled' do + before do + expect_next_instance_of(Gitlab::GitalyClient::CommitService) do |service| + expect(service) + .to receive(:list_commits) + .with([new_commit, '--not', '--all']) + .and_call_original + end + end + + it_behaves_like 'an enumeration of new commits' + end + end + describe '#count_commits_between' do subject { repository.count_commits_between('feature', 'master') } diff --git a/spec/lib/gitlab/gitaly_client/commit_service_spec.rb b/spec/lib/gitlab/gitaly_client/commit_service_spec.rb index ac4c42d57ee..22c29403255 100644 --- a/spec/lib/gitlab/gitaly_client/commit_service_spec.rb +++ b/spec/lib/gitlab/gitaly_client/commit_service_spec.rb @@ -287,6 +287,39 @@ RSpec.describe Gitlab::GitalyClient::CommitService do end end + describe '#list_commits' do + shared_examples 'a ListCommits request' do + before do + ::Gitlab::GitalyClient.clear_stubs! + end + + it 'sends a list_commits message' do + expect_next_instance_of(Gitaly::CommitService::Stub) do |service| + expect(service) + .to receive(:list_commits) + .with(gitaly_request_with_params(expected_params), kind_of(Hash)) + .and_return([]) + end + + client.list_commits(revisions) + end + end + + context 'with a single revision' do + let(:revisions) { 'master' } + let(:expected_params) { %w[master] } + + it_behaves_like 'a ListCommits request' + end + + context 'with multiple revisions' do + let(:revisions) { %w[master --not --all] } + let(:expected_params) { %w[master --not --all] } + + it_behaves_like 'a ListCommits request' + end + end + describe '#commit_stats' do let(:request) do Gitaly::CommitStatsRequest.new( diff --git a/spec/lib/gitlab/sidekiq_middleware/size_limiter/validator_spec.rb b/spec/lib/gitlab/sidekiq_middleware/size_limiter/validator_spec.rb index 4fbe59c3c27..440eca10a88 100644 --- a/spec/lib/gitlab/sidekiq_middleware/size_limiter/validator_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware/size_limiter/validator_spec.rb @@ -230,11 +230,11 @@ RSpec.describe Gitlab::SidekiqMiddleware::SizeLimiter::Validator do end context 'in compress mode' do + let(:size_limit) { 50 } + let(:compression_threshold) { 30 } let(:mode) { 'compress' } context 'when job size is less than compression threshold' do - let(:size_limit) { 50 } - let(:compression_threshold) { 30 } let(:job) { job_payload(a: 'a' * 10) } it 'does not raise an exception' do @@ -244,8 +244,6 @@ RSpec.describe Gitlab::SidekiqMiddleware::SizeLimiter::Validator do end context 'when job size is bigger than compression threshold and less than size limit after compressed' do - let(:size_limit) { 50 } - let(:compression_threshold) { 30 } let(:args) { { a: 'a' * 300 } } let(:job) { job_payload(args) } @@ -260,9 +258,20 @@ RSpec.describe Gitlab::SidekiqMiddleware::SizeLimiter::Validator do end end + context 'when the job was already compressed' do + let(:job) do + job_payload({ a: 'a' * 10 }) + .merge(Gitlab::SidekiqMiddleware::SizeLimiter::Compressor::COMPRESSED_KEY => true) + end + + it 'does not compress the arguments again' do + expect(Gitlab::SidekiqMiddleware::SizeLimiter::Compressor).not_to receive(:compress) + + expect { validate.call(TestSizeLimiterWorker, job) }.not_to raise_error + end + end + context 'when job size is bigger than compression threshold and bigger than size limit after compressed' do - let(:size_limit) { 50 } - let(:compression_threshold) { 30 } let(:args) { { a: 'a' * 3000 } } let(:job) { job_payload(args) } diff --git a/spec/lib/gitlab/usage/metric_definition_spec.rb b/spec/lib/gitlab/usage/metric_definition_spec.rb index 1ed639b2f7d..f3c3e5fc550 100644 --- a/spec/lib/gitlab/usage/metric_definition_spec.rb +++ b/spec/lib/gitlab/usage/metric_definition_spec.rb @@ -17,7 +17,8 @@ RSpec.describe Gitlab::Usage::MetricDefinition do data_source: 'database', distribution: %w(ee ce), tier: %w(free starter premium ultimate bronze silver gold), - name: 'count_boards' + name: 'uuid', + data_category: 'Standard' } end @@ -63,6 +64,7 @@ RSpec.describe Gitlab::Usage::MetricDefinition do :value_type | nil :value_type | 'test' :status | nil + :data_category | nil :key_path | nil :product_group | nil :time_frame | nil @@ -196,7 +198,8 @@ RSpec.describe Gitlab::Usage::MetricDefinition do time_frame: 'none', data_source: 'database', distribution: %w(ee ce), - tier: %w(free starter premium ultimate bronze silver gold) + tier: %w(free starter premium ultimate bronze silver gold), + data_category: 'Optional' } end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index e634800320d..efa269cdb5c 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -3156,35 +3156,19 @@ RSpec.describe Project, factory_default: :keep do end end - describe '#change_head' do - let_it_be(:project) { create(:project, :repository) } - - it 'returns error if branch does not exist' do - expect(project.change_head('unexisted-branch')).to be false - expect(project.errors.size).to eq(1) - end - - it 'calls the before_change_head and after_change_head methods' do - expect(project.repository).to receive(:before_change_head) - expect(project.repository).to receive(:after_change_head) - - project.change_head(project.default_branch) - end + describe '#after_repository_change_head' do + let_it_be(:project) { create(:project) } it 'updates commit count' do expect(ProjectCacheWorker).to receive(:perform_async).with(project.id, [], [:commit_count]) - project.change_head(project.default_branch) - end - - it 'copies the gitattributes' do - expect(project.repository).to receive(:copy_gitattributes).with(project.default_branch) - project.change_head(project.default_branch) + project.after_repository_change_head end it 'reloads the default branch' do expect(project).to receive(:reload_default_branch) - project.change_head(project.default_branch) + + project.after_repository_change_head end end diff --git a/spec/models/repository_spec.rb b/spec/models/repository_spec.rb index 4b0435baa98..452eafe733f 100644 --- a/spec/models/repository_spec.rb +++ b/spec/models/repository_spec.rb @@ -2141,6 +2141,12 @@ RSpec.describe Repository do repository.after_change_head end + + it 'calls after_repository_change_head on container' do + expect(repository.container).to receive(:after_repository_change_head) + + repository.after_change_head + end end describe '#expires_caches_for_tags' do @@ -3266,4 +3272,30 @@ RSpec.describe Repository do settings.save! end end + + describe '#change_head' do + let(:branch) { repository.container.default_branch } + + it 'adds an error to container if branch does not exist' do + expect(repository.change_head('unexisted-branch')).to be false + expect(repository.container.errors.size).to eq(1) + end + + it 'calls the before_change_head and after_change_head methods' do + expect(repository).to receive(:before_change_head) + expect(repository).to receive(:after_change_head) + + repository.change_head(branch) + end + + it 'copies the gitattributes' do + expect(repository).to receive(:copy_gitattributes).with(branch) + repository.change_head(branch) + end + + it 'reloads the default branch' do + expect(repository.container).to receive(:reload_default_branch) + repository.change_head(branch) + end + end end diff --git a/spec/policies/project_policy_spec.rb b/spec/policies/project_policy_spec.rb index 6c09abc7705..59123c3695a 100644 --- a/spec/policies/project_policy_spec.rb +++ b/spec/policies/project_policy_spec.rb @@ -4,6 +4,7 @@ require 'spec_helper' RSpec.describe ProjectPolicy do include ExternalAuthorizationServiceHelpers + include AdminModeHelper include_context 'ProjectPolicy context' let(:project) { public_project } @@ -1486,7 +1487,12 @@ RSpec.describe ProjectPolicy do describe 'container_image policies' do using RSpec::Parameterized::TableSyntax - let(:guest_operations_permissions) { [:read_container_image] } + # These are permissions that admins should not have when the project is private + # or the container registry is private. + let(:admin_excluded_permissions) { [:build_read_container_image] } + + let(:anonymous_operations_permissions) { [:read_container_image] } + let(:guest_operations_permissions) { anonymous_operations_permissions + [:build_read_container_image] } let(:developer_operations_permissions) do guest_operations_permissions + [ @@ -1500,47 +1506,67 @@ RSpec.describe ProjectPolicy do ] end + let(:all_permissions) { maintainer_operations_permissions } + where(:project_visibility, :access_level, :role, :allowed) do + :public | ProjectFeature::ENABLED | :admin | true + :public | ProjectFeature::ENABLED | :owner | true :public | ProjectFeature::ENABLED | :maintainer | true :public | ProjectFeature::ENABLED | :developer | true :public | ProjectFeature::ENABLED | :reporter | true :public | ProjectFeature::ENABLED | :guest | true :public | ProjectFeature::ENABLED | :anonymous | true + :public | ProjectFeature::PRIVATE | :admin | true + :public | ProjectFeature::PRIVATE | :owner | true :public | ProjectFeature::PRIVATE | :maintainer | true :public | ProjectFeature::PRIVATE | :developer | true :public | ProjectFeature::PRIVATE | :reporter | true :public | ProjectFeature::PRIVATE | :guest | false :public | ProjectFeature::PRIVATE | :anonymous | false + :public | ProjectFeature::DISABLED | :admin | false + :public | ProjectFeature::DISABLED | :owner | false :public | ProjectFeature::DISABLED | :maintainer | false :public | ProjectFeature::DISABLED | :developer | false :public | ProjectFeature::DISABLED | :reporter | false :public | ProjectFeature::DISABLED | :guest | false :public | ProjectFeature::DISABLED | :anonymous | false + :internal | ProjectFeature::ENABLED | :admin | true + :internal | ProjectFeature::ENABLED | :owner | true :internal | ProjectFeature::ENABLED | :maintainer | true :internal | ProjectFeature::ENABLED | :developer | true :internal | ProjectFeature::ENABLED | :reporter | true :internal | ProjectFeature::ENABLED | :guest | true :internal | ProjectFeature::ENABLED | :anonymous | false + :internal | ProjectFeature::PRIVATE | :admin | true + :internal | ProjectFeature::PRIVATE | :owner | true :internal | ProjectFeature::PRIVATE | :maintainer | true :internal | ProjectFeature::PRIVATE | :developer | true :internal | ProjectFeature::PRIVATE | :reporter | true :internal | ProjectFeature::PRIVATE | :guest | false :internal | ProjectFeature::PRIVATE | :anonymous | false + :internal | ProjectFeature::DISABLED | :admin | false + :internal | ProjectFeature::DISABLED | :owner | false :internal | ProjectFeature::DISABLED | :maintainer | false :internal | ProjectFeature::DISABLED | :developer | false :internal | ProjectFeature::DISABLED | :reporter | false :internal | ProjectFeature::DISABLED | :guest | false :internal | ProjectFeature::DISABLED | :anonymous | false + :private | ProjectFeature::ENABLED | :admin | true + :private | ProjectFeature::ENABLED | :owner | true :private | ProjectFeature::ENABLED | :maintainer | true :private | ProjectFeature::ENABLED | :developer | true :private | ProjectFeature::ENABLED | :reporter | true :private | ProjectFeature::ENABLED | :guest | false :private | ProjectFeature::ENABLED | :anonymous | false + :private | ProjectFeature::PRIVATE | :admin | true + :private | ProjectFeature::PRIVATE | :owner | true :private | ProjectFeature::PRIVATE | :maintainer | true :private | ProjectFeature::PRIVATE | :developer | true :private | ProjectFeature::PRIVATE | :reporter | true :private | ProjectFeature::PRIVATE | :guest | false :private | ProjectFeature::PRIVATE | :anonymous | false + :private | ProjectFeature::DISABLED | :admin | false + :private | ProjectFeature::DISABLED | :owner | false :private | ProjectFeature::DISABLED | :maintainer | false :private | ProjectFeature::DISABLED | :developer | false :private | ProjectFeature::DISABLED | :reporter | false @@ -1552,24 +1578,44 @@ RSpec.describe ProjectPolicy do let(:current_user) { send(role) } let(:project) { send("#{project_visibility}_project") } - it 'allows/disallows the abilities based on the container_registry feature access level' do + before do + enable_admin_mode!(admin) if role == :admin project.project_feature.update!(container_registry_access_level: access_level) + end + it 'allows/disallows the abilities based on the container_registry feature access level' do if allowed expect_allowed(*permissions_abilities(role)) + expect_disallowed(*(all_permissions - permissions_abilities(role))) else - expect_disallowed(*permissions_abilities(role)) + expect_disallowed(*all_permissions) + end + end + + it 'allows build_read_container_image to admins who are also team members' do + if allowed && role == :admin + project.add_reporter(current_user) + + expect_allowed(:build_read_container_image) end end def permissions_abilities(role) case role - when :maintainer + when :admin + if project_visibility == :private || access_level == ProjectFeature::PRIVATE + maintainer_operations_permissions - admin_excluded_permissions + else + maintainer_operations_permissions + end + when :maintainer, :owner maintainer_operations_permissions when :developer developer_operations_permissions - when :reporter, :guest, :anonymous + when :reporter, :guest guest_operations_permissions + when :anonymous + anonymous_operations_permissions else raise "Unknown role #{role}" end diff --git a/spec/rubocop/cop/worker_data_consistency_spec.rb b/spec/rubocop/cop/worker_data_consistency_spec.rb new file mode 100644 index 00000000000..5fa42bf2b87 --- /dev/null +++ b/spec/rubocop/cop/worker_data_consistency_spec.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' +require_relative '../../../rubocop/cop/worker_data_consistency' + +RSpec.describe RuboCop::Cop::WorkerDataConsistency do + subject(:cop) { described_class.new } + + before do + allow(cop) + .to receive(:in_worker?) + .and_return(true) + end + + it 'adds an offense when not defining data_consistency' do + expect_offense(<<~CODE) + class SomeWorker + ^^^^^^^^^^^^^^^^ Should define data_consistency expectation.[...] + include ApplicationWorker + + queue_namespace :pipeline_hooks + feature_category :continuous_integration + urgency :high + end + CODE + end + + it 'adds no offense when defining data_consistency' do + expect_no_offenses(<<~CODE) + class SomeWorker + include ApplicationWorker + + queue_namespace :pipeline_hooks + feature_category :continuous_integration + data_consistency :delayed + urgency :high + end + CODE + end + + it 'adds no offense when worker is not an ApplicationWorker' do + expect_no_offenses(<<~CODE) + class SomeWorker + queue_namespace :pipeline_hooks + feature_category :continuous_integration + urgency :high + end + CODE + end +end diff --git a/spec/support/shared_examples/models/concerns/has_repository_shared_examples.rb b/spec/support/shared_examples/models/concerns/has_repository_shared_examples.rb index 1be4d9b80a4..a403a27adef 100644 --- a/spec/support/shared_examples/models/concerns/has_repository_shared_examples.rb +++ b/spec/support/shared_examples/models/concerns/has_repository_shared_examples.rb @@ -152,4 +152,20 @@ RSpec.shared_examples 'model with repository' do it { is_expected.to respond_to(:disk_path) } it { is_expected.to respond_to(:gitlab_shell) } end + + describe '#change_head' do + it 'delegates #change_head to repository' do + expect(stubbed_container.repository).to receive(:change_head).with('foo') + + stubbed_container.change_head('foo') + end + end + + describe '#after_repository_change_head' do + it 'calls #reload_default_branch' do + expect(stubbed_container).to receive(:reload_default_branch) + + stubbed_container.after_repository_change_head + end + end end diff --git a/spec/support/shared_examples/services/container_registry_auth_service_shared_examples.rb b/spec/support/shared_examples/services/container_registry_auth_service_shared_examples.rb index c98d3768748..eafcbd77040 100644 --- a/spec/support/shared_examples/services/container_registry_auth_service_shared_examples.rb +++ b/spec/support/shared_examples/services/container_registry_auth_service_shared_examples.rb @@ -629,6 +629,22 @@ RSpec.shared_examples 'a container registry auth service' do end end end + + context 'for project with private container registry' do + let_it_be(:project, reload: true) { create(:project, :public) } + + before do + project.project_feature.update!(container_registry_access_level: ProjectFeature::PRIVATE) + end + + it_behaves_like 'pullable for being team member' + + context 'when you are admin' do + let_it_be(:current_user) { create(:admin) } + + it_behaves_like 'pullable for being team member' + end + end end context 'when pushing' do diff --git a/spec/workers/jira_connect/sync_branch_worker_spec.rb b/spec/workers/jira_connect/sync_branch_worker_spec.rb index f70a09c1ee2..349ccd10694 100644 --- a/spec/workers/jira_connect/sync_branch_worker_spec.rb +++ b/spec/workers/jira_connect/sync_branch_worker_spec.rb @@ -7,7 +7,6 @@ RSpec.describe JiraConnect::SyncBranchWorker do it_behaves_like 'worker with data consistency', described_class, - feature_flag: :load_balancing_for_jira_connect_workers, data_consistency: :delayed describe '#perform' do diff --git a/spec/workers/jira_connect/sync_builds_worker_spec.rb b/spec/workers/jira_connect/sync_builds_worker_spec.rb index e60d1c58966..9be0cccae2b 100644 --- a/spec/workers/jira_connect/sync_builds_worker_spec.rb +++ b/spec/workers/jira_connect/sync_builds_worker_spec.rb @@ -7,7 +7,6 @@ RSpec.describe ::JiraConnect::SyncBuildsWorker do it_behaves_like 'worker with data consistency', described_class, - feature_flag: :load_balancing_for_jira_connect_workers, data_consistency: :delayed describe '#perform' do diff --git a/spec/workers/jira_connect/sync_deployments_worker_spec.rb b/spec/workers/jira_connect/sync_deployments_worker_spec.rb index 62b879e0130..86ba11ebe9c 100644 --- a/spec/workers/jira_connect/sync_deployments_worker_spec.rb +++ b/spec/workers/jira_connect/sync_deployments_worker_spec.rb @@ -7,7 +7,6 @@ RSpec.describe ::JiraConnect::SyncDeploymentsWorker do it_behaves_like 'worker with data consistency', described_class, - feature_flag: :load_balancing_for_jira_connect_workers, data_consistency: :delayed describe '#perform' do diff --git a/spec/workers/jira_connect/sync_feature_flags_worker_spec.rb b/spec/workers/jira_connect/sync_feature_flags_worker_spec.rb index 1c2f9126b71..6763aefcbec 100644 --- a/spec/workers/jira_connect/sync_feature_flags_worker_spec.rb +++ b/spec/workers/jira_connect/sync_feature_flags_worker_spec.rb @@ -7,7 +7,6 @@ RSpec.describe ::JiraConnect::SyncFeatureFlagsWorker do it_behaves_like 'worker with data consistency', described_class, - feature_flag: :load_balancing_for_jira_connect_workers, data_consistency: :delayed describe '#perform' do diff --git a/spec/workers/jira_connect/sync_merge_request_worker_spec.rb b/spec/workers/jira_connect/sync_merge_request_worker_spec.rb index 7e403de54bb..65976566b22 100644 --- a/spec/workers/jira_connect/sync_merge_request_worker_spec.rb +++ b/spec/workers/jira_connect/sync_merge_request_worker_spec.rb @@ -7,7 +7,6 @@ RSpec.describe JiraConnect::SyncMergeRequestWorker do it_behaves_like 'worker with data consistency', described_class, - feature_flag: :load_balancing_for_jira_connect_workers, data_consistency: :delayed describe '#perform' do diff --git a/spec/workers/jira_connect/sync_project_worker_spec.rb b/spec/workers/jira_connect/sync_project_worker_spec.rb index 727d22d8b08..d172bde2400 100644 --- a/spec/workers/jira_connect/sync_project_worker_spec.rb +++ b/spec/workers/jira_connect/sync_project_worker_spec.rb @@ -7,7 +7,6 @@ RSpec.describe JiraConnect::SyncProjectWorker, factory_default: :keep do it_behaves_like 'worker with data consistency', described_class, - feature_flag: :load_balancing_for_jira_connect_workers, data_consistency: :delayed describe '#perform' do diff --git a/yarn.lock b/yarn.lock index c316b7d9d00..5b86288f11a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -847,10 +847,10 @@ exec-sh "^0.3.2" minimist "^1.2.0" -"@eslint/eslintrc@^0.4.2": - version "0.4.2" - resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-0.4.2.tgz#f63d0ef06f5c0c57d76c4ab5f63d3835c51b0179" - integrity sha512-8nmGq/4ycLpIwzvhI4tNDmQztZ8sp+hI7cyG8i1nQDhkAbRzHpXPidRAHlNvCZQpJTKw5ItIpMw9RSToGF00mg== +"@eslint/eslintrc@^0.4.3": + version "0.4.3" + resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-0.4.3.tgz#9e42981ef035beb3dd49add17acb96e8ff6f394c" + integrity sha512-J6KFFz5QCYUJq3pf0mjEcCJVERbzv71PUIDczuh9JkwGEzced6CO5ADLHB1rbf/+oPBtoPfMYNOpGDzCANlbXw== dependencies: ajv "^6.12.4" debug "^4.1.1" @@ -5048,13 +5048,13 @@ eslint-visitor-keys@^2.0.0: resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-2.0.0.tgz#21fdc8fbcd9c795cc0321f0563702095751511a8" integrity sha512-QudtT6av5WXels9WjIM7qz1XD1cWGvX4gGXvp/zBn9nXG02D0utdU3Em2m/QjTnrsk6bBjmCygl3rmj118msQQ== -eslint@7.30.0: - version "7.30.0" - resolved "https://registry.yarnpkg.com/eslint/-/eslint-7.30.0.tgz#6d34ab51aaa56112fd97166226c9a97f505474f8" - integrity sha512-VLqz80i3as3NdloY44BQSJpFw534L9Oh+6zJOUaViV4JPd+DaHwutqP7tcpkW3YiXbK6s05RZl7yl7cQn+lijg== +eslint@7.31.0: + version "7.31.0" + resolved "https://registry.yarnpkg.com/eslint/-/eslint-7.31.0.tgz#f972b539424bf2604907a970860732c5d99d3aca" + integrity sha512-vafgJpSh2ia8tnTkNUkwxGmnumgckLh5aAbLa1xRmIn9+owi8qBNGKL+B881kNKNTy7FFqTEkpNkUvmw0n6PkA== dependencies: "@babel/code-frame" "7.12.11" - "@eslint/eslintrc" "^0.4.2" + "@eslint/eslintrc" "^0.4.3" "@humanwhocodes/config-array" "^0.5.0" ajv "^6.10.0" chalk "^4.0.0" |