diff options
36 files changed, 576 insertions, 126 deletions
diff --git a/.rubocop.yml b/.rubocop.yml index 7ea9db1cfef..aa36d3f03ff 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -566,9 +566,6 @@ Graphql/Descriptions: RSpec/ImplicitSubject: Enabled: false -RSpec/DescribedClass: - Enabled: false - RSpec/ReceiveNever: Enabled: false diff --git a/.rubocop_todo/layout/hash_alignment.yml b/.rubocop_todo/layout/hash_alignment.yml index 9ecfe11d765..7646c1cb56d 100644 --- a/.rubocop_todo/layout/hash_alignment.yml +++ b/.rubocop_todo/layout/hash_alignment.yml @@ -89,28 +89,6 @@ Layout/HashAlignment: - 'spec/controllers/projects/registry/tags_controller_spec.rb' - 'spec/controllers/projects/service_desk_controller_spec.rb' - 'spec/db/migration_spec.rb' - - 'spec/factories/ci/builds.rb' - - 'spec/factories/environments.rb' - - 'spec/factories/groups.rb' - - 'spec/features/dashboard/datetime_on_tooltips_spec.rb' - - 'spec/features/dashboard/todos/todos_sorting_spec.rb' - - 'spec/features/jira_oauth_provider_authorize_spec.rb' - - 'spec/features/merge_request/user_sees_deployment_widget_spec.rb' - - 'spec/features/profiles/active_sessions_spec.rb' - - 'spec/features/projects/badges/coverage_spec.rb' - - 'spec/features/projects/environments/environment_spec.rb' - - 'spec/features/projects/feature_flags/user_sees_feature_flag_list_spec.rb' - - 'spec/features/projects/feature_flags/user_updates_feature_flag_spec.rb' - - 'spec/features/projects/jobs/user_browses_jobs_spec.rb' - - 'spec/features/projects/milestones/milestones_sorting_spec.rb' - - 'spec/features/projects/new_project_spec.rb' - - 'spec/features/projects/pipelines/legacy_pipeline_spec.rb' - - 'spec/features/projects/pipelines/legacy_pipelines_spec.rb' - - 'spec/features/projects/pipelines/pipeline_spec.rb' - - 'spec/features/projects/pipelines/pipelines_spec.rb' - - 'spec/features/snippets/user_creates_snippet_spec.rb' - - 'spec/features/users/email_verification_on_login_spec.rb' - - 'spec/features/users/login_spec.rb' - 'spec/finders/ci/pipelines_for_merge_request_finder_spec.rb' - 'spec/finders/group_descendants_finder_spec.rb' - 'spec/finders/group_members_finder_spec.rb' diff --git a/.rubocop_todo/rspec/described_class.yml b/.rubocop_todo/rspec/described_class.yml new file mode 100644 index 00000000000..8304704985b --- /dev/null +++ b/.rubocop_todo/rspec/described_class.yml @@ -0,0 +1,282 @@ +--- +# Cop supports --auto-correct. +RSpec/DescribedClass: + Exclude: + - 'ee/spec/controllers/concerns/gitlab_subscriptions/seat_count_alert_spec.rb' + - 'ee/spec/controllers/concerns/registrations/verification_spec.rb' + - 'ee/spec/controllers/concerns/routable_actions_spec.rb' + - 'ee/spec/controllers/repositories/git_http_controller_spec.rb' + - 'ee/spec/frontend/fixtures/epic.rb' + - 'ee/spec/graphql/ee/types/todoable_interface_spec.rb' + - 'ee/spec/lib/ee/gitlab/background_migration/migrate_shared_vulnerability_scanners_spec.rb' + - 'ee/spec/lib/ee/gitlab/elastic/helper_spec.rb' + - 'ee/spec/lib/ee/gitlab/git_access_snippet_spec.rb' + - 'ee/spec/lib/gitlab/auth/saml/config_spec.rb' + - 'ee/spec/lib/gitlab/checks/changes_access_spec.rb' + - 'ee/spec/lib/gitlab/geo/log_helpers_spec.rb' + - 'ee/spec/lib/gitlab/geo/replicator_spec.rb' + - 'ee/spec/lib/gitlab/geo_spec.rb' + - 'ee/spec/lib/gitlab/gl_repository/repo_type_spec.rb' + - 'ee/spec/lib/gitlab/instrumentation/elasticsearch_transport_spec.rb' + - 'ee/spec/lib/gitlab/vulnerabilities/findings_preloader_spec.rb' + - 'ee/spec/lib/omni_auth/strategies/group_saml_spec.rb' + - 'ee/spec/models/ci/minutes/namespace_monthly_usage_spec.rb' + - 'ee/spec/models/ci/processable_spec.rb' + - 'ee/spec/models/concerns/elastic/issue_spec.rb' + - 'ee/spec/models/concerns/elastic/merge_request_spec.rb' + - 'ee/spec/models/concerns/elastic/note_spec.rb' + - 'ee/spec/models/concerns/elastic/project_spec.rb' + - 'ee/spec/models/concerns/elastic/repository_spec.rb' + - 'ee/spec/models/concerns/elastic/snippet_spec.rb' + - 'ee/spec/models/dast_scanner_profile_spec.rb' + - 'ee/spec/models/dast_site_profile_spec.rb' + - 'ee/spec/models/ee/ci/job_artifact_spec.rb' + - 'ee/spec/models/ee/ci/pending_build_spec.rb' + - 'ee/spec/models/ee/ci/runner_spec.rb' + - 'ee/spec/models/ee/gpg_key_spec.rb' + - 'ee/spec/models/ee/group_spec.rb' + - 'ee/spec/models/ee/iteration_spec.rb' + - 'ee/spec/models/ee/merge_request_diff_spec.rb' + - 'ee/spec/models/ee/namespace/storage/notification_spec.rb' + - 'ee/spec/models/ee/vulnerability_spec.rb' + - 'ee/spec/models/epic_issue_spec.rb' + - 'ee/spec/models/epic_spec.rb' + - 'ee/spec/models/geo/container_repository_registry_spec.rb' + - 'ee/spec/models/geo/design_registry_spec.rb' + - 'ee/spec/models/geo/package_file_registry_spec.rb' + - 'ee/spec/models/geo/project_registry_spec.rb' + - 'ee/spec/models/geo/secondary_usage_data_spec.rb' + - 'ee/spec/models/incident_management/escalation_policy_spec.rb' + - 'ee/spec/models/issuable_metric_image_spec.rb' + - 'ee/spec/models/issue_spec.rb' + - 'ee/spec/models/license_spec.rb' + - 'ee/spec/models/merge_train_spec.rb' + - 'ee/spec/models/plan_spec.rb' + - 'ee/spec/models/project_import_state_spec.rb' + - 'ee/spec/models/project_spec.rb' + - 'ee/spec/models/release_highlight_spec.rb' + - 'ee/spec/models/requirements_management/requirement_spec.rb' + - 'ee/spec/models/requirements_management/test_report_spec.rb' + - 'ee/spec/models/resource_weight_event_spec.rb' + - 'ee/spec/models/uploads/local_spec.rb' + - 'ee/spec/models/vulnerabilities/finding_spec.rb' + - 'ee/spec/models/vulnerabilities/flag_spec.rb' + - 'ee/spec/models/vulnerabilities/read_spec.rb' + - 'ee/spec/services/arkose/blocked_users_report_service_spec.rb' + - 'ee/spec/services/arkose/user_verification_service_spec.rb' + - 'ee/spec/services/ee/resource_events/synthetic_weight_notes_builder_service_spec.rb' + - 'ee/spec/services/ee/users/reject_service_spec.rb' + - 'ee/spec/services/resource_access_tokens/create_service_spec.rb' + - 'ee/spec/services/security/ingestion/tasks/update_vulnerability_uuids_spec.rb' + - 'ee/spec/services/users/captcha_challenge_service_spec.rb' + - 'ee/spec/services/vulnerabilities/create_from_security_finding_service_spec.rb' + - 'ee/spec/workers/concerns/elastic/indexing_control_spec.rb' + - 'ee/spec/workers/elastic/migration_worker_spec.rb' + - 'ee/spec/workers/geo/secondary/registry_consistency_worker_spec.rb' + - 'ee/spec/workers/geo/verification_state_backfill_worker_spec.rb' + - 'qa/spec/service/docker_run/base_spec.rb' + - 'qa/spec/support/formatters/test_stats_formatter_spec.rb' + - 'qa/spec/support/loglinking_spec.rb' + - 'qa/spec/support/page_error_checker_spec.rb' + - 'spec/config/settings_spec.rb' + - 'spec/controllers/concerns/confirm_email_warning_spec.rb' + - 'spec/controllers/concerns/controller_with_cross_project_access_check_spec.rb' + - 'spec/controllers/concerns/enforces_admin_authentication_spec.rb' + - 'spec/controllers/concerns/graceful_timeout_handling_spec.rb' + - 'spec/controllers/concerns/group_tree_spec.rb' + - 'spec/controllers/concerns/metrics_dashboard_spec.rb' + - 'spec/controllers/concerns/product_analytics_tracking_spec.rb' + - 'spec/controllers/concerns/redirects_for_missing_path_on_tree_spec.rb' + - 'spec/controllers/concerns/redis_tracking_spec.rb' + - 'spec/controllers/concerns/renders_commits_spec.rb' + - 'spec/controllers/concerns/routable_actions_spec.rb' + - 'spec/controllers/concerns/sourcegraph_decorator_spec.rb' + - 'spec/controllers/concerns/spammable_actions/akismet_mark_as_spam_action_spec.rb' + - 'spec/controllers/concerns/spammable_actions/captcha_check/html_format_actions_support_spec.rb' + - 'spec/controllers/concerns/spammable_actions/captcha_check/json_format_actions_support_spec.rb' + - 'spec/controllers/concerns/static_object_external_storage_spec.rb' + - 'spec/controllers/repositories/git_http_controller_spec.rb' + - 'spec/experiments/application_experiment_spec.rb' + - 'spec/experiments/concerns/project_commit_count_spec.rb' + - 'spec/frontend/fixtures/metrics_dashboard.rb' + - 'spec/frontend/fixtures/timezones.rb' + - 'spec/frontend/fixtures/u2f.rb' + - 'spec/frontend/fixtures/webauthn.rb' + - 'spec/graphql/gitlab_schema_spec.rb' + - 'spec/graphql/graphql_triggers_spec.rb' + - 'spec/graphql/types/global_id_type_spec.rb' + - 'spec/initializers/google_api_client_spec.rb' + - 'spec/lib/api/entities/project_spec.rb' + - 'spec/lib/api/helpers_spec.rb' + - 'spec/lib/bulk_imports/groups/stage_spec.rb' + - 'spec/lib/bulk_imports/projects/stage_spec.rb' + - 'spec/lib/expand_variables_spec.rb' + - 'spec/lib/feature_spec.rb' + - 'spec/lib/gitlab/application_context_spec.rb' + - 'spec/lib/gitlab/asciidoc/html5_converter_spec.rb' + - 'spec/lib/gitlab/buffered_io_spec.rb' + - 'spec/lib/gitlab/checks/snippet_check_spec.rb' + - 'spec/lib/gitlab/ci/tags/bulk_insert_spec.rb' + - 'spec/lib/gitlab/ci/trace/chunked_io_spec.rb' + - 'spec/lib/gitlab/ci/variables/collection/item_spec.rb' + - 'spec/lib/gitlab/ci/variables/collection/sort_spec.rb' + - 'spec/lib/gitlab/ci/variables/collection_spec.rb' + - 'spec/lib/gitlab/ci/yaml_processor_spec.rb' + - 'spec/lib/gitlab/config/entry/composable_array_spec.rb' + - 'spec/lib/gitlab/config/entry/composable_hash_spec.rb' + - 'spec/lib/gitlab/current_settings_spec.rb' + - 'spec/lib/gitlab/database/background_migration/health_status_spec.rb' + - 'spec/lib/gitlab/database/load_balancing/host_spec.rb' + - 'spec/lib/gitlab/database/load_balancing/primary_host_spec.rb' + - 'spec/lib/gitlab/database/migrations/lock_retry_mixin_spec.rb' + - 'spec/lib/gitlab/database/query_analyzers/gitlab_schemas_validate_connection_spec.rb' + - 'spec/lib/gitlab/database/reindexing_spec.rb' + - 'spec/lib/gitlab/database/similarity_score_spec.rb' + - 'spec/lib/gitlab/database_spec.rb' + - 'spec/lib/gitlab/email/handler_spec.rb' + - 'spec/lib/gitlab/email/hook/smime_signature_interceptor_spec.rb' + - 'spec/lib/gitlab/encrypted_configuration_spec.rb' + - 'spec/lib/gitlab/error_tracking/logger_spec.rb' + - 'spec/lib/gitlab/experimentation/controller_concern_spec.rb' + - 'spec/lib/gitlab/git/blame_spec.rb' + - 'spec/lib/gitlab/git/blob_spec.rb' + - 'spec/lib/gitlab/git/commit_spec.rb' + - 'spec/lib/gitlab/git/compare_spec.rb' + - 'spec/lib/gitlab/git/diff_collection_spec.rb' + - 'spec/lib/gitlab/git/repository_spec.rb' + - 'spec/lib/gitlab/git/tree_spec.rb' + - 'spec/lib/gitlab/git_access_snippet_spec.rb' + - 'spec/lib/gitlab/gl_repository/repo_type_spec.rb' + - 'spec/lib/gitlab/import_formatter_spec.rb' + - 'spec/lib/gitlab/inactive_projects_deletion_warning_tracker_spec.rb' + - 'spec/lib/gitlab/kubernetes/kube_client_spec.rb' + - 'spec/lib/gitlab/no_cache_headers_spec.rb' + - 'spec/lib/gitlab/pagination/keyset/order_spec.rb' + - 'spec/lib/gitlab/relative_positioning/range_spec.rb' + - 'spec/lib/gitlab/runtime_spec.rb' + - 'spec/lib/gitlab/search_context/controller_concern_spec.rb' + - 'spec/lib/gitlab/seeder_spec.rb' + - 'spec/lib/gitlab/serverless/service_spec.rb' + - 'spec/lib/gitlab/sidekiq_middleware/size_limiter/client_spec.rb' + - 'spec/lib/gitlab/suggestions/commit_message_spec.rb' + - 'spec/lib/gitlab/usage/metrics/aggregates/sources/redis_hll_spec.rb' + - 'spec/lib/gitlab/utils/measuring_spec.rb' + - 'spec/lib/gitlab/webpack/file_loader_spec.rb' + - 'spec/lib/gitlab/webpack/manifest_spec.rb' + - 'spec/lib/gitlab/x509/commit_spec.rb' + - 'spec/lib/gitlab/x509/signature_spec.rb' + - 'spec/lib/peek/views/active_record_spec.rb' + - 'spec/lib/service_ping/devops_report_spec.rb' + - 'spec/lib/sidebars/panel_spec.rb' + - 'spec/mailers/emails/service_desk_spec.rb' + - 'spec/metrics_server/metrics_server_spec.rb' + - 'spec/migrations/remove_records_without_group_from_webhooks_table_spec.rb' + - 'spec/models/active_session_spec.rb' + - 'spec/models/alert_management/alert_spec.rb' + - 'spec/models/application_record_spec.rb' + - 'spec/models/application_setting_spec.rb' + - 'spec/models/awareness_session_spec.rb' + - 'spec/models/broadcast_message_spec.rb' + - 'spec/models/chat_name_spec.rb' + - 'spec/models/ci/build_runner_session_spec.rb' + - 'spec/models/ci/build_spec.rb' + - 'spec/models/ci/group_variable_spec.rb' + - 'spec/models/ci/job_artifact_spec.rb' + - 'spec/models/ci/namespace_mirror_spec.rb' + - 'spec/models/ci/pipeline_artifact_spec.rb' + - 'spec/models/ci/pipeline_spec.rb' + - 'spec/models/ci/processable_spec.rb' + - 'spec/models/ci/ref_spec.rb' + - 'spec/models/ci/variable_spec.rb' + - 'spec/models/ci_platform_metric_spec.rb' + - 'spec/models/clusters/cluster_spec.rb' + - 'spec/models/commit_spec.rb' + - 'spec/models/commit_status_spec.rb' + - 'spec/models/concerns/blocks_unsafe_serialization_spec.rb' + - 'spec/models/concerns/bulk_insertable_associations_spec.rb' + - 'spec/models/concerns/counter_attribute_spec.rb' + - 'spec/models/concerns/has_user_type_spec.rb' + - 'spec/models/concerns/integrations/enable_ssl_verification_spec.rb' + - 'spec/models/concerns/integrations/reset_secret_fields_spec.rb' + - 'spec/models/concerns/issuable_spec.rb' + - 'spec/models/concerns/mentionable_spec.rb' + - 'spec/models/concerns/resolvable_note_spec.rb' + - 'spec/models/concerns/token_authenticatable_spec.rb' + - 'spec/models/customer_relations/contact_spec.rb' + - 'spec/models/customer_relations/organization_spec.rb' + - 'spec/models/dependency_proxy/manifest_spec.rb' + - 'spec/models/deployment_spec.rb' + - 'spec/models/experiment_spec.rb' + - 'spec/models/group_spec.rb' + - 'spec/models/import_failure_spec.rb' + - 'spec/models/integration_spec.rb' + - 'spec/models/internal_id_spec.rb' + - 'spec/models/issue_assignee_spec.rb' + - 'spec/models/issue_spec.rb' + - 'spec/models/jira_import_state_spec.rb' + - 'spec/models/label_link_spec.rb' + - 'spec/models/lfs_objects_project_spec.rb' + - 'spec/models/loose_foreign_keys/deleted_record_spec.rb' + - 'spec/models/member_spec.rb' + - 'spec/models/merge_request_assignee_spec.rb' + - 'spec/models/merge_request_diff_commit_spec.rb' + - 'spec/models/merge_request_diff_file_spec.rb' + - 'spec/models/merge_request_spec.rb' + - 'spec/models/milestone_spec.rb' + - 'spec/models/namespace_spec.rb' + - 'spec/models/namespace_statistics_spec.rb' + - 'spec/models/oauth_access_token_spec.rb' + - 'spec/models/packages/dependency_spec.rb' + - 'spec/models/packages/maven/metadatum_spec.rb' + - 'spec/models/packages/package_spec.rb' + - 'spec/models/pages_deployment_spec.rb' + - 'spec/models/pages_domain_spec.rb' + - 'spec/models/performance_monitoring/prometheus_dashboard_spec.rb' + - 'spec/models/performance_monitoring/prometheus_metric_spec.rb' + - 'spec/models/performance_monitoring/prometheus_panel_group_spec.rb' + - 'spec/models/performance_monitoring/prometheus_panel_spec.rb' + - 'spec/models/plan_spec.rb' + - 'spec/models/postgresql/detached_partition_spec.rb' + - 'spec/models/preloaders/user_max_access_level_in_projects_preloader_spec.rb' + - 'spec/models/preloaders/users_max_access_level_in_projects_preloader_spec.rb' + - 'spec/models/project_feature_usage_spec.rb' + - 'spec/models/project_setting_spec.rb' + - 'spec/models/project_spec.rb' + - 'spec/models/projects/topic_spec.rb' + - 'spec/models/release_highlight_spec.rb' + - 'spec/models/release_spec.rb' + - 'spec/models/route_spec.rb' + - 'spec/models/todo_spec.rb' + - 'spec/models/u2f_registration_spec.rb' + - 'spec/models/user_custom_attribute_spec.rb' + - 'spec/models/user_preference_spec.rb' + - 'spec/models/user_spec.rb' + - 'spec/models/users/merge_request_interaction_spec.rb' + - 'spec/models/users_statistics_spec.rb' + - 'spec/models/wiki_directory_spec.rb' + - 'spec/models/work_items/type_spec.rb' + - 'spec/serializers/context_commits_diff_entity_spec.rb' + - 'spec/services/alert_management/alerts/todo/create_service_spec.rb' + - 'spec/services/auth/dependency_proxy_authentication_service_spec.rb' + - 'spec/services/ci/register_job_service_spec.rb' + - 'spec/services/incident_management/issuable_escalation_statuses/after_update_service_spec.rb' + - 'spec/services/incident_management/issuable_escalation_statuses/prepare_update_service_spec.rb' + - 'spec/services/issuable/process_assignees_spec.rb' + - 'spec/services/loose_foreign_keys/cleaner_service_spec.rb' + - 'spec/services/merge_requests/update_service_spec.rb' + - 'spec/services/notification_service_spec.rb' + - 'spec/services/projects/create_service_spec.rb' + - 'spec/services/resource_access_tokens/create_service_spec.rb' + - 'spec/services/resource_events/synthetic_label_notes_builder_service_spec.rb' + - 'spec/services/service_ping/submit_service_ping_service_spec.rb' + - 'spec/services/snippets/update_service_spec.rb' + - 'spec/services/system_hooks_service_spec.rb' + - 'spec/services/user_project_access_changed_service_spec.rb' + - 'spec/services/webauthn/authenticate_service_spec.rb' + - 'spec/services/webauthn/register_service_spec.rb' + - 'spec/support_specs/helpers/active_record/query_recorder_spec.rb' + - 'spec/support_specs/helpers/graphql_helpers_spec.rb' + - 'spec/support_specs/matchers/exceed_query_limit_helpers_spec.rb' + - 'spec/validators/html_safety_validator_spec.rb' + - 'spec/workers/bulk_imports/entity_worker_spec.rb' + - 'spec/workers/jira_connect/retry_request_worker_spec.rb' @@ -384,7 +384,7 @@ group :development, :test do gem 'gitlab-styles', '~> 7.1.0', require: false - gem 'haml_lint', '~> 0.36.0', require: false + gem 'haml_lint', '~> 0.40.0', require: false gem 'bundler-audit', '~> 0.7.0.1', require: false gem 'benchmark-ips', '~> 2.3.0', require: false diff --git a/Gemfile.lock b/Gemfile.lock index d9e9b2f04a8..0d729b0232e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -649,10 +649,10 @@ GEM guard (~> 2.1) guard-compat (~> 1.1) rspec (>= 2.99.0, < 4.0) - haml (5.1.2) + haml (5.2.2) temple (>= 0.8.0) tilt - haml_lint (0.36.0) + haml_lint (0.40.1) haml (>= 4.0, < 5.3) parallel (~> 1.10) rainbow @@ -1606,7 +1606,7 @@ DEPENDENCIES grpc (~> 1.42.0) gssapi guard-rspec - haml_lint (~> 0.36.0) + haml_lint (~> 0.40.0) hamlit (~> 2.15.0) hangouts-chat (~> 0.0.5) hashie diff --git a/app/workers/analytics/usage_trends/counter_job_worker.rb b/app/workers/analytics/usage_trends/counter_job_worker.rb index b3a8f7dd3c2..e6de623f784 100644 --- a/app/workers/analytics/usage_trends/counter_job_worker.rb +++ b/app/workers/analytics/usage_trends/counter_job_worker.rb @@ -3,6 +3,8 @@ module Analytics module UsageTrends class CounterJobWorker + TIMEOUT = 250.seconds + extend ::Gitlab::Utils::Override include ApplicationWorker @@ -15,24 +17,27 @@ module Analytics idempotent! - def perform(measurement_identifier, min_id, max_id, recorded_at) + def perform(measurement_identifier, min_id, max_id, recorded_at, partial_results = nil) query_scope = ::Analytics::UsageTrends::Measurement.identifier_query_mapping[measurement_identifier].call - count = if min_id.nil? || max_id.nil? # table is empty - 0 - else - counter(query_scope, min_id, max_id) - end + result = counter(query_scope, min_id, max_id, partial_results) + + # If the batch counter timed out, schedule a job to continue counting later + if result[:status] == :timeout + return self.class.perform_async(measurement_identifier, result[:continue_from], max_id, recorded_at, result[:partial_results]) + end - return if count == Gitlab::Database::BatchCounter::FALLBACK + return if result[:status] != :completed - UsageTrends::Measurement.insert_all([{ recorded_at: recorded_at, count: count, identifier: measurement_identifier }]) + UsageTrends::Measurement.insert_all([{ recorded_at: recorded_at, count: result[:count], identifier: measurement_identifier }]) end private - def counter(query_scope, min_id, max_id) - Gitlab::Database::BatchCount.batch_count(query_scope, start: min_id, finish: max_id) + def counter(query_scope, min_id, max_id, partial_results) + return { status: :completed, count: 0 } if min_id.nil? || max_id.nil? # table is empty + + Gitlab::Database::BatchCount.batch_count_with_timeout(query_scope, start: min_id, finish: max_id, timeout: TIMEOUT, partial_results: partial_results) end end end diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index 2dd0db54d8e..30cdd9fc188 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -18836,6 +18836,7 @@ Represents a vulnerability scanner. | <a id="vulnerabilityscannerid"></a>`id` | [`ID`](#id) | ID of the scanner. | | <a id="vulnerabilityscannername"></a>`name` | [`String`](#string) | Name of the vulnerability scanner. | | <a id="vulnerabilityscannerreporttype"></a>`reportType` | [`VulnerabilityReportType`](#vulnerabilityreporttype) | Type of the vulnerability report. | +| <a id="vulnerabilityscannerreporttypehumanized"></a>`reportTypeHumanized` | [`String`](#string) | Humanized type of the vulnerability report. | | <a id="vulnerabilityscannervendor"></a>`vendor` | [`String`](#string) | Vendor of the vulnerability scanner. | ### `VulnerabilitySeveritiesCount` diff --git a/lib/gitlab/database/batch_count.rb b/lib/gitlab/database/batch_count.rb index 92a41bb36ee..b33f39e3cbe 100644 --- a/lib/gitlab/database/batch_count.rb +++ b/lib/gitlab/database/batch_count.rb @@ -35,6 +35,10 @@ module Gitlab BatchCounter.new(relation, column: column).count(batch_size: batch_size, start: start, finish: finish) end + def batch_count_with_timeout(relation, column = nil, batch_size: nil, start: nil, finish: nil, timeout: nil, partial_results: nil) + BatchCounter.new(relation, column: column).count_with_timeout(batch_size: batch_size, start: start, finish: finish, timeout: timeout, partial_results: partial_results) + end + def batch_distinct_count(relation, column = nil, batch_size: nil, start: nil, finish: nil) BatchCounter.new(relation, column: column).count(mode: :distinct, batch_size: batch_size, start: start, finish: finish) end diff --git a/lib/gitlab/database/batch_counter.rb b/lib/gitlab/database/batch_counter.rb index 522b598cd9d..aa4abf15755 100644 --- a/lib/gitlab/database/batch_counter.rb +++ b/lib/gitlab/database/batch_counter.rb @@ -33,6 +33,14 @@ module Gitlab end def count(batch_size: nil, mode: :itself, start: nil, finish: nil) + result = count_with_timeout(batch_size: batch_size, mode: mode, start: start, finish: finish, timeout: nil) + + return FALLBACK if result[:status] != :completed + + result[:count] + end + + def count_with_timeout(batch_size: nil, mode: :itself, start: nil, finish: nil, timeout: nil, partial_results: nil) raise 'BatchCount can not be run inside a transaction' if transaction_open? check_mode!(mode) @@ -44,12 +52,20 @@ module Gitlab finish = actual_finish(finish) raise "Batch counting expects positive values only for #{@column}" if start < 0 || finish < 0 - return FALLBACK if unwanted_configuration?(finish, batch_size, start) + return { status: :bad_config } if unwanted_configuration?(finish, batch_size, start) - results = nil + results = partial_results batch_start = start + start_time = ::Gitlab::Metrics::System.monotonic_time.seconds + while batch_start < finish + + # Timeout elapsed, return partial result so the caller can continue later + if timeout && ::Gitlab::Metrics::System.monotonic_time.seconds - start_time > timeout + return { status: :timeout, partial_results: results, continue_from: batch_start } + end + begin batch_end = [batch_start + batch_size, finish].min batch_relation = build_relation_batch(batch_start, batch_end, mode) @@ -62,14 +78,14 @@ module Gitlab batch_size /= 2 else log_canceled_batch_fetch(batch_start, mode, batch_relation.to_sql, error) - return FALLBACK + return { status: :cancelled } end end sleep(SLEEP_TIME_IN_SECONDS) end - results + { status: :completed, count: results } end def transaction_open? diff --git a/qa/qa/page/project/settings/pages.rb b/qa/qa/page/project/settings/pages.rb index 1417f7ec1b5..c5b8560ba9a 100644 --- a/qa/qa/page/project/settings/pages.rb +++ b/qa/qa/page/project/settings/pages.rb @@ -14,6 +14,7 @@ module QA def go_to_access_page within_element(:access_page_container) do find('a').click + page.driver.browser.switch_to.window(page.driver.browser.window_handles.last) end end end diff --git a/qa/qa/resource/user.rb b/qa/qa/resource/user.rb index 2fb8b18b71f..82d12fd69b5 100644 --- a/qa/qa/resource/user.rb +++ b/qa/qa/resource/user.rb @@ -232,3 +232,5 @@ module QA end end end + +QA::Resource::User.prepend_mod_with('Resource::User', namespace: QA) diff --git a/qa/qa/specs/features/browser_ui/3_create/pages/pages_pipeline_spec.rb b/qa/qa/specs/features/browser_ui/3_create/pages/new_static_page_spec.rb index 191c4a096e7..9c912080c5f 100644 --- a/qa/qa/specs/features/browser_ui/3_create/pages/pages_pipeline_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/pages/new_static_page_spec.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true - module QA - RSpec.describe 'Release', :runner do + RSpec.describe 'Create', :runner, only: { subdomain: :staging } do # TODO: Convert back to :smoke once proved to be stable. Related issue: https://gitlab.com/gitlab-org/gitlab/-/issues/300906 describe 'Pages' do let!(:project) do @@ -27,12 +26,11 @@ module QA runner.project = project runner.executor = :docker end - pipeline.visit! end - it('runs a Pages-specific pipeline', - testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347669') do + it 'creates a Pages website', + testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347669' do Page::Project::Pipeline::Show.perform do |show| expect(show).to have_job(:pages) show.click_job(:pages) @@ -41,6 +39,15 @@ module QA Page::Project::Job::Show.perform do |show| expect(show).to have_passed(timeout: 300) end + + Page::Project::Show.perform(&:go_to_pages_settings) + QA::Page::Project::Settings::Pages.perform do |pages| + pages.go_to_access_page + Support::Waiter.wait_until(sleep_interval: 2, max_duration: 60, reload_page: page, + retry_on_exception: true) do + expect(page).to have_content('Write an awesome description for your new site here.') + end + end end end end diff --git a/spec/factories/ci/builds.rb b/spec/factories/ci/builds.rb index d684f79a518..8c2edc8cd9f 100644 --- a/spec/factories/ci/builds.rb +++ b/spec/factories/ci/builds.rb @@ -129,8 +129,8 @@ FactoryBot.define do { script: %w(ls), environment: { name: 'staging', - action: 'stop', - url: 'http://staging.example.com/$CI_JOB_NAME' } + action: 'stop', + url: 'http://staging.example.com/$CI_JOB_NAME' } } end end @@ -141,9 +141,9 @@ FactoryBot.define do { script: %w(ls), environment: { name: 'test_portal', - action: 'start', - url: 'http://staging.example.com/$CI_JOB_NAME', - deployment_tier: 'testing' } + action: 'start', + url: 'http://staging.example.com/$CI_JOB_NAME', + deployment_tier: 'testing' } } end end @@ -155,7 +155,7 @@ FactoryBot.define do { script: %w(ls), environment: { name: 'production', - url: 'http://prd.example.com/$CI_JOB_NAME' } + url: 'http://prd.example.com/$CI_JOB_NAME' } } end end @@ -167,8 +167,8 @@ FactoryBot.define do { script: %w(ls), environment: { name: 'review/$CI_COMMIT_REF_NAME', - url: 'http://staging.example.com/$CI_JOB_NAME', - on_stop: 'stop_review_app' } + url: 'http://staging.example.com/$CI_JOB_NAME', + on_stop: 'stop_review_app' } } end end @@ -181,8 +181,8 @@ FactoryBot.define do { script: %w(ls), environment: { name: 'review/$CI_COMMIT_REF_NAME', - url: 'http://staging.example.com/$CI_JOB_NAME', - action: 'stop' } + url: 'http://staging.example.com/$CI_JOB_NAME', + action: 'stop' } } end end diff --git a/spec/factories/environments.rb b/spec/factories/environments.rb index ccd2011eb8d..34843dab0fe 100644 --- a/spec/factories/environments.rb +++ b/spec/factories/environments.rb @@ -47,7 +47,7 @@ FactoryBot.define do pipeline = create(:ci_pipeline, project: environment.project) deployable = create(:ci_build, :success, name: "#{environment.name}:deploy", - pipeline: pipeline) + pipeline: pipeline) deployment = create(:deployment, :success, diff --git a/spec/factories/groups.rb b/spec/factories/groups.rb index 152ae061605..6f9cf0ef895 100644 --- a/spec/factories/groups.rb +++ b/spec/factories/groups.rb @@ -106,9 +106,9 @@ FactoryBot.define do end create_graph( - parent: group, + parent: group, children: evaluator.children, - depth: evaluator.depth + depth: evaluator.depth ) end end diff --git a/spec/features/dashboard/datetime_on_tooltips_spec.rb b/spec/features/dashboard/datetime_on_tooltips_spec.rb index bf9f6895d24..48a6976f263 100644 --- a/spec/features/dashboard/datetime_on_tooltips_spec.rb +++ b/spec/features/dashboard/datetime_on_tooltips_spec.rb @@ -15,7 +15,7 @@ RSpec.describe 'Tooltips on .timeago dates', :js do context 'on the activity tab' do before do Event.create!( project: project, author_id: user.id, action: :joined, - updated_at: created_date, created_at: created_date) + updated_at: created_date, created_at: created_date) sign_in user visit user_activity_path(user) diff --git a/spec/features/dashboard/todos/todos_sorting_spec.rb b/spec/features/dashboard/todos/todos_sorting_spec.rb index d593031590e..a0fa53b761b 100644 --- a/spec/features/dashboard/todos/todos_sorting_spec.rb +++ b/spec/features/dashboard/todos/todos_sorting_spec.rb @@ -28,7 +28,7 @@ RSpec.describe 'Dashboard > User sorts todos' do create(:todo, user: user, project: project, target: issue_3, created_at: 3.hours.ago, updated_at: 2.minutes.ago) create(:todo, user: user, project: project, target: issue_1, created_at: 2.hours.ago, updated_at: 2.hours.ago) create(:todo, user: user, project: project, target: merge_request_1, created_at: 1.hour.ago, - updated_at: 1.hour.ago) + updated_at: 1.hour.ago) merge_request_1.labels << label_1 issue_3.labels << label_1 diff --git a/spec/features/jira_oauth_provider_authorize_spec.rb b/spec/features/jira_oauth_provider_authorize_spec.rb index a216d2d44b2..eb26440aff9 100644 --- a/spec/features/jira_oauth_provider_authorize_spec.rb +++ b/spec/features/jira_oauth_provider_authorize_spec.rb @@ -10,10 +10,10 @@ RSpec.describe 'JIRA OAuth Provider' do sign_in(user) visit oauth_jira_dvcs_authorize_path(client_id: application.uid, - redirect_uri: oauth_jira_dvcs_callback_url, - response_type: 'code', - state: 'my_state', - scope: 'read_user') + redirect_uri: oauth_jira_dvcs_callback_url, + response_type: 'code', + state: 'my_state', + scope: 'read_user') end it_behaves_like 'Secure OAuth Authorizations' diff --git a/spec/features/merge_request/user_sees_deployment_widget_spec.rb b/spec/features/merge_request/user_sees_deployment_widget_spec.rb index c02149eed87..63ac7862b06 100644 --- a/spec/features/merge_request/user_sees_deployment_widget_spec.rb +++ b/spec/features/merge_request/user_sees_deployment_widget_spec.rb @@ -111,7 +111,7 @@ RSpec.describe 'Merge request > User sees deployment widget', :js do context 'with stop action' do let(:manual) do create(:ci_build, :manual, pipeline: pipeline, - name: 'close_app', environment: environment.name) + name: 'close_app', environment: environment.name) end before do diff --git a/spec/features/profiles/active_sessions_spec.rb b/spec/features/profiles/active_sessions_spec.rb index 24c9225532b..d0819bb5363 100644 --- a/spec/features/profiles/active_sessions_spec.rb +++ b/spec/features/profiles/active_sessions_spec.rb @@ -59,7 +59,7 @@ RSpec.describe 'Profile > Active Sessions', :clean_gitlab_redis_shared_state do expect(page).to( have_selector('ul.list-group li.list-group-item', text: 'Signed in on', - count: 2)) + count: 2)) expect(page).to have_content( '127.0.0.1 ' \ diff --git a/spec/features/projects/badges/coverage_spec.rb b/spec/features/projects/badges/coverage_spec.rb index 5c1bc1ad239..7555e567c37 100644 --- a/spec/features/projects/badges/coverage_spec.rb +++ b/spec/features/projects/badges/coverage_spec.rb @@ -191,7 +191,7 @@ RSpec.describe 'test coverage badge' do def show_test_coverage_badge(job: nil, min_good: nil, min_acceptable: nil, min_medium: nil) visit coverage_project_badges_path(project, ref: :master, job: job, min_good: min_good, - min_acceptable: min_acceptable, min_medium: min_medium, format: :svg) + min_acceptable: min_acceptable, min_medium: min_medium, format: :svg) end def expect_coverage_badge(coverage) diff --git a/spec/features/projects/environments/environment_spec.rb b/spec/features/projects/environments/environment_spec.rb index a53e8beb555..be4b21dfff4 100644 --- a/spec/features/projects/environments/environment_spec.rb +++ b/spec/features/projects/environments/environment_spec.rb @@ -264,9 +264,7 @@ RSpec.describe 'Environment' do let(:build) { create(:ci_build, :success, pipeline: pipeline, environment: environment.name) } let(:action) do - create(:ci_build, :manual, pipeline: pipeline, - name: 'close_app', - environment: environment.name) + create(:ci_build, :manual, pipeline: pipeline, name: 'close_app', environment: environment.name) end let(:deployment) do @@ -278,8 +276,7 @@ RSpec.describe 'Environment' do context 'when user has ability to stop environment' do let(:permissions) do - create(:protected_branch, :developers_can_merge, - name: action.ref, project: project) + create(:protected_branch, :developers_can_merge, name: action.ref, project: project) end it 'allows to stop environment', :js do diff --git a/spec/features/projects/feature_flags/user_sees_feature_flag_list_spec.rb b/spec/features/projects/feature_flags/user_sees_feature_flag_list_spec.rb index 221f07a2f75..c05707d2921 100644 --- a/spec/features/projects/feature_flags/user_sees_feature_flag_list_spec.rb +++ b/spec/features/projects/feature_flags/user_sees_feature_flag_list_spec.rb @@ -30,8 +30,7 @@ RSpec.describe 'User sees feature flag list', :js do create(:operations_scope, strategy: strat, environment_scope: 'production') end end - create(:operations_feature_flag, :new_version_flag, project: project, - name: 'my_flag', active: false) + create(:operations_feature_flag, :new_version_flag, project: project, name: 'my_flag', active: false) end it 'shows the user the first flag' do diff --git a/spec/features/projects/feature_flags/user_updates_feature_flag_spec.rb b/spec/features/projects/feature_flags/user_updates_feature_flag_spec.rb index 71c9d89fbde..eb9ac078662 100644 --- a/spec/features/projects/feature_flags/user_updates_feature_flag_spec.rb +++ b/spec/features/projects/feature_flags/user_updates_feature_flag_spec.rb @@ -18,13 +18,12 @@ RSpec.describe 'User updates feature flag', :js do context 'with a new version feature flag' do let!(:feature_flag) do - create_flag(project, 'test_flag', false, version: Operations::FeatureFlag.versions['new_version_flag'], - description: 'For testing') + create_flag(project, 'test_flag', false, + version: Operations::FeatureFlag.versions['new_version_flag'], description: 'For testing') end let!(:strategy) do - create(:operations_strategy, feature_flag: feature_flag, - name: 'default', parameters: {}) + create(:operations_strategy, feature_flag: feature_flag, name: 'default', parameters: {}) end let!(:scope) do diff --git a/spec/features/projects/jobs/user_browses_jobs_spec.rb b/spec/features/projects/jobs/user_browses_jobs_spec.rb index 289ab8cffa5..995f4a1e3d2 100644 --- a/spec/features/projects/jobs/user_browses_jobs_spec.rb +++ b/spec/features/projects/jobs/user_browses_jobs_spec.rb @@ -58,8 +58,7 @@ RSpec.describe 'User browses jobs' do context 'when a job can be canceled' do let!(:job) do - create(:ci_build, pipeline: pipeline, - stage: 'test') + create(:ci_build, pipeline: pipeline, stage: 'test') end before do @@ -81,7 +80,7 @@ RSpec.describe 'User browses jobs' do context 'when a job can be retried' do let!(:job) do create(:ci_build, pipeline: pipeline, - stage: 'test') + stage: 'test') end before do @@ -190,7 +189,7 @@ RSpec.describe 'User browses jobs' do context 'column links' do let!(:job) do create(:ci_build, pipeline: pipeline, - stage: 'test') + stage: 'test') end before do diff --git a/spec/features/projects/milestones/milestones_sorting_spec.rb b/spec/features/projects/milestones/milestones_sorting_spec.rb index 2ad820e4a06..c47350fb663 100644 --- a/spec/features/projects/milestones/milestones_sorting_spec.rb +++ b/spec/features/projects/milestones/milestones_sorting_spec.rb @@ -8,7 +8,7 @@ RSpec.describe 'Milestones sorting', :js do let(:milestones_for_sort_by) do { 'Due later' => %w[b c a], - 'Name, ascending' => %w[a b c], + 'Name, ascending' => %w[a b c], 'Name, descending' => %w[c b a], 'Start later' => %w[a c b], 'Start soon' => %w[b c a], diff --git a/spec/features/projects/new_project_spec.rb b/spec/features/projects/new_project_spec.rb index f45025d079a..7cf05242a23 100644 --- a/spec/features/projects/new_project_spec.rb +++ b/spec/features/projects/new_project_spec.rb @@ -424,9 +424,10 @@ RSpec.describe 'New project', :js do it 'keeps "Import project" tab open after form validation error' do collision_project = create(:project, name: 'test-name-collision', namespace: user.namespace) - stub_request(:get, "http://foo/bar/info/refs?service=git-upload-pack").to_return({ status: 200, - body: '001e# service=git-upload-pack', - headers: { 'Content-Type': 'application/x-git-upload-pack-advertisement' } }) + stub_request(:get, "http://foo/bar/info/refs?service=git-upload-pack").to_return( + { status: 200, + body: '001e# service=git-upload-pack', + headers: { 'Content-Type': 'application/x-git-upload-pack-advertisement' } }) fill_in 'project_import_url', with: 'http://foo/bar' fill_in 'project_name', with: collision_project.name @@ -465,9 +466,10 @@ RSpec.describe 'New project', :js do end it 'initiates import when valid repo url is provided' do - stub_request(:get, "http://foo/bar/info/refs?service=git-upload-pack").to_return({ status: 200, - body: '001e# service=git-upload-pack', - headers: { 'Content-Type': 'application/x-git-upload-pack-advertisement' } }) + stub_request(:get, "http://foo/bar/info/refs?service=git-upload-pack").to_return( + { status: 200, + body: '001e# service=git-upload-pack', + headers: { 'Content-Type': 'application/x-git-upload-pack-advertisement' } }) fill_in 'project_import_url', with: 'http://foo/bar' diff --git a/spec/features/projects/pipelines/legacy_pipeline_spec.rb b/spec/features/projects/pipelines/legacy_pipeline_spec.rb index 14f60dfe061..f1d16f95a8c 100644 --- a/spec/features/projects/pipelines/legacy_pipeline_spec.rb +++ b/spec/features/projects/pipelines/legacy_pipeline_spec.rb @@ -769,13 +769,17 @@ RSpec.describe 'Pipeline', :js do let(:resource_group) { create(:ci_resource_group, project: project) } let!(:test_job) do - create(:ci_build, :pending, stage: 'test', name: 'test', - stage_idx: 1, pipeline: pipeline, project: project) + create(:ci_build, :pending, stage: 'test', name: 'test', stage_idx: 1, pipeline: pipeline, project: project) end let!(:deploy_job) do - create(:ci_build, :created, stage: 'deploy', name: 'deploy', - stage_idx: 2, pipeline: pipeline, project: project, resource_group: resource_group) + create(:ci_build, :created, + stage: 'deploy', + name: 'deploy', + stage_idx: 2, + pipeline: pipeline, + project: project, + resource_group: resource_group) end describe 'GET /:project/-/pipelines/:id' do @@ -873,8 +877,14 @@ RSpec.describe 'Pipeline', :js do context 'when deploy job is a bridge to trigger a downstream pipeline' do let!(:deploy_job) do - create(:ci_bridge, :created, stage: 'deploy', name: 'deploy', - stage_idx: 2, pipeline: pipeline, project: project, resource_group: resource_group) + create(:ci_bridge, :created, + stage: 'deploy', + name: 'deploy', + stage_idx: 2, + pipeline: pipeline, + project: project, + resource_group: resource_group + ) end it 'shows deploy job as waiting for resource' do @@ -895,8 +905,14 @@ RSpec.describe 'Pipeline', :js do context 'when deploy job is a bridge to trigger a downstream pipeline' do let!(:deploy_job) do - create(:ci_bridge, :created, stage: 'deploy', name: 'deploy', - stage_idx: 2, pipeline: pipeline, project: project, resource_group: resource_group) + create(:ci_bridge, :created, + stage: 'deploy', + name: 'deploy', + stage_idx: 2, + pipeline: pipeline, + project: project, + resource_group: resource_group + ) end it 'shows deploy job as waiting for resource' do diff --git a/spec/features/projects/pipelines/legacy_pipelines_spec.rb b/spec/features/projects/pipelines/legacy_pipelines_spec.rb index eb8f2de3aba..c903fe60fdb 100644 --- a/spec/features/projects/pipelines/legacy_pipelines_spec.rb +++ b/spec/features/projects/pipelines/legacy_pipelines_spec.rb @@ -546,8 +546,8 @@ RSpec.describe 'Pipelines', :js do context 'for a failed pipeline' do let!(:build) do create(:ci_build, :failed, pipeline: pipeline, - stage: 'build', - name: 'build') + stage: 'build', + name: 'build') end it 'displays the failure reason' do diff --git a/spec/features/projects/pipelines/pipeline_spec.rb b/spec/features/projects/pipelines/pipeline_spec.rb index cfdd851cb80..937e807b5f1 100644 --- a/spec/features/projects/pipelines/pipeline_spec.rb +++ b/spec/features/projects/pipelines/pipeline_spec.rb @@ -895,12 +895,12 @@ RSpec.describe 'Pipeline', :js do let!(:test_job) do create(:ci_build, :pending, stage: 'test', name: 'test', - stage_idx: 1, pipeline: pipeline, project: project) + stage_idx: 1, pipeline: pipeline, project: project) end let!(:deploy_job) do create(:ci_build, :created, stage: 'deploy', name: 'deploy', - stage_idx: 2, pipeline: pipeline, project: project, resource_group: resource_group) + stage_idx: 2, pipeline: pipeline, project: project, resource_group: resource_group) end describe 'GET /:project/-/pipelines/:id' do @@ -998,8 +998,14 @@ RSpec.describe 'Pipeline', :js do context 'when deploy job is a bridge to trigger a downstream pipeline' do let!(:deploy_job) do - create(:ci_bridge, :created, stage: 'deploy', name: 'deploy', - stage_idx: 2, pipeline: pipeline, project: project, resource_group: resource_group) + create(:ci_bridge, :created, + stage: 'deploy', + name: 'deploy', + stage_idx: 2, + pipeline: pipeline, + project: project, + resource_group: resource_group + ) end it 'shows deploy job as waiting for resource' do diff --git a/spec/features/projects/pipelines/pipelines_spec.rb b/spec/features/projects/pipelines/pipelines_spec.rb index bf521971ae0..d4f58813534 100644 --- a/spec/features/projects/pipelines/pipelines_spec.rb +++ b/spec/features/projects/pipelines/pipelines_spec.rb @@ -536,8 +536,8 @@ RSpec.describe 'Pipelines', :js do context 'for a failed pipeline' do let!(:build) do create(:ci_build, :failed, pipeline: pipeline, - stage: 'build', - name: 'build') + stage: 'build', + name: 'build') end it 'displays the failure reason' do diff --git a/spec/features/snippets/user_creates_snippet_spec.rb b/spec/features/snippets/user_creates_snippet_spec.rb index 628468a2abe..fd95516090a 100644 --- a/spec/features/snippets/user_creates_snippet_spec.rb +++ b/spec/features/snippets/user_creates_snippet_spec.rb @@ -80,7 +80,7 @@ RSpec.describe 'User creates snippet', :js do context 'when snippets default visibility level is restricted' do before do stub_application_setting(restricted_visibility_levels: [Gitlab::VisibilityLevel::PRIVATE], - default_snippet_visibility: Gitlab::VisibilityLevel::PRIVATE) + default_snippet_visibility: Gitlab::VisibilityLevel::PRIVATE) end it 'creates a snippet using the lowest available visibility level as default' do diff --git a/spec/features/users/email_verification_on_login_spec.rb b/spec/features/users/email_verification_on_login_spec.rb index 55954d121e1..1bfc59c3e73 100644 --- a/spec/features/users/email_verification_on_login_spec.rb +++ b/spec/features/users/email_verification_on_login_spec.rb @@ -349,9 +349,9 @@ RSpec.describe 'Email Verification On Login', :clean_gitlab_redis_rate_limiting expect(Gitlab::AppLogger).to have_received(:info) .exactly(times).times .with(message || hash_including(message: 'Email Verification', - event: event, - username: user.username, - ip: '127.0.0.1', - reason: reason)) + event: event, + username: user.username, + ip: '127.0.0.1', + reason: reason)) end end diff --git a/spec/features/users/login_spec.rb b/spec/features/users/login_spec.rb index b875dbe1340..4c294063a4e 100644 --- a/spec/features/users/login_spec.rb +++ b/spec/features/users/login_spec.rb @@ -862,7 +862,7 @@ RSpec.describe 'Login', :clean_gitlab_redis_sessions do context 'when the user already enabled 2FA' do before do user.update!(otp_required_for_login: true, - otp_secret: User.generate_otp_secret(32)) + otp_secret: User.generate_otp_secret(32)) end it 'asks the user to accept the terms' do diff --git a/spec/lib/gitlab/database/batch_count_spec.rb b/spec/lib/gitlab/database/batch_count_spec.rb index 811d4fad95c..72a4af692b4 100644 --- a/spec/lib/gitlab/database/batch_count_spec.rb +++ b/spec/lib/gitlab/database/batch_count_spec.rb @@ -86,48 +86,48 @@ RSpec.describe Gitlab::Database::BatchCount do query: batch_count_query, message: 'Query has been canceled with message: query timed out' ) - expect(subject.call(model, column, batch_size: batch_size, start: 0)).to eq(-1) + expect(subject.call(model, column, batch_size: batch_size, start: 0)).to eq(fallback) end end end describe '#batch_count' do it 'counts table' do - expect(described_class.batch_count(model)).to eq(5) + expect(described_class.batch_count(model)).to eq(model.count) end it 'counts with :id field' do - expect(described_class.batch_count(model, :id)).to eq(5) + expect(described_class.batch_count(model, :id)).to eq(model.count) end it 'counts with "id" field' do - expect(described_class.batch_count(model, 'id')).to eq(5) + expect(described_class.batch_count(model, 'id')).to eq(model.count) end it 'counts with table.id field' do - expect(described_class.batch_count(model, "#{model.table_name}.id")).to eq(5) + expect(described_class.batch_count(model, "#{model.table_name}.id")).to eq(model.count) end it 'counts with Arel column' do - expect(described_class.batch_count(model, model.arel_table[:id])).to eq(5) + expect(described_class.batch_count(model, model.arel_table[:id])).to eq(model.count) end it 'counts table with batch_size 50K' do - expect(described_class.batch_count(model, batch_size: 50_000)).to eq(5) + expect(described_class.batch_count(model, batch_size: 50_000)).to eq(model.count) end it 'will not count table with a batch size less than allowed' do expect(described_class.batch_count(model, batch_size: small_batch_size)).to eq(fallback) end - it 'counts with a small edge case batch_sizes than result' do + it 'produces the same result with different batch sizes' do stub_const('Gitlab::Database::BatchCounter::MIN_REQUIRED_BATCH_SIZE', 0) - [1, 2, 4, 5, 6].each { |i| expect(described_class.batch_count(model, batch_size: i)).to eq(5) } + [1, 2, 4, 5, 6].each { |i| expect(described_class.batch_count(model, batch_size: i)).to eq(model.count) } end it 'counts with a start and finish' do - expect(described_class.batch_count(model, start: model.minimum(:id), finish: model.maximum(:id))).to eq(5) + expect(described_class.batch_count(model, start: model.minimum(:id), finish: model.maximum(:id))).to eq(model.count) end it 'stops counting when finish value is reached' do @@ -217,6 +217,113 @@ RSpec.describe Gitlab::Database::BatchCount do end end + describe '#batch_count_with_timeout' do + it 'counts table' do + expect(described_class.batch_count_with_timeout(model)).to eq({ status: :completed, count: model.count }) + end + + it 'counts with :id field' do + expect(described_class.batch_count_with_timeout(model, :id)).to eq({ status: :completed, count: model.count }) + end + + it 'counts with "id" field' do + expect(described_class.batch_count_with_timeout(model, 'id')).to eq({ status: :completed, count: model.count }) + end + + it 'counts with table.id field' do + expect(described_class.batch_count_with_timeout(model, "#{model.table_name}.id")).to eq({ status: :completed, count: model.count }) + end + + it 'counts with Arel column' do + expect(described_class.batch_count_with_timeout(model, model.arel_table[:id])).to eq({ status: :completed, count: model.count }) + end + + it 'counts table with batch_size 50K' do + expect(described_class.batch_count_with_timeout(model, batch_size: 50_000)).to eq({ status: :completed, count: model.count }) + end + + it 'will not count table with a batch size less than allowed' do + expect(described_class.batch_count_with_timeout(model, batch_size: small_batch_size)).to eq({ status: :bad_config }) + end + + it 'produces the same result with different batch sizes' do + stub_const('Gitlab::Database::BatchCounter::MIN_REQUIRED_BATCH_SIZE', 0) + + [1, 2, 4, 5, 6].each { |i| expect(described_class.batch_count_with_timeout(model, batch_size: i)).to eq({ status: :completed, count: model.count }) } + end + + it 'counts with a start and finish' do + expect(described_class.batch_count_with_timeout(model, start: model.minimum(:id), finish: model.maximum(:id))).to eq({ status: :completed, count: model.count }) + end + + it 'stops counting when finish value is reached' do + stub_const('Gitlab::Database::BatchCounter::MIN_REQUIRED_BATCH_SIZE', 0) + + expect(described_class.batch_count_with_timeout(model, + start: model.minimum(:id), + finish: model.maximum(:id) - 1, # Do not count the last record + batch_size: model.count - 2 # Ensure there are multiple batches + )).to eq({ status: :completed, count: model.count - 1 }) + end + + it 'returns a partial count when timeout elapses' do + stub_const('Gitlab::Database::BatchCounter::MIN_REQUIRED_BATCH_SIZE', 0) + + expect(::Gitlab::Metrics::System).to receive(:monotonic_time).and_return(1, 10, 300) + + expect( + described_class.batch_count_with_timeout(model, batch_size: 1, timeout: 250.seconds) + ).to eq({ status: :timeout, partial_results: 1, continue_from: model.minimum(:id) + 1 }) + end + + it 'starts counting from a given partial result' do + expect(described_class.batch_count_with_timeout(model, partial_results: 3)).to eq({ status: :completed, count: 3 + model.count }) + end + + it_behaves_like 'when a transaction is open' do + subject { described_class.batch_count_with_timeout(model) } + end + + it_behaves_like 'when batch fetch query is canceled' do + let(:mode) { :itself } + let(:operation) { :count } + let(:operation_args) { nil } + let(:column) { nil } + let(:fallback) { { status: :cancelled } } + + subject { described_class.method(:batch_count_with_timeout) } + end + + context 'disallowed_configurations' do + include_examples 'disallowed configurations', :batch_count do + let(:args) { [Issue] } + let(:default_batch_size) { Gitlab::Database::BatchCounter::DEFAULT_BATCH_SIZE } + end + + it 'raises an error if distinct count is requested' do + expect { described_class.batch_count_with_timeout(model.distinct(column)) }.to raise_error 'Use distinct count for optimized distinct counting' + end + end + + context 'when a relation is grouped' do + let!(:one_more_issue) { create(:issue, author: user, project: model.first.project) } + + before do + stub_const('Gitlab::Database::BatchCounter::MIN_REQUIRED_BATCH_SIZE', 1) + end + + context 'count by default column' do + let(:count) do + described_class.batch_count_with_timeout(model.group(column), batch_size: 2) + end + + it 'counts grouped records' do + expect(count).to eq({ status: :completed, count: { user.id => 4, another_user.id => 2 } }) + end + end + end + end + describe '#batch_distinct_count' do it 'counts with column field' do expect(described_class.batch_distinct_count(model, column)).to eq(2) @@ -242,7 +349,7 @@ RSpec.describe Gitlab::Database::BatchCount do expect(described_class.batch_distinct_count(model, column, batch_size: small_batch_size)).to eq(fallback) end - it 'counts with a small edge case batch_sizes than result' do + it 'produces the same result with different batch sizes' do stub_const('Gitlab::Database::BatchCounter::MIN_REQUIRED_BATCH_SIZE', 0) [1, 2, 4, 5, 6].each { |i| expect(described_class.batch_distinct_count(model, column, batch_size: i)).to eq(2) } diff --git a/spec/workers/analytics/usage_trends/counter_job_worker_spec.rb b/spec/workers/analytics/usage_trends/counter_job_worker_spec.rb index c45ec20fe5a..ee1bbafa9b5 100644 --- a/spec/workers/analytics/usage_trends/counter_job_worker_spec.rb +++ b/spec/workers/analytics/usage_trends/counter_job_worker_spec.rb @@ -48,11 +48,43 @@ RSpec.describe Analytics::UsageTrends::CounterJobWorker do end it 'does not insert anything when BatchCount returns error' do - allow(Gitlab::Database::BatchCount).to receive(:batch_count).and_return(Gitlab::Database::BatchCounter::FALLBACK) + allow(Gitlab::Database::BatchCount).to receive(:batch_count_with_timeout) + .and_return({ status: :canceled }) expect { subject }.not_to change { Analytics::UsageTrends::Measurement.count } end + context 'when the timeout elapses' do + let(:min_id) { 1 } + let(:max_id) { 12345 } + let(:continue_from) { 321 } + let(:partial_results) { 42 } + let(:final_count) { 123 } + + subject { described_class.new.perform(users_measurement_identifier, min_id, max_id, recorded_at) } + + it 'continues counting later when the timeout elapses' do + expect(Gitlab::Database::BatchCount).to receive(:batch_count_with_timeout) + .with(anything, start: min_id, finish: max_id, timeout: 250.seconds, partial_results: nil) + .and_return({ status: :timeout, partial_results: partial_results, continue_from: continue_from }) + + expect(described_class).to receive(:perform_async).with(anything, continue_from, max_id, recorded_at, partial_results) do |*args| + described_class.new.perform(*args) + end + + expect(Gitlab::Database::BatchCount).to receive(:batch_count_with_timeout) + .with(anything, start: continue_from, finish: max_id, timeout: 250.seconds, partial_results: partial_results) + .and_return({ status: :completed, count: final_count }) + + expect { subject }.to change { Analytics::UsageTrends::Measurement.count } + + measurement = Analytics::UsageTrends::Measurement.users.last + expect(measurement.recorded_at).to be_like_time(recorded_at) + expect(measurement.identifier).to eq('users') + expect(measurement.count).to eq(final_count) + end + end + context 'when pipelines_succeeded identifier is passed' do let_it_be(:pipeline) { create(:ci_pipeline, :success) } |