diff options
26 files changed, 349 insertions, 138 deletions
diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml index bd2c68068d4..d3844d01213 100644 --- a/.gitlab/ci/frontend.gitlab-ci.yml +++ b/.gitlab/ci/frontend.gitlab-ci.yml @@ -133,6 +133,7 @@ update-storybook-yarn-cache: paths: - tmp/tests/frontend/ - knapsack/ + - crystalball/ # Builds FOSS, and EE fixtures in the EE project. # Builds FOSS fixtures in the FOSS project. diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION index c44cd17461b..42d8be2ef60 100644 --- a/GITALY_SERVER_VERSION +++ b/GITALY_SERVER_VERSION @@ -1 +1 @@ -06ec7a17f320497d13efdc06f7798b919f45fa9d +8d7e242576249154697fed7876cd9fe2a31ffdc3 diff --git a/app/models/loose_foreign_keys/deleted_record.rb b/app/models/loose_foreign_keys/deleted_record.rb index 01d6d068ca4..c3b3e76f67b 100644 --- a/app/models/loose_foreign_keys/deleted_record.rb +++ b/app/models/loose_foreign_keys/deleted_record.rb @@ -4,29 +4,29 @@ class LooseForeignKeys::DeletedRecord < ApplicationRecord self.primary_key = :id scope :for_table, -> (table) { where(fully_qualified_table_name: table) } - scope :ordered_by_id, -> { order(:id, :primary_key_value) } - # This needs to be parameterized once we start adding partitions - scope :for_partition, -> { where(partition: 1) } + scope :consume_order, -> { order(:partition, :consume_after, :id) } enum status: { pending: 1, processed: 2 }, _prefix: :status def self.load_batch_for_table(table, batch_size) for_table(table) - .for_partition .status_pending - .ordered_by_id + .consume_order .limit(batch_size) .to_a end - def self.mark_records_processed_for_table_between(table, from_record, to_record) - from = from_record.id - to = to_record.id + def self.mark_records_processed(all_records) + # Run a query for each partition to optimize the row lookup by primary key (partition, id) + update_count = 0 - for_table(table) - .for_partition - .status_pending - .where(id: from..to) - .update_all(status: :processed) + all_records.group_by(&:partition).each do |partition, records_within_partition| + update_count += status_pending + .where(partition: partition) + .where(id: records_within_partition.pluck(:id)) + .update_all(status: :processed) + end + + update_count end end diff --git a/app/services/labels/transfer_service.rb b/app/services/labels/transfer_service.rb index a05090d6bfb..19d419609a5 100644 --- a/app/services/labels/transfer_service.rb +++ b/app/services/labels/transfer_service.rb @@ -50,21 +50,32 @@ module Labels # rubocop: disable CodeReuse/ActiveRecord def group_labels_applied_to_issues - @group_labels_applied_to_issues ||= Label.joins(:issues) - .where( - issues: { project_id: project.id }, - labels: { group_id: old_group.self_and_ancestors } - ) + @labels_applied_to_issues ||= if use_optimized_group_labels_query? + Label.joins(:issues) + .joins("INNER JOIN namespaces on namespaces.id = labels.group_id AND namespaces.type = 'Group'" ) + .where(issues: { project_id: project.id }).reorder(nil) + else + Label.joins(:issues).where( + issues: { project_id: project.id }, + labels: { group_id: old_group.self_and_ancestors } + ) + end end # rubocop: enable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord def group_labels_applied_to_merge_requests - @group_labels_applied_to_merge_requests ||= Label.joins(:merge_requests) - .where( - merge_requests: { target_project_id: project.id }, - labels: { group_id: old_group.self_and_ancestors } - ) + @labels_applied_to_mrs ||= if use_optimized_group_labels_query? + Label.joins(:merge_requests) + .joins("INNER JOIN namespaces on namespaces.id = labels.group_id AND namespaces.type = 'Group'" ) + .where(merge_requests: { target_project_id: project.id }).reorder(nil) + else + Label.joins(:merge_requests) + .where( + merge_requests: { target_project_id: project.id }, + labels: { group_id: old_group.self_and_ancestors } + ) + end end # rubocop: enable CodeReuse/ActiveRecord @@ -88,5 +99,9 @@ module Labels .update_all(label_id: new_label_id) end # rubocop: enable CodeReuse/ActiveRecord + + def use_optimized_group_labels_query? + Feature.enabled?(:use_optimized_group_labels_query, project.root_namespace, default_enabled: :yaml) + end end end diff --git a/app/services/loose_foreign_keys/batch_cleaner_service.rb b/app/services/loose_foreign_keys/batch_cleaner_service.rb index caee2dcf753..06c05e8ff54 100644 --- a/app/services/loose_foreign_keys/batch_cleaner_service.rb +++ b/app/services/loose_foreign_keys/batch_cleaner_service.rb @@ -25,8 +25,7 @@ module LooseForeignKeys return if modification_tracker.over_limit? # At this point, all associations are cleaned up, we can update the status of the parent records - update_count = LooseForeignKeys::DeletedRecord - .mark_records_processed_for_table_between(deleted_parent_records.first.fully_qualified_table_name, deleted_parent_records.first, deleted_parent_records.last) + update_count = LooseForeignKeys::DeletedRecord.mark_records_processed(deleted_parent_records) deleted_records_counter.increment({ table: parent_klass.table_name, db_config_name: LooseForeignKeys::DeletedRecord.connection.pool.db_config.name }, update_count) end diff --git a/config/feature_flags/development/use_optimized_group_labels_query.yml b/config/feature_flags/development/use_optimized_group_labels_query.yml new file mode 100644 index 00000000000..37e2525d03e --- /dev/null +++ b/config/feature_flags/development/use_optimized_group_labels_query.yml @@ -0,0 +1,8 @@ +--- +name: use_optimized_group_labels_query +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/73501 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/344957 +milestone: '14.5' +type: development +group: group::workspace +default_enabled: false diff --git a/data/deprecations/disable_strict_host_key_checking.yml b/data/deprecations/disable_strict_host_key_checking.yml index 5a14120ea06..62011795bb9 100644 --- a/data/deprecations/disable_strict_host_key_checking.yml +++ b/data/deprecations/disable_strict_host_key_checking.yml @@ -3,11 +3,10 @@ announcement_date: "2021-11-22" removal_milestone: "15.0" # the milestone when this feature is planned to be removed body: | # Do not modify this line, instead modify the lines below. - In [GitLab 14.3](https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/3074), we added a configuration setting in the GitLab Runner config.toml. This setting, [`[runners.ssh.disable_strict_host_key_checking]`](https://docs.gitlab.com/runner/executors/ssh.html#security), controls whether or not to use strict host key checking with the SSH executor. + In [GitLab 14.3](https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/3074), we added a configuration setting in the GitLab Runner `config.toml` file. This setting, [`[runners.ssh.disable_strict_host_key_checking]`](https://docs.gitlab.com/runner/executors/ssh.html#security), controls whether or not to use strict host key checking with the SSH executor. In GitLab 15.0 and later, the default value for this configuration option will change from `true` to `false`. This means that strict host key checking will be enforced when using the GitLab Runner SSH executor. stage: Verify tiers: [Core, Premium, Ultimate] issue_url: https://gitlab.com/gitlab-org/gitlab-runner/-/issues/28192 documentation_url: https://docs.gitlab.com/runner/executors/ssh.html#security - diff --git a/db/migrate/20211109100050_add_consume_after_to_loose_fk_deleted_records.rb b/db/migrate/20211109100050_add_consume_after_to_loose_fk_deleted_records.rb new file mode 100644 index 00000000000..6b8e8c0d4f3 --- /dev/null +++ b/db/migrate/20211109100050_add_consume_after_to_loose_fk_deleted_records.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class AddConsumeAfterToLooseFkDeletedRecords < Gitlab::Database::Migration[1.0] + enable_lock_retries! + + def up + add_column :loose_foreign_keys_deleted_records, :consume_after, :datetime_with_timezone, default: -> { 'NOW()' } + end + + def down + remove_column :loose_foreign_keys_deleted_records, :consume_after + end +end diff --git a/db/migrate/20211109101010_support_partition_query_in_loose_fk_table.rb b/db/migrate/20211109101010_support_partition_query_in_loose_fk_table.rb new file mode 100644 index 00000000000..fccb1fc5cac --- /dev/null +++ b/db/migrate/20211109101010_support_partition_query_in_loose_fk_table.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class SupportPartitionQueryInLooseFkTable < Gitlab::Database::Migration[1.0] + include Gitlab::Database::PartitioningMigrationHelpers + + disable_ddl_transaction! + + INDEX_NAME = 'index_loose_foreign_keys_deleted_records_for_partitioned_query' + + def up + add_concurrent_partitioned_index :loose_foreign_keys_deleted_records, + %I[partition fully_qualified_table_name consume_after id], + where: 'status = 1', + name: INDEX_NAME + end + + def down + remove_concurrent_partitioned_index_by_name :loose_foreign_keys_deleted_records, INDEX_NAME + end +end diff --git a/db/post_migrate/20211029102822_add_open_source_plan.rb b/db/post_migrate/20211029102822_add_open_source_plan.rb new file mode 100644 index 00000000000..00266640f03 --- /dev/null +++ b/db/post_migrate/20211029102822_add_open_source_plan.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +class AddOpenSourcePlan < Gitlab::Database::Migration[1.0] + class Plan < ActiveRecord::Base + self.inheritance_column = :_type_disabled + + has_one :limits, class_name: 'PlanLimits' + + def actual_limits + self.limits || self.build_limits + end + end + + class PlanLimits < ActiveRecord::Base + self.inheritance_column = :_type_disabled + + belongs_to :plan + end + + def create_plan_limits(plan_limit_name, plan) + plan_limit = Plan.find_or_initialize_by(name: plan_limit_name).actual_limits.dup + plan_limit.plan = plan + plan_limit.save! + end + + def up + return unless Gitlab.dev_env_or_com? + + opensource = Plan.create!(name: 'opensource', title: 'Open Source Program') + + create_plan_limits('ultimate', opensource) + end + + def down + return unless Gitlab.dev_env_or_com? + + Plan.where(name: 'opensource').delete_all + end +end diff --git a/db/post_migrate/20211109112454_drop_old_loose_fk_deleted_records_index.rb b/db/post_migrate/20211109112454_drop_old_loose_fk_deleted_records_index.rb new file mode 100644 index 00000000000..ef5a70713d0 --- /dev/null +++ b/db/post_migrate/20211109112454_drop_old_loose_fk_deleted_records_index.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class DropOldLooseFkDeletedRecordsIndex < Gitlab::Database::Migration[1.0] + include Gitlab::Database::PartitioningMigrationHelpers + + disable_ddl_transaction! + + INDEX_NAME = 'index_loose_foreign_keys_deleted_records_for_loading_records' + + def up + remove_concurrent_partitioned_index_by_name :loose_foreign_keys_deleted_records, INDEX_NAME + end + + def down + add_concurrent_partitioned_index :loose_foreign_keys_deleted_records, + %I[fully_qualified_table_name id primary_key_value partition], + where: 'status = 1', + name: INDEX_NAME + end +end diff --git a/db/schema_migrations/20211029102822 b/db/schema_migrations/20211029102822 new file mode 100644 index 00000000000..72198f302b3 --- /dev/null +++ b/db/schema_migrations/20211029102822 @@ -0,0 +1 @@ +e1f9d87287048010e9816fd5b4a9a2d30b64d2ad150226852f6679b950031914
\ No newline at end of file diff --git a/db/schema_migrations/20211109100050 b/db/schema_migrations/20211109100050 new file mode 100644 index 00000000000..94f9612277f --- /dev/null +++ b/db/schema_migrations/20211109100050 @@ -0,0 +1 @@ +86aa6ad1759a00c2cc5cb6dc2e381aead2910a24f0e37933a5e72af56d08101a
\ No newline at end of file diff --git a/db/schema_migrations/20211109101010 b/db/schema_migrations/20211109101010 new file mode 100644 index 00000000000..ea24f5e7e37 --- /dev/null +++ b/db/schema_migrations/20211109101010 @@ -0,0 +1 @@ +30eb98b8fdb24bc5de357b0ec14a6b92d520db025c82bd7b9448f71542c7d7e3
\ No newline at end of file diff --git a/db/schema_migrations/20211109112454 b/db/schema_migrations/20211109112454 new file mode 100644 index 00000000000..6fdb1e344b9 --- /dev/null +++ b/db/schema_migrations/20211109112454 @@ -0,0 +1 @@ +1bc48cdae55eea5a5963edd3a138d7d6859afa6caafe0b793c553fdfabe9f488
\ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 4279fbfd448..4fd96e5d3c5 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -1021,6 +1021,7 @@ CREATE TABLE loose_foreign_keys_deleted_records ( status smallint DEFAULT 1 NOT NULL, created_at timestamp with time zone DEFAULT now() NOT NULL, fully_qualified_table_name text NOT NULL, + consume_after timestamp with time zone DEFAULT now(), CONSTRAINT check_1a541f3235 CHECK ((char_length(fully_qualified_table_name) <= 150)) ) PARTITION BY LIST (partition); @@ -1041,6 +1042,7 @@ CREATE TABLE gitlab_partitions_static.loose_foreign_keys_deleted_records_1 ( status smallint DEFAULT 1 NOT NULL, created_at timestamp with time zone DEFAULT now() NOT NULL, fully_qualified_table_name text NOT NULL, + consume_after timestamp with time zone DEFAULT now(), CONSTRAINT check_1a541f3235 CHECK ((char_length(fully_qualified_table_name) <= 150)) ); ALTER TABLE ONLY loose_foreign_keys_deleted_records ATTACH PARTITION gitlab_partitions_static.loose_foreign_keys_deleted_records_1 FOR VALUES IN ('1'); @@ -24073,6 +24075,10 @@ CREATE INDEX index_merge_request_stage_events_project_duration ON ONLY analytics CREATE INDEX index_006f943df6 ON gitlab_partitions_static.analytics_cycle_analytics_merge_request_stage_events_16 USING btree (stage_event_hash_id, project_id, end_event_timestamp, merge_request_id, start_event_timestamp) WHERE (end_event_timestamp IS NOT NULL); +CREATE INDEX index_loose_foreign_keys_deleted_records_for_partitioned_query ON ONLY loose_foreign_keys_deleted_records USING btree (partition, fully_qualified_table_name, consume_after, id) WHERE (status = 1); + +CREATE INDEX index_01e3390fac ON gitlab_partitions_static.loose_foreign_keys_deleted_records_1 USING btree (partition, fully_qualified_table_name, consume_after, id) WHERE (status = 1); + CREATE INDEX index_02749b504c ON gitlab_partitions_static.analytics_cycle_analytics_merge_request_stage_events_11 USING btree (stage_event_hash_id, project_id, end_event_timestamp, merge_request_id, start_event_timestamp) WHERE (end_event_timestamp IS NOT NULL); CREATE INDEX index_merge_request_stage_events_group_duration ON ONLY analytics_cycle_analytics_merge_request_stage_events USING btree (stage_event_hash_id, group_id, end_event_timestamp, merge_request_id, start_event_timestamp) WHERE (end_event_timestamp IS NOT NULL); @@ -24391,10 +24397,6 @@ CREATE INDEX index_8a0fc3de4f ON gitlab_partitions_static.analytics_cycle_analyt CREATE INDEX index_8b9f9a19a4 ON gitlab_partitions_static.analytics_cycle_analytics_merge_request_stage_events_18 USING btree (stage_event_hash_id, group_id, end_event_timestamp, merge_request_id, start_event_timestamp) WHERE (end_event_timestamp IS NOT NULL); -CREATE INDEX index_loose_foreign_keys_deleted_records_for_loading_records ON ONLY loose_foreign_keys_deleted_records USING btree (fully_qualified_table_name, id, primary_key_value, partition) WHERE (status = 1); - -CREATE INDEX index_8be8640437 ON gitlab_partitions_static.loose_foreign_keys_deleted_records_1 USING btree (fully_qualified_table_name, id, primary_key_value, partition) WHERE (status = 1); - CREATE INDEX index_8fb48e72ce ON gitlab_partitions_static.analytics_cycle_analytics_issue_stage_events_26 USING btree (stage_event_hash_id, group_id, end_event_timestamp, issue_id, start_event_timestamp) WHERE (end_event_timestamp IS NOT NULL); CREATE INDEX index_9201b952a0 ON gitlab_partitions_static.analytics_cycle_analytics_issue_stage_events_13 USING btree (stage_event_hash_id, group_id, end_event_timestamp, issue_id, start_event_timestamp) WHERE (end_event_timestamp IS NOT NULL); @@ -27911,6 +27913,8 @@ ALTER INDEX index_issue_stage_events_project_duration ATTACH PARTITION gitlab_pa ALTER INDEX index_merge_request_stage_events_project_duration ATTACH PARTITION gitlab_partitions_static.index_006f943df6; +ALTER INDEX index_loose_foreign_keys_deleted_records_for_partitioned_query ATTACH PARTITION gitlab_partitions_static.index_01e3390fac; + ALTER INDEX index_merge_request_stage_events_project_duration ATTACH PARTITION gitlab_partitions_static.index_02749b504c; ALTER INDEX index_merge_request_stage_events_group_duration ATTACH PARTITION gitlab_partitions_static.index_0287f5ba09; @@ -28217,8 +28221,6 @@ ALTER INDEX index_issue_stage_events_project_in_progress_duration ATTACH PARTITI ALTER INDEX index_merge_request_stage_events_group_duration ATTACH PARTITION gitlab_partitions_static.index_8b9f9a19a4; -ALTER INDEX index_loose_foreign_keys_deleted_records_for_loading_records ATTACH PARTITION gitlab_partitions_static.index_8be8640437; - ALTER INDEX index_issue_stage_events_group_duration ATTACH PARTITION gitlab_partitions_static.index_8fb48e72ce; ALTER INDEX index_issue_stage_events_group_duration ATTACH PARTITION gitlab_partitions_static.index_9201b952a0; diff --git a/doc/update/deprecations.md b/doc/update/deprecations.md index 2e209b2787b..37c925b4e9d 100644 --- a/doc/update/deprecations.md +++ b/doc/update/deprecations.md @@ -66,6 +66,14 @@ We decided to remove the GitLab Serverless features as they never really resonat Announced: 2021-09-22 +### Known host required for GitLab Runner SSH executor + +In [GitLab 14.3](https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/3074), we added a configuration setting in the GitLab Runner `config.toml` file. This setting, [`[runners.ssh.disable_strict_host_key_checking]`](https://docs.gitlab.com/runner/executors/ssh.html#security), controls whether or not to use strict host key checking with the SSH executor. + +In GitLab 15.0 and later, the default value for this configuration option will change from `true` to `false`. This means that strict host key checking will be enforced when using the GitLab Runner SSH executor. + +Announced: 2021-11-22 + ### Legacy database configuration The syntax of [GitLabs database](https://docs.gitlab.com/omnibus/settings/database.html) diff --git a/doc/user/application_security/secret_detection/index.md b/doc/user/application_security/secret_detection/index.md index dc6fe2e92fc..140f660d729 100644 --- a/doc/user/application_security/secret_detection/index.md +++ b/doc/user/application_security/secret_detection/index.md @@ -202,8 +202,9 @@ Secret Detection can be customized by defining available CI/CD variables: | CI/CD variable | Default value | Description | |-----------------------------------|---------------|-------------| -| `SECRET_DETECTION_COMMIT_FROM` | - | The commit a Gitleaks scan starts at. | -| `SECRET_DETECTION_COMMIT_TO` | - | The commit a Gitleaks scan ends at. | +| `SECRET_DETECTION_COMMIT_FROM` | - | The commit a Gitleaks scan starts at. [Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/243564) in GitLab 13.5. Replaced with `SECRET_DETECTION_COMMITS`. | +| `SECRET_DETECTION_COMMIT_TO` | - | The commit a Gitleaks scan ends at. [Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/243564) in GitLab 13.5. Replaced with `SECRET_DETECTION_COMMITS`. | +| `SECRET_DETECTION_COMMITS` | - | The list of commits that Gitleaks should scan. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/243564) in GitLab 13.5. | | `SECRET_DETECTION_EXCLUDED_PATHS` | "" | Exclude vulnerabilities from output based on the paths. This is a comma-separated list of patterns. Patterns can be globs, or file or folder paths (for example, `doc,spec` ). Parent directories also match patterns. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/225273) in GitLab 13.3. | | `SECRET_DETECTION_HISTORIC_SCAN` | false | Flag to enable a historic Gitleaks scan. | diff --git a/lib/gitlab/ci/parsers/security/common.rb b/lib/gitlab/ci/parsers/security/common.rb index 1cf4f252ab9..2086ac5668c 100644 --- a/lib/gitlab/ci/parsers/security/common.rb +++ b/lib/gitlab/ci/parsers/security/common.rb @@ -33,8 +33,7 @@ module Gitlab report_data rescue JSON::ParserError raise SecurityReportParserError, 'JSON parsing failed' - rescue StandardError => e - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) + rescue StandardError raise SecurityReportParserError, "#{report.type} security report parsing failed" end diff --git a/lib/gitlab/database/reindexing/index_selection.rb b/lib/gitlab/database/reindexing/index_selection.rb index 2186384e7d7..2d384f2f9e2 100644 --- a/lib/gitlab/database/reindexing/index_selection.rb +++ b/lib/gitlab/database/reindexing/index_selection.rb @@ -9,8 +9,8 @@ module Gitlab # Only reindex indexes with a relative bloat level (bloat estimate / size) higher than this MINIMUM_RELATIVE_BLOAT = 0.2 - # Only consider indexes with a total ondisk size in this range (before reindexing) - INDEX_SIZE_RANGE = (1.gigabyte..100.gigabyte).freeze + # Only consider indexes beyond this size (before reindexing) + INDEX_SIZE_MINIMUM = 1.gigabyte delegate :each, to: :indexes @@ -32,7 +32,7 @@ module Gitlab @indexes ||= candidates .not_recently_reindexed - .where(ondisk_size_bytes: INDEX_SIZE_RANGE) + .where('ondisk_size_bytes >= ?', INDEX_SIZE_MINIMUM) .sort_by(&:relative_bloat_level) # forced N+1 .reverse .select { |candidate| candidate.relative_bloat_level >= MINIMUM_RELATIVE_BLOAT } diff --git a/lib/gitlab/database/reindexing/reindex_concurrently.rb b/lib/gitlab/database/reindexing/reindex_concurrently.rb index 1a2e8bbcb2d..152935bd734 100644 --- a/lib/gitlab/database/reindexing/reindex_concurrently.rb +++ b/lib/gitlab/database/reindexing/reindex_concurrently.rb @@ -8,7 +8,7 @@ module Gitlab ReindexError = Class.new(StandardError) TEMPORARY_INDEX_PATTERN = '\_ccnew[0-9]*' - STATEMENT_TIMEOUT = 9.hours + STATEMENT_TIMEOUT = 24.hours PG_MAX_INDEX_NAME_LENGTH = 63 attr_reader :index, :logger diff --git a/spec/lib/gitlab/database/reindexing/index_selection_spec.rb b/spec/lib/gitlab/database/reindexing/index_selection_spec.rb index ee3f2b1b415..2ae9037959d 100644 --- a/spec/lib/gitlab/database/reindexing/index_selection_spec.rb +++ b/spec/lib/gitlab/database/reindexing/index_selection_spec.rb @@ -46,14 +46,14 @@ RSpec.describe Gitlab::Database::Reindexing::IndexSelection do expect(subject).not_to include(excluded.index) end - it 'excludes indexes larger than 100 GB ondisk size' do - excluded = create( + it 'includes indexes larger than 100 GB ondisk size' do + included = create( :postgres_index_bloat_estimate, index: create(:postgres_index, ondisk_size_bytes: 101.gigabytes), bloat_size_bytes: 25.gigabyte ) - expect(subject).not_to include(excluded.index) + expect(subject).to include(included.index) end context 'with time frozen' do diff --git a/spec/lib/gitlab/database/reindexing/reindex_concurrently_spec.rb b/spec/lib/gitlab/database/reindexing/reindex_concurrently_spec.rb index 6f87475fc94..db267ff4f14 100644 --- a/spec/lib/gitlab/database/reindexing/reindex_concurrently_spec.rb +++ b/spec/lib/gitlab/database/reindexing/reindex_concurrently_spec.rb @@ -62,7 +62,7 @@ RSpec.describe Gitlab::Database::Reindexing::ReindexConcurrently, '#perform' do it 'recreates the index using REINDEX with a long statement timeout' do expect_to_execute_in_order( - "SET statement_timeout TO '32400s'", + "SET statement_timeout TO '86400s'", "REINDEX INDEX CONCURRENTLY \"public\".\"#{index.name}\"", "RESET statement_timeout" ) @@ -84,7 +84,7 @@ RSpec.describe Gitlab::Database::Reindexing::ReindexConcurrently, '#perform' do it 'drops the dangling indexes while controlling lock_timeout' do expect_to_execute_in_order( # Regular index rebuild - "SET statement_timeout TO '32400s'", + "SET statement_timeout TO '86400s'", "REINDEX INDEX CONCURRENTLY \"public\".\"#{index_name}\"", "RESET statement_timeout", # Drop _ccnew index diff --git a/spec/migrations/add_open_source_plan_spec.rb b/spec/migrations/add_open_source_plan_spec.rb new file mode 100644 index 00000000000..04b26662f82 --- /dev/null +++ b/spec/migrations/add_open_source_plan_spec.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +require 'spec_helper' + +require_migration! + +RSpec.describe AddOpenSourcePlan, :migration do + describe '#up' do + before do + allow(Gitlab).to receive(:dev_env_or_com?).and_return true + end + + it 'creates 1 entry within the plans table' do + expect { migrate! }.to change { AddOpenSourcePlan::Plan.count }.by 1 + expect(AddOpenSourcePlan::Plan.last.name).to eql('opensource') + end + + it 'creates 1 entry for plan limits' do + expect { migrate! }.to change { AddOpenSourcePlan::PlanLimits.count }.by 1 + end + + context 'when the plan limits for gold and silver exists' do + before do + table(:plans).create!(id: 1, name: 'ultimate', title: 'Ultimate') + table(:plan_limits).create!(id: 1, plan_id: 1, storage_size_limit: 2000) + end + + it 'duplicates the gold and silvers plan limits entries' do + migrate! + + opensource_limits = AddOpenSourcePlan::Plan.find_by(name: 'opensource').limits + expect(opensource_limits.storage_size_limit).to be 2000 + end + end + + context 'when the instance is not SaaS' do + before do + allow(Gitlab).to receive(:dev_env_or_com?).and_return false + end + + it 'does not create plans and plan limits and returns' do + expect { migrate! }.not_to change { AddOpenSourcePlan::Plan.count } + end + end + end + + describe '#down' do + before do + table(:plans).create!(id: 3, name: 'other') + table(:plan_limits).create!(plan_id: 3) + end + + context 'when the instance is SaaS' do + before do + allow(Gitlab).to receive(:dev_env_or_com?).and_return true + end + + it 'removes the newly added opensource entry' do + migrate! + + expect { described_class.new.down }.to change { AddOpenSourcePlan::Plan.count }.by(-1) + expect(AddOpenSourcePlan::Plan.find_by(name: 'opensource')).to be_nil + + other_plan = AddOpenSourcePlan::Plan.find_by(name: 'other') + expect(other_plan).to be_persisted + expect(AddOpenSourcePlan::PlanLimits.count).to eq(1) + expect(AddOpenSourcePlan::PlanLimits.first.plan_id).to eq(other_plan.id) + end + end + + context 'when the instance is not SaaS' do + before do + allow(Gitlab).to receive(:dev_env_or_com?).and_return false + table(:plans).create!(id: 1, name: 'opensource', title: 'Open Source Program') + table(:plan_limits).create!(id: 1, plan_id: 1) + end + + it 'does not delete plans and plan limits and returns' do + migrate! + + expect { described_class.new.down }.not_to change { AddOpenSourcePlan::Plan.count } + expect(AddOpenSourcePlan::PlanLimits.count).to eq(2) + end + end + end +end diff --git a/spec/models/loose_foreign_keys/deleted_record_spec.rb b/spec/models/loose_foreign_keys/deleted_record_spec.rb index 8494cd91c6e..cd5068bdb52 100644 --- a/spec/models/loose_foreign_keys/deleted_record_spec.rb +++ b/spec/models/loose_foreign_keys/deleted_record_spec.rb @@ -24,26 +24,9 @@ RSpec.describe LooseForeignKeys::DeletedRecord, type: :model do end end - describe '.mark_records_processed_for_table_between' do - it 'marks processed exactly one record' do - described_class.mark_records_processed_for_table_between(table, deleted_record_2, deleted_record_2) - - expect(described_class.status_pending.count).to eq(3) - expect(described_class.status_processed.count).to eq(1) - - processed_record = described_class.status_processed.first - expect(processed_record).to eq(deleted_record_2) - end - - it 'deletes two records' do - described_class.mark_records_processed_for_table_between(table, deleted_record_2, deleted_record_4) - - expect(described_class.status_pending.count).to eq(2) - expect(described_class.status_processed.count).to eq(2) - end - - it 'deletes all records' do - described_class.mark_records_processed_for_table_between(table, deleted_record_1, deleted_record_4) + describe '.mark_records_processed' do + it 'updates all records' do + described_class.mark_records_processed([deleted_record_1, deleted_record_2, deleted_record_4]) expect(described_class.status_pending.count).to eq(1) expect(described_class.status_processed.count).to eq(3) diff --git a/spec/services/labels/transfer_service_spec.rb b/spec/services/labels/transfer_service_spec.rb index 18fd401f383..05190accb33 100644 --- a/spec/services/labels/transfer_service_spec.rb +++ b/spec/services/labels/transfer_service_spec.rb @@ -3,107 +3,121 @@ require 'spec_helper' RSpec.describe Labels::TransferService do - describe '#execute' do - let_it_be(:user) { create(:user) } + shared_examples 'transfer labels' do + describe '#execute' do + let_it_be(:user) { create(:user) } - let_it_be(:old_group_ancestor) { create(:group) } - let_it_be(:old_group) { create(:group, parent: old_group_ancestor) } + let_it_be(:old_group_ancestor) { create(:group) } + let_it_be(:old_group) { create(:group, parent: old_group_ancestor) } - let_it_be(:new_group) { create(:group) } + let_it_be(:new_group) { create(:group) } - let_it_be(:project) { create(:project, :repository, group: new_group) } + let_it_be(:project) { create(:project, :repository, group: new_group) } - subject(:service) { described_class.new(user, old_group, project) } + subject(:service) { described_class.new(user, old_group, project) } - before do - old_group_ancestor.add_developer(user) - new_group.add_developer(user) - end + before do + old_group_ancestor.add_developer(user) + new_group.add_developer(user) + end - it 'recreates missing group labels at project level and assigns them to the issuables' do - old_group_label_1 = create(:group_label, group: old_group) - old_group_label_2 = create(:group_label, group: old_group) + it 'recreates missing group labels at project level and assigns them to the issuables' do + old_group_label_1 = create(:group_label, group: old_group) + old_group_label_2 = create(:group_label, group: old_group) - labeled_issue = create(:labeled_issue, project: project, labels: [old_group_label_1]) - labeled_merge_request = create(:labeled_merge_request, source_project: project, labels: [old_group_label_2]) + labeled_issue = create(:labeled_issue, project: project, labels: [old_group_label_1]) + labeled_merge_request = create(:labeled_merge_request, source_project: project, labels: [old_group_label_2]) - expect { service.execute }.to change(project.labels, :count).by(2) - expect(labeled_issue.reload.labels).to contain_exactly(project.labels.find_by_title(old_group_label_1.title)) - expect(labeled_merge_request.reload.labels).to contain_exactly(project.labels.find_by_title(old_group_label_2.title)) - end + expect { service.execute }.to change(project.labels, :count).by(2) + expect(labeled_issue.reload.labels).to contain_exactly(project.labels.find_by_title(old_group_label_1.title)) + expect(labeled_merge_request.reload.labels).to contain_exactly(project.labels.find_by_title(old_group_label_2.title)) + end - it 'recreates missing ancestor group labels at project level and assigns them to the issuables' do - old_group_ancestor_label_1 = create(:group_label, group: old_group_ancestor) - old_group_ancestor_label_2 = create(:group_label, group: old_group_ancestor) + it 'recreates missing ancestor group labels at project level and assigns them to the issuables' do + old_group_ancestor_label_1 = create(:group_label, group: old_group_ancestor) + old_group_ancestor_label_2 = create(:group_label, group: old_group_ancestor) - labeled_issue = create(:labeled_issue, project: project, labels: [old_group_ancestor_label_1]) - labeled_merge_request = create(:labeled_merge_request, source_project: project, labels: [old_group_ancestor_label_2]) + labeled_issue = create(:labeled_issue, project: project, labels: [old_group_ancestor_label_1]) + labeled_merge_request = create(:labeled_merge_request, source_project: project, labels: [old_group_ancestor_label_2]) - expect { service.execute }.to change(project.labels, :count).by(2) - expect(labeled_issue.reload.labels).to contain_exactly(project.labels.find_by_title(old_group_ancestor_label_1.title)) - expect(labeled_merge_request.reload.labels).to contain_exactly(project.labels.find_by_title(old_group_ancestor_label_2.title)) - end + expect { service.execute }.to change(project.labels, :count).by(2) + expect(labeled_issue.reload.labels).to contain_exactly(project.labels.find_by_title(old_group_ancestor_label_1.title)) + expect(labeled_merge_request.reload.labels).to contain_exactly(project.labels.find_by_title(old_group_ancestor_label_2.title)) + end - it 'recreates label priorities related to the missing group labels' do - old_group_label = create(:group_label, group: old_group) - create(:labeled_issue, project: project, labels: [old_group_label]) - create(:label_priority, project: project, label: old_group_label, priority: 1) + it 'recreates label priorities related to the missing group labels' do + old_group_label = create(:group_label, group: old_group) + create(:labeled_issue, project: project, labels: [old_group_label]) + create(:label_priority, project: project, label: old_group_label, priority: 1) - service.execute + service.execute - new_project_label = project.labels.find_by(title: old_group_label.title) - expect(new_project_label.id).not_to eq old_group_label.id - expect(new_project_label.priorities).not_to be_empty - end + new_project_label = project.labels.find_by(title: old_group_label.title) + expect(new_project_label.id).not_to eq old_group_label.id + expect(new_project_label.priorities).not_to be_empty + end - it 'does not recreate missing group labels that are not applied to issues or merge requests' do - old_group_label = create(:group_label, group: old_group) + it 'does not recreate missing group labels that are not applied to issues or merge requests' do + old_group_label = create(:group_label, group: old_group) - service.execute + service.execute - expect(project.labels.where(title: old_group_label.title)).to be_empty - end + expect(project.labels.where(title: old_group_label.title)).to be_empty + end - it 'does not recreate missing group labels that already exist in the project group' do - old_group_label = create(:group_label, group: old_group) - labeled_issue = create(:labeled_issue, project: project, labels: [old_group_label]) + it 'does not recreate missing group labels that already exist in the project group' do + old_group_label = create(:group_label, group: old_group) + labeled_issue = create(:labeled_issue, project: project, labels: [old_group_label]) - new_group_label = create(:group_label, group: new_group, title: old_group_label.title) + new_group_label = create(:group_label, group: new_group, title: old_group_label.title) - service.execute + service.execute - expect(project.labels.where(title: old_group_label.title)).to be_empty - expect(labeled_issue.reload.labels).to contain_exactly(new_group_label) - end + expect(project.labels.where(title: old_group_label.title)).to be_empty + expect(labeled_issue.reload.labels).to contain_exactly(new_group_label) + end - it 'updates only label links in the given project' do - old_group_label = create(:group_label, group: old_group) - other_project = create(:project, group: old_group) + it 'updates only label links in the given project' do + old_group_label = create(:group_label, group: old_group) + other_project = create(:project, group: old_group) - labeled_issue = create(:labeled_issue, project: project, labels: [old_group_label]) - other_project_labeled_issue = create(:labeled_issue, project: other_project, labels: [old_group_label]) + labeled_issue = create(:labeled_issue, project: project, labels: [old_group_label]) + other_project_labeled_issue = create(:labeled_issue, project: other_project, labels: [old_group_label]) - service.execute + service.execute - expect(labeled_issue.reload.labels).not_to include(old_group_label) - expect(other_project_labeled_issue.reload.labels).to contain_exactly(old_group_label) - end + expect(labeled_issue.reload.labels).not_to include(old_group_label) + expect(other_project_labeled_issue.reload.labels).to contain_exactly(old_group_label) + end - context 'when moving within the same ancestor group' do - let(:other_subgroup) { create(:group, parent: old_group_ancestor) } - let(:project) { create(:project, :repository, group: other_subgroup) } + context 'when moving within the same ancestor group' do + let(:other_subgroup) { create(:group, parent: old_group_ancestor) } + let(:project) { create(:project, :repository, group: other_subgroup) } - it 'does not recreate ancestor group labels' do - old_group_ancestor_label_1 = create(:group_label, group: old_group_ancestor) - old_group_ancestor_label_2 = create(:group_label, group: old_group_ancestor) + it 'does not recreate ancestor group labels' do + old_group_ancestor_label_1 = create(:group_label, group: old_group_ancestor) + old_group_ancestor_label_2 = create(:group_label, group: old_group_ancestor) - labeled_issue = create(:labeled_issue, project: project, labels: [old_group_ancestor_label_1]) - labeled_merge_request = create(:labeled_merge_request, source_project: project, labels: [old_group_ancestor_label_2]) + labeled_issue = create(:labeled_issue, project: project, labels: [old_group_ancestor_label_1]) + labeled_merge_request = create(:labeled_merge_request, source_project: project, labels: [old_group_ancestor_label_2]) - expect { service.execute }.not_to change(project.labels, :count) - expect(labeled_issue.reload.labels).to contain_exactly(old_group_ancestor_label_1) - expect(labeled_merge_request.reload.labels).to contain_exactly(old_group_ancestor_label_2) + expect { service.execute }.not_to change(project.labels, :count) + expect(labeled_issue.reload.labels).to contain_exactly(old_group_ancestor_label_1) + expect(labeled_merge_request.reload.labels).to contain_exactly(old_group_ancestor_label_2) + end end end end + + context 'with use_optimized_group_labels_query FF on' do + it_behaves_like 'transfer labels' + end + + context 'with use_optimized_group_labels_query FF off' do + before do + stub_feature_flags(use_optimized_group_labels_query: false) + end + + it_behaves_like 'transfer labels' + end end |