diff options
Diffstat (limited to 'spec/lib/gitlab')
261 files changed, 5469 insertions, 6453 deletions
diff --git a/spec/lib/gitlab/analytics/cycle_analytics/request_params_spec.rb b/spec/lib/gitlab/analytics/cycle_analytics/request_params_spec.rb index 3c171d684d6..9b362debb10 100644 --- a/spec/lib/gitlab/analytics/cycle_analytics/request_params_spec.rb +++ b/spec/lib/gitlab/analytics/cycle_analytics/request_params_spec.rb @@ -17,10 +17,24 @@ RSpec.describe Gitlab::Analytics::CycleAnalytics::RequestParams, feature_categor expect(attributes).to match(hash_including({ namespace: { name: project.name, - full_path: project.full_path + full_path: project.full_path, + type: "Project" } })) end + + context 'with a subgroup project' do + let_it_be(:sub_group) { create(:group, parent: root_group) } + let_it_be_with_refind(:subgroup_project) { create(:project, group: sub_group) } + let(:namespace) { subgroup_project.project_namespace } + + it 'includes the correct group_path' do + expect(attributes).to match(hash_including({ + group_path: "groups/#{subgroup_project.namespace.full_path}", + full_path: subgroup_project.full_path + })) + end + end end end end diff --git a/spec/lib/gitlab/app_logger_spec.rb b/spec/lib/gitlab/app_logger_spec.rb index e3415f4ad8c..149c3d1f19f 100644 --- a/spec/lib/gitlab/app_logger_spec.rb +++ b/spec/lib/gitlab/app_logger_spec.rb @@ -2,31 +2,12 @@ require 'spec_helper' -RSpec.describe Gitlab::AppLogger do +RSpec.describe Gitlab::AppLogger, feature_category: :shared do subject { described_class } - context 'when UNSTRUCTURED_RAILS_LOG is enabled' do - before do - stub_env('UNSTRUCTURED_RAILS_LOG', 'true') - end + specify { expect(described_class.primary_logger).to be Gitlab::AppJsonLogger } - it 'builds two Logger instances' do - expect(Gitlab::Logger).to receive(:new).and_call_original - expect(Gitlab::JsonLogger).to receive(:new).and_call_original - - subject.info('Hello World!') - end - - it 'logs info to AppLogger and AppJsonLogger' do - expect_any_instance_of(Gitlab::AppTextLogger).to receive(:info).and_call_original - expect_any_instance_of(Gitlab::AppJsonLogger).to receive(:info).and_call_original - - subject.info('Hello World!') - end - end - - it 'logs info to only the AppJsonLogger when unstructured logs are disabled' do - expect_any_instance_of(Gitlab::AppTextLogger).not_to receive(:info).and_call_original + it 'logs to AppJsonLogger' do expect_any_instance_of(Gitlab::AppJsonLogger).to receive(:info).and_call_original subject.info('Hello World!') diff --git a/spec/lib/gitlab/auth/o_auth/auth_hash_spec.rb b/spec/lib/gitlab/auth/o_auth/auth_hash_spec.rb index c94f962ee93..8c50b2acac6 100644 --- a/spec/lib/gitlab/auth/o_auth/auth_hash_spec.rb +++ b/spec/lib/gitlab/auth/o_auth/auth_hash_spec.rb @@ -2,14 +2,19 @@ require 'spec_helper' -RSpec.describe Gitlab::Auth::OAuth::AuthHash do +RSpec.describe Gitlab::Auth::OAuth::AuthHash, feature_category: :user_management do let(:provider) { 'ldap' } let(:auth_hash) do described_class.new( OmniAuth::AuthHash.new( provider: provider, uid: uid_ascii, - info: info_hash + info: info_hash, + extra: { + raw_info: { + 'https://example.com/claims/username': username_claim_utf8 + } + } ) ) end @@ -24,6 +29,7 @@ RSpec.describe Gitlab::Auth::OAuth::AuthHash do let(:first_name_raw) { +'Onur' } let(:last_name_raw) { +"K\xC3\xBC\xC3\xA7\xC3\xBCk" } let(:name_raw) { +"Onur K\xC3\xBC\xC3\xA7\xC3\xBCk" } + let(:username_claim_raw) { +'onur.partner' } let(:uid_ascii) { uid_raw.force_encoding(Encoding::ASCII_8BIT) } let(:email_ascii) { email_raw.force_encoding(Encoding::ASCII_8BIT) } @@ -37,6 +43,7 @@ RSpec.describe Gitlab::Auth::OAuth::AuthHash do let(:nickname_utf8) { nickname_ascii.force_encoding(Encoding::UTF_8) } let(:name_utf8) { name_ascii.force_encoding(Encoding::UTF_8) } let(:first_name_utf8) { first_name_ascii.force_encoding(Encoding::UTF_8) } + let(:username_claim_utf8) { username_claim_raw.force_encoding(Encoding::ASCII_8BIT) } let(:info_hash) do { @@ -98,10 +105,16 @@ RSpec.describe Gitlab::Auth::OAuth::AuthHash do allow(Gitlab::Auth::OAuth::Provider).to receive(:config_for).and_return(provider_config) end - it 'uses the custom field for the username' do + it 'uses the custom field for the username within info' do expect(auth_hash.username).to eql first_name_utf8 end + it 'uses the custom field for the username within extra.raw_info' do + provider_config['args']['gitlab_username_claim'] = 'https://example.com/claims/username' + + expect(auth_hash.username).to eql username_claim_utf8 + end + it 'uses the default claim for the username when the custom claim is not found' do provider_config['args']['gitlab_username_claim'] = 'nonexistent' @@ -146,4 +159,66 @@ RSpec.describe Gitlab::Auth::OAuth::AuthHash do expect(auth_hash.password.encoding).to eql Encoding::UTF_8 end end + + describe '#get_from_auth_hash_or_info' do + context 'for a key not within auth_hash' do + let(:auth_hash) do + described_class.new( + OmniAuth::AuthHash.new( + provider: provider, + uid: uid_ascii, + info: info_hash + ) + ) + end + + let(:info_hash) { { nickname: nickname_ascii } } + + it 'provides username from info_hash' do + expect(auth_hash.username).to eql nickname_utf8 + end + end + + context 'for a key within auth_hash' do + let(:auth_hash) do + described_class.new( + OmniAuth::AuthHash.new( + provider: provider, + uid: uid_ascii, + info: info_hash, + username: nickname_ascii + ) + ) + end + + let(:info_hash) { { something: nickname_ascii } } + + it 'provides username from auth_hash' do + expect(auth_hash.username).to eql nickname_utf8 + end + end + + context 'for a key within auth_hash extra' do + let(:auth_hash) do + described_class.new( + OmniAuth::AuthHash.new( + provider: provider, + uid: uid_ascii, + info: info_hash, + extra: { + raw_info: { + nickname: nickname_ascii + } + } + ) + ) + end + + let(:info_hash) { { something: nickname_ascii } } + + it 'provides username from auth_hash extra' do + expect(auth_hash.username).to eql nickname_utf8 + end + end + end end diff --git a/spec/lib/gitlab/auth/u2f_webauthn_converter_spec.rb b/spec/lib/gitlab/auth/u2f_webauthn_converter_spec.rb deleted file mode 100644 index deddc7f5294..00000000000 --- a/spec/lib/gitlab/auth/u2f_webauthn_converter_spec.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Auth::U2fWebauthnConverter do - let_it_be(:u2f_registration) do - device = U2F::FakeU2F.new(FFaker::BaconIpsum.characters(5)) - create(:u2f_registration, name: 'u2f_device', - certificate: Base64.strict_encode64(device.cert_raw), - key_handle: U2F.urlsafe_encode64(device.key_handle_raw), - public_key: Base64.strict_encode64(device.origin_public_key_raw)) - end - - it 'converts u2f registration' do - webauthn_credential = WebAuthn::U2fMigrator.new( - app_id: Gitlab.config.gitlab.url, - certificate: u2f_registration.certificate, - key_handle: u2f_registration.key_handle, - public_key: u2f_registration.public_key, - counter: u2f_registration.counter - ).credential - - converted_webauthn = described_class.new(u2f_registration).convert - - expect(converted_webauthn).to( - include(user_id: u2f_registration.user_id, - credential_xid: Base64.strict_encode64(webauthn_credential.id))) - end -end diff --git a/spec/lib/gitlab/auth_spec.rb b/spec/lib/gitlab/auth_spec.rb index 11e9ecdb878..36c87fb4557 100644 --- a/spec/lib/gitlab/auth_spec.rb +++ b/spec/lib/gitlab/auth_spec.rb @@ -21,6 +21,10 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching, feature_cate expect(subject::REPOSITORY_SCOPES).to match_array %i[read_repository write_repository] end + it 'OBSERVABILITY_SCOPES contains all scopes for Observability access' do + expect(subject::OBSERVABILITY_SCOPES).to match_array %i[read_observability write_observability] + end + it 'OPENID_SCOPES contains all scopes for OpenID Connect' do expect(subject::OPENID_SCOPES).to match_array [:openid] end @@ -31,54 +35,103 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching, feature_cate end context 'available_scopes' do - it 'contains all non-default scopes' do + before do stub_container_registry_config(enabled: true) + end - expect(subject.all_available_scopes).to match_array %i[api read_user read_api read_repository write_repository read_registry write_registry sudo admin_mode] + it 'contains all non-default scopes' do + expect(subject.all_available_scopes).to match_array %i[api read_user read_api read_repository write_repository read_registry write_registry sudo admin_mode read_observability write_observability] end - it 'contains for non-admin user all non-default scopes without ADMIN access' do - stub_container_registry_config(enabled: true) - user = create(:user, admin: false) + it 'contains for non-admin user all non-default scopes without ADMIN access and without observability scopes' do + user = build_stubbed(:user, admin: false) expect(subject.available_scopes_for(user)).to match_array %i[api read_user read_api read_repository write_repository read_registry write_registry] end - it 'contains for admin user all non-default scopes with ADMIN access' do - stub_container_registry_config(enabled: true) - user = create(:user, admin: true) + it 'contains for admin user all non-default scopes with ADMIN access and without observability scopes' do + user = build_stubbed(:user, admin: true) expect(subject.available_scopes_for(user)).to match_array %i[api read_user read_api read_repository write_repository read_registry write_registry sudo admin_mode] end + it 'contains for project all resource bot scopes without observability scopes' do + expect(subject.available_scopes_for(project)).to match_array %i[api read_api read_repository write_repository read_registry write_registry] + end + + it 'contains for group all resource bot scopes' do + group = build_stubbed(:group) + + expect(subject.available_scopes_for(group)).to match_array %i[api read_api read_repository write_repository read_registry write_registry read_observability write_observability] + end + + it 'contains for unsupported type no scopes' do + expect(subject.available_scopes_for(:something)).to be_empty + end + it 'optional_scopes contains all non-default scopes' do - stub_container_registry_config(enabled: true) + expect(subject.optional_scopes).to match_array %i[read_user read_api read_repository write_repository read_registry write_registry sudo admin_mode openid profile email read_observability write_observability] + end + + context 'with observability_group_tab feature flag' do + context 'when disabled' do + before do + stub_feature_flags(observability_group_tab: false) + end + + it 'contains for group all resource bot scopes without observability scopes' do + group = build_stubbed(:group) - expect(subject.optional_scopes).to match_array %i[read_user read_api read_repository write_repository read_registry write_registry sudo admin_mode openid profile email] + expect(subject.available_scopes_for(group)).to match_array %i[api read_api read_repository write_repository read_registry write_registry] + end + end + + context 'when enabled for specific group' do + let(:group) { build_stubbed(:group) } + + before do + stub_feature_flags(observability_group_tab: group) + end + + it 'contains for other group all resource bot scopes including observability scopes' do + expect(subject.available_scopes_for(group)).to match_array %i[api read_api read_repository write_repository read_registry write_registry read_observability write_observability] + end + + it 'contains for admin user all non-default scopes with ADMIN access and without observability scopes' do + user = build_stubbed(:user, admin: true) + + expect(subject.available_scopes_for(user)).to match_array %i[api read_user read_api read_repository write_repository read_registry write_registry sudo admin_mode] + end + + it 'contains for project all resource bot scopes without observability scopes' do + expect(subject.available_scopes_for(project)).to match_array %i[api read_api read_repository write_repository read_registry write_registry] + end + + it 'contains for other group all resource bot scopes without observability scopes' do + other_group = build_stubbed(:group) + + expect(subject.available_scopes_for(other_group)).to match_array %i[api read_api read_repository write_repository read_registry write_registry] + end + end end - context 'with feature flag disabled' do + context 'with admin_mode_for_api feature flag disabled' do before do stub_feature_flags(admin_mode_for_api: false) end it 'contains all non-default scopes' do - stub_container_registry_config(enabled: true) - - expect(subject.all_available_scopes).to match_array %i[api read_user read_api read_repository write_repository read_registry write_registry sudo admin_mode] + expect(subject.all_available_scopes).to match_array %i[api read_user read_api read_repository write_repository read_registry write_registry sudo admin_mode read_observability write_observability] end - it 'contains for admin user all non-default scopes with ADMIN access' do - stub_container_registry_config(enabled: true) - user = create(:user, admin: true) + it 'contains for admin user all non-default scopes with ADMIN access and without observability scopes' do + user = build_stubbed(:user, admin: true) expect(subject.available_scopes_for(user)).to match_array %i[api read_user read_api read_repository write_repository read_registry write_registry sudo] end it 'optional_scopes contains all non-default scopes' do - stub_container_registry_config(enabled: true) - - expect(subject.optional_scopes).to match_array %i[read_user read_api read_repository write_repository read_registry write_registry sudo admin_mode openid profile email] + expect(subject.optional_scopes).to match_array %i[read_user read_api read_repository write_repository read_registry write_registry sudo admin_mode openid profile email read_observability write_observability] end end @@ -120,8 +173,8 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching, feature_cate end end - it 'raises an IpBlacklisted exception' do - expect { subject }.to raise_error(Gitlab::Auth::IpBlacklisted) + it 'raises an IpBlocked exception' do + expect { subject }.to raise_error(Gitlab::Auth::IpBlocked) end end @@ -314,15 +367,17 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching, feature_cate using RSpec::Parameterized::TableSyntax where(:scopes, :abilities) do - 'api' | described_class.full_authentication_abilities - 'read_api' | described_class.read_only_authentication_abilities - 'read_repository' | [:download_code] - 'write_repository' | [:download_code, :push_code] - 'read_user' | [] - 'sudo' | [] - 'openid' | [] - 'profile' | [] - 'email' | [] + 'api' | described_class.full_authentication_abilities + 'read_api' | described_class.read_only_authentication_abilities + 'read_repository' | [:download_code] + 'write_repository' | [:download_code, :push_code] + 'read_user' | [] + 'sudo' | [] + 'openid' | [] + 'profile' | [] + 'email' | [] + 'read_observability' | [] + 'write_observability' | [] end with_them do @@ -1024,6 +1079,7 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching, feature_cate it { is_expected.to include(*described_class::API_SCOPES - [:read_user]) } it { is_expected.to include(*described_class::REPOSITORY_SCOPES) } it { is_expected.to include(*described_class.registry_scopes) } + it { is_expected.to include(*described_class::OBSERVABILITY_SCOPES) } end private diff --git a/spec/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens_spec.rb b/spec/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens_spec.rb index d2da6867773..92fec48454c 100644 --- a/spec/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens_spec.rb @@ -24,8 +24,12 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillAdminModeScopeForPersonalAcc personal_access_tokens.create!(name: 'admin 4', user_id: admin.id, scopes: "---\n- admin_mode\n") end - let!(:pat_admin_2) { personal_access_tokens.create!(name: 'admin 5', user_id: admin.id, scopes: "---\n- read_api\n") } - let!(:pat_not_in_range) { personal_access_tokens.create!(name: 'admin 6', user_id: admin.id, scopes: "---\n- api\n") } + let!(:pat_with_symbol_in_scopes) do + personal_access_tokens.create!(name: 'admin 5', user_id: admin.id, scopes: "---\n- :api\n") + end + + let!(:pat_admin_2) { personal_access_tokens.create!(name: 'admin 6', user_id: admin.id, scopes: "---\n- read_api\n") } + let!(:pat_not_in_range) { personal_access_tokens.create!(name: 'admin 7', user_id: admin.id, scopes: "---\n- api\n") } subject do described_class.new( @@ -47,6 +51,7 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillAdminModeScopeForPersonalAcc expect(pat_revoked.reload.scopes).to eq("---\n- api\n") expect(pat_expired.reload.scopes).to eq("---\n- api\n") expect(pat_admin_mode.reload.scopes).to eq("---\n- admin_mode\n") + expect(pat_with_symbol_in_scopes.reload.scopes).to eq("---\n- api\n- admin_mode\n") expect(pat_admin_2.reload.scopes).to eq("---\n- read_api\n- admin_mode\n") expect(pat_not_in_range.reload.scopes).to eq("---\n- api\n") end diff --git a/spec/lib/gitlab/background_migration/backfill_ci_queuing_tables_spec.rb b/spec/lib/gitlab/background_migration/backfill_ci_queuing_tables_spec.rb deleted file mode 100644 index aaf8c124a83..00000000000 --- a/spec/lib/gitlab/background_migration/backfill_ci_queuing_tables_spec.rb +++ /dev/null @@ -1,245 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::BackfillCiQueuingTables, :migration, - :suppress_gitlab_schemas_validate_connection, schema: 20220208115439 do - let(:namespaces) { table(:namespaces) } - let(:projects) { table(:projects) } - let(:ci_cd_settings) { table(:project_ci_cd_settings) } - let(:builds) { table(:ci_builds) } - let(:queuing_entries) { table(:ci_pending_builds) } - let(:tags) { table(:tags) } - let(:taggings) { table(:taggings) } - - subject { described_class.new } - - describe '#perform' do - let!(:namespace) do - namespaces.create!( - id: 10, - name: 'namespace10', - path: 'namespace10', - traversal_ids: [10]) - end - - let!(:other_namespace) do - namespaces.create!( - id: 11, - name: 'namespace11', - path: 'namespace11', - traversal_ids: [11]) - end - - let!(:project) do - projects.create!(id: 5, namespace_id: 10, name: 'test1', path: 'test1') - end - - let!(:ci_cd_setting) do - ci_cd_settings.create!(id: 5, project_id: 5, group_runners_enabled: true) - end - - let!(:other_project) do - projects.create!(id: 7, namespace_id: 11, name: 'test2', path: 'test2') - end - - let!(:other_ci_cd_setting) do - ci_cd_settings.create!(id: 7, project_id: 7, group_runners_enabled: false) - end - - let!(:another_project) do - projects.create!(id: 9, namespace_id: 10, name: 'test3', path: 'test3', shared_runners_enabled: false) - end - - let!(:ruby_tag) do - tags.create!(id: 22, name: 'ruby') - end - - let!(:postgres_tag) do - tags.create!(id: 23, name: 'postgres') - end - - it 'creates ci_pending_builds for all pending builds in range' do - builds.create!(id: 50, status: :pending, name: 'test1', project_id: 5, type: 'Ci::Build') - builds.create!(id: 51, status: :created, name: 'test2', project_id: 5, type: 'Ci::Build') - builds.create!(id: 52, status: :pending, name: 'test3', project_id: 5, protected: true, type: 'Ci::Build') - - taggings.create!(taggable_id: 52, taggable_type: 'CommitStatus', tag_id: 22) - taggings.create!(taggable_id: 52, taggable_type: 'CommitStatus', tag_id: 23) - - builds.create!(id: 60, status: :pending, name: 'test1', project_id: 7, type: 'Ci::Build') - builds.create!(id: 61, status: :running, name: 'test2', project_id: 7, protected: true, type: 'Ci::Build') - builds.create!(id: 62, status: :pending, name: 'test3', project_id: 7, type: 'Ci::Build') - - taggings.create!(taggable_id: 60, taggable_type: 'CommitStatus', tag_id: 23) - taggings.create!(taggable_id: 62, taggable_type: 'CommitStatus', tag_id: 22) - - builds.create!(id: 70, status: :pending, name: 'test1', project_id: 9, protected: true, type: 'Ci::Build') - builds.create!(id: 71, status: :failed, name: 'test2', project_id: 9, type: 'Ci::Build') - builds.create!(id: 72, status: :pending, name: 'test3', project_id: 9, type: 'Ci::Build') - - taggings.create!(taggable_id: 71, taggable_type: 'CommitStatus', tag_id: 22) - - subject.perform(1, 100) - - expect(queuing_entries.all).to contain_exactly( - an_object_having_attributes( - build_id: 50, - project_id: 5, - namespace_id: 10, - protected: false, - instance_runners_enabled: true, - minutes_exceeded: false, - tag_ids: [], - namespace_traversal_ids: [10]), - an_object_having_attributes( - build_id: 52, - project_id: 5, - namespace_id: 10, - protected: true, - instance_runners_enabled: true, - minutes_exceeded: false, - tag_ids: match_array([22, 23]), - namespace_traversal_ids: [10]), - an_object_having_attributes( - build_id: 60, - project_id: 7, - namespace_id: 11, - protected: false, - instance_runners_enabled: true, - minutes_exceeded: false, - tag_ids: [23], - namespace_traversal_ids: []), - an_object_having_attributes( - build_id: 62, - project_id: 7, - namespace_id: 11, - protected: false, - instance_runners_enabled: true, - minutes_exceeded: false, - tag_ids: [22], - namespace_traversal_ids: []), - an_object_having_attributes( - build_id: 70, - project_id: 9, - namespace_id: 10, - protected: true, - instance_runners_enabled: false, - minutes_exceeded: false, - tag_ids: [], - namespace_traversal_ids: []), - an_object_having_attributes( - build_id: 72, - project_id: 9, - namespace_id: 10, - protected: false, - instance_runners_enabled: false, - minutes_exceeded: false, - tag_ids: [], - namespace_traversal_ids: []) - ) - end - - it 'skips builds that already have ci_pending_builds' do - builds.create!(id: 50, status: :pending, name: 'test1', project_id: 5, type: 'Ci::Build') - builds.create!(id: 51, status: :created, name: 'test2', project_id: 5, type: 'Ci::Build') - builds.create!(id: 52, status: :pending, name: 'test3', project_id: 5, protected: true, type: 'Ci::Build') - - taggings.create!(taggable_id: 50, taggable_type: 'CommitStatus', tag_id: 22) - taggings.create!(taggable_id: 52, taggable_type: 'CommitStatus', tag_id: 23) - - queuing_entries.create!(build_id: 50, project_id: 5, namespace_id: 10) - - subject.perform(1, 100) - - expect(queuing_entries.all).to contain_exactly( - an_object_having_attributes( - build_id: 50, - project_id: 5, - namespace_id: 10, - protected: false, - instance_runners_enabled: false, - minutes_exceeded: false, - tag_ids: [], - namespace_traversal_ids: []), - an_object_having_attributes( - build_id: 52, - project_id: 5, - namespace_id: 10, - protected: true, - instance_runners_enabled: true, - minutes_exceeded: false, - tag_ids: [23], - namespace_traversal_ids: [10]) - ) - end - - it 'upserts values in case of conflicts' do - builds.create!(id: 50, status: :pending, name: 'test1', project_id: 5, type: 'Ci::Build') - queuing_entries.create!(build_id: 50, project_id: 5, namespace_id: 10) - - build = described_class::Ci::Build.find(50) - described_class::Ci::PendingBuild.upsert_from_build!(build) - - expect(queuing_entries.all).to contain_exactly( - an_object_having_attributes( - build_id: 50, - project_id: 5, - namespace_id: 10, - protected: false, - instance_runners_enabled: true, - minutes_exceeded: false, - tag_ids: [], - namespace_traversal_ids: [10]) - ) - end - end - - context 'Ci::Build' do - describe '.each_batch' do - let(:model) { described_class::Ci::Build } - - before do - builds.create!(id: 1, status: :pending, name: 'test1', project_id: 5, type: 'Ci::Build') - builds.create!(id: 2, status: :pending, name: 'test2', project_id: 5, type: 'Ci::Build') - builds.create!(id: 3, status: :pending, name: 'test3', project_id: 5, type: 'Ci::Build') - builds.create!(id: 4, status: :pending, name: 'test4', project_id: 5, type: 'Ci::Build') - builds.create!(id: 5, status: :pending, name: 'test5', project_id: 5, type: 'Ci::Build') - end - - it 'yields an ActiveRecord::Relation when a block is given' do - model.each_batch do |relation| - expect(relation).to be_a_kind_of(ActiveRecord::Relation) - end - end - - it 'yields a batch index as the second argument' do - model.each_batch do |_, index| - expect(index).to eq(1) - end - end - - it 'accepts a custom batch size' do - amount = 0 - - model.each_batch(of: 1) { amount += 1 } - - expect(amount).to eq(5) - end - - it 'does not include ORDER BYs in the yielded relations' do - model.each_batch do |relation| - expect(relation.to_sql).not_to include('ORDER BY') - end - end - - it 'orders ascending' do - ids = [] - - model.each_batch(of: 1) { |rel| ids.concat(rel.ids) } - - expect(ids).to eq(ids.sort) - end - end - end -end diff --git a/spec/lib/gitlab/background_migration/backfill_group_features_spec.rb b/spec/lib/gitlab/background_migration/backfill_group_features_spec.rb index e0be5a785b8..2c2740434de 100644 --- a/spec/lib/gitlab/background_migration/backfill_group_features_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_group_features_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::BackfillGroupFeatures, :migration, schema: 20220302114046 do +RSpec.describe Gitlab::BackgroundMigration::BackfillGroupFeatures, :migration, schema: 20220314184009 do let(:group_features) { table(:group_features) } let(:namespaces) { table(:namespaces) } diff --git a/spec/lib/gitlab/background_migration/backfill_integrations_type_new_spec.rb b/spec/lib/gitlab/background_migration/backfill_integrations_type_new_spec.rb deleted file mode 100644 index e6588644b4f..00000000000 --- a/spec/lib/gitlab/background_migration/backfill_integrations_type_new_spec.rb +++ /dev/null @@ -1,67 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::BackfillIntegrationsTypeNew, :migration, schema: 20220212120735 do - let(:migration) { described_class.new } - let(:integrations) { table(:integrations) } - - let(:namespaced_integrations) do - Set.new( - %w[ - Asana Assembla Bamboo Bugzilla Buildkite Campfire Confluence CustomIssueTracker Datadog - Discord DroneCi EmailsOnPush Ewm ExternalWiki Flowdock HangoutsChat Harbor Irker Jenkins Jira Mattermost - MattermostSlashCommands MicrosoftTeams MockCi MockMonitoring Packagist PipelinesEmail Pivotaltracker - Prometheus Pushover Redmine Shimo Slack SlackSlashCommands Teamcity UnifyCircuit WebexTeams Youtrack Zentao - Github GitlabSlackApplication - ]).freeze - end - - before do - integrations.connection.execute 'ALTER TABLE integrations DISABLE TRIGGER "trigger_type_new_on_insert"' - - namespaced_integrations.each_with_index do |type, i| - integrations.create!(id: i + 1, type: "#{type}Service") - end - - integrations.create!(id: namespaced_integrations.size + 1, type: 'LegacyService') - ensure - integrations.connection.execute 'ALTER TABLE integrations ENABLE TRIGGER "trigger_type_new_on_insert"' - end - - it 'backfills `type_new` for the selected records' do - # We don't want to mock `Kernel.sleep`, so instead we mock it on the migration - # class before it gets forwarded. - expect(migration).to receive(:sleep).with(0.05).exactly(5).times - - queries = ActiveRecord::QueryRecorder.new do - migration.perform(2, 10, :integrations, :id, 2, 50) - end - - expect(queries.count).to be(16) - expect(queries.log.grep(/^SELECT/).size).to be(11) - expect(queries.log.grep(/^UPDATE/).size).to be(5) - expect(queries.log.grep(/^UPDATE/).join.scan(/WHERE .*/)).to eq( - [ - 'WHERE integrations.id BETWEEN 2 AND 3', - 'WHERE integrations.id BETWEEN 4 AND 5', - 'WHERE integrations.id BETWEEN 6 AND 7', - 'WHERE integrations.id BETWEEN 8 AND 9', - 'WHERE integrations.id BETWEEN 10 AND 10' - ]) - - expect(integrations.where(id: 2..10).pluck(:type, :type_new)).to contain_exactly( - ['AssemblaService', 'Integrations::Assembla'], - ['BambooService', 'Integrations::Bamboo'], - ['BugzillaService', 'Integrations::Bugzilla'], - ['BuildkiteService', 'Integrations::Buildkite'], - ['CampfireService', 'Integrations::Campfire'], - ['ConfluenceService', 'Integrations::Confluence'], - ['CustomIssueTrackerService', 'Integrations::CustomIssueTracker'], - ['DatadogService', 'Integrations::Datadog'], - ['DiscordService', 'Integrations::Discord'] - ) - - expect(integrations.where.not(id: 2..10)).to all(have_attributes(type_new: nil)) - end -end diff --git a/spec/lib/gitlab/background_migration/backfill_member_namespace_for_group_members_spec.rb b/spec/lib/gitlab/background_migration/backfill_member_namespace_for_group_members_spec.rb index e1ef12a1479..ea07079f9ee 100644 --- a/spec/lib/gitlab/background_migration/backfill_member_namespace_for_group_members_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_member_namespace_for_group_members_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::BackfillMemberNamespaceForGroupMembers, :migration, schema: 20220120211832 do +RSpec.describe Gitlab::BackgroundMigration::BackfillMemberNamespaceForGroupMembers, :migration, schema: 20220314184009 do let(:migration) { described_class.new } let(:members_table) { table(:members) } let(:namespaces_table) { table(:namespaces) } diff --git a/spec/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route_spec.rb b/spec/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route_spec.rb index b821efcadb0..f4e8fa1bbac 100644 --- a/spec/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::BackfillNamespaceIdForNamespaceRoute, :migration, schema: 20220120123800 do +RSpec.describe Gitlab::BackgroundMigration::BackfillNamespaceIdForNamespaceRoute, :migration, schema: 20220314184009 do let(:migration) { described_class.new } let(:namespaces_table) { table(:namespaces) } let(:projects_table) { table(:projects) } diff --git a/spec/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children_spec.rb b/spec/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children_spec.rb deleted file mode 100644 index 876eb070745..00000000000 --- a/spec/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children_spec.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::BackfillNamespaceTraversalIdsChildren, :migration, schema: 20210826171758 do - let(:namespaces_table) { table(:namespaces) } - - let!(:user_namespace) { namespaces_table.create!(id: 1, name: 'user', path: 'user', type: nil) } - let!(:root_group) { namespaces_table.create!(id: 2, name: 'group', path: 'group', type: 'Group', parent_id: nil) } - let!(:sub_group) { namespaces_table.create!(id: 3, name: 'subgroup', path: 'subgroup', type: 'Group', parent_id: 2) } - - describe '#perform' do - it 'backfills traversal_ids for child namespaces' do - described_class.new.perform(1, 3, 5) - - expect(user_namespace.reload.traversal_ids).to eq([]) - expect(root_group.reload.traversal_ids).to eq([]) - expect(sub_group.reload.traversal_ids).to eq([root_group.id, sub_group.id]) - end - end -end diff --git a/spec/lib/gitlab/background_migration/backfill_namespace_traversal_ids_roots_spec.rb b/spec/lib/gitlab/background_migration/backfill_namespace_traversal_ids_roots_spec.rb deleted file mode 100644 index ad9b54608c6..00000000000 --- a/spec/lib/gitlab/background_migration/backfill_namespace_traversal_ids_roots_spec.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::BackfillNamespaceTraversalIdsRoots, :migration, schema: 20210826171758 do - let(:namespaces_table) { table(:namespaces) } - - let!(:user_namespace) { namespaces_table.create!(id: 1, name: 'user', path: 'user', type: nil) } - let!(:root_group) { namespaces_table.create!(id: 2, name: 'group', path: 'group', type: 'Group', parent_id: nil) } - let!(:sub_group) { namespaces_table.create!(id: 3, name: 'subgroup', path: 'subgroup', type: 'Group', parent_id: 2) } - - describe '#perform' do - it 'backfills traversal_ids for root namespaces' do - described_class.new.perform(1, 3, 5) - - expect(user_namespace.reload.traversal_ids).to eq([user_namespace.id]) - expect(root_group.reload.traversal_ids).to eq([root_group.id]) - expect(sub_group.reload.traversal_ids).to eq([]) - end - end -end diff --git a/spec/lib/gitlab/background_migration/backfill_partitioned_table_spec.rb b/spec/lib/gitlab/background_migration/backfill_partitioned_table_spec.rb new file mode 100644 index 00000000000..53216cc780b --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_partitioned_table_spec.rb @@ -0,0 +1,140 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillPartitionedTable, feature_category: :database do + subject(:backfill_job) do + described_class.new( + start_id: 1, + end_id: 3, + batch_table: source_table, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + job_arguments: [destination_table], + connection: connection + ) + end + + let(:connection) { ApplicationRecord.connection } + let(:source_table) { '_test_source_table' } + let(:destination_table) { "#{source_table}_partitioned" } + let(:source_model) { Class.new(ApplicationRecord) } + let(:destination_model) { Class.new(ApplicationRecord) } + + describe '#perform' do + context 'without the destination table' do + let(:expected_error_message) do + "exiting backfill migration because partitioned table #{destination_table} does not exist. " \ + "This could be due to rollback of the migration which created the partitioned table." + end + + it 'raises an exception' do + expect { backfill_job.perform }.to raise_error(expected_error_message) + end + end + + context 'with destination table being not partitioned' do + before do + connection.execute(<<~SQL) + CREATE TABLE #{destination_table} ( + id serial NOT NULL, + col1 int NOT NULL, + col2 text NOT NULL, + created_at timestamptz NOT NULL, + PRIMARY KEY (id, created_at) + ) + SQL + end + + after do + connection.drop_table destination_table + end + + let(:expected_error_message) do + "exiting backfill migration because the given destination table is not partitioned." + end + + it 'raises an exception' do + expect { backfill_job.perform }.to raise_error(expected_error_message) + end + end + + context 'when the destination table exists' do + before do + connection.execute(<<~SQL) + CREATE TABLE #{source_table} ( + id serial NOT NULL PRIMARY KEY, + col1 int NOT NULL, + col2 text NOT NULL, + created_at timestamptz NOT NULL + ) + SQL + + connection.execute(<<~SQL) + CREATE TABLE #{destination_table} ( + id serial NOT NULL, + col1 int NOT NULL, + col2 text NOT NULL, + created_at timestamptz NOT NULL, + PRIMARY KEY (id, created_at) + ) PARTITION BY RANGE (created_at) + SQL + + connection.execute(<<~SQL) + CREATE TABLE #{destination_table}_202001 PARTITION OF #{destination_table} + FOR VALUES FROM ('2020-01-01') TO ('2020-02-01') + SQL + + connection.execute(<<~SQL) + CREATE TABLE #{destination_table}_202002 PARTITION OF #{destination_table} + FOR VALUES FROM ('2020-02-01') TO ('2020-03-01') + SQL + + source_model.table_name = source_table + destination_model.table_name = destination_table + end + + after do + connection.drop_table source_table + connection.drop_table destination_table + end + + let(:timestamp) { Time.utc(2020, 1, 2).round } + let!(:source1) { create_source_record(timestamp) } + let!(:source2) { create_source_record(timestamp + 1.day) } + let!(:source3) { create_source_record(timestamp + 1.month) } + + it 'copies data into the destination table idempotently' do + expect(destination_model.count).to eq(0) + + backfill_job.perform + + expect(destination_model.count).to eq(3) + + source_model.find_each do |source_record| + destination_record = destination_model.find_by_id(source_record.id) + + expect(destination_record.attributes).to eq(source_record.attributes) + end + + backfill_job.perform + + expect(destination_model.count).to eq(3) + end + + it 'breaks the assigned batch into smaller sub batches' do + expect_next_instance_of(Gitlab::Database::PartitioningMigrationHelpers::BulkCopy) do |bulk_copy| + expect(bulk_copy).to receive(:copy_between).with(source1.id, source2.id) + expect(bulk_copy).to receive(:copy_between).with(source3.id, source3.id) + end + + backfill_job.perform + end + end + end + + def create_source_record(timestamp) + source_model.create!(col1: 123, col2: 'original value', created_at: timestamp) + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_prepared_at_merge_requests_spec.rb b/spec/lib/gitlab/background_migration/backfill_prepared_at_merge_requests_spec.rb index b33a1a31c40..28ecfae1bd4 100644 --- a/spec/lib/gitlab/background_migration/backfill_prepared_at_merge_requests_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_prepared_at_merge_requests_spec.rb @@ -14,18 +14,6 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillPreparedAtMergeRequests, :mi projects.create!(name: 'proj1', path: 'proj1', namespace_id: namespace.id, project_namespace_id: proj_namespace.id) end - let(:test_worker) do - described_class.new( - start_id: 1, - end_id: 100, - batch_table: :merge_requests, - batch_column: :id, - sub_batch_size: 10, - pause_ms: 0, - connection: ApplicationRecord.connection - ) - end - it 'updates merge requests with prepared_at nil' do time = Time.current @@ -40,6 +28,16 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillPreparedAtMergeRequests, :mi mr_5 = mr_table.create!(target_project_id: project.id, source_branch: 'master', target_branch: 'feature', prepared_at: time, merge_status: 'preparing') + test_worker = described_class.new( + start_id: mr_1.id, + end_id: [(mr_5.id + 1), 100].max, + batch_table: :merge_requests, + batch_column: :id, + sub_batch_size: 10, + pause_ms: 0, + connection: ApplicationRecord.connection + ) + expect(mr_1.prepared_at).to be_nil expect(mr_2.prepared_at).to be_nil expect(mr_3.prepared_at.to_i).to eq(time.to_i) diff --git a/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb b/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb index 80fd86e90bb..6f6ff9232e0 100644 --- a/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::BackfillSnippetRepositories, :migration, schema: 20210826171758, +RSpec.describe Gitlab::BackgroundMigration::BackfillSnippetRepositories, :migration, schema: 20220314184009, feature_category: :source_code_management do let(:gitlab_shell) { Gitlab::Shell.new } let(:users) { table(:users) } diff --git a/spec/lib/gitlab/background_migration/backfill_upvotes_count_on_issues_spec.rb b/spec/lib/gitlab/background_migration/backfill_upvotes_count_on_issues_spec.rb deleted file mode 100644 index 7142aea3ab2..00000000000 --- a/spec/lib/gitlab/background_migration/backfill_upvotes_count_on_issues_spec.rb +++ /dev/null @@ -1,46 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::BackfillUpvotesCountOnIssues, schema: 20210826171758 do - let(:award_emoji) { table(:award_emoji) } - - let!(:namespace) { table(:namespaces).create!(name: 'namespace', path: 'namespace') } - let!(:project1) { table(:projects).create!(namespace_id: namespace.id) } - let!(:project2) { table(:projects).create!(namespace_id: namespace.id) } - let!(:issue1) { table(:issues).create!(project_id: project1.id) } - let!(:issue2) { table(:issues).create!(project_id: project2.id) } - let!(:issue3) { table(:issues).create!(project_id: project2.id) } - let!(:issue4) { table(:issues).create!(project_id: project2.id) } - - describe '#perform' do - before do - add_upvotes(issue1, :thumbsdown, 1) - add_upvotes(issue2, :thumbsup, 2) - add_upvotes(issue2, :thumbsdown, 1) - add_upvotes(issue3, :thumbsup, 3) - add_upvotes(issue4, :thumbsup, 4) - end - - it 'updates upvotes_count' do - subject.perform(issue1.id, issue4.id) - - expect(issue1.reload.upvotes_count).to eq(0) - expect(issue2.reload.upvotes_count).to eq(2) - expect(issue3.reload.upvotes_count).to eq(3) - expect(issue4.reload.upvotes_count).to eq(4) - end - end - - private - - def add_upvotes(issue, name, count) - count.times do - award_emoji.create!( - name: name.to_s, - awardable_type: 'Issue', - awardable_id: issue.id - ) - end - end -end diff --git a/spec/lib/gitlab/background_migration/backfill_user_namespace_spec.rb b/spec/lib/gitlab/background_migration/backfill_user_namespace_spec.rb deleted file mode 100644 index 395248b786d..00000000000 --- a/spec/lib/gitlab/background_migration/backfill_user_namespace_spec.rb +++ /dev/null @@ -1,39 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::BackfillUserNamespace, :migration, schema: 20210930211936 do - let(:migration) { described_class.new } - let(:namespaces_table) { table(:namespaces) } - - let(:table_name) { 'namespaces' } - let(:batch_column) { :id } - let(:sub_batch_size) { 100 } - let(:pause_ms) { 0 } - - subject(:perform_migration) { migration.perform(1, 10, table_name, batch_column, sub_batch_size, pause_ms) } - - before do - namespaces_table.create!(id: 1, name: 'test1', path: 'test1', type: nil) - namespaces_table.create!(id: 2, name: 'test2', path: 'test2', type: 'User') - namespaces_table.create!(id: 3, name: 'test3', path: 'test3', type: 'Group') - namespaces_table.create!(id: 4, name: 'test4', path: 'test4', type: nil) - namespaces_table.create!(id: 11, name: 'test11', path: 'test11', type: nil) - end - - it 'backfills `type` for the selected records', :aggregate_failures do - queries = ActiveRecord::QueryRecorder.new do - perform_migration - end - - expect(queries.count).to eq(3) - expect(namespaces_table.where(type: 'User').count).to eq 3 - expect(namespaces_table.where(type: 'User').pluck(:id)).to match_array([1, 2, 4]) - end - - it 'tracks timings of queries' do - expect(migration.batch_metrics.timings).to be_empty - - expect { perform_migration }.to change { migration.batch_metrics.timings } - end -end diff --git a/spec/lib/gitlab/background_migration/cleanup_orphaned_lfs_objects_projects_spec.rb b/spec/lib/gitlab/background_migration/cleanup_orphaned_lfs_objects_projects_spec.rb deleted file mode 100644 index 5ffe665f0ad..00000000000 --- a/spec/lib/gitlab/background_migration/cleanup_orphaned_lfs_objects_projects_spec.rb +++ /dev/null @@ -1,85 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::CleanupOrphanedLfsObjectsProjects, schema: 20210826171758 do - let(:lfs_objects_projects) { table(:lfs_objects_projects) } - let(:lfs_objects) { table(:lfs_objects) } - let(:projects) { table(:projects) } - let(:namespaces) { table(:namespaces) } - - let(:namespace) { namespaces.create!(name: 'namespace', path: 'namespace') } - let(:project) { projects.create!(namespace_id: namespace.id) } - let(:another_project) { projects.create!(namespace_id: namespace.id) } - let(:lfs_object) { lfs_objects.create!(oid: 'abcdef', size: 1) } - let(:another_lfs_object) { lfs_objects.create!(oid: '1abcde', size: 2) } - - let!(:without_object1) { create_object(project_id: project.id) } - let!(:without_object2) { create_object(project_id: another_project.id) } - let!(:without_object3) { create_object(project_id: another_project.id) } - let!(:with_project_and_object1) { create_object(project_id: project.id, lfs_object_id: lfs_object.id) } - let!(:with_project_and_object2) { create_object(project_id: project.id, lfs_object_id: another_lfs_object.id) } - let!(:with_project_and_object3) { create_object(project_id: another_project.id, lfs_object_id: another_lfs_object.id) } - let!(:without_project1) { create_object(lfs_object_id: lfs_object.id) } - let!(:without_project2) { create_object(lfs_object_id: another_lfs_object.id) } - let!(:without_project_and_object) { create_object } - - def create_object(project_id: non_existing_record_id, lfs_object_id: non_existing_record_id) - lfs_objects_project = nil - - ActiveRecord::Base.connection.disable_referential_integrity do - lfs_objects_project = lfs_objects_projects.create!(project_id: project_id, lfs_object_id: lfs_object_id) - end - - lfs_objects_project - end - - subject { described_class.new } - - describe '#perform' do - it 'lfs_objects_projects without an existing lfs object or project are removed' do - subject.perform(without_object1.id, without_object3.id) - - expect(lfs_objects_projects.all).to match_array( - [ - with_project_and_object1, with_project_and_object2, with_project_and_object3, - without_project1, without_project2, without_project_and_object - ]) - - subject.perform(with_project_and_object1.id, with_project_and_object3.id) - - expect(lfs_objects_projects.all).to match_array( - [ - with_project_and_object1, with_project_and_object2, with_project_and_object3, - without_project1, without_project2, without_project_and_object - ]) - - subject.perform(without_project1.id, without_project_and_object.id) - - expect(lfs_objects_projects.all).to match_array( - [ - with_project_and_object1, with_project_and_object2, with_project_and_object3 - ]) - - expect(lfs_objects.ids).to contain_exactly(lfs_object.id, another_lfs_object.id) - expect(projects.ids).to contain_exactly(project.id, another_project.id) - end - - it 'cache for affected projects is being reset' do - expect(ProjectCacheWorker).to receive(:bulk_perform_in) do |delay, args| - expect(delay).to eq(1.minute) - expect(args).to match_array([[project.id, [], [:lfs_objects_size]], [another_project.id, [], [:lfs_objects_size]]]) - end - - subject.perform(without_object1.id, with_project_and_object1.id) - - expect(ProjectCacheWorker).not_to receive(:bulk_perform_in) - - subject.perform(with_project_and_object1.id, with_project_and_object3.id) - - expect(ProjectCacheWorker).not_to receive(:bulk_perform_in) - - subject.perform(without_project1.id, without_project_and_object.id) - end - end -end diff --git a/spec/lib/gitlab/background_migration/delete_orphaned_deployments_spec.rb b/spec/lib/gitlab/background_migration/delete_orphaned_deployments_spec.rb deleted file mode 100644 index 8f058c875a2..00000000000 --- a/spec/lib/gitlab/background_migration/delete_orphaned_deployments_spec.rb +++ /dev/null @@ -1,54 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::DeleteOrphanedDeployments, :migration, schema: 20210826171758 do - let!(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') } - let!(:project) { table(:projects).create!(namespace_id: namespace.id) } - let!(:environment) { table(:environments).create!(name: 'production', slug: 'production', project_id: project.id) } - let(:background_migration_jobs) { table(:background_migration_jobs) } - - before do - create_deployment!(environment.id, project.id) - end - - it 'deletes only orphaned deployments' do - expect(valid_deployments.pluck(:id)).not_to be_empty - - subject.perform(table(:deployments).minimum(:id), table(:deployments).maximum(:id)) - - expect(valid_deployments.pluck(:id)).not_to be_empty - end - - it 'marks jobs as done' do - first_job = background_migration_jobs.create!( - class_name: 'DeleteOrphanedDeployments', - arguments: [table(:deployments).minimum(:id), table(:deployments).minimum(:id)] - ) - - subject.perform(table(:deployments).minimum(:id), table(:deployments).minimum(:id)) - - expect(first_job.reload.status).to eq(Gitlab::Database::BackgroundMigrationJob.statuses[:succeeded]) - end - - private - - def valid_deployments - table(:deployments).where('EXISTS (SELECT 1 FROM environments WHERE deployments.environment_id = environments.id)') - end - - def orphaned_deployments - table(:deployments).where('NOT EXISTS (SELECT 1 FROM environments WHERE deployments.environment_id = environments.id)') - end - - def create_deployment!(environment_id, project_id) - table(:deployments).create!( - environment_id: environment_id, - project_id: project_id, - ref: 'master', - tag: false, - sha: 'x', - status: 1, - iid: table(:deployments).count + 1) - end -end diff --git a/spec/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images_spec.rb b/spec/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images_spec.rb deleted file mode 100644 index e7b0471810d..00000000000 --- a/spec/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images_spec.rb +++ /dev/null @@ -1,142 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::DisableExpirationPoliciesLinkedToNoContainerImages, :migration, schema: 20220326161803 do # rubocop:disable Layout/LineLength - let!(:projects) { table(:projects) } - let!(:container_expiration_policies) { table(:container_expiration_policies) } - let!(:container_repositories) { table(:container_repositories) } - let!(:namespaces) { table(:namespaces) } - - let!(:namespace) { namespaces.create!(name: 'test', path: 'test') } - - let!(:policy1) { create_expiration_policy(project_id: 1, enabled: true) } - let!(:policy2) { create_expiration_policy(project_id: 2, enabled: false) } - let!(:policy3) { create_expiration_policy(project_id: 3, enabled: false) } - let!(:policy4) { create_expiration_policy(project_id: 4, enabled: true, with_images: true) } - let!(:policy5) { create_expiration_policy(project_id: 5, enabled: false, with_images: true) } - let!(:policy6) { create_expiration_policy(project_id: 6, enabled: false) } - let!(:policy7) { create_expiration_policy(project_id: 7, enabled: true) } - let!(:policy8) { create_expiration_policy(project_id: 8, enabled: true, with_images: true) } - let!(:policy9) { create_expiration_policy(project_id: 9, enabled: true) } - - describe '#perform' do - subject { described_class.new.perform(from_id, to_id) } - - shared_examples 'disabling policies with no images' do - it 'disables the proper policies' do - subject - - rows = container_expiration_policies.order(:project_id).to_h do |row| - [row.project_id, row.enabled] - end - expect(rows).to eq(expected_rows) - end - end - - context 'the whole range' do - let(:from_id) { 1 } - let(:to_id) { 9 } - - it_behaves_like 'disabling policies with no images' do - let(:expected_rows) do - { - 1 => false, - 2 => false, - 3 => false, - 4 => true, - 5 => false, - 6 => false, - 7 => false, - 8 => true, - 9 => false - } - end - end - end - - context 'a range with no policies to disable' do - let(:from_id) { 2 } - let(:to_id) { 6 } - - it_behaves_like 'disabling policies with no images' do - let(:expected_rows) do - { - 1 => true, - 2 => false, - 3 => false, - 4 => true, - 5 => false, - 6 => false, - 7 => true, - 8 => true, - 9 => true - } - end - end - end - - context 'a range with only images' do - let(:from_id) { 4 } - let(:to_id) { 5 } - - it_behaves_like 'disabling policies with no images' do - let(:expected_rows) do - { - 1 => true, - 2 => false, - 3 => false, - 4 => true, - 5 => false, - 6 => false, - 7 => true, - 8 => true, - 9 => true - } - end - end - end - - context 'a range with a single element' do - let(:from_id) { 9 } - let(:to_id) { 9 } - - it_behaves_like 'disabling policies with no images' do - let(:expected_rows) do - { - 1 => true, - 2 => false, - 3 => false, - 4 => true, - 5 => false, - 6 => false, - 7 => true, - 8 => true, - 9 => false - } - end - end - end - end - - def create_expiration_policy(project_id:, enabled:, with_images: false) - projects.create!(id: project_id, namespace_id: namespace.id, name: "gitlab-#{project_id}") - - if with_images - container_repositories.create!(project_id: project_id, name: "image-#{project_id}") - end - - container_expiration_policies.create!( - enabled: enabled, - project_id: project_id - ) - end - - def enabled_policies - container_expiration_policies.where(enabled: true) - end - - def disabled_policies - container_expiration_policies.where(enabled: false) - end -end diff --git a/spec/lib/gitlab/background_migration/drop_invalid_security_findings_spec.rb b/spec/lib/gitlab/background_migration/drop_invalid_security_findings_spec.rb deleted file mode 100644 index 5fdd8683d06..00000000000 --- a/spec/lib/gitlab/background_migration/drop_invalid_security_findings_spec.rb +++ /dev/null @@ -1,57 +0,0 @@ -# frozen_string_literal: true -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::DropInvalidSecurityFindings, :suppress_gitlab_schemas_validate_connection, - schema: 20211108211434 do - let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user', type: Namespaces::UserNamespace.sti_name) } - let(:project) { table(:projects).create!(namespace_id: namespace.id) } - - let(:pipelines) { table(:ci_pipelines) } - let!(:pipeline) { pipelines.create!(project_id: project.id) } - - let(:ci_builds) { table(:ci_builds) } - let!(:ci_build) { ci_builds.create! } - - let(:security_scans) { table(:security_scans) } - let!(:security_scan) do - security_scans.create!( - scan_type: 1, - status: 1, - build_id: ci_build.id, - project_id: project.id, - pipeline_id: pipeline.id - ) - end - - let(:vulnerability_scanners) { table(:vulnerability_scanners) } - let!(:vulnerability_scanner) { vulnerability_scanners.create!(project_id: project.id, external_id: 'test 1', name: 'test scanner 1') } - - let(:security_findings) { table(:security_findings) } - let!(:security_finding_without_uuid) do - security_findings.create!( - severity: 1, - confidence: 1, - scan_id: security_scan.id, - scanner_id: vulnerability_scanner.id, - uuid: nil - ) - end - - let!(:security_finding_with_uuid) do - security_findings.create!( - severity: 1, - confidence: 1, - scan_id: security_scan.id, - scanner_id: vulnerability_scanner.id, - uuid: 'bd95c085-71aa-51d7-9bb6-08ae669c262e' - ) - end - - let(:sub_batch_size) { 10_000 } - - subject { described_class.new.perform(security_finding_without_uuid.id, security_finding_with_uuid.id, sub_batch_size) } - - it 'drops Security::Finding objects with no UUID' do - expect { subject }.to change(security_findings, :count).from(2).to(1) - end -end diff --git a/spec/lib/gitlab/background_migration/drop_invalid_vulnerabilities_spec.rb b/spec/lib/gitlab/background_migration/drop_invalid_vulnerabilities_spec.rb deleted file mode 100644 index 8f3ef44e00c..00000000000 --- a/spec/lib/gitlab/background_migration/drop_invalid_vulnerabilities_spec.rb +++ /dev/null @@ -1,126 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::DropInvalidVulnerabilities, schema: 20210826171758 do - let!(:background_migration_jobs) { table(:background_migration_jobs) } - let!(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') } - let!(:users) { table(:users) } - let!(:user) { create_user! } - let!(:project) { table(:projects).create!(id: 123, namespace_id: namespace.id) } - - let!(:scanners) { table(:vulnerability_scanners) } - let!(:scanner) { scanners.create!(project_id: project.id, external_id: 'test 1', name: 'test scanner 1') } - let!(:different_scanner) { scanners.create!(project_id: project.id, external_id: 'test 2', name: 'test scanner 2') } - - let!(:vulnerabilities) { table(:vulnerabilities) } - let!(:vulnerability_with_finding) do - create_vulnerability!( - project_id: project.id, - author_id: user.id - ) - end - - let!(:vulnerability_without_finding) do - create_vulnerability!( - project_id: project.id, - author_id: user.id - ) - end - - let!(:vulnerability_identifiers) { table(:vulnerability_identifiers) } - let!(:primary_identifier) do - vulnerability_identifiers.create!( - project_id: project.id, - external_type: 'uuid-v5', - external_id: 'uuid-v5', - fingerprint: '7e394d1b1eb461a7406d7b1e08f057a1cf11287a', - name: 'Identifier for UUIDv5') - end - - let!(:vulnerabilities_findings) { table(:vulnerability_occurrences) } - let!(:finding) do - create_finding!( - vulnerability_id: vulnerability_with_finding.id, - project_id: project.id, - scanner_id: scanner.id, - primary_identifier_id: primary_identifier.id - ) - end - - let(:succeeded_status) { 1 } - let(:pending_status) { 0 } - - it 'drops Vulnerabilities without any Findings' do - expect(vulnerabilities.pluck(:id)).to eq([vulnerability_with_finding.id, vulnerability_without_finding.id]) - - expect { subject.perform(vulnerability_with_finding.id, vulnerability_without_finding.id) }.to change(vulnerabilities, :count).by(-1) - - expect(vulnerabilities.pluck(:id)).to eq([vulnerability_with_finding.id]) - end - - it 'marks jobs as done' do - background_migration_jobs.create!( - class_name: 'DropInvalidVulnerabilities', - arguments: [vulnerability_with_finding.id, vulnerability_with_finding.id] - ) - - background_migration_jobs.create!( - class_name: 'DropInvalidVulnerabilities', - arguments: [vulnerability_without_finding.id, vulnerability_without_finding.id] - ) - - subject.perform(vulnerability_with_finding.id, vulnerability_with_finding.id) - - expect(background_migration_jobs.first.status).to eq(succeeded_status) - expect(background_migration_jobs.second.status).to eq(pending_status) - end - - private - - def create_vulnerability!(project_id:, author_id:, title: 'test', severity: 7, confidence: 7, report_type: 0) - vulnerabilities.create!( - project_id: project_id, - author_id: author_id, - title: title, - severity: severity, - confidence: confidence, - report_type: report_type - ) - end - - # rubocop:disable Metrics/ParameterLists - def create_finding!( - vulnerability_id:, project_id:, scanner_id:, primary_identifier_id:, - name: "test", severity: 7, confidence: 7, report_type: 0, - project_fingerprint: '123qweasdzxc', location_fingerprint: 'test', - metadata_version: 'test', raw_metadata: 'test', uuid: SecureRandom.uuid) - vulnerabilities_findings.create!( - vulnerability_id: vulnerability_id, - project_id: project_id, - name: name, - severity: severity, - confidence: confidence, - report_type: report_type, - project_fingerprint: project_fingerprint, - scanner_id: scanner_id, - primary_identifier_id: primary_identifier_id, - location_fingerprint: location_fingerprint, - metadata_version: metadata_version, - raw_metadata: raw_metadata, - uuid: uuid - ) - end - # rubocop:enable Metrics/ParameterLists - - def create_user!(name: "Example User", email: "user@example.com", user_type: nil) - users.create!( - name: name, - email: email, - username: name, - projects_limit: 0, - user_type: user_type, - confirmed_at: Time.current - ) - end -end diff --git a/spec/lib/gitlab/background_migration/encrypt_ci_trigger_token_spec.rb b/spec/lib/gitlab/background_migration/encrypt_ci_trigger_token_spec.rb index b52f30a5e21..dd3e7877f8a 100644 --- a/spec/lib/gitlab/background_migration/encrypt_ci_trigger_token_spec.rb +++ b/spec/lib/gitlab/background_migration/encrypt_ci_trigger_token_spec.rb @@ -10,8 +10,7 @@ RSpec.describe Gitlab::BackgroundMigration::EncryptCiTriggerToken, feature_categ mode: :per_attribute_iv, key: ::Settings.attr_encrypted_db_key_base_32, algorithm: 'aes-256-gcm', - encode: false, - encode_iv: false + encode: false end end @@ -52,6 +51,7 @@ RSpec.describe Gitlab::BackgroundMigration::EncryptCiTriggerToken, feature_categ already_encrypted_token = Ci::Trigger.find(with_encryption.id) expect(already_encrypted_token.encrypted_token).to eq(with_encryption.encrypted_token) expect(already_encrypted_token.encrypted_token_iv).to eq(with_encryption.encrypted_token_iv) + expect(already_encrypted_token.token).to eq(already_encrypted_token.encrypted_token_tmp) expect(with_encryption.token).to eq(with_encryption.encrypted_token_tmp) end end diff --git a/spec/lib/gitlab/background_migration/encrypt_integration_properties_spec.rb b/spec/lib/gitlab/background_migration/encrypt_integration_properties_spec.rb deleted file mode 100644 index c788b701d79..00000000000 --- a/spec/lib/gitlab/background_migration/encrypt_integration_properties_spec.rb +++ /dev/null @@ -1,63 +0,0 @@ -# frozen_string_literal: true -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::EncryptIntegrationProperties, schema: 20220415124804 do - let(:integrations) do - table(:integrations) do |integrations| - integrations.send :attr_encrypted, :encrypted_properties_tmp, - attribute: :encrypted_properties, - mode: :per_attribute_iv, - key: ::Settings.attr_encrypted_db_key_base_32, - algorithm: 'aes-256-gcm', - marshal: true, - marshaler: ::Gitlab::Json, - encode: false, - encode_iv: false - end - end - - let!(:no_properties) { integrations.create! } - let!(:with_plaintext_1) { integrations.create!(properties: json_props(1)) } - let!(:with_plaintext_2) { integrations.create!(properties: json_props(2)) } - let!(:with_encrypted) do - x = integrations.new - x.properties = nil - x.encrypted_properties_tmp = some_props(3) - x.save! - x - end - - let(:start_id) { integrations.minimum(:id) } - let(:end_id) { integrations.maximum(:id) } - - it 'ensures all properties are encrypted', :aggregate_failures do - described_class.new.perform(start_id, end_id) - - props = integrations.all.to_h do |record| - [record.id, [Gitlab::Json.parse(record.properties), record.encrypted_properties_tmp]] - end - - expect(integrations.count).to eq(4) - - expect(props).to match( - no_properties.id => both(be_nil), - with_plaintext_1.id => both(eq some_props(1)), - with_plaintext_2.id => both(eq some_props(2)), - with_encrypted.id => match([be_nil, eq(some_props(3))]) - ) - end - - private - - def both(obj) - match [obj, obj] - end - - def some_props(id) - HashWithIndifferentAccess.new({ id: id, foo: 1, bar: true, baz: %w[a string array] }) - end - - def json_props(id) - some_props(id).to_json - end -end diff --git a/spec/lib/gitlab/background_migration/encrypt_static_object_token_spec.rb b/spec/lib/gitlab/background_migration/encrypt_static_object_token_spec.rb deleted file mode 100644 index 4e7b97d33f6..00000000000 --- a/spec/lib/gitlab/background_migration/encrypt_static_object_token_spec.rb +++ /dev/null @@ -1,64 +0,0 @@ -# frozen_string_literal: true -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::EncryptStaticObjectToken do - let(:users) { table(:users) } - let!(:user_without_tokens) { create_user!(name: 'notoken') } - let!(:user_with_plaintext_token_1) { create_user!(name: 'plaintext_1', token: 'token') } - let!(:user_with_plaintext_token_2) { create_user!(name: 'plaintext_2', token: 'TOKEN') } - let!(:user_with_plaintext_empty_token) { create_user!(name: 'plaintext_3', token: '') } - let!(:user_with_encrypted_token) { create_user!(name: 'encrypted', encrypted_token: 'encrypted') } - let!(:user_with_both_tokens) { create_user!(name: 'both', token: 'token2', encrypted_token: 'encrypted2') } - - before do - allow(Gitlab::CryptoHelper).to receive(:aes256_gcm_encrypt).and_call_original - allow(Gitlab::CryptoHelper).to receive(:aes256_gcm_encrypt).with('token') { 'secure_token' } - allow(Gitlab::CryptoHelper).to receive(:aes256_gcm_encrypt).with('TOKEN') { 'SECURE_TOKEN' } - end - - subject { described_class.new.perform(start_id, end_id) } - - let(:start_id) { users.minimum(:id) } - let(:end_id) { users.maximum(:id) } - - it 'backfills encrypted tokens to users with plaintext token only', :aggregate_failures do - subject - - new_state = users.pluck(:id, :static_object_token, :static_object_token_encrypted).to_h do |row| - [row[0], [row[1], row[2]]] - end - - expect(new_state.count).to eq(6) - - expect(new_state[user_with_plaintext_token_1.id]).to match_array(%w[token secure_token]) - expect(new_state[user_with_plaintext_token_2.id]).to match_array(%w[TOKEN SECURE_TOKEN]) - - expect(new_state[user_with_plaintext_empty_token.id]).to match_array(['', nil]) - expect(new_state[user_without_tokens.id]).to match_array([nil, nil]) - expect(new_state[user_with_both_tokens.id]).to match_array(%w[token2 encrypted2]) - expect(new_state[user_with_encrypted_token.id]).to match_array([nil, 'encrypted']) - end - - context 'when id range does not include existing user ids' do - let(:arguments) { [non_existing_record_id, non_existing_record_id.succ] } - - it_behaves_like 'marks background migration job records' do - subject { described_class.new } - end - end - - private - - def create_user!(name:, token: nil, encrypted_token: nil) - email = "#{name}@example.com" - - table(:users).create!( - name: name, - email: email, - username: name, - projects_limit: 0, - static_object_token: token, - static_object_token_encrypted: encrypted_token - ) - end -end diff --git a/spec/lib/gitlab/background_migration/extract_project_topics_into_separate_table_spec.rb b/spec/lib/gitlab/background_migration/extract_project_topics_into_separate_table_spec.rb deleted file mode 100644 index 586e75ffb37..00000000000 --- a/spec/lib/gitlab/background_migration/extract_project_topics_into_separate_table_spec.rb +++ /dev/null @@ -1,46 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::ExtractProjectTopicsIntoSeparateTable, - :suppress_gitlab_schemas_validate_connection, schema: 20210826171758 do - it 'correctly extracts project topics into separate table' do - namespaces = table(:namespaces) - projects = table(:projects) - taggings = table(:taggings) - tags = table(:tags) - project_topics = table(:project_topics) - topics = table(:topics) - - namespace = namespaces.create!(name: 'foo', path: 'foo') - project = projects.create!(namespace_id: namespace.id) - tag_1 = tags.create!(name: 'Topic1') - tag_2 = tags.create!(name: 'Topic2') - tag_3 = tags.create!(name: 'Topic3') - topic_3 = topics.create!(name: 'Topic3') - tagging_1 = taggings.create!(taggable_type: 'Project', taggable_id: project.id, context: 'topics', tag_id: tag_1.id) - tagging_2 = taggings.create!(taggable_type: 'Project', taggable_id: project.id, context: 'topics', tag_id: tag_2.id) - other_tagging = taggings.create!(taggable_type: 'Other', taggable_id: project.id, context: 'topics', tag_id: tag_1.id) - tagging_3 = taggings.create!(taggable_type: 'Project', taggable_id: project.id, context: 'topics', tag_id: tag_3.id) - tagging_4 = taggings.create!(taggable_type: 'Project', taggable_id: -1, context: 'topics', tag_id: tag_1.id) - tagging_5 = taggings.create!(taggable_type: 'Project', taggable_id: project.id, context: 'topics', tag_id: -1) - - subject.perform(tagging_1.id, tagging_5.id) - - # Tagging records - expect { tagging_1.reload }.to raise_error(ActiveRecord::RecordNotFound) - expect { tagging_2.reload }.to raise_error(ActiveRecord::RecordNotFound) - expect { other_tagging.reload }.not_to raise_error - expect { tagging_3.reload }.to raise_error(ActiveRecord::RecordNotFound) - expect { tagging_4.reload }.to raise_error(ActiveRecord::RecordNotFound) - expect { tagging_5.reload }.to raise_error(ActiveRecord::RecordNotFound) - - # Topic records - topic_1 = topics.find_by(name: 'Topic1') - topic_2 = topics.find_by(name: 'Topic2') - expect(topics.all).to contain_exactly(topic_1, topic_2, topic_3) - - # ProjectTopic records - expect(project_topics.all.map(&:topic_id)).to contain_exactly(topic_1.id, topic_2.id, topic_3.id) - end -end diff --git a/spec/lib/gitlab/background_migration/fix_first_mentioned_in_commit_at_spec.rb b/spec/lib/gitlab/background_migration/fix_first_mentioned_in_commit_at_spec.rb deleted file mode 100644 index 7f15aceca42..00000000000 --- a/spec/lib/gitlab/background_migration/fix_first_mentioned_in_commit_at_spec.rb +++ /dev/null @@ -1,166 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require Rails.root.join('db', 'post_migrate', '20211004110500_add_temporary_index_to_issue_metrics.rb') - -RSpec.describe Gitlab::BackgroundMigration::FixFirstMentionedInCommitAt, :migration, schema: 20211004110500 do - let(:namespaces) { table(:namespaces) } - let(:projects) { table(:projects) } - let(:users) { table(:users) } - let(:merge_requests) { table(:merge_requests) } - let(:issues) { table(:issues) } - let(:issue_metrics) { table(:issue_metrics) } - let(:merge_requests_closing_issues) { table(:merge_requests_closing_issues) } - let(:diffs) { table(:merge_request_diffs) } - let(:ten_days_ago) { 10.days.ago } - let(:commits) do - table(:merge_request_diff_commits).tap do |t| - t.extend(SuppressCompositePrimaryKeyWarning) - end - end - - let(:namespace) { namespaces.create!(name: 'ns', path: 'ns') } - let(:project) { projects.create!(namespace_id: namespace.id) } - - let!(:issue1) do - issues.create!( - title: 'issue', - description: 'description', - project_id: project.id - ) - end - - let!(:issue2) do - issues.create!( - title: 'issue', - description: 'description', - project_id: project.id - ) - end - - let!(:merge_request1) do - merge_requests.create!( - source_branch: 'a', - target_branch: 'master', - target_project_id: project.id - ) - end - - let!(:merge_request2) do - merge_requests.create!( - source_branch: 'b', - target_branch: 'master', - target_project_id: project.id - ) - end - - let!(:merge_request_closing_issue1) do - merge_requests_closing_issues.create!(issue_id: issue1.id, merge_request_id: merge_request1.id) - end - - let!(:merge_request_closing_issue2) do - merge_requests_closing_issues.create!(issue_id: issue2.id, merge_request_id: merge_request2.id) - end - - let!(:diff1) { diffs.create!(merge_request_id: merge_request1.id) } - let!(:diff2) { diffs.create!(merge_request_id: merge_request1.id) } - - let!(:other_diff) { diffs.create!(merge_request_id: merge_request2.id) } - - let!(:commit1) do - commits.create!( - merge_request_diff_id: diff2.id, - relative_order: 0, - sha: Gitlab::Database::ShaAttribute.serialize('aaa'), - authored_date: 5.days.ago - ) - end - - let!(:commit2) do - commits.create!( - merge_request_diff_id: diff2.id, - relative_order: 1, - sha: Gitlab::Database::ShaAttribute.serialize('aaa'), - authored_date: 10.days.ago - ) - end - - let!(:commit3) do - commits.create!( - merge_request_diff_id: other_diff.id, - relative_order: 1, - sha: Gitlab::Database::ShaAttribute.serialize('aaa'), - authored_date: 5.days.ago - ) - end - - def run_migration - described_class - .new - .perform(issue_metrics.minimum(:issue_id), issue_metrics.maximum(:issue_id)) - end - - shared_examples 'fixes first_mentioned_in_commit_at' do - it "marks successful slices as completed" do - min_issue_id = issue_metrics.minimum(:issue_id) - max_issue_id = issue_metrics.maximum(:issue_id) - - expect(subject).to receive(:mark_job_as_succeeded).with(min_issue_id, max_issue_id) - - subject.perform(min_issue_id, max_issue_id) - end - - context 'when the persisted first_mentioned_in_commit_at is later than the first commit authored_date' do - it 'updates the issue_metrics record' do - record1 = issue_metrics.create!(issue_id: issue1.id, first_mentioned_in_commit_at: Time.current) - record2 = issue_metrics.create!(issue_id: issue2.id, first_mentioned_in_commit_at: Time.current) - - run_migration - record1.reload - record2.reload - - expect(record1.first_mentioned_in_commit_at).to be_within(2.seconds).of(commit2.authored_date) - expect(record2.first_mentioned_in_commit_at).to be_within(2.seconds).of(commit3.authored_date) - end - end - - context 'when the persisted first_mentioned_in_commit_at is earlier than the first commit authored_date' do - it 'does not update the issue_metrics record' do - record = issue_metrics.create!(issue_id: issue1.id, first_mentioned_in_commit_at: 20.days.ago) - - expect { run_migration }.not_to change { record.reload.first_mentioned_in_commit_at } - end - end - - context 'when the first_mentioned_in_commit_at is null' do - it 'does nothing' do - record = issue_metrics.create!(issue_id: issue1.id, first_mentioned_in_commit_at: nil) - - expect { run_migration }.not_to change { record.reload.first_mentioned_in_commit_at } - end - end - end - - describe 'running the migration when first_mentioned_in_commit_at is timestamp without time zone' do - it_behaves_like 'fixes first_mentioned_in_commit_at' - end - - describe 'running the migration when first_mentioned_in_commit_at is timestamp with time zone' do - around do |example| - AddTemporaryIndexToIssueMetrics.new.down - - ActiveRecord::Base.connection.execute "ALTER TABLE issue_metrics ALTER first_mentioned_in_commit_at type timestamp with time zone" - Gitlab::BackgroundMigration::FixFirstMentionedInCommitAt::TmpIssueMetrics.reset_column_information - AddTemporaryIndexToIssueMetrics.new.up - - example.run - - AddTemporaryIndexToIssueMetrics.new.down - ActiveRecord::Base.connection.execute "ALTER TABLE issue_metrics ALTER first_mentioned_in_commit_at type timestamp without time zone" - Gitlab::BackgroundMigration::FixFirstMentionedInCommitAt::TmpIssueMetrics.reset_column_information - AddTemporaryIndexToIssueMetrics.new.up - end - - it_behaves_like 'fixes first_mentioned_in_commit_at' - end -end diff --git a/spec/lib/gitlab/background_migration/fix_merge_request_diff_commit_users_spec.rb b/spec/lib/gitlab/background_migration/fix_merge_request_diff_commit_users_spec.rb deleted file mode 100644 index 99df21562b0..00000000000 --- a/spec/lib/gitlab/background_migration/fix_merge_request_diff_commit_users_spec.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -# rubocop: disable RSpec/FactoriesInMigrationSpecs -RSpec.describe Gitlab::BackgroundMigration::FixMergeRequestDiffCommitUsers do - let(:migration) { described_class.new } - - describe '#perform' do - context 'when the project exists' do - it 'does nothing' do - project = create(:project) - - expect { migration.perform(project.id) }.not_to raise_error - end - end - - context 'when the project does not exist' do - it 'does nothing' do - expect { migration.perform(-1) }.not_to raise_error - end - end - end -end -# rubocop: enable RSpec/FactoriesInMigrationSpecs diff --git a/spec/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata_spec.rb b/spec/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata_spec.rb index af551861d47..3cbc05b762a 100644 --- a/spec/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata_spec.rb +++ b/spec/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::FixVulnerabilityOccurrencesWithHashesAsRawMetadata, schema: 20211209203821 do +RSpec.describe Gitlab::BackgroundMigration::FixVulnerabilityOccurrencesWithHashesAsRawMetadata, schema: 20220314184009 do let(:users) { table(:users) } let(:namespaces) { table(:namespaces) } let(:projects) { table(:projects) } diff --git a/spec/lib/gitlab/background_migration/merge_topics_with_same_name_spec.rb b/spec/lib/gitlab/background_migration/merge_topics_with_same_name_spec.rb deleted file mode 100644 index 2c2c048992f..00000000000 --- a/spec/lib/gitlab/background_migration/merge_topics_with_same_name_spec.rb +++ /dev/null @@ -1,148 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::MergeTopicsWithSameName, schema: 20220331133802 do - def set_avatar(topic_id, avatar) - topic = ::Projects::Topic.find(topic_id) - topic.avatar = avatar - topic.save! - topic.avatar.absolute_path - end - - it 'merges project topics with same case insensitive name' do - namespaces = table(:namespaces) - projects = table(:projects) - topics = table(:topics) - project_topics = table(:project_topics) - - group_1 = namespaces.create!(name: 'space1', type: 'Group', path: 'space1') - group_2 = namespaces.create!(name: 'space2', type: 'Group', path: 'space2') - group_3 = namespaces.create!(name: 'space3', type: 'Group', path: 'space3') - proj_space_1 = namespaces.create!(name: 'proj1', path: 'proj1', type: 'Project', parent_id: group_1.id) - proj_space_2 = namespaces.create!(name: 'proj2', path: 'proj2', type: 'Project', parent_id: group_2.id) - proj_space_3 = namespaces.create!(name: 'proj3', path: 'proj3', type: 'Project', parent_id: group_3.id) - project_1 = projects.create!(namespace_id: group_1.id, project_namespace_id: proj_space_1.id, visibility_level: 20) - project_2 = projects.create!(namespace_id: group_2.id, project_namespace_id: proj_space_2.id, visibility_level: 10) - project_3 = projects.create!(namespace_id: group_3.id, project_namespace_id: proj_space_3.id, visibility_level: 0) - topic_1_keep = topics.create!( - name: 'topic1', - title: 'Topic 1', - description: 'description 1 to keep', - total_projects_count: 2, - non_private_projects_count: 2 - ) - topic_1_remove = topics.create!( - name: 'TOPIC1', - title: 'Topic 1', - description: 'description 1 to remove', - total_projects_count: 2, - non_private_projects_count: 1 - ) - topic_2_remove = topics.create!( - name: 'topic2', - title: 'Topic 2', - total_projects_count: 0 - ) - topic_2_keep = topics.create!( - name: 'TOPIC2', - title: 'Topic 2', - description: 'description 2 to keep', - total_projects_count: 1 - ) - topic_3_remove_1 = topics.create!( - name: 'topic3', - title: 'Topic 3', - total_projects_count: 2, - non_private_projects_count: 1 - ) - topic_3_keep = topics.create!( - name: 'Topic3', - title: 'Topic 3', - total_projects_count: 2, - non_private_projects_count: 2 - ) - topic_3_remove_2 = topics.create!( - name: 'TOPIC3', - title: 'Topic 3', - description: 'description 3 to keep', - total_projects_count: 2, - non_private_projects_count: 1 - ) - topic_4_keep = topics.create!( - name: 'topic4', - title: 'Topic 4' - ) - - project_topics_1 = [] - project_topics_3 = [] - project_topics_removed = [] - - project_topics_1 << project_topics.create!(topic_id: topic_1_keep.id, project_id: project_1.id) - project_topics_1 << project_topics.create!(topic_id: topic_1_keep.id, project_id: project_2.id) - project_topics_removed << project_topics.create!(topic_id: topic_1_remove.id, project_id: project_2.id) - project_topics_1 << project_topics.create!(topic_id: topic_1_remove.id, project_id: project_3.id) - - project_topics_3 << project_topics.create!(topic_id: topic_3_keep.id, project_id: project_1.id) - project_topics_3 << project_topics.create!(topic_id: topic_3_keep.id, project_id: project_2.id) - project_topics_removed << project_topics.create!(topic_id: topic_3_remove_1.id, project_id: project_1.id) - project_topics_3 << project_topics.create!(topic_id: topic_3_remove_1.id, project_id: project_3.id) - project_topics_removed << project_topics.create!(topic_id: topic_3_remove_2.id, project_id: project_1.id) - project_topics_removed << project_topics.create!(topic_id: topic_3_remove_2.id, project_id: project_3.id) - - avatar_paths = { - topic_1_keep: set_avatar(topic_1_keep.id, fixture_file_upload('spec/fixtures/avatars/avatar1.png')), - topic_1_remove: set_avatar(topic_1_remove.id, fixture_file_upload('spec/fixtures/avatars/avatar2.png')), - topic_2_remove: set_avatar(topic_2_remove.id, fixture_file_upload('spec/fixtures/avatars/avatar3.png')), - topic_3_remove_1: set_avatar(topic_3_remove_1.id, fixture_file_upload('spec/fixtures/avatars/avatar4.png')), - topic_3_remove_2: set_avatar(topic_3_remove_2.id, fixture_file_upload('spec/fixtures/avatars/avatar5.png')) - } - - subject.perform(%w[topic1 topic2 topic3 topic4]) - - # Topics - [topic_1_keep, topic_2_keep, topic_3_keep, topic_4_keep].each(&:reload) - expect(topic_1_keep.name).to eq('topic1') - expect(topic_1_keep.description).to eq('description 1 to keep') - expect(topic_1_keep.total_projects_count).to eq(3) - expect(topic_1_keep.non_private_projects_count).to eq(2) - expect(topic_2_keep.name).to eq('TOPIC2') - expect(topic_2_keep.description).to eq('description 2 to keep') - expect(topic_2_keep.total_projects_count).to eq(0) - expect(topic_2_keep.non_private_projects_count).to eq(0) - expect(topic_3_keep.name).to eq('Topic3') - expect(topic_3_keep.description).to eq('description 3 to keep') - expect(topic_3_keep.total_projects_count).to eq(3) - expect(topic_3_keep.non_private_projects_count).to eq(2) - expect(topic_4_keep.reload.name).to eq('topic4') - - [topic_1_remove, topic_2_remove, topic_3_remove_1, topic_3_remove_2].each do |topic| - expect { topic.reload }.to raise_error(ActiveRecord::RecordNotFound) - end - - # Topic avatars - expect(topic_1_keep.avatar).to eq('avatar1.png') - expect(File.exist?(::Projects::Topic.find(topic_1_keep.id).avatar.absolute_path)).to be_truthy - expect(topic_2_keep.avatar).to eq('avatar3.png') - expect(File.exist?(::Projects::Topic.find(topic_2_keep.id).avatar.absolute_path)).to be_truthy - expect(topic_3_keep.avatar).to eq('avatar4.png') - expect(File.exist?(::Projects::Topic.find(topic_3_keep.id).avatar.absolute_path)).to be_truthy - - [:topic_1_remove, :topic_2_remove, :topic_3_remove_1, :topic_3_remove_2].each do |topic| - expect(File.exist?(avatar_paths[topic])).to be_falsey - end - - # Project Topic assignments - project_topics_1.each do |project_topic| - expect(project_topic.reload.topic_id).to eq(topic_1_keep.id) - end - - project_topics_3.each do |project_topic| - expect(project_topic.reload.topic_id).to eq(topic_3_keep.id) - end - - project_topics_removed.each do |project_topic| - expect { project_topic.reload }.to raise_error(ActiveRecord::RecordNotFound) - end - end -end diff --git a/spec/lib/gitlab/background_migration/migrate_evidences_for_vulnerability_findings_spec.rb b/spec/lib/gitlab/background_migration/migrate_evidences_for_vulnerability_findings_spec.rb index b70044ab2a4..28e16a5820d 100644 --- a/spec/lib/gitlab/background_migration/migrate_evidences_for_vulnerability_findings_spec.rb +++ b/spec/lib/gitlab/background_migration/migrate_evidences_for_vulnerability_findings_spec.rb @@ -38,8 +38,6 @@ RSpec.describe Gitlab::BackgroundMigration::MigrateEvidencesForVulnerabilityFind end it 'does not create any evidence' do - expect(Gitlab::AppLogger).not_to receive(:error) - expect { perform_migration }.not_to change { vulnerability_finding_evidences.count } end end @@ -50,8 +48,6 @@ RSpec.describe Gitlab::BackgroundMigration::MigrateEvidencesForVulnerabilityFind end it 'does not create any evidence' do - expect(Gitlab::AppLogger).not_to receive(:error) - expect { perform_migration }.not_to change { vulnerability_finding_evidences.count } end end @@ -61,32 +57,15 @@ RSpec.describe Gitlab::BackgroundMigration::MigrateEvidencesForVulnerabilityFind let!(:finding2) { create_finding!(project1.id, scanner1.id, { evidence: evidence_hash }) } it 'creates new evidence for each finding' do - expect(Gitlab::AppLogger).not_to receive(:error) - expect { perform_migration }.to change { vulnerability_finding_evidences.count }.by(2) end - context 'when create throws exception StandardError' do - before do - allow(migration).to receive(:create_evidences).and_raise(StandardError) - end - - it 'logs StandardError' do - expect(Gitlab::AppLogger).to receive(:error).with({ - class: described_class.name, message: StandardError.to_s - }) - expect { perform_migration }.not_to change { vulnerability_finding_evidences.count } - end - end - context 'when parse throws exception JSON::ParserError' do before do allow(Gitlab::Json).to receive(:parse).and_raise(JSON::ParserError) end - it 'does not log this error nor create new records' do - expect(Gitlab::AppLogger).not_to receive(:error) - + it 'does not create new records' do expect { perform_migration }.not_to change { vulnerability_finding_evidences.count } end end @@ -100,8 +79,6 @@ RSpec.describe Gitlab::BackgroundMigration::MigrateEvidencesForVulnerabilityFind end it 'does not create new evidence' do - expect(Gitlab::AppLogger).not_to receive(:error) - expect { perform_migration }.not_to change { vulnerability_finding_evidences.count } end @@ -109,8 +86,6 @@ RSpec.describe Gitlab::BackgroundMigration::MigrateEvidencesForVulnerabilityFind let!(:finding3) { create_finding!(project1.id, scanner1.id, { evidence: { url: 'http://secondary.com' } }) } it 'creates a new evidence only to the non-existing evidence' do - expect(Gitlab::AppLogger).not_to receive(:error) - expect { perform_migration }.to change { vulnerability_finding_evidences.count }.by(1) end end diff --git a/spec/lib/gitlab/background_migration/migrate_links_for_vulnerability_findings_spec.rb b/spec/lib/gitlab/background_migration/migrate_links_for_vulnerability_findings_spec.rb index fd2e3ffb670..9a90af968e2 100644 --- a/spec/lib/gitlab/background_migration/migrate_links_for_vulnerability_findings_spec.rb +++ b/spec/lib/gitlab/background_migration/migrate_links_for_vulnerability_findings_spec.rb @@ -56,6 +56,64 @@ RSpec.describe Gitlab::BackgroundMigration::MigrateLinksForVulnerabilityFindings end end + context 'with links equals to a string' do + before do + create_finding!(project1.id, scanner1.id, { links: "wrong format" }) + end + + it 'does not create any link' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_finding_links.count } + end + end + + context 'with some elements which do not contain the key url' do + let!(:finding) do + create_finding!(project1.id, scanner1.id, { links: [link_hash, "wrong format", {}] }) + end + + it 'creates links only to valid elements' do + expect(Gitlab::AppLogger).not_to receive(:error) + + perform_migration + + expect(vulnerability_finding_links.all).to contain_exactly(have_attributes( + url: link_hash[:url], + vulnerability_occurrence_id: finding.id)) + end + end + + context 'when link name is too long' do + let!(:finding) do + create_finding!(project1.id, scanner1.id, { links: [{ name: 'A' * 300, url: 'https://foo' }] }) + end + + it 'skips creation of link and logs error' do + expect(Gitlab::AppLogger).to receive(:error).with({ + class: described_class.name, + message: /check_55f0a95439/, + model_id: finding.id + }) + expect { perform_migration }.not_to change { vulnerability_finding_links.count } + end + end + + context 'when link url is too long' do + let!(:finding) do + create_finding!(project1.id, scanner1.id, { links: [{ url: "https://f#{'o' * 2050}" }] }) + end + + it 'skips creation of link and logs error' do + expect(Gitlab::AppLogger).to receive(:error).with({ + class: described_class.name, + message: /check_b7fe886df6/, + model_id: finding.id + }) + expect { perform_migration }.not_to change { vulnerability_finding_links.count } + end + end + context 'with links equals to an array of duplicated elements' do let!(:finding) do create_finding!(project1.id, scanner1.id, { links: [link_hash, link_hash] }) @@ -64,7 +122,11 @@ RSpec.describe Gitlab::BackgroundMigration::MigrateLinksForVulnerabilityFindings it 'creates one new link' do expect(Gitlab::AppLogger).not_to receive(:error) - expect { perform_migration }.to change { vulnerability_finding_links.count }.by(1) + perform_migration + + expect(vulnerability_finding_links.all).to contain_exactly(have_attributes( + url: link_hash[:url], + vulnerability_occurrence_id: finding.id)) end end diff --git a/spec/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users_spec.rb b/spec/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users_spec.rb deleted file mode 100644 index c3ae2cc060c..00000000000 --- a/spec/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users_spec.rb +++ /dev/null @@ -1,413 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::MigrateMergeRequestDiffCommitUsers, schema: 20211012134316 do - let(:namespaces) { table(:namespaces) } - let(:projects) { table(:projects) } - let(:users) { table(:users) } - let(:merge_requests) { table(:merge_requests) } - let(:diffs) { table(:merge_request_diffs) } - let(:commits) do - table(:merge_request_diff_commits).tap do |t| - t.extend(SuppressCompositePrimaryKeyWarning) - end - end - - let(:commit_users) { described_class::MergeRequestDiffCommitUser } - - let(:namespace) { namespaces.create!(name: 'foo', path: 'foo') } - let(:project) { projects.create!(namespace_id: namespace.id) } - let(:merge_request) do - merge_requests.create!( - source_branch: 'x', - target_branch: 'master', - target_project_id: project.id - ) - end - - let(:diff) { diffs.create!(merge_request_id: merge_request.id) } - let(:migration) { described_class.new } - - describe 'MergeRequestDiffCommit' do - describe '.each_row_to_migrate' do - it 'yields the rows to migrate for a given range' do - commit1 = commits.create!( - merge_request_diff_id: diff.id, - relative_order: 0, - sha: Gitlab::Database::ShaAttribute.serialize('123abc'), - author_name: 'bob', - author_email: 'bob@example.com', - committer_name: 'bob', - committer_email: 'bob@example.com' - ) - - commit2 = commits.create!( - merge_request_diff_id: diff.id, - relative_order: 1, - sha: Gitlab::Database::ShaAttribute.serialize('123abc'), - author_name: 'Alice', - author_email: 'alice@example.com', - committer_name: 'Alice', - committer_email: 'alice@example.com' - ) - - # We stub this constant to make sure we run at least two pagination - # queries for getting the data. This way we can test if the pagination - # is actually working properly. - stub_const( - 'Gitlab::BackgroundMigration::MigrateMergeRequestDiffCommitUsers::COMMIT_ROWS_PER_QUERY', - 1 - ) - - rows = [] - - described_class::MergeRequestDiffCommit.each_row_to_migrate(diff.id, diff.id + 1) do |row| - rows << row - end - - expect(rows.length).to eq(2) - - expect(rows[0].author_name).to eq(commit1.author_name) - expect(rows[1].author_name).to eq(commit2.author_name) - end - end - end - - describe 'MergeRequestDiffCommitUser' do - describe '.union' do - it 'produces a union of the given queries' do - alice = commit_users.create!(name: 'Alice', email: 'alice@example.com') - bob = commit_users.create!(name: 'Bob', email: 'bob@example.com') - users = commit_users.union( - [ - commit_users.where(name: 'Alice').to_sql, - commit_users.where(name: 'Bob').to_sql - ]) - - expect(users).to include(alice) - expect(users).to include(bob) - end - end - end - - describe '#perform' do - it 'skips jobs that have already been completed' do - Gitlab::Database::BackgroundMigrationJob.create!( - class_name: 'MigrateMergeRequestDiffCommitUsers', - arguments: [1, 10], - status: :succeeded - ) - - expect(migration).not_to receive(:get_data_to_update) - - migration.perform(1, 10) - end - - it 'migrates the data in the range' do - commits.create!( - merge_request_diff_id: diff.id, - relative_order: 0, - sha: Gitlab::Database::ShaAttribute.serialize('123abc'), - author_name: 'bob', - author_email: 'bob@example.com', - committer_name: 'bob', - committer_email: 'bob@example.com' - ) - - migration.perform(diff.id, diff.id + 1) - - bob = commit_users.find_by(name: 'bob') - commit = commits.first - - expect(commit.commit_author_id).to eq(bob.id) - expect(commit.committer_id).to eq(bob.id) - end - - it 'treats empty names and Emails the same as NULL values' do - commits.create!( - merge_request_diff_id: diff.id, - relative_order: 0, - sha: Gitlab::Database::ShaAttribute.serialize('123abc'), - author_name: 'bob', - author_email: 'bob@example.com', - committer_name: '', - committer_email: '' - ) - - migration.perform(diff.id, diff.id + 1) - - bob = commit_users.find_by(name: 'bob') - commit = commits.first - - expect(commit.commit_author_id).to eq(bob.id) - expect(commit.committer_id).to be_nil - end - - it 'does not update rows without a committer and author' do - commits.create!( - merge_request_diff_id: diff.id, - relative_order: 0, - sha: Gitlab::Database::ShaAttribute.serialize('123abc') - ) - - migration.perform(diff.id, diff.id + 1) - - commit = commits.first - - expect(commit_users.count).to eq(0) - expect(commit.commit_author_id).to be_nil - expect(commit.committer_id).to be_nil - end - - it 'marks the background job as done' do - Gitlab::Database::BackgroundMigrationJob.create!( - class_name: 'MigrateMergeRequestDiffCommitUsers', - arguments: [diff.id, diff.id + 1] - ) - - migration.perform(diff.id, diff.id + 1) - - job = Gitlab::Database::BackgroundMigrationJob.first - - expect(job.status).to eq('succeeded') - end - end - - describe '#get_data_to_update' do - it 'returns the users and commit rows to update' do - commits.create!( - merge_request_diff_id: diff.id, - relative_order: 0, - sha: Gitlab::Database::ShaAttribute.serialize('123abc'), - author_name: 'bob' + ('a' * 510), - author_email: 'bob@example.com', - committer_name: 'bob' + ('a' * 510), - committer_email: 'bob@example.com' - ) - - commits.create!( - merge_request_diff_id: diff.id, - relative_order: 1, - sha: Gitlab::Database::ShaAttribute.serialize('456abc'), - author_name: 'alice', - author_email: 'alice@example.com', - committer_name: 'alice', - committer_email: 'alice@example.com' - ) - - users, to_update = migration.get_data_to_update(diff.id, diff.id + 1) - - bob_name = 'bob' + ('a' * 509) - - expect(users).to include(%w[alice alice@example.com]) - expect(users).to include([bob_name, 'bob@example.com']) - - expect(to_update[[diff.id, 0]]) - .to eq([[bob_name, 'bob@example.com'], [bob_name, 'bob@example.com']]) - - expect(to_update[[diff.id, 1]]) - .to eq([%w[alice alice@example.com], %w[alice alice@example.com]]) - end - - it 'does not include a user if both the name and Email are missing' do - commits.create!( - merge_request_diff_id: diff.id, - relative_order: 0, - sha: Gitlab::Database::ShaAttribute.serialize('123abc'), - author_name: nil, - author_email: nil, - committer_name: 'bob', - committer_email: 'bob@example.com' - ) - - users, _ = migration.get_data_to_update(diff.id, diff.id + 1) - - expect(users).to eq([%w[bob bob@example.com]].to_set) - end - end - - describe '#get_user_rows_in_batches' do - it 'retrieves all existing users' do - alice = commit_users.create!(name: 'alice', email: 'alice@example.com') - bob = commit_users.create!(name: 'bob', email: 'bob@example.com') - - users = [[alice.name, alice.email], [bob.name, bob.email]] - mapping = {} - - migration.get_user_rows_in_batches(users, mapping) - - expect(mapping[%w[alice alice@example.com]]).to eq(alice) - expect(mapping[%w[bob bob@example.com]]).to eq(bob) - end - end - - describe '#create_missing_users' do - it 'creates merge request diff commit users that are missing' do - alice = commit_users.create!(name: 'alice', email: 'alice@example.com') - users = [%w[alice alice@example.com], %w[bob bob@example.com]] - mapping = { %w[alice alice@example.com] => alice } - - migration.create_missing_users(users, mapping) - - expect(mapping[%w[alice alice@example.com]]).to eq(alice) - expect(mapping[%w[bob bob@example.com]].name).to eq('bob') - expect(mapping[%w[bob bob@example.com]].email).to eq('bob@example.com') - end - end - - describe '#update_commit_rows' do - it 'updates the merge request diff commit rows' do - to_update = { [42, 0] => [%w[alice alice@example.com], []] } - user_mapping = { %w[alice alice@example.com] => double(:user, id: 1) } - - expect(migration) - .to receive(:bulk_update_commit_rows) - .with({ [42, 0] => [1, nil] }) - - migration.update_commit_rows(to_update, user_mapping) - end - end - - describe '#bulk_update_commit_rows' do - context 'when there are no authors and committers' do - it 'does not update any rows' do - migration.bulk_update_commit_rows({ [1, 0] => [] }) - - expect(described_class::MergeRequestDiffCommit.connection) - .not_to receive(:execute) - end - end - - context 'when there are only authors' do - it 'only updates the author IDs' do - author = commit_users.create!(name: 'Alice', email: 'alice@example.com') - commit = commits.create!( - merge_request_diff_id: diff.id, - relative_order: 0, - sha: Gitlab::Database::ShaAttribute.serialize('123abc') - ) - - mapping = { - [commit.merge_request_diff_id, commit.relative_order] => - [author.id, nil] - } - - migration.bulk_update_commit_rows(mapping) - - commit = commits.first - - expect(commit.commit_author_id).to eq(author.id) - expect(commit.committer_id).to be_nil - end - end - - context 'when there are only committers' do - it 'only updates the committer IDs' do - committer = - commit_users.create!(name: 'Alice', email: 'alice@example.com') - - commit = commits.create!( - merge_request_diff_id: diff.id, - relative_order: 0, - sha: Gitlab::Database::ShaAttribute.serialize('123abc') - ) - - mapping = { - [commit.merge_request_diff_id, commit.relative_order] => - [nil, committer.id] - } - - migration.bulk_update_commit_rows(mapping) - - commit = commits.first - - expect(commit.committer_id).to eq(committer.id) - expect(commit.commit_author_id).to be_nil - end - end - - context 'when there are both authors and committers' do - it 'updates both the author and committer IDs' do - author = commit_users.create!(name: 'Bob', email: 'bob@example.com') - committer = - commit_users.create!(name: 'Alice', email: 'alice@example.com') - - commit = commits.create!( - merge_request_diff_id: diff.id, - relative_order: 0, - sha: Gitlab::Database::ShaAttribute.serialize('123abc') - ) - - mapping = { - [commit.merge_request_diff_id, commit.relative_order] => - [author.id, committer.id] - } - - migration.bulk_update_commit_rows(mapping) - - commit = commits.first - - expect(commit.commit_author_id).to eq(author.id) - expect(commit.committer_id).to eq(committer.id) - end - end - - context 'when there are multiple commit rows to update' do - it 'updates all the rows' do - author = commit_users.create!(name: 'Bob', email: 'bob@example.com') - committer = - commit_users.create!(name: 'Alice', email: 'alice@example.com') - - commit1 = commits.create!( - merge_request_diff_id: diff.id, - relative_order: 0, - sha: Gitlab::Database::ShaAttribute.serialize('123abc') - ) - - commit2 = commits.create!( - merge_request_diff_id: diff.id, - relative_order: 1, - sha: Gitlab::Database::ShaAttribute.serialize('456abc') - ) - - mapping = { - [commit1.merge_request_diff_id, commit1.relative_order] => - [author.id, committer.id], - - [commit2.merge_request_diff_id, commit2.relative_order] => - [author.id, nil] - } - - migration.bulk_update_commit_rows(mapping) - - commit1 = commits.find_by(relative_order: 0) - commit2 = commits.find_by(relative_order: 1) - - expect(commit1.commit_author_id).to eq(author.id) - expect(commit1.committer_id).to eq(committer.id) - - expect(commit2.commit_author_id).to eq(author.id) - expect(commit2.committer_id).to be_nil - end - end - end - - describe '#primary_key' do - it 'returns the primary key for the commits table' do - key = migration.primary_key - - expect(key.to_sql).to eq('("merge_request_diff_commits"."merge_request_diff_id", "merge_request_diff_commits"."relative_order")') - end - end - - describe '#prepare' do - it 'trims a value to at most 512 characters' do - expect(migration.prepare('€' * 1_000)).to eq('€' * 512) - end - - it 'returns nil if the value is an empty string' do - expect(migration.prepare('')).to be_nil - end - end -end diff --git a/spec/lib/gitlab/background_migration/migrate_personal_namespace_project_maintainer_to_owner_spec.rb b/spec/lib/gitlab/background_migration/migrate_personal_namespace_project_maintainer_to_owner_spec.rb index 07e77bdbc13..90d05ccbe1a 100644 --- a/spec/lib/gitlab/background_migration/migrate_personal_namespace_project_maintainer_to_owner_spec.rb +++ b/spec/lib/gitlab/background_migration/migrate_personal_namespace_project_maintainer_to_owner_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::MigratePersonalNamespaceProjectMaintainerToOwner, :migration, schema: 20220208080921 do +RSpec.describe Gitlab::BackgroundMigration::MigratePersonalNamespaceProjectMaintainerToOwner, :migration, schema: 20220314184009 do let(:migration) { described_class.new } let(:users_table) { table(:users) } let(:members_table) { table(:members) } diff --git a/spec/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics_spec.rb b/spec/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics_spec.rb deleted file mode 100644 index b252df4ecff..00000000000 --- a/spec/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics_spec.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::MigrateProjectTaggingsContextFromTagsToTopics, - :suppress_gitlab_schemas_validate_connection, schema: 20210826171758 do - it 'correctly migrates project taggings context from tags to topics' do - taggings = table(:taggings) - - project_old_tagging_1 = taggings.create!(taggable_type: 'Project', context: 'tags') - project_new_tagging_1 = taggings.create!(taggable_type: 'Project', context: 'topics') - project_other_context_tagging_1 = taggings.create!(taggable_type: 'Project', context: 'other') - project_old_tagging_2 = taggings.create!(taggable_type: 'Project', context: 'tags') - project_old_tagging_3 = taggings.create!(taggable_type: 'Project', context: 'tags') - - subject.perform(project_old_tagging_1.id, project_old_tagging_2.id) - - project_old_tagging_1.reload - project_new_tagging_1.reload - project_other_context_tagging_1.reload - project_old_tagging_2.reload - project_old_tagging_3.reload - - expect(project_old_tagging_1.context).to eq('topics') - expect(project_new_tagging_1.context).to eq('topics') - expect(project_other_context_tagging_1.context).to eq('other') - expect(project_old_tagging_2.context).to eq('topics') - expect(project_old_tagging_3.context).to eq('tags') - end -end diff --git a/spec/lib/gitlab/background_migration/migrate_u2f_webauthn_spec.rb b/spec/lib/gitlab/background_migration/migrate_u2f_webauthn_spec.rb deleted file mode 100644 index 08fde0d0ff4..00000000000 --- a/spec/lib/gitlab/background_migration/migrate_u2f_webauthn_spec.rb +++ /dev/null @@ -1,67 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -require 'webauthn/u2f_migrator' - -RSpec.describe Gitlab::BackgroundMigration::MigrateU2fWebauthn, :migration, schema: 20210826171758 do - let(:users) { table(:users) } - - let(:user) { users.create!(email: 'email@email.com', name: 'foo', username: 'foo', projects_limit: 0) } - - let(:u2f_registrations) { table(:u2f_registrations) } - let(:webauthn_registrations) { table(:webauthn_registrations) } - - let!(:u2f_registration_not_migrated) { create_u2f_registration(1, 'reg1') } - let!(:u2f_registration_not_migrated_no_name) { create_u2f_registration(2, nil, 2) } - let!(:u2f_registration_migrated) { create_u2f_registration(3, 'reg3') } - - subject { described_class.new.perform(1, 3) } - - before do - converted_credential = convert_credential_for(u2f_registration_migrated) - webauthn_registrations.create!(converted_credential) - end - - it 'migrates all records' do - expect { subject }.to change { webauthn_registrations.count }.from(1).to(3) - - all_webauthn_registrations = webauthn_registrations.all.map(&:attributes) - - [u2f_registration_not_migrated, u2f_registration_not_migrated_no_name].each do |u2f_registration| - expected_credential = convert_credential_for(u2f_registration).except(:created_at).stringify_keys - expect(all_webauthn_registrations).to include(a_hash_including(expected_credential)) - end - end - - def create_u2f_registration(id, name, counter = 5) - device = U2F::FakeU2F.new(FFaker::BaconIpsum.characters(5)) - u2f_registrations.create!({ id: id, - certificate: Base64.strict_encode64(device.cert_raw), - key_handle: U2F.urlsafe_encode64(device.key_handle_raw), - public_key: Base64.strict_encode64(device.origin_public_key_raw), - counter: counter, - name: name, - user_id: user.id }) - end - - def convert_credential_for(u2f_registration) - converted_credential = WebAuthn::U2fMigrator.new( - app_id: Gitlab.config.gitlab.url, - certificate: u2f_registration.certificate, - key_handle: u2f_registration.key_handle, - public_key: u2f_registration.public_key, - counter: u2f_registration.counter - ).credential - - { - credential_xid: Base64.strict_encode64(converted_credential.id), - public_key: Base64.strict_encode64(converted_credential.public_key), - counter: u2f_registration.counter, - name: u2f_registration.name || '', - user_id: u2f_registration.user_id, - u2f_registration_id: u2f_registration.id, - created_at: u2f_registration.created_at - } - end -end diff --git a/spec/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature_spec.rb b/spec/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature_spec.rb deleted file mode 100644 index 71cf58a933f..00000000000 --- a/spec/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature_spec.rb +++ /dev/null @@ -1,98 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::MoveContainerRegistryEnabledToProjectFeature, :migration, schema: 20210826171758 do - let(:enabled) { 20 } - let(:disabled) { 0 } - - let(:namespaces) { table(:namespaces) } - let(:project_features) { table(:project_features) } - let(:projects) { table(:projects) } - - let(:namespace) { namespaces.create!(name: 'user', path: 'user') } - let!(:project1) { projects.create!(namespace_id: namespace.id) } - let!(:project2) { projects.create!(namespace_id: namespace.id) } - let!(:project3) { projects.create!(namespace_id: namespace.id) } - let!(:project4) { projects.create!(namespace_id: namespace.id) } - - # pages_access_level cannot be null. - let(:non_null_project_features) { { pages_access_level: enabled } } - let!(:project_feature1) { project_features.create!(project_id: project1.id, **non_null_project_features) } - let!(:project_feature2) { project_features.create!(project_id: project2.id, **non_null_project_features) } - let!(:project_feature3) { project_features.create!(project_id: project3.id, **non_null_project_features) } - - describe '#perform' do - before do - project1.update!(container_registry_enabled: true) - project2.update!(container_registry_enabled: false) - project3.update!(container_registry_enabled: nil) - project4.update!(container_registry_enabled: true) - end - - it 'copies values to project_features' do - table(:background_migration_jobs).create!( - class_name: 'MoveContainerRegistryEnabledToProjectFeature', - arguments: [project1.id, project4.id] - ) - table(:background_migration_jobs).create!( - class_name: 'MoveContainerRegistryEnabledToProjectFeature', - arguments: [-1, -3] - ) - - expect(project1.container_registry_enabled).to eq(true) - expect(project2.container_registry_enabled).to eq(false) - expect(project3.container_registry_enabled).to eq(nil) - expect(project4.container_registry_enabled).to eq(true) - - expect(project_feature1.container_registry_access_level).to eq(disabled) - expect(project_feature2.container_registry_access_level).to eq(disabled) - expect(project_feature3.container_registry_access_level).to eq(disabled) - - expect_next_instance_of(Gitlab::BackgroundMigration::Logger) do |logger| - expect(logger).to receive(:info) - .with(message: "#{described_class}: Copied container_registry_enabled values for projects with IDs between #{project1.id}..#{project4.id}") - - expect(logger).not_to receive(:info) - end - - subject.perform(project1.id, project4.id) - - expect(project1.reload.container_registry_enabled).to eq(true) - expect(project2.reload.container_registry_enabled).to eq(false) - expect(project3.reload.container_registry_enabled).to eq(nil) - expect(project4.container_registry_enabled).to eq(true) - - expect(project_feature1.reload.container_registry_access_level).to eq(enabled) - expect(project_feature2.reload.container_registry_access_level).to eq(disabled) - expect(project_feature3.reload.container_registry_access_level).to eq(disabled) - - expect(table(:background_migration_jobs).first.status).to eq(1) # succeeded - expect(table(:background_migration_jobs).second.status).to eq(0) # pending - end - - context 'when no projects exist in range' do - it 'does not fail' do - expect(project1.container_registry_enabled).to eq(true) - expect(project_feature1.container_registry_access_level).to eq(disabled) - - expect { subject.perform(-1, -2) }.not_to raise_error - - expect(project1.container_registry_enabled).to eq(true) - expect(project_feature1.container_registry_access_level).to eq(disabled) - end - end - - context 'when projects in range all have nil container_registry_enabled' do - it 'does not fail' do - expect(project3.container_registry_enabled).to eq(nil) - expect(project_feature3.container_registry_access_level).to eq(disabled) - - expect { subject.perform(project3.id, project3.id) }.not_to raise_error - - expect(project3.container_registry_enabled).to eq(nil) - expect(project_feature3.container_registry_access_level).to eq(disabled) - end - end - end -end diff --git a/spec/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds_spec.rb b/spec/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds_spec.rb index 2f0eef3c399..7c78350e697 100644 --- a/spec/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds_spec.rb +++ b/spec/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::BackgroundMigration::NullifyOrphanRunnerIdOnCiBuilds, - :suppress_gitlab_schemas_validate_connection, migration: :gitlab_ci, schema: 20220223112304 do + :suppress_gitlab_schemas_validate_connection, migration: :gitlab_ci, schema: 20220314184009 do let(:namespaces) { table(:namespaces) } let(:projects) { table(:projects) } let(:ci_runners) { table(:ci_runners) } diff --git a/spec/lib/gitlab/background_migration/populate_namespace_statistics_spec.rb b/spec/lib/gitlab/background_migration/populate_namespace_statistics_spec.rb deleted file mode 100644 index 4a7d52ee784..00000000000 --- a/spec/lib/gitlab/background_migration/populate_namespace_statistics_spec.rb +++ /dev/null @@ -1,71 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::PopulateNamespaceStatistics do - let!(:namespaces) { table(:namespaces) } - let!(:namespace_statistics) { table(:namespace_statistics) } - let!(:dependency_proxy_manifests) { table(:dependency_proxy_manifests) } - let!(:dependency_proxy_blobs) { table(:dependency_proxy_blobs) } - - let!(:group1) { namespaces.create!(id: 10, type: 'Group', name: 'group1', path: 'group1') } - let!(:group2) { namespaces.create!(id: 20, type: 'Group', name: 'group2', path: 'group2') } - - let!(:group1_manifest) do - dependency_proxy_manifests.create!(group_id: 10, size: 20, file_name: 'test-file', file: 'test', digest: 'abc123') - end - - let!(:group2_manifest) do - dependency_proxy_manifests.create!(group_id: 20, size: 20, file_name: 'test-file', file: 'test', digest: 'abc123') - end - - let!(:group1_stats) { namespace_statistics.create!(id: 10, namespace_id: 10) } - - let(:ids) { namespaces.pluck(:id) } - let(:statistics) { [] } - - subject(:perform) { described_class.new.perform(ids, statistics) } - - it 'creates/updates all namespace_statistics and updates root storage statistics', :aggregate_failures do - expect(Namespaces::ScheduleAggregationWorker).to receive(:perform_async).with(group1.id) - expect(Namespaces::ScheduleAggregationWorker).to receive(:perform_async).with(group2.id) - - expect { perform }.to change(namespace_statistics, :count).from(1).to(2) - - namespace_statistics.all.each do |stat| - expect(stat.dependency_proxy_size).to eq 20 - expect(stat.storage_size).to eq 20 - end - end - - context 'when just a stat is passed' do - let(:statistics) { [:dependency_proxy_size] } - - it 'calls the statistics update service with just that stat' do - expect(Groups::UpdateStatisticsService) - .to receive(:new) - .with(anything, statistics: [:dependency_proxy_size]) - .twice.and_call_original - - perform - end - end - - context 'when a statistics update fails' do - before do - error_response = instance_double(ServiceResponse, message: 'an error', error?: true) - - allow_next_instance_of(Groups::UpdateStatisticsService) do |instance| - allow(instance).to receive(:execute).and_return(error_response) - end - end - - it 'logs an error' do - expect_next_instance_of(Gitlab::BackgroundMigration::Logger) do |instance| - expect(instance).to receive(:error).twice - end - - perform - end - end -end diff --git a/spec/lib/gitlab/background_migration/populate_topics_non_private_projects_count_spec.rb b/spec/lib/gitlab/background_migration/populate_topics_non_private_projects_count_spec.rb deleted file mode 100644 index e72e3392210..00000000000 --- a/spec/lib/gitlab/background_migration/populate_topics_non_private_projects_count_spec.rb +++ /dev/null @@ -1,50 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::PopulateTopicsNonPrivateProjectsCount, schema: 20220125122640 do - it 'correctly populates the non private projects counters' do - namespaces = table(:namespaces) - projects = table(:projects) - topics = table(:topics) - project_topics = table(:project_topics) - - group = namespaces.create!(name: 'group', path: 'group') - project_public = projects.create!(namespace_id: group.id, visibility_level: Gitlab::VisibilityLevel::PUBLIC) - project_internal = projects.create!(namespace_id: group.id, visibility_level: Gitlab::VisibilityLevel::INTERNAL) - project_private = projects.create!(namespace_id: group.id, visibility_level: Gitlab::VisibilityLevel::PRIVATE) - topic_1 = topics.create!(name: 'Topic1') - topic_2 = topics.create!(name: 'Topic2') - topic_3 = topics.create!(name: 'Topic3') - topic_4 = topics.create!(name: 'Topic4') - topic_5 = topics.create!(name: 'Topic5') - topic_6 = topics.create!(name: 'Topic6') - topic_7 = topics.create!(name: 'Topic7') - topic_8 = topics.create!(name: 'Topic8') - - project_topics.create!(topic_id: topic_1.id, project_id: project_public.id) - project_topics.create!(topic_id: topic_2.id, project_id: project_internal.id) - project_topics.create!(topic_id: topic_3.id, project_id: project_private.id) - project_topics.create!(topic_id: topic_4.id, project_id: project_public.id) - project_topics.create!(topic_id: topic_4.id, project_id: project_internal.id) - project_topics.create!(topic_id: topic_5.id, project_id: project_public.id) - project_topics.create!(topic_id: topic_5.id, project_id: project_private.id) - project_topics.create!(topic_id: topic_6.id, project_id: project_internal.id) - project_topics.create!(topic_id: topic_6.id, project_id: project_private.id) - project_topics.create!(topic_id: topic_7.id, project_id: project_public.id) - project_topics.create!(topic_id: topic_7.id, project_id: project_internal.id) - project_topics.create!(topic_id: topic_7.id, project_id: project_private.id) - project_topics.create!(topic_id: topic_8.id, project_id: project_public.id) - - subject.perform(topic_1.id, topic_7.id) - - expect(topic_1.reload.non_private_projects_count).to eq(1) - expect(topic_2.reload.non_private_projects_count).to eq(1) - expect(topic_3.reload.non_private_projects_count).to eq(0) - expect(topic_4.reload.non_private_projects_count).to eq(2) - expect(topic_5.reload.non_private_projects_count).to eq(1) - expect(topic_6.reload.non_private_projects_count).to eq(1) - expect(topic_7.reload.non_private_projects_count).to eq(2) - expect(topic_8.reload.non_private_projects_count).to eq(0) - end -end diff --git a/spec/lib/gitlab/background_migration/populate_topics_total_projects_count_cache_spec.rb b/spec/lib/gitlab/background_migration/populate_topics_total_projects_count_cache_spec.rb deleted file mode 100644 index 8e07b43f5b9..00000000000 --- a/spec/lib/gitlab/background_migration/populate_topics_total_projects_count_cache_spec.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::PopulateTopicsTotalProjectsCountCache, schema: 20211006060436 do - it 'correctly populates total projects count cache' do - namespaces = table(:namespaces) - projects = table(:projects) - topics = table(:topics) - project_topics = table(:project_topics) - - group = namespaces.create!(name: 'group', path: 'group') - project_1 = projects.create!(namespace_id: group.id) - project_2 = projects.create!(namespace_id: group.id) - project_3 = projects.create!(namespace_id: group.id) - topic_1 = topics.create!(name: 'Topic1') - topic_2 = topics.create!(name: 'Topic2') - topic_3 = topics.create!(name: 'Topic3') - topic_4 = topics.create!(name: 'Topic4') - - project_topics.create!(project_id: project_1.id, topic_id: topic_1.id) - project_topics.create!(project_id: project_1.id, topic_id: topic_3.id) - project_topics.create!(project_id: project_2.id, topic_id: topic_3.id) - project_topics.create!(project_id: project_1.id, topic_id: topic_4.id) - project_topics.create!(project_id: project_2.id, topic_id: topic_4.id) - project_topics.create!(project_id: project_3.id, topic_id: topic_4.id) - - subject.perform(topic_1.id, topic_4.id) - - expect(topic_1.reload.total_projects_count).to eq(1) - expect(topic_2.reload.total_projects_count).to eq(0) - expect(topic_3.reload.total_projects_count).to eq(2) - expect(topic_4.reload.total_projects_count).to eq(3) - end -end diff --git a/spec/lib/gitlab/background_migration/populate_vulnerability_reads_spec.rb b/spec/lib/gitlab/background_migration/populate_vulnerability_reads_spec.rb deleted file mode 100644 index c0470f26d9e..00000000000 --- a/spec/lib/gitlab/background_migration/populate_vulnerability_reads_spec.rb +++ /dev/null @@ -1,93 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::PopulateVulnerabilityReads, :migration, schema: 20220326161803 do - let(:vulnerabilities) { table(:vulnerabilities) } - let(:vulnerability_reads) { table(:vulnerability_reads) } - let(:vulnerabilities_findings) { table(:vulnerability_occurrences) } - let(:vulnerability_issue_links) { table(:vulnerability_issue_links) } - let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') } - let(:user) { table(:users).create!(email: 'author@example.com', username: 'author', projects_limit: 10) } - let(:project) { table(:projects).create!(namespace_id: namespace.id) } - let(:scanner) { table(:vulnerability_scanners).create!(project_id: project.id, external_id: 'test 1', name: 'test scanner 1') } - let(:sub_batch_size) { 1000 } - - before do - vulnerabilities_findings.connection.execute 'ALTER TABLE vulnerability_occurrences DISABLE TRIGGER "trigger_insert_or_update_vulnerability_reads_from_occurrences"' - vulnerabilities.connection.execute 'ALTER TABLE vulnerabilities DISABLE TRIGGER "trigger_update_vulnerability_reads_on_vulnerability_update"' - vulnerability_issue_links.connection.execute 'ALTER TABLE vulnerability_issue_links DISABLE TRIGGER "trigger_update_has_issues_on_vulnerability_issue_links_update"' - - 10.times.each do |x| - vulnerability = create_vulnerability!( - project_id: project.id, - report_type: 7, - author_id: user.id - ) - identifier = table(:vulnerability_identifiers).create!( - project_id: project.id, - external_type: 'uuid-v5', - external_id: 'uuid-v5', - fingerprint: Digest::SHA1.hexdigest(vulnerability.id.to_s), - name: 'Identifier for UUIDv5') - - create_finding!( - vulnerability_id: vulnerability.id, - project_id: project.id, - scanner_id: scanner.id, - primary_identifier_id: identifier.id - ) - end - end - - it 'creates vulnerability_reads for the given records' do - described_class.new.perform(vulnerabilities.first.id, vulnerabilities.last.id, sub_batch_size) - - expect(vulnerability_reads.count).to eq(10) - end - - it 'does not create new records when records already exists' do - described_class.new.perform(vulnerabilities.first.id, vulnerabilities.last.id, sub_batch_size) - described_class.new.perform(vulnerabilities.first.id, vulnerabilities.last.id, sub_batch_size) - - expect(vulnerability_reads.count).to eq(10) - end - - private - - def create_vulnerability!(project_id:, author_id:, title: 'test', severity: 7, confidence: 7, report_type: 0) - vulnerabilities.create!( - project_id: project_id, - author_id: author_id, - title: title, - severity: severity, - confidence: confidence, - report_type: report_type - ) - end - - # rubocop:disable Metrics/ParameterLists - def create_finding!( - project_id:, scanner_id:, primary_identifier_id:, vulnerability_id: nil, - name: "test", severity: 7, confidence: 7, report_type: 0, - project_fingerprint: '123qweasdzxc', location: { "image" => "alpine:3.4" }, location_fingerprint: 'test', - metadata_version: 'test', raw_metadata: 'test', uuid: SecureRandom.uuid) - vulnerabilities_findings.create!( - vulnerability_id: vulnerability_id, - project_id: project_id, - name: name, - severity: severity, - confidence: confidence, - report_type: report_type, - project_fingerprint: project_fingerprint, - scanner_id: scanner_id, - primary_identifier_id: primary_identifier_id, - location: location, - location_fingerprint: location_fingerprint, - metadata_version: metadata_version, - raw_metadata: raw_metadata, - uuid: uuid - ) - end - # rubocop:enable Metrics/ParameterLists -end diff --git a/spec/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid_spec.rb b/spec/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid_spec.rb deleted file mode 100644 index 2271bbfb2f3..00000000000 --- a/spec/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid_spec.rb +++ /dev/null @@ -1,530 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -def create_background_migration_job(ids, status) - proper_status = case status - when :pending - Gitlab::Database::BackgroundMigrationJob.statuses['pending'] - when :succeeded - Gitlab::Database::BackgroundMigrationJob.statuses['succeeded'] - else - raise ArgumentError - end - - background_migration_jobs.create!( - class_name: 'RecalculateVulnerabilitiesOccurrencesUuid', - arguments: Array(ids), - status: proper_status, - created_at: Time.now.utc - ) -end - -RSpec.describe Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid, :suppress_gitlab_schemas_validate_connection, schema: 20211124132705 do - let(:background_migration_jobs) { table(:background_migration_jobs) } - let(:pending_jobs) { background_migration_jobs.where(status: Gitlab::Database::BackgroundMigrationJob.statuses['pending']) } - let(:succeeded_jobs) { background_migration_jobs.where(status: Gitlab::Database::BackgroundMigrationJob.statuses['succeeded']) } - let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') } - let(:users) { table(:users) } - let(:user) { create_user! } - let(:project) { table(:projects).create!(id: 123, namespace_id: namespace.id) } - let(:scanners) { table(:vulnerability_scanners) } - let(:scanner) { scanners.create!(project_id: project.id, external_id: 'test 1', name: 'test scanner 1') } - let(:scanner2) { scanners.create!(project_id: project.id, external_id: 'test 2', name: 'test scanner 2') } - let(:vulnerabilities) { table(:vulnerabilities) } - let(:vulnerability_findings) { table(:vulnerability_occurrences) } - let(:vulnerability_finding_pipelines) { table(:vulnerability_occurrence_pipelines) } - let(:vulnerability_finding_signatures) { table(:vulnerability_finding_signatures) } - let(:vulnerability_identifiers) { table(:vulnerability_identifiers) } - - let(:identifier_1) { 'identifier-1' } - let!(:vulnerability_identifier) do - vulnerability_identifiers.create!( - project_id: project.id, - external_type: identifier_1, - external_id: identifier_1, - fingerprint: Gitlab::Database::ShaAttribute.serialize('ff9ef548a6e30a0462795d916f3f00d1e2b082ca'), - name: 'Identifier 1') - end - - let(:identifier_2) { 'identifier-2' } - let!(:vulnerability_identfier2) do - vulnerability_identifiers.create!( - project_id: project.id, - external_type: identifier_2, - external_id: identifier_2, - fingerprint: Gitlab::Database::ShaAttribute.serialize('4299e8ddd819f9bde9cfacf45716724c17b5ddf7'), - name: 'Identifier 2') - end - - let(:identifier_3) { 'identifier-3' } - let!(:vulnerability_identifier3) do - vulnerability_identifiers.create!( - project_id: project.id, - external_type: identifier_3, - external_id: identifier_3, - fingerprint: Gitlab::Database::ShaAttribute.serialize('8e91632f9c6671e951834a723ee221c44cc0d844'), - name: 'Identifier 3') - end - - let(:known_uuid_v4) { "b3cc2518-5446-4dea-871c-89d5e999c1ac" } - let(:known_uuid_v5) { "05377088-dc26-5161-920e-52a7159fdaa1" } - let(:desired_uuid_v5) { "f3e9a23f-9181-54bf-a5ab-c5bc7a9b881a" } - - subject { described_class.new.perform(start_id, end_id) } - - context "when finding has a UUIDv4" do - before do - @uuid_v4 = create_finding!( - vulnerability_id: nil, - project_id: project.id, - scanner_id: scanner2.id, - primary_identifier_id: vulnerability_identfier2.id, - report_type: 0, # "sast" - location_fingerprint: Gitlab::Database::ShaAttribute.serialize("fa18f432f1d56675f4098d318739c3cd5b14eb3e"), - uuid: known_uuid_v4 - ) - end - - let(:start_id) { @uuid_v4.id } - let(:end_id) { @uuid_v4.id } - - it "replaces it with UUIDv5" do - expect(vulnerability_findings.pluck(:uuid)).to match_array([known_uuid_v4]) - - subject - - expect(vulnerability_findings.pluck(:uuid)).to match_array([desired_uuid_v5]) - end - - it 'logs recalculation' do - expect_next_instance_of(Gitlab::BackgroundMigration::Logger) do |instance| - expect(instance).to receive(:info).twice - end - - subject - end - end - - context "when finding has a UUIDv5" do - before do - @uuid_v5 = create_finding!( - vulnerability_id: nil, - project_id: project.id, - scanner_id: scanner.id, - primary_identifier_id: vulnerability_identifier.id, - report_type: 0, # "sast" - location_fingerprint: Gitlab::Database::ShaAttribute.serialize("838574be0210968bf6b9f569df9c2576242cbf0a"), - uuid: known_uuid_v5 - ) - end - - let(:start_id) { @uuid_v5.id } - let(:end_id) { @uuid_v5.id } - - it "stays the same" do - expect(vulnerability_findings.pluck(:uuid)).to match_array([known_uuid_v5]) - - subject - - expect(vulnerability_findings.pluck(:uuid)).to match_array([known_uuid_v5]) - end - end - - context 'if a duplicate UUID would be generated' do # rubocop: disable RSpec/MultipleMemoizedHelpers - let(:v1) do - create_vulnerability!( - project_id: project.id, - author_id: user.id - ) - end - - let!(:finding_with_incorrect_uuid) do - create_finding!( - vulnerability_id: v1.id, - project_id: project.id, - scanner_id: scanner.id, - primary_identifier_id: vulnerability_identifier.id, - report_type: 0, # "sast" - location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis') - uuid: 'bd95c085-71aa-51d7-9bb6-08ae669c262e' - ) - end - - let(:v2) do - create_vulnerability!( - project_id: project.id, - author_id: user.id - ) - end - - let!(:finding_with_correct_uuid) do - create_finding!( - vulnerability_id: v2.id, - project_id: project.id, - primary_identifier_id: vulnerability_identifier.id, - scanner_id: scanner2.id, - report_type: 0, # "sast" - location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis') - uuid: '91984483-5efe-5215-b471-d524ac5792b1' - ) - end - - let(:v3) do - create_vulnerability!( - project_id: project.id, - author_id: user.id - ) - end - - let!(:finding_with_incorrect_uuid2) do - create_finding!( - vulnerability_id: v3.id, - project_id: project.id, - scanner_id: scanner.id, - primary_identifier_id: vulnerability_identfier2.id, - report_type: 0, # "sast" - location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis') - uuid: '00000000-1111-2222-3333-444444444444' - ) - end - - let(:v4) do - create_vulnerability!( - project_id: project.id, - author_id: user.id - ) - end - - let!(:finding_with_correct_uuid2) do - create_finding!( - vulnerability_id: v4.id, - project_id: project.id, - scanner_id: scanner2.id, - primary_identifier_id: vulnerability_identfier2.id, - report_type: 0, # "sast" - location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis') - uuid: '1edd751e-ef9a-5391-94db-a832c8635bfc' - ) - end - - let!(:finding_with_incorrect_uuid3) do - create_finding!( - vulnerability_id: nil, - project_id: project.id, - scanner_id: scanner.id, - primary_identifier_id: vulnerability_identifier3.id, - report_type: 0, # "sast" - location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis') - uuid: '22222222-3333-4444-5555-666666666666' - ) - end - - let!(:duplicate_not_in_the_same_batch) do - create_finding!( - id: 99999, - vulnerability_id: nil, - project_id: project.id, - scanner_id: scanner2.id, - primary_identifier_id: vulnerability_identifier3.id, - report_type: 0, # "sast" - location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis') - uuid: '4564f9d5-3c6b-5cc3-af8c-7c25285362a7' - ) - end - - let(:start_id) { finding_with_incorrect_uuid.id } - let(:end_id) { finding_with_incorrect_uuid3.id } - - before do - 4.times do - create_finding_pipeline!(project_id: project.id, finding_id: finding_with_incorrect_uuid.id) - create_finding_pipeline!(project_id: project.id, finding_id: finding_with_correct_uuid.id) - create_finding_pipeline!(project_id: project.id, finding_id: finding_with_incorrect_uuid2.id) - create_finding_pipeline!(project_id: project.id, finding_id: finding_with_correct_uuid2.id) - end - end - - it 'drops duplicates and related records', :aggregate_failures do - expect(vulnerability_findings.pluck(:id)).to match_array( - [ - finding_with_correct_uuid.id, - finding_with_incorrect_uuid.id, - finding_with_correct_uuid2.id, - finding_with_incorrect_uuid2.id, - finding_with_incorrect_uuid3.id, - duplicate_not_in_the_same_batch.id - ]) - - expect { subject }.to change(vulnerability_finding_pipelines, :count).from(16).to(8) - .and change(vulnerability_findings, :count).from(6).to(3) - .and change(vulnerabilities, :count).from(4).to(2) - - expect(vulnerability_findings.pluck(:id)).to match_array([finding_with_incorrect_uuid.id, finding_with_incorrect_uuid2.id, finding_with_incorrect_uuid3.id]) - end - - context 'if there are conflicting UUID values within the batch' do # rubocop: disable RSpec/MultipleMemoizedHelpers - let(:end_id) { finding_with_broken_data_integrity.id } - let(:vulnerability_5) { create_vulnerability!(project_id: project.id, author_id: user.id) } - let(:different_project) { table(:projects).create!(namespace_id: namespace.id) } - let!(:identifier_with_broken_data_integrity) do - vulnerability_identifiers.create!( - project_id: different_project.id, - external_type: identifier_2, - external_id: identifier_2, - fingerprint: Gitlab::Database::ShaAttribute.serialize('4299e8ddd819f9bde9cfacf45716724c17b5ddf7'), - name: 'Identifier 2') - end - - let(:finding_with_broken_data_integrity) do - create_finding!( - vulnerability_id: vulnerability_5, - project_id: project.id, - scanner_id: scanner.id, - primary_identifier_id: identifier_with_broken_data_integrity.id, - report_type: 0, # "sast" - location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis') - uuid: SecureRandom.uuid - ) - end - - it 'deletes the conflicting record' do - expect { subject }.to change { vulnerability_findings.find_by_id(finding_with_broken_data_integrity.id) }.to(nil) - end - end - - context 'if a conflicting UUID is found during the migration' do # rubocop:disable RSpec/MultipleMemoizedHelpers - let(:finding_class) { Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid::VulnerabilitiesFinding } - let(:uuid) { '4564f9d5-3c6b-5cc3-af8c-7c25285362a7' } - - before do - exception = ActiveRecord::RecordNotUnique.new("(uuid)=(#{uuid})") - - call_count = 0 - allow(::Gitlab::Database::BulkUpdate).to receive(:execute) do - call_count += 1 - call_count.eql?(1) ? raise(exception) : {} - end - - allow(finding_class).to receive(:find_by).with(uuid: uuid).and_return(duplicate_not_in_the_same_batch) - end - - it 'retries the recalculation' do - subject - - expect(Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid::VulnerabilitiesFinding) - .to have_received(:find_by).with(uuid: uuid).once - end - - it 'logs the conflict' do - expect_next_instance_of(Gitlab::BackgroundMigration::Logger) do |instance| - expect(instance).to receive(:info).exactly(6).times - end - - subject - end - - it 'marks the job as done' do - create_background_migration_job([start_id, end_id], :pending) - - subject - - expect(pending_jobs.count).to eq(0) - expect(succeeded_jobs.count).to eq(1) - end - end - - it 'logs an exception if a different uniquness problem was found' do - exception = ActiveRecord::RecordNotUnique.new("Totally not an UUID uniqueness problem") - allow(::Gitlab::Database::BulkUpdate).to receive(:execute).and_raise(exception) - allow(Gitlab::ErrorTracking).to receive(:track_and_raise_exception) - - subject - - expect(Gitlab::ErrorTracking).to have_received(:track_and_raise_exception).with(exception).once - end - - it 'logs a duplicate found message' do - expect_next_instance_of(Gitlab::BackgroundMigration::Logger) do |instance| - expect(instance).to receive(:info).exactly(3).times - end - - subject - end - end - - context 'when finding has a signature' do - before do - @f1 = create_finding!( - vulnerability_id: nil, - project_id: project.id, - scanner_id: scanner.id, - primary_identifier_id: vulnerability_identifier.id, - report_type: 0, # "sast" - location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis') - uuid: 'd15d774d-e4b1-5a1b-929b-19f2a53e35ec' - ) - - vulnerability_finding_signatures.create!( - finding_id: @f1.id, - algorithm_type: 2, # location - signature_sha: Gitlab::Database::ShaAttribute.serialize('57d4e05205f6462a73f039a5b2751aa1ab344e6e') # sha1('youshouldusethis') - ) - - vulnerability_finding_signatures.create!( - finding_id: @f1.id, - algorithm_type: 1, # hash - signature_sha: Gitlab::Database::ShaAttribute.serialize('c554d8d8df1a7a14319eafdaae24af421bf5b587') # sha1('andnotthis') - ) - - @f2 = create_finding!( - vulnerability_id: nil, - project_id: project.id, - scanner_id: scanner.id, - primary_identifier_id: vulnerability_identfier2.id, - report_type: 0, # "sast" - location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis') - uuid: '4be029b5-75e5-5ac0-81a2-50ab41726135' - ) - - vulnerability_finding_signatures.create!( - finding_id: @f2.id, - algorithm_type: 2, # location - signature_sha: Gitlab::Database::ShaAttribute.serialize('57d4e05205f6462a73f039a5b2751aa1ab344e6e') # sha1('youshouldusethis') - ) - - vulnerability_finding_signatures.create!( - finding_id: @f2.id, - algorithm_type: 1, # hash - signature_sha: Gitlab::Database::ShaAttribute.serialize('c554d8d8df1a7a14319eafdaae24af421bf5b587') # sha1('andnotthis') - ) - end - - let(:start_id) { @f1.id } - let(:end_id) { @f2.id } - - let(:uuids_before) { [@f1.uuid, @f2.uuid] } - let(:uuids_after) { %w[d3b60ddd-d312-5606-b4d3-ad058eebeacb 349d9bec-c677-5530-a8ac-5e58889c3b1a] } - - it 'is recalculated using signature' do - expect(vulnerability_findings.pluck(:uuid)).to match_array(uuids_before) - - subject - - expect(vulnerability_findings.pluck(:uuid)).to match_array(uuids_after) - end - end - - context 'if all records are removed before the job ran' do - let(:start_id) { 1 } - let(:end_id) { 9 } - - before do - create_background_migration_job([start_id, end_id], :pending) - end - - it 'does not error out' do - expect { subject }.not_to raise_error - end - - it 'marks the job as done' do - subject - - expect(pending_jobs.count).to eq(0) - expect(succeeded_jobs.count).to eq(1) - end - end - - context 'when recalculation fails' do - before do - @uuid_v4 = create_finding!( - vulnerability_id: nil, - project_id: project.id, - scanner_id: scanner2.id, - primary_identifier_id: vulnerability_identfier2.id, - report_type: 0, # "sast" - location_fingerprint: Gitlab::Database::ShaAttribute.serialize("fa18f432f1d56675f4098d318739c3cd5b14eb3e"), - uuid: known_uuid_v4 - ) - - allow(Gitlab::ErrorTracking).to receive(:track_and_raise_exception) - allow(::Gitlab::Database::BulkUpdate).to receive(:execute).and_raise(expected_error) - end - - let(:start_id) { @uuid_v4.id } - let(:end_id) { @uuid_v4.id } - let(:expected_error) { RuntimeError.new } - - it 'captures the errors and does not crash entirely' do - expect { subject }.not_to raise_error - - allow(Gitlab::ErrorTracking).to receive(:track_and_raise_exception) - expect(Gitlab::ErrorTracking).to have_received(:track_and_raise_exception).with(expected_error).once - end - - it_behaves_like 'marks background migration job records' do - let(:arguments) { [1, 4] } - subject { described_class.new } - end - end - - it_behaves_like 'marks background migration job records' do - let(:arguments) { [1, 4] } - subject { described_class.new } - end - - private - - def create_vulnerability!(project_id:, author_id:, title: 'test', severity: 7, confidence: 7, report_type: 0) - vulnerabilities.create!( - project_id: project_id, - author_id: author_id, - title: title, - severity: severity, - confidence: confidence, - report_type: report_type - ) - end - - # rubocop:disable Metrics/ParameterLists - def create_finding!( - vulnerability_id:, project_id:, scanner_id:, primary_identifier_id:, id: nil, - name: "test", severity: 7, confidence: 7, report_type: 0, - project_fingerprint: '123qweasdzxc', location_fingerprint: 'test', - metadata_version: 'test', raw_metadata: 'test', uuid: SecureRandom.uuid) - vulnerability_findings.create!({ - id: id, - vulnerability_id: vulnerability_id, - project_id: project_id, - name: name, - severity: severity, - confidence: confidence, - report_type: report_type, - project_fingerprint: project_fingerprint, - scanner_id: scanner_id, - primary_identifier_id: primary_identifier_id, - location_fingerprint: location_fingerprint, - metadata_version: metadata_version, - raw_metadata: raw_metadata, - uuid: uuid - }.compact - ) - end - # rubocop:enable Metrics/ParameterLists - - def create_user!(name: "Example User", email: "user@example.com", user_type: nil, created_at: Time.zone.now, confirmed_at: Time.zone.now) - users.create!( - name: name, - email: email, - username: name, - projects_limit: 0, - user_type: user_type, - confirmed_at: confirmed_at - ) - end - - def create_finding_pipeline!(project_id:, finding_id:) - pipeline = table(:ci_pipelines).create!(project_id: project_id) - vulnerability_finding_pipelines.create!(pipeline_id: pipeline.id, occurrence_id: finding_id) - end -end diff --git a/spec/lib/gitlab/background_migration/remove_all_trace_expiration_dates_spec.rb b/spec/lib/gitlab/background_migration/remove_all_trace_expiration_dates_spec.rb deleted file mode 100644 index eabc012f98b..00000000000 --- a/spec/lib/gitlab/background_migration/remove_all_trace_expiration_dates_spec.rb +++ /dev/null @@ -1,54 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::RemoveAllTraceExpirationDates, :migration, - :suppress_gitlab_schemas_validate_connection, schema: 20220131000001 do - subject(:perform) { migration.perform(1, 99) } - - let(:migration) { described_class.new } - - let(:trace_in_range) { create_trace!(id: 10, created_at: Date.new(2020, 06, 20), expire_at: Date.new(2021, 01, 22)) } - let(:trace_outside_range) { create_trace!(id: 40, created_at: Date.new(2020, 06, 22), expire_at: Date.new(2021, 01, 22)) } - let(:trace_without_expiry) { create_trace!(id: 30, created_at: Date.new(2020, 06, 21), expire_at: nil) } - let(:archive_in_range) { create_archive!(id: 10, created_at: Date.new(2020, 06, 20), expire_at: Date.new(2021, 01, 22)) } - let(:trace_outside_id_range) { create_trace!(id: 100, created_at: Date.new(2020, 06, 20), expire_at: Date.new(2021, 01, 22)) } - - before do - table(:namespaces).create!(id: 1, name: 'the-namespace', path: 'the-path') - table(:projects).create!(id: 1, name: 'the-project', namespace_id: 1) - table(:ci_builds).create!(id: 1, allow_failure: false) - end - - context 'for self-hosted instances' do - it 'sets expire_at for artifacts in range to nil' do - expect { perform }.not_to change { trace_in_range.reload.expire_at } - end - - it 'does not change expire_at timestamps that are not set to midnight' do - expect { perform }.not_to change { trace_outside_range.reload.expire_at } - end - - it 'does not change expire_at timestamps that are set to midnight on a day other than the 22nd' do - expect { perform }.not_to change { trace_without_expiry.reload.expire_at } - end - - it 'does not touch artifacts outside id range' do - expect { perform }.not_to change { archive_in_range.reload.expire_at } - end - - it 'does not touch artifacts outside date range' do - expect { perform }.not_to change { trace_outside_id_range.reload.expire_at } - end - end - - private - - def create_trace!(**args) - table(:ci_job_artifacts).create!(**args, project_id: 1, job_id: 1, file_type: 3) - end - - def create_archive!(**args) - table(:ci_job_artifacts).create!(**args, project_id: 1, job_id: 1, file_type: 1) - end -end diff --git a/spec/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings_spec.rb b/spec/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings_spec.rb deleted file mode 100644 index ed08ae22245..00000000000 --- a/spec/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings_spec.rb +++ /dev/null @@ -1,171 +0,0 @@ -# frozen_string_literal: true -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::RemoveDuplicateVulnerabilitiesFindings, :migration, schema: 20220326161803 do - let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') } - let(:users) { table(:users) } - let(:user) { create_user! } - let(:project) { table(:projects).create!(id: 14219619, namespace_id: namespace.id) } - let(:scanners) { table(:vulnerability_scanners) } - let!(:scanner1) { scanners.create!(project_id: project.id, external_id: 'test 1', name: 'test scanner 1') } - let!(:scanner2) { scanners.create!(project_id: project.id, external_id: 'test 2', name: 'test scanner 2') } - let!(:scanner3) { scanners.create!(project_id: project.id, external_id: 'test 3', name: 'test scanner 3') } - let!(:unrelated_scanner) { scanners.create!(project_id: project.id, external_id: 'unreleated_scanner', name: 'unrelated scanner') } - let(:vulnerabilities) { table(:vulnerabilities) } - let(:vulnerability_findings) { table(:vulnerability_occurrences) } - let(:vulnerability_identifiers) { table(:vulnerability_identifiers) } - let(:vulnerability_identifier) do - vulnerability_identifiers.create!( - id: 1244459, - project_id: project.id, - external_type: 'vulnerability-identifier', - external_id: 'vulnerability-identifier', - fingerprint: '0a203e8cd5260a1948edbedc76c7cb91ad6a2e45', - name: 'vulnerability identifier') - end - - let!(:vulnerability_for_first_duplicate) do - create_vulnerability!( - project_id: project.id, - author_id: user.id - ) - end - - let!(:first_finding_duplicate) do - create_finding!( - id: 5606961, - uuid: "bd95c085-71aa-51d7-9bb6-08ae669c262e", - vulnerability_id: vulnerability_for_first_duplicate.id, - report_type: 0, - location_fingerprint: '00049d5119c2cb3bfb3d1ee1f6e031fe925aed75', - primary_identifier_id: vulnerability_identifier.id, - scanner_id: scanner1.id, - project_id: project.id - ) - end - - let!(:vulnerability_for_second_duplicate) do - create_vulnerability!( - project_id: project.id, - author_id: user.id - ) - end - - let!(:second_finding_duplicate) do - create_finding!( - id: 8765432, - uuid: "5b714f58-1176-5b26-8fd5-e11dfcb031b5", - vulnerability_id: vulnerability_for_second_duplicate.id, - report_type: 0, - location_fingerprint: '00049d5119c2cb3bfb3d1ee1f6e031fe925aed75', - primary_identifier_id: vulnerability_identifier.id, - scanner_id: scanner2.id, - project_id: project.id - ) - end - - let!(:vulnerability_for_third_duplicate) do - create_vulnerability!( - project_id: project.id, - author_id: user.id - ) - end - - let!(:third_finding_duplicate) do - create_finding!( - id: 8832995, - uuid: "cfe435fa-b25b-5199-a56d-7b007cc9e2d4", - vulnerability_id: vulnerability_for_third_duplicate.id, - report_type: 0, - location_fingerprint: '00049d5119c2cb3bfb3d1ee1f6e031fe925aed75', - primary_identifier_id: vulnerability_identifier.id, - scanner_id: scanner3.id, - project_id: project.id - ) - end - - let!(:unrelated_finding) do - create_finding!( - id: 9999999, - uuid: Gitlab::UUID.v5(SecureRandom.hex), - vulnerability_id: nil, - report_type: 1, - location_fingerprint: 'random_location_fingerprint', - primary_identifier_id: vulnerability_identifier.id, - scanner_id: unrelated_scanner.id, - project_id: project.id - ) - end - - subject { described_class.new.perform(first_finding_duplicate.id, unrelated_finding.id) } - - before do - stub_const("#{described_class}::DELETE_BATCH_SIZE", 1) - end - - it "removes entries which would result in duplicate UUIDv5" do - expect(vulnerability_findings.count).to eq(4) - - expect { subject }.to change { vulnerability_findings.count }.from(4).to(2) - - expect(vulnerability_findings.pluck(:id)).to match_array([third_finding_duplicate.id, unrelated_finding.id]) - end - - it "removes vulnerabilites without findings" do - expect(vulnerabilities.count).to eq(3) - - expect { subject }.to change { vulnerabilities.count }.from(3).to(1) - - expect(vulnerabilities.pluck(:id)).to match_array([vulnerability_for_third_duplicate.id]) - end - - private - - def create_vulnerability!(project_id:, author_id:, title: 'test', severity: 7, confidence: 7, report_type: 0) - vulnerabilities.create!( - project_id: project_id, - author_id: author_id, - title: title, - severity: severity, - confidence: confidence, - report_type: report_type - ) - end - - # rubocop:disable Metrics/ParameterLists - def create_finding!( - vulnerability_id:, project_id:, scanner_id:, primary_identifier_id:, id: nil, - name: "test", severity: 7, confidence: 7, report_type: 0, - project_fingerprint: '123qweasdzxc', location_fingerprint: 'test', - metadata_version: 'test', raw_metadata: 'test', uuid: SecureRandom.uuid) - params = { - vulnerability_id: vulnerability_id, - project_id: project_id, - name: name, - severity: severity, - confidence: confidence, - report_type: report_type, - project_fingerprint: project_fingerprint, - scanner_id: scanner_id, - primary_identifier_id: vulnerability_identifier.id, - location_fingerprint: location_fingerprint, - metadata_version: metadata_version, - raw_metadata: raw_metadata, - uuid: uuid - } - params[:id] = id unless id.nil? - vulnerability_findings.create!(params) - end - # rubocop:enable Metrics/ParameterLists - - def create_user!(name: "Example User", email: "user@example.com", user_type: nil, created_at: Time.zone.now, confirmed_at: Time.zone.now) - users.create!( - name: name, - email: email, - username: name, - projects_limit: 0, - user_type: user_type, - confirmed_at: confirmed_at - ) - end -end diff --git a/spec/lib/gitlab/background_migration/remove_project_group_link_with_missing_groups_spec.rb b/spec/lib/gitlab/background_migration/remove_project_group_link_with_missing_groups_spec.rb new file mode 100644 index 00000000000..c45c402ab9d --- /dev/null +++ b/spec/lib/gitlab/background_migration/remove_project_group_link_with_missing_groups_spec.rb @@ -0,0 +1,124 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::RemoveProjectGroupLinkWithMissingGroups, :migration, + feature_category: :subgroups, schema: 20230206172702 do + let(:projects) { table(:projects) } + let(:namespaces) { table(:namespaces) } + let(:project_group_links) { table(:project_group_links) } + + let!(:group) do + namespaces.create!( + name: 'Group0', type: 'Group', path: 'space0' + ) + end + + let!(:group_1) do + namespaces.create!( + name: 'Group1', type: 'Group', path: 'space1' + ) + end + + let!(:group_2) do + namespaces.create!( + name: 'Group2', type: 'Group', path: 'space2' + ) + end + + let!(:group_3) do + namespaces.create!( + name: 'Group3', type: 'Group', path: 'space3' + ) + end + + let!(:project_namespace_1) do + namespaces.create!( + name: 'project_1', path: 'project_1', type: 'Project' + ) + end + + let!(:project_namespace_2) do + namespaces.create!( + name: 'project_2', path: 'project_2', type: 'Project' + ) + end + + let!(:project_namespace_3) do + namespaces.create!( + name: 'project_3', path: 'project_3', type: 'Project' + ) + end + + let!(:project_1) do + projects.create!( + name: 'project_1', path: 'project_1', namespace_id: group.id, project_namespace_id: project_namespace_1.id + ) + end + + let!(:project_2) do + projects.create!( + name: 'project_2', path: 'project_2', namespace_id: group.id, project_namespace_id: project_namespace_2.id + ) + end + + let!(:project_3) do + projects.create!( + name: 'project_3', path: 'project_3', namespace_id: group.id, project_namespace_id: project_namespace_3.id + ) + end + + let!(:project_group_link_1) do + project_group_links.create!( + project_id: project_1.id, group_id: group_1.id, group_access: Gitlab::Access::DEVELOPER + ) + end + + let!(:project_group_link_2) do + project_group_links.create!( + project_id: project_2.id, group_id: group_2.id, group_access: Gitlab::Access::DEVELOPER + ) + end + + let!(:project_group_link_3) do + project_group_links.create!( + project_id: project_3.id, group_id: group_3.id, group_access: Gitlab::Access::DEVELOPER + ) + end + + let!(:project_group_link_4) do + project_group_links.create!( + project_id: project_3.id, group_id: group_2.id, group_access: Gitlab::Access::DEVELOPER + ) + end + + subject do + described_class.new( + start_id: project_group_link_1.id, + end_id: project_group_link_4.id, + batch_table: :project_group_links, + batch_column: :id, + sub_batch_size: 1, + pause_ms: 0, + connection: ApplicationRecord.connection + ).perform + end + + it 'removes the `project_group_links` records whose associated group does not exist anymore' do + group_2.delete + + # Schema is fixed to `20230206172702` on this spec. + # This expectation is needed to make sure that the orphaned records are indeed deleted via the migration + # and not via the foreign_key relationship introduced after `20230206172702`, in `20230207002330` + expect(project_group_links.count).to eq(4) + + expect { subject } + .to change { project_group_links.count }.from(4).to(2) + .and change { + project_group_links.where(project_id: project_2.id, group_id: group_2.id).present? + }.from(true).to(false) + .and change { + project_group_links.where(project_id: project_3.id, group_id: group_2.id).present? + }.from(true).to(false) + end +end diff --git a/spec/lib/gitlab/background_migration/remove_vulnerability_finding_links_spec.rb b/spec/lib/gitlab/background_migration/remove_vulnerability_finding_links_spec.rb index 918df8f4442..da14381aae2 100644 --- a/spec/lib/gitlab/background_migration/remove_vulnerability_finding_links_spec.rb +++ b/spec/lib/gitlab/background_migration/remove_vulnerability_finding_links_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::RemoveVulnerabilityFindingLinks, :migration, schema: 20211104165220 do +RSpec.describe Gitlab::BackgroundMigration::RemoveVulnerabilityFindingLinks, :migration, schema: 20220314184009 do let(:vulnerability_findings) { table(:vulnerability_occurrences) } let(:finding_links) { table(:vulnerability_finding_links) } diff --git a/spec/lib/gitlab/background_migration/steal_migrate_merge_request_diff_commit_users_spec.rb b/spec/lib/gitlab/background_migration/steal_migrate_merge_request_diff_commit_users_spec.rb deleted file mode 100644 index 841a7f306d7..00000000000 --- a/spec/lib/gitlab/background_migration/steal_migrate_merge_request_diff_commit_users_spec.rb +++ /dev/null @@ -1,50 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::StealMigrateMergeRequestDiffCommitUsers, schema: 20211012134316 do - let(:migration) { described_class.new } - - describe '#perform' do - it 'processes the background migration' do - spy = instance_spy( - Gitlab::BackgroundMigration::MigrateMergeRequestDiffCommitUsers - ) - - allow(Gitlab::BackgroundMigration::MigrateMergeRequestDiffCommitUsers) - .to receive(:new) - .and_return(spy) - - expect(spy).to receive(:perform).with(1, 4) - expect(migration).to receive(:schedule_next_job) - - migration.perform(1, 4) - end - end - - describe '#schedule_next_job' do - it 'schedules the next job in ascending order' do - Gitlab::Database::BackgroundMigrationJob.create!( - class_name: 'MigrateMergeRequestDiffCommitUsers', - arguments: [10, 20] - ) - - Gitlab::Database::BackgroundMigrationJob.create!( - class_name: 'MigrateMergeRequestDiffCommitUsers', - arguments: [40, 50] - ) - - expect(BackgroundMigrationWorker) - .to receive(:perform_in) - .with(5.minutes, 'StealMigrateMergeRequestDiffCommitUsers', [10, 20]) - - migration.schedule_next_job - end - - it 'does not schedule any new jobs when there are none' do - expect(BackgroundMigrationWorker).not_to receive(:perform_in) - - migration.schedule_next_job - end - end -end diff --git a/spec/lib/gitlab/background_migration/update_timelogs_null_spent_at_spec.rb b/spec/lib/gitlab/background_migration/update_timelogs_null_spent_at_spec.rb deleted file mode 100644 index 908f11aabc3..00000000000 --- a/spec/lib/gitlab/background_migration/update_timelogs_null_spent_at_spec.rb +++ /dev/null @@ -1,40 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::UpdateTimelogsNullSpentAt, schema: 20211215090620 do - let!(:previous_time) { 10.days.ago } - let!(:namespace) { table(:namespaces).create!(name: 'namespace', path: 'namespace') } - let!(:project) { table(:projects).create!(namespace_id: namespace.id) } - let!(:issue) { table(:issues).create!(project_id: project.id) } - let!(:merge_request) { table(:merge_requests).create!(target_project_id: project.id, source_branch: 'master', target_branch: 'feature') } - let!(:timelog1) { create_timelog!(issue_id: issue.id) } - let!(:timelog2) { create_timelog!(merge_request_id: merge_request.id) } - let!(:timelog3) { create_timelog!(issue_id: issue.id, spent_at: previous_time) } - let!(:timelog4) { create_timelog!(merge_request_id: merge_request.id, spent_at: previous_time) } - - subject(:background_migration) { described_class.new } - - before do - table(:timelogs).where.not(id: [timelog3.id, timelog4.id]).update_all(spent_at: nil) - end - - describe '#perform' do - it 'sets correct spent_at' do - background_migration.perform(timelog1.id, timelog4.id) - - expect(timelog1.reload.spent_at).to be_like_time(timelog1.created_at) - expect(timelog2.reload.spent_at).to be_like_time(timelog2.created_at) - expect(timelog3.reload.spent_at).to be_like_time(previous_time) - expect(timelog4.reload.spent_at).to be_like_time(previous_time) - expect(timelog3.reload.spent_at).not_to be_like_time(timelog3.created_at) - expect(timelog4.reload.spent_at).not_to be_like_time(timelog4.created_at) - end - end - - private - - def create_timelog!(**args) - table(:timelogs).create!(**args, time_spent: 1) - end -end diff --git a/spec/lib/gitlab/background_migration/update_timelogs_project_id_spec.rb b/spec/lib/gitlab/background_migration/update_timelogs_project_id_spec.rb deleted file mode 100644 index b8c3bf8f3ac..00000000000 --- a/spec/lib/gitlab/background_migration/update_timelogs_project_id_spec.rb +++ /dev/null @@ -1,52 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::UpdateTimelogsProjectId, schema: 20210826171758 do - let!(:namespace) { table(:namespaces).create!(name: 'namespace', path: 'namespace') } - let!(:project1) { table(:projects).create!(namespace_id: namespace.id) } - let!(:project2) { table(:projects).create!(namespace_id: namespace.id) } - let!(:issue1) { table(:issues).create!(project_id: project1.id) } - let!(:issue2) { table(:issues).create!(project_id: project2.id) } - let!(:merge_request1) { table(:merge_requests).create!(target_project_id: project1.id, source_branch: 'master', target_branch: 'feature') } - let!(:merge_request2) { table(:merge_requests).create!(target_project_id: project2.id, source_branch: 'master', target_branch: 'feature') } - let!(:timelog1) { table(:timelogs).create!(issue_id: issue1.id, time_spent: 60) } - let!(:timelog2) { table(:timelogs).create!(issue_id: issue1.id, time_spent: 60) } - let!(:timelog3) { table(:timelogs).create!(issue_id: issue2.id, time_spent: 60) } - let!(:timelog4) { table(:timelogs).create!(merge_request_id: merge_request1.id, time_spent: 600) } - let!(:timelog5) { table(:timelogs).create!(merge_request_id: merge_request1.id, time_spent: 600) } - let!(:timelog6) { table(:timelogs).create!(merge_request_id: merge_request2.id, time_spent: 600) } - let!(:timelog7) { table(:timelogs).create!(issue_id: issue2.id, time_spent: 60, project_id: project1.id) } - let!(:timelog8) { table(:timelogs).create!(merge_request_id: merge_request2.id, time_spent: 600, project_id: project1.id) } - - describe '#perform' do - context 'when timelogs belong to issues' do - it 'sets correct project_id' do - subject.perform(timelog1.id, timelog3.id) - - expect(timelog1.reload.project_id).to eq(issue1.project_id) - expect(timelog2.reload.project_id).to eq(issue1.project_id) - expect(timelog3.reload.project_id).to eq(issue2.project_id) - end - end - - context 'when timelogs belong to merge requests' do - it 'sets correct project ids' do - subject.perform(timelog4.id, timelog6.id) - - expect(timelog4.reload.project_id).to eq(merge_request1.target_project_id) - expect(timelog5.reload.project_id).to eq(merge_request1.target_project_id) - expect(timelog6.reload.project_id).to eq(merge_request2.target_project_id) - end - end - - context 'when timelogs already belong to projects' do - it 'does not update the project id' do - subject.perform(timelog7.id, timelog8.id) - - expect(timelog7.reload.project_id).to eq(project1.id) - expect(timelog8.reload.project_id).to eq(project1.id) - end - end - end -end diff --git a/spec/lib/gitlab/background_migration/update_users_where_two_factor_auth_required_from_group_spec.rb b/spec/lib/gitlab/background_migration/update_users_where_two_factor_auth_required_from_group_spec.rb deleted file mode 100644 index f16ae489b78..00000000000 --- a/spec/lib/gitlab/background_migration/update_users_where_two_factor_auth_required_from_group_spec.rb +++ /dev/null @@ -1,84 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::UpdateUsersWhereTwoFactorAuthRequiredFromGroup, :migration, schema: 20210826171758 do - include MigrationHelpers::NamespacesHelpers - - let(:group_with_2fa_parent) { create_namespace('parent', Gitlab::VisibilityLevel::PRIVATE, require_two_factor_authentication: true) } - let(:group_with_2fa_child) { create_namespace('child', Gitlab::VisibilityLevel::PRIVATE, parent_id: group_with_2fa_parent.id) } - let(:members_table) { table(:members) } - let(:users_table) { table(:users) } - - subject { described_class.new } - - describe '#perform' do - context 'with group members' do - let(:user_1) { create_user('user@example.com') } - let!(:member) { create_group_member(user_1, group_with_2fa_parent) } - let!(:user_without_group) { create_user('user_without@example.com') } - let(:user_other) { create_user('user_other@example.com') } - let!(:member_other) { create_group_member(user_other, group_with_2fa_parent) } - - it 'updates user when user should be required to establish two factor authentication' do - subject.perform(user_1.id, user_without_group.id) - - expect(user_1.reload.require_two_factor_authentication_from_group).to eq(true) - end - - it 'does not update user who is not in current batch' do - subject.perform(user_1.id, user_without_group.id) - - expect(user_other.reload.require_two_factor_authentication_from_group).to eq(false) - end - - it 'updates all users in current batch' do - subject.perform(user_1.id, user_other.id) - - expect(user_other.reload.require_two_factor_authentication_from_group).to eq(true) - end - - it 'updates user when user is member of group in which parent group requires two factor authentication' do - member.destroy! - - subgroup = create_namespace('subgroup', Gitlab::VisibilityLevel::PRIVATE, require_two_factor_authentication: false, parent_id: group_with_2fa_child.id) - create_group_member(user_1, subgroup) - - subject.perform(user_1.id, user_other.id) - - expect(user_1.reload.require_two_factor_authentication_from_group).to eq(true) - end - - it 'updates user when user is member of a group and the subgroup requires two factor authentication' do - member.destroy! - - parent = create_namespace('other_parent', Gitlab::VisibilityLevel::PRIVATE, require_two_factor_authentication: false) - create_namespace('other_subgroup', Gitlab::VisibilityLevel::PRIVATE, require_two_factor_authentication: true, parent_id: parent.id) - create_group_member(user_1, parent) - - subject.perform(user_1.id, user_other.id) - - expect(user_1.reload.require_two_factor_authentication_from_group).to eq(true) - end - - it 'does not update user when not a member of a group that requires two factor authentication' do - member_other.destroy! - - other_group = create_namespace('other_group', Gitlab::VisibilityLevel::PRIVATE, require_two_factor_authentication: false) - create_group_member(user_other, other_group) - - subject.perform(user_1.id, user_other.id) - - expect(user_other.reload.require_two_factor_authentication_from_group).to eq(false) - end - end - end - - def create_user(email, require_2fa: false) - users_table.create!(email: email, projects_limit: 10, require_two_factor_authentication_from_group: require_2fa) - end - - def create_group_member(user, group) - members_table.create!(user_id: user.id, source_id: group.id, access_level: GroupMember::MAINTAINER, source_type: "Namespace", type: "GroupMember", notification_level: 3) - end -end diff --git a/spec/lib/gitlab/bullet/exclusions_spec.rb b/spec/lib/gitlab/bullet/exclusions_spec.rb index 325b0167f58..ccedfee28c7 100644 --- a/spec/lib/gitlab/bullet/exclusions_spec.rb +++ b/spec/lib/gitlab/bullet/exclusions_spec.rb @@ -3,7 +3,7 @@ require 'fast_spec_helper' require 'tempfile' -RSpec.describe Gitlab::Bullet::Exclusions do +RSpec.describe Gitlab::Bullet::Exclusions, feature_category: :application_performance do let(:config_file) do file = Tempfile.new('bullet.yml') File.basename(file) @@ -78,6 +78,19 @@ RSpec.describe Gitlab::Bullet::Exclusions do expect(described_class.new('_some_bogus_file_').execute).to match([]) end end + + context 'with a Symbol' do + let(:exclude) { [] } + let(:config) { { exclusions: { abc: { exclude: exclude } } } } + + before do + File.write(config_file, YAML.dump(config)) + end + + it 'raises an exception' do + expect { executor }.to raise_error(Psych::DisallowedClass) + end + end end describe '#validate_paths!' do diff --git a/spec/lib/gitlab/cache/client_spec.rb b/spec/lib/gitlab/cache/client_spec.rb index ec22fcdee7e..638fed1a905 100644 --- a/spec/lib/gitlab/cache/client_spec.rb +++ b/spec/lib/gitlab/cache/client_spec.rb @@ -8,14 +8,12 @@ RSpec.describe Gitlab::Cache::Client, feature_category: :source_code_management let(:backend) { Rails.cache } let(:metadata) do Gitlab::Cache::Metadata.new( - caller_id: caller_id, cache_identifier: cache_identifier, feature_category: feature_category, backing_resource: backing_resource ) end - let(:caller_id) { 'caller-id' } let(:cache_identifier) { 'MyClass#cache' } let(:feature_category) { :source_code_management } let(:backing_resource) { :cpu } @@ -32,7 +30,6 @@ RSpec.describe Gitlab::Cache::Client, feature_category: :source_code_management describe '.build_with_metadata' do it 'builds a cache client with metrics support' do attributes = { - caller_id: caller_id, cache_identifier: cache_identifier, feature_category: feature_category, backing_resource: backing_resource diff --git a/spec/lib/gitlab/cache/metadata_spec.rb b/spec/lib/gitlab/cache/metadata_spec.rb index 2e8af7a9c44..d2b79fb8b08 100644 --- a/spec/lib/gitlab/cache/metadata_spec.rb +++ b/spec/lib/gitlab/cache/metadata_spec.rb @@ -5,24 +5,18 @@ require 'spec_helper' RSpec.describe Gitlab::Cache::Metadata, feature_category: :source_code_management do subject(:attributes) do described_class.new( - caller_id: caller_id, cache_identifier: cache_identifier, feature_category: feature_category, backing_resource: backing_resource ) end - let(:caller_id) { 'caller-id' } let(:cache_identifier) { 'ApplicationController#show' } let(:feature_category) { :source_code_management } let(:backing_resource) { :unknown } describe '#initialize' do context 'when optional arguments are not set' do - before do - Gitlab::ApplicationContext.push(caller_id: 'context-id') - end - it 'sets default value for them' do attributes = described_class.new( cache_identifier: cache_identifier, @@ -30,7 +24,6 @@ RSpec.describe Gitlab::Cache::Metadata, feature_category: :source_code_managemen ) expect(attributes.backing_resource).to eq(:unknown) - expect(attributes.caller_id).to eq('context-id') end end @@ -68,12 +61,6 @@ RSpec.describe Gitlab::Cache::Metadata, feature_category: :source_code_managemen end end - describe '#caller_id' do - subject { attributes.caller_id } - - it { is_expected.to eq caller_id } - end - describe '#cache_identifier' do subject { attributes.cache_identifier } diff --git a/spec/lib/gitlab/cache/metrics_spec.rb b/spec/lib/gitlab/cache/metrics_spec.rb index 24b274f4209..76ec0dbfa0b 100644 --- a/spec/lib/gitlab/cache/metrics_spec.rb +++ b/spec/lib/gitlab/cache/metrics_spec.rb @@ -7,14 +7,12 @@ RSpec.describe Gitlab::Cache::Metrics do let(:metadata) do Gitlab::Cache::Metadata.new( - caller_id: caller_id, cache_identifier: cache_identifier, feature_category: feature_category, backing_resource: backing_resource ) end - let(:caller_id) { 'caller-id' } let(:cache_identifier) { 'ApplicationController#show' } let(:feature_category) { :source_code_management } let(:backing_resource) { :unknown } @@ -37,7 +35,6 @@ RSpec.describe Gitlab::Cache::Metrics do .to receive(:increment) .with( { - caller_id: caller_id, cache_identifier: cache_identifier, feature_category: feature_category, backing_resource: backing_resource, @@ -57,7 +54,6 @@ RSpec.describe Gitlab::Cache::Metrics do .to receive(:increment) .with( { - caller_id: caller_id, cache_identifier: cache_identifier, feature_category: feature_category, backing_resource: backing_resource, @@ -86,7 +82,6 @@ RSpec.describe Gitlab::Cache::Metrics do :redis_cache_generation_duration_seconds, 'Duration of Redis cache generation', { - caller_id: caller_id, cache_identifier: cache_identifier, feature_category: feature_category, backing_resource: backing_resource diff --git a/spec/lib/gitlab/ci/ansi2json/signed_state_spec.rb b/spec/lib/gitlab/ci/ansi2json/signed_state_spec.rb new file mode 100644 index 00000000000..671efdf5256 --- /dev/null +++ b/spec/lib/gitlab/ci/ansi2json/signed_state_spec.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Ansi2json::SignedState, feature_category: :continuous_integration do + def build_state(state_class) + state_class.new('', 1000).tap do |state| + state.offset = 1 + state.new_line!(style: { fg: 'some-fg', bg: 'some-bg', mask: 1234 }) + state.set_last_line_offset + state.open_section('hello', 111, {}) + end + end + + let(:state) { build_state(described_class) } + + describe '#initialize' do + it 'restores valid prior state', :aggregate_failures do + new_state = described_class.new(state.encode, 1000) + + expect(new_state.offset).to eq(1) + expect(new_state.inherited_style).to eq({ + bg: 'some-bg', + fg: 'some-fg', + mask: 1234 + }) + expect(new_state.open_sections).to eq({ 'hello' => 111 }) + end + + it 'ignores unsigned prior state', :aggregate_failures do + unsigned = build_state(Gitlab::Ci::Ansi2json::State).encode + expect(::Gitlab::AppLogger).to( + receive(:warn).with( + message: a_string_matching(/signature missing or invalid/), + invalid_state: unsigned + ) + ) + + new_state = described_class.new(unsigned, 0) + + expect(new_state.offset).to eq(0) + expect(new_state.inherited_style).to eq({}) + expect(new_state.open_sections).to eq({}) + end + + it 'ignores bad input', :aggregate_failures do + expect(::Gitlab::AppLogger).to( + receive(:warn).with( + message: a_string_matching(/signature missing or invalid/), + invalid_state: 'abcd' + ) + ) + + new_state = described_class.new('abcd', 0) + + expect(new_state.offset).to eq(0) + expect(new_state.inherited_style).to eq({}) + expect(new_state.open_sections).to eq({}) + end + end + + describe '#encode' do + it 'deterministically signs the state' do + expect(state.encode).to eq state.encode + end + end +end diff --git a/spec/lib/gitlab/ci/ansi2json/state_spec.rb b/spec/lib/gitlab/ci/ansi2json/state_spec.rb new file mode 100644 index 00000000000..9b14231f1be --- /dev/null +++ b/spec/lib/gitlab/ci/ansi2json/state_spec.rb @@ -0,0 +1,83 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Ansi2json::State, feature_category: :continuous_integration do + def build_state(state_class) + state_class.new('', 1000).tap do |state| + state.offset = 1 + state.new_line!(style: { fg: 'some-fg', bg: 'some-bg', mask: 1234 }) + state.set_last_line_offset + state.open_section('hello', 111, {}) + end + end + + let(:state) { build_state(described_class) } + + describe '#initialize' do + it 'restores valid prior state', :aggregate_failures do + new_state = described_class.new(state.encode, 1000) + + expect(new_state.offset).to eq(1) + expect(new_state.inherited_style).to eq({ + bg: 'some-bg', + fg: 'some-fg', + mask: 1234 + }) + expect(new_state.open_sections).to eq({ 'hello' => 111 }) + end + + it 'ignores signed state' do + signed_state = Gitlab::Ci::Ansi2json::SignedState.new('', 1000) + signed_state.offset = 1 + signed_state.new_line!(style: { fg: 'some-fg', bg: 'some-bg', mask: 1234 }) + signed_state.set_last_line_offset + signed_state.open_section('hello', 111, {}) + + encoded = signed_state.encode + expect(::Gitlab::AppLogger).to( + receive(:warn).with( + message: a_string_matching(/decode error/), + invalid_state: encoded, + error: an_instance_of(JSON::ParserError) + ) + ) + new_state = described_class.new(encoded, 1000) + expect(new_state.offset).to eq(0) + expect(new_state.inherited_style).to eq({}) + expect(new_state.open_sections).to eq({}) + end + + it 'ignores invalid Base64 and logs a warning', :aggregate_failures do + expect(::Gitlab::AppLogger).to( + receive(:warn).with( + message: a_string_matching(/decode error/), + invalid_state: '.', + error: an_instance_of(ArgumentError) + ) + ) + + new_state = described_class.new('.', 0) + + expect(new_state.offset).to eq(0) + expect(new_state.inherited_style).to eq({}) + expect(new_state.open_sections).to eq({}) + end + + it 'ignores invalid JSON and logs a warning', :aggregate_failures do + encoded = Base64.urlsafe_encode64('.') + expect(::Gitlab::AppLogger).to( + receive(:warn).with( + message: a_string_matching(/decode error/), + invalid_state: encoded, + error: an_instance_of(JSON::ParserError) + ) + ) + + new_state = described_class.new(encoded, 0) + expect(new_state.offset).to eq(0) + expect(new_state.inherited_style).to eq({}) + expect(new_state.open_sections).to eq({}) + end + end +end diff --git a/spec/lib/gitlab/ci/ansi2json_spec.rb b/spec/lib/gitlab/ci/ansi2json_spec.rb index 0f8f3759834..12eeb8f6cac 100644 --- a/spec/lib/gitlab/ci/ansi2json_spec.rb +++ b/spec/lib/gitlab/ci/ansi2json_spec.rb @@ -2,10 +2,26 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Ansi2json do +RSpec.describe Gitlab::Ci::Ansi2json, feature_category: :continuous_integration do subject { described_class } describe 'lines' do + describe 'verify_state' do + it 'uses SignedState when true' do + expect(Gitlab::Ci::Ansi2json::State).not_to receive(:new) + expect(Gitlab::Ci::Ansi2json::SignedState).to receive(:new).and_call_original + + described_class.convert(StringIO.new('data'), verify_state: true) + end + + it 'uses State when false' do + expect(Gitlab::Ci::Ansi2json::State).to receive(:new).and_call_original + expect(Gitlab::Ci::Ansi2json::SignedState).not_to receive(:new) + + described_class.convert(StringIO.new('data'), verify_state: false) + end + end + it 'prints non-ansi as-is' do expect(convert_json('Hello')).to eq([{ offset: 0, content: [{ text: 'Hello' }] }]) end diff --git a/spec/lib/gitlab/ci/build/cache_spec.rb b/spec/lib/gitlab/ci/build/cache_spec.rb index a8fa14b4b4c..bfb8fb7f21c 100644 --- a/spec/lib/gitlab/ci/build/cache_spec.rb +++ b/spec/lib/gitlab/ci/build/cache_spec.rb @@ -3,16 +3,21 @@ require 'spec_helper' RSpec.describe Gitlab::Ci::Build::Cache do + let(:cache_config) { [] } + let(:pipeline) { double(::Ci::Pipeline) } + let(:cache_seed_a) { double(Gitlab::Ci::Pipeline::Seed::Build::Cache) } + let(:cache_seed_b) { double(Gitlab::Ci::Pipeline::Seed::Build::Cache) } + + subject(:cache) { described_class.new(cache_config, pipeline) } + describe '.initialize' do context 'when the cache is an array' do + let(:cache_config) { [{ key: 'key-a' }, { key: 'key-b' }] } + it 'instantiates an array of cache seeds' do - cache_config = [{ key: 'key-a' }, { key: 'key-b' }] - pipeline = double(::Ci::Pipeline) - cache_seed_a = double(Gitlab::Ci::Pipeline::Seed::Build::Cache) - cache_seed_b = double(Gitlab::Ci::Pipeline::Seed::Build::Cache) allow(Gitlab::Ci::Pipeline::Seed::Build::Cache).to receive(:new).and_return(cache_seed_a, cache_seed_b) - cache = described_class.new(cache_config, pipeline) + cache expect(Gitlab::Ci::Pipeline::Seed::Build::Cache).to have_received(:new).with(pipeline, { key: 'key-a' }, 0) expect(Gitlab::Ci::Pipeline::Seed::Build::Cache).to have_received(:new).with(pipeline, { key: 'key-b' }, 1) @@ -21,16 +26,49 @@ RSpec.describe Gitlab::Ci::Build::Cache do end context 'when the cache is a hash' do + let(:cache_config) { { key: 'key-a' } } + it 'instantiates a cache seed' do - cache_config = { key: 'key-a' } - pipeline = double(::Ci::Pipeline) - cache_seed = double(Gitlab::Ci::Pipeline::Seed::Build::Cache) - allow(Gitlab::Ci::Pipeline::Seed::Build::Cache).to receive(:new).and_return(cache_seed) + allow(Gitlab::Ci::Pipeline::Seed::Build::Cache).to receive(:new).and_return(cache_seed_a) - cache = described_class.new(cache_config, pipeline) + cache expect(Gitlab::Ci::Pipeline::Seed::Build::Cache).to have_received(:new).with(pipeline, cache_config, 0) - expect(cache.instance_variable_get(:@cache)).to eq([cache_seed]) + expect(cache.instance_variable_get(:@cache)).to eq([cache_seed_a]) + end + end + + context 'when the cache is an array with files inside hashes' do + let(:cache_config) { [{ key: { files: ['file1.json'] } }, { key: { files: ['file1.json', 'file2.json'] } }] } + + context 'with ci_fix_for_runner_cache_prefix disabled' do + before do + stub_feature_flags(ci_fix_for_runner_cache_prefix: false) + end + + it 'instantiates a cache seed' do + allow(Gitlab::Ci::Pipeline::Seed::Build::Cache).to receive(:new).and_return(cache_seed_a, cache_seed_b) + + cache + + expect(Gitlab::Ci::Pipeline::Seed::Build::Cache).to have_received(:new) + .with(pipeline, cache_config.first, 0) + expect(Gitlab::Ci::Pipeline::Seed::Build::Cache).to have_received(:new) + .with(pipeline, cache_config.second, 1) + expect(cache.instance_variable_get(:@cache)).to match_array([cache_seed_a, cache_seed_b]) + end + end + + it 'instantiates a cache seed' do + allow(Gitlab::Ci::Pipeline::Seed::Build::Cache).to receive(:new).and_return(cache_seed_a, cache_seed_b) + + cache + + expect(Gitlab::Ci::Pipeline::Seed::Build::Cache).to have_received(:new) + .with(pipeline, cache_config.first, '0_file1') + expect(Gitlab::Ci::Pipeline::Seed::Build::Cache).to have_received(:new) + .with(pipeline, cache_config.second, '1_file1_file2') + expect(cache.instance_variable_get(:@cache)).to match_array([cache_seed_a, cache_seed_b]) end end end @@ -38,10 +76,6 @@ RSpec.describe Gitlab::Ci::Build::Cache do describe '#cache_attributes' do context 'when there are no caches' do it 'returns an empty hash' do - cache_config = [] - pipeline = double(::Ci::Pipeline) - cache = described_class.new(cache_config, pipeline) - attributes = cache.cache_attributes expect(attributes).to eq({}) @@ -51,7 +85,6 @@ RSpec.describe Gitlab::Ci::Build::Cache do context 'when there are caches' do it 'returns the structured attributes for the caches' do cache_config = [{ key: 'key-a' }, { key: 'key-b' }] - pipeline = double(::Ci::Pipeline) cache = described_class.new(cache_config, pipeline) attributes = cache.cache_attributes diff --git a/spec/lib/gitlab/ci/build/context/build_spec.rb b/spec/lib/gitlab/ci/build/context/build_spec.rb index 4fdeffb033a..d4a2af0015f 100644 --- a/spec/lib/gitlab/ci/build/context/build_spec.rb +++ b/spec/lib/gitlab/ci/build/context/build_spec.rb @@ -13,14 +13,29 @@ RSpec.describe Gitlab::Ci::Build::Context::Build, feature_category: :pipeline_co it { is_expected.to include('CI_PIPELINE_IID' => pipeline.iid.to_s) } it { is_expected.to include('CI_PROJECT_PATH' => pipeline.project.full_path) } it { is_expected.to include('CI_JOB_NAME' => 'some-job') } - it { is_expected.to include('CI_BUILD_REF_NAME' => 'master') } + + context 'when FF `ci_remove_legacy_predefined_variables` is disabled' do + before do + stub_feature_flags(ci_remove_legacy_predefined_variables: false) + end + + it { is_expected.to include('CI_BUILD_REF_NAME' => 'master') } + end context 'without passed build-specific attributes' do let(:context) { described_class.new(pipeline) } - it { is_expected.to include('CI_JOB_NAME' => nil) } - it { is_expected.to include('CI_BUILD_REF_NAME' => 'master') } - it { is_expected.to include('CI_PROJECT_PATH' => pipeline.project.full_path) } + it { is_expected.to include('CI_JOB_NAME' => nil) } + it { is_expected.to include('CI_COMMIT_REF_NAME' => 'master') } + it { is_expected.to include('CI_PROJECT_PATH' => pipeline.project.full_path) } + + context 'when FF `ci_remove_legacy_predefined_variables` is disabled' do + before do + stub_feature_flags(ci_remove_legacy_predefined_variables: false) + end + + it { is_expected.to include('CI_BUILD_REF_NAME' => 'master') } + end end context 'when environment:name is provided' do diff --git a/spec/lib/gitlab/ci/build/context/global_spec.rb b/spec/lib/gitlab/ci/build/context/global_spec.rb index d4141eb8389..328b5eb62fa 100644 --- a/spec/lib/gitlab/ci/build/context/global_spec.rb +++ b/spec/lib/gitlab/ci/build/context/global_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Build::Context::Global do +RSpec.describe Gitlab::Ci::Build::Context::Global, feature_category: :pipeline_composition do let(:pipeline) { create(:ci_pipeline) } let(:yaml_variables) { {} } @@ -14,7 +14,14 @@ RSpec.describe Gitlab::Ci::Build::Context::Global do it { is_expected.to include('CI_PROJECT_PATH' => pipeline.project.full_path) } it { is_expected.not_to have_key('CI_JOB_NAME') } - it { is_expected.not_to have_key('CI_BUILD_REF_NAME') } + + context 'when FF `ci_remove_legacy_predefined_variables` is disabled' do + before do + stub_feature_flags(ci_remove_legacy_predefined_variables: false) + end + + it { is_expected.not_to have_key('CI_BUILD_REF_NAME') } + end context 'with passed yaml variables' do let(:yaml_variables) { [{ key: 'SUPPORTED', value: 'parsed', public: true }] } diff --git a/spec/lib/gitlab/ci/components/header_spec.rb b/spec/lib/gitlab/ci/components/header_spec.rb deleted file mode 100644 index b1af4ca9238..00000000000 --- a/spec/lib/gitlab/ci/components/header_spec.rb +++ /dev/null @@ -1,50 +0,0 @@ -# frozen_string_literal: true - -require 'fast_spec_helper' - -RSpec.describe Gitlab::Ci::Components::Header, feature_category: :pipeline_composition do - subject { described_class.new(spec) } - - context 'when spec is valid' do - let(:spec) do - { - spec: { - inputs: { - website: nil, - run: { - options: %w[opt1 opt2] - } - } - } - } - end - - it 'fabricates a spec from valid data' do - expect(subject).not_to be_empty - end - - describe '#inputs' do - it 'fabricates input data' do - input = subject.inputs({ website: 'https//gitlab.com', run: 'opt1' }) - - expect(input.count).to eq 2 - end - end - - describe '#context' do - it 'fabricates interpolation context' do - ctx = subject.context({ website: 'https//gitlab.com', run: 'opt1' }) - - expect(ctx).to be_valid - end - end - end - - context 'when spec is empty' do - let(:spec) { { spec: {} } } - - it 'returns an empty header' do - expect(subject).to be_empty - end - end -end diff --git a/spec/lib/gitlab/ci/components/instance_path_spec.rb b/spec/lib/gitlab/ci/components/instance_path_spec.rb index fbe5e0b9d42..e037c37c817 100644 --- a/spec/lib/gitlab/ci/components/instance_path_spec.rb +++ b/spec/lib/gitlab/ci/components/instance_path_spec.rb @@ -98,6 +98,37 @@ RSpec.describe Gitlab::Ci::Components::InstancePath, feature_category: :pipeline end end + context 'when version is `~latest`' do + let(:version) { '~latest' } + + context 'when project is a catalog resource' do + before do + create(:catalog_resource, project: existing_project) + end + + context 'when project has releases' do + let_it_be(:releases) do + [ + create(:release, project: existing_project, sha: 'sha-1', released_at: Time.zone.now - 1.day), + create(:release, project: existing_project, sha: 'sha-2', released_at: Time.zone.now) + ] + end + + it 'returns the sha of the latest release' do + expect(path.sha).to eq(releases.last.sha) + end + end + + context 'when project does not have releases' do + it { expect(path.sha).to be_nil } + end + end + + context 'when project is not a catalog resource' do + it { expect(path.sha).to be_nil } + end + end + context 'when project does not exist' do let(:project_path) { 'non-existent/project' } diff --git a/spec/lib/gitlab/ci/config/entry/job_spec.rb b/spec/lib/gitlab/ci/config/entry/job_spec.rb index c8b4a8b8a0e..39a88fc7721 100644 --- a/spec/lib/gitlab/ci/config/entry/job_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/job_spec.rb @@ -595,6 +595,39 @@ RSpec.describe Gitlab::Ci::Config::Entry::Job, feature_category: :pipeline_compo end end end + + context 'when job is not a pages job' do + let(:name) { :rspec } + + context 'if the config contains a publish entry' do + let(:entry) { described_class.new({ script: 'echo', publish: 'foo' }, name: name) } + + it 'is invalid' do + expect(entry).not_to be_valid + expect(entry.errors).to include /job publish can only be used within a `pages` job/ + end + end + end + + context 'when job is a pages job' do + let(:name) { :pages } + + context 'when it does not have a publish entry' do + let(:entry) { described_class.new({ script: 'echo' }, name: name) } + + it 'is valid' do + expect(entry).to be_valid + end + end + + context 'when it has a publish entry' do + let(:entry) { described_class.new({ script: 'echo', publish: 'foo' }, name: name) } + + it 'is valid' do + expect(entry).to be_valid + end + end + end end describe '#relevant?' do diff --git a/spec/lib/gitlab/ci/config/entry/publish_spec.rb b/spec/lib/gitlab/ci/config/entry/publish_spec.rb new file mode 100644 index 00000000000..53ad868a05e --- /dev/null +++ b/spec/lib/gitlab/ci/config/entry/publish_spec.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Config::Entry::Publish, feature_category: :pages do + let(:publish) { described_class.new(config) } + + describe 'validations' do + context 'when publish config value is correct' do + let(:config) { 'dist/static' } + + describe '#config' do + it 'returns the publish directory' do + expect(publish.config).to eq config + end + end + + describe '#valid?' do + it 'is valid' do + expect(publish).to be_valid + end + end + end + + context 'when the value has a wrong type' do + let(:config) { { test: true } } + + it 'reports an error' do + expect(publish.errors) + .to include 'publish config should be a string' + end + end + end + + describe '.default' do + it 'returns the default value' do + expect(described_class.default).to eq 'public' + end + end +end diff --git a/spec/lib/gitlab/ci/config/external/file/artifact_spec.rb b/spec/lib/gitlab/ci/config/external/file/artifact_spec.rb index 52b8dcbcb44..ea1e42de901 100644 --- a/spec/lib/gitlab/ci/config/external/file/artifact_spec.rb +++ b/spec/lib/gitlab/ci/config/external/file/artifact_spec.rb @@ -4,9 +4,11 @@ require 'spec_helper' RSpec.describe Gitlab::Ci::Config::External::File::Artifact, feature_category: :pipeline_composition do let(:parent_pipeline) { create(:ci_pipeline) } + let(:project) { parent_pipeline.project } let(:variables) {} let(:context) do - Gitlab::Ci::Config::External::Context.new(variables: variables, parent_pipeline: parent_pipeline) + Gitlab::Ci::Config::External::Context + .new(variables: variables, parent_pipeline: parent_pipeline, project: project) end let(:external_file) { described_class.new(params, context) } @@ -43,7 +45,7 @@ RSpec.describe Gitlab::Ci::Config::External::File::Artifact, feature_category: : end describe 'when used in non child pipeline context' do - let(:parent_pipeline) { nil } + let(:context) { Gitlab::Ci::Config::External::Context.new } let(:params) { { artifact: 'generated.yml' } } let(:expected_error) do @@ -201,7 +203,7 @@ RSpec.describe Gitlab::Ci::Config::External::File::Artifact, feature_category: : it { is_expected.to eq( - context_project: nil, + context_project: project.full_path, context_sha: nil, type: :artifact, location: 'generated.yml', @@ -218,7 +220,7 @@ RSpec.describe Gitlab::Ci::Config::External::File::Artifact, feature_category: : it { is_expected.to eq( - context_project: nil, + context_project: project.full_path, context_sha: nil, type: :artifact, location: 'generated.yml', @@ -227,4 +229,35 @@ RSpec.describe Gitlab::Ci::Config::External::File::Artifact, feature_category: : } end end + + describe '#to_hash' do + context 'when interpolation is being used' do + let!(:job) { create(:ci_build, name: 'generator', pipeline: parent_pipeline) } + let!(:artifacts) { create(:ci_job_artifact, :archive, job: job) } + let!(:metadata) { create(:ci_job_artifact, :metadata, job: job) } + + before do + allow_next_instance_of(Gitlab::Ci::ArtifactFileReader) do |reader| + allow(reader).to receive(:read).and_return(template) + end + end + + let(:template) do + <<~YAML + spec: + inputs: + env: + --- + deploy: + script: deploy $[[ inputs.env ]] + YAML + end + + let(:params) { { artifact: 'generated.yml', job: 'generator', with: { env: 'production' } } } + + it 'correctly interpolates content' do + expect(external_file.to_hash).to eq({ deploy: { script: 'deploy production' } }) + end + end + end end diff --git a/spec/lib/gitlab/ci/config/external/file/base_spec.rb b/spec/lib/gitlab/ci/config/external/file/base_spec.rb index 959dcdf31af..1c5918f77ca 100644 --- a/spec/lib/gitlab/ci/config/external/file/base_spec.rb +++ b/spec/lib/gitlab/ci/config/external/file/base_spec.rb @@ -3,14 +3,15 @@ require 'spec_helper' RSpec.describe Gitlab::Ci::Config::External::File::Base, feature_category: :pipeline_composition do + let_it_be(:project) { create(:project) } let(:variables) {} - let(:context_params) { { sha: 'HEAD', variables: variables } } - let(:context) { Gitlab::Ci::Config::External::Context.new(**context_params) } + let(:context_params) { { sha: 'HEAD', variables: variables, project: project } } + let(:ctx) { Gitlab::Ci::Config::External::Context.new(**context_params) } let(:test_class) do Class.new(described_class) do - def initialize(params, context) - @location = params + def initialize(params, ctx) + @location = params[:location] super end @@ -18,15 +19,18 @@ RSpec.describe Gitlab::Ci::Config::External::File::Base, feature_category: :pipe def validate_context! # no-op end + + def content + params[:content] + end end end - subject(:file) { test_class.new(location, context) } + let(:content) { 'key: value' } - before do - allow_any_instance_of(test_class) - .to receive(:content).and_return('key: value') + subject(:file) { test_class.new({ location: location, content: content }, ctx) } + before do allow_any_instance_of(Gitlab::Ci::Config::External::Context) .to receive(:check_execution_time!) end @@ -51,7 +55,7 @@ RSpec.describe Gitlab::Ci::Config::External::File::Base, feature_category: :pipe describe '#valid?' do subject(:valid?) do - Gitlab::Ci::Config::External::Mapper::Verifier.new(context).process([file]) + Gitlab::Ci::Config::External::Mapper::Verifier.new(ctx).process([file]) file.valid? end @@ -87,7 +91,12 @@ RSpec.describe Gitlab::Ci::Config::External::File::Base, feature_category: :pipe context 'when there are YAML syntax errors' do let(:location) { 'some/file/secret_file_name.yml' } - let(:variables) { Gitlab::Ci::Variables::Collection.new([{ 'key' => 'GITLAB_TOKEN', 'value' => 'secret_file_name', 'masked' => true }]) } + + let(:variables) do + Gitlab::Ci::Variables::Collection.new( + [{ 'key' => 'GITLAB_TOKEN', 'value' => 'secret_file_name', 'masked' => true }] + ) + end before do allow_any_instance_of(test_class) @@ -96,15 +105,16 @@ RSpec.describe Gitlab::Ci::Config::External::File::Base, feature_category: :pipe it 'is not a valid file' do expect(valid?).to be_falsy - expect(file.error_message).to eq('Included file `some/file/xxxxxxxxxxxxxxxx.yml` does not have valid YAML syntax!') + expect(file.error_message) + .to eq('`some/file/xxxxxxxxxxxxxxxx.yml`: content does not have a valid YAML syntax') end end context 'when the class has no validate_context!' do let(:test_class) do Class.new(described_class) do - def initialize(params, context) - @location = params + def initialize(params, ctx) + @location = params[:location] super end @@ -117,6 +127,88 @@ RSpec.describe Gitlab::Ci::Config::External::File::Base, feature_category: :pipe expect { valid? }.to raise_error(NotImplementedError) end end + + context 'when interpolation is disabled but there is a spec header' do + before do + stub_feature_flags(ci_includable_files_interpolation: false) + end + + let(:location) { 'some-location.yml' } + + let(:content) do + <<~YAML + spec: + include: + website: + --- + run: + script: deploy $[[ inputs.website ]] + YAML + end + + it 'returns an error saying that interpolation is disabled' do + expect(valid?).to be_falsy + expect(file.errors) + .to include('`some-location.yml`: can not evaluate included file because interpolation is disabled') + end + end + + context 'when interpolation was unsuccessful' do + let(:location) { 'some-location.yml' } + + context 'when context key is missing' do + let(:content) do + <<~YAML + spec: + inputs: + --- + run: + script: deploy $[[ inputs.abcd ]] + YAML + end + + it 'surfaces interpolation errors' do + expect(valid?).to be_falsy + expect(file.errors) + .to include('`some-location.yml`: interpolation interrupted by errors, unknown interpolation key: `abcd`') + end + end + + context 'when header is invalid' do + let(:content) do + <<~YAML + spec: + a: abc + --- + run: + script: deploy $[[ inputs.abcd ]] + YAML + end + + it 'surfaces header errors' do + expect(valid?).to be_falsy + expect(file.errors) + .to include('`some-location.yml`: header:spec config contains unknown keys: a') + end + end + + context 'when header is not a hash' do + let(:content) do + <<~YAML + spec: abcd + --- + run: + script: deploy $[[ inputs.abcd ]] + YAML + end + + it 'surfaces header errors' do + expect(valid?).to be_falsy + expect(file.errors) + .to contain_exactly('`some-location.yml`: header:spec config should be a hash') + end + end + end end describe '#to_hash' do @@ -142,7 +234,7 @@ RSpec.describe Gitlab::Ci::Config::External::File::Base, feature_category: :pipe it { is_expected.to eq( - context_project: nil, + context_project: project.full_path, context_sha: 'HEAD' ) } @@ -154,13 +246,13 @@ RSpec.describe Gitlab::Ci::Config::External::File::Base, feature_category: :pipe subject(:eql) { file.eql?(other_file) } context 'when the other file has the same params' do - let(:other_file) { test_class.new(location, context) } + let(:other_file) { test_class.new({ location: location, content: content }, ctx) } it { is_expected.to eq(true) } end context 'when the other file has not the same params' do - let(:other_file) { test_class.new('some/other/file', context) } + let(:other_file) { test_class.new({ location: 'some/other/file', content: content }, ctx) } it { is_expected.to eq(false) } end @@ -172,14 +264,15 @@ RSpec.describe Gitlab::Ci::Config::External::File::Base, feature_category: :pipe subject(:filehash) { file.hash } context 'with a project' do - let(:project) { create(:project) } let(:context_params) { { project: project, sha: 'HEAD', variables: variables } } - it { is_expected.to eq([location, project.full_path, 'HEAD'].hash) } + it { is_expected.to eq([{ location: location, content: content }, project.full_path, 'HEAD'].hash) } end context 'without a project' do - it { is_expected.to eq([location, nil, 'HEAD'].hash) } + let(:context_params) { { sha: 'HEAD', variables: variables } } + + it { is_expected.to eq([{ location: location, content: content }, nil, 'HEAD'].hash) } end end end diff --git a/spec/lib/gitlab/ci/config/external/file/component_spec.rb b/spec/lib/gitlab/ci/config/external/file/component_spec.rb index 1562e571060..fe811bce9fe 100644 --- a/spec/lib/gitlab/ci/config/external/file/component_spec.rb +++ b/spec/lib/gitlab/ci/config/external/file/component_spec.rb @@ -121,7 +121,7 @@ RSpec.describe Gitlab::Ci::Config::External::File::Component, feature_category: it 'is invalid' do expect(subject).to be_falsy - expect(external_resource.error_message).to match(/does not have valid YAML syntax/) + expect(external_resource.error_message).to match(/does not have a valid YAML syntax/) end end end @@ -176,4 +176,35 @@ RSpec.describe Gitlab::Ci::Config::External::File::Component, feature_category: variables: context.variables) end end + + describe '#to_hash' do + context 'when interpolation is being used' do + let(:response) do + ServiceResponse.success(payload: { content: content, path: path }) + end + + let(:path) do + instance_double(::Gitlab::Ci::Components::InstancePath, project: project, sha: '12345') + end + + let(:content) do + <<~YAML + spec: + inputs: + env: + --- + deploy: + script: deploy $[[ inputs.env ]] + YAML + end + + let(:params) do + { component: 'gitlab.com/acme/components/my-component@1.0', with: { env: 'production' } } + end + + it 'correctly interpolates the content' do + expect(external_resource.to_hash).to eq({ deploy: { script: 'deploy production' } }) + end + end + end end diff --git a/spec/lib/gitlab/ci/config/external/file/local_spec.rb b/spec/lib/gitlab/ci/config/external/file/local_spec.rb index 2bac8a6968b..6c0242050a6 100644 --- a/spec/lib/gitlab/ci/config/external/file/local_spec.rb +++ b/spec/lib/gitlab/ci/config/external/file/local_spec.rb @@ -228,6 +228,34 @@ RSpec.describe Gitlab::Ci::Config::External::File::Local, feature_category: :pip expect(local_file.to_hash).to include(:rspec) end end + + context 'when interpolaton is being used' do + let(:local_file_content) do + <<~YAML + spec: + inputs: + website: + --- + test: + script: cap deploy $[[ inputs.website ]] + YAML + end + + let(:location) { '/lib/gitlab/ci/templates/existent-file.yml' } + let(:params) { { local: location, with: { website: 'gitlab.com' } } } + + before do + allow_any_instance_of(described_class) + .to receive(:fetch_local_content) + .and_return(local_file_content) + end + + it 'correctly interpolates the local template' do + expect(local_file).to be_valid + expect(local_file.to_hash) + .to eq({ test: { script: 'cap deploy gitlab.com' } }) + end + end end describe '#metadata' do diff --git a/spec/lib/gitlab/ci/config/external/file/project_spec.rb b/spec/lib/gitlab/ci/config/external/file/project_spec.rb index 0ef39a22932..59522e7ab7d 100644 --- a/spec/lib/gitlab/ci/config/external/file/project_spec.rb +++ b/spec/lib/gitlab/ci/config/external/file/project_spec.rb @@ -289,4 +289,37 @@ RSpec.describe Gitlab::Ci::Config::External::File::Project, feature_category: :p } end end + + describe '#to_hash' do + context 'when interpolation is being used' do + before do + project.repository.create_file( + user, + 'template-file.yml', + template, + message: 'Add template', + branch_name: 'master' + ) + end + + let(:template) do + <<~YAML + spec: + inputs: + name: + --- + rspec: + script: rspec --suite $[[ inputs.name ]] + YAML + end + + let(:params) do + { file: 'template-file.yml', ref: 'master', project: project.full_path, with: { name: 'abc' } } + end + + it 'correctly interpolates the content' do + expect(project_file.to_hash).to eq({ rspec: { script: 'rspec --suite abc' } }) + end + end + end end diff --git a/spec/lib/gitlab/ci/config/external/file/remote_spec.rb b/spec/lib/gitlab/ci/config/external/file/remote_spec.rb index f8986e8fa10..30a407d3a8f 100644 --- a/spec/lib/gitlab/ci/config/external/file/remote_spec.rb +++ b/spec/lib/gitlab/ci/config/external/file/remote_spec.rb @@ -234,15 +234,13 @@ RSpec.describe Gitlab::Ci::Config::External::File::Remote, feature_category: :pi end describe '#to_hash' do - subject(:to_hash) { remote_file.to_hash } - before do stub_full_request(location).to_return(body: remote_file_content) end context 'with a valid remote file' do it 'returns the content as a hash' do - expect(to_hash).to eql( + expect(remote_file.to_hash).to eql( before_script: ["apt-get update -qq && apt-get install -y -qq sqlite3 libsqlite3-dev nodejs", "ruby -v", "which ruby", @@ -262,7 +260,7 @@ RSpec.describe Gitlab::Ci::Config::External::File::Remote, feature_category: :pi end it 'returns the content as a hash' do - expect(to_hash).to eql( + expect(remote_file.to_hash).to eql( include: [ { local: 'another-file.yml', rules: [{ exists: ['Dockerfile'] }] } @@ -270,5 +268,38 @@ RSpec.describe Gitlab::Ci::Config::External::File::Remote, feature_category: :pi ) end end + + context 'when interpolation has been used' do + let_it_be(:project) { create(:project) } + + let(:remote_file_content) do + <<~YAML + spec: + inputs: + include: + --- + include: + - local: $[[ inputs.include ]] + rules: + - exists: [Dockerfile] + YAML + end + + let(:params) { { remote: location, with: { include: 'some-file.yml' } } } + + let(:context_params) do + { sha: '12345', variables: variables, project: project } + end + + it 'returns the content as a hash' do + expect(remote_file).to be_valid + expect(remote_file.to_hash).to eql( + include: [ + { local: 'some-file.yml', + rules: [{ exists: ['Dockerfile'] }] } + ] + ) + end + end end end diff --git a/spec/lib/gitlab/ci/config/external/file/template_spec.rb b/spec/lib/gitlab/ci/config/external/file/template_spec.rb index 79fd4203c3e..89b8240ce9b 100644 --- a/spec/lib/gitlab/ci/config/external/file/template_spec.rb +++ b/spec/lib/gitlab/ci/config/external/file/template_spec.rb @@ -130,4 +130,37 @@ RSpec.describe Gitlab::Ci::Config::External::File::Template, feature_category: : ) } end + + describe '#to_hash' do + context 'when interpolation is being used' do + before do + allow(Gitlab::Template::GitlabCiYmlTemplate) + .to receive(:find) + .and_return(template_double) + end + + let(:template_double) do + instance_double(Gitlab::Template::GitlabCiYmlTemplate, content: template_content) + end + + let(:template_content) do + <<~YAML + spec: + inputs: + env: + --- + deploy: + script: deploy $[[ inputs.env ]] + YAML + end + + let(:params) do + { template: template, with: { env: 'production' } } + end + + it 'correctly interpolates the content' do + expect(template_file.to_hash).to eq({ deploy: { script: 'deploy production' } }) + end + end + end end diff --git a/spec/lib/gitlab/ci/config/external/interpolator_spec.rb b/spec/lib/gitlab/ci/config/external/interpolator_spec.rb new file mode 100644 index 00000000000..b274e5950e4 --- /dev/null +++ b/spec/lib/gitlab/ci/config/external/interpolator_spec.rb @@ -0,0 +1,312 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Config::External::Interpolator, feature_category: :pipeline_composition do + let_it_be(:project) { create(:project) } + + let(:ctx) { instance_double(Gitlab::Ci::Config::External::Context, project: project) } + let(:result) { ::Gitlab::Ci::Config::Yaml::Result.new(config: [header, content]) } + + subject { described_class.new(result, arguments, ctx) } + + context 'when input data is valid' do + let(:header) do + { spec: { inputs: { website: nil } } } + end + + let(:content) do + { test: 'deploy $[[ inputs.website ]]' } + end + + let(:arguments) do + { website: 'gitlab.com' } + end + + it 'correctly interpolates the config' do + subject.interpolate! + + expect(subject).to be_valid + expect(subject.to_hash).to eq({ test: 'deploy gitlab.com' }) + end + end + + context 'when config has a syntax error' do + let(:result) { ::Gitlab::Ci::Config::Yaml::Result.new(error: ArgumentError.new) } + + let(:arguments) do + { website: 'gitlab.com' } + end + + it 'surfaces an error about invalid config' do + subject.interpolate! + + expect(subject).not_to be_valid + expect(subject.error_message).to eq subject.errors.first + expect(subject.errors).to include 'content does not have a valid YAML syntax' + end + end + + context 'when spec header is invalid' do + let(:header) do + { spec: { arguments: { website: nil } } } + end + + let(:content) do + { test: 'deploy $[[ inputs.website ]]' } + end + + let(:arguments) do + { website: 'gitlab.com' } + end + + it 'surfaces an error about invalid header' do + subject.interpolate! + + expect(subject).not_to be_valid + expect(subject.error_message).to eq subject.errors.first + expect(subject.errors).to include('header:spec config contains unknown keys: arguments') + end + end + + context 'when interpolation block is invalid' do + let(:header) do + { spec: { inputs: { website: nil } } } + end + + let(:content) do + { test: 'deploy $[[ inputs.abc ]]' } + end + + let(:arguments) do + { website: 'gitlab.com' } + end + + it 'correctly interpolates the config' do + subject.interpolate! + + expect(subject).not_to be_valid + expect(subject.errors).to include 'unknown interpolation key: `abc`' + expect(subject.error_message).to eq 'interpolation interrupted by errors, unknown interpolation key: `abc`' + end + end + + context 'when provided interpolation argument is invalid' do + let(:header) do + { spec: { inputs: { website: nil } } } + end + + let(:content) do + { test: 'deploy $[[ inputs.website ]]' } + end + + let(:arguments) do + { website: ['gitlab.com'] } + end + + it 'correctly interpolates the config' do + subject.interpolate! + + expect(subject).not_to be_valid + expect(subject.error_message).to eq subject.errors.first + expect(subject.errors).to include 'unsupported value in input argument `website`' + end + end + + context 'when multiple interpolation blocks are invalid' do + let(:header) do + { spec: { inputs: { website: nil } } } + end + + let(:content) do + { test: 'deploy $[[ inputs.something.abc ]] $[[ inputs.cde ]] $[[ efg ]]' } + end + + let(:arguments) do + { website: 'gitlab.com' } + end + + it 'correctly interpolates the config' do + subject.interpolate! + + expect(subject).not_to be_valid + expect(subject.error_message).to eq 'interpolation interrupted by errors, unknown interpolation key: `something`' + end + end + + describe '#to_hash' do + context 'when interpolation is disabled' do + before do + stub_feature_flags(ci_includable_files_interpolation: false) + end + + let(:header) do + { spec: { inputs: { website: nil } } } + end + + let(:content) do + { test: 'deploy $[[ inputs.website ]]' } + end + + let(:arguments) { {} } + + it 'returns an empty hash' do + subject.interpolate! + + expect(subject.to_hash).to be_empty + end + end + + context 'when interpolation is not used' do + let(:result) do + ::Gitlab::Ci::Config::Yaml::Result.new(config: content) + end + + let(:content) do + { test: 'deploy production' } + end + + let(:arguments) { nil } + + it 'returns original content' do + subject.interpolate! + + expect(subject.to_hash).to eq(content) + end + end + + context 'when interpolation is available' do + let(:header) do + { spec: { inputs: { website: nil } } } + end + + let(:content) do + { test: 'deploy $[[ inputs.website ]]' } + end + + let(:arguments) do + { website: 'gitlab.com' } + end + + it 'correctly interpolates content' do + subject.interpolate! + + expect(subject.to_hash).to eq({ test: 'deploy gitlab.com' }) + end + end + end + + describe '#ready?' do + let(:header) do + { spec: { inputs: { website: nil } } } + end + + let(:content) do + { test: 'deploy $[[ inputs.website ]]' } + end + + let(:arguments) do + { website: 'gitlab.com' } + end + + it 'returns false if interpolation has not been done yet' do + expect(subject).not_to be_ready + end + + it 'returns true if interpolation has been performed' do + subject.interpolate! + + expect(subject).to be_ready + end + + context 'when interpolation can not be performed' do + let(:result) do + ::Gitlab::Ci::Config::Yaml::Result.new(error: ArgumentError.new) + end + + it 'returns true if interpolator has preliminary errors' do + expect(subject).to be_ready + end + + it 'returns true if interpolation has been attempted' do + subject.interpolate! + + expect(subject).to be_ready + end + end + end + + describe '#interpolate?' do + let(:header) do + { spec: { inputs: { website: nil } } } + end + + let(:content) do + { test: 'deploy $[[ inputs.something.abc ]] $[[ inputs.cde ]] $[[ efg ]]' } + end + + let(:arguments) do + { website: 'gitlab.com' } + end + + context 'when interpolation can be performed' do + it 'will perform interpolation' do + expect(subject.interpolate?).to eq true + end + end + + context 'when interpolation is disabled' do + before do + stub_feature_flags(ci_includable_files_interpolation: false) + end + + it 'will not perform interpolation' do + expect(subject.interpolate?).to eq false + end + end + + context 'when an interpolation header is missing' do + let(:header) { nil } + + it 'will not perform interpolation' do + expect(subject.interpolate?).to eq false + end + end + + context 'when interpolator has preliminary errors' do + let(:result) do + ::Gitlab::Ci::Config::Yaml::Result.new(error: ArgumentError.new) + end + + it 'will not perform interpolation' do + expect(subject.interpolate?).to eq false + end + end + end + + describe '#has_header?' do + let(:content) do + { test: 'deploy $[[ inputs.something.abc ]] $[[ inputs.cde ]] $[[ efg ]]' } + end + + let(:arguments) do + { website: 'gitlab.com' } + end + + context 'when header is an empty hash' do + let(:header) { {} } + + it 'does not have a header available' do + expect(subject).not_to have_header + end + end + + context 'when header is not specified' do + let(:header) { nil } + + it 'does not have a header available' do + expect(subject).not_to have_header + end + end + end +end diff --git a/spec/lib/gitlab/ci/config/external/mapper/matcher_spec.rb b/spec/lib/gitlab/ci/config/external/mapper/matcher_spec.rb index 6ca4fd24e61..719c75dca80 100644 --- a/spec/lib/gitlab/ci/config/external/mapper/matcher_spec.rb +++ b/spec/lib/gitlab/ci/config/external/mapper/matcher_spec.rb @@ -16,28 +16,56 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper::Matcher, feature_category: subject(:matcher) { described_class.new(context) } describe '#process' do - let(:locations) do - [ - { local: 'file.yml' }, - { file: 'file.yml', project: 'namespace/project' }, - { component: 'gitlab.com/org/component@1.0' }, - { remote: 'https://example.com/.gitlab-ci.yml' }, - { template: 'file.yml' }, - { artifact: 'generated.yml', job: 'test' } - ] + subject(:process) { matcher.process(locations) } + + context 'with ci_include_components FF disabled' do + before do + stub_feature_flags(ci_include_components: false) + end + + let(:locations) do + [ + { local: 'file.yml' }, + { file: 'file.yml', project: 'namespace/project' }, + { remote: 'https://example.com/.gitlab-ci.yml' }, + { template: 'file.yml' }, + { artifact: 'generated.yml', job: 'test' } + ] + end + + it 'returns an array of file objects' do + is_expected.to contain_exactly( + an_instance_of(Gitlab::Ci::Config::External::File::Local), + an_instance_of(Gitlab::Ci::Config::External::File::Project), + an_instance_of(Gitlab::Ci::Config::External::File::Remote), + an_instance_of(Gitlab::Ci::Config::External::File::Template), + an_instance_of(Gitlab::Ci::Config::External::File::Artifact) + ) + end end - subject(:process) { matcher.process(locations) } + context 'with ci_include_components FF enabled' do + let(:locations) do + [ + { local: 'file.yml' }, + { file: 'file.yml', project: 'namespace/project' }, + { component: 'gitlab.com/org/component@1.0' }, + { remote: 'https://example.com/.gitlab-ci.yml' }, + { template: 'file.yml' }, + { artifact: 'generated.yml', job: 'test' } + ] + end - it 'returns an array of file objects' do - is_expected.to contain_exactly( - an_instance_of(Gitlab::Ci::Config::External::File::Local), - an_instance_of(Gitlab::Ci::Config::External::File::Project), - an_instance_of(Gitlab::Ci::Config::External::File::Component), - an_instance_of(Gitlab::Ci::Config::External::File::Remote), - an_instance_of(Gitlab::Ci::Config::External::File::Template), - an_instance_of(Gitlab::Ci::Config::External::File::Artifact) - ) + it 'returns an array of file objects' do + is_expected.to contain_exactly( + an_instance_of(Gitlab::Ci::Config::External::File::Local), + an_instance_of(Gitlab::Ci::Config::External::File::Project), + an_instance_of(Gitlab::Ci::Config::External::File::Component), + an_instance_of(Gitlab::Ci::Config::External::File::Remote), + an_instance_of(Gitlab::Ci::Config::External::File::Template), + an_instance_of(Gitlab::Ci::Config::External::File::Artifact) + ) + end end context 'when a location is not valid' do diff --git a/spec/lib/gitlab/ci/config/external/mapper/variables_expander_spec.rb b/spec/lib/gitlab/ci/config/external/mapper/variables_expander_spec.rb index e27e8034faa..5def516bb1e 100644 --- a/spec/lib/gitlab/ci/config/external/mapper/variables_expander_spec.rb +++ b/spec/lib/gitlab/ci/config/external/mapper/variables_expander_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::External::Mapper::VariablesExpander, feature_category: :pipeline_composition do +RSpec.describe Gitlab::Ci::Config::External::Mapper::VariablesExpander, feature_category: :secrets_management do let_it_be(:variables) do Gitlab::Ci::Variables::Collection.new.tap do |variables| variables.append(key: 'VARIABLE1', value: 'hello') diff --git a/spec/lib/gitlab/ci/config/external/processor_spec.rb b/spec/lib/gitlab/ci/config/external/processor_spec.rb index 97f600baf25..74afb3b1e97 100644 --- a/spec/lib/gitlab/ci/config/external/processor_spec.rb +++ b/spec/lib/gitlab/ci/config/external/processor_spec.rb @@ -221,7 +221,7 @@ RSpec.describe Gitlab::Ci::Config::External::Processor, feature_category: :pipel it 'raises an error' do expect { processor.perform }.to raise_error( described_class::IncludeError, - "Included file `lib/gitlab/ci/templates/template.yml` does not have valid YAML syntax!" + '`lib/gitlab/ci/templates/template.yml`: content does not have a valid YAML syntax' ) end end diff --git a/spec/lib/gitlab/ci/config/header/spec_spec.rb b/spec/lib/gitlab/ci/config/header/spec_spec.rb index cb4237f84ce..74cfb39dfd5 100644 --- a/spec/lib/gitlab/ci/config/header/spec_spec.rb +++ b/spec/lib/gitlab/ci/config/header/spec_spec.rb @@ -28,6 +28,18 @@ RSpec.describe Gitlab::Ci::Config::Header::Spec, feature_category: :pipeline_com end end + context 'when spec contains a required value' do + let(:spec_hash) do + { inputs: { foo: nil } } + end + + it 'parses the config correctly' do + expect(config).to be_valid + expect(config.errors).to be_empty + expect(config.value).to eq({ inputs: { foo: {} } }) + end + end + context 'when spec contains unknown keywords' do let(:spec_hash) { { test: 123 } } let(:expected_errors) { ['spec config contains unknown keys: test'] } diff --git a/spec/lib/gitlab/ci/config/yaml/result_spec.rb b/spec/lib/gitlab/ci/config/yaml/result_spec.rb index eda15ee9eb2..72d96349668 100644 --- a/spec/lib/gitlab/ci/config/yaml/result_spec.rb +++ b/spec/lib/gitlab/ci/config/yaml/result_spec.rb @@ -9,11 +9,34 @@ RSpec.describe Gitlab::Ci::Config::Yaml::Result, feature_category: :pipeline_com expect(result).not_to have_header end - it 'has a header when config is an array of hashes' do - result = described_class.new(config: [{ a: 1 }, { b: 2 }]) + context 'when config is an array of hashes' do + context 'when first document matches the header schema' do + it 'has a header' do + result = described_class.new(config: [{ spec: { inputs: {} } }, { b: 2 }]) + + expect(result).to have_header + expect(result.header).to eq({ spec: { inputs: {} } }) + expect(result.content).to eq({ b: 2 }) + end + end + + context 'when first document does not match the header schema' do + it 'does not have header' do + result = described_class.new(config: [{ a: 1 }, { b: 2 }]) + + expect(result).not_to have_header + expect(result.content).to eq({ a: 1 }) + end + end + end + + context 'when the first document is undefined' do + it 'does not have header' do + result = described_class.new(config: [nil, { a: 1 }]) - expect(result).to have_header - expect(result.header).to eq({ a: 1 }) + expect(result).not_to have_header + expect(result.content).to be_nil + end end it 'raises an error when reading a header when there is none' do diff --git a/spec/lib/gitlab/ci/config/yaml_spec.rb b/spec/lib/gitlab/ci/config/yaml_spec.rb index f4b70069bbe..beb872071d2 100644 --- a/spec/lib/gitlab/ci/config/yaml_spec.rb +++ b/spec/lib/gitlab/ci/config/yaml_spec.rb @@ -113,18 +113,85 @@ RSpec.describe Gitlab::Ci::Config::Yaml, feature_category: :pipeline_composition end describe '.load_result!' do + let_it_be(:project) { create(:project) } + + subject(:result) { described_class.load_result!(yaml, project: project) } + context 'when syntax is invalid' do let(:yaml) { 'some: invalid: syntax' } it 'returns an invalid result object' do - result = described_class.load_result!(yaml) - expect(result).not_to be_valid expect(result.error).to be_a ::Gitlab::Config::Loader::FormatError end end - context 'when syntax is valid and contains a header document' do + context 'when the first document is a header' do + context 'with explicit document start marker' do + let(:yaml) do + <<~YAML + --- + spec: + --- + b: 2 + YAML + end + + it 'considers the first document as header and the second as content' do + expect(result).to be_valid + expect(result.error).to be_nil + expect(result.header).to eq({ spec: nil }) + expect(result.content).to eq({ b: 2 }) + end + end + end + + context 'when first document is empty' do + let(:yaml) do + <<~YAML + --- + --- + b: 2 + YAML + end + + it 'considers the first document as header and the second as content' do + expect(result).not_to have_header + end + end + + context 'when first document is an empty hash' do + let(:yaml) do + <<~YAML + {} + --- + b: 2 + YAML + end + + it 'returns second document as a content' do + expect(result).not_to have_header + expect(result.content).to eq({ b: 2 }) + end + end + + context 'when first an array' do + let(:yaml) do + <<~YAML + --- + - a + - b + --- + b: 2 + YAML + end + + it 'considers the first document as header and the second as content' do + expect(result).not_to have_header + end + end + + context 'when the first document is not a header' do let(:yaml) do <<~YAML a: 1 @@ -133,15 +200,62 @@ RSpec.describe Gitlab::Ci::Config::Yaml, feature_category: :pipeline_composition YAML end - let(:project) { create(:project) } + it 'considers the first document as content for backwards compatibility' do + expect(result).to be_valid + expect(result.error).to be_nil + expect(result).not_to have_header + expect(result.content).to eq({ a: 1 }) + end + + context 'with explicit document start marker' do + let(:yaml) do + <<~YAML + --- + a: 1 + --- + b: 2 + YAML + end + + it 'considers the first document as content for backwards compatibility' do + expect(result).to be_valid + expect(result.error).to be_nil + expect(result).not_to have_header + expect(result.content).to eq({ a: 1 }) + end + end + end - it 'returns a result object' do - result = described_class.load_result!(yaml, project: project) + context 'when the first document is not a header and second document is empty' do + let(:yaml) do + <<~YAML + a: 1 + --- + YAML + end + it 'considers the first document as content' do expect(result).to be_valid expect(result.error).to be_nil - expect(result.header).to eq({ a: 1 }) - expect(result.content).to eq({ b: 2 }) + expect(result).not_to have_header + expect(result.content).to eq({ a: 1 }) + end + + context 'with explicit document start marker' do + let(:yaml) do + <<~YAML + --- + a: 1 + --- + YAML + end + + it 'considers the first document as content' do + expect(result).to be_valid + expect(result.error).to be_nil + expect(result).not_to have_header + expect(result.content).to eq({ a: 1 }) + end end end end diff --git a/spec/lib/gitlab/ci/input/arguments/default_spec.rb b/spec/lib/gitlab/ci/input/arguments/default_spec.rb index 6b5dd441eb7..bc0cee6ac4e 100644 --- a/spec/lib/gitlab/ci/input/arguments/default_spec.rb +++ b/spec/lib/gitlab/ci/input/arguments/default_spec.rb @@ -27,6 +27,12 @@ RSpec.describe Gitlab::Ci::Input::Arguments::Default, feature_category: :pipelin expect(argument.to_hash).to eq({ website: 'https://gitlab.com' }) end + it 'returns an error if the default argument has not been recognized' do + argument = described_class.new(:website, { default: ['gitlab.com'] }, 'abc') + + expect(argument).not_to be_valid + end + it 'returns an error if the argument has not been fabricated correctly' do argument = described_class.new(:website, { required: 'https://gitlab.com' }, 'https://example.gitlab.com') @@ -40,6 +46,8 @@ RSpec.describe Gitlab::Ci::Input::Arguments::Default, feature_category: :pipelin it 'does not match specs different configuration keyword' do expect(described_class.matches?({ options: %w[a b] })).to be false + expect(described_class.matches?('a b c')).to be false + expect(described_class.matches?(%w[default a])).to be false end end end diff --git a/spec/lib/gitlab/ci/input/arguments/options_spec.rb b/spec/lib/gitlab/ci/input/arguments/options_spec.rb index afa279ad48d..17e3469b294 100644 --- a/spec/lib/gitlab/ci/input/arguments/options_spec.rb +++ b/spec/lib/gitlab/ci/input/arguments/options_spec.rb @@ -29,7 +29,7 @@ RSpec.describe Gitlab::Ci::Input::Arguments::Options, feature_category: :pipelin argument = described_class.new(:website, { options: { a: 1 } }, 'opt1') expect(argument).not_to be_valid - expect(argument.errors.first).to eq '`website` input: argument value opt1 not allowlisted' + expect(argument.errors.first).to eq '`website` input: argument specification invalid' end it 'returns an empty value if it is allowlisted' do @@ -47,6 +47,8 @@ RSpec.describe Gitlab::Ci::Input::Arguments::Options, feature_category: :pipelin it 'does not match specs different configuration keyword' do expect(described_class.matches?({ default: 'abc' })).to be false + expect(described_class.matches?(['options'])).to be false + expect(described_class.matches?('options')).to be false end end end diff --git a/spec/lib/gitlab/ci/input/arguments/required_spec.rb b/spec/lib/gitlab/ci/input/arguments/required_spec.rb index 0c2ffc282ea..847272998c2 100644 --- a/spec/lib/gitlab/ci/input/arguments/required_spec.rb +++ b/spec/lib/gitlab/ci/input/arguments/required_spec.rb @@ -34,6 +34,10 @@ RSpec.describe Gitlab::Ci::Input::Arguments::Required, feature_category: :pipeli expect(described_class.matches?('')).to be true end + it 'matches specs with an empty hash configuration' do + expect(described_class.matches?({})).to be true + end + it 'does not match specs with configuration' do expect(described_class.matches?({ options: %w[a b] })).to be false end diff --git a/spec/lib/gitlab/ci/jwt_v2_spec.rb b/spec/lib/gitlab/ci/jwt_v2_spec.rb index 5eeab658a8e..21fd7e3adcf 100644 --- a/spec/lib/gitlab/ci/jwt_v2_spec.rb +++ b/spec/lib/gitlab/ci/jwt_v2_spec.rb @@ -5,7 +5,13 @@ require 'spec_helper' RSpec.describe Gitlab::Ci::JwtV2 do let(:namespace) { build_stubbed(:namespace) } let(:project) { build_stubbed(:project, namespace: namespace) } - let(:user) { build_stubbed(:user) } + let(:user) do + build_stubbed( + :user, + identities: [build_stubbed(:identity, extern_uid: '1', provider: 'github')] + ) + end + let(:pipeline) { build_stubbed(:ci_pipeline, ref: 'auto-deploy-2020-03-19') } let(:aud) { described_class::DEFAULT_AUD } @@ -33,6 +39,18 @@ RSpec.describe Gitlab::Ci::JwtV2 do end end + it 'includes user identities when enabled' do + expect(user).to receive(:pass_user_identities_to_ci_jwt).and_return(true) + identities = payload[:user_identities].map { |identity| identity.slice(:extern_uid, :provider) } + expect(identities).to eq([{ extern_uid: '1', provider: 'github' }]) + end + + it 'does not include user identities when disabled' do + expect(user).to receive(:pass_user_identities_to_ci_jwt).and_return(false) + + expect(payload).not_to include(:user_identities) + end + context 'when given an aud' do let(:aud) { 'AWS' } diff --git a/spec/lib/gitlab/ci/parsers/security/sast_spec.rb b/spec/lib/gitlab/ci/parsers/security/sast_spec.rb index f6113308201..d1ce6808d23 100644 --- a/spec/lib/gitlab/ci/parsers/security/sast_spec.rb +++ b/spec/lib/gitlab/ci/parsers/security/sast_spec.rb @@ -13,8 +13,8 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Sast do context "when passing valid report" do # rubocop: disable Layout/LineLength where(:report_format, :report_version, :scanner_length, :finding_length, :identifier_length, :file_path, :start_line, :end_line, :primary_identifiers_length) do - :sast | '14.0.0' | 1 | 5 | 6 | 'groovy/src/main/java/com/gitlab/security_products/tests/App.groovy' | 47 | 47 | nil - :sast_semgrep_for_multiple_findings | '14.0.4' | 1 | 2 | 6 | 'app/app.py' | 39 | nil | 2 + :sast | '15.0.0' | 1 | 5 | 6 | 'groovy/src/main/java/com/gitlab/security_products/tests/App.groovy' | 47 | 47 | nil + :sast_semgrep_for_multiple_findings | '15.0.4' | 1 | 2 | 6 | 'app/app.py' | 39 | nil | 2 end # rubocop: enable Layout/LineLength diff --git a/spec/lib/gitlab/ci/parsers/security/secret_detection_spec.rb b/spec/lib/gitlab/ci/parsers/security/secret_detection_spec.rb index e8f1d617cb7..13999b2a9e5 100644 --- a/spec/lib/gitlab/ci/parsers/security/secret_detection_spec.rb +++ b/spec/lib/gitlab/ci/parsers/security/secret_detection_spec.rb @@ -39,7 +39,7 @@ RSpec.describe Gitlab::Ci::Parsers::Security::SecretDetection do end it "generates expected metadata_version" do - expect(report.findings.first.metadata_version).to eq('14.1.2') + expect(report.findings.first.metadata_version).to eq('15.0.0') end end end diff --git a/spec/lib/gitlab/ci/pipeline/seed/build/cache_spec.rb b/spec/lib/gitlab/ci/pipeline/seed/build/cache_spec.rb index c264ea3bece..e6ff82810ae 100644 --- a/spec/lib/gitlab/ci/pipeline/seed/build/cache_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/seed/build/cache_spec.rb @@ -7,8 +7,9 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build::Cache do let_it_be(:head_sha) { project.repository.head_commit.id } let_it_be(:pipeline) { create(:ci_pipeline, project: project, sha: head_sha) } let(:index) { 1 } + let(:cache_prefix) { index } - let(:processor) { described_class.new(pipeline, config, index) } + let(:processor) { described_class.new(pipeline, config, cache_prefix) } describe '#attributes' do subject { processor.attributes } @@ -32,18 +33,38 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build::Cache do } end - it { is_expected.to include(config.merge(key: "a_key")) } + it { is_expected.to include(config.merge(key: 'a_key')) } end context 'with cache:key:files' do + context 'with ci_fix_for_runner_cache_prefix disabled' do + before do + stub_feature_flags(ci_fix_for_runner_cache_prefix: false) + end + + shared_examples 'default key' do + let(:config) do + { key: { files: files } } + end + + context 'without a prefix' do + it 'uses default key with an index as a prefix' do + expected = { key: '1-default' } + + is_expected.to include(expected) + end + end + end + end + shared_examples 'default key' do let(:config) do { key: { files: files } } end context 'without a prefix' do - it 'uses default key with an index as a prefix' do - expected = { key: '1-default' } + it 'uses default key with an index and file names as a prefix' do + expected = { key: "#{cache_prefix}-default" } is_expected.to include(expected) end @@ -61,9 +82,9 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build::Cache do end context 'without a prefix' do - it 'builds a string key with an index as a prefix' do + it 'builds a string key with an index and file names as a prefix' do expected = { - key: '1-703ecc8fef1635427a1f86a8a1a308831c122392', + key: "#{cache_prefix}-703ecc8fef1635427a1f86a8a1a308831c122392", paths: ['vendor/ruby'] } @@ -74,30 +95,41 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build::Cache do context 'with existing files' do let(:files) { ['VERSION', 'Gemfile.zip'] } + let(:cache_prefix) { '1_VERSION_Gemfile' } it_behaves_like 'version and gemfile files' end context 'with files starting with ./' do let(:files) { ['Gemfile.zip', './VERSION'] } + let(:cache_prefix) { '1_Gemfile_' } it_behaves_like 'version and gemfile files' end + context 'with no files' do + let(:files) { [] } + + it_behaves_like 'default key' + end + context 'with files ending with /' do let(:files) { ['Gemfile.zip/'] } + let(:cache_prefix) { '1_Gemfile' } it_behaves_like 'default key' end context 'with new line in filenames' do - let(:files) { ["Gemfile.zip\nVERSION"] } + let(:files) { ['Gemfile.zip\nVERSION'] } + let(:cache_prefix) { '1_Gemfile' } it_behaves_like 'default key' end context 'with missing files' do let(:files) { ['project-gemfile.lock', ''] } + let(:cache_prefix) { '1_project-gemfile_' } it_behaves_like 'default key' end @@ -113,8 +145,8 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build::Cache do end context 'without a prefix' do - it 'builds a string key with an index as a prefix' do - expected = { key: '1-74bf43fb1090f161bdd4e265802775dbda2f03d1' } + it 'builds a string key with an index and file names as a prefix' do + expected = { key: "#{cache_prefix}-74bf43fb1090f161bdd4e265802775dbda2f03d1" } is_expected.to include(expected) end @@ -123,18 +155,21 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build::Cache do context 'with directory' do let(:files) { ['foo/bar'] } + let(:cache_prefix) { '1_foo/bar' } it_behaves_like 'foo/bar directory key' end context 'with directory ending in slash' do let(:files) { ['foo/bar/'] } + let(:cache_prefix) { '1_foo/bar/' } it_behaves_like 'foo/bar directory key' end context 'with directories ending in slash star' do let(:files) { ['foo/bar/*'] } + let(:cache_prefix) { '1_foo/bar/*' } it_behaves_like 'foo/bar directory key' end diff --git a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb index ce68e741d00..86a11111283 100644 --- a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb @@ -152,7 +152,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build, feature_category: :pipeline_co it 'includes cache options' do cache_options = { options: { - cache: [a_hash_including(key: '0-f155568ad0933d8358f66b846133614f76dd0ca4')] + cache: [a_hash_including(key: '0_VERSION-f155568ad0933d8358f66b846133614f76dd0ca4')] } } @@ -798,7 +798,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build, feature_category: :pipeline_co [ [[{ if: '$CI_JOB_NAME == "rspec" && $VAR == null', when: 'on_failure' }]], [[{ if: '$VARIABLE != null', when: 'delayed', start_in: '1 day' }, { if: '$CI_JOB_NAME == "rspec"', when: 'on_failure' }]], - [[{ if: '$VARIABLE == "the wrong value"', when: 'delayed', start_in: '1 day' }, { if: '$CI_BUILD_NAME == "rspec"', when: 'on_failure' }]] + [[{ if: '$VARIABLE == "the wrong value"', when: 'delayed', start_in: '1 day' }, { if: '$CI_JOB_NAME == "rspec"', when: 'on_failure' }]] ] end @@ -811,6 +811,30 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build, feature_category: :pipeline_co end end + context 'when FF `ci_remove_legacy_predefined_variables` is disabled' do + before do + stub_feature_flags(ci_remove_legacy_predefined_variables: false) + end + + context 'with an explicit `when: on_failure`' do + where(:rule_set) do + [ + [[{ if: '$CI_JOB_NAME == "rspec" && $VAR == null', when: 'on_failure' }]], + [[{ if: '$VARIABLE != null', when: 'delayed', start_in: '1 day' }, { if: '$CI_JOB_NAME == "rspec"', when: 'on_failure' }]], + [[{ if: '$VARIABLE == "the wrong value"', when: 'delayed', start_in: '1 day' }, { if: '$CI_BUILD_NAME == "rspec"', when: 'on_failure' }]] + ] + end + + with_them do + it { is_expected.to be_included } + + it 'correctly populates when:' do + expect(seed_build.attributes).to include(when: 'on_failure') + end + end + end + end + context 'with an explicit `when: delayed`' do where(:rule_set) do [ diff --git a/spec/lib/gitlab/ci/status/composite_spec.rb b/spec/lib/gitlab/ci/status/composite_spec.rb index cceabc35e85..cbf0976c976 100644 --- a/spec/lib/gitlab/ci/status/composite_spec.rb +++ b/spec/lib/gitlab/ci/status/composite_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Status::Composite do +RSpec.describe Gitlab::Ci::Status::Composite, feature_category: :continuous_integration do let_it_be(:pipeline) { create(:ci_pipeline) } before_all do @@ -15,6 +15,18 @@ RSpec.describe Gitlab::Ci::Status::Composite do end end + describe '.initialize' do + subject(:composite_status) { described_class.new(all_statuses) } + + context 'when passing a single status' do + let(:all_statuses) { @statuses[:success] } + + it 'raises ArgumentError' do + expect { composite_status }.to raise_error(ArgumentError, 'all_jobs needs to respond to `.pluck`') + end + end + end + describe '#status' do using RSpec::Parameterized::TableSyntax @@ -51,8 +63,8 @@ RSpec.describe Gitlab::Ci::Status::Composite do %i(created success pending) | false | 'running' | false %i(skipped success failed) | false | 'failed' | false %i(skipped success failed) | true | 'skipped' | false - %i(success manual) | true | 'pending' | false - %i(success failed created) | true | 'pending' | false + %i(success manual) | true | 'manual' | false + %i(success failed created) | true | 'running' | false end with_them do diff --git a/spec/lib/gitlab/ci/status/processable/waiting_for_resource_spec.rb b/spec/lib/gitlab/ci/status/processable/waiting_for_resource_spec.rb index 26087fd771c..e1baa1097e4 100644 --- a/spec/lib/gitlab/ci/status/processable/waiting_for_resource_spec.rb +++ b/spec/lib/gitlab/ci/status/processable/waiting_for_resource_spec.rb @@ -2,12 +2,25 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Status::Processable::WaitingForResource do +RSpec.describe Gitlab::Ci::Status::Processable::WaitingForResource, feature_category: :continuous_integration do let(:user) { create(:user) } + let(:processable) { create(:ci_build, :waiting_for_resource, :resource_group) } - subject do - processable = create(:ci_build, :waiting_for_resource, :resource_group) - described_class.new(Gitlab::Ci::Status::Core.new(processable, user)) + subject { described_class.new(Gitlab::Ci::Status::Core.new(processable, user)) } + + it 'fabricates status with correct details' do + expect(subject.has_action?).to eq false + end + + context 'when resource is retained by a build' do + before do + processable.resource_group.assign_resource_to(create(:ci_build)) + end + + it 'fabricates status with correct details' do + expect(subject.has_action?).to eq true + expect(subject.action_path).to include 'jobs' + end end describe '#illustration' do diff --git a/spec/lib/gitlab/ci/trace/chunked_io_spec.rb b/spec/lib/gitlab/ci/trace/chunked_io_spec.rb index 63625244fe8..7a926a06f16 100644 --- a/spec/lib/gitlab/ci/trace/chunked_io_spec.rb +++ b/spec/lib/gitlab/ci/trace/chunked_io_spec.rb @@ -446,15 +446,5 @@ RSpec.describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do expect(Ci::BuildTraceChunk.where(build: build).count).to eq(0) end - - context 'when the job does not have archived trace' do - it 'leaves a message in sidekiq log' do - expect(Sidekiq.logger).to receive(:warn).with( - message: 'The job does not have archived trace but going to be destroyed.', - job_id: build.id).and_call_original - - subject - end - end end end diff --git a/spec/lib/gitlab/ci/variables/builder/pipeline_spec.rb b/spec/lib/gitlab/ci/variables/builder/pipeline_spec.rb index f8770457083..0a079a69682 100644 --- a/spec/lib/gitlab/ci/variables/builder/pipeline_spec.rb +++ b/spec/lib/gitlab/ci/variables/builder/pipeline_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Variables::Builder::Pipeline, feature_category: :pipeline_composition do +RSpec.describe Gitlab::Ci::Variables::Builder::Pipeline, feature_category: :secrets_management do let_it_be(:project) { create_default(:project, :repository, create_tag: 'test').freeze } let_it_be(:user) { create(:user) } @@ -30,15 +30,13 @@ RSpec.describe Gitlab::Ci::Variables::Builder::Pipeline, feature_category: :pipe CI_COMMIT_REF_PROTECTED CI_COMMIT_TIMESTAMP CI_COMMIT_AUTHOR - CI_BUILD_REF - CI_BUILD_BEFORE_SHA - CI_BUILD_REF_NAME - CI_BUILD_REF_SLUG ]) end - context 'when the pipeline is running for a tag' do - let(:pipeline) { build(:ci_empty_pipeline, :created, project: project, ref: 'test', tag: true) } + context 'when FF `ci_remove_legacy_predefined_variables` is disabled' do + before do + stub_feature_flags(ci_remove_legacy_predefined_variables: false) + end it 'includes all predefined variables in a valid order' do keys = subject.pluck(:key) @@ -52,6 +50,7 @@ RSpec.describe Gitlab::Ci::Variables::Builder::Pipeline, feature_category: :pipe CI_COMMIT_BEFORE_SHA CI_COMMIT_REF_NAME CI_COMMIT_REF_SLUG + CI_COMMIT_BRANCH CI_COMMIT_MESSAGE CI_COMMIT_TITLE CI_COMMIT_DESCRIPTION @@ -62,11 +61,69 @@ RSpec.describe Gitlab::Ci::Variables::Builder::Pipeline, feature_category: :pipe CI_BUILD_BEFORE_SHA CI_BUILD_REF_NAME CI_BUILD_REF_SLUG + ]) + end + end + + context 'when the pipeline is running for a tag' do + let(:pipeline) { build(:ci_empty_pipeline, :created, project: project, ref: 'test', tag: true) } + + it 'includes all predefined variables in a valid order' do + keys = subject.pluck(:key) + + expect(keys).to contain_exactly(*%w[ + CI_PIPELINE_IID + CI_PIPELINE_SOURCE + CI_PIPELINE_CREATED_AT + CI_COMMIT_SHA + CI_COMMIT_SHORT_SHA + CI_COMMIT_BEFORE_SHA + CI_COMMIT_REF_NAME + CI_COMMIT_REF_SLUG + CI_COMMIT_MESSAGE + CI_COMMIT_TITLE + CI_COMMIT_DESCRIPTION + CI_COMMIT_REF_PROTECTED + CI_COMMIT_TIMESTAMP + CI_COMMIT_AUTHOR CI_COMMIT_TAG CI_COMMIT_TAG_MESSAGE - CI_BUILD_TAG ]) end + + context 'when FF `ci_remove_legacy_predefined_variables` is disabled' do + before do + stub_feature_flags(ci_remove_legacy_predefined_variables: false) + end + + it 'includes all predefined variables in a valid order' do + keys = subject.pluck(:key) + + expect(keys).to contain_exactly(*%w[ + CI_PIPELINE_IID + CI_PIPELINE_SOURCE + CI_PIPELINE_CREATED_AT + CI_COMMIT_SHA + CI_COMMIT_SHORT_SHA + CI_COMMIT_BEFORE_SHA + CI_COMMIT_REF_NAME + CI_COMMIT_REF_SLUG + CI_COMMIT_MESSAGE + CI_COMMIT_TITLE + CI_COMMIT_DESCRIPTION + CI_COMMIT_REF_PROTECTED + CI_COMMIT_TIMESTAMP + CI_COMMIT_AUTHOR + CI_BUILD_REF + CI_BUILD_BEFORE_SHA + CI_BUILD_REF_NAME + CI_BUILD_REF_SLUG + CI_COMMIT_TAG + CI_COMMIT_TAG_MESSAGE + CI_BUILD_TAG + ]) + end + end end context 'when merge request is present' do @@ -305,10 +362,24 @@ RSpec.describe Gitlab::Ci::Variables::Builder::Pipeline, feature_category: :pipe expect(subject.to_hash.keys) .not_to include( 'CI_COMMIT_TAG', - 'CI_COMMIT_TAG_MESSAGE', - 'CI_BUILD_TAG' + 'CI_COMMIT_TAG_MESSAGE' ) end + + context 'when FF `ci_remove_legacy_predefined_variables` is disabled' do + before do + stub_feature_flags(ci_remove_legacy_predefined_variables: false) + end + + it 'does not expose tag variables' do + expect(subject.to_hash.keys) + .not_to include( + 'CI_COMMIT_TAG', + 'CI_COMMIT_TAG_MESSAGE', + 'CI_BUILD_TAG' + ) + end + end end context 'without a commit' do diff --git a/spec/lib/gitlab/ci/variables/builder_spec.rb b/spec/lib/gitlab/ci/variables/builder_spec.rb index 215b18ea614..10974993fa4 100644 --- a/spec/lib/gitlab/ci/variables/builder_spec.rb +++ b/spec/lib/gitlab/ci/variables/builder_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Variables::Builder, :clean_gitlab_redis_cache, feature_category: :pipeline_composition do +RSpec.describe Gitlab::Ci::Variables::Builder, :clean_gitlab_redis_cache, feature_category: :secrets_management do include Ci::TemplateHelpers let_it_be(:group) { create(:group) } let_it_be(:project) { create(:project, :repository, namespace: group) } @@ -35,10 +35,6 @@ RSpec.describe Gitlab::Ci::Variables::Builder, :clean_gitlab_redis_cache, featur value: '1' }, { key: 'CI_ENVIRONMENT_NAME', value: 'test' }, - { key: 'CI_BUILD_NAME', - value: 'rspec:test 1' }, - { key: 'CI_BUILD_STAGE', - value: job.stage_name }, { key: 'CI', value: 'true' }, { key: 'GITLAB_CI', @@ -51,6 +47,10 @@ RSpec.describe Gitlab::Ci::Variables::Builder, :clean_gitlab_redis_cache, featur value: Gitlab.config.gitlab.port.to_s }, { key: 'CI_SERVER_PROTOCOL', value: Gitlab.config.gitlab.protocol }, + { key: 'CI_SERVER_SHELL_SSH_HOST', + value: Gitlab.config.gitlab_shell.ssh_host.to_s }, + { key: 'CI_SERVER_SHELL_SSH_PORT', + value: Gitlab.config.gitlab_shell.ssh_port.to_s }, { key: 'CI_SERVER_NAME', value: 'GitLab' }, { key: 'CI_SERVER_VERSION', @@ -101,6 +101,8 @@ RSpec.describe Gitlab::Ci::Variables::Builder, :clean_gitlab_redis_cache, featur value: project.pages_url }, { key: 'CI_API_V4_URL', value: API::Helpers::Version.new('v4').root_url }, + { key: 'CI_API_GRAPHQL_URL', + value: Gitlab::Routing.url_helpers.api_graphql_url }, { key: 'CI_TEMPLATE_REGISTRY_HOST', value: template_registry_host }, { key: 'CI_PIPELINE_IID', @@ -133,14 +135,6 @@ RSpec.describe Gitlab::Ci::Variables::Builder, :clean_gitlab_redis_cache, featur value: pipeline.git_commit_timestamp }, { key: 'CI_COMMIT_AUTHOR', value: pipeline.git_author_full_text }, - { key: 'CI_BUILD_REF', - value: job.sha }, - { key: 'CI_BUILD_BEFORE_SHA', - value: job.before_sha }, - { key: 'CI_BUILD_REF_NAME', - value: job.ref }, - { key: 'CI_BUILD_REF_SLUG', - value: job.ref_slug }, { key: 'YAML_VARIABLE', value: 'value' }, { key: 'GITLAB_USER_ID', @@ -160,6 +154,151 @@ RSpec.describe Gitlab::Ci::Variables::Builder, :clean_gitlab_redis_cache, featur it { expect(subject.to_runner_variables).to eq(predefined_variables) } + context 'when FF `ci_remove_legacy_predefined_variables` is disabled' do + before do + stub_feature_flags(ci_remove_legacy_predefined_variables: false) + end + + let(:predefined_variables) do + [ + { key: 'CI_JOB_NAME', + value: 'rspec:test 1' }, + { key: 'CI_JOB_NAME_SLUG', + value: 'rspec-test-1' }, + { key: 'CI_JOB_STAGE', + value: job.stage_name }, + { key: 'CI_NODE_TOTAL', + value: '1' }, + { key: 'CI_ENVIRONMENT_NAME', + value: 'test' }, + { key: 'CI_BUILD_NAME', + value: 'rspec:test 1' }, + { key: 'CI_BUILD_STAGE', + value: job.stage_name }, + { key: 'CI', + value: 'true' }, + { key: 'GITLAB_CI', + value: 'true' }, + { key: 'CI_SERVER_URL', + value: Gitlab.config.gitlab.url }, + { key: 'CI_SERVER_HOST', + value: Gitlab.config.gitlab.host }, + { key: 'CI_SERVER_PORT', + value: Gitlab.config.gitlab.port.to_s }, + { key: 'CI_SERVER_PROTOCOL', + value: Gitlab.config.gitlab.protocol }, + { key: 'CI_SERVER_SHELL_SSH_HOST', + value: Gitlab.config.gitlab_shell.ssh_host.to_s }, + { key: 'CI_SERVER_SHELL_SSH_PORT', + value: Gitlab.config.gitlab_shell.ssh_port.to_s }, + { key: 'CI_SERVER_NAME', + value: 'GitLab' }, + { key: 'CI_SERVER_VERSION', + value: Gitlab::VERSION }, + { key: 'CI_SERVER_VERSION_MAJOR', + value: Gitlab.version_info.major.to_s }, + { key: 'CI_SERVER_VERSION_MINOR', + value: Gitlab.version_info.minor.to_s }, + { key: 'CI_SERVER_VERSION_PATCH', + value: Gitlab.version_info.patch.to_s }, + { key: 'CI_SERVER_REVISION', + value: Gitlab.revision }, + { key: 'GITLAB_FEATURES', + value: project.licensed_features.join(',') }, + { key: 'CI_PROJECT_ID', + value: project.id.to_s }, + { key: 'CI_PROJECT_NAME', + value: project.path }, + { key: 'CI_PROJECT_TITLE', + value: project.title }, + { key: 'CI_PROJECT_DESCRIPTION', + value: project.description }, + { key: 'CI_PROJECT_PATH', + value: project.full_path }, + { key: 'CI_PROJECT_PATH_SLUG', + value: project.full_path_slug }, + { key: 'CI_PROJECT_NAMESPACE', + value: project.namespace.full_path }, + { key: 'CI_PROJECT_NAMESPACE_ID', + value: project.namespace.id.to_s }, + { key: 'CI_PROJECT_ROOT_NAMESPACE', + value: project.namespace.root_ancestor.path }, + { key: 'CI_PROJECT_URL', + value: project.web_url }, + { key: 'CI_PROJECT_VISIBILITY', + value: "private" }, + { key: 'CI_PROJECT_REPOSITORY_LANGUAGES', + value: project.repository_languages.map(&:name).join(',').downcase }, + { key: 'CI_PROJECT_CLASSIFICATION_LABEL', + value: project.external_authorization_classification_label }, + { key: 'CI_DEFAULT_BRANCH', + value: project.default_branch }, + { key: 'CI_CONFIG_PATH', + value: project.ci_config_path_or_default }, + { key: 'CI_PAGES_DOMAIN', + value: Gitlab.config.pages.host }, + { key: 'CI_PAGES_URL', + value: project.pages_url }, + { key: 'CI_API_V4_URL', + value: API::Helpers::Version.new('v4').root_url }, + { key: 'CI_API_GRAPHQL_URL', + value: Gitlab::Routing.url_helpers.api_graphql_url }, + { key: 'CI_TEMPLATE_REGISTRY_HOST', + value: template_registry_host }, + { key: 'CI_PIPELINE_IID', + value: pipeline.iid.to_s }, + { key: 'CI_PIPELINE_SOURCE', + value: pipeline.source }, + { key: 'CI_PIPELINE_CREATED_AT', + value: pipeline.created_at.iso8601 }, + { key: 'CI_COMMIT_SHA', + value: job.sha }, + { key: 'CI_COMMIT_SHORT_SHA', + value: job.short_sha }, + { key: 'CI_COMMIT_BEFORE_SHA', + value: job.before_sha }, + { key: 'CI_COMMIT_REF_NAME', + value: job.ref }, + { key: 'CI_COMMIT_REF_SLUG', + value: job.ref_slug }, + { key: 'CI_COMMIT_BRANCH', + value: job.ref }, + { key: 'CI_COMMIT_MESSAGE', + value: pipeline.git_commit_message }, + { key: 'CI_COMMIT_TITLE', + value: pipeline.git_commit_title }, + { key: 'CI_COMMIT_DESCRIPTION', + value: pipeline.git_commit_description }, + { key: 'CI_COMMIT_REF_PROTECTED', + value: (!!pipeline.protected_ref?).to_s }, + { key: 'CI_COMMIT_TIMESTAMP', + value: pipeline.git_commit_timestamp }, + { key: 'CI_COMMIT_AUTHOR', + value: pipeline.git_author_full_text }, + { key: 'CI_BUILD_REF', + value: job.sha }, + { key: 'CI_BUILD_BEFORE_SHA', + value: job.before_sha }, + { key: 'CI_BUILD_REF_NAME', + value: job.ref }, + { key: 'CI_BUILD_REF_SLUG', + value: job.ref_slug }, + { key: 'YAML_VARIABLE', + value: 'value' }, + { key: 'GITLAB_USER_ID', + value: user.id.to_s }, + { key: 'GITLAB_USER_EMAIL', + value: user.email }, + { key: 'GITLAB_USER_LOGIN', + value: user.username }, + { key: 'GITLAB_USER_NAME', + value: user.name } + ].map { |var| var.merge(public: true, masked: false) } + end + + it { expect(subject.to_runner_variables).to eq(predefined_variables) } + end + context 'variables ordering' do def var(name, value) { key: name, value: value.to_s, public: true, masked: false } diff --git a/spec/lib/gitlab/ci/variables/collection_spec.rb b/spec/lib/gitlab/ci/variables/collection_spec.rb index 668f1173675..181e37de9b9 100644 --- a/spec/lib/gitlab/ci/variables/collection_spec.rb +++ b/spec/lib/gitlab/ci/variables/collection_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Variables::Collection, feature_category: :pipeline_composition do +RSpec.describe Gitlab::Ci::Variables::Collection, feature_category: :secrets_management do describe '.new' do it 'can be initialized with an array' do variable = { key: 'VAR', value: 'value', public: true, masked: false } diff --git a/spec/lib/gitlab/ci/yaml_processor_spec.rb b/spec/lib/gitlab/ci/yaml_processor_spec.rb index b00d9b46bc7..d7dcfe64c74 100644 --- a/spec/lib/gitlab/ci/yaml_processor_spec.rb +++ b/spec/lib/gitlab/ci/yaml_processor_spec.rb @@ -2395,10 +2395,16 @@ module Gitlab end end - context 'undefined need' do + context 'when need is an undefined job' do let(:needs) { ['undefined'] } it_behaves_like 'returns errors', 'test1 job: undefined need: undefined' + + context 'when need is optional' do + let(:needs) { [{ job: 'undefined', optional: true }] } + + it { is_expected.to be_valid } + end end context 'needs to deploy' do diff --git a/spec/lib/gitlab/config/loader/multi_doc_yaml_spec.rb b/spec/lib/gitlab/config/loader/multi_doc_yaml_spec.rb index f63aacecce6..438f3e5b17a 100644 --- a/spec/lib/gitlab/config/loader/multi_doc_yaml_spec.rb +++ b/spec/lib/gitlab/config/loader/multi_doc_yaml_spec.rb @@ -3,7 +3,8 @@ require 'spec_helper' RSpec.describe Gitlab::Config::Loader::MultiDocYaml, feature_category: :pipeline_composition do - let(:loader) { described_class.new(yml, max_documents: 2) } + let(:loader) { described_class.new(yml, max_documents: 2, reject_empty: reject_empty) } + let(:reject_empty) { false } describe '#load!' do context 'when a simple single delimiter is being used' do @@ -141,6 +142,27 @@ RSpec.describe Gitlab::Config::Loader::MultiDocYaml, feature_category: :pipeline expect(loader.load!).to contain_exactly({ a: 1 }, { b: 2 }) end end + + context 'when the YAML contains empty documents' do + let(:yml) do + <<~YAML + a: 1 + --- + YAML + end + + it 'raises an error' do + expect { loader.load! }.to raise_error(::Gitlab::Config::Loader::Yaml::NotHashError) + end + + context 'when reject_empty: true' do + let(:reject_empty) { true } + + it 'loads only non empty documents' do + expect(loader.load!).to contain_exactly({ a: 1 }) + end + end + end end describe '#load_raw!' do diff --git a/spec/lib/gitlab/content_security_policy/config_loader_spec.rb b/spec/lib/gitlab/content_security_policy/config_loader_spec.rb index ffb651fe23c..b40829d72a0 100644 --- a/spec/lib/gitlab/content_security_policy/config_loader_spec.rb +++ b/spec/lib/gitlab/content_security_policy/config_loader_spec.rb @@ -178,53 +178,6 @@ RSpec.describe Gitlab::ContentSecurityPolicy::ConfigLoader do end end - context 'when KAS is configured' do - before do - stub_config_setting(host: 'gitlab.example.com') - allow(::Gitlab::Kas).to receive(:enabled?).and_return true - end - - context 'when user access feature flag is disabled' do - before do - stub_feature_flags(kas_user_access: false) - end - - it 'does not add KAS url to CSP' do - expect(directives['connect_src']).not_to eq("'self' ws://gitlab.example.com #{::Gitlab::Kas.tunnel_url}") - end - end - - context 'when user access feature flag is enabled' do - before do - stub_feature_flags(kas_user_access: true) - end - - context 'when KAS is on same domain as rails' do - let_it_be(:kas_tunnel_url) { "ws://gitlab.example.com/-/k8s-proxy/" } - - before do - allow(::Gitlab::Kas).to receive(:tunnel_url).and_return(kas_tunnel_url) - end - - it 'does not add KAS url to CSP' do - expect(directives['connect_src']).not_to eq("'self' ws://gitlab.example.com #{::Gitlab::Kas.tunnel_url}") - end - end - - context 'when KAS is on subdomain' do - let_it_be(:kas_tunnel_url) { "ws://kas.gitlab.example.com/k8s-proxy/" } - - before do - allow(::Gitlab::Kas).to receive(:tunnel_url).and_return(kas_tunnel_url) - end - - it 'does add KAS url to CSP' do - expect(directives['connect_src']).to eq("'self' ws://gitlab.example.com #{kas_tunnel_url}") - end - end - end - end - context 'when CUSTOMER_PORTAL_URL is set' do let(:customer_portal_url) { 'https://customers.example.com' } diff --git a/spec/lib/gitlab/database/async_indexes/migration_helpers_spec.rb b/spec/lib/gitlab/database/async_indexes/migration_helpers_spec.rb index 7c5c368fcb5..b2ba1a60fbb 100644 --- a/spec/lib/gitlab/database/async_indexes/migration_helpers_spec.rb +++ b/spec/lib/gitlab/database/async_indexes/migration_helpers_spec.rb @@ -143,6 +143,92 @@ RSpec.describe Gitlab::Database::AsyncIndexes::MigrationHelpers, feature_categor end end + describe '#prepare_async_index_from_sql' do + let(:index_definition) { "CREATE INDEX CONCURRENTLY #{index_name} ON #{table_name} USING btree(id)" } + + subject(:prepare_async_index_from_sql) do + migration.prepare_async_index_from_sql(index_definition) + end + + before do + connection.create_table(table_name) + + allow(Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas).to receive(:require_ddl_mode!).and_call_original + end + + it 'requires ddl mode' do + prepare_async_index_from_sql + + expect(Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas).to have_received(:require_ddl_mode!) + end + + context 'when the given index is invalid' do + let(:index_definition) { "SELECT FROM users" } + + it 'raises a RuntimeError' do + expect { prepare_async_index_from_sql }.to raise_error(RuntimeError, 'Index statement not found!') + end + end + + context 'when the given index is valid' do + context 'when the index algorithm is not concurrent' do + let(:index_definition) { "CREATE INDEX #{index_name} ON #{table_name} USING btree(id)" } + + it 'raises a RuntimeError' do + expect { prepare_async_index_from_sql }.to raise_error(RuntimeError, 'Index must be created concurrently!') + end + end + + context 'when the index algorithm is concurrent' do + context 'when the statement tries to create an index for non-existing table' do + let(:index_definition) { "CREATE INDEX CONCURRENTLY #{index_name} ON foo_table USING btree(id)" } + + it 'raises a RuntimeError' do + expect { prepare_async_index_from_sql }.to raise_error(RuntimeError, 'Table does not exist!') + end + end + + context 'when the statement tries to create an index for an existing table' do + context 'when the async index creation is not available' do + before do + connection.drop_table(:postgres_async_indexes) + end + + it 'does not raise an error' do + expect { prepare_async_index_from_sql }.not_to raise_error + end + end + + context 'when the async index creation is available' do + context 'when there is already an index with the given name' do + before do + connection.add_index(table_name, 'id', name: index_name) + end + + it 'does not create the async index record' do + expect { prepare_async_index_from_sql }.not_to change { index_model.where(name: index_name).count } + end + end + + context 'when there is no index with the given name' do + let(:async_index) { index_model.find_by(name: index_name) } + + it 'creates the async index record' do + expect { prepare_async_index_from_sql }.to change { index_model.where(name: index_name).count }.by(1) + end + + it 'sets the async index attributes correctly' do + prepare_async_index_from_sql + + expect(async_index).to have_attributes(table_name: table_name, definition: index_definition) + end + end + end + end + end + end + end + describe '#prepare_async_index_removal' do before do connection.create_table(table_name) diff --git a/spec/lib/gitlab/database/background_migration/batched_job_spec.rb b/spec/lib/gitlab/database/background_migration/batched_job_spec.rb index 073a30e7839..d9b81a2be30 100644 --- a/spec/lib/gitlab/database/background_migration/batched_job_spec.rb +++ b/spec/lib/gitlab/database/background_migration/batched_job_spec.rb @@ -378,41 +378,27 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedJob, type: :model d let(:attempts) { 0 } let(:batch_size) { 10 } let(:sub_batch_size) { 6 } - let(:feature_flag) { :reduce_sub_batch_size_on_timeouts } let(:job) do create(:batched_background_migration_job, attempts: attempts, batch_size: batch_size, sub_batch_size: sub_batch_size) end - where(:feature_flag_state, :within_boundaries, :outside_boundaries, :limit_reached) do - [ - [true, true, false, false], - [false, false, false, false] - ] - end - - with_them do - before do - stub_feature_flags(feature_flag => feature_flag_state) - end + context 'when the number of attempts is lower than the limit and batch size are within boundaries' do + let(:attempts) { 1 } - context 'when the number of attempts is lower than the limit and batch size are within boundaries' do - let(:attempts) { 1 } - - it { expect(job.can_reduce_sub_batch_size?).to be(within_boundaries) } - end + it { expect(job.can_reduce_sub_batch_size?).to be(true) } + end - context 'when the number of attempts is lower than the limit and batch size are outside boundaries' do - let(:batch_size) { 1 } + context 'when the number of attempts is lower than the limit and batch size are outside boundaries' do + let(:batch_size) { 1 } - it { expect(job.can_reduce_sub_batch_size?).to be(outside_boundaries) } - end + it { expect(job.can_reduce_sub_batch_size?).to be(false) } + end - context 'when the number of attempts is greater than the limit and batch size are within boundaries' do - let(:attempts) { 3 } + context 'when the number of attempts is greater than the limit and batch size are within boundaries' do + let(:attempts) { 3 } - it { expect(job.can_reduce_sub_batch_size?).to be(limit_reached) } - end + it { expect(job.can_reduce_sub_batch_size?).to be(false) } end end diff --git a/spec/lib/gitlab/database/background_migration/batched_migration_spec.rb b/spec/lib/gitlab/database/background_migration/batched_migration_spec.rb index d132559acea..546f9353808 100644 --- a/spec/lib/gitlab/database/background_migration/batched_migration_spec.rb +++ b/spec/lib/gitlab/database/background_migration/batched_migration_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigration, type: :model do +RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigration, type: :model, feature_category: :database do it_behaves_like 'having unique enum values' it { is_expected.to be_a Gitlab::Database::SharedModel } @@ -328,6 +328,17 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigration, type: :m end end + describe '.finalizing' do + let!(:migration1) { create(:batched_background_migration, :active) } + let!(:migration2) { create(:batched_background_migration, :paused) } + let!(:migration3) { create(:batched_background_migration, :finalizing) } + let!(:migration4) { create(:batched_background_migration, :finished) } + + it 'returns only finalizing migrations' do + expect(described_class.finalizing).to contain_exactly(migration3) + end + end + describe '.successful_rows_counts' do let!(:migration1) { create(:batched_background_migration) } let!(:migration2) { create(:batched_background_migration) } diff --git a/spec/lib/gitlab/database/background_migration/health_status/indicators/patroni_apdex_spec.rb b/spec/lib/gitlab/database/background_migration/health_status/indicators/patroni_apdex_spec.rb new file mode 100644 index 00000000000..d3102a105ea --- /dev/null +++ b/spec/lib/gitlab/database/background_migration/health_status/indicators/patroni_apdex_spec.rb @@ -0,0 +1,148 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::BackgroundMigration::HealthStatus::Indicators::PatroniApdex, :aggregate_failures, feature_category: :database do # rubocop:disable Layout/LineLength + let(:schema) { :main } + let(:connection) { Gitlab::Database.database_base_models[schema].connection } + + around do |example| + Gitlab::Database::SharedModel.using_connection(connection) do + example.run + end + end + + describe '#evaluate' do + let(:prometheus_url) { 'http://thanos:9090' } + let(:prometheus_config) { [prometheus_url, { allow_local_requests: true, verify: true }] } + + let(:prometheus_client) { instance_double(Gitlab::PrometheusClient) } + + let(:context) do + Gitlab::Database::BackgroundMigration::HealthStatus::Context + .new(connection, ['users'], gitlab_schema) + end + + let(:gitlab_schema) { "gitlab_#{schema}" } + let(:client_ready) { true } + let(:database_apdex_sli_query_main) { 'Apdex query for main' } + let(:database_apdex_sli_query_ci) { 'Apdex query for ci' } + let(:database_apdex_slo_main) { 0.99 } + let(:database_apdex_slo_ci) { 0.95 } + let(:database_apdex_settings) do + { + prometheus_api_url: prometheus_url, + apdex_sli_query: { + main: database_apdex_sli_query_main, + ci: database_apdex_sli_query_ci + }, + apdex_slo: { + main: database_apdex_slo_main, + ci: database_apdex_slo_ci + } + } + end + + subject(:evaluate) { described_class.new(context).evaluate } + + before do + stub_application_setting(database_apdex_settings: database_apdex_settings) + + allow(Gitlab::PrometheusClient).to receive(:new).with(*prometheus_config).and_return(prometheus_client) + allow(prometheus_client).to receive(:ready?).and_return(client_ready) + end + + shared_examples 'Patroni Apdex Evaluator' do |schema| + context "with #{schema} schema" do + let(:schema) { schema } + let(:apdex_slo_above_sli) { { main: 0.991, ci: 0.951 } } + let(:apdex_slo_below_sli) { { main: 0.989, ci: 0.949 } } + + it 'returns NoSignal signal in case the feature flag is disabled' do + stub_feature_flags(batched_migrations_health_status_patroni_apdex: false) + + expect(evaluate).to be_a(Gitlab::Database::BackgroundMigration::HealthStatus::Signals::NotAvailable) + expect(evaluate.reason).to include('indicator disabled') + end + + context 'without database_apdex_settings' do + let(:database_apdex_settings) { nil } + + it 'returns Unknown signal' do + expect(evaluate).to be_a(Gitlab::Database::BackgroundMigration::HealthStatus::Signals::Unknown) + expect(evaluate.reason).to include('Patroni Apdex Settings not configured') + end + end + + context 'when Prometheus client is not ready' do + let(:client_ready) { false } + + it 'returns Unknown signal' do + expect(evaluate).to be_a(Gitlab::Database::BackgroundMigration::HealthStatus::Signals::Unknown) + expect(evaluate.reason).to include('Prometheus client is not ready') + end + end + + context 'when apdex SLI query is not configured' do + let(:"database_apdex_sli_query_#{schema}") { nil } + + it 'returns Unknown signal' do + expect(evaluate).to be_a(Gitlab::Database::BackgroundMigration::HealthStatus::Signals::Unknown) + expect(evaluate.reason).to include('Apdex SLI query is not configured') + end + end + + context 'when slo is not configured' do + let(:"database_apdex_slo_#{schema}") { nil } + + it 'returns Unknown signal' do + expect(evaluate).to be_a(Gitlab::Database::BackgroundMigration::HealthStatus::Signals::Unknown) + expect(evaluate.reason).to include('Apdex SLO is not configured') + end + end + + it 'returns Normal signal when Patroni apdex SLI is above SLO' do + expect(prometheus_client).to receive(:query) + .with(send("database_apdex_sli_query_#{schema}")) + .and_return([{ "value" => [1662423310.878, apdex_slo_above_sli[schema]] }]) + expect(evaluate).to be_a(Gitlab::Database::BackgroundMigration::HealthStatus::Signals::Normal) + expect(evaluate.reason).to include('Patroni service apdex is above SLO') + end + + it 'returns Stop signal when Patroni apdex is below SLO' do + expect(prometheus_client).to receive(:query) + .with(send("database_apdex_sli_query_#{schema}")) + .and_return([{ "value" => [1662423310.878, apdex_slo_below_sli[schema]] }]) + expect(evaluate).to be_a(Gitlab::Database::BackgroundMigration::HealthStatus::Signals::Stop) + expect(evaluate.reason).to include('Patroni service apdex is below SLO') + end + + context 'when Patroni apdex can not be calculated' do + where(:result) do + [ + nil, + [], + [{}], + [{ 'value' => 1 }], + [{ 'value' => [1] }] + ] + end + + with_them do + it 'returns Unknown signal' do + expect(prometheus_client).to receive(:query).and_return(result) + expect(evaluate).to be_a(Gitlab::Database::BackgroundMigration::HealthStatus::Signals::Unknown) + expect(evaluate.reason).to include('Patroni service apdex can not be calculated') + end + end + end + end + end + + Gitlab::Database.database_base_models.each do |database_base_model, connection| + next unless connection.present? + + it_behaves_like 'Patroni Apdex Evaluator', database_base_model.to_sym + end + end +end diff --git a/spec/lib/gitlab/database/background_migration/health_status_spec.rb b/spec/lib/gitlab/database/background_migration/health_status_spec.rb index 8bc04d80fa1..e14440f1fb4 100644 --- a/spec/lib/gitlab/database/background_migration/health_status_spec.rb +++ b/spec/lib/gitlab/database/background_migration/health_status_spec.rb @@ -19,8 +19,10 @@ RSpec.describe Gitlab::Database::BackgroundMigration::HealthStatus do let(:health_status) { Gitlab::Database::BackgroundMigration::HealthStatus } let(:autovacuum_indicator_class) { health_status::Indicators::AutovacuumActiveOnTable } let(:wal_indicator_class) { health_status::Indicators::WriteAheadLog } + let(:patroni_apdex_indicator_class) { health_status::Indicators::PatroniApdex } let(:autovacuum_indicator) { instance_double(autovacuum_indicator_class) } let(:wal_indicator) { instance_double(wal_indicator_class) } + let(:patroni_apdex_indicator) { instance_double(patroni_apdex_indicator_class) } before do allow(autovacuum_indicator_class).to receive(:new).with(migration.health_context).and_return(autovacuum_indicator) @@ -36,8 +38,11 @@ RSpec.describe Gitlab::Database::BackgroundMigration::HealthStatus do expect(autovacuum_indicator).to receive(:evaluate).and_return(normal_signal) expect(wal_indicator_class).to receive(:new).with(migration.health_context).and_return(wal_indicator) expect(wal_indicator).to receive(:evaluate).and_return(not_available_signal) + expect(patroni_apdex_indicator_class).to receive(:new).with(migration.health_context) + .and_return(patroni_apdex_indicator) + expect(patroni_apdex_indicator).to receive(:evaluate).and_return(not_available_signal) - expect(evaluate).to contain_exactly(normal_signal, not_available_signal) + expect(evaluate).to contain_exactly(normal_signal, not_available_signal, not_available_signal) end end diff --git a/spec/lib/gitlab/database/consistency_checker_spec.rb b/spec/lib/gitlab/database/consistency_checker_spec.rb index c0f0c349ddd..be03bd00619 100644 --- a/spec/lib/gitlab/database/consistency_checker_spec.rb +++ b/spec/lib/gitlab/database/consistency_checker_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::ConsistencyChecker, feature_category: :pods do +RSpec.describe Gitlab::Database::ConsistencyChecker, feature_category: :cell do let(:batch_size) { 10 } let(:max_batches) { 4 } let(:max_runtime) { described_class::MAX_RUNTIME } diff --git a/spec/lib/gitlab/database/load_balancing/action_cable_callbacks_spec.rb b/spec/lib/gitlab/database/load_balancing/action_cable_callbacks_spec.rb index 768855464c1..a57f02b22df 100644 --- a/spec/lib/gitlab/database/load_balancing/action_cable_callbacks_spec.rb +++ b/spec/lib/gitlab/database/load_balancing/action_cable_callbacks_spec.rb @@ -2,18 +2,13 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::LoadBalancing::ActionCableCallbacks, :request_store do +RSpec.describe Gitlab::Database::LoadBalancing::ActionCableCallbacks, :request_store, feature_category: :shared do describe '.wrapper' do - it 'uses primary and then releases the connection and clears the session' do + it 'releases the connection and clears the session' do expect(Gitlab::Database::LoadBalancing).to receive(:release_hosts) expect(Gitlab::Database::LoadBalancing::Session).to receive(:clear_session) - described_class.wrapper.call( - nil, - lambda do - expect(Gitlab::Database::LoadBalancing::Session.current.use_primary?).to eq(true) - end - ) + described_class.wrapper.call(nil, lambda {}) end context 'with an exception' do diff --git a/spec/lib/gitlab/database/load_balancing/sidekiq_client_middleware_spec.rb b/spec/lib/gitlab/database/load_balancing/sidekiq_client_middleware_spec.rb index 7eb20f77417..83fc614bde3 100644 --- a/spec/lib/gitlab/database/load_balancing/sidekiq_client_middleware_spec.rb +++ b/spec/lib/gitlab/database/load_balancing/sidekiq_client_middleware_spec.rb @@ -67,16 +67,6 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqClientMiddleware, feature let(:location) { '0/D525E3A8' } - context 'when feature flag is disabled' do - let(:expected_consistency) { :always } - - before do - stub_feature_flags(load_balancing_for_test_data_consistency_worker: false) - end - - include_examples 'does not pass database locations' - end - context 'when write was not performed' do before do allow(Gitlab::Database::LoadBalancing::Session.current).to receive(:use_primary?).and_return(false) @@ -106,7 +96,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqClientMiddleware, feature expected_location = {} Gitlab::Database::LoadBalancing.each_load_balancer do |lb| - expect(lb).to receive(:host).and_return(nil) + expect(lb).to receive(:host).at_least(:once).and_return(nil) expect(lb).to receive(:primary_write_location).and_return(location) expected_location[lb.name] = location diff --git a/spec/lib/gitlab/database/load_balancing/sidekiq_server_middleware_spec.rb b/spec/lib/gitlab/database/load_balancing/sidekiq_server_middleware_spec.rb index abf10456d0a..7ad0ddbca8e 100644 --- a/spec/lib/gitlab/database/load_balancing/sidekiq_server_middleware_spec.rb +++ b/spec/lib/gitlab/database/load_balancing/sidekiq_server_middleware_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware, :clean_gitlab_redis_queues do +RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware, :clean_gitlab_redis_queues, feature_category: :scalability do let(:middleware) { described_class.new } let(:worker) { worker_class.new } let(:location) { '0/D525E3A8' } @@ -15,6 +15,8 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware, :clean_ replication_lag!(false) Gitlab::Database::LoadBalancing::Session.clear_session + + stub_const("#{described_class.name}::REPLICA_WAIT_SLEEP_SECONDS", 0.0) end after do @@ -76,14 +78,6 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware, :clean_ end shared_examples_for 'sticks based on data consistency' do - context 'when load_balancing_for_test_data_consistency_worker is disabled' do - before do - stub_feature_flags(load_balancing_for_test_data_consistency_worker: false) - end - - include_examples 'stick to the primary', 'primary' - end - context 'when database wal location is set' do let(:job) { { 'job_id' => 'a180b47c-3fd6-41b8-81e9-34da61c3400e', 'wal_locations' => wal_locations } } @@ -119,9 +113,9 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware, :clean_ end end - shared_examples_for 'sleeps when necessary' do + shared_examples_for 'essential sleep' do context 'when WAL locations are blank', :freeze_time do - let(:job) { { "retry" => 3, "job_id" => "a180b47c-3fd6-41b8-81e9-34da61c3400e", "wal_locations" => {}, "created_at" => Time.current.to_f - (described_class::MINIMUM_DELAY_INTERVAL_SECONDS - 0.3) } } + let(:job) { { "retry" => 3, "job_id" => "a180b47c-3fd6-41b8-81e9-34da61c3400e", "wal_locations" => {}, "created_at" => Time.current.to_f - (described_class::REPLICA_WAIT_SLEEP_SECONDS + 0.2) } } it 'does not sleep' do expect(middleware).not_to receive(:sleep) @@ -134,7 +128,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware, :clean_ let(:job) { { "retry" => 3, "job_id" => "a180b47c-3fd6-41b8-81e9-34da61c3400e", 'wal_locations' => wal_locations, "created_at" => Time.current.to_f - elapsed_time } } context 'when delay interval has not elapsed' do - let(:elapsed_time) { described_class::MINIMUM_DELAY_INTERVAL_SECONDS - 0.3 } + let(:elapsed_time) { described_class::REPLICA_WAIT_SLEEP_SECONDS + 0.2 } context 'when replica is up to date' do before do @@ -158,30 +152,24 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware, :clean_ end it 'sleeps until the minimum delay is reached' do - expect(middleware).to receive(:sleep).with(be_within(0.01).of(described_class::MINIMUM_DELAY_INTERVAL_SECONDS - elapsed_time)) + expect(middleware).to receive(:sleep).with(described_class::REPLICA_WAIT_SLEEP_SECONDS) run_middleware end end - end - - context 'when delay interval has elapsed' do - let(:elapsed_time) { described_class::MINIMUM_DELAY_INTERVAL_SECONDS + 0.3 } - - it 'does not sleep' do - expect(middleware).not_to receive(:sleep) - run_middleware - end - end - - context 'when created_at is in the future' do - let(:elapsed_time) { -5 } + context 'when replica is never not up to date' do + before do + Gitlab::Database::LoadBalancing.each_load_balancer do |lb| + allow(lb).to receive(:select_up_to_date_host).and_return(false, false) + end + end - it 'does not sleep' do - expect(middleware).not_to receive(:sleep) + it 'sleeps until the maximum delay is reached' do + expect(middleware).to receive(:sleep).exactly(3).times.with(described_class::REPLICA_WAIT_SLEEP_SECONDS) - run_middleware + run_middleware + end end end end @@ -200,7 +188,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware, :clean_ context 'when delay interval has not elapsed', :freeze_time do let(:job) { { "retry" => 3, "job_id" => "a180b47c-3fd6-41b8-81e9-34da61c3400e", 'wal_locations' => wal_locations, "created_at" => Time.current.to_f - elapsed_time } } - let(:elapsed_time) { described_class::MINIMUM_DELAY_INTERVAL_SECONDS - 0.3 } + let(:elapsed_time) { described_class::REPLICA_WAIT_SLEEP_SECONDS + 0.2 } it 'does not sleep' do expect(middleware).not_to receive(:sleep) @@ -214,7 +202,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware, :clean_ include_context 'data consistency worker class', :delayed, :load_balancing_for_test_data_consistency_worker include_examples 'sticks based on data consistency' - include_examples 'sleeps when necessary' + include_examples 'essential sleep' context 'when replica is not up to date' do before do @@ -263,7 +251,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware, :clean_ include_context 'data consistency worker class', :sticky, :load_balancing_for_test_data_consistency_worker include_examples 'sticks based on data consistency' - include_examples 'sleeps when necessary' + include_examples 'essential sleep' context 'when replica is not up to date' do before do diff --git a/spec/lib/gitlab/database/load_balancing_spec.rb b/spec/lib/gitlab/database/load_balancing_spec.rb index 59e16e6ca8b..7196b4bc337 100644 --- a/spec/lib/gitlab/database/load_balancing_spec.rb +++ b/spec/lib/gitlab/database/load_balancing_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::LoadBalancing, :suppress_gitlab_schemas_validate_connection, feature_category: :pods do +RSpec.describe Gitlab::Database::LoadBalancing, :suppress_gitlab_schemas_validate_connection, feature_category: :cell do describe '.base_models' do it 'returns the models to apply load balancing to' do models = described_class.base_models @@ -497,14 +497,14 @@ RSpec.describe Gitlab::Database::LoadBalancing, :suppress_gitlab_schemas_validat where(:queries, :expected_role) do [ # Reload cache. The schema loading queries should be handled by - # primary. + # replica. [ -> { model.connection.clear_cache! model.connection.schema_cache.add('users') model.connection.pool.release_connection }, - :primary + :replica ], # Call model's connection method diff --git a/spec/lib/gitlab/database/lock_writes_manager_spec.rb b/spec/lib/gitlab/database/lock_writes_manager_spec.rb index c06c463d918..2aa95372338 100644 --- a/spec/lib/gitlab/database/lock_writes_manager_spec.rb +++ b/spec/lib/gitlab/database/lock_writes_manager_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::LockWritesManager, :delete, feature_category: :pods do +RSpec.describe Gitlab::Database::LockWritesManager, :delete, feature_category: :cell do let(:connection) { ApplicationRecord.connection } let(:test_table) { '_test_table' } let(:logger) { instance_double(Logger) } @@ -122,6 +122,13 @@ RSpec.describe Gitlab::Database::LockWritesManager, :delete, feature_category: : } end + it 'returns result hash with action skipped' do + subject.lock_writes + + expect(subject.lock_writes).to eq({ action: "skipped", database: "main", dry_run: false, +table: test_table }) + end + context 'when running in dry_run mode' do let(:dry_run) { true } @@ -146,6 +153,11 @@ RSpec.describe Gitlab::Database::LockWritesManager, :delete, feature_category: : connection.execute("truncate #{test_table}") end.not_to raise_error end + + it 'returns result hash with action locked' do + expect(subject.lock_writes).to eq({ action: "locked", database: "main", dry_run: dry_run, +table: test_table }) + end end end @@ -186,6 +198,11 @@ RSpec.describe Gitlab::Database::LockWritesManager, :delete, feature_category: : subject.unlock_writes end + it 'returns result hash with action unlocked' do + expect(subject.unlock_writes).to eq({ action: "unlocked", database: "main", dry_run: dry_run, +table: test_table }) + end + context 'when running in dry_run mode' do let(:dry_run) { true } @@ -206,6 +223,11 @@ RSpec.describe Gitlab::Database::LockWritesManager, :delete, feature_category: : connection.execute("delete from #{test_table}") end.to raise_error(ActiveRecord::StatementInvalid, /Table: "#{test_table}" is write protected/) end + + it 'returns result hash with dry_run true' do + expect(subject.unlock_writes).to eq({ action: "unlocked", database: "main", dry_run: dry_run, +table: test_table }) + end end end diff --git a/spec/lib/gitlab/database/loose_foreign_keys_spec.rb b/spec/lib/gitlab/database/loose_foreign_keys_spec.rb index 3c2d9ca82f2..552df64096a 100644 --- a/spec/lib/gitlab/database/loose_foreign_keys_spec.rb +++ b/spec/lib/gitlab/database/loose_foreign_keys_spec.rb @@ -85,31 +85,40 @@ RSpec.describe Gitlab::Database::LooseForeignKeys do end end - describe '.definitions' do - subject(:definitions) { described_class.definitions } - - it 'contains at least all parent tables that have triggers' do - all_definition_parent_tables = definitions.map { |d| d.to_table }.to_set + context 'all tables have correct triggers installed' do + let(:all_tables_from_yaml) { described_class.definitions.pluck(:to_table).uniq } + let(:all_tables_with_triggers) do triggers_query = <<~SQL - SELECT event_object_table, trigger_name - FROM information_schema.triggers + SELECT event_object_table FROM information_schema.triggers WHERE trigger_name LIKE '%_loose_fk_trigger' - GROUP BY event_object_table, trigger_name SQL - all_triggers = ApplicationRecord.connection.execute(triggers_query) - - all_triggers.each do |trigger| - table = trigger['event_object_table'] - trigger_name = trigger['trigger_name'] - error_message = <<~END - Missing a loose foreign key definition for parent table: #{table} with trigger: #{trigger_name}. - Loose foreign key definitions must be added before triggers are added and triggers must be removed before removing the loose foreign key definition. - Read more at https://docs.gitlab.com/ee/development/database/loose_foreign_keys.html ." - END - expect(all_definition_parent_tables).to include(table), error_message - end + ApplicationRecord.connection.execute(triggers_query) + .pluck('event_object_table').uniq + end + + it 'all YAML tables do have `track_record_deletions` installed' do + missing_trigger_tables = all_tables_from_yaml - all_tables_with_triggers + + expect(missing_trigger_tables).to be_empty, <<~END + The loose foreign keys definitions require using `track_record_deletions` + for the following tables: #{missing_trigger_tables}. + Read more at https://docs.gitlab.com/ee/development/database/loose_foreign_keys.html." + END + end + + it 'no extra tables have `track_record_deletions` installed' do + extra_trigger_tables = all_tables_with_triggers - all_tables_from_yaml + + pending 'This result of this test is informatory, and not critical' if extra_trigger_tables.any? + + expect(extra_trigger_tables).to be_empty, <<~END + The following tables have unused `track_record_deletions` triggers installed, + but they are not referenced by any of the loose foreign key definitions: #{extra_trigger_tables}. + You can remove them in one of the future releases as part of `db/post_migrate`. + Read more at https://docs.gitlab.com/ee/development/database/loose_foreign_keys.html." + END end end diff --git a/spec/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables_spec.rb b/spec/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables_spec.rb index be9346e3829..090a9f53523 100644 --- a/spec/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables_spec.rb +++ b/spec/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::Database::MigrationHelpers::AutomaticLockWritesOnTables, - :reestablished_active_record_base, :delete, query_analyzers: false, feature_category: :pods do + :reestablished_active_record_base, :delete, query_analyzers: false, feature_category: :cell do using RSpec::Parameterized::TableSyntax let(:schema_class) { Class.new(Gitlab::Database::Migration[2.1]) } @@ -86,7 +86,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers::AutomaticLockWritesOnTables, let(:create_gitlab_shared_table_migration_class) { create_table_migration(gitlab_shared_table_name) } before do - skip_if_multiple_databases_are_setup(:ci) + skip_if_database_exists(:ci) end it 'does not lock any newly created tables' do @@ -106,7 +106,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers::AutomaticLockWritesOnTables, context 'when multiple databases' do before do - skip_if_multiple_databases_not_setup(:ci) + skip_if_shared_database(:ci) end let(:migration_class) { create_table_migration(table_name, skip_automatic_lock_on_writes) } @@ -238,7 +238,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers::AutomaticLockWritesOnTables, context 'when renaming a table' do before do - skip_if_multiple_databases_not_setup(:ci) + skip_if_shared_database(:ci) create_table_migration(old_table_name).migrate(:up) # create the table first before renaming it end @@ -277,7 +277,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers::AutomaticLockWritesOnTables, let(:config_model) { Gitlab::Database.database_base_models[:main] } before do - skip_if_multiple_databases_are_setup(:ci) + skip_if_database_exists(:ci) end it 'does not lock any newly created tables' do @@ -305,7 +305,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers::AutomaticLockWritesOnTables, context 'when multiple databases' do before do - skip_if_multiple_databases_not_setup(:ci) + skip_if_shared_database(:ci) migration_class.connection.execute("CREATE TABLE #{table_name}()") migration_class.migrate(:up) end diff --git a/spec/lib/gitlab/database/migration_helpers/convert_to_bigint_spec.rb b/spec/lib/gitlab/database/migration_helpers/convert_to_bigint_spec.rb index b1971977e7c..cee5f54bd6a 100644 --- a/spec/lib/gitlab/database/migration_helpers/convert_to_bigint_spec.rb +++ b/spec/lib/gitlab/database/migration_helpers/convert_to_bigint_spec.rb @@ -7,9 +7,9 @@ RSpec.describe Gitlab::Database::MigrationHelpers::ConvertToBigint, feature_cate using RSpec::Parameterized::TableSyntax where(:dot_com, :dev_or_test, :jh, :expectation) do - true | true | true | false + true | true | true | true true | false | true | false - false | true | true | false + false | true | true | true false | false | true | false true | true | false | true true | false | false | true diff --git a/spec/lib/gitlab/database/migration_helpers/loose_foreign_key_helpers_spec.rb b/spec/lib/gitlab/database/migration_helpers/loose_foreign_key_helpers_spec.rb index 25fc676d09e..2b58cdff931 100644 --- a/spec/lib/gitlab/database/migration_helpers/loose_foreign_key_helpers_spec.rb +++ b/spec/lib/gitlab/database/migration_helpers/loose_foreign_key_helpers_spec.rb @@ -7,20 +7,22 @@ RSpec.describe Gitlab::Database::MigrationHelpers::LooseForeignKeyHelpers do ActiveRecord::Migration.new.extend(described_class) end + let_it_be(:table_name) { :_test_loose_fk_test_table } + let(:model) do Class.new(ApplicationRecord) do - self.table_name = '_test_loose_fk_test_table' + self.table_name = :_test_loose_fk_test_table end end before(:all) do - migration.create_table :_test_loose_fk_test_table do |t| + migration.create_table table_name do |t| t.timestamps end end after(:all) do - migration.drop_table :_test_loose_fk_test_table + migration.drop_table table_name end before do @@ -33,11 +35,13 @@ RSpec.describe Gitlab::Database::MigrationHelpers::LooseForeignKeyHelpers do expect(LooseForeignKeys::DeletedRecord.count).to eq(0) end + + it { expect(migration.has_loose_foreign_key?(table_name)).to be_falsy } end context 'when the record deletion tracker trigger is installed' do before do - migration.track_record_deletions(:_test_loose_fk_test_table) + migration.track_record_deletions(table_name) end it 'stores the record deletion' do @@ -55,7 +59,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers::LooseForeignKeyHelpers do .first expect(deleted_record.primary_key_value).to eq(record_to_be_deleted.id) - expect(deleted_record.fully_qualified_table_name).to eq('public._test_loose_fk_test_table') + expect(deleted_record.fully_qualified_table_name).to eq("public.#{table_name}") expect(deleted_record.partition_number).to eq(1) end @@ -64,5 +68,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers::LooseForeignKeyHelpers do expect(LooseForeignKeys::DeletedRecord.count).to eq(3) end + + it { expect(migration.has_loose_foreign_key?(table_name)).to be_truthy } end end diff --git a/spec/lib/gitlab/database/migration_helpers/restrict_gitlab_schema_spec.rb b/spec/lib/gitlab/database/migration_helpers/restrict_gitlab_schema_spec.rb index 714fbab5aff..faf0447c054 100644 --- a/spec/lib/gitlab/database/migration_helpers/restrict_gitlab_schema_spec.rb +++ b/spec/lib/gitlab/database/migration_helpers/restrict_gitlab_schema_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::Database::MigrationHelpers::RestrictGitlabSchema, query_analyzers: false, - stub_feature_flags: false, feature_category: :pods do + stub_feature_flags: false, feature_category: :cell do let(:schema_class) { Class.new(Gitlab::Database::Migration[1.0]).include(described_class) } # We keep only the GitlabSchemasValidateConnection analyzer running @@ -506,7 +506,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers::RestrictGitlabSchema, query_a def down; end end, query_matcher: /FROM ci_builds/, - setup: -> (_) { skip_if_multiple_databases_not_setup }, + setup: -> (_) { skip_if_shared_database(:ci) }, expected: { no_gitlab_schema: { main: :cross_schema_error, diff --git a/spec/lib/gitlab/database/migration_helpers/wraparound_vacuum_helpers_spec.rb b/spec/lib/gitlab/database/migration_helpers/wraparound_vacuum_helpers_spec.rb new file mode 100644 index 00000000000..eb67e81f677 --- /dev/null +++ b/spec/lib/gitlab/database/migration_helpers/wraparound_vacuum_helpers_spec.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::MigrationHelpers::WraparoundVacuumHelpers, feature_category: :database do + include Database::DatabaseHelpers + + let(:table_name) { 'ci_builds' } + + describe '#check_if_wraparound_in_progress' do + let(:migration) do + ActiveRecord::Migration.new.extend(described_class) + end + + subject { migration.check_if_wraparound_in_progress(table_name) } + + it 'delegates to the wraparound class' do + expect(described_class::WraparoundCheck) + .to receive(:new) + .with(table_name, migration: migration) + .and_call_original + + expect { subject }.not_to raise_error + end + end + + describe described_class::WraparoundCheck do + let(:migration) do + ActiveRecord::Migration.new.extend(Gitlab::Database::MigrationHelpers::WraparoundVacuumHelpers) + end + + describe '#execute' do + subject do + described_class.new(table_name, migration: migration).execute + end + + context 'with wraparound vacuuum running' do + before do + swapout_view_for_table(:pg_stat_activity, connection: migration.connection) + + migration.connection.execute(<<~SQL.squish) + INSERT INTO pg_stat_activity ( + datid, datname, pid, backend_start, xact_start, query_start, + state_change, wait_event_type, wait_event, state, backend_xmin, + query, backend_type) + VALUES ( + 16401, 'gitlabhq_dblab', 178, '2023-03-30 08:10:50.851322+00', + '2023-03-30 08:10:50.890485+00', now() - '150 minutes'::interval, + '2023-03-30 08:10:50.890485+00', 'IO', 'DataFileRead', 'active','3214790381'::xid, + 'autovacuum: VACUUM public.ci_builds (to prevent wraparound)', 'autovacuum worker') + SQL + end + + it 'outputs a message related to autovacuum' do + expect { subject } + .to output(/Autovacuum with wraparound prevention mode is running on `ci_builds`/).to_stdout + end + + it { expect { subject }.to output(/autovacuum: VACUUM public.ci_builds \(to prevent wraparound\)/).to_stdout } + it { expect { subject }.to output(/Current duration: 2 hours, 30 minutes/).to_stdout } + it { expect { subject }.to output(/Process id: 178/).to_stdout } + it { expect { subject }.to output(/`select pg_cancel_backend\(178\);`/).to_stdout } + + context 'when GITLAB_MIGRATIONS_DISABLE_WRAPAROUND_CHECK is set' do + before do + stub_env('GITLAB_MIGRATIONS_DISABLE_WRAPAROUND_CHECK' => 'true') + end + + it { expect { subject }.not_to output(/autovacuum/i).to_stdout } + + it 'is disabled on .com' do + expect(Gitlab).to receive(:com?).and_return(true) + + expect { subject }.not_to raise_error + end + end + + context 'when executed by self-managed' do + before do + allow(Gitlab).to receive(:com?).and_return(false) + allow(Gitlab).to receive(:dev_or_test_env?).and_return(false) + end + + it { expect { subject }.not_to output(/autovacuum/i).to_stdout } + end + end + + context 'with wraparound vacuuum not running' do + it { expect { subject }.not_to output(/autovacuum/i).to_stdout } + end + + context 'when the table does not exist' do + let(:table_name) { :no_table } + + it { expect { subject }.to raise_error described_class::WraparoundError, /no_table/ } + end + end + end +end diff --git a/spec/lib/gitlab/database/migration_helpers_spec.rb b/spec/lib/gitlab/database/migration_helpers_spec.rb index 3f6528558b1..a3eab560c67 100644 --- a/spec/lib/gitlab/database/migration_helpers_spec.rb +++ b/spec/lib/gitlab/database/migration_helpers_spec.rb @@ -14,6 +14,8 @@ RSpec.describe Gitlab::Database::MigrationHelpers, feature_category: :database d allow(model).to receive(:puts) end + it { expect(model.singleton_class.ancestors).to include(described_class::WraparoundVacuumHelpers) } + describe 'overridden dynamic model helpers' do let(:test_table) { '_test_batching_table' } @@ -120,157 +122,6 @@ RSpec.describe Gitlab::Database::MigrationHelpers, feature_category: :database d end end - describe '#create_table_with_constraints' do - let(:table_name) { :test_table } - let(:column_attributes) do - [ - { name: 'id', sql_type: 'bigint', null: false, default: nil }, - { name: 'created_at', sql_type: 'timestamp with time zone', null: false, default: nil }, - { name: 'updated_at', sql_type: 'timestamp with time zone', null: false, default: nil }, - { name: 'some_id', sql_type: 'integer', null: false, default: nil }, - { name: 'active', sql_type: 'boolean', null: false, default: 'true' }, - { name: 'name', sql_type: 'text', null: true, default: nil } - ] - end - - before do - allow(model).to receive(:transaction_open?).and_return(true) - end - - context 'when no check constraints are defined' do - it 'creates the table as expected' do - model.create_table_with_constraints table_name do |t| - t.timestamps_with_timezone - t.integer :some_id, null: false - t.boolean :active, null: false, default: true - t.text :name - end - - expect_table_columns_to_match(column_attributes, table_name) - end - end - - context 'when check constraints are defined' do - context 'when the text_limit is explicity named' do - it 'creates the table as expected' do - model.create_table_with_constraints table_name do |t| - t.timestamps_with_timezone - t.integer :some_id, null: false - t.boolean :active, null: false, default: true - t.text :name - - t.text_limit :name, 255, name: 'check_name_length' - t.check_constraint :some_id_is_positive, 'some_id > 0' - end - - expect_table_columns_to_match(column_attributes, table_name) - - expect_check_constraint(table_name, 'check_name_length', 'char_length(name) <= 255') - expect_check_constraint(table_name, 'some_id_is_positive', 'some_id > 0') - end - end - - context 'when the text_limit is not named' do - it 'creates the table as expected, naming the text limit' do - model.create_table_with_constraints table_name do |t| - t.timestamps_with_timezone - t.integer :some_id, null: false - t.boolean :active, null: false, default: true - t.text :name - - t.text_limit :name, 255 - t.check_constraint :some_id_is_positive, 'some_id > 0' - end - - expect_table_columns_to_match(column_attributes, table_name) - - expect_check_constraint(table_name, 'check_cda6f69506', 'char_length(name) <= 255') - expect_check_constraint(table_name, 'some_id_is_positive', 'some_id > 0') - end - end - - it 'runs the change within a with_lock_retries' do - expect(model).to receive(:with_lock_retries).ordered.and_yield - expect(model).to receive(:create_table).ordered.and_call_original - expect(model).to receive(:execute).with(<<~SQL).ordered - ALTER TABLE "#{table_name}"\nADD CONSTRAINT "check_cda6f69506" CHECK (char_length("name") <= 255) - SQL - - model.create_table_with_constraints table_name do |t| - t.text :name - t.text_limit :name, 255 - end - end - - context 'when with_lock_retries re-runs the block' do - it 'only creates constraint for unique definitions' do - expected_sql = <<~SQL - ALTER TABLE "#{table_name}"\nADD CONSTRAINT "check_cda6f69506" CHECK (char_length("name") <= 255) - SQL - - expect(model).to receive(:create_table).twice.and_call_original - - expect(model).to receive(:execute).with(expected_sql).and_raise(ActiveRecord::LockWaitTimeout) - expect(model).to receive(:execute).with(expected_sql).and_call_original - - model.create_table_with_constraints table_name do |t| - t.timestamps_with_timezone - t.integer :some_id, null: false - t.boolean :active, null: false, default: true - t.text :name - - t.text_limit :name, 255 - end - - expect_table_columns_to_match(column_attributes, table_name) - - expect_check_constraint(table_name, 'check_cda6f69506', 'char_length(name) <= 255') - end - end - - context 'when constraints are given invalid names' do - let(:expected_max_length) { described_class::MAX_IDENTIFIER_NAME_LENGTH } - let(:expected_error_message) { "The maximum allowed constraint name is #{expected_max_length} characters" } - - context 'when the explicit text limit name is not valid' do - it 'raises an error' do - too_long_length = expected_max_length + 1 - - expect do - model.create_table_with_constraints table_name do |t| - t.timestamps_with_timezone - t.integer :some_id, null: false - t.boolean :active, null: false, default: true - t.text :name - - t.text_limit :name, 255, name: ('a' * too_long_length) - t.check_constraint :some_id_is_positive, 'some_id > 0' - end - end.to raise_error(expected_error_message) - end - end - - context 'when a check constraint name is not valid' do - it 'raises an error' do - too_long_length = expected_max_length + 1 - - expect do - model.create_table_with_constraints table_name do |t| - t.timestamps_with_timezone - t.integer :some_id, null: false - t.boolean :active, null: false, default: true - t.text :name - - t.text_limit :name, 255 - t.check_constraint ('a' * too_long_length), 'some_id > 0' - end - end.to raise_error(expected_error_message) - end - end - end - end - end - describe '#add_concurrent_index' do context 'outside a transaction' do before do @@ -1199,6 +1050,38 @@ RSpec.describe Gitlab::Database::MigrationHelpers, feature_category: :database d it_behaves_like 'foreign key checks' end + context 'if the schema cache does not include the constrained_columns column' do + let(:target_table) { nil } + + around do |ex| + model.transaction do + require_migration!('add_columns_to_postgres_foreign_keys') + AddColumnsToPostgresForeignKeys.new.down + Gitlab::Database::PostgresForeignKey.reset_column_information + Gitlab::Database::PostgresForeignKey.columns_hash # Force populate the column hash in the old schema + AddColumnsToPostgresForeignKeys.new.up + + # Rolling back reverts the schema cache information, so we need to run the example here before the rollback. + ex.run + + raise ActiveRecord::Rollback + end + + # make sure that we're resetting the schema cache here so that we don't leak the change to other tests. + Gitlab::Database::PostgresForeignKey.reset_column_information + # Double-check that the column information is back to normal + expect(Gitlab::Database::PostgresForeignKey.columns_hash.keys).to include('constrained_columns') + end + + # This test verifies that the situation we're trying to set up for the shared examples is actually being + # set up correctly + it 'correctly sets up the test without the column in the columns_hash' do + expect(Gitlab::Database::PostgresForeignKey.columns_hash.keys).not_to include('constrained_columns') + end + + it_behaves_like 'foreign key checks' + end + it 'compares by target table if no column given' do expect(model.foreign_key_exists?(:projects, :other_table)).to be_falsey end diff --git a/spec/lib/gitlab/database/migrations/pg_backend_pid_spec.rb b/spec/lib/gitlab/database/migrations/pg_backend_pid_spec.rb new file mode 100644 index 00000000000..515f59345ee --- /dev/null +++ b/spec/lib/gitlab/database/migrations/pg_backend_pid_spec.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::Migrations::PgBackendPid, feature_category: :database do + describe Gitlab::Database::Migrations::PgBackendPid::MigratorPgBackendPid do + let(:klass) do + Class.new do + def with_advisory_lock_connection + yield :conn + end + end + end + + it 're-yields with same arguments and wraps it with calls to .say' do + patched_instance = klass.prepend(described_class).new + expect(Gitlab::Database::Migrations::PgBackendPid).to receive(:say).twice + + expect { |b| patched_instance.with_advisory_lock_connection(&b) }.to yield_with_args(:conn) + end + end + + describe '.patch!' do + it 'patches ActiveRecord::Migrator' do + expect(ActiveRecord::Migrator).to receive(:prepend).with(described_class::MigratorPgBackendPid) + + described_class.patch! + end + end + + describe '.say' do + it 'outputs the connection information' do + conn = ActiveRecord::Base.connection + + expect(conn).to receive(:object_id).and_return(9876) + expect(conn).to receive(:select_value).with('SELECT pg_backend_pid()').and_return(12345) + expect(Gitlab::Database).to receive(:db_config_name).with(conn).and_return('main') + + expected_output = "main: == [advisory_lock_connection] object_id: 9876, pg_backend_pid: 12345\n" + + expect { described_class.say(conn) }.to output(expected_output).to_stdout + end + end +end diff --git a/spec/lib/gitlab/database/migrations/runner_spec.rb b/spec/lib/gitlab/database/migrations/runner_spec.rb index 66eb5a5de51..7c71076e8f3 100644 --- a/spec/lib/gitlab/database/migrations/runner_spec.rb +++ b/spec/lib/gitlab/database/migrations/runner_spec.rb @@ -65,7 +65,7 @@ RSpec.describe Gitlab::Database::Migrations::Runner, :reestablished_active_recor end before do - skip_if_multiple_databases_not_setup unless database == :main + skip_if_shared_database(database) stub_const('Gitlab::Database::Migrations::Runner::BASE_RESULT_DIR', base_result_dir) allow(ActiveRecord::Migrator).to receive(:new) do |dir, _all_migrations, _schema_migration_class, version_to_migrate| diff --git a/spec/lib/gitlab/database/partitioning/convert_table_to_first_list_partition_spec.rb b/spec/lib/gitlab/database/partitioning/convert_table_to_first_list_partition_spec.rb index cd3a94f5737..f4b13033270 100644 --- a/spec/lib/gitlab/database/partitioning/convert_table_to_first_list_partition_spec.rb +++ b/spec/lib/gitlab/database/partitioning/convert_table_to_first_list_partition_spec.rb @@ -2,11 +2,15 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::Partitioning::ConvertTableToFirstListPartition do +RSpec.describe Gitlab::Database::Partitioning::ConvertTableToFirstListPartition, feature_category: :database do include Gitlab::Database::DynamicModelHelpers include Database::TableSchemaHelpers - let(:migration_context) { Gitlab::Database::Migration[2.0].new } + let(:migration_context) do + Gitlab::Database::Migration[2.0].new.tap do |migration| + migration.extend Gitlab::Database::PartitioningMigrationHelpers::TableManagementHelpers + end + end let(:connection) { migration_context.connection } let(:table_name) { '_test_table_to_partition' } @@ -73,7 +77,9 @@ RSpec.describe Gitlab::Database::Partitioning::ConvertTableToFirstListPartition end describe "#prepare_for_partitioning" do - subject(:prepare) { converter.prepare_for_partitioning } + subject(:prepare) { converter.prepare_for_partitioning(async: async) } + + let(:async) { false } it 'adds a check constraint' do expect { prepare }.to change { @@ -83,9 +89,100 @@ RSpec.describe Gitlab::Database::Partitioning::ConvertTableToFirstListPartition .count }.from(0).to(1) end + + context 'when it fails to add constraint' do + before do + allow(migration_context).to receive(:add_check_constraint) + end + + it 'raises UnableToPartition error' do + expect { prepare } + .to raise_error(described_class::UnableToPartition) + .and change { + Gitlab::Database::PostgresConstraint + .check_constraints + .by_table_identifier(table_identifier) + .count + }.by(0) + end + end + + context 'when async' do + let(:async) { true } + + it 'adds a NOT VALID check constraint' do + expect { prepare }.to change { + Gitlab::Database::PostgresConstraint + .check_constraints + .by_table_identifier(table_identifier) + .count + }.from(0).to(1) + + constraint = + Gitlab::Database::PostgresConstraint + .check_constraints + .by_table_identifier(table_identifier) + .last + + expect(constraint.definition).to end_with('NOT VALID') + end + + it 'adds a PostgresAsyncConstraintValidation record' do + expect { prepare }.to change { + Gitlab::Database::AsyncConstraints::PostgresAsyncConstraintValidation.count + }.from(0).to(1) + + record = Gitlab::Database::AsyncConstraints::PostgresAsyncConstraintValidation.last + expect(record.name).to eq described_class::PARTITIONING_CONSTRAINT_NAME + expect(record).to be_check_constraint + end + + context 'when constraint exists but is not valid' do + before do + converter.prepare_for_partitioning(async: true) + end + + it 'validates the check constraint' do + expect { prepare }.to change { + Gitlab::Database::PostgresConstraint + .check_constraints + .by_table_identifier(table_identifier).first.constraint_valid? + }.from(false).to(true) + end + + context 'when it fails to validate constraint' do + before do + allow(migration_context).to receive(:validate_check_constraint) + end + + it 'raises UnableToPartition error' do + expect { prepare } + .to raise_error(described_class::UnableToPartition, + starting_with('Error validating partitioning constraint')) + .and change { + Gitlab::Database::PostgresConstraint + .check_constraints + .by_table_identifier(table_identifier) + .count + }.by(0) + end + end + end + + context 'when constraint exists and is valid' do + before do + converter.prepare_for_partitioning(async: false) + end + + it 'raises UnableToPartition error' do + expect(Gitlab::AppLogger).to receive(:info).with(starting_with('Nothing to do')) + prepare + end + end + end end - describe '#revert_prepare_for_partitioning' do + describe '#revert_preparation_for_partitioning' do before do converter.prepare_for_partitioning end @@ -102,11 +199,13 @@ RSpec.describe Gitlab::Database::Partitioning::ConvertTableToFirstListPartition end end - describe "#convert_to_zero_partition" do + describe "#partition" do subject(:partition) { converter.partition } + let(:async) { false } + before do - converter.prepare_for_partitioning + converter.prepare_for_partitioning(async: async) end context 'when the primary key is incorrect' do @@ -130,7 +229,15 @@ RSpec.describe Gitlab::Database::Partitioning::ConvertTableToFirstListPartition end it 'throws a reasonable error message' do - expect { partition }.to raise_error(described_class::UnableToPartition, /constraint /) + expect { partition }.to raise_error(described_class::UnableToPartition, /is not ready for partitioning./) + end + end + + context 'when supporting check constraint is not valid' do + let(:async) { true } + + it 'throws a reasonable error message' do + expect { partition }.to raise_error(described_class::UnableToPartition, /is not ready for partitioning./) end end @@ -203,7 +310,7 @@ RSpec.describe Gitlab::Database::Partitioning::ConvertTableToFirstListPartition proc do allow(migration_context.connection).to receive(:add_foreign_key).and_call_original expect(migration_context.connection).to receive(:add_foreign_key).with(from_table, to_table, any_args) - .and_wrap_original(&fail_first_time) + .and_wrap_original(&fail_first_time) end end @@ -231,9 +338,24 @@ RSpec.describe Gitlab::Database::Partitioning::ConvertTableToFirstListPartition end end end + + context 'when table has LFK triggers' do + before do + migration_context.track_record_deletions(table_name) + end + + it 'moves the trigger on the parent table', :aggregate_failures do + expect(migration_context.has_loose_foreign_key?(table_name)).to be_truthy + + expect { partition }.not_to raise_error + + expect(migration_context.has_loose_foreign_key?(table_name)).to be_truthy + expect(migration_context.has_loose_foreign_key?(parent_table_name)).to be_truthy + end + end end - describe '#revert_conversion_to_zero_partition' do + describe '#revert_partitioning' do before do converter.prepare_for_partitioning converter.partition @@ -269,5 +391,21 @@ RSpec.describe Gitlab::Database::Partitioning::ConvertTableToFirstListPartition expect { revert_conversion }.to change { converter.send(:sequences_owned_by, table_name).count }.from(0) .and change { converter.send(:sequences_owned_by, parent_table_name).count }.to(0) end + + context 'when table has LFK triggers' do + before do + migration_context.track_record_deletions(parent_table_name) + migration_context.track_record_deletions(table_name) + end + + it 'restores the trigger on the partition', :aggregate_failures do + expect(migration_context.has_loose_foreign_key?(table_name)).to be_truthy + expect(migration_context.has_loose_foreign_key?(parent_table_name)).to be_truthy + + expect { revert_conversion }.not_to raise_error + + expect(migration_context.has_loose_foreign_key?(table_name)).to be_truthy + end + end end end diff --git a/spec/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table_spec.rb b/spec/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table_spec.rb index 1885e84ac4c..fc279051800 100644 --- a/spec/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table_spec.rb +++ b/spec/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table_spec.rb @@ -54,6 +54,11 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::BackfillPartition allow(backfill_job).to receive(:sleep) end + after do + connection.drop_table source_table + connection.drop_table destination_table + end + let(:source_model) { Class.new(ActiveRecord::Base) } let(:destination_model) { Class.new(ActiveRecord::Base) } let(:timestamp) { Time.utc(2020, 1, 2).round } @@ -82,7 +87,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::BackfillPartition end it 'breaks the assigned batch into smaller batches' do - expect_next_instance_of(described_class::BulkCopy) do |bulk_copy| + expect_next_instance_of(Gitlab::Database::PartitioningMigrationHelpers::BulkCopy) do |bulk_copy| expect(bulk_copy).to receive(:copy_between).with(source1.id, source2.id) expect(bulk_copy).to receive(:copy_between).with(source3.id, source3.id) end diff --git a/spec/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers_spec.rb b/spec/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers_spec.rb index e76b1da3834..d87ef7a0953 100644 --- a/spec/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers_spec.rb +++ b/spec/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers_spec.rb @@ -2,10 +2,11 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHelpers do +RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHelpers, feature_category: :database do include Database::PartitioningHelpers include Database::TriggerHelpers include Database::TableSchemaHelpers + include MigrationsHelpers let(:migration) do ActiveRecord::Migration.new.extend(described_class) @@ -98,7 +99,8 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe migration.prepare_constraint_for_list_partitioning(table_name: source_table, partitioning_column: partition_column, parent_table_name: partitioned_table, - initial_partitioning_value: min_date) + initial_partitioning_value: min_date, + async: false) end end end @@ -484,17 +486,15 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe end context 'when records exist in the source table' do - let(:migration_class) { '::Gitlab::Database::PartitioningMigrationHelpers::BackfillPartitionedTable' } + let(:migration_class) { described_class::MIGRATION } let(:sub_batch_size) { described_class::SUB_BATCH_SIZE } - let(:pause_seconds) { described_class::PAUSE_SECONDS } let!(:first_id) { source_model.create!(name: 'Bob', age: 20).id } let!(:second_id) { source_model.create!(name: 'Alice', age: 30).id } let!(:third_id) { source_model.create!(name: 'Sam', age: 40).id } before do stub_const("#{described_class.name}::BATCH_SIZE", 2) - - expect(migration).to receive(:queue_background_migration_jobs_by_range_at_intervals).and_call_original + stub_const("#{described_class.name}::SUB_BATCH_SIZE", 1) end it 'enqueues jobs to copy each batch of data' do @@ -503,13 +503,13 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe Sidekiq::Testing.fake! do migration.enqueue_partitioning_data_migration source_table - expect(BackgroundMigrationWorker.jobs.size).to eq(2) - - first_job_arguments = [first_id, second_id, source_table.to_s, partitioned_table, 'id'] - expect(BackgroundMigrationWorker.jobs[0]['args']).to eq([migration_class, first_job_arguments]) - - second_job_arguments = [third_id, third_id, source_table.to_s, partitioned_table, 'id'] - expect(BackgroundMigrationWorker.jobs[1]['args']).to eq([migration_class, second_job_arguments]) + expect(migration_class).to have_scheduled_batched_migration( + table_name: source_table, + column_name: :id, + job_arguments: [partitioned_table], + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE + ) end end end diff --git a/spec/lib/gitlab/database/partitioning_spec.rb b/spec/lib/gitlab/database/partitioning_spec.rb index 4c0fde46b2f..4aa9d5f6df0 100644 --- a/spec/lib/gitlab/database/partitioning_spec.rb +++ b/spec/lib/gitlab/database/partitioning_spec.rb @@ -2,11 +2,11 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::Partitioning do +RSpec.describe Gitlab::Database::Partitioning, feature_category: :database do include Database::PartitioningHelpers include Database::TableSchemaHelpers - let(:connection) { ApplicationRecord.connection } + let(:main_connection) { ApplicationRecord.connection } around do |example| previously_registered_models = described_class.registered_models.dup @@ -84,7 +84,7 @@ RSpec.describe Gitlab::Database::Partitioning do before do table_names.each do |table_name| - connection.execute(<<~SQL) + execute_on_each_database(<<~SQL) CREATE TABLE #{table_name} ( id serial not null, created_at timestamptz not null, @@ -101,32 +101,12 @@ RSpec.describe Gitlab::Database::Partitioning do end context 'with multiple databases' do - before do - table_names.each do |table_name| - ci_connection.execute("DROP TABLE IF EXISTS #{table_name}") - - ci_connection.execute(<<~SQL) - CREATE TABLE #{table_name} ( - id serial not null, - created_at timestamptz not null, - PRIMARY KEY (id, created_at)) - PARTITION BY RANGE (created_at); - SQL - end - end - - after do - table_names.each do |table_name| - ci_connection.execute("DROP TABLE IF EXISTS #{table_name}") - end - end - it 'creates partitions in each database' do - skip_if_multiple_databases_not_setup(:ci) + skip_if_shared_database(:ci) expect { described_class.sync_partitions(models) } - .to change { find_partitions(table_names.first, conn: connection).size }.from(0) - .and change { find_partitions(table_names.last, conn: connection).size }.from(0) + .to change { find_partitions(table_names.first, conn: main_connection).size }.from(0) + .and change { find_partitions(table_names.last, conn: main_connection).size }.from(0) .and change { find_partitions(table_names.first, conn: ci_connection).size }.from(0) .and change { find_partitions(table_names.last, conn: ci_connection).size }.from(0) end @@ -161,10 +141,12 @@ RSpec.describe Gitlab::Database::Partitioning do end before do + skip_if_shared_database(:ci) + (table_names + ['partitioning_test3']).each do |table_name| - ci_connection.execute("DROP TABLE IF EXISTS #{table_name}") + execute_on_each_database("DROP TABLE IF EXISTS #{table_name}") - ci_connection.execute(<<~SQL) + execute_on_each_database(<<~SQL) CREATE TABLE #{table_name} ( id serial not null, created_at timestamptz not null, @@ -181,14 +163,12 @@ RSpec.describe Gitlab::Database::Partitioning do end it 'manages partitions for models for the given database', :aggregate_failures do - skip_if_multiple_databases_not_setup(:ci) - expect { described_class.sync_partitions([models.first, ci_model], only_on: 'ci') } .to change { find_partitions(ci_model.table_name, conn: ci_connection).size }.from(0) - expect(find_partitions(models.first.table_name).size).to eq(0) + expect(find_partitions(models.first.table_name, conn: main_connection).size).to eq(0) expect(find_partitions(models.first.table_name, conn: ci_connection).size).to eq(0) - expect(find_partitions(ci_model.table_name).size).to eq(0) + expect(find_partitions(ci_model.table_name, conn: main_connection).size).to eq(0) end end end diff --git a/spec/lib/gitlab/database/postgres_foreign_key_spec.rb b/spec/lib/gitlab/database/postgres_foreign_key_spec.rb index c128c56c708..03343c134ae 100644 --- a/spec/lib/gitlab/database/postgres_foreign_key_spec.rb +++ b/spec/lib/gitlab/database/postgres_foreign_key_spec.rb @@ -203,7 +203,7 @@ RSpec.describe Gitlab::Database::PostgresForeignKey, type: :model, feature_categ end end - context 'when supporting foreign keys to inherited tables' do + context 'when supporting foreign keys on partitioned tables' do before do ApplicationRecord.connection.execute(<<~SQL) create table #{schema_table_name('parent')} ( @@ -246,6 +246,40 @@ RSpec.describe Gitlab::Database::PostgresForeignKey, type: :model, feature_categ end end + context 'with two tables both partitioned' do + before do + ApplicationRecord.connection.execute(<<~SQL) + create table #{table_name('parent')} ( + id bigserial primary key not null + ) partition by hash(id); + + create table #{table_name('child')} + partition of #{table_name('parent')} for values with (remainder 1, modulus 2); + + create table #{table_name('ref_parent')} ( + id bigserial primary key not null + ) partition by hash(id); + + create table #{table_name('ref_child_1')} + partition of #{table_name('ref_parent')} for values with (remainder 1, modulus 3); + + create table #{table_name('ref_child_2')} + partition of #{table_name('ref_parent')} for values with (remainder 2, modulus 3); + + alter table #{table_name('parent')} add constraint fk foreign key (id) references #{table_name('ref_parent')} (id); + SQL + end + + describe '#child_foreign_keys' do + it 'is the child foreign keys of the partitioned parent fk' do + fk = described_class.by_constrained_table_name(table_name('parent')).first + children = fk.child_foreign_keys + expect(children.count).to eq(1) + expect(children.first.constrained_table_name).to eq(table_name('child')) + end + end + end + def schema_table_name(name) "public.#{table_name(name)}" end diff --git a/spec/lib/gitlab/database/query_analyzers/gitlab_schemas_validate_connection_spec.rb b/spec/lib/gitlab/database/query_analyzers/gitlab_schemas_validate_connection_spec.rb index d31be6cb883..ed05d1ce169 100644 --- a/spec/lib/gitlab/database/query_analyzers/gitlab_schemas_validate_connection_spec.rb +++ b/spec/lib/gitlab/database/query_analyzers/gitlab_schemas_validate_connection_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::Database::QueryAnalyzers::GitlabSchemasValidateConnection, query_analyzers: false, - feature_category: :pods do + feature_category: :cell do let(:analyzer) { described_class } # We keep only the GitlabSchemasValidateConnection analyzer running @@ -28,19 +28,19 @@ RSpec.describe Gitlab::Database::QueryAnalyzers::GitlabSchemasValidateConnection model: ApplicationRecord, sql: "SELECT 1 FROM projects LEFT JOIN ci_builds ON ci_builds.project_id=projects.id", expect_error: /The query tried to access \["projects", "ci_builds"\]/, - setup: -> (_) { skip_if_multiple_databases_not_setup(:ci) } + setup: -> (_) { skip_if_shared_database(:ci) } }, "for query accessing gitlab_ci and gitlab_main the gitlab_schemas is always ordered" => { model: ApplicationRecord, sql: "SELECT 1 FROM ci_builds LEFT JOIN projects ON ci_builds.project_id=projects.id", expect_error: /The query tried to access \["ci_builds", "projects"\]/, - setup: -> (_) { skip_if_multiple_databases_not_setup(:ci) } + setup: -> (_) { skip_if_shared_database(:ci) } }, "for query accessing main table from CI database" => { model: Ci::ApplicationRecord, sql: "SELECT 1 FROM projects", expect_error: /The query tried to access \["projects"\]/, - setup: -> (_) { skip_if_multiple_databases_not_setup(:ci) } + setup: -> (_) { skip_if_shared_database(:ci) } }, "for query accessing CI database" => { model: Ci::ApplicationRecord, @@ -51,13 +51,13 @@ RSpec.describe Gitlab::Database::QueryAnalyzers::GitlabSchemasValidateConnection model: ::ApplicationRecord, sql: "SELECT 1 FROM ci_builds", expect_error: /The query tried to access \["ci_builds"\]/, - setup: -> (_) { skip_if_multiple_databases_not_setup(:ci) } + setup: -> (_) { skip_if_shared_database(:ci) } }, "for query accessing unknown gitlab_schema" => { model: ::ApplicationRecord, sql: "SELECT 1 FROM new_table", expect_error: /The query tried to access \["new_table"\] \(of undefined_new_table\)/, - setup: -> (_) { skip_if_multiple_databases_not_setup(:ci) } + setup: -> (_) { skip_if_shared_database(:ci) } } } end @@ -77,7 +77,7 @@ RSpec.describe Gitlab::Database::QueryAnalyzers::GitlabSchemasValidateConnection context "when analyzer is enabled for tests", :query_analyzers do before do - skip_if_multiple_databases_not_setup(:ci) + skip_if_shared_database(:ci) end it "throws an error when trying to access a table that belongs to the gitlab_main schema from the ci database" do diff --git a/spec/lib/gitlab/database/query_analyzers/prevent_cross_database_modification_spec.rb b/spec/lib/gitlab/database/query_analyzers/prevent_cross_database_modification_spec.rb index a4322689bf9..887dd7c9838 100644 --- a/spec/lib/gitlab/database/query_analyzers/prevent_cross_database_modification_spec.rb +++ b/spec/lib/gitlab/database/query_analyzers/prevent_cross_database_modification_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::Database::QueryAnalyzers::PreventCrossDatabaseModification, query_analyzers: false, - feature_category: :pods do + feature_category: :cell do let_it_be(:pipeline, refind: true) { create(:ci_pipeline) } let_it_be(:project, refind: true) { create(:project) } @@ -118,6 +118,18 @@ RSpec.describe Gitlab::Database::QueryAnalyzers::PreventCrossDatabaseModificatio end end + context 'when ci_pipelines are ignored for cross modification' do + it 'does not raise error' do + Project.transaction do + expect do + described_class.temporary_ignore_tables_in_transaction(%w[ci_pipelines], url: 'TODO') do + run_queries + end + end.not_to raise_error + end + end + end + context 'when data modification happens in nested transactions' do it 'raises error' do Project.transaction(requires_new: true) do diff --git a/spec/lib/gitlab/database/schema_validation/adapters/column_database_adapter_spec.rb b/spec/lib/gitlab/database/schema_validation/adapters/column_database_adapter_spec.rb new file mode 100644 index 00000000000..13c4bc0b054 --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/adapters/column_database_adapter_spec.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::Adapters::ColumnDatabaseAdapter, feature_category: :database do + subject(:adapter) { described_class.new(db_result) } + + let(:column_name) { 'email' } + let(:column_default) { "'no-reply@gitlab.com'::character varying" } + let(:not_null) { true } + let(:db_result) do + { + 'table_name' => 'projects', + 'column_name' => column_name, + 'data_type' => 'character varying', + 'column_default' => column_default, + 'not_null' => not_null + } + end + + describe '#name' do + it { expect(adapter.name).to eq('email') } + end + + describe '#table_name' do + it { expect(adapter.table_name).to eq('projects') } + end + + describe '#data_type' do + it { expect(adapter.data_type).to eq('character varying') } + end + + describe '#default' do + context "when there's no default value in the column" do + let(:column_default) { nil } + + it { expect(adapter.default).to be_nil } + end + + context 'when the column name is id' do + let(:column_name) { 'id' } + + it { expect(adapter.default).to be_nil } + end + + context 'when the column default includes nextval' do + let(:column_default) { "nextval('my_seq'::regclass)" } + + it { expect(adapter.default).to be_nil } + end + + it { expect(adapter.default).to eq("DEFAULT 'no-reply@gitlab.com'::character varying") } + end + + describe '#nullable' do + context 'when column is not null' do + it { expect(adapter.nullable).to eq('NOT NULL') } + end + + context 'when column is nullable' do + let(:not_null) { false } + + it { expect(adapter.nullable).to be_nil } + end + end +end diff --git a/spec/lib/gitlab/database/schema_validation/adapters/column_structure_sql_adapter_spec.rb b/spec/lib/gitlab/database/schema_validation/adapters/column_structure_sql_adapter_spec.rb new file mode 100644 index 00000000000..d7e5c6e896e --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/adapters/column_structure_sql_adapter_spec.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::Adapters::ColumnStructureSqlAdapter, feature_category: :database do + subject(:adapter) { described_class.new(table_name, column_def) } + + let(:table_name) { 'my_table' } + let(:file_path) { Rails.root.join('spec/fixtures/structure.sql') } + let(:table_stmts) { PgQuery.parse(File.read(file_path)).tree.stmts.filter_map { |s| s.stmt.create_stmt } } + let(:column_stmts) { table_stmts.find { |table| table.relation.relname == 'test_table' }.table_elts } + let(:column_def) { column_stmts.find { |col| col.column_def.colname == column_name }.column_def } + + where(:column_name, :data_type, :default_value, :nullable) do + [ + ['id', 'bigint', nil, 'NOT NULL'], + ['integer_column', 'integer', nil, nil], + ['integer_with_default_column', 'integer', 'DEFAULT 1', nil], + ['smallint_with_default_column', 'smallint', 'DEFAULT 0', 'NOT NULL'], + ['double_precision_with_default_column', 'double precision', 'DEFAULT 1.0', nil], + ['numeric_with_default_column', 'numeric', 'DEFAULT 1.0', 'NOT NULL'], + ['boolean_with_default_colum', 'boolean', 'DEFAULT true', 'NOT NULL'], + ['varying_with_default_column', 'character varying', "DEFAULT 'DEFAULT'::character varying", 'NOT NULL'], + ['varying_with_limit_and_default_column', 'character varying(255)', "DEFAULT 'DEFAULT'::character varying", nil], + ['text_with_default_column', 'text', "DEFAULT ''::text", 'NOT NULL'], + ['array_with_default_column', 'character varying(255)[]', "DEFAULT '{one,two}'::character varying[]", 'NOT NULL'], + ['jsonb_with_default_column', 'jsonb', "DEFAULT '[]'::jsonb", 'NOT NULL'], + ['timestamptz_with_default_column', 'timestamp(6) with time zone', "DEFAULT now()", nil], + ['timestamp_with_default_column', 'timestamp(6) without time zone', + "DEFAULT '2022-01-23 00:00:00+00'::timestamp without time zone", 'NOT NULL'], + ['date_with_default_column', 'date', 'DEFAULT 2023-04-05', nil], + ['inet_with_default_column', 'inet', "DEFAULT '0.0.0.0'::inet", 'NOT NULL'], + ['macaddr_with_default_column', 'macaddr', "DEFAULT '00-00-00-00-00-000'::macaddr", 'NOT NULL'], + ['uuid_with_default_column', 'uuid', "DEFAULT '00000000-0000-0000-0000-000000000000'::uuid", 'NOT NULL'], + ['bytea_with_default_column', 'bytea', "DEFAULT '\\xDEADBEEF'::bytea", nil] + ] + end + + with_them do + describe '#name' do + it { expect(adapter.name).to eq(column_name) } + end + + describe '#table_name' do + it { expect(adapter.table_name).to eq(table_name) } + end + + describe '#data_type' do + it { expect(adapter.data_type).to eq(data_type) } + end + + describe '#nullable' do + it { expect(adapter.nullable).to eq(nullable) } + end + + describe '#default' do + it { expect(adapter.default).to eq(default_value) } + end + end + + context 'when the data type is not mapped' do + let(:column_name) { 'unmapped_column_type' } + let(:error_class) { Gitlab::Database::SchemaValidation::Adapters::UndefinedPGType } + + describe '#data_type' do + it { expect { adapter.data_type }.to raise_error(error_class) } + end + end +end diff --git a/spec/lib/gitlab/database/schema_validation/database_spec.rb b/spec/lib/gitlab/database/schema_validation/database_spec.rb index eadaf683a29..8fd98382625 100644 --- a/spec/lib/gitlab/database/schema_validation/database_spec.rb +++ b/spec/lib/gitlab/database/schema_validation/database_spec.rb @@ -2,109 +2,90 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::SchemaValidation::Database, feature_category: :database do +RSpec.shared_examples 'database schema assertions for' do |fetch_by_name_method, exists_method, all_objects_method| subject(:database) { described_class.new(connection) } let(:database_model) { Gitlab::Database.database_base_models['main'] } let(:connection) { database_model.connection } - context 'when having indexes' do - let(:schema_object) { Gitlab::Database::SchemaValidation::SchemaObjects::Index } - let(:results) do - [['index', 'CREATE UNIQUE INDEX "index" ON public.achievements USING btree (namespace_id, lower(name))']] - end + before do + allow(connection).to receive(:select_rows).and_return(results) + allow(connection).to receive(:exec_query).and_return(results) + end - before do - allow(connection).to receive(:select_rows).and_return(results) + describe "##{fetch_by_name_method}" do + it 'returns nil when schema object does not exists' do + expect(database.public_send(fetch_by_name_method, 'invalid-object-name')).to be_nil end - describe '#fetch_index_by_name' do - context 'when index does not exist' do - it 'returns nil' do - index = database.fetch_index_by_name('non_existing_index') - - expect(index).to be_nil - end - end - - it 'returns index by name' do - index = database.fetch_index_by_name('index') - - expect(index.name).to eq('index') - end + it 'returns the schema object by name' do + expect(database.public_send(fetch_by_name_method, valid_schema_object_name).name).to eq(valid_schema_object_name) end + end - describe '#index_exists?' do - context 'when index exists' do - it 'returns true' do - index_exists = database.index_exists?('index') + describe "##{exists_method}" do + it 'returns true when schema object exists' do + expect(database.public_send(exists_method, valid_schema_object_name)).to be_truthy + end - expect(index_exists).to be_truthy - end - end + it 'returns false when schema object does not exists' do + expect(database.public_send(exists_method, 'invalid-object')).to be_falsey + end + end - context 'when index does not exist' do - it 'returns false' do - index_exists = database.index_exists?('non_existing_index') + describe "##{all_objects_method}" do + it 'returns all the schema objects' do + schema_objects = database.public_send(all_objects_method) - expect(index_exists).to be_falsey - end - end + expect(schema_objects).to all(be_a(schema_object)) + expect(schema_objects.map(&:name)).to eq([valid_schema_object_name]) end + end +end - describe '#indexes' do - it 'returns indexes' do - indexes = database.indexes - - expect(indexes).to all(be_a(schema_object)) - expect(indexes.map(&:name)).to eq(['index']) - end +RSpec.describe Gitlab::Database::SchemaValidation::Database, feature_category: :database do + context 'when having indexes' do + let(:schema_object) { Gitlab::Database::SchemaValidation::SchemaObjects::Index } + let(:valid_schema_object_name) { 'index' } + let(:results) do + [['index', 'CREATE UNIQUE INDEX "index" ON public.achievements USING btree (namespace_id, lower(name))']] end + + include_examples 'database schema assertions for', 'fetch_index_by_name', 'index_exists?', 'indexes' end context 'when having triggers' do let(:schema_object) { Gitlab::Database::SchemaValidation::SchemaObjects::Trigger } + let(:valid_schema_object_name) { 'my_trigger' } let(:results) do - { 'my_trigger' => 'CREATE TRIGGER my_trigger BEFORE INSERT ON todos FOR EACH ROW EXECUTE FUNCTION trigger()' } + [['my_trigger', 'CREATE TRIGGER my_trigger BEFORE INSERT ON todos FOR EACH ROW EXECUTE FUNCTION trigger()']] end - before do - allow(database).to receive(:fetch_triggers).and_return(results) - end - - describe '#fetch_trigger_by_name' do - context 'when trigger does not exist' do - it 'returns nil' do - expect(database.fetch_trigger_by_name('non_existing_trigger')).to be_nil - end - end - - it 'returns trigger by name' do - expect(database.fetch_trigger_by_name('my_trigger').name).to eq('my_trigger') - end - end + include_examples 'database schema assertions for', 'fetch_trigger_by_name', 'trigger_exists?', 'triggers' + end - describe '#trigger_exists?' do - context 'when trigger exists' do - it 'returns true' do - expect(database.trigger_exists?('my_trigger')).to be_truthy - end - end - - context 'when trigger does not exist' do - it 'returns false' do - expect(database.trigger_exists?('non_existing_trigger')).to be_falsey - end - end + context 'when having tables' do + let(:schema_object) { Gitlab::Database::SchemaValidation::SchemaObjects::Table } + let(:valid_schema_object_name) { 'my_table' } + let(:results) do + [ + { + 'table_name' => 'my_table', + 'column_name' => 'id', + 'not_null' => true, + 'data_type' => 'bigint', + 'column_default' => "nextval('audit_events_id_seq'::regclass)" + }, + { + 'table_name' => 'my_table', + 'column_name' => 'details', + 'not_null' => false, + 'data_type' => 'text', + 'column_default' => nil + } + ] end - describe '#triggers' do - it 'returns triggers' do - triggers = database.triggers - - expect(triggers).to all(be_a(schema_object)) - expect(triggers.map(&:name)).to eq(['my_trigger']) - end - end + include_examples 'database schema assertions for', 'fetch_table_by_name', 'table_exists?', 'tables' end end diff --git a/spec/lib/gitlab/database/schema_validation/inconsistency_spec.rb b/spec/lib/gitlab/database/schema_validation/inconsistency_spec.rb new file mode 100644 index 00000000000..cb3df75b3fb --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/inconsistency_spec.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::Inconsistency, feature_category: :database do + let(:validator) { Gitlab::Database::SchemaValidation::Validators::DifferentDefinitionIndexes } + + let(:database_statement) { 'CREATE INDEX index_name ON public.achievements USING btree (namespace_id)' } + let(:structure_sql_statement) { 'CREATE INDEX index_name ON public.achievements USING btree (id)' } + + let(:structure_stmt) { PgQuery.parse(structure_sql_statement).tree.stmts.first.stmt.index_stmt } + let(:database_stmt) { PgQuery.parse(database_statement).tree.stmts.first.stmt.index_stmt } + + let(:structure_sql_object) { Gitlab::Database::SchemaValidation::SchemaObjects::Index.new(structure_stmt) } + let(:database_object) { Gitlab::Database::SchemaValidation::SchemaObjects::Index.new(database_stmt) } + + subject(:inconsistency) { described_class.new(validator, structure_sql_object, database_object) } + + describe '#object_name' do + it 'returns the index name' do + expect(inconsistency.object_name).to eq('index_name') + end + end + + describe '#diff' do + it 'returns a diff between the structure.sql and the database' do + expect(inconsistency.diff).to be_a(Diffy::Diff) + expect(inconsistency.diff.string1).to eq("#{structure_sql_statement}\n") + expect(inconsistency.diff.string2).to eq("#{database_statement}\n") + end + end + + describe '#error_message' do + it 'returns the error message' do + stub_const "#{validator}::ERROR_MESSAGE", 'error message %s' + + expect(inconsistency.error_message).to eq('error message index_name') + end + end + + describe '#type' do + it 'returns the type of the validator' do + expect(inconsistency.type).to eq('different_definition_indexes') + end + end + + describe '#table_name' do + it 'returns the table name' do + expect(inconsistency.table_name).to eq('achievements') + end + end + + describe '#inspect' do + let(:expected_output) do + <<~MSG + ------------------------------------------------------ + The index_name index has a different statement between structure.sql and database + Diff: + \e[31m-CREATE INDEX index_name ON public.achievements USING btree (id)\e[0m + \e[32m+CREATE INDEX index_name ON public.achievements USING btree (namespace_id)\e[0m + + ------------------------------------------------------ + MSG + end + + it 'prints the inconsistency message' do + expect(inconsistency.inspect).to eql(expected_output) + end + end +end diff --git a/spec/lib/gitlab/database/schema_validation/runner_spec.rb b/spec/lib/gitlab/database/schema_validation/runner_spec.rb index ddbdedcd8b4..f5d1c6ba31b 100644 --- a/spec/lib/gitlab/database/schema_validation/runner_spec.rb +++ b/spec/lib/gitlab/database/schema_validation/runner_spec.rb @@ -28,7 +28,7 @@ RSpec.describe Gitlab::Database::SchemaValidation::Runner, feature_category: :da subject(:inconsistencies) { described_class.new(structure_sql, database, validators: validators).execute } let(:class_name) { 'Gitlab::Database::SchemaValidation::Validators::ExtraIndexes' } - let(:inconsistency_class_name) { 'Gitlab::Database::SchemaValidation::Validators::BaseValidator::Inconsistency' } + let(:inconsistency_class_name) { 'Gitlab::Database::SchemaValidation::Inconsistency' } let(:extra_indexes) { class_double(class_name) } let(:instace_extra_index) { instance_double(class_name, execute: [inconsistency]) } diff --git a/spec/lib/gitlab/database/schema_validation/schema_inconsistency_spec.rb b/spec/lib/gitlab/database/schema_validation/schema_inconsistency_spec.rb new file mode 100644 index 00000000000..7d6a279def9 --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/schema_inconsistency_spec.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::SchemaInconsistency, type: :model, feature_category: :database do + it { is_expected.to be_a ApplicationRecord } + + describe 'associations' do + it { is_expected.to belong_to(:issue) } + end + + describe "Validations" do + it { is_expected.to validate_presence_of(:object_name) } + it { is_expected.to validate_presence_of(:valitador_name) } + it { is_expected.to validate_presence_of(:table_name) } + end +end diff --git a/spec/lib/gitlab/database/schema_validation/schema_objects/column_spec.rb b/spec/lib/gitlab/database/schema_validation/schema_objects/column_spec.rb new file mode 100644 index 00000000000..74bc5f43b50 --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/schema_objects/column_spec.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::SchemaObjects::Column, feature_category: :database do + subject(:column) { described_class.new(adapter) } + + let(:database_adapter) { 'Gitlab::Database::SchemaValidation::Adapters::ColumnDatabaseAdapter' } + let(:adapter) do + instance_double(database_adapter, name: 'id', table_name: 'projects', + data_type: 'bigint', default: nil, nullable: 'NOT NULL') + end + + describe '#name' do + it { expect(column.name).to eq('id') } + end + + describe '#table_name' do + it { expect(column.table_name).to eq('projects') } + end + + describe '#statement' do + it { expect(column.statement).to eq('id bigint NOT NULL') } + end +end diff --git a/spec/lib/gitlab/database/schema_validation/schema_objects/index_spec.rb b/spec/lib/gitlab/database/schema_validation/schema_objects/index_spec.rb index 1aaa994e3bb..43d8fa38ec8 100644 --- a/spec/lib/gitlab/database/schema_validation/schema_objects/index_spec.rb +++ b/spec/lib/gitlab/database/schema_validation/schema_objects/index_spec.rb @@ -5,6 +5,7 @@ require 'spec_helper' RSpec.describe Gitlab::Database::SchemaValidation::SchemaObjects::Index, feature_category: :database do let(:statement) { 'CREATE INDEX index_name ON public.achievements USING btree (namespace_id)' } let(:name) { 'index_name' } + let(:table_name) { 'achievements' } include_examples 'schema objects assertions for', 'index_stmt' end diff --git a/spec/lib/gitlab/database/schema_validation/schema_objects/table_spec.rb b/spec/lib/gitlab/database/schema_validation/schema_objects/table_spec.rb new file mode 100644 index 00000000000..6c2efee056b --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/schema_objects/table_spec.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::SchemaObjects::Table, feature_category: :database do + subject(:table) { described_class.new(name, columns) } + + let(:name) { 'my_table' } + let(:column_class) { 'Gitlab::Database::SchemaValidation::SchemaObjects::Column' } + let(:columns) do + [ + instance_double(column_class, name: 'id', statement: 'id bigint NOT NULL'), + instance_double(column_class, name: 'col', statement: 'col text') + ] + end + + describe '#name' do + it { expect(table.name).to eq('my_table') } + end + + describe '#table_name' do + it { expect(table.table_name).to eq('my_table') } + end + + describe '#statement' do + it { expect(table.statement).to eq('CREATE TABLE my_table (id bigint NOT NULL, col text)') } + end + + describe '#fetch_column_by_name' do + it { expect(table.fetch_column_by_name('col')).not_to be_nil } + + it { expect(table.fetch_column_by_name('invalid')).to be_nil } + end + + describe '#column_exists?' do + it { expect(table.column_exists?('col')).to eq(true) } + + it { expect(table.column_exists?('invalid')).to eq(false) } + end +end diff --git a/spec/lib/gitlab/database/schema_validation/schema_objects/trigger_spec.rb b/spec/lib/gitlab/database/schema_validation/schema_objects/trigger_spec.rb index 8000a54ee27..3c2481dfae0 100644 --- a/spec/lib/gitlab/database/schema_validation/schema_objects/trigger_spec.rb +++ b/spec/lib/gitlab/database/schema_validation/schema_objects/trigger_spec.rb @@ -5,6 +5,7 @@ require 'spec_helper' RSpec.describe Gitlab::Database::SchemaValidation::SchemaObjects::Trigger, feature_category: :database do let(:statement) { 'CREATE TRIGGER my_trigger BEFORE INSERT ON todos FOR EACH ROW EXECUTE FUNCTION trigger()' } let(:name) { 'my_trigger' } + let(:table_name) { 'todos' } include_examples 'schema objects assertions for', 'create_trig_stmt' end diff --git a/spec/lib/gitlab/database/schema_validation/structure_sql_spec.rb b/spec/lib/gitlab/database/schema_validation/structure_sql_spec.rb index cc0bd4125ef..b0c056ff5db 100644 --- a/spec/lib/gitlab/database/schema_validation/structure_sql_spec.rb +++ b/spec/lib/gitlab/database/schema_validation/structure_sql_spec.rb @@ -2,81 +2,65 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::SchemaValidation::StructureSql, feature_category: :database do - let(:structure_file_path) { Rails.root.join('spec/fixtures/structure.sql') } - let(:schema_name) { 'public' } - +RSpec.shared_examples 'structure sql schema assertions for' do |object_exists_method, all_objects_method| subject(:structure_sql) { described_class.new(structure_file_path, schema_name) } - context 'when having indexes' do - describe '#index_exists?' do - subject(:index_exists) { structure_sql.index_exists?(index_name) } + let(:structure_file_path) { Rails.root.join('spec/fixtures/structure.sql') } + let(:schema_name) { 'public' } - context 'when the index does not exist' do - let(:index_name) { 'non-existent-index' } + describe "##{object_exists_method}" do + it 'returns true when schema object exists' do + expect(structure_sql.public_send(object_exists_method, valid_schema_object_name)).to be_truthy + end - it 'returns false' do - expect(index_exists).to be_falsey - end - end + it 'returns false when schema object does not exists' do + expect(structure_sql.public_send(object_exists_method, 'invalid-object-name')).to be_falsey + end + end - context 'when the index exists' do - let(:index_name) { 'index' } + describe "##{all_objects_method}" do + it 'returns all the schema objects' do + schema_objects = structure_sql.public_send(all_objects_method) - it 'returns true' do - expect(index_exists).to be_truthy - end - end + expect(schema_objects).to all(be_a(schema_object)) + expect(schema_objects.map(&:name)).to eq(expected_objects) end + end +end - describe '#indexes' do - it 'returns indexes' do - indexes = structure_sql.indexes +RSpec.describe Gitlab::Database::SchemaValidation::StructureSql, feature_category: :database do + let(:structure_file_path) { Rails.root.join('spec/fixtures/structure.sql') } + let(:schema_name) { 'public' } - expected_indexes = %w[ - missing_index - wrong_index - index - index_namespaces_public_groups_name_id - index_on_deploy_keys_id_and_type_and_public - index_users_on_public_email_excluding_null_and_empty - ] + subject(:structure_sql) { described_class.new(structure_file_path, schema_name) } - expect(indexes).to all(be_a(Gitlab::Database::SchemaValidation::SchemaObjects::Index)) - expect(indexes.map(&:name)).to eq(expected_indexes) - end + context 'when having indexes' do + let(:schema_object) { Gitlab::Database::SchemaValidation::SchemaObjects::Index } + let(:valid_schema_object_name) { 'index' } + let(:expected_objects) do + %w[missing_index wrong_index index index_namespaces_public_groups_name_id + index_on_deploy_keys_id_and_type_and_public index_users_on_public_email_excluding_null_and_empty] end + + include_examples 'structure sql schema assertions for', 'index_exists?', 'indexes' end context 'when having triggers' do - describe '#trigger_exists?' do - subject(:trigger_exists) { structure_sql.trigger_exists?(name) } + let(:schema_object) { Gitlab::Database::SchemaValidation::SchemaObjects::Trigger } + let(:valid_schema_object_name) { 'trigger' } + let(:expected_objects) { %w[trigger wrong_trigger missing_trigger_1 projects_loose_fk_trigger] } - context 'when the trigger does not exist' do - let(:name) { 'non-existent-trigger' } - - it 'returns false' do - expect(trigger_exists).to be_falsey - end - end - - context 'when the trigger exists' do - let(:name) { 'trigger' } + include_examples 'structure sql schema assertions for', 'trigger_exists?', 'triggers' + end - it 'returns true' do - expect(trigger_exists).to be_truthy - end - end + context 'when having tables' do + let(:schema_object) { Gitlab::Database::SchemaValidation::SchemaObjects::Table } + let(:valid_schema_object_name) { 'test_table' } + let(:expected_objects) do + %w[test_table ci_project_mirrors wrong_table extra_table_columns missing_table missing_table_columns + operations_user_lists] end - describe '#triggers' do - it 'returns triggers' do - triggers = structure_sql.triggers - expected_triggers = %w[trigger wrong_trigger missing_trigger_1 projects_loose_fk_trigger] - - expect(triggers).to all(be_a(Gitlab::Database::SchemaValidation::SchemaObjects::Trigger)) - expect(triggers.map(&:name)).to eq(expected_triggers) - end - end + include_examples 'structure sql schema assertions for', 'table_exists?', 'tables' end end diff --git a/spec/lib/gitlab/database/schema_validation/track_inconsistency_spec.rb b/spec/lib/gitlab/database/schema_validation/track_inconsistency_spec.rb new file mode 100644 index 00000000000..84db721fc2d --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/track_inconsistency_spec.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::TrackInconsistency, feature_category: :database do + describe '#execute' do + let(:validator) { Gitlab::Database::SchemaValidation::Validators::DifferentDefinitionIndexes } + + let(:database_statement) { 'CREATE INDEX index_name ON public.achievements USING btree (namespace_id)' } + let(:structure_sql_statement) { 'CREATE INDEX index_name ON public.achievements USING btree (id)' } + + let(:structure_stmt) { PgQuery.parse(structure_sql_statement).tree.stmts.first.stmt.index_stmt } + let(:database_stmt) { PgQuery.parse(database_statement).tree.stmts.first.stmt.index_stmt } + + let(:structure_sql_object) { Gitlab::Database::SchemaValidation::SchemaObjects::Index.new(structure_stmt) } + let(:database_object) { Gitlab::Database::SchemaValidation::SchemaObjects::Index.new(database_stmt) } + + let(:inconsistency) do + Gitlab::Database::SchemaValidation::Inconsistency.new(validator, structure_sql_object, database_object) + end + + let_it_be(:project) { create(:project) } + let_it_be(:user) { create(:user) } + + subject(:execute) { described_class.new(inconsistency, project, user).execute } + + before do + stub_spam_services + end + + context 'when is not GitLab.com' do + it 'does not create a schema inconsistency record' do + allow(Gitlab).to receive(:com?).and_return(false) + + expect { execute }.not_to change { Gitlab::Database::SchemaValidation::SchemaInconsistency.count } + end + end + + context 'when the issue creation fails' do + let(:issue_creation) { instance_double(Mutations::Issues::Create, resolve: { errors: 'error' }) } + + before do + allow(Mutations::Issues::Create).to receive(:new).and_return(issue_creation) + end + + it 'does not create a schema inconsistency record' do + allow(Gitlab).to receive(:com?).and_return(true) + + expect { execute }.not_to change { Gitlab::Database::SchemaValidation::SchemaInconsistency.count } + end + end + + context 'when a new inconsistency is found' do + before do + project.add_developer(user) + end + + it 'creates a new schema inconsistency record' do + allow(Gitlab).to receive(:com?).and_return(true) + + expect { execute }.to change { Gitlab::Database::SchemaValidation::SchemaInconsistency.count } + end + end + + context 'when the schema inconsistency already exists' do + before do + project.add_developer(user) + end + + let!(:schema_inconsistency) do + create(:schema_inconsistency, object_name: 'index_name', table_name: 'achievements', + valitador_name: 'different_definition_indexes') + end + + it 'does not create a schema inconsistency record' do + allow(Gitlab).to receive(:com?).and_return(true) + + expect { execute }.not_to change { Gitlab::Database::SchemaValidation::SchemaInconsistency.count } + end + end + end +end diff --git a/spec/lib/gitlab/database/schema_validation/validators/base_validator_spec.rb b/spec/lib/gitlab/database/schema_validation/validators/base_validator_spec.rb index 2f38c25cf68..036ad6424f0 100644 --- a/spec/lib/gitlab/database/schema_validation/validators/base_validator_spec.rb +++ b/spec/lib/gitlab/database/schema_validation/validators/base_validator_spec.rb @@ -8,10 +8,15 @@ RSpec.describe Gitlab::Database::SchemaValidation::Validators::BaseValidator, fe it 'returns an array of all validators' do expect(all_validators).to eq([ + Gitlab::Database::SchemaValidation::Validators::ExtraTables, + Gitlab::Database::SchemaValidation::Validators::ExtraTableColumns, Gitlab::Database::SchemaValidation::Validators::ExtraIndexes, Gitlab::Database::SchemaValidation::Validators::ExtraTriggers, + Gitlab::Database::SchemaValidation::Validators::MissingTables, + Gitlab::Database::SchemaValidation::Validators::MissingTableColumns, Gitlab::Database::SchemaValidation::Validators::MissingIndexes, Gitlab::Database::SchemaValidation::Validators::MissingTriggers, + Gitlab::Database::SchemaValidation::Validators::DifferentDefinitionTables, Gitlab::Database::SchemaValidation::Validators::DifferentDefinitionIndexes, Gitlab::Database::SchemaValidation::Validators::DifferentDefinitionTriggers ]) diff --git a/spec/lib/gitlab/database/schema_validation/validators/different_definition_tables_spec.rb b/spec/lib/gitlab/database/schema_validation/validators/different_definition_tables_spec.rb new file mode 100644 index 00000000000..746418b757e --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/validators/different_definition_tables_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::Validators::DifferentDefinitionTables, feature_category: :database do + include_examples 'table validators', described_class, ['wrong_table'] +end diff --git a/spec/lib/gitlab/database/schema_validation/validators/extra_table_columns_spec.rb b/spec/lib/gitlab/database/schema_validation/validators/extra_table_columns_spec.rb new file mode 100644 index 00000000000..9d17a2fffa9 --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/validators/extra_table_columns_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::Validators::ExtraTableColumns, feature_category: :database do + include_examples 'table validators', described_class, ['extra_table_columns'] +end diff --git a/spec/lib/gitlab/database/schema_validation/validators/extra_tables_spec.rb b/spec/lib/gitlab/database/schema_validation/validators/extra_tables_spec.rb new file mode 100644 index 00000000000..edaf79e3c93 --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/validators/extra_tables_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::Validators::ExtraTables, feature_category: :database do + include_examples 'table validators', described_class, ['extra_table'] +end diff --git a/spec/lib/gitlab/database/schema_validation/validators/missing_table_columns_spec.rb b/spec/lib/gitlab/database/schema_validation/validators/missing_table_columns_spec.rb new file mode 100644 index 00000000000..de2956b4dd9 --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/validators/missing_table_columns_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::Validators::MissingTableColumns, feature_category: :database do + include_examples 'table validators', described_class, ['missing_table_columns'] +end diff --git a/spec/lib/gitlab/database/schema_validation/validators/missing_tables_spec.rb b/spec/lib/gitlab/database/schema_validation/validators/missing_tables_spec.rb new file mode 100644 index 00000000000..7c80923e860 --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/validators/missing_tables_spec.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::Validators::MissingTables, feature_category: :database do + missing_tables = %w[ci_project_mirrors missing_table operations_user_lists test_table] + + include_examples 'table validators', described_class, missing_tables +end diff --git a/spec/lib/gitlab/database/tables_locker_spec.rb b/spec/lib/gitlab/database/tables_locker_spec.rb index 30f0f9376c8..aaafe27f7ca 100644 --- a/spec/lib/gitlab/database/tables_locker_spec.rb +++ b/spec/lib/gitlab/database/tables_locker_spec.rb @@ -3,9 +3,13 @@ require 'spec_helper' RSpec.describe Gitlab::Database::TablesLocker, :suppress_gitlab_schemas_validate_connection, :silence_stdout, - feature_category: :pods do + feature_category: :cell do let(:default_lock_writes_manager) do - instance_double(Gitlab::Database::LockWritesManager, lock_writes: nil, unlock_writes: nil) + instance_double( + Gitlab::Database::LockWritesManager, + lock_writes: { action: 'any action' }, + unlock_writes: { action: 'unlocked' } + ) end before do @@ -81,6 +85,10 @@ RSpec.describe Gitlab::Database::TablesLocker, :suppress_gitlab_schemas_validate subject end + + it 'returns list of actions' do + expect(subject).to include({ action: 'any action' }) + end end shared_examples "unlock tables" do |gitlab_schema, database_name| @@ -110,6 +118,10 @@ RSpec.describe Gitlab::Database::TablesLocker, :suppress_gitlab_schemas_validate subject end + + it 'returns list of actions' do + expect(subject).to include({ action: 'unlocked' }) + end end shared_examples "lock partitions" do |partition_identifier, database_name| @@ -154,7 +166,7 @@ RSpec.describe Gitlab::Database::TablesLocker, :suppress_gitlab_schemas_validate context 'when running on single database' do before do - skip_if_multiple_databases_are_setup(:ci) + skip_if_database_exists(:ci) end describe '#lock_writes' do @@ -191,7 +203,7 @@ RSpec.describe Gitlab::Database::TablesLocker, :suppress_gitlab_schemas_validate context 'when running on multiple databases' do before do - skip_if_multiple_databases_not_setup(:ci) + skip_if_shared_database(:ci) end describe '#lock_writes' do diff --git a/spec/lib/gitlab/database/tables_truncate_spec.rb b/spec/lib/gitlab/database/tables_truncate_spec.rb index 3bb2f4e982c..bcbed0332e2 100644 --- a/spec/lib/gitlab/database/tables_truncate_spec.rb +++ b/spec/lib/gitlab/database/tables_truncate_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::Database::TablesTruncate, :reestablished_active_record_base, - :suppress_gitlab_schemas_validate_connection, feature_category: :pods do + :suppress_gitlab_schemas_validate_connection, feature_category: :cell do include MigrationsHelpers let(:min_batch_size) { 1 } @@ -48,7 +48,7 @@ RSpec.describe Gitlab::Database::TablesTruncate, :reestablished_active_record_ba end before do - skip_if_multiple_databases_not_setup(:ci) + skip_if_shared_database(:ci) # Creating some test tables on the main database main_tables_sql = <<~SQL @@ -79,8 +79,7 @@ RSpec.describe Gitlab::Database::TablesTruncate, :reestablished_active_record_ba ALTER TABLE _test_gitlab_hook_logs DETACH PARTITION gitlab_partitions_dynamic._test_gitlab_hook_logs_202201; SQL - main_connection.execute(main_tables_sql) - ci_connection.execute(main_tables_sql) + execute_on_each_database(main_tables_sql) ci_tables_sql = <<~SQL CREATE TABLE _test_gitlab_ci_items (id serial NOT NULL PRIMARY KEY); @@ -92,15 +91,13 @@ RSpec.describe Gitlab::Database::TablesTruncate, :reestablished_active_record_ba ); SQL - main_connection.execute(ci_tables_sql) - ci_connection.execute(ci_tables_sql) + execute_on_each_database(ci_tables_sql) internal_tables_sql = <<~SQL CREATE TABLE _test_gitlab_shared_items (id serial NOT NULL PRIMARY KEY); SQL - main_connection.execute(internal_tables_sql) - ci_connection.execute(internal_tables_sql) + execute_on_each_database(internal_tables_sql) # Filling the tables 5.times do |i| @@ -314,8 +311,7 @@ RSpec.describe Gitlab::Database::TablesTruncate, :reestablished_active_record_ba context 'when running with multiple shared databases' do before do skip_if_multiple_databases_not_setup(:ci) - ci_db_config = Ci::ApplicationRecord.connection_db_config - allow(::Gitlab::Database).to receive(:db_config_share_with).with(ci_db_config).and_return('main') + skip_if_database_exists(:ci) end it 'raises an error when truncating the main database that it is a single database setup' do diff --git a/spec/lib/gitlab/database/transaction_timeout_settings_spec.rb b/spec/lib/gitlab/database/transaction_timeout_settings_spec.rb index 5b68f9a3757..2725b22ca9d 100644 --- a/spec/lib/gitlab/database/transaction_timeout_settings_spec.rb +++ b/spec/lib/gitlab/database/transaction_timeout_settings_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::TransactionTimeoutSettings, feature_category: :pods do +RSpec.describe Gitlab::Database::TransactionTimeoutSettings, feature_category: :cell do let(:connection) { ActiveRecord::Base.connection } subject { described_class.new(connection) } diff --git a/spec/lib/gitlab/database_spec.rb b/spec/lib/gitlab/database_spec.rb index 26d6ff431ec..f2be888e6eb 100644 --- a/spec/lib/gitlab/database_spec.rb +++ b/spec/lib/gitlab/database_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Database do +RSpec.describe Gitlab::Database, feature_category: :database do before do stub_const('MigrationTest', Class.new { include Gitlab::Database }) end @@ -66,6 +66,48 @@ RSpec.describe Gitlab::Database do end end + describe '.has_database?' do + context 'three tier database config' do + it 'returns true for main' do + expect(described_class.has_database?(:main)).to eq(true) + end + + it 'returns false for shared database' do + skip_if_multiple_databases_not_setup(:ci) + skip_if_database_exists(:ci) + + expect(described_class.has_database?(:ci)).to eq(false) + end + + it 'returns false for non-existent' do + expect(described_class.has_database?(:nonexistent)).to eq(false) + end + end + end + + describe '.database_mode' do + context 'three tier database config' do + it 'returns single-database if ci is not configured' do + skip_if_multiple_databases_are_setup(:ci) + + expect(described_class.database_mode).to eq(::Gitlab::Database::MODE_SINGLE_DATABASE) + end + + it 'returns single-database-ci-connection if ci is shared with main database' do + skip_if_multiple_databases_not_setup(:ci) + skip_if_database_exists(:ci) + + expect(described_class.database_mode).to eq(::Gitlab::Database::MODE_SINGLE_DATABASE_CI_CONNECTION) + end + + it 'returns multiple-database if ci has its own database' do + skip_if_shared_database(:ci) + + expect(described_class.database_mode).to eq(::Gitlab::Database::MODE_MULTIPLE_DATABASES) + end + end + end + describe '.check_for_non_superuser' do subject { described_class.check_for_non_superuser } diff --git a/spec/lib/gitlab/diff/highlight_cache_spec.rb b/spec/lib/gitlab/diff/highlight_cache_spec.rb index 33e9360ee01..43e4f28b4df 100644 --- a/spec/lib/gitlab/diff/highlight_cache_spec.rb +++ b/spec/lib/gitlab/diff/highlight_cache_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Diff::HighlightCache, :clean_gitlab_redis_cache do +RSpec.describe Gitlab::Diff::HighlightCache, :clean_gitlab_redis_cache, feature_category: :source_code_management do let_it_be(:merge_request) { create(:merge_request_with_diffs) } let(:diff_hash) do @@ -282,17 +282,7 @@ RSpec.describe Gitlab::Diff::HighlightCache, :clean_gitlab_redis_cache do end it 'returns cache key' do - is_expected.to eq("highlighted-diff-files:#{cache.diffable.cache_key}:2:#{options_hash([cache.diff_options, true, true])}") - end - - context 'when the `use_marker_ranges` feature flag is disabled' do - before do - stub_feature_flags(use_marker_ranges: false) - end - - it 'returns the original version of the cache' do - is_expected.to eq("highlighted-diff-files:#{cache.diffable.cache_key}:2:#{options_hash([cache.diff_options, false, true])}") - end + is_expected.to eq("highlighted-diff-files:#{cache.diffable.cache_key}:2:#{options_hash([cache.diff_options, true])}") end context 'when the `diff_line_syntax_highlighting` feature flag is disabled' do @@ -301,7 +291,7 @@ RSpec.describe Gitlab::Diff::HighlightCache, :clean_gitlab_redis_cache do end it 'returns the original version of the cache' do - is_expected.to eq("highlighted-diff-files:#{cache.diffable.cache_key}:2:#{options_hash([cache.diff_options, true, false])}") + is_expected.to eq("highlighted-diff-files:#{cache.diffable.cache_key}:2:#{options_hash([cache.diff_options, false])}") end end end diff --git a/spec/lib/gitlab/diff/highlight_spec.rb b/spec/lib/gitlab/diff/highlight_spec.rb index c378ecb8134..233dddbdad7 100644 --- a/spec/lib/gitlab/diff/highlight_spec.rb +++ b/spec/lib/gitlab/diff/highlight_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Diff::Highlight do +RSpec.describe Gitlab::Diff::Highlight, feature_category: :source_code_management do include RepoHelpers let_it_be(:project) { create(:project, :repository) } @@ -15,7 +15,6 @@ RSpec.describe Gitlab::Diff::Highlight do let(:code) { '<h2 onmouseover="alert(2)">Test</h2>' } before do - allow(Gitlab::Diff::InlineDiff).to receive(:for_lines).and_return([]) allow_any_instance_of(Gitlab::Diff::Line).to receive(:text).and_return(code) end @@ -121,18 +120,6 @@ RSpec.describe Gitlab::Diff::Highlight do end end - context 'when `use_marker_ranges` feature flag is disabled' do - it 'returns the same result' do - with_feature_flag = described_class.new(diff_file, repository: project.repository).highlight - - stub_feature_flags(use_marker_ranges: false) - - without_feature_flag = described_class.new(diff_file, repository: project.repository).highlight - - expect(with_feature_flag.map(&:rich_text)).to eq(without_feature_flag.map(&:rich_text)) - end - end - context 'when no inline diffs' do it_behaves_like 'without inline diffs' end diff --git a/spec/lib/gitlab/email/hook/silent_mode_interceptor_spec.rb b/spec/lib/gitlab/email/hook/silent_mode_interceptor_spec.rb new file mode 100644 index 00000000000..cc371643bee --- /dev/null +++ b/spec/lib/gitlab/email/hook/silent_mode_interceptor_spec.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Email::Hook::SilentModeInterceptor, :mailer, feature_category: :geo_replication do + let_it_be(:user) { create(:user) } + + before do + Mail.register_interceptor(described_class) + end + + after do + Mail.unregister_interceptor(described_class) + end + + context 'when silent mode is enabled' do + it 'prevents mail delivery' do + stub_application_setting(silent_mode_enabled: true) + + deliver_mails(user) + + should_not_email_anyone + end + + it 'logs the suppression' do + stub_application_setting(silent_mode_enabled: true) + + expect(Gitlab::AppJsonLogger).to receive(:info).with( + message: 'SilentModeInterceptor prevented sending mail', + mail_subject: 'Two-factor authentication disabled', + silent_mode_enabled: true + ) + expect(Gitlab::AppJsonLogger).to receive(:info).with( + message: 'SilentModeInterceptor prevented sending mail', + mail_subject: 'Welcome to GitLab!', + silent_mode_enabled: true + ) + + deliver_mails(user) + end + end + + context 'when silent mode is disabled' do + it 'does not prevent mail delivery' do + stub_application_setting(silent_mode_enabled: false) + + deliver_mails(user) + + should_email(user, times: 2) + end + + it 'debug logs the no-op' do + stub_application_setting(silent_mode_enabled: false) + + expect(Gitlab::AppJsonLogger).to receive(:debug).with( + message: 'SilentModeInterceptor did nothing', + mail_subject: 'Two-factor authentication disabled', + silent_mode_enabled: false + ) + expect(Gitlab::AppJsonLogger).to receive(:debug).with( + message: 'SilentModeInterceptor did nothing', + mail_subject: 'Welcome to GitLab!', + silent_mode_enabled: false + ) + + deliver_mails(user) + end + end + + def deliver_mails(user) + Notify.disabled_two_factor_email(user).deliver_now + DeviseMailer.user_admin_approval(user).deliver_now + end +end diff --git a/spec/lib/gitlab/incoming_email_spec.rb b/spec/lib/gitlab/email/incoming_email_spec.rb index acd6634058f..123b050aee7 100644 --- a/spec/lib/gitlab/incoming_email_spec.rb +++ b/spec/lib/gitlab/email/incoming_email_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::IncomingEmail do +RSpec.describe Gitlab::Email::IncomingEmail, feature_category: :service_desk do let(:setting_name) { :incoming_email } it_behaves_like 'common email methods' diff --git a/spec/lib/gitlab/email/receiver_spec.rb b/spec/lib/gitlab/email/receiver_spec.rb index 865e40d4ecb..e58da2478bf 100644 --- a/spec/lib/gitlab/email/receiver_spec.rb +++ b/spec/lib/gitlab/email/receiver_spec.rb @@ -11,9 +11,10 @@ RSpec.describe Gitlab::Email::Receiver do shared_examples 'successful receive' do let(:handler) { double(:handler, project: project, execute: true, metrics_event: nil, metrics_params: nil) } let(:client_id) { 'email/jake@example.com' } + let(:mail_key) { 'gitlabhq/gitlabhq+auth_token' } it 'correctly finds the mail key' do - expect(Gitlab::Email::Handler).to receive(:for).with(an_instance_of(Mail::Message), 'gitlabhq/gitlabhq+auth_token').and_return(handler) + expect(Gitlab::Email::Handler).to receive(:for).with(an_instance_of(Mail::Message), mail_key).and_return(handler) receiver.execute end @@ -92,6 +93,16 @@ RSpec.describe Gitlab::Email::Receiver do it_behaves_like 'successful receive' end + context 'when mail key is in the references header with a comma' do + let(:email_raw) { fixture_file('emails/valid_reply_with_references_in_comma.eml') } + let(:meta_key) { :references } + let(:meta_value) { ['"<reply-59d8df8370b7e95c5a49fbf86aeb2c93@localhost>,<issue_1@localhost>,<exchange@microsoft.com>"'] } + + it_behaves_like 'successful receive' do + let(:mail_key) { '59d8df8370b7e95c5a49fbf86aeb2c93' } + end + end + context 'when all other headers are missing' do let(:email_raw) { fixture_file('emails/missing_delivered_to_header.eml') } let(:meta_key) { :received_recipients } diff --git a/spec/lib/gitlab/email/reply_parser_spec.rb b/spec/lib/gitlab/email/reply_parser_spec.rb index e4c68dbba92..35065b74eff 100644 --- a/spec/lib/gitlab/email/reply_parser_spec.rb +++ b/spec/lib/gitlab/email/reply_parser_spec.rb @@ -3,7 +3,7 @@ require "spec_helper" # Inspired in great part by Discourse's Email::Receiver -RSpec.describe Gitlab::Email::ReplyParser do +RSpec.describe Gitlab::Email::ReplyParser, feature_category: :team_planning do describe '#execute' do def test_parse_body(mail_string, params = {}) described_class.new(Mail::Message.new(mail_string), **params).execute @@ -188,67 +188,36 @@ RSpec.describe Gitlab::Email::ReplyParser do ) end - context 'properly renders email reply from gmail web client' do - context 'when feature flag is enabled' do - it do - expect(test_parse_body(fixture_file("emails/html_only.eml"))) - .to eq( - <<-BODY.strip_heredoc.chomp - ### This is a reply from standard GMail in Google Chrome. - - The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. - - Here's some **bold** text, **strong** text and *italic* in Markdown. - - Here's a link http://example.com - - Here's an img <details> - <summary> - One</summary> - Some details</details> - - <details> - <summary> - Two</summary> - Some details</details> - - Test reply. - - First paragraph. - - Second paragraph. - BODY - ) - end - end - - context 'when feature flag is disabled' do - before do - stub_feature_flags(service_desk_html_to_text_email_handler: false) - end + context 'properly renders email reply from gmail web client', feature_category: :service_desk do + it do + expect(test_parse_body(fixture_file("emails/html_only.eml"))) + .to eq( + <<-BODY.strip_heredoc.chomp + ### This is a reply from standard GMail in Google Chrome. - it do - expect(test_parse_body(fixture_file("emails/html_only.eml"))) - .to eq( - <<-BODY.strip_heredoc.chomp - ### This is a reply from standard GMail in Google Chrome. + The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. - The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. + Here's some **bold** text, **strong** text and *italic* in Markdown. - Here's some **bold** text, strong text and italic in Markdown. + Here's a link http://example.com - Here's a link http://example.com + Here's an img <details> + <summary> + One</summary> + Some details</details> - Here's an img [Miro]One Some details Two Some details + <details> + <summary> + Two</summary> + Some details</details> - Test reply. + Test reply. - First paragraph. + First paragraph. - Second paragraph. - BODY - ) - end + Second paragraph. + BODY + ) end end diff --git a/spec/lib/gitlab/service_desk_email_spec.rb b/spec/lib/gitlab/email/service_desk_email_spec.rb index 69569c0f194..d59b8aa2cf7 100644 --- a/spec/lib/gitlab/service_desk_email_spec.rb +++ b/spec/lib/gitlab/email/service_desk_email_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::ServiceDeskEmail do +RSpec.describe Gitlab::Email::ServiceDeskEmail, feature_category: :service_desk do let(:setting_name) { :service_desk_email } it_behaves_like 'common email methods' diff --git a/spec/lib/gitlab/emoji_spec.rb b/spec/lib/gitlab/emoji_spec.rb index 0db3b5f3b11..44b2ec12246 100644 --- a/spec/lib/gitlab/emoji_spec.rb +++ b/spec/lib/gitlab/emoji_spec.rb @@ -3,23 +3,6 @@ require 'spec_helper' RSpec.describe Gitlab::Emoji do - describe '.emoji_image_tag' do - it 'returns emoji image tag' do - emoji_image = described_class.emoji_image_tag('emoji_one', 'src_url') - - expect(emoji_image).to eq("<img class=\"emoji\" src=\"src_url\" title=\":emoji_one:\" alt=\":emoji_one:\" height=\"20\" width=\"20\" align=\"absmiddle\" />") - end - - it 'escapes emoji image attrs to prevent XSS' do - xss_payload = "<script>alert(1)</script>" - escaped_xss_payload = html_escape(xss_payload) - - emoji_image = described_class.emoji_image_tag(xss_payload, 'http://aaa#' + xss_payload) - - expect(emoji_image).to eq("<img class=\"emoji\" src=\"http://aaa##{escaped_xss_payload}\" title=\":#{escaped_xss_payload}:\" alt=\":#{escaped_xss_payload}:\" height=\"20\" width=\"20\" align=\"absmiddle\" />") - end - end - describe '.gl_emoji_tag' do it 'returns gl emoji tag if emoji is found' do emoji = TanukiEmoji.find_by_alpha_code('small_airplane') diff --git a/spec/lib/gitlab/error_tracking_spec.rb b/spec/lib/gitlab/error_tracking_spec.rb index 0f056ee9eac..79016335a40 100644 --- a/spec/lib/gitlab/error_tracking_spec.rb +++ b/spec/lib/gitlab/error_tracking_spec.rb @@ -5,7 +5,7 @@ require 'spec_helper' require 'raven/transports/dummy' require 'sentry/transport/dummy_transport' -RSpec.describe Gitlab::ErrorTracking do +RSpec.describe Gitlab::ErrorTracking, feature_category: :shared do let(:exception) { RuntimeError.new('boom') } let(:issue_url) { 'http://gitlab.com/gitlab-org/gitlab-foss/issues/1' } let(:extra) { { issue_url: issue_url, some_other_info: 'info' } } @@ -58,7 +58,7 @@ RSpec.describe Gitlab::ErrorTracking do stub_feature_flags(enable_new_sentry_integration: true) stub_sentry_settings - allow(described_class).to receive(:sentry_configurable?) { true } + allow(described_class).to receive(:sentry_configurable?).and_return(true) allow(Labkit::Correlation::CorrelationId).to receive(:current_id).and_return('cid') allow(I18n).to receive(:locale).and_return('en') @@ -82,7 +82,7 @@ RSpec.describe Gitlab::ErrorTracking do describe '.track_and_raise_for_dev_exception' do context 'when exceptions for dev should be raised' do before do - expect(described_class).to receive(:should_raise_for_dev?).and_return(true) + allow(described_class).to receive(:should_raise_for_dev?).and_return(true) end it 'raises the exception' do @@ -101,7 +101,7 @@ RSpec.describe Gitlab::ErrorTracking do context 'when exceptions for dev should not be raised' do before do - expect(described_class).to receive(:should_raise_for_dev?).and_return(false) + allow(described_class).to receive(:should_raise_for_dev?).and_return(false) end it 'logs the exception with all attributes passed' do @@ -219,7 +219,7 @@ RSpec.describe Gitlab::ErrorTracking do end end - context 'the exception implements :sentry_extra_data' do + context 'when the exception implements :sentry_extra_data' do let(:extra_info) { { event: 'explosion', size: :massive } } before do @@ -239,7 +239,7 @@ RSpec.describe Gitlab::ErrorTracking do end end - context 'the exception implements :sentry_extra_data, which returns nil' do + context 'when the exception implements :sentry_extra_data, which returns nil' do let(:extra) { { issue_url: issue_url } } before do @@ -260,7 +260,7 @@ RSpec.describe Gitlab::ErrorTracking do end end - context 'event processors' do + describe 'event processors' do subject(:track_exception) { described_class.track_exception(exception, extra) } before do @@ -269,7 +269,16 @@ RSpec.describe Gitlab::ErrorTracking do allow(Gitlab::ErrorTracking::Logger).to receive(:error) end - context 'custom GitLab context when using Raven.capture_exception directly' do + # This is a workaround for restoring Raven's user context below. + # Raven.user_context(&block) does not restore the user context correctly. + around do |example| + previous_user_context = Raven.context.user.dup + example.run + ensure + Raven.context.user = previous_user_context + end + + context 'with custom GitLab context when using Raven.capture_exception directly' do subject(:track_exception) { Raven.capture_exception(exception) } it 'merges a default set of tags into the existing tags' do @@ -289,7 +298,7 @@ RSpec.describe Gitlab::ErrorTracking do end end - context 'custom GitLab context when using Sentry.capture_exception directly' do + context 'with custom GitLab context when using Sentry.capture_exception directly' do subject(:track_exception) { Sentry.capture_exception(exception) } it 'merges a default set of tags into the existing tags' do @@ -401,15 +410,17 @@ RSpec.describe Gitlab::ErrorTracking do end ['Gitlab::SidekiqMiddleware::RetryError', 'SubclassRetryError'].each do |ex| - let(:exception) { ex.constantize.new } + context "with #{ex} exception" do + let(:exception) { ex.constantize.new } - it "does not report #{ex} exception to Sentry" do - expect(Gitlab::ErrorTracking::Logger).to receive(:error) + it "does not report exception to Sentry" do + expect(Gitlab::ErrorTracking::Logger).to receive(:error) - track_exception + track_exception - expect(Raven.client.transport.events).to eq([]) - expect(Sentry.get_current_client.transport.events).to eq([]) + expect(Raven.client.transport.events).to eq([]) + expect(Sentry.get_current_client.transport.events).to eq([]) + end end end end @@ -491,7 +502,7 @@ RSpec.describe Gitlab::ErrorTracking do end end - context 'Sentry performance monitoring' do + describe 'Sentry performance monitoring' do context 'when ENABLE_SENTRY_PERFORMANCE_MONITORING env is disabled' do before do stub_env('ENABLE_SENTRY_PERFORMANCE_MONITORING', false) diff --git a/spec/lib/gitlab/favicon_spec.rb b/spec/lib/gitlab/favicon_spec.rb index 884425dab3b..033fa5d1b42 100644 --- a/spec/lib/gitlab/favicon_spec.rb +++ b/spec/lib/gitlab/favicon_spec.rb @@ -40,14 +40,22 @@ RSpec.describe Gitlab::Favicon, :request_store do end end - describe '.status_overlay' do - subject { described_class.status_overlay('favicon_status_created') } + describe '.ci_status_overlay' do + subject { described_class.ci_status_overlay('favicon_status_created') } it 'returns the overlay for the status' do expect(subject).to match_asset_path '/assets/ci_favicons/favicon_status_created.png' end end + describe '.mr_status_overlay' do + subject { described_class.mr_status_overlay('favicon_status_merged') } + + it 'returns the overlay for the status' do + expect(subject).to match_asset_path '/assets/mr_favicons/favicon_status_merged.png' + end + end + describe '.available_status_names' do subject { described_class.available_status_names } diff --git a/spec/lib/gitlab/git/blame_mode_spec.rb b/spec/lib/gitlab/git/blame_mode_spec.rb new file mode 100644 index 00000000000..1fc6f12c552 --- /dev/null +++ b/spec/lib/gitlab/git/blame_mode_spec.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Git::BlameMode, feature_category: :source_code_management do + subject(:blame_mode) { described_class.new(project, params) } + + let_it_be(:project) { build(:project) } + let(:params) { {} } + + describe '#streaming_supported?' do + subject { blame_mode.streaming_supported? } + + it { is_expected.to be_truthy } + + context 'when `blame_page_streaming` is disabled' do + before do + stub_feature_flags(blame_page_streaming: false) + end + + it { is_expected.to be_falsey } + end + end + + describe '#streaming?' do + subject { blame_mode.streaming? } + + it { is_expected.to be_falsey } + + context 'when streaming param is provided' do + let(:params) { { streaming: true } } + + it { is_expected.to be_truthy } + + context 'when `blame_page_streaming` is disabled' do + before do + stub_feature_flags(blame_page_streaming: false) + end + + it { is_expected.to be_falsey } + end + end + end + + describe '#pagination?' do + subject { blame_mode.pagination? } + + it { is_expected.to be_truthy } + + context 'when `streaming` params is enabled' do + let(:params) { { streaming: true } } + + it { is_expected.to be_falsey } + end + + context 'when `no_pagination` param is provided' do + let(:params) { { no_pagination: true } } + + it { is_expected.to be_falsey } + end + + context 'when `blame_page_pagination` is disabled' do + before do + stub_feature_flags(blame_page_pagination: false) + end + + it { is_expected.to be_falsey } + end + end + + describe '#full?' do + subject { blame_mode.full? } + + it { is_expected.to be_falsey } + + context 'when `blame_page_pagination` is disabled' do + before do + stub_feature_flags(blame_page_pagination: false) + end + + it { is_expected.to be_truthy } + end + end +end diff --git a/spec/lib/gitlab/git/blame_pagination_spec.rb b/spec/lib/gitlab/git/blame_pagination_spec.rb new file mode 100644 index 00000000000..1f3c0c0342e --- /dev/null +++ b/spec/lib/gitlab/git/blame_pagination_spec.rb @@ -0,0 +1,175 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Git::BlamePagination, feature_category: :source_code_management do + subject(:blame_pagination) { described_class.new(blob, blame_mode, params) } + + let_it_be(:project) { create(:project, :repository) } + let_it_be(:commit) { project.repository.commit } + let_it_be(:blob) { project.repository.blob_at('HEAD', 'README.md') } + + let(:blame_mode) do + instance_double( + 'Gitlab::Git::BlameMode', + 'streaming?' => streaming_mode, + 'full?' => full_mode + ) + end + + let(:params) { { page: page } } + let(:page) { 1 } + let(:streaming_mode) { false } + let(:full_mode) { false } + + using RSpec::Parameterized::TableSyntax + + describe '#page' do + subject { blame_pagination.page } + + where(:page, :expected_page) do + nil | 1 + 1 | 1 + 5 | 5 + -1 | 1 + 'a' | 1 + end + + with_them do + it { is_expected.to eq(expected_page) } + end + end + + describe '#per_page' do + subject { blame_pagination.per_page } + + it { is_expected.to eq(described_class::PAGINATION_PER_PAGE) } + + context 'when blame mode is streaming' do + let(:streaming_mode) { true } + + it { is_expected.to eq(described_class::STREAMING_PER_PAGE) } + end + end + + describe '#total_pages' do + subject { blame_pagination.total_pages } + + before do + stub_const("#{described_class.name}::PAGINATION_PER_PAGE", 2) + end + + it { is_expected.to eq(2) } + end + + describe '#total_extra_pages' do + subject { blame_pagination.total_extra_pages } + + before do + stub_const("#{described_class.name}::PAGINATION_PER_PAGE", 2) + end + + it { is_expected.to eq(1) } + end + + describe '#pagination' do + subject { blame_pagination.paginator } + + before do + stub_const("#{described_class.name}::PAGINATION_PER_PAGE", 2) + end + + it 'returns a pagination object' do + is_expected.to be_kind_of(Kaminari::PaginatableArray) + + expect(subject.current_page).to eq(1) + expect(subject.total_pages).to eq(2) + expect(subject.total_count).to eq(4) + end + + context 'when user disabled the pagination' do + let(:full_mode) { true } + + it { is_expected.to be_nil } + end + + context 'when user chose streaming' do + let(:streaming_mode) { true } + + it { is_expected.to be_nil } + end + + context 'when per_page is above the global max per page limit' do + before do + stub_const("#{described_class.name}::PAGINATION_PER_PAGE", 1000) + allow(blob).to receive_message_chain(:data, :lines, :count) { 500 } + end + + it 'returns a correct pagination object' do + is_expected.to be_kind_of(Kaminari::PaginatableArray) + + expect(subject.current_page).to eq(1) + expect(subject.total_pages).to eq(1) + expect(subject.total_count).to eq(500) + end + end + + describe 'Pagination attributes' do + where(:page, :current_page, :total_pages) do + 1 | 1 | 2 + 2 | 2 | 2 + 0 | 1 | 2 # Incorrect + end + + with_them do + it 'returns the correct pagination attributes' do + expect(subject.current_page).to eq(current_page) + expect(subject.total_pages).to eq(total_pages) + end + end + end + end + + describe '#blame_range' do + subject { blame_pagination.blame_range } + + before do + stub_const("#{described_class.name}::PAGINATION_PER_PAGE", 2) + end + + where(:page, :expected_range) do + 1 | (1..2) + 2 | (3..4) + 0 | (1..2) + end + + with_them do + it { is_expected.to eq(expected_range) } + end + + context 'when user disabled the pagination' do + let(:full_mode) { true } + + it { is_expected.to be_nil } + end + + context 'when streaming is enabled' do + let(:streaming_mode) { true } + + before do + stub_const("#{described_class.name}::STREAMING_FIRST_PAGE_SIZE", 1) + stub_const("#{described_class.name}::STREAMING_PER_PAGE", 1) + end + + where(:page, :expected_range) do + 1 | (1..1) + 2 | (2..2) + 0 | (1..1) + end + + with_them do + it { is_expected.to eq(expected_range) } + end + end + end +end diff --git a/spec/lib/gitlab/git/repository_spec.rb b/spec/lib/gitlab/git/repository_spec.rb index 15bce16bd7f..e78e01ae129 100644 --- a/spec/lib/gitlab/git/repository_spec.rb +++ b/spec/lib/gitlab/git/repository_spec.rb @@ -2452,107 +2452,6 @@ RSpec.describe Gitlab::Git::Repository, feature_category: :source_code_managemen end end - describe '#squash' do - let(:branch_name) { 'fix' } - let(:start_sha) { TestEnv::BRANCH_SHA['master'] } - let(:end_sha) { '12d65c8dd2b2676fa3ac47d955accc085a37a9c1' } - - subject do - opts = { - branch: branch_name, - start_sha: start_sha, - end_sha: end_sha, - author: user, - message: 'Squash commit message' - } - - repository.squash(user, opts) - end - - # Should be ported to gitaly-ruby rspec suite https://gitlab.com/gitlab-org/gitaly/issues/1234 - skip 'sparse checkout' do - let(:expected_files) { %w(files files/js files/js/application.js) } - - it 'checks out only the files in the diff' do - allow(repository).to receive(:with_worktree).and_wrap_original do |m, *args| - m.call(*args) do - worktree_path = args[0] - files_pattern = File.join(worktree_path, '**', '*') - expected = expected_files.map do |path| - File.expand_path(path, worktree_path) - end - - expect(Dir[files_pattern]).to eq(expected) - end - end - - subject - end - - context 'when the diff contains a rename' do - let(:end_sha) do - repository.commit_files( - user, - branch_name: repository.root_ref, - message: 'Move CHANGELOG to encoding/', - actions: [{ - action: :move, - previous_path: 'CHANGELOG', - file_path: 'encoding/CHANGELOG', - content: 'CHANGELOG' - }] - ).newrev - end - - after do - # Erase our commits so other tests get the original repo - repository.write_ref(repository.root_ref, TestEnv::BRANCH_SHA['master']) - end - - it 'does not include the renamed file in the sparse checkout' do - allow(repository).to receive(:with_worktree).and_wrap_original do |m, *args| - m.call(*args) do - worktree_path = args[0] - files_pattern = File.join(worktree_path, '**', '*') - - expect(Dir[files_pattern]).not_to include('CHANGELOG') - expect(Dir[files_pattern]).not_to include('encoding/CHANGELOG') - end - end - - subject - end - end - end - - # Should be ported to gitaly-ruby rspec suite https://gitlab.com/gitlab-org/gitaly/issues/1234 - skip 'with an ASCII-8BIT diff' do - let(:diff) { "diff --git a/README.md b/README.md\nindex faaf198..43c5edf 100644\n--- a/README.md\n+++ b/README.md\n@@ -1,4 +1,4 @@\n-testme\n+✓ testme\n ======\n \n Sample repo for testing gitlab features\n" } - - it 'applies a ASCII-8BIT diff' do - allow(repository).to receive(:run_git!).and_call_original - allow(repository).to receive(:run_git!).with(%W(diff --binary #{start_sha}...#{end_sha})).and_return(diff.force_encoding('ASCII-8BIT')) - - expect(subject).to match(/\h{40}/) - end - end - - # Should be ported to gitaly-ruby rspec suite https://gitlab.com/gitlab-org/gitaly/issues/1234 - skip 'with trailing whitespace in an invalid patch' do - let(:diff) { "diff --git a/README.md b/README.md\nindex faaf198..43c5edf 100644\n--- a/README.md\n+++ b/README.md\n@@ -1,4 +1,4 @@\n-testme\n+ \n ====== \n \n Sample repo for testing gitlab features\n" } - - it 'does not include whitespace warnings in the error' do - allow(repository).to receive(:run_git!).and_call_original - allow(repository).to receive(:run_git!).with(%W(diff --binary #{start_sha}...#{end_sha})).and_return(diff.force_encoding('ASCII-8BIT')) - - expect { subject }.to raise_error do |error| - expect(error).to be_a(described_class::GitError) - expect(error.message).not_to include('trailing whitespace') - end - end - end - end - def create_remote_branch(remote_name, branch_name, source_branch_name) source_branch = repository.find_branch(source_branch_name) repository.write_ref("refs/remotes/#{remote_name}/#{branch_name}", source_branch.dereferenced_target.sha) diff --git a/spec/lib/gitlab/git_ref_validator_spec.rb b/spec/lib/gitlab/git_ref_validator_spec.rb index 03dd4e7b89b..1a79817130c 100644 --- a/spec/lib/gitlab/git_ref_validator_spec.rb +++ b/spec/lib/gitlab/git_ref_validator_spec.rb @@ -37,6 +37,11 @@ RSpec.describe Gitlab::GitRefValidator do it { expect(described_class.validate("\xA0\u0000\xB0")).to be false } it { expect(described_class.validate("")).to be false } it { expect(described_class.validate(nil)).to be false } + it { expect(described_class.validate('HEAD')).to be false } + + context 'when skip_head_ref_check is true' do + it { expect(described_class.validate('HEAD', skip_head_ref_check: true)).to be true } + end end describe '.validate_merge_request_branch' do diff --git a/spec/lib/gitlab/github_import/bulk_importing_spec.rb b/spec/lib/gitlab/github_import/bulk_importing_spec.rb index 136ddb566aa..28fbd4d883f 100644 --- a/spec/lib/gitlab/github_import/bulk_importing_spec.rb +++ b/spec/lib/gitlab/github_import/bulk_importing_spec.rb @@ -13,6 +13,8 @@ RSpec.describe Gitlab::GithubImport::BulkImporting, feature_category: :importers :object_type end + private + def model Label end @@ -26,85 +28,153 @@ RSpec.describe Gitlab::GithubImport::BulkImporting, feature_category: :importers end describe '#build_database_rows' do - it 'returns an Array containing the rows to insert and validation errors if object invalid' do - object = double(:object, title: 'Foo') - - expect(importer) - .to receive(:build_attributes) - .with(object) - .and_return({ title: 'Foo' }) - - expect(Label) - .to receive(:new) - .with({ title: 'Foo' }) - .and_return(label) - - expect(importer) - .to receive(:already_imported?) - .with(object) - .and_return(false) - - expect(Gitlab::Import::Logger) - .to receive(:info) - .with( - import_type: :github, - project_id: 1, - importer: 'MyImporter', - message: '1 object_types fetched' - ) - - expect(Gitlab::GithubImport::ObjectCounter) - .to receive(:increment) - .with( - project, - :object_type, - :fetched, - value: 1 - ) - - enum = [[object, 1]].to_enum - - rows, errors = importer.build_database_rows(enum) + context 'without validation errors' do + let(:object) { double(:object, title: 'Foo') } + + it 'returns an array containing the rows to insert' do + expect(importer) + .to receive(:build_attributes) + .with(object) + .and_return({ title: 'Foo' }) + + expect(Label) + .to receive(:new) + .with({ title: 'Foo' }) + .and_return(label) + + expect(importer) + .to receive(:already_imported?) + .with(object) + .and_return(false) + + expect(Gitlab::Import::Logger) + .to receive(:info) + .with( + import_type: :github, + project_id: 1, + importer: 'MyImporter', + message: '1 object_types fetched' + ) + + expect(Gitlab::GithubImport::ObjectCounter) + .to receive(:increment) + .with( + project, + :object_type, + :fetched, + value: 1 + ) + + enum = [[object, 1]].to_enum + + rows, errors = importer.build_database_rows(enum) + + expect(rows).to match_array([{ title: 'Foo' }]) + expect(errors).to be_empty + end - expect(rows).to match_array([{ title: 'Foo' }]) - expect(errors).to be_empty + it 'does not import objects that have already been imported' do + expect(importer) + .not_to receive(:build_attributes) + + expect(importer) + .to receive(:already_imported?) + .with(object) + .and_return(true) + + expect(Gitlab::Import::Logger) + .to receive(:info) + .with( + import_type: :github, + project_id: 1, + importer: 'MyImporter', + message: '0 object_types fetched' + ) + + expect(Gitlab::GithubImport::ObjectCounter) + .to receive(:increment) + .with( + project, + :object_type, + :fetched, + value: 0 + ) + + enum = [[object, 1]].to_enum + + rows, errors = importer.build_database_rows(enum) + + expect(rows).to be_empty + expect(errors).to be_empty + end end - it 'does not import objects that have already been imported' do - object = double(:object, title: 'Foo') - - expect(importer) - .not_to receive(:build_attributes) + context 'with validation errors' do + let(:object) { double(:object, id: 12345, title: 'bug,bug') } - expect(importer) - .to receive(:already_imported?) - .with(object) - .and_return(true) + before do + allow(importer) + .to receive(:already_imported?) + .with(object) + .and_return(false) - expect(Gitlab::Import::Logger) - .to receive(:info) - .with( - import_type: :github, - project_id: 1, - importer: 'MyImporter', - message: '0 object_types fetched' - ) - - expect(Gitlab::GithubImport::ObjectCounter) - .to receive(:increment) - .with( - project, - :object_type, - :fetched, - value: 0 - ) + allow(importer) + .to receive(:build_attributes) + .with(object) + .and_return({ title: 'bug,bug' }) + end - enum = [[object, 1]].to_enum + context 'without implemented github_identifiers method' do + it 'raises NotImplementedError' do + enum = [[object, 1]].to_enum - rows, errors = importer.build_database_rows(enum) + expect { importer.build_database_rows(enum) }.to raise_error(NotImplementedError) + end + end - expect(rows).to be_empty - expect(errors).to be_empty + context 'with implemented github_identifiers method' do + it 'returns an array containing the validation errors and logs them' do + expect(importer) + .to receive(:github_identifiers) + .with(object) + .and_return( + { + id: object.id, + title: object.title, + object_type: importer.object_type + } + ) + + expect(Gitlab::Import::Logger) + .to receive(:error) + .with( + import_type: :github, + project_id: 1, + importer: 'MyImporter', + message: ['Title is invalid'], + github_identifiers: { id: 12345, title: 'bug,bug', object_type: :object_type } + ) + + expect(Gitlab::GithubImport::ObjectCounter) + .to receive(:increment) + .with( + project, + :object_type, + :fetched, + value: 0 + ) + + enum = [[object, 1]].to_enum + + rows, errors = importer.build_database_rows(enum) + + expect(rows).to be_empty + expect(errors).not_to be_empty + + expect(errors[0][:validation_errors].full_messages).to match_array(['Title is invalid']) + expect(errors[0][:github_identifiers]).to eq({ id: 12345, title: 'bug,bug', object_type: :object_type }) + end + end end end @@ -157,7 +227,8 @@ RSpec.describe Gitlab::GithubImport::BulkImporting, feature_category: :importers exception_message: 'Title invalid', correlation_id_value: 'cid', retry_count: nil, - created_at: Time.zone.now + created_at: Time.zone.now, + external_identifiers: { id: 123456 } }] end @@ -170,8 +241,23 @@ RSpec.describe Gitlab::GithubImport::BulkImporting, feature_category: :importers expect(import_failures).to receive(:insert_all).with(formatted_errors) expect(Labkit::Correlation::CorrelationId).to receive(:current_or_new_id).and_return('cid') - importer.bulk_insert_failures([error]) + importer.bulk_insert_failures([{ + validation_errors: error, + github_identifiers: { id: 123456 } + }]) end end end + + describe '#object_type' do + let(:importer_class) do + Class.new do + include Gitlab::GithubImport::BulkImporting + end + end + + it 'raises NotImplementedError' do + expect { importer.object_type }.to raise_error(NotImplementedError) + end + end end diff --git a/spec/lib/gitlab/github_import/importer/attachments/issues_importer_spec.rb b/spec/lib/gitlab/github_import/importer/attachments/issues_importer_spec.rb index 85bc67376d3..7890561bf2d 100644 --- a/spec/lib/gitlab/github_import/importer/attachments/issues_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/attachments/issues_importer_spec.rb @@ -17,6 +17,8 @@ RSpec.describe Gitlab::GithubImport::Importer::Attachments::IssuesImporter do let(:importer_attrs) { [instance_of(Gitlab::GithubImport::Representation::NoteText), project, client] } it 'imports each project issue attachments' do + expect(project.issues).to receive(:select).with(:id, :description, :iid).and_call_original + expect_next_instances_of( Gitlab::GithubImport::Importer::NoteAttachmentsImporter, 2, false, *importer_attrs ) do |note_attachments_importer| diff --git a/spec/lib/gitlab/github_import/importer/attachments/merge_requests_importer_spec.rb b/spec/lib/gitlab/github_import/importer/attachments/merge_requests_importer_spec.rb index e4718c2d17c..e5aa17dd81e 100644 --- a/spec/lib/gitlab/github_import/importer/attachments/merge_requests_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/attachments/merge_requests_importer_spec.rb @@ -17,6 +17,8 @@ RSpec.describe Gitlab::GithubImport::Importer::Attachments::MergeRequestsImporte let(:importer_attrs) { [instance_of(Gitlab::GithubImport::Representation::NoteText), project, client] } it 'imports each project merge request attachments' do + expect(project.merge_requests).to receive(:select).with(:id, :description, :iid).and_call_original + expect_next_instances_of( Gitlab::GithubImport::Importer::NoteAttachmentsImporter, 2, false, *importer_attrs ) do |note_attachments_importer| diff --git a/spec/lib/gitlab/github_import/importer/attachments/releases_importer_spec.rb b/spec/lib/gitlab/github_import/importer/attachments/releases_importer_spec.rb index b989345ae09..e1b009c3eeb 100644 --- a/spec/lib/gitlab/github_import/importer/attachments/releases_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/attachments/releases_importer_spec.rb @@ -17,6 +17,8 @@ RSpec.describe Gitlab::GithubImport::Importer::Attachments::ReleasesImporter do let(:importer_attrs) { [instance_of(Gitlab::GithubImport::Representation::NoteText), project, client] } it 'imports each project release' do + expect(project.releases).to receive(:select).with(:id, :description, :tag).and_call_original + expect(Gitlab::GithubImport::Importer::NoteAttachmentsImporter).to receive(:new) .with(*importer_attrs).twice.and_return(importer_stub) expect(importer_stub).to receive(:execute).twice diff --git a/spec/lib/gitlab/github_import/importer/labels_importer_spec.rb b/spec/lib/gitlab/github_import/importer/labels_importer_spec.rb index 9e295ab215a..fc8d9cee066 100644 --- a/spec/lib/gitlab/github_import/importer/labels_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/labels_importer_spec.rb @@ -56,14 +56,14 @@ feature_category: :importers do project_id: project.id, importer: described_class.name, message: ['Title is invalid'], - github_identifier: 1 + github_identifiers: { title: 'bug,bug', object_type: :label } ) rows, errors = importer.build_labels expect(rows).to be_empty expect(errors.length).to eq(1) - expect(errors[0].full_messages).to match_array(['Title is invalid']) + expect(errors[0][:validation_errors].full_messages).to match_array(['Title is invalid']) end end diff --git a/spec/lib/gitlab/github_import/importer/milestones_importer_spec.rb b/spec/lib/gitlab/github_import/importer/milestones_importer_spec.rb index 47b9a41c364..cf44d510c80 100644 --- a/spec/lib/gitlab/github_import/importer/milestones_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/milestones_importer_spec.rb @@ -74,7 +74,7 @@ RSpec.describe Gitlab::GithubImport::Importer::MilestonesImporter, :clean_gitlab end it 'does not build milestones that are invalid' do - milestone = { id: 1, title: nil } + milestone = { id: 123456, title: nil, number: 2 } expect(importer) .to receive(:each_milestone) @@ -86,14 +86,14 @@ RSpec.describe Gitlab::GithubImport::Importer::MilestonesImporter, :clean_gitlab project_id: project.id, importer: described_class.name, message: ["Title can't be blank"], - github_identifier: 1 + github_identifiers: { iid: 2, object_type: :milestone, title: nil } ) rows, errors = importer.build_milestones expect(rows).to be_empty expect(errors.length).to eq(1) - expect(errors[0].full_messages).to match_array(["Title can't be blank"]) + expect(errors[0][:validation_errors].full_messages).to match_array(["Title can't be blank"]) end end diff --git a/spec/lib/gitlab/github_import/importer/pull_requests/review_requests_importer_spec.rb b/spec/lib/gitlab/github_import/importer/pull_requests/review_requests_importer_spec.rb index 536983fea06..9e9d6c6e9cd 100644 --- a/spec/lib/gitlab/github_import/importer/pull_requests/review_requests_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/pull_requests/review_requests_importer_spec.rb @@ -86,6 +86,7 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequests::ReviewRequestsImpor project.id, { merge_request_id: merge_request_1.id, + merge_request_iid: merge_request_1.iid, users: [ { id: 4, login: 'alice' }, { id: 5, login: 'bob' } @@ -97,6 +98,7 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequests::ReviewRequestsImpor project.id, { merge_request_id: merge_request_2.id, + merge_request_iid: merge_request_2.iid, users: [ { id: 4, login: 'alice' } ] diff --git a/spec/lib/gitlab/github_import/importer/pull_requests_reviews_importer_spec.rb b/spec/lib/gitlab/github_import/importer/pull_requests_reviews_importer_spec.rb index 5f9c73cbfff..92f7d906f61 100644 --- a/spec/lib/gitlab/github_import/importer/pull_requests_reviews_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/pull_requests_reviews_importer_spec.rb @@ -53,6 +53,7 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestsReviewsImporter do subject.each_object_to_import {} expect(review[:merge_request_id]).to eq(merge_request.id) + expect(review[:merge_request_iid]).to eq(merge_request.iid) end it 'skips cached pages' do diff --git a/spec/lib/gitlab/github_import/importer/releases_importer_spec.rb b/spec/lib/gitlab/github_import/importer/releases_importer_spec.rb index fe4d3e9d90b..a3d20af22c7 100644 --- a/spec/lib/gitlab/github_import/importer/releases_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/releases_importer_spec.rb @@ -18,6 +18,7 @@ RSpec.describe Gitlab::GithubImport::Importer::ReleasesImporter, feature_categor let(:github_release) do { + id: 123456, tag_name: '1.0', name: github_release_name, body: 'This is my release', @@ -144,7 +145,10 @@ RSpec.describe Gitlab::GithubImport::Importer::ReleasesImporter, feature_categor expect(releases).to be_empty expect(errors.length).to eq(1) - expect(errors[0].full_messages).to match_array(['Description is too long (maximum is 1000000 characters)']) + expect(errors[0][:validation_errors].full_messages).to match_array( + ['Description is too long (maximum is 1000000 characters)'] + ) + expect(errors[0][:github_identifiers]).to eq({ tag: '1.0', object_type: :release }) end end diff --git a/spec/lib/gitlab/github_import/representation/collaborator_spec.rb b/spec/lib/gitlab/github_import/representation/collaborator_spec.rb index d5952f9459b..cc52c34ec74 100644 --- a/spec/lib/gitlab/github_import/representation/collaborator_spec.rb +++ b/spec/lib/gitlab/github_import/representation/collaborator_spec.rb @@ -20,6 +20,17 @@ RSpec.describe Gitlab::GithubImport::Representation::Collaborator, feature_categ it 'includes the role' do expect(collaborator.role_name).to eq('maintainer') end + + describe '#github_identifiers' do + it 'returns a hash with needed identifiers' do + expect(collaborator.github_identifiers).to eq( + { + id: 42, + login: 'alice' + } + ) + end + end end end diff --git a/spec/lib/gitlab/github_import/representation/issue_event_spec.rb b/spec/lib/gitlab/github_import/representation/issue_event_spec.rb index 0dd281cb3b0..33f0c6d3c64 100644 --- a/spec/lib/gitlab/github_import/representation/issue_event_spec.rb +++ b/spec/lib/gitlab/github_import/representation/issue_event_spec.rb @@ -156,7 +156,11 @@ RSpec.describe Gitlab::GithubImport::Representation::IssueEvent do describe '#github_identifiers' do it 'returns a hash with needed identifiers' do - expect(issue_event.github_identifiers).to eq({ id: 6501124486 }) + expect(issue_event.github_identifiers).to eq( + id: 6501124486, + iid: 2, + event: 'closed' + ) end end end diff --git a/spec/lib/gitlab/github_import/representation/issue_spec.rb b/spec/lib/gitlab/github_import/representation/issue_spec.rb index 263ef8b1708..39447da0fac 100644 --- a/spec/lib/gitlab/github_import/representation/issue_spec.rb +++ b/spec/lib/gitlab/github_import/representation/issue_spec.rb @@ -192,7 +192,8 @@ RSpec.describe Gitlab::GithubImport::Representation::Issue do it 'returns a hash with needed identifiers' do github_identifiers = { iid: 42, - issuable_type: 'MergeRequest' + issuable_type: 'MergeRequest', + title: 'Implement cool feature' } other_attributes = { pull_request: true, something_else: '_something_else_' } issue = described_class.new(github_identifiers.merge(other_attributes)) diff --git a/spec/lib/gitlab/github_import/representation/lfs_object_spec.rb b/spec/lib/gitlab/github_import/representation/lfs_object_spec.rb index 6663a7366a5..799a77afb0c 100644 --- a/spec/lib/gitlab/github_import/representation/lfs_object_spec.rb +++ b/spec/lib/gitlab/github_import/representation/lfs_object_spec.rb @@ -6,7 +6,8 @@ RSpec.describe Gitlab::GithubImport::Representation::LfsObject do describe '#github_identifiers' do it 'returns a hash with needed identifiers' do github_identifiers = { - oid: 42 + oid: 42, + size: 123456 } other_attributes = { something_else: '_something_else_' } lfs_object = described_class.new(github_identifiers.merge(other_attributes)) diff --git a/spec/lib/gitlab/github_import/representation/note_text_spec.rb b/spec/lib/gitlab/github_import/representation/note_text_spec.rb index 8b57c9a0373..7aa458a1c33 100644 --- a/spec/lib/gitlab/github_import/representation/note_text_spec.rb +++ b/spec/lib/gitlab/github_import/representation/note_text_spec.rb @@ -22,35 +22,45 @@ RSpec.describe Gitlab::GithubImport::Representation::NoteText do end describe '.from_db_record' do + let(:representation) { described_class.from_db_record(record) } + context 'with Release' do - let(:record) { build_stubbed(:release, id: 42, description: 'Some text here..') } + let(:record) { build_stubbed(:release, id: 42, description: 'Some text here..', tag: 'v1.0') } + + it_behaves_like 'a Note text data', 'Release' - it_behaves_like 'a Note text data', 'Release' do - let(:representation) { described_class.from_db_record(record) } + it 'includes tag' do + expect(representation.tag).to eq 'v1.0' end end context 'with Issue' do - let(:record) { build_stubbed(:issue, id: 42, description: 'Some text here..') } + let(:record) { build_stubbed(:issue, id: 42, iid: 2, description: 'Some text here..') } + + it_behaves_like 'a Note text data', 'Issue' - it_behaves_like 'a Note text data', 'Issue' do - let(:representation) { described_class.from_db_record(record) } + it 'includes noteable iid' do + expect(representation.iid).to eq 2 end end context 'with MergeRequest' do - let(:record) { build_stubbed(:merge_request, id: 42, description: 'Some text here..') } + let(:record) { build_stubbed(:merge_request, id: 42, iid: 2, description: 'Some text here..') } - it_behaves_like 'a Note text data', 'MergeRequest' do - let(:representation) { described_class.from_db_record(record) } + it_behaves_like 'a Note text data', 'MergeRequest' + + it 'includes noteable iid' do + expect(representation.iid).to eq 2 end end context 'with Note' do - let(:record) { build_stubbed(:note, id: 42, note: 'Some text here..') } + let(:record) { build_stubbed(:note, id: 42, note: 'Some text here..', noteable_type: 'Issue') } + + it_behaves_like 'a Note text data', 'Note' - it_behaves_like 'a Note text data', 'Note' do - let(:representation) { described_class.from_db_record(record) } + it 'includes noteable type' do + expect(representation.noteable_type).to eq 'Issue' end end end @@ -61,7 +71,8 @@ RSpec.describe Gitlab::GithubImport::Representation::NoteText do { 'record_db_id' => 42, 'record_type' => 'Release', - 'text' => 'Some text here..' + 'text' => 'Some text here..', + 'tag' => 'v1.0' } end @@ -70,11 +81,76 @@ RSpec.describe Gitlab::GithubImport::Representation::NoteText do end describe '#github_identifiers' do - it 'returns a hash with needed identifiers' do - record_id = rand(100) - representation = described_class.new(record_db_id: record_id, text: 'text') + let(:iid) { nil } + let(:tag) { nil } + let(:noteable_type) { nil } + let(:hash) do + { + 'record_db_id' => 42, + 'record_type' => record_type, + 'text' => 'Some text here..', + 'iid' => iid, + 'tag' => tag, + 'noteable_type' => noteable_type + } + end + + subject { described_class.from_json_hash(hash) } + + context 'with Release' do + let(:record_type) { 'Release' } + let(:tag) { 'v1.0' } + + it 'returns a hash with needed identifiers' do + expect(subject.github_identifiers).to eq( + { + db_id: 42, + tag: 'v1.0' + } + ) + end + end + + context 'with Issue' do + let(:record_type) { 'Issue' } + let(:iid) { 2 } + + it 'returns a hash with needed identifiers' do + expect(subject.github_identifiers).to eq( + { + db_id: 42, + noteable_iid: 2 + } + ) + end + end - expect(representation.github_identifiers).to eq({ db_id: record_id }) + context 'with Merge Request' do + let(:record_type) { 'MergeRequest' } + let(:iid) { 3 } + + it 'returns a hash with needed identifiers' do + expect(subject.github_identifiers).to eq( + { + db_id: 42, + noteable_iid: 3 + } + ) + end + end + + context 'with Note' do + let(:record_type) { 'Note' } + let(:noteable_type) { 'MergeRequest' } + + it 'returns a hash with needed identifiers' do + expect(subject.github_identifiers).to eq( + { + db_id: 42, + noteable_type: 'MergeRequest' + } + ) + end end end end diff --git a/spec/lib/gitlab/github_import/representation/pull_request_review_spec.rb b/spec/lib/gitlab/github_import/representation/pull_request_review_spec.rb index 0203da9f4fb..8925f466e27 100644 --- a/spec/lib/gitlab/github_import/representation/pull_request_review_spec.rb +++ b/spec/lib/gitlab/github_import/representation/pull_request_review_spec.rb @@ -77,7 +77,7 @@ RSpec.describe Gitlab::GithubImport::Representation::PullRequestReview do it 'returns a hash with needed identifiers' do github_identifiers = { review_id: 999, - merge_request_id: 42 + merge_request_iid: 1 } other_attributes = { something_else: '_something_else_' } review = described_class.new(github_identifiers.merge(other_attributes)) diff --git a/spec/lib/gitlab/github_import/representation/pull_request_spec.rb b/spec/lib/gitlab/github_import/representation/pull_request_spec.rb index b8c1c67e07c..4b8e7401e9d 100644 --- a/spec/lib/gitlab/github_import/representation/pull_request_spec.rb +++ b/spec/lib/gitlab/github_import/representation/pull_request_spec.rb @@ -287,7 +287,8 @@ RSpec.describe Gitlab::GithubImport::Representation::PullRequest do describe '#github_identifiers' do it 'returns a hash with needed identifiers' do github_identifiers = { - iid: 1 + iid: 1, + title: 'My Pull Request' } other_attributes = { something_else: '_something_else_' } pr = described_class.new(github_identifiers.merge(other_attributes)) diff --git a/spec/lib/gitlab/github_import/representation/pull_requests/review_requests_spec.rb b/spec/lib/gitlab/github_import/representation/pull_requests/review_requests_spec.rb index 0393f692a69..0259fbedee3 100644 --- a/spec/lib/gitlab/github_import/representation/pull_requests/review_requests_spec.rb +++ b/spec/lib/gitlab/github_import/representation/pull_requests/review_requests_spec.rb @@ -46,4 +46,27 @@ RSpec.describe Gitlab::GithubImport::Representation::PullRequests::ReviewRequest let(:review_requests) { described_class.from_json_hash(response) } end end + + describe '#github_identifiers' do + it 'returns a hash with needed identifiers' do + review_requests = { + merge_request_iid: 2, + merge_request_id: merge_request_id, + users: [ + { id: 4, login: 'alice' }, + { id: 5, login: 'bob' } + ] + } + + github_identifiers = { + merge_request_iid: 2, + requested_reviewers: %w[alice bob] + } + + other_attributes = { merge_request_id: 123, something_else: '_something_else_' } + review_requests = described_class.new(review_requests.merge(other_attributes)) + + expect(review_requests.github_identifiers).to eq(github_identifiers) + end + end end diff --git a/spec/lib/gitlab/github_import/user_finder_spec.rb b/spec/lib/gitlab/github_import/user_finder_spec.rb index d77aaa0e846..b6e369cb35b 100644 --- a/spec/lib/gitlab/github_import/user_finder_spec.rb +++ b/spec/lib/gitlab/github_import/user_finder_spec.rb @@ -259,6 +259,41 @@ RSpec.describe Gitlab::GithubImport::UserFinder, :clean_gitlab_redis_cache do expect(finder.email_for_github_username('kittens')).to be_nil end + + context 'when a username does not exist on GitHub' do + context 'when github username inexistence is not cached' do + it 'caches github username inexistence' do + expect(client) + .to receive(:user) + .with('kittens') + .and_raise(::Octokit::NotFound) + + expect(Gitlab::Cache::Import::Caching) + .to receive(:write).with( + described_class::INEXISTENCE_OF_GITHUB_USERNAME_CACHE_KEY % 'kittens', true + ) + + expect(finder.email_for_github_username('kittens')).to be_nil + end + end + + context 'when github username inexistence is already cached' do + it 'does not make request to the client' do + expect(Gitlab::Cache::Import::Caching) + .to receive(:read).with(described_class::EMAIL_FOR_USERNAME_CACHE_KEY % 'kittens') + + expect(Gitlab::Cache::Import::Caching) + .to receive(:read).with( + described_class::INEXISTENCE_OF_GITHUB_USERNAME_CACHE_KEY % 'kittens' + ).and_return('true') + + expect(client) + .not_to receive(:user) + + expect(finder.email_for_github_username('kittens')).to be_nil + end + end + end end end diff --git a/spec/lib/gitlab/gl_repository/repo_type_spec.rb b/spec/lib/gitlab/gl_repository/repo_type_spec.rb index 0ec94563cbb..40dcbe16688 100644 --- a/spec/lib/gitlab/gl_repository/repo_type_spec.rb +++ b/spec/lib/gitlab/gl_repository/repo_type_spec.rb @@ -136,7 +136,7 @@ RSpec.describe Gitlab::GlRepository::RepoType do let(:expected_identifier) { "design-#{project.id}" } let(:expected_id) { project.id } let(:expected_suffix) { '.design' } - let(:expected_repository) { ::DesignManagement::Repository.new(project) } + let(:expected_repository) { project.design_management_repository } let(:expected_container) { project } end diff --git a/spec/lib/gitlab/graphql/authorize/authorize_resource_spec.rb b/spec/lib/gitlab/graphql/authorize/authorize_resource_spec.rb index ac512e28e7b..1cd93d7b364 100644 --- a/spec/lib/gitlab/graphql/authorize/authorize_resource_spec.rb +++ b/spec/lib/gitlab/graphql/authorize/authorize_resource_spec.rb @@ -76,13 +76,17 @@ RSpec.describe Gitlab::Graphql::Authorize::AuthorizeResource do end end - context 'when the class does not define #find_object' do + describe '#find_object' do let(:fake_class) do Class.new { include Gitlab::Graphql::Authorize::AuthorizeResource } end - it 'raises a comprehensive error message' do - expect { fake_class.new.find_object }.to raise_error(/Implement #find_object in #{fake_class.name}/) + let(:id) { "id" } + let(:return_value) { "return value" } + + it 'calls GitlabSchema.find_by_gid' do + expect(GitlabSchema).to receive(:find_by_gid).with(id).and_return(return_value) + expect(fake_class.new.find_object(id: id)).to be return_value end end diff --git a/spec/lib/gitlab/graphql/deprecations/deprecation_spec.rb b/spec/lib/gitlab/graphql/deprecations/deprecation_spec.rb index 55650b0480e..172872fd7eb 100644 --- a/spec/lib/gitlab/graphql/deprecations/deprecation_spec.rb +++ b/spec/lib/gitlab/graphql/deprecations/deprecation_spec.rb @@ -55,7 +55,7 @@ RSpec.describe ::Gitlab::Graphql::Deprecations::Deprecation, feature_category: : it 'raises an error' do expect { parsed_deprecation }.to raise_error(ArgumentError, - '`alpha` and `deprecated` arguments cannot be passed at the same time' + '`experiment` and `deprecated` arguments cannot be passed at the same time' ) end end diff --git a/spec/lib/gitlab/graphql/known_operations_spec.rb b/spec/lib/gitlab/graphql/known_operations_spec.rb index 3ebfefbb43c..c7bc47e1e6a 100644 --- a/spec/lib/gitlab/graphql/known_operations_spec.rb +++ b/spec/lib/gitlab/graphql/known_operations_spec.rb @@ -2,7 +2,6 @@ require 'fast_spec_helper' require 'rspec-parameterized' -require "support/graphql/fake_query_type" RSpec.describe Gitlab::Graphql::KnownOperations do using RSpec::Parameterized::TableSyntax diff --git a/spec/lib/gitlab/graphql/loaders/lazy_relation_loader/registry_spec.rb b/spec/lib/gitlab/graphql/loaders/lazy_relation_loader/registry_spec.rb new file mode 100644 index 00000000000..265839d1236 --- /dev/null +++ b/spec/lib/gitlab/graphql/loaders/lazy_relation_loader/registry_spec.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Graphql::Loaders::LazyRelationLoader::Registry, feature_category: :vulnerability_management do + describe '#respond_to?' do + let(:relation) { Project.all } + let(:registry) { described_class.new(relation) } + + subject { registry.respond_to?(method_name) } + + context 'when the relation responds to given method' do + let(:method_name) { :sorted_by_updated_asc } + + it { is_expected.to be_truthy } + end + + context 'when the relation does not respond to given method' do + let(:method_name) { :this_method_does_not_exist } + + it { is_expected.to be_falsey } + end + end +end diff --git a/spec/lib/gitlab/graphql/loaders/lazy_relation_loader/relation_proxy_spec.rb b/spec/lib/gitlab/graphql/loaders/lazy_relation_loader/relation_proxy_spec.rb new file mode 100644 index 00000000000..f54fb6e77c5 --- /dev/null +++ b/spec/lib/gitlab/graphql/loaders/lazy_relation_loader/relation_proxy_spec.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Graphql::Loaders::LazyRelationLoader::RelationProxy, feature_category: :vulnerability_management do + describe '#respond_to?' do + let(:object) { double } + let(:registry) { instance_double(Gitlab::Graphql::Loaders::LazyRelationLoader::Registry) } + let(:relation_proxy) { described_class.new(object, registry) } + + subject { relation_proxy.respond_to?(:foo) } + + before do + allow(registry).to receive(:respond_to?).with(:foo, false).and_return(responds_to?) + end + + context 'when the registry responds to given method' do + let(:responds_to?) { true } + + it { is_expected.to be_truthy } + end + + context 'when the registry does not respond to given method' do + let(:responds_to?) { false } + + it { is_expected.to be_falsey } + end + end +end diff --git a/spec/lib/gitlab/graphql/loaders/lazy_relation_loader_spec.rb b/spec/lib/gitlab/graphql/loaders/lazy_relation_loader_spec.rb new file mode 100644 index 00000000000..e56cb68c6cb --- /dev/null +++ b/spec/lib/gitlab/graphql/loaders/lazy_relation_loader_spec.rb @@ -0,0 +1,123 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Graphql::Loaders::LazyRelationLoader, feature_category: :vulnerability_management do + let(:query_context) { {} } + let(:args) { {} } + + let_it_be(:project) { create(:project) } + + let(:loader) { loader_class.new(query_context, project, **args) } + + describe '#load' do + subject(:load_relation) { loader.load } + + context 'when the association is has many' do + let_it_be(:public_issue) { create(:issue, project: project) } + let_it_be(:confidential_issue) { create(:issue, :confidential, project: project) } + + let(:loader_class) do + Class.new(described_class) do + self.model = Project + self.association = :issues + + def relation(public_only: false) + relation = base_relation + relation = relation.public_only if public_only + + relation + end + end + end + + it { is_expected.to be_an_instance_of(described_class::RelationProxy) } + + describe '#relation' do + subject { load_relation.load } + + context 'without arguments' do + it { is_expected.to contain_exactly(public_issue, confidential_issue) } + end + + context 'with arguments' do + let(:args) { { public_only: true } } + + it { is_expected.to contain_exactly(public_issue) } + end + end + + describe 'using the same context for different records' do + let_it_be(:another_project) { create(:project) } + + let(:loader_for_another_project) { loader_class.new(query_context, another_project, **args) } + let(:records_for_another_project) { loader_for_another_project.load.load } + let(:records_for_project) { load_relation.load } + + before do + loader # register the original loader to query context + end + + it 'does not mix associated records' do + expect(records_for_another_project).to be_empty + expect(records_for_project).to contain_exactly(public_issue, confidential_issue) + end + + it 'does not cause N+1 queries' do + expect { records_for_another_project }.not_to exceed_query_limit(1) + end + end + + describe 'using Active Record querying methods' do + subject { load_relation.limit(1).load.count } + + it { is_expected.to be(1) } + end + + describe 'using Active Record finder methods' do + subject { load_relation.last(2) } + + it { is_expected.to contain_exactly(public_issue, confidential_issue) } + end + + describe 'calling a method that returns a non relation object' do + subject { load_relation.limit(1).limit_value } + + it { is_expected.to be(1) } + end + + describe 'calling a prohibited method' do + subject(:count) { load_relation.count } + + it 'raises a `PrematureQueryExecutionTriggered` error' do + expect { count }.to raise_error(described_class::Registry::PrematureQueryExecutionTriggered) + end + end + end + + context 'when the association is has one' do + let!(:project_setting) { create(:project_setting, project: project) } + let(:loader_class) do + Class.new(described_class) do + self.model = Project + self.association = :project_setting + end + end + + it { is_expected.to eq(project_setting) } + end + + context 'when the association is belongs to' do + let(:loader_class) do + Class.new(described_class) do + self.model = Project + self.association = :namespace + end + end + + it 'raises error' do + expect { load_relation }.to raise_error(RuntimeError) + end + end + end +end diff --git a/spec/lib/gitlab/graphql/subscriptions/action_cable_with_load_balancing_spec.rb b/spec/lib/gitlab/graphql/subscriptions/action_cable_with_load_balancing_spec.rb new file mode 100644 index 00000000000..a9bf3f1dca9 --- /dev/null +++ b/spec/lib/gitlab/graphql/subscriptions/action_cable_with_load_balancing_spec.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Graphql::Subscriptions::ActionCableWithLoadBalancing, feature_category: :shared do + let(:session_class) { ::Gitlab::Database::LoadBalancing::Session } + let(:session) { instance_double(session_class) } + let(:event) { instance_double(::GraphQL::Subscriptions::Event) } + + subject(:subscriptions) { described_class.new(schema: GitlabSchema) } + + it 'forces use of DB primary when executing subscription updates' do + expect(session_class).to receive(:current).and_return(session) + expect(session).to receive(:use_primary!) + + subscriptions.execute_update('sub:123', event, {}) + end +end diff --git a/spec/lib/gitlab/graphql/tracers/metrics_tracer_spec.rb b/spec/lib/gitlab/graphql/tracers/metrics_tracer_spec.rb index 168f5aa529e..f0312293469 100644 --- a/spec/lib/gitlab/graphql/tracers/metrics_tracer_spec.rb +++ b/spec/lib/gitlab/graphql/tracers/metrics_tracer_spec.rb @@ -2,7 +2,6 @@ require 'spec_helper' require 'rspec-parameterized' -require "support/graphql/fake_query_type" RSpec.describe Gitlab::Graphql::Tracers::MetricsTracer do using RSpec::Parameterized::TableSyntax diff --git a/spec/lib/gitlab/graphql/tracers/timer_tracer_spec.rb b/spec/lib/gitlab/graphql/tracers/timer_tracer_spec.rb index 986120dcd95..e42883aafd8 100644 --- a/spec/lib/gitlab/graphql/tracers/timer_tracer_spec.rb +++ b/spec/lib/gitlab/graphql/tracers/timer_tracer_spec.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true require "fast_spec_helper" -require "support/graphql/fake_tracer" -require "support/graphql/fake_query_type" RSpec.describe Gitlab::Graphql::Tracers::TimerTracer do let(:expected_duration) { 5 } diff --git a/spec/lib/gitlab/harbor/client_spec.rb b/spec/lib/gitlab/harbor/client_spec.rb index 4e80b8b53e3..745e22191bd 100644 --- a/spec/lib/gitlab/harbor/client_spec.rb +++ b/spec/lib/gitlab/harbor/client_spec.rb @@ -265,18 +265,20 @@ RSpec.describe Gitlab::Harbor::Client do end end - describe '#ping' do + describe '#check_project_availability' do before do - stub_request(:get, "https://demo.goharbor.io/api/v2.0/ping") + stub_request(:head, "https://demo.goharbor.io/api/v2.0/projects?project_name=testproject") .with( headers: { + 'Accept': 'application/json', + 'Authorization': 'Basic aGFyYm9ydXNlcm5hbWU6aGFyYm9ycGFzc3dvcmQ=', 'Content-Type': 'application/json' }) - .to_return(status: 200, body: 'pong') + .to_return(status: 200, body: '', headers: {}) end - it "calls api/v2.0/ping successfully" do - expect(client.ping).to eq(success: true) + it "calls api/v2.0/projects successfully" do + expect(client.check_project_availability).to eq(success: true) end end diff --git a/spec/lib/gitlab/http_connection_adapter_spec.rb b/spec/lib/gitlab/http_connection_adapter_spec.rb index dbf0252da46..8b8097f4885 100644 --- a/spec/lib/gitlab/http_connection_adapter_spec.rb +++ b/spec/lib/gitlab/http_connection_adapter_spec.rb @@ -111,20 +111,6 @@ RSpec.describe Gitlab::HTTPConnectionAdapter do end end - context 'when http(s) environment variable is set' do - before do - stub_env('https_proxy' => 'https://my.proxy') - end - - it 'sets up the connection' do - expect(connection).to be_a(Gitlab::NetHttpAdapter) - expect(connection.address).to eq('example.org') - expect(connection.hostname_override).to eq(nil) - expect(connection.addr_port).to eq('example.org') - expect(connection.port).to eq(443) - end - end - context 'when URL scheme is not HTTP/HTTPS' do let(:uri) { URI('ssh://example.org') } diff --git a/spec/lib/gitlab/import/metrics_spec.rb b/spec/lib/gitlab/import/metrics_spec.rb index 1a988af0dbd..9a7eb7b875e 100644 --- a/spec/lib/gitlab/import/metrics_spec.rb +++ b/spec/lib/gitlab/import/metrics_spec.rb @@ -42,11 +42,11 @@ RSpec.describe Gitlab::Import::Metrics, :aggregate_failures do it 'does not emit importer metrics' do expect(subject).not_to receive(:track_usage_event) expect_no_snowplow_event( - category: :test_importer, + category: 'Import::GithubService', action: 'create', label: 'github_import_project_state', project: project, - extra: { import_type: 'github', state: 'failed' } + import_type: 'github', state: 'failed' ) subject.track_failed_import @@ -65,11 +65,11 @@ RSpec.describe Gitlab::Import::Metrics, :aggregate_failures do subject.track_failed_import expect_snowplow_event( - category: :test_importer, + category: 'Import::GithubService', action: 'create', label: 'github_import_project_state', project: project, - extra: { import_type: 'github', state: 'failed' } + import_type: 'github', state: 'failed' ) end end @@ -102,11 +102,11 @@ RSpec.describe Gitlab::Import::Metrics, :aggregate_failures do subject.track_finished_import expect_snowplow_event( - category: :test_importer, + category: 'Import::GithubService', action: 'create', label: 'github_import_project_state', project: project, - extra: { import_type: 'github', state: 'completed' } + import_type: 'github', state: 'completed' ) expect(subject.duration).not_to be_nil @@ -123,11 +123,11 @@ RSpec.describe Gitlab::Import::Metrics, :aggregate_failures do subject.track_finished_import expect_snowplow_event( - category: :test_importer, + category: 'Import::GithubService', action: 'create', label: 'github_import_project_state', project: project, - extra: { import_type: 'github', state: 'partially completed' } + import_type: 'github', state: 'partially completed' ) end end @@ -140,11 +140,11 @@ RSpec.describe Gitlab::Import::Metrics, :aggregate_failures do subject.track_finished_import expect_no_snowplow_event( - category: :test_importer, + category: 'Import::GithubService', action: 'create', label: 'github_import_project_state', project: project, - extra: { import_type: 'github', state: 'completed' } + import_type: 'github', state: 'completed' ) end end @@ -155,11 +155,11 @@ RSpec.describe Gitlab::Import::Metrics, :aggregate_failures do it 'does not emit importer metrics' do expect(subject).not_to receive(:track_usage_event) expect_no_snowplow_event( - category: :test_importer, + category: 'Import::GithubService', action: 'create', label: 'github_import_project_state', project: project, - extra: { import_type: 'github', state: 'canceled' } + import_type: 'github', state: 'canceled' ) subject.track_canceled_import @@ -178,11 +178,11 @@ RSpec.describe Gitlab::Import::Metrics, :aggregate_failures do subject.track_canceled_import expect_snowplow_event( - category: :test_importer, + category: 'Import::GithubService', action: 'create', label: 'github_import_project_state', project: project, - extra: { import_type: 'github', state: 'canceled' } + import_type: 'github', state: 'canceled' ) end end diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index f6d6a791e8c..66b57deb643 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -14,6 +14,7 @@ issues: - resource_milestone_events - resource_state_events - resource_iteration_events +- assignment_events - sent_notifications - sentry_issue - issuable_severity @@ -180,6 +181,7 @@ merge_requests: - resource_milestone_events - resource_state_events - resource_iteration_events +- assignment_events - label_links - labels - last_edited_by @@ -216,7 +218,7 @@ merge_requests: - approver_groups - approved_by_users - draft_notes -- merge_train +- merge_train_car - blocks_as_blocker - blocks_as_blockee - blocking_merge_requests @@ -260,6 +262,11 @@ ci_pipelines: - statuses - statuses_order_id_desc - latest_statuses_ordered_by_stage +- latest_statuses +- all_jobs +- current_jobs +- all_processable_jobs +- current_processable_jobs - builds - bridges - processables @@ -308,7 +315,6 @@ ci_pipelines: - latest_builds_report_results - messages - pipeline_artifacts -- latest_statuses - dast_profile - dast_profiles_pipeline - dast_site_profile @@ -396,8 +402,8 @@ builds: - job_artifacts_cluster_image_scanning - job_artifacts_cyclonedx - job_artifacts_requirements_v2 -- runner_machine -- runner_machine_build +- runner_manager +- runner_manager_build - runner_session - trace_metadata - terraform_state_versions @@ -493,6 +499,7 @@ container_repositories: - project - name project: +- catalog_resource - external_status_checks - base_tags - project_topics @@ -706,13 +713,14 @@ project: - packages - package_files - rpm_repository_files +- npm_metadata_caches - packages_cleanup_policy - alerting_setting - project_setting - webide_pipelines - reviews - incident_management_setting -- merge_trains +- merge_train_cars - designs - project_aliases - external_pull_requests @@ -724,6 +732,7 @@ project: - downstream_project_subscriptions - service_desk_setting - service_desk_custom_email_verification +- service_desk_custom_email_credential - security_setting - import_failures - container_expiration_policy @@ -779,6 +788,7 @@ project: - sbom_occurrences - analytics_dashboards_configuration_project - analytics_dashboards_pointer +- design_management_repository award_emoji: - awardable - user @@ -865,6 +875,7 @@ incident_management_setting: - project merge_trains: - project +merge_train_cars: - merge_request boards: - group @@ -998,3 +1009,22 @@ resource_iteration_events: iterations_cadence: - group - iterations +catalog_resource: + - project +approval_rules: + - users + - groups + - group_users + - security_orchestration_policy_configuration + - protected_branches + - approval_merge_request_rule_sources + - approval_merge_request_rules + - approval_project_rules_users + - approval_project_rules_protected_branches + - scan_result_policy_read +approval_project_rules_users: + - user + - approval_project_rule +approval_project_rules_protected_branches: + - protected_branch + - approval_project_rule diff --git a/spec/lib/gitlab/import_export/attributes_finder_spec.rb b/spec/lib/gitlab/import_export/attributes_finder_spec.rb index 767b7a3c84e..f12cbe4f82f 100644 --- a/spec/lib/gitlab/import_export/attributes_finder_spec.rb +++ b/spec/lib/gitlab/import_export/attributes_finder_spec.rb @@ -177,7 +177,8 @@ RSpec.describe Gitlab::ImportExport::AttributesFinder, feature_category: :import end def setup_yaml(hash) - allow(YAML).to receive(:load_file).with(test_config).and_return(hash) + allow(YAML).to receive(:safe_load_file) + .with(test_config, aliases: true, permitted_classes: [Symbol]).and_return(hash) end end end diff --git a/spec/lib/gitlab/import_export/fork_spec.rb b/spec/lib/gitlab/import_export/fork_spec.rb deleted file mode 100644 index 9d766eb3af1..00000000000 --- a/spec/lib/gitlab/import_export/fork_spec.rb +++ /dev/null @@ -1,59 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe 'forked project import' do - include ProjectForksHelper - - let(:user) { create(:user) } - let!(:project_with_repo) { create(:project, :repository, name: 'test-repo-restorer', path: 'test-repo-restorer') } - let!(:project) { create(:project, name: 'test-repo-restorer-no-repo', path: 'test-repo-restorer-no-repo') } - let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" } - let(:shared) { project.import_export_shared } - let(:forked_from_project) { create(:project, :repository) } - let(:forked_project) { fork_project(project_with_repo, nil, repository: true) } - let(:repo_saver) { Gitlab::ImportExport::RepoSaver.new(exportable: project_with_repo, shared: shared) } - let(:bundle_path) { File.join(shared.export_path, Gitlab::ImportExport.project_bundle_filename) } - - let(:repo_restorer) do - Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: bundle_path, shared: shared, importable: project) - end - - let!(:merge_request) do - create(:merge_request, source_project: forked_project, target_project: project_with_repo) - end - - let(:saver) do - Gitlab::ImportExport::Project::TreeSaver.new(project: project_with_repo, current_user: user, shared: shared) - end - - let(:restorer) do - Gitlab::ImportExport::Project::TreeRestorer.new(user: user, shared: shared, project: project) - end - - before do - stub_feature_flags(project_export_as_ndjson: false) - - allow_next_instance_of(Gitlab::ImportExport) do |instance| - allow(instance).to receive(:storage_path).and_return(export_path) - end - - saver.save # rubocop:disable Rails/SaveBang - repo_saver.save # rubocop:disable Rails/SaveBang - - repo_restorer.restore - restorer.restore - end - - after do - FileUtils.rm_rf(export_path) - project_with_repo.repository.remove - project.repository.remove - end - - it 'can access the MR', :sidekiq_might_not_need_inline do - project.merge_requests.first.fetch_ref! - - expect(project.repository.ref_exists?('refs/merge-requests/1/head')).to be_truthy - end -end diff --git a/spec/lib/gitlab/import_export/group/relation_tree_restorer_spec.rb b/spec/lib/gitlab/import_export/group/relation_tree_restorer_spec.rb index 07971d6271c..495cefa002a 100644 --- a/spec/lib/gitlab/import_export/group/relation_tree_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/group/relation_tree_restorer_spec.rb @@ -14,20 +14,26 @@ RSpec.describe Gitlab::ImportExport::Group::RelationTreeRestorer, feature_catego let(:importable) { create(:group, parent: group) } include_context 'relation tree restorer shared context' do - let(:importable_name) { nil } + let(:importable_name) { 'groups/4353' } end - let(:path) { 'spec/fixtures/lib/gitlab/import_export/group_exports/no_children/group.json' } + let(:path) { Rails.root.join('spec/fixtures/lib/gitlab/import_export/group_exports/no_children/tree') } let(:relation_reader) do - Gitlab::ImportExport::Json::LegacyReader::File.new( - path, - relation_names: reader.group_relation_names) + Gitlab::ImportExport::Json::NdjsonReader.new(path) end let(:reader) do Gitlab::ImportExport::Reader.new( shared: shared, - config: Gitlab::ImportExport::Config.new(config: Gitlab::ImportExport.legacy_group_config_file).to_h + config: Gitlab::ImportExport::Config.new(config: Gitlab::ImportExport.group_config_file).to_h + ) + end + + let(:members_mapper) do + Gitlab::ImportExport::MembersMapper.new( + exported_members: relation_reader.consume_relation(importable_name, 'members').map(&:first), + user: user, + importable: importable ) end @@ -41,7 +47,7 @@ RSpec.describe Gitlab::ImportExport::Group::RelationTreeRestorer, feature_catego relation_factory: Gitlab::ImportExport::Group::RelationFactory, reader: reader, importable: importable, - importable_path: nil, + importable_path: importable_name, importable_attributes: attributes ) end @@ -62,20 +68,13 @@ RSpec.describe Gitlab::ImportExport::Group::RelationTreeRestorer, feature_catego end describe 'relation object saving' do - let(:importable) { create(:group) } - let(:relation_reader) do - Gitlab::ImportExport::Json::LegacyReader::File.new( - path, - relation_names: [:labels]) - end - before do allow(shared.logger).to receive(:info).and_call_original allow(relation_reader).to receive(:consume_relation).and_call_original allow(relation_reader) .to receive(:consume_relation) - .with(nil, 'labels') + .with(importable_name, 'labels') .and_return([[label, 0]]) end diff --git a/spec/lib/gitlab/import_export/group/tree_restorer_spec.rb b/spec/lib/gitlab/import_export/group/tree_restorer_spec.rb index aa30e24296e..a6afd0a36ec 100644 --- a/spec/lib/gitlab/import_export/group/tree_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/group/tree_restorer_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::ImportExport::Group::TreeRestorer, feature: :subgroups do +RSpec.describe Gitlab::ImportExport::Group::TreeRestorer, feature: :subgroups, feature_category: :importers do include ImportExport::CommonUtil shared_examples 'group restoration' do @@ -171,7 +171,7 @@ RSpec.describe Gitlab::ImportExport::Group::TreeRestorer, feature: :subgroups do allow(shared).to receive(:export_path).and_return(tmpdir) expect(group_tree_restorer.restore).to eq(false) - expect(shared.errors).to include('Incorrect JSON format') + expect(shared.errors).to include('Invalid file') end end end diff --git a/spec/lib/gitlab/import_export/import_export_equivalence_spec.rb b/spec/lib/gitlab/import_export/import_export_equivalence_spec.rb deleted file mode 100644 index 6c997dc1361..00000000000 --- a/spec/lib/gitlab/import_export/import_export_equivalence_spec.rb +++ /dev/null @@ -1,67 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -# Verifies that given an exported project meta-data tree, when importing this -# tree and then exporting it again, we should obtain the initial tree. -# -# This equivalence only works up to a certain extent, for instance we need -# to ignore: -# -# - row IDs and foreign key IDs -# - some timestamps -# - randomly generated fields like tokens -# -# as these are expected to change between import/export cycles. -RSpec.describe Gitlab::ImportExport, feature_category: :importers do - include ImportExport::CommonUtil - include ConfigurationHelper - include ImportExport::ProjectTreeExpectations - - let(:json_fixture) { 'complex' } - - before do - stub_feature_flags(project_export_as_ndjson: false) - end - - it 'yields the initial tree when importing and exporting it again' do - project = create(:project) - user = create(:user, :admin) - - # We first generate a test fixture dynamically from a seed-fixture, so as to - # account for any fields in the initial fixture that are missing and set to - # defaults during import (ideally we should have realistic test fixtures - # that "honestly" represent exports) - expect( - restore_then_save_project( - project, - user, - import_path: seed_fixture_path, - export_path: test_fixture_path) - ).to be true - # Import, then export again from the generated fixture. Any residual changes - # in the JSON will count towards comparison i.e. test failures. - expect( - restore_then_save_project( - project, - user, - import_path: test_fixture_path, - export_path: test_tmp_path) - ).to be true - - imported_json = Gitlab::Json.parse(File.read("#{test_fixture_path}/project.json")) - exported_json = Gitlab::Json.parse(File.read("#{test_tmp_path}/project.json")) - - assert_relations_match(imported_json, exported_json) - end - - private - - def seed_fixture_path - "#{fixtures_path}/#{json_fixture}" - end - - def test_fixture_path - "#{test_tmp_path}/#{json_fixture}" - end -end diff --git a/spec/lib/gitlab/import_export/json/legacy_reader/file_spec.rb b/spec/lib/gitlab/import_export/json/legacy_reader/file_spec.rb deleted file mode 100644 index 793b3ebfb9e..00000000000 --- a/spec/lib/gitlab/import_export/json/legacy_reader/file_spec.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require_relative 'shared_example' - -RSpec.describe Gitlab::ImportExport::Json::LegacyReader::File do - it_behaves_like 'import/export json legacy reader' do - let(:valid_path) { 'spec/fixtures/lib/gitlab/import_export/light/project.json' } - let(:data) { valid_path } - let(:json_data) { Gitlab::Json.parse(File.read(valid_path)) } - end - - describe '#exist?' do - let(:legacy_reader) do - described_class.new(path, relation_names: []) - end - - subject { legacy_reader.exist? } - - context 'given valid path' do - let(:path) { 'spec/fixtures/lib/gitlab/import_export/light/project.json' } - - it { is_expected.to be true } - end - - context 'given invalid path' do - let(:path) { 'spec/non-existing-folder/do-not-create-this-file.json' } - - it { is_expected.to be false } - end - end -end diff --git a/spec/lib/gitlab/import_export/json/legacy_reader/hash_spec.rb b/spec/lib/gitlab/import_export/json/legacy_reader/hash_spec.rb deleted file mode 100644 index 57d66dc0f50..00000000000 --- a/spec/lib/gitlab/import_export/json/legacy_reader/hash_spec.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require_relative 'shared_example' - -RSpec.describe Gitlab::ImportExport::Json::LegacyReader::Hash do - it_behaves_like 'import/export json legacy reader' do - let(:path) { 'spec/fixtures/lib/gitlab/import_export/light/project.json' } - - # the hash is modified by the `LegacyReader` - # we need to deep-dup it - let(:json_data) { Gitlab::Json.parse(File.read(path)) } - let(:data) { Gitlab::Json.parse(File.read(path)) } - end - - describe '#exist?' do - let(:legacy_reader) do - described_class.new(tree_hash, relation_names: []) - end - - subject { legacy_reader.exist? } - - context 'tree_hash is nil' do - let(:tree_hash) { nil } - - it { is_expected.to be_falsey } - end - - context 'tree_hash presents' do - let(:tree_hash) { { "issues": [] } } - - it { is_expected.to be_truthy } - end - end -end diff --git a/spec/lib/gitlab/import_export/json/legacy_reader/shared_example.rb b/spec/lib/gitlab/import_export/json/legacy_reader/shared_example.rb deleted file mode 100644 index 3e9bd3fe741..00000000000 --- a/spec/lib/gitlab/import_export/json/legacy_reader/shared_example.rb +++ /dev/null @@ -1,102 +0,0 @@ -# frozen_string_literal: true - -RSpec.shared_examples 'import/export json legacy reader' do - let(:relation_names) { [] } - - let(:legacy_reader) do - described_class.new( - data, - relation_names: relation_names, - allowed_path: "project") - end - - describe '#consume_attributes' do - context 'when valid path is passed' do - subject { legacy_reader.consume_attributes("project") } - - context 'no excluded attributes' do - let(:relation_names) { [] } - - it 'returns the whole tree from parsed JSON' do - expect(subject).to eq(json_data) - end - end - - context 'some attributes are excluded' do - let(:relation_names) { %w[milestones labels] } - - it 'returns hash without excluded attributes and relations' do - expect(subject).not_to include('milestones', 'labels') - end - end - end - - context 'when invalid path is passed' do - it 'raises an exception' do - expect { legacy_reader.consume_attributes("invalid-path") } - .to raise_error(ArgumentError) - end - end - end - - describe '#consume_relation' do - context 'when valid path is passed' do - let(:key) { 'labels' } - - subject { legacy_reader.consume_relation("project", key) } - - context 'key has not been consumed' do - it 'returns an Enumerator' do - expect(subject).to be_an_instance_of(Enumerator) - end - - context 'value is nil' do - before do - expect(legacy_reader).to receive(:relations).and_return({ key => nil }) - end - - it 'yields nothing to the Enumerator' do - expect(subject.to_a).to eq([]) - end - end - - context 'value is an array' do - before do - expect(legacy_reader).to receive(:relations).and_return({ key => %w[label1 label2] }) - end - - it 'yields every relation value to the Enumerator' do - expect(subject.to_a).to eq([['label1', 0], ['label2', 1]]) - end - end - - context 'value is not array' do - before do - expect(legacy_reader).to receive(:relations).and_return({ key => 'non-array value' }) - end - - it 'yields the value with index 0 to the Enumerator' do - expect(subject.to_a).to eq([['non-array value', 0]]) - end - end - end - - context 'key has been consumed' do - before do - legacy_reader.consume_relation("project", key).first - end - - it 'yields nothing to the Enumerator' do - expect(subject.to_a).to eq([]) - end - end - end - - context 'when invalid path is passed' do - it 'raises an exception' do - expect { legacy_reader.consume_relation("invalid") } - .to raise_error(ArgumentError) - end - end - end -end diff --git a/spec/lib/gitlab/import_export/json/legacy_writer_spec.rb b/spec/lib/gitlab/import_export/json/legacy_writer_spec.rb deleted file mode 100644 index 2c0f023ad2c..00000000000 --- a/spec/lib/gitlab/import_export/json/legacy_writer_spec.rb +++ /dev/null @@ -1,102 +0,0 @@ -# frozen_string_literal: true - -require 'fast_spec_helper' -require 'tmpdir' - -RSpec.describe Gitlab::ImportExport::Json::LegacyWriter, feature_category: :importers do - let(:path) { "#{Dir.tmpdir}/legacy_writer_spec/test.json" } - - subject do - described_class.new(path, allowed_path: "project") - end - - after do - FileUtils.rm_rf(path) - end - - describe "#write_attributes" do - it "writes correct json" do - expected_hash = { "key" => "value_1", "key_1" => "value_2" } - subject.write_attributes("project", expected_hash) - - expect(subject_json).to eq(expected_hash) - end - - context 'when invalid path is used' do - it 'raises an exception' do - expect { subject.write_attributes("invalid", { "key" => "value" }) } - .to raise_error(ArgumentError) - end - end - end - - describe "#write_relation" do - context "when key is already written" do - it "raises exception" do - subject.write_relation("project", "key", "old value") - - expect { subject.write_relation("project", "key", "new value") } - .to raise_exception("key 'key' already written") - end - end - - context "when key is not already written" do - context "when multiple key value pairs are stored" do - it "writes correct json" do - expected_hash = { "key" => "value_1", "key_1" => "value_2" } - expected_hash.each do |key, value| - subject.write_relation("project", key, value) - end - - expect(subject_json).to eq(expected_hash) - end - end - end - - context 'when invalid path is used' do - it 'raises an exception' do - expect { subject.write_relation("invalid", "key", "value") } - .to raise_error(ArgumentError) - end - end - end - - describe "#write_relation_array" do - context 'when array is used' do - it 'writes correct json' do - subject.write_relation_array("project", "key", ["value"]) - - expect(subject_json).to eq({ "key" => ["value"] }) - end - end - - context 'when enumerable is used' do - it 'writes correct json' do - values = %w(value1 value2) - - enumerator = Enumerator.new do |items| - values.each { |value| items << value } - end - - subject.write_relation_array("project", "key", enumerator) - - expect(subject_json).to eq({ "key" => values }) - end - end - - context "when key is already written" do - it "raises an exception" do - subject.write_relation_array("project", "key", %w(old_value)) - - expect { subject.write_relation_array("project", "key", %w(new_value)) } - .to raise_error(ArgumentError) - end - end - end - - def subject_json - subject.close - - ::JSON.parse(File.read(subject.path)) - end -end diff --git a/spec/lib/gitlab/import_export/json/ndjson_reader_spec.rb b/spec/lib/gitlab/import_export/json/ndjson_reader_spec.rb index 0ca4c4ccc87..98afe01c08b 100644 --- a/spec/lib/gitlab/import_export/json/ndjson_reader_spec.rb +++ b/spec/lib/gitlab/import_export/json/ndjson_reader_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::ImportExport::Json::NdjsonReader do +RSpec.describe Gitlab::ImportExport::Json::NdjsonReader, feature_category: :importers do include ImportExport::CommonUtil let(:fixture) { 'spec/fixtures/lib/gitlab/import_export/light/tree' } @@ -26,14 +26,6 @@ RSpec.describe Gitlab::ImportExport::Json::NdjsonReader do end end - describe '#legacy?' do - let(:dir_path) { fixture } - - subject { ndjson_reader.legacy? } - - it { is_expected.to be false } - end - describe '#consume_attributes' do let(:dir_path) { fixture } @@ -42,6 +34,20 @@ RSpec.describe Gitlab::ImportExport::Json::NdjsonReader do it 'returns the whole root tree from parsed JSON' do expect(subject).to eq(root_tree) end + + context 'when project.json is symlink' do + it 'raises error an error' do + Dir.mktmpdir do |tmpdir| + FileUtils.touch(File.join(tmpdir, 'passwd')) + File.symlink(File.join(tmpdir, 'passwd'), File.join(tmpdir, 'project.json')) + + ndjson_reader = described_class.new(tmpdir) + + expect { ndjson_reader.consume_attributes(importable_path) } + .to raise_error(Gitlab::ImportExport::Error, 'Invalid file') + end + end + end end describe '#consume_relation' do @@ -91,6 +97,22 @@ RSpec.describe Gitlab::ImportExport::Json::NdjsonReader do end end + context 'when relation file is a symlink' do + it 'yields nothing to the Enumerator' do + Dir.mktmpdir do |tmpdir| + Dir.mkdir(File.join(tmpdir, 'project')) + File.write(File.join(tmpdir, 'passwd'), "{}\n{}") + File.symlink(File.join(tmpdir, 'passwd'), File.join(tmpdir, 'project', 'issues.ndjson')) + + ndjson_reader = described_class.new(tmpdir) + + result = ndjson_reader.consume_relation(importable_path, 'issues') + + expect(result.to_a).to eq([]) + end + end + end + context 'relation file is empty' do let(:key) { 'empty' } diff --git a/spec/lib/gitlab/import_export/json/streaming_serializer_spec.rb b/spec/lib/gitlab/import_export/json/streaming_serializer_spec.rb index 103d3512e8b..f4c9189030b 100644 --- a/spec/lib/gitlab/import_export/json/streaming_serializer_spec.rb +++ b/spec/lib/gitlab/import_export/json/streaming_serializer_spec.rb @@ -28,7 +28,7 @@ RSpec.describe Gitlab::ImportExport::Json::StreamingSerializer, feature_category let(:exportable_path) { 'project' } let(:logger) { Gitlab::Export::Logger.build } - let(:json_writer) { instance_double('Gitlab::ImportExport::Json::LegacyWriter') } + let(:json_writer) { instance_double('Gitlab::ImportExport::Json::NdjsonWriter') } let(:hash) { { name: exportable.name, description: exportable.description }.stringify_keys } let(:include) { [] } let(:custom_orderer) { nil } diff --git a/spec/lib/gitlab/import_export/model_configuration_spec.rb b/spec/lib/gitlab/import_export/model_configuration_spec.rb index ce965a05a32..8e5fe96f3b4 100644 --- a/spec/lib/gitlab/import_export/model_configuration_spec.rb +++ b/spec/lib/gitlab/import_export/model_configuration_spec.rb @@ -9,7 +9,7 @@ RSpec.describe 'Import/Export model configuration', feature_category: :importers include ConfigurationHelper let(:all_models_yml) { 'spec/lib/gitlab/import_export/all_models.yml' } - let(:all_models_hash) { YAML.load_file(all_models_yml) } + let(:all_models_hash) { YAML.safe_load_file(all_models_yml, aliases: true) } let(:current_models) { setup_models } let(:model_names) { relation_names_for(:project) } diff --git a/spec/lib/gitlab/import_export/project/exported_relations_merger_spec.rb b/spec/lib/gitlab/import_export/project/exported_relations_merger_spec.rb index d70e89c6856..f8018e75879 100644 --- a/spec/lib/gitlab/import_export/project/exported_relations_merger_spec.rb +++ b/spec/lib/gitlab/import_export/project/exported_relations_merger_spec.rb @@ -64,8 +64,8 @@ RSpec.describe Gitlab::ImportExport::Project::ExportedRelationsMerger do expect(result).to eq(false) expect(shared.errors).to match_array( [ - "undefined method `export_file' for nil:NilClass", - "undefined method `export_file' for nil:NilClass" + /^undefined method `export_file' for nil:NilClass/, + /^undefined method `export_file' for nil:NilClass/ ] ) end diff --git a/spec/lib/gitlab/import_export/project/relation_tree_restorer_spec.rb b/spec/lib/gitlab/import_export/project/relation_tree_restorer_spec.rb index 75012aa80ec..180a6b6ff0a 100644 --- a/spec/lib/gitlab/import_export/project/relation_tree_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/project/relation_tree_restorer_spec.rb @@ -55,54 +55,19 @@ RSpec.describe Gitlab::ImportExport::Project::RelationTreeRestorer, feature_cate end end - context 'with legacy reader' do - let(:path) { 'spec/fixtures/lib/gitlab/import_export/complex/project.json' } - let(:relation_reader) do - Gitlab::ImportExport::Json::LegacyReader::File.new( - path, - relation_names: reader.project_relation_names, - allowed_path: 'project' - ) - end - - let(:attributes) { relation_reader.consume_attributes('project') } - - it_behaves_like 'import project successfully' - - context 'with logging of relations creation' do - let_it_be(:group) { create(:group).tap { |g| g.add_maintainer(user) } } - let_it_be(:importable) do - create(:project, :builds_enabled, :issues_disabled, name: 'project', path: 'project', group: group) - end - - it 'logs top-level relation creation' do - expect(shared.logger) - .to receive(:info) - .with(hash_including(message: '[Project/Group Import] Created new object relation')) - .at_least(:once) - - subject - end - end - end - - context 'with ndjson reader' do + context 'when inside a group' do let(:path) { 'spec/fixtures/lib/gitlab/import_export/complex/tree' } let(:relation_reader) { Gitlab::ImportExport::Json::NdjsonReader.new(path) } - it_behaves_like 'import project successfully' - - context 'when inside a group' do - let_it_be(:group) do - create(:group, :disabled_and_unoverridable).tap { |g| g.add_maintainer(user) } - end - - before do - importable.update!(shared_runners_enabled: false, group: group) - end + let_it_be(:group) do + create(:group, :disabled_and_unoverridable).tap { |g| g.add_maintainer(user) } + end - it_behaves_like 'import project successfully' + before do + importable.update!(shared_runners_enabled: false, group: group) end + + it_behaves_like 'import project successfully' end context 'with invalid relations' do diff --git a/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb b/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb index a07fe4fd29c..5aa16f9508d 100644 --- a/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb @@ -12,7 +12,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i let(:shared) { project.import_export_shared } - RSpec.shared_examples 'project tree restorer work properly' do |reader, ndjson_enabled| + RSpec.shared_examples 'project tree restorer work properly' do describe 'restore project tree' do before_all do # Using an admin for import, so we can check assignment of existing members @@ -27,10 +27,9 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i @shared = @project.import_export_shared stub_all_feature_flags - stub_feature_flags(project_import_ndjson: ndjson_enabled) setup_import_export_config('complex') - setup_reader(reader) + setup_reader allow_any_instance_of(Repository).to receive(:fetch_source_branch!).and_return(true) allow_any_instance_of(Gitlab::Git::Repository).to receive(:branch_exists?).and_return(false) @@ -606,23 +605,15 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i end end - context 'project.json file access check' do + context 'when expect tree structure is not present in the export path' do let(:user) { create(:user) } - let!(:project) { create(:project, :builds_disabled, :issues_disabled, name: 'project', path: 'project') } - let(:project_tree_restorer) do - described_class.new(user: user, shared: shared, project: project) - end + let_it_be(:project) { create(:project, :builds_disabled, :issues_disabled, name: 'project', path: 'project') } - let(:restored_project_json) { project_tree_restorer.restore } + it 'fails to restore the project' do + result = described_class.new(user: user, shared: shared, project: project).restore - it 'does not read a symlink' do - Dir.mktmpdir do |tmpdir| - setup_symlink(tmpdir, 'project.json') - allow(shared).to receive(:export_path).and_call_original - - expect(project_tree_restorer.restore).to eq(false) - expect(shared.errors).to include('invalid import format') - end + expect(result).to eq(false) + expect(shared.errors).to include('invalid import format') end end @@ -635,7 +626,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i context 'with a simple project' do before do setup_import_export_config('light') - setup_reader(reader) + setup_reader expect(restored_project_json).to eq(true) end @@ -670,7 +661,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i context 'multiple pipelines reference the same external pull request' do before do setup_import_export_config('multi_pipeline_ref_one_external_pr') - setup_reader(reader) + setup_reader expect(restored_project_json).to eq(true) end @@ -698,7 +689,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i before do setup_import_export_config('light') - setup_reader(reader) + setup_reader expect(project).to receive(:merge_requests).and_call_original expect(project).to receive(:merge_requests).and_raise(exception) @@ -715,7 +706,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i before do setup_import_export_config('light') - setup_reader(reader) + setup_reader expect(project).to receive(:merge_requests).and_call_original expect(project).to receive(:merge_requests).and_raise(exception) @@ -747,7 +738,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i context 'when the project has overridden params in import data' do before do setup_import_export_config('light') - setup_reader(reader) + setup_reader end it 'handles string versions of visibility_level' do @@ -813,7 +804,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i before do setup_import_export_config('group') - setup_reader(reader) + setup_reader expect(restored_project_json).to eq(true) end @@ -849,7 +840,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i before do setup_import_export_config('light') - setup_reader(reader) + setup_reader end it 'imports labels' do @@ -885,7 +876,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i before do setup_import_export_config('milestone-iid') - setup_reader(reader) + setup_reader end it 'preserves the project milestone IID' do @@ -901,7 +892,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i context 'with external authorization classification labels' do before do setup_import_export_config('light') - setup_reader(reader) + setup_reader end it 'converts empty external classification authorization labels to nil' do @@ -928,76 +919,80 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i described_class.new(user: user, shared: shared, project: project) end - before do - allow_any_instance_of(Gitlab::ImportExport::Json::LegacyReader::File).to receive(:exist?).and_return(true) - allow_any_instance_of(Gitlab::ImportExport::Json::NdjsonReader).to receive(:exist?).and_return(false) - allow_any_instance_of(Gitlab::ImportExport::Json::LegacyReader::File).to receive(:tree_hash) { tree_hash } - end - - context 'no group visibility' do - let(:visibility) { Gitlab::VisibilityLevel::PRIVATE } + describe 'visibility level' do + before do + setup_import_export_config('light') - it 'uses the project visibility' do - expect(restorer.restore).to eq(true) - expect(restorer.project.visibility_level).to eq(visibility) + allow_next_instance_of(Gitlab::ImportExport::Json::NdjsonReader) do |relation_reader| + allow(relation_reader).to receive(:consume_attributes).and_return(tree_hash) + end end - end - - context 'with restricted internal visibility' do - describe 'internal project' do - let(:visibility) { Gitlab::VisibilityLevel::INTERNAL } - it 'uses private visibility' do - stub_application_setting(restricted_visibility_levels: [Gitlab::VisibilityLevel::INTERNAL]) + context 'no group visibility' do + let(:visibility) { Gitlab::VisibilityLevel::PRIVATE } + it 'uses the project visibility' do expect(restorer.restore).to eq(true) - expect(restorer.project.visibility_level).to eq(Gitlab::VisibilityLevel::PRIVATE) + expect(restorer.project.visibility_level).to eq(visibility) end end - end - context 'with group visibility' do - before do - group = create(:group, visibility_level: group_visibility) - group.add_members([user], GroupMember::MAINTAINER) - project.update!(group: group) - end + context 'with restricted internal visibility' do + describe 'internal project' do + let(:visibility) { Gitlab::VisibilityLevel::INTERNAL } - context 'private group visibility' do - let(:group_visibility) { Gitlab::VisibilityLevel::PRIVATE } - let(:visibility) { Gitlab::VisibilityLevel::PUBLIC } + it 'uses private visibility' do + stub_application_setting(restricted_visibility_levels: [Gitlab::VisibilityLevel::INTERNAL]) - it 'uses the group visibility' do - expect(restorer.restore).to eq(true) - expect(restorer.project.visibility_level).to eq(group_visibility) + expect(restorer.restore).to eq(true) + expect(restorer.project.visibility_level).to eq(Gitlab::VisibilityLevel::PRIVATE) + end end end - context 'public group visibility' do - let(:group_visibility) { Gitlab::VisibilityLevel::PUBLIC } - let(:visibility) { Gitlab::VisibilityLevel::PRIVATE } + context 'with group visibility' do + before do + group = create(:group, visibility_level: group_visibility) + group.add_members([user], GroupMember::MAINTAINER) + project.update!(group: group) + end - it 'uses the project visibility' do - expect(restorer.restore).to eq(true) - expect(restorer.project.visibility_level).to eq(visibility) + context 'private group visibility' do + let(:group_visibility) { Gitlab::VisibilityLevel::PRIVATE } + let(:visibility) { Gitlab::VisibilityLevel::PUBLIC } + + it 'uses the group visibility' do + expect(restorer.restore).to eq(true) + expect(restorer.project.visibility_level).to eq(group_visibility) + end end - end - context 'internal group visibility' do - let(:group_visibility) { Gitlab::VisibilityLevel::INTERNAL } - let(:visibility) { Gitlab::VisibilityLevel::PUBLIC } + context 'public group visibility' do + let(:group_visibility) { Gitlab::VisibilityLevel::PUBLIC } + let(:visibility) { Gitlab::VisibilityLevel::PRIVATE } - it 'uses the group visibility' do - expect(restorer.restore).to eq(true) - expect(restorer.project.visibility_level).to eq(group_visibility) + it 'uses the project visibility' do + expect(restorer.restore).to eq(true) + expect(restorer.project.visibility_level).to eq(visibility) + end end - context 'with restricted internal visibility' do - it 'sets private visibility' do - stub_application_setting(restricted_visibility_levels: [Gitlab::VisibilityLevel::INTERNAL]) + context 'internal group visibility' do + let(:group_visibility) { Gitlab::VisibilityLevel::INTERNAL } + let(:visibility) { Gitlab::VisibilityLevel::PUBLIC } + it 'uses the group visibility' do expect(restorer.restore).to eq(true) - expect(restorer.project.visibility_level).to eq(Gitlab::VisibilityLevel::PRIVATE) + expect(restorer.project.visibility_level).to eq(group_visibility) + end + + context 'with restricted internal visibility' do + it 'sets private visibility' do + stub_application_setting(restricted_visibility_levels: [Gitlab::VisibilityLevel::INTERNAL]) + + expect(restorer.restore).to eq(true) + expect(restorer.project.visibility_level).to eq(Gitlab::VisibilityLevel::PRIVATE) + end end end end @@ -1008,24 +1003,35 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i let(:user2) { create(:user) } let(:project_members) do [ - { - "id" => 2, - "access_level" => 40, - "source_type" => "Project", - "notification_level" => 3, - "user" => { - "id" => user2.id, - "email" => user2.email, - "username" => 'test' - } - } + [ + { + "id" => 2, + "access_level" => 40, + "source_type" => "Project", + "notification_level" => 3, + "user" => { + "id" => user2.id, + "email" => user2.email, + "username" => 'test' + } + }, + 0 + ] ] end - let(:tree_hash) { { 'project_members' => project_members } } - before do project.add_maintainer(user) + + setup_import_export_config('light') + + allow_next_instance_of(Gitlab::ImportExport::Json::NdjsonReader) do |relation_reader| + allow(relation_reader).to receive(:consume_relation).and_call_original + + allow(relation_reader).to receive(:consume_relation) + .with('project', 'project_members') + .and_return(project_members) + end end it 'restores project members' do @@ -1045,7 +1051,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i before do setup_import_export_config('with_invalid_records') - setup_reader(reader) + setup_reader subject end @@ -1138,13 +1144,5 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i end end - context 'enable ndjson import' do - it_behaves_like 'project tree restorer work properly', :legacy_reader, true - - it_behaves_like 'project tree restorer work properly', :ndjson_reader, true - end - - context 'disable ndjson import' do - it_behaves_like 'project tree restorer work properly', :legacy_reader, false - end + it_behaves_like 'project tree restorer work properly' end diff --git a/spec/lib/gitlab/import_export/project/tree_saver_spec.rb b/spec/lib/gitlab/import_export/project/tree_saver_spec.rb index b87992c4594..4166eba4e8e 100644 --- a/spec/lib/gitlab/import_export/project/tree_saver_spec.rb +++ b/spec/lib/gitlab/import_export/project/tree_saver_spec.rb @@ -9,28 +9,21 @@ RSpec.describe Gitlab::ImportExport::Project::TreeSaver, :with_license, feature_ let_it_be(:group) { create(:group) } let_it_be(:project) { setup_project } - shared_examples 'saves project tree successfully' do |ndjson_enabled| + shared_examples 'saves project tree successfully' do include ImportExport::CommonUtil - subject { get_json(full_path, exportable_path, relation_name, ndjson_enabled) } + subject { get_json(full_path, exportable_path, relation_name) } describe 'saves project tree attributes' do let_it_be(:shared) { project.import_export_shared } let(:relation_name) { :projects } - let_it_be(:full_path) do - if ndjson_enabled - File.join(shared.export_path, 'tree') - else - File.join(shared.export_path, Gitlab::ImportExport.project_filename) - end - end + let_it_be(:full_path) { File.join(shared.export_path, 'tree') } before_all do RSpec::Mocks.with_temporary_scope do stub_all_feature_flags - stub_feature_flags(project_export_as_ndjson: ndjson_enabled) project.add_maintainer(user) @@ -300,13 +293,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeSaver, :with_license, feature_ let_it_be(:group) { create(:group) } let(:project) { setup_project } - let(:full_path) do - if ndjson_enabled - File.join(shared.export_path, 'tree') - else - File.join(shared.export_path, Gitlab::ImportExport.project_filename) - end - end + let(:full_path) { File.join(shared.export_path, 'tree') } let(:shared) { project.import_export_shared } let(:params) { {} } @@ -314,7 +301,6 @@ RSpec.describe Gitlab::ImportExport::Project::TreeSaver, :with_license, feature_ let(:project_tree_saver ) { described_class.new(project: project, current_user: user, shared: shared, params: params) } before do - stub_feature_flags(project_export_as_ndjson: ndjson_enabled) project.add_maintainer(user) FileUtils.rm_rf(export_path) @@ -425,13 +411,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeSaver, :with_license, feature_ end end - context 'with JSON' do - it_behaves_like "saves project tree successfully", false - end - - context 'with NDJSON' do - it_behaves_like "saves project tree successfully", true - end + it_behaves_like "saves project tree successfully" context 'when streaming has to retry', :aggregate_failures do let(:shared) { double('shared', export_path: exportable_path) } diff --git a/spec/lib/gitlab/import_export/safe_model_attributes.yml b/spec/lib/gitlab/import_export/safe_model_attributes.yml index 2384baabb6b..854909fd592 100644 --- a/spec/lib/gitlab/import_export/safe_model_attributes.yml +++ b/spec/lib/gitlab/import_export/safe_model_attributes.yml @@ -1060,3 +1060,21 @@ ResourceIterationEvent: - action Iterations::Cadence: - title +ApprovalProjectRule: + - approvals_required + - name + - rule_type + - scanners + - vulnerabilities_allowed + - severity_levels + - report_type + - vulnerability_states + - orchestration_policy_idx + - applies_to_all_protected_branches +ApprovalProjectRulesUser: + - user_id + - approval_project_rule_id +ApprovalProjectRulesProtectedBranch: + - protected_branch_id + - approval_project_rule_id + - branch_name diff --git a/spec/lib/gitlab/jwt_authenticatable_spec.rb b/spec/lib/gitlab/jwt_authenticatable_spec.rb index 92d5feceb75..9a06f9b91df 100644 --- a/spec/lib/gitlab/jwt_authenticatable_spec.rb +++ b/spec/lib/gitlab/jwt_authenticatable_spec.rb @@ -172,11 +172,17 @@ RSpec.describe Gitlab::JwtAuthenticatable do end it 'raises an error if iat is invalid' do - encoded_message = JWT.encode(payload.merge(iat: 'wrong'), test_class.secret, 'HS256') + encoded_message = JWT.encode(payload.merge(iat: Time.current.to_i + 1), test_class.secret, 'HS256') expect { test_class.decode_jwt(encoded_message, iat_after: true) }.to raise_error(JWT::DecodeError) end + it 'raises InvalidPayload exception if iat is a string' do + expect do + JWT.encode(payload.merge(iat: 'wrong'), test_class.secret, 'HS256') + end.to raise_error(JWT::InvalidPayload) + end + it 'raises an error if iat is absent' do encoded_message = JWT.encode(payload, test_class.secret, 'HS256') diff --git a/spec/lib/gitlab/kas/user_access_spec.rb b/spec/lib/gitlab/kas/user_access_spec.rb index 8795ad565d0..a8296d23a18 100644 --- a/spec/lib/gitlab/kas/user_access_spec.rb +++ b/spec/lib/gitlab/kas/user_access_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Kas::UserAccess, feature_category: :kubernetes_management do +RSpec.describe Gitlab::Kas::UserAccess, feature_category: :deployment_management do describe '.enabled?' do subject { described_class.enabled? } diff --git a/spec/lib/gitlab/kubernetes/helm/api_spec.rb b/spec/lib/gitlab/kubernetes/helm/api_spec.rb deleted file mode 100644 index e022f5bd912..00000000000 --- a/spec/lib/gitlab/kubernetes/helm/api_spec.rb +++ /dev/null @@ -1,269 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Kubernetes::Helm::API do - let(:client) { double('kubernetes client') } - let(:helm) { described_class.new(client) } - let(:gitlab_namespace) { Gitlab::Kubernetes::Helm::NAMESPACE } - let(:gitlab_namespace_labels) { Gitlab::Kubernetes::Helm::NAMESPACE_LABELS } - let(:namespace) { Gitlab::Kubernetes::Namespace.new(gitlab_namespace, client, labels: gitlab_namespace_labels) } - let(:application_name) { 'app-name' } - let(:rbac) { false } - let(:files) { {} } - - let(:command) do - Gitlab::Kubernetes::Helm::V2::InstallCommand.new( - name: application_name, - chart: 'chart-name', - rbac: rbac, - files: files - ) - end - - subject { helm } - - before do - allow(Gitlab::Kubernetes::Namespace).to( - receive(:new).with(gitlab_namespace, client, labels: gitlab_namespace_labels).and_return(namespace) - ) - allow(client).to receive(:create_config_map) - end - - describe '#initialize' do - it 'creates a namespace object' do - expect(Gitlab::Kubernetes::Namespace).to( - receive(:new).with(gitlab_namespace, client, labels: gitlab_namespace_labels) - ) - - subject - end - end - - describe '#uninstall' do - before do - allow(client).to receive(:create_pod).and_return(nil) - allow(client).to receive(:get_config_map).and_return(nil) - allow(client).to receive(:create_config_map).and_return(nil) - allow(client).to receive(:delete_pod).and_return(nil) - allow(namespace).to receive(:ensure_exists!).once - end - - it 'ensures the namespace exists before creating the POD' do - expect(namespace).to receive(:ensure_exists!).once.ordered - expect(client).to receive(:create_pod).once.ordered - - subject.uninstall(command) - end - - it 'removes an existing pod before installing' do - expect(client).to receive(:delete_pod).with('install-app-name', 'gitlab-managed-apps').once.ordered - expect(client).to receive(:create_pod).once.ordered - - subject.uninstall(command) - end - - context 'with a ConfigMap' do - let(:resource) { Gitlab::Kubernetes::ConfigMap.new(application_name, files).generate } - - it 'creates a ConfigMap on kubeclient' do - expect(client).to receive(:create_config_map).with(resource).once - - subject.install(command) - end - - context 'config map already exists' do - before do - expect(client).to receive(:get_config_map).with("values-content-configuration-#{application_name}", gitlab_namespace).and_return(resource) - end - - it 'updates the config map' do - expect(client).to receive(:update_config_map).with(resource).once - - subject.install(command) - end - end - end - end - - describe '#install' do - before do - allow(client).to receive(:create_pod).and_return(nil) - allow(client).to receive(:get_config_map).and_return(nil) - allow(client).to receive(:create_config_map).and_return(nil) - allow(client).to receive(:create_service_account).and_return(nil) - allow(client).to receive(:delete_pod).and_return(nil) - allow(namespace).to receive(:ensure_exists!).once - end - - it 'ensures the namespace exists before creating the POD' do - expect(namespace).to receive(:ensure_exists!).once.ordered - expect(client).to receive(:create_pod).once.ordered - - subject.install(command) - end - - it 'removes an existing pod before installing' do - expect(client).to receive(:delete_pod).with('install-app-name', 'gitlab-managed-apps').once.ordered - expect(client).to receive(:create_pod).once.ordered - - subject.install(command) - end - - context 'with a ConfigMap' do - let(:resource) { Gitlab::Kubernetes::ConfigMap.new(application_name, files).generate } - - it 'creates a ConfigMap on kubeclient' do - expect(client).to receive(:create_config_map).with(resource).once - - subject.install(command) - end - - context 'config map already exists' do - before do - expect(client).to receive(:get_config_map).with("values-content-configuration-#{application_name}", gitlab_namespace).and_return(resource) - end - - it 'updates the config map' do - expect(client).to receive(:update_config_map).with(resource).once - - subject.install(command) - end - end - end - - context 'without a service account' do - it 'does not create a service account on kubeclient' do - expect(client).not_to receive(:create_service_account) - expect(client).not_to receive(:update_cluster_role_binding) - - subject.install(command) - end - end - - context 'with a service account' do - let(:command) { Gitlab::Kubernetes::Helm::V2::InitCommand.new(name: application_name, files: files, rbac: rbac) } - - context 'rbac-enabled cluster' do - let(:rbac) { true } - - let(:service_account_resource) do - Kubeclient::Resource.new(metadata: { name: 'tiller', namespace: 'gitlab-managed-apps' }) - end - - let(:cluster_role_binding_resource) do - Kubeclient::Resource.new( - metadata: { name: 'tiller-admin' }, - roleRef: { apiGroup: 'rbac.authorization.k8s.io', kind: 'ClusterRole', name: 'cluster-admin' }, - subjects: [{ kind: 'ServiceAccount', name: 'tiller', namespace: 'gitlab-managed-apps' }] - ) - end - - context 'service account does not exist' do - before do - expect(client).to receive(:get_service_account).with('tiller', 'gitlab-managed-apps').and_raise(Kubeclient::ResourceNotFoundError.new(404, 'Not found', nil)) - end - - it 'creates a service account, followed the cluster role binding on kubeclient' do - expect(client).to receive(:create_service_account).with(service_account_resource).once.ordered - expect(client).to receive(:update_cluster_role_binding).with(cluster_role_binding_resource).once.ordered - - subject.install(command) - end - end - - context 'service account already exists' do - before do - expect(client).to receive(:get_service_account).with('tiller', 'gitlab-managed-apps').and_return(service_account_resource) - end - - it 'updates the service account, followed by creating the cluster role binding' do - expect(client).to receive(:update_service_account).with(service_account_resource).once.ordered - expect(client).to receive(:update_cluster_role_binding).with(cluster_role_binding_resource).once.ordered - - subject.install(command) - end - end - - context 'a non-404 error is thrown' do - before do - expect(client).to receive(:get_service_account).with('tiller', 'gitlab-managed-apps').and_raise(Kubeclient::HttpError.new(401, 'Unauthorized', nil)) - end - - it 'raises an error' do - expect { subject.install(command) }.to raise_error(Kubeclient::HttpError) - end - end - end - - context 'legacy abac cluster' do - it 'does not create a service account on kubeclient' do - expect(client).not_to receive(:create_service_account) - expect(client).not_to receive(:update_cluster_role_binding) - - subject.install(command) - end - end - end - end - - describe '#status' do - let(:phase) { Gitlab::Kubernetes::Pod::RUNNING } - let(:pod) { Kubeclient::Resource.new(status: { phase: phase }) } # partial representation - - it 'fetches POD phase from kubernetes cluster' do - expect(client).to receive(:get_pod).with(command.pod_name, gitlab_namespace).once.and_return(pod) - - expect(subject.status(command.pod_name)).to eq(phase) - end - end - - describe '#log' do - let(:log) { 'some output' } - let(:response) { RestClient::Response.new(log) } - - it 'fetches POD phase from kubernetes cluster' do - expect(client).to receive(:get_pod_log).with(command.pod_name, gitlab_namespace).once.and_return(response) - - expect(subject.log(command.pod_name)).to eq(log) - end - end - - describe '#delete_pod!' do - it 'deletes the POD from kubernetes cluster' do - expect(client).to receive(:delete_pod).with('install-app-name', 'gitlab-managed-apps').once - - subject.delete_pod!('install-app-name') - end - - context 'when the resource being deleted does not exist' do - it 'catches the error' do - expect(client).to receive(:delete_pod).with('install-app-name', 'gitlab-managed-apps') - .and_raise(Kubeclient::ResourceNotFoundError.new(404, 'Not found', nil)) - - subject.delete_pod!('install-app-name') - end - end - end - - describe '#get_config_map' do - before do - allow(namespace).to receive(:ensure_exists!).once - allow(client).to receive(:get_config_map).and_return(nil) - end - - it 'ensures the namespace exists before retrieving the config map' do - expect(namespace).to receive(:ensure_exists!).once - - subject.get_config_map('example-config-map-name') - end - - it 'gets the config map on kubeclient' do - expect(client).to receive(:get_config_map) - .with('example-config-map-name', namespace.name) - .once - - subject.get_config_map('example-config-map-name') - end - end -end diff --git a/spec/lib/gitlab/kubernetes/helm/pod_spec.rb b/spec/lib/gitlab/kubernetes/helm/pod_spec.rb deleted file mode 100644 index 8aa755bffce..00000000000 --- a/spec/lib/gitlab/kubernetes/helm/pod_spec.rb +++ /dev/null @@ -1,89 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Kubernetes::Helm::Pod do - describe '#generate' do - using RSpec::Parameterized::TableSyntax - - where(:helm_major_version, :expected_helm_version, :expected_command_env) do - 2 | '2.17.0' | [:TILLER_NAMESPACE] - 3 | '3.2.4' | nil - end - - with_them do - let(:cluster) { create(:cluster, helm_major_version: helm_major_version) } - let(:app) { create(:clusters_applications_knative, cluster: cluster) } - let(:command) { app.install_command } - let(:namespace) { Gitlab::Kubernetes::Helm::NAMESPACE } - let(:service_account_name) { nil } - - subject { described_class.new(command, namespace, service_account_name: service_account_name) } - - context 'with a command' do - it 'generates a Kubeclient::Resource' do - expect(subject.generate).to be_a_kind_of(Kubeclient::Resource) - end - - it 'generates the appropriate metadata' do - metadata = subject.generate.metadata - expect(metadata.name).to eq("install-#{app.name}") - expect(metadata.namespace).to eq('gitlab-managed-apps') - expect(metadata.labels['gitlab.org/action']).to eq('install') - expect(metadata.labels['gitlab.org/application']).to eq(app.name) - end - - it 'generates a container spec' do - spec = subject.generate.spec - expect(spec.containers.count).to eq(1) - end - - it 'generates the appropriate specifications for the container' do - container = subject.generate.spec.containers.first - expect(container.name).to eq('helm') - expect(container.image).to eq("registry.gitlab.com/gitlab-org/cluster-integration/helm-install-image/releases/#{expected_helm_version}-kube-1.13.12-alpine-3.12") - expect(container.env.map(&:name)).to include(:HELM_VERSION, :COMMAND_SCRIPT, *expected_command_env) - expect(container.command).to match_array(["/bin/sh"]) - expect(container.args).to match_array(["-c", "$(COMMAND_SCRIPT)"]) - end - - it 'includes a never restart policy' do - spec = subject.generate.spec - expect(spec.restartPolicy).to eq('Never') - end - - it 'includes volumes for the container' do - container = subject.generate.spec.containers.first - expect(container.volumeMounts.first['name']).to eq('configuration-volume') - expect(container.volumeMounts.first['mountPath']).to eq("/data/helm/#{app.name}/config") - end - - it 'includes a volume inside the specification' do - spec = subject.generate.spec - expect(spec.volumes.first['name']).to eq('configuration-volume') - end - - it 'mounts configMap specification in the volume' do - volume = subject.generate.spec.volumes.first - expect(volume.configMap['name']).to eq("values-content-configuration-#{app.name}") - expect(volume.configMap['items'].first['key']).to eq(:'values.yaml') - expect(volume.configMap['items'].first['path']).to eq(:'values.yaml') - end - - it 'has no serviceAccountName' do - spec = subject.generate.spec - expect(spec.serviceAccountName).to be_nil - end - - context 'with a service_account_name' do - let(:service_account_name) { 'sa' } - - it 'uses the serviceAccountName provided' do - spec = subject.generate.spec - expect(spec.serviceAccountName).to eq(service_account_name) - end - end - end - end - end -end diff --git a/spec/lib/gitlab/kubernetes/helm/v2/base_command_spec.rb b/spec/lib/gitlab/kubernetes/helm/v2/base_command_spec.rb deleted file mode 100644 index 3d2b36b9094..00000000000 --- a/spec/lib/gitlab/kubernetes/helm/v2/base_command_spec.rb +++ /dev/null @@ -1,50 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Kubernetes::Helm::V2::BaseCommand do - subject(:base_command) do - test_class.new(rbac) - end - - let(:application) { create(:clusters_applications_helm) } - let(:rbac) { false } - - let(:test_class) do - Class.new(described_class) do - def initialize(rbac) - super( - name: 'test-class-name', - rbac: rbac, - files: { some: 'value' } - ) - end - end - end - - describe 'HELM_VERSION' do - subject { described_class::HELM_VERSION } - - it { is_expected.to match /^2\.\d+\.\d+$/ } - end - - describe '#env' do - subject { base_command.env } - - it { is_expected.to include(TILLER_NAMESPACE: 'gitlab-managed-apps') } - end - - it_behaves_like 'helm command generator' do - let(:commands) { '' } - end - - describe '#pod_name' do - subject { base_command.pod_name } - - it { is_expected.to eq('install-test-class-name') } - end - - it_behaves_like 'helm command' do - let(:command) { base_command } - end -end diff --git a/spec/lib/gitlab/kubernetes/helm/v2/certificate_spec.rb b/spec/lib/gitlab/kubernetes/helm/v2/certificate_spec.rb deleted file mode 100644 index 698b88c9fa1..00000000000 --- a/spec/lib/gitlab/kubernetes/helm/v2/certificate_spec.rb +++ /dev/null @@ -1,28 +0,0 @@ -# frozen_string_literal: true -require 'fast_spec_helper' - -RSpec.describe Gitlab::Kubernetes::Helm::V2::Certificate do - describe '.generate_root' do - subject { described_class.generate_root } - - it 'generates a root CA that expires a long way in the future' do - expect(subject.cert.not_after).to be > 999.years.from_now - end - end - - describe '#issue' do - subject { described_class.generate_root.issue } - - it 'generates a cert that expires soon' do - expect(subject.cert.not_after).to be < 60.minutes.from_now - end - - context 'passing in INFINITE_EXPIRY' do - subject { described_class.generate_root.issue(expires_in: described_class::INFINITE_EXPIRY) } - - it 'generates a cert that expires a long way in the future' do - expect(subject.cert.not_after).to be > 999.years.from_now - end - end - end -end diff --git a/spec/lib/gitlab/kubernetes/helm/v2/delete_command_spec.rb b/spec/lib/gitlab/kubernetes/helm/v2/delete_command_spec.rb deleted file mode 100644 index 4a3a41dba4a..00000000000 --- a/spec/lib/gitlab/kubernetes/helm/v2/delete_command_spec.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Kubernetes::Helm::V2::DeleteCommand do - subject(:delete_command) { described_class.new(name: app_name, rbac: rbac, files: files) } - - let(:app_name) { 'app-name' } - let(:rbac) { true } - let(:files) { {} } - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - export HELM_HOST="localhost:44134" - tiller -listen ${HELM_HOST} -alsologtostderr & - helm init --client-only - helm delete --purge app-name - EOS - end - end - - describe '#pod_name' do - subject { delete_command.pod_name } - - it { is_expected.to eq('uninstall-app-name') } - end - - it_behaves_like 'helm command' do - let(:command) { delete_command } - end - - describe '#delete_command' do - it 'deletes the release' do - expect(subject.delete_command).to eq('helm delete --purge app-name') - end - end -end diff --git a/spec/lib/gitlab/kubernetes/helm/v2/init_command_spec.rb b/spec/lib/gitlab/kubernetes/helm/v2/init_command_spec.rb deleted file mode 100644 index 8ae78ada15c..00000000000 --- a/spec/lib/gitlab/kubernetes/helm/v2/init_command_spec.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Kubernetes::Helm::V2::InitCommand do - subject(:init_command) { described_class.new(name: application.name, files: files, rbac: rbac) } - - let(:application) { create(:clusters_applications_helm) } - let(:rbac) { false } - let(:files) { {} } - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - helm init --tiller-tls --tiller-tls-verify --tls-ca-cert /data/helm/helm/config/ca.pem --tiller-tls-cert /data/helm/helm/config/cert.pem --tiller-tls-key /data/helm/helm/config/key.pem - EOS - end - end - - context 'on a rbac-enabled cluster' do - let(:rbac) { true } - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - helm init --tiller-tls --tiller-tls-verify --tls-ca-cert /data/helm/helm/config/ca.pem --tiller-tls-cert /data/helm/helm/config/cert.pem --tiller-tls-key /data/helm/helm/config/key.pem --service-account tiller - EOS - end - end - end - - it_behaves_like 'helm command' do - let(:command) { init_command } - end -end diff --git a/spec/lib/gitlab/kubernetes/helm/v2/install_command_spec.rb b/spec/lib/gitlab/kubernetes/helm/v2/install_command_spec.rb deleted file mode 100644 index 250d1a82e7a..00000000000 --- a/spec/lib/gitlab/kubernetes/helm/v2/install_command_spec.rb +++ /dev/null @@ -1,183 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Kubernetes::Helm::V2::InstallCommand do - subject(:install_command) do - described_class.new( - name: 'app-name', - chart: 'chart-name', - rbac: rbac, - files: files, - version: version, - repository: repository, - preinstall: preinstall, - postinstall: postinstall - ) - end - - let(:files) { { 'ca.pem': 'some file content' } } - let(:repository) { 'https://repository.example.com' } - let(:rbac) { false } - let(:version) { '1.2.3' } - let(:preinstall) { nil } - let(:postinstall) { nil } - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - export HELM_HOST="localhost:44134" - tiller -listen ${HELM_HOST} -alsologtostderr & - helm init --client-only - helm repo add app-name https://repository.example.com - helm repo update - #{helm_install_comand} - EOS - end - - let(:helm_install_comand) do - <<~EOS.squish - helm upgrade app-name chart-name - --install - --atomic - --cleanup-on-fail - --reset-values - --version 1.2.3 - --set rbac.create\\=false,rbac.enabled\\=false - --namespace gitlab-managed-apps - -f /data/helm/app-name/config/values.yaml - EOS - end - end - - context 'when rbac is true' do - let(:rbac) { true } - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - export HELM_HOST="localhost:44134" - tiller -listen ${HELM_HOST} -alsologtostderr & - helm init --client-only - helm repo add app-name https://repository.example.com - helm repo update - #{helm_install_command} - EOS - end - - let(:helm_install_command) do - <<~EOS.squish - helm upgrade app-name chart-name - --install - --atomic - --cleanup-on-fail - --reset-values - --version 1.2.3 - --set rbac.create\\=true,rbac.enabled\\=true - --namespace gitlab-managed-apps - -f /data/helm/app-name/config/values.yaml - EOS - end - end - end - - context 'when there is a pre-install script' do - let(:preinstall) { ['/bin/date', '/bin/true'] } - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - export HELM_HOST="localhost:44134" - tiller -listen ${HELM_HOST} -alsologtostderr & - helm init --client-only - helm repo add app-name https://repository.example.com - helm repo update - /bin/date - /bin/true - #{helm_install_command} - EOS - end - - let(:helm_install_command) do - <<~EOS.squish - helm upgrade app-name chart-name - --install - --atomic - --cleanup-on-fail - --reset-values - --version 1.2.3 - --set rbac.create\\=false,rbac.enabled\\=false - --namespace gitlab-managed-apps - -f /data/helm/app-name/config/values.yaml - EOS - end - end - end - - context 'when there is a post-install script' do - let(:postinstall) { ['/bin/date', "/bin/false\n"] } - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - export HELM_HOST="localhost:44134" - tiller -listen ${HELM_HOST} -alsologtostderr & - helm init --client-only - helm repo add app-name https://repository.example.com - helm repo update - #{helm_install_command} - /bin/date - /bin/false - EOS - end - - let(:helm_install_command) do - <<~EOS.squish - helm upgrade app-name chart-name - --install - --atomic - --cleanup-on-fail - --reset-values - --version 1.2.3 - --set rbac.create\\=false,rbac.enabled\\=false - --namespace gitlab-managed-apps - -f /data/helm/app-name/config/values.yaml - EOS - end - end - end - - context 'when there is no version' do - let(:version) { nil } - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - export HELM_HOST="localhost:44134" - tiller -listen ${HELM_HOST} -alsologtostderr & - helm init --client-only - helm repo add app-name https://repository.example.com - helm repo update - #{helm_install_command} - EOS - end - - let(:helm_install_command) do - <<~EOS.squish - helm upgrade app-name chart-name - --install - --atomic - --cleanup-on-fail - --reset-values - --set rbac.create\\=false,rbac.enabled\\=false - --namespace gitlab-managed-apps - -f /data/helm/app-name/config/values.yaml - EOS - end - end - end - - it_behaves_like 'helm command' do - let(:command) { install_command } - end -end diff --git a/spec/lib/gitlab/kubernetes/helm/v2/patch_command_spec.rb b/spec/lib/gitlab/kubernetes/helm/v2/patch_command_spec.rb deleted file mode 100644 index 98eb77d397c..00000000000 --- a/spec/lib/gitlab/kubernetes/helm/v2/patch_command_spec.rb +++ /dev/null @@ -1,87 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Kubernetes::Helm::V2::PatchCommand do - let(:files) { { 'ca.pem': 'some file content' } } - let(:repository) { 'https://repository.example.com' } - let(:rbac) { false } - let(:version) { '1.2.3' } - - subject(:patch_command) do - described_class.new( - name: 'app-name', - chart: 'chart-name', - rbac: rbac, - files: files, - version: version, - repository: repository - ) - end - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - export HELM_HOST="localhost:44134" - tiller -listen ${HELM_HOST} -alsologtostderr & - helm init --client-only - helm repo add app-name https://repository.example.com - helm repo update - #{helm_upgrade_comand} - EOS - end - - let(:helm_upgrade_comand) do - <<~EOS.squish - helm upgrade app-name chart-name - --reuse-values - --version 1.2.3 - --namespace gitlab-managed-apps - -f /data/helm/app-name/config/values.yaml - EOS - end - end - - context 'when rbac is true' do - let(:rbac) { true } - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - export HELM_HOST="localhost:44134" - tiller -listen ${HELM_HOST} -alsologtostderr & - helm init --client-only - helm repo add app-name https://repository.example.com - helm repo update - #{helm_upgrade_command} - EOS - end - - let(:helm_upgrade_command) do - <<~EOS.squish - helm upgrade app-name chart-name - --reuse-values - --version 1.2.3 - --namespace gitlab-managed-apps - -f /data/helm/app-name/config/values.yaml - EOS - end - end - end - - context 'when there is no version' do - let(:version) { nil } - - it { expect { patch_command }.to raise_error(ArgumentError, 'version is required') } - end - - describe '#pod_name' do - subject { patch_command.pod_name } - - it { is_expected.to eq 'install-app-name' } - end - - it_behaves_like 'helm command' do - let(:command) { patch_command } - end -end diff --git a/spec/lib/gitlab/kubernetes/helm/v2/reset_command_spec.rb b/spec/lib/gitlab/kubernetes/helm/v2/reset_command_spec.rb deleted file mode 100644 index 2a3a4cec2b0..00000000000 --- a/spec/lib/gitlab/kubernetes/helm/v2/reset_command_spec.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Kubernetes::Helm::V2::ResetCommand do - subject(:reset_command) { described_class.new(name: name, rbac: rbac, files: files) } - - let(:rbac) { true } - let(:name) { 'helm' } - let(:files) { {} } - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - export HELM_HOST="localhost:44134" - tiller -listen ${HELM_HOST} -alsologtostderr & - helm init --client-only - helm reset --force - EOS - end - end - - describe '#pod_name' do - subject { reset_command.pod_name } - - it { is_expected.to eq('uninstall-helm') } - end - - it_behaves_like 'helm command' do - let(:command) { reset_command } - end -end diff --git a/spec/lib/gitlab/kubernetes/helm/v3/base_command_spec.rb b/spec/lib/gitlab/kubernetes/helm/v3/base_command_spec.rb deleted file mode 100644 index ad5ff13b4c9..00000000000 --- a/spec/lib/gitlab/kubernetes/helm/v3/base_command_spec.rb +++ /dev/null @@ -1,44 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Kubernetes::Helm::V3::BaseCommand do - subject(:base_command) do - test_class.new(rbac) - end - - let(:application) { create(:clusters_applications_helm) } - let(:rbac) { false } - - let(:test_class) do - Class.new(described_class) do - def initialize(rbac) - super( - name: 'test-class-name', - rbac: rbac, - files: { some: 'value' } - ) - end - end - end - - describe 'HELM_VERSION' do - subject { described_class::HELM_VERSION } - - it { is_expected.to match /^3\.\d+\.\d+$/ } - end - - it_behaves_like 'helm command generator' do - let(:commands) { '' } - end - - describe '#pod_name' do - subject { base_command.pod_name } - - it { is_expected.to eq('install-test-class-name') } - end - - it_behaves_like 'helm command' do - let(:command) { base_command } - end -end diff --git a/spec/lib/gitlab/kubernetes/helm/v3/delete_command_spec.rb b/spec/lib/gitlab/kubernetes/helm/v3/delete_command_spec.rb deleted file mode 100644 index 63e7a8d2f25..00000000000 --- a/spec/lib/gitlab/kubernetes/helm/v3/delete_command_spec.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Kubernetes::Helm::V3::DeleteCommand do - subject(:delete_command) { described_class.new(name: app_name, rbac: rbac, files: files) } - - let(:app_name) { 'app-name' } - let(:rbac) { true } - let(:files) { {} } - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - helm uninstall app-name --namespace gitlab-managed-apps - EOS - end - end - - describe '#pod_name' do - subject { delete_command.pod_name } - - it { is_expected.to eq('uninstall-app-name') } - end - - it_behaves_like 'helm command' do - let(:command) { delete_command } - end - - describe '#delete_command' do - it 'deletes the release' do - expect(subject.delete_command).to eq('helm uninstall app-name --namespace gitlab-managed-apps') - end - end -end diff --git a/spec/lib/gitlab/kubernetes/helm/v3/install_command_spec.rb b/spec/lib/gitlab/kubernetes/helm/v3/install_command_spec.rb deleted file mode 100644 index 2bf1f713b3f..00000000000 --- a/spec/lib/gitlab/kubernetes/helm/v3/install_command_spec.rb +++ /dev/null @@ -1,168 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Kubernetes::Helm::V3::InstallCommand do - subject(:install_command) do - described_class.new( - name: 'app-name', - chart: 'chart-name', - rbac: rbac, - files: files, - version: version, - repository: repository, - preinstall: preinstall, - postinstall: postinstall - ) - end - - let(:files) { { 'ca.pem': 'some file content' } } - let(:repository) { 'https://repository.example.com' } - let(:rbac) { false } - let(:version) { '1.2.3' } - let(:preinstall) { nil } - let(:postinstall) { nil } - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - helm repo add app-name https://repository.example.com - helm repo update - #{helm_install_comand} - EOS - end - - let(:helm_install_comand) do - <<~EOS.squish - helm upgrade app-name chart-name - --install - --atomic - --cleanup-on-fail - --reset-values - --version 1.2.3 - --set rbac.create\\=false,rbac.enabled\\=false - --namespace gitlab-managed-apps - -f /data/helm/app-name/config/values.yaml - EOS - end - end - - context 'when rbac is true' do - let(:rbac) { true } - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - helm repo add app-name https://repository.example.com - helm repo update - #{helm_install_command} - EOS - end - - let(:helm_install_command) do - <<~EOS.squish - helm upgrade app-name chart-name - --install - --atomic - --cleanup-on-fail - --reset-values - --version 1.2.3 - --set rbac.create\\=true,rbac.enabled\\=true - --namespace gitlab-managed-apps - -f /data/helm/app-name/config/values.yaml - EOS - end - end - end - - context 'when there is a pre-install script' do - let(:preinstall) { ['/bin/date', '/bin/true'] } - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - helm repo add app-name https://repository.example.com - helm repo update - /bin/date - /bin/true - #{helm_install_command} - EOS - end - - let(:helm_install_command) do - <<~EOS.squish - helm upgrade app-name chart-name - --install - --atomic - --cleanup-on-fail - --reset-values - --version 1.2.3 - --set rbac.create\\=false,rbac.enabled\\=false - --namespace gitlab-managed-apps - -f /data/helm/app-name/config/values.yaml - EOS - end - end - end - - context 'when there is a post-install script' do - let(:postinstall) { ['/bin/date', "/bin/false\n"] } - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - helm repo add app-name https://repository.example.com - helm repo update - #{helm_install_command} - /bin/date - /bin/false - EOS - end - - let(:helm_install_command) do - <<~EOS.squish - helm upgrade app-name chart-name - --install - --atomic - --cleanup-on-fail - --reset-values - --version 1.2.3 - --set rbac.create\\=false,rbac.enabled\\=false - --namespace gitlab-managed-apps - -f /data/helm/app-name/config/values.yaml - EOS - end - end - end - - context 'when there is no version' do - let(:version) { nil } - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - helm repo add app-name https://repository.example.com - helm repo update - #{helm_install_command} - EOS - end - - let(:helm_install_command) do - <<~EOS.squish - helm upgrade app-name chart-name - --install - --atomic - --cleanup-on-fail - --reset-values - --set rbac.create\\=false,rbac.enabled\\=false - --namespace gitlab-managed-apps - -f /data/helm/app-name/config/values.yaml - EOS - end - end - end - - it_behaves_like 'helm command' do - let(:command) { install_command } - end -end diff --git a/spec/lib/gitlab/kubernetes/helm/v3/patch_command_spec.rb b/spec/lib/gitlab/kubernetes/helm/v3/patch_command_spec.rb deleted file mode 100644 index 2f22e0f2e77..00000000000 --- a/spec/lib/gitlab/kubernetes/helm/v3/patch_command_spec.rb +++ /dev/null @@ -1,81 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Kubernetes::Helm::V3::PatchCommand do - let(:files) { { 'ca.pem': 'some file content' } } - let(:repository) { 'https://repository.example.com' } - let(:rbac) { false } - let(:version) { '1.2.3' } - - subject(:patch_command) do - described_class.new( - name: 'app-name', - chart: 'chart-name', - rbac: rbac, - files: files, - version: version, - repository: repository - ) - end - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - helm repo add app-name https://repository.example.com - helm repo update - #{helm_upgrade_comand} - EOS - end - - let(:helm_upgrade_comand) do - <<~EOS.squish - helm upgrade app-name chart-name - --reuse-values - --version 1.2.3 - --namespace gitlab-managed-apps - -f /data/helm/app-name/config/values.yaml - EOS - end - end - - context 'when rbac is true' do - let(:rbac) { true } - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - helm repo add app-name https://repository.example.com - helm repo update - #{helm_upgrade_command} - EOS - end - - let(:helm_upgrade_command) do - <<~EOS.squish - helm upgrade app-name chart-name - --reuse-values - --version 1.2.3 - --namespace gitlab-managed-apps - -f /data/helm/app-name/config/values.yaml - EOS - end - end - end - - context 'when there is no version' do - let(:version) { nil } - - it { expect { patch_command }.to raise_error(ArgumentError, 'version is required') } - end - - describe '#pod_name' do - subject { patch_command.pod_name } - - it { is_expected.to eq 'install-app-name' } - end - - it_behaves_like 'helm command' do - let(:command) { patch_command } - end -end diff --git a/spec/lib/gitlab/legacy_github_import/user_formatter_spec.rb b/spec/lib/gitlab/legacy_github_import/user_formatter_spec.rb index bc127f74e84..0844ab7eccc 100644 --- a/spec/lib/gitlab/legacy_github_import/user_formatter_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/user_formatter_spec.rb @@ -5,14 +5,15 @@ require 'spec_helper' RSpec.describe Gitlab::LegacyGithubImport::UserFormatter do let(:client) { double } let(:octocat) { { id: 123456, login: 'octocat', email: 'octocat@example.com' } } + let(:gitea_ghost) { { id: -1, login: 'Ghost', email: '' } } - subject(:user) { described_class.new(client, octocat) } + describe '#gitlab_id' do + subject(:user) { described_class.new(client, octocat) } - before do - allow(client).to receive(:user).and_return(octocat) - end + before do + allow(client).to receive(:user).and_return(octocat) + end - describe '#gitlab_id' do context 'when GitHub user is a GitLab user' do it 'return GitLab user id when user associated their account with GitHub' do gl_user = create(:omniauth_user, extern_uid: octocat[:id], provider: 'github') @@ -51,4 +52,16 @@ RSpec.describe Gitlab::LegacyGithubImport::UserFormatter do expect(user.gitlab_id).to be_nil end end + + describe '.email' do + subject(:user) { described_class.new(client, gitea_ghost) } + + before do + allow(client).to receive(:user).and_return(gitea_ghost) + end + + it 'assigns a dummy email address when user is a Ghost gitea user' do + expect(subject.send(:email)).to eq described_class::GITEA_GHOST_EMAIL + end + end end diff --git a/spec/lib/gitlab/metrics/subscribers/action_cable_spec.rb b/spec/lib/gitlab/metrics/subscribers/action_cable_spec.rb index 08437920e0c..77c42f57f3c 100644 --- a/spec/lib/gitlab/metrics/subscribers/action_cable_spec.rb +++ b/spec/lib/gitlab/metrics/subscribers/action_cable_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Metrics::Subscribers::ActionCable, :request_store do +RSpec.describe Gitlab::Metrics::Subscribers::ActionCable, :request_store, feature_category: :application_performance do let(:subscriber) { described_class.new } let(:counter) { double(:counter) } let(:data) { { 'result' => { 'data' => { 'event' => 'updated' } } } } @@ -55,7 +55,6 @@ RSpec.describe Gitlab::Metrics::Subscribers::ActionCable, :request_store do { event: :updated } end - let(:broadcasting) { 'issues:Z2lkOi8vZ2l0bGFiL0lzc3VlLzQ0Ng' } let(:payload) do { broadcasting: broadcasting, @@ -64,14 +63,40 @@ RSpec.describe Gitlab::Metrics::Subscribers::ActionCable, :request_store do } end - it 'tracks the broadcast event' do + before do allow(::Gitlab::Metrics).to receive(:counter).with( :action_cable_broadcasts_total, /broadcast/ ).and_return(counter) + end - expect(counter).to receive(:increment) + context 'when broadcast is for a GraphQL event' do + let(:broadcasting) { 'graphql-event::issuableEpicUpdated:issuableId:Z2lkOi8vZ2l0bGFiL0lzc3VlLzM' } + + it 'tracks the event with broadcasting set to event topic' do + expect(counter).to receive(:increment).with({ broadcasting: 'graphql-event:issuableEpicUpdated' }) + + subscriber.broadcast(event) + end + end + + context 'when broadcast is for a GraphQL channel subscription' do + let(:broadcasting) { 'graphql-subscription:09ae595a-45c4-4ae0-b765-4e503203211d' } + + it 'strips out subscription ID from broadcasting' do + expect(counter).to receive(:increment).with({ broadcasting: 'graphql-subscription' }) + + subscriber.broadcast(event) + end + end + + context 'when broadcast is something else' do + let(:broadcasting) { 'unknown-topic' } + + it 'tracks the event as "other"' do + expect(counter).to receive(:increment).with({ broadcasting: 'other' }) - subscriber.broadcast(event) + subscriber.broadcast(event) + end end end diff --git a/spec/lib/gitlab/metrics/subscribers/active_record_spec.rb b/spec/lib/gitlab/metrics/subscribers/active_record_spec.rb index 7ce5cbec18d..afb029a96cb 100644 --- a/spec/lib/gitlab/metrics/subscribers/active_record_spec.rb +++ b/spec/lib/gitlab/metrics/subscribers/active_record_spec.rb @@ -226,7 +226,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::ActiveRecord do # Emulate Marginalia pre-pending comments def sql(query, comments: true) - if comments && !%w[BEGIN COMMIT].include?(query) + if comments "/*application:web,controller:badges,action:pipeline,correlation_id:01EYN39K9VMJC56Z7808N7RSRH*/ #{query}" else query @@ -244,8 +244,9 @@ RSpec.describe Gitlab::Metrics::Subscribers::ActiveRecord do 'SQL' | 'UPDATE users SET admin = true WHERE id = 10' | true | true | false 'CACHE' | 'SELECT * FROM users WHERE id = 10' | true | false | true 'SCHEMA' | "SELECT attr.attname FROM pg_attribute attr INNER JOIN pg_constraint cons ON attr.attrelid = cons.conrelid AND attr.attnum = any(cons.conkey) WHERE cons.contype = 'p' AND cons.conrelid = '\"projects\"'::regclass" | false | false | false - nil | 'BEGIN' | false | false | false - nil | 'COMMIT' | false | false | false + 'TRANSACTION' | 'BEGIN' | false | false | false + 'TRANSACTION' | 'COMMIT' | false | false | false + 'TRANSACTION' | 'ROLLBACK' | false | false | false end with_them do @@ -291,7 +292,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::ActiveRecord do # Emulate Marginalia pre-pending comments def sql(query, comments: true) - if comments && !%w[BEGIN COMMIT].include?(query) + if comments "/*application:web,controller:badges,action:pipeline,correlation_id:01EYN39K9VMJC56Z7808N7RSRH*/ #{query}" else query @@ -313,8 +314,9 @@ RSpec.describe Gitlab::Metrics::Subscribers::ActiveRecord do 'CACHE' | 'SELECT pg_last_wal_replay_lsn()::text AS location' | true | false | true | true 'CACHE' | 'SELECT * FROM users WHERE id = 10' | true | false | true | false 'SCHEMA' | "SELECT attr.attname FROM pg_attribute attr INNER JOIN pg_constraint cons ON attr.attrelid = cons.conrelid AND attr.attnum = any(cons.conkey) WHERE cons.contype = 'p' AND cons.conrelid = '\"projects\"'::regclass" | false | false | false | false - nil | 'BEGIN' | false | false | false | false - nil | 'COMMIT' | false | false | false | false + 'TRANSACTION' | 'BEGIN' | false | false | false | false + 'TRANSACTION' | 'COMMIT' | false | false | false | false + 'TRANSACTION' | 'ROLLBACK' | false | false | false | false end with_them do diff --git a/spec/lib/gitlab/metrics/subscribers/external_http_spec.rb b/spec/lib/gitlab/metrics/subscribers/external_http_spec.rb index e489ac97b9c..18a5d2c2c3f 100644 --- a/spec/lib/gitlab/metrics/subscribers/external_http_spec.rb +++ b/spec/lib/gitlab/metrics/subscribers/external_http_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Metrics::Subscribers::ExternalHttp, :request_store do +RSpec.describe Gitlab::Metrics::Subscribers::ExternalHttp, :request_store, feature_category: :logging do let(:transaction) { Gitlab::Metrics::WebTransaction.new({}) } let(:subscriber) { described_class.new } @@ -15,7 +15,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::ExternalHttp, :request_store do :event, payload: { method: 'POST', code: "200", duration: 0.321, - scheme: 'https', host: 'gitlab.com', port: 80, path: '/api/v4/projects', + scheme: 'https', host: 'gitlab.com', port: 443, path: '/api/v4/projects', query: 'current=true' }, time: Time.current @@ -95,6 +95,47 @@ RSpec.describe Gitlab::Metrics::Subscribers::ExternalHttp, :request_store do expect(described_class.payload).to eql(external_http_count: 7, external_http_duration_s: 1.2) end end + + context 'with multiple requests' do + let(:slow_requests) do + [ + { + method: 'POST', + host: 'gitlab.com', + port: 80, + path: '/api/v4/projects/2/issues', + duration_s: 5.3 + }, + { + method: 'POST', + host: 'gitlab.com', + port: 443, + path: '/api/v4/projects', + duration_s: 0.321 + } + ] + end + + before do + stub_const("#{described_class}::MAX_SLOW_REQUESTS", 2) + stub_const("#{described_class}::THRESHOLD_SLOW_REQUEST_S", 0.01) + + subscriber.request(event_1) + subscriber.request(event_2) + subscriber.request(event_3) + end + + it 'returns a payload containing a limited set of slow requests' do + expect(described_class.payload).to eq( + external_http_count: 3, + external_http_duration_s: 5.741, + external_http_slow_requests: slow_requests + ) + expect(described_class.top_slowest_requests).to eq(slow_requests) + + expect(Gitlab::SafeRequestStore[:external_http_slow_requests].length).to eq(3) + end + end end describe '#request' do @@ -153,7 +194,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::ExternalHttp, :request_store do expect(Gitlab::SafeRequestStore[:external_http_detail_store][0]).to match a_hash_including( start: be_like_time(Time.current), method: 'POST', code: "200", duration: 0.321, - scheme: 'https', host: 'gitlab.com', port: 80, path: '/api/v4/projects', + scheme: 'https', host: 'gitlab.com', port: 443, path: '/api/v4/projects', query: 'current=true', exception_object: nil, backtrace: be_a(Array) ) diff --git a/spec/lib/gitlab/metrics/subscribers/load_balancing_spec.rb b/spec/lib/gitlab/metrics/subscribers/load_balancing_spec.rb index b401b7cc996..c2c3bb29b16 100644 --- a/spec/lib/gitlab/metrics/subscribers/load_balancing_spec.rb +++ b/spec/lib/gitlab/metrics/subscribers/load_balancing_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Metrics::Subscribers::LoadBalancing, :request_store, feature_category: :pods do +RSpec.describe Gitlab::Metrics::Subscribers::LoadBalancing, :request_store, feature_category: :cell do let(:subscriber) { described_class.new } describe '#caught_up_replica_pick' do diff --git a/spec/lib/gitlab/middleware/go_spec.rb b/spec/lib/gitlab/middleware/go_spec.rb index aaa274e252d..83d4d3fb612 100644 --- a/spec/lib/gitlab/middleware/go_spec.rb +++ b/spec/lib/gitlab/middleware/go_spec.rb @@ -138,7 +138,7 @@ RSpec.describe Gitlab::Middleware::Go, feature_category: :source_code_management context 'with a blacklisted ip' do it 'returns forbidden' do - expect(Gitlab::Auth).to receive(:find_for_git_client).and_raise(Gitlab::Auth::IpBlacklisted) + expect(Gitlab::Auth).to receive(:find_for_git_client).and_raise(Gitlab::Auth::IpBlocked) response = go expect(response[0]).to eq(403) diff --git a/spec/lib/gitlab/middleware/multipart_spec.rb b/spec/lib/gitlab/middleware/multipart_spec.rb index 294a5ee82ed..509a4bb921b 100644 --- a/spec/lib/gitlab/middleware/multipart_spec.rb +++ b/spec/lib/gitlab/middleware/multipart_spec.rb @@ -175,7 +175,7 @@ RSpec.describe Gitlab::Middleware::Multipart do end it 'raises an error' do - expect { subject }.to raise_error(JWT::VerificationError, 'Signature verification raised') + expect { subject }.to raise_error(JWT::VerificationError, 'Signature verification failed') end end @@ -191,7 +191,7 @@ RSpec.describe Gitlab::Middleware::Multipart do end it 'raises an error' do - expect { subject }.to raise_error(JWT::VerificationError, 'Signature verification raised') + expect { subject }.to raise_error(JWT::VerificationError, 'Signature verification failed') end end end diff --git a/spec/lib/gitlab/octokit/middleware_spec.rb b/spec/lib/gitlab/octokit/middleware_spec.rb index 5555990b113..f7063f2c4f2 100644 --- a/spec/lib/gitlab/octokit/middleware_spec.rb +++ b/spec/lib/gitlab/octokit/middleware_spec.rb @@ -6,7 +6,7 @@ RSpec.describe Gitlab::Octokit::Middleware, feature_category: :importers do let(:app) { double(:app) } let(:middleware) { described_class.new(app) } - shared_examples 'Public URL' do + shared_examples 'Allowed URL' do it 'does not raise an error' do expect(app).to receive(:call).with(env) @@ -14,7 +14,7 @@ RSpec.describe Gitlab::Octokit::Middleware, feature_category: :importers do end end - shared_examples 'Local URL' do + shared_examples 'Blocked URL' do it 'raises an error' do expect { middleware.call(env) }.to raise_error(Gitlab::UrlBlocker::BlockedUrlError) end @@ -24,7 +24,24 @@ RSpec.describe Gitlab::Octokit::Middleware, feature_category: :importers do context 'when the URL is a public URL' do let(:env) { { url: 'https://public-url.com' } } - it_behaves_like 'Public URL' + it_behaves_like 'Allowed URL' + + context 'with failed address check' do + before do + stub_env('RSPEC_ALLOW_INVALID_URLS', 'false') + allow(Addrinfo).to receive(:getaddrinfo).and_raise(SocketError) + end + + it_behaves_like 'Blocked URL' + + context 'with disabled dns rebinding check' do + before do + stub_application_setting(dns_rebinding_protection_enabled: false) + end + + it_behaves_like 'Allowed URL' + end + end end context 'when the URL is a localhost address' do @@ -35,7 +52,7 @@ RSpec.describe Gitlab::Octokit::Middleware, feature_category: :importers do stub_application_setting(allow_local_requests_from_web_hooks_and_services: false) end - it_behaves_like 'Local URL' + it_behaves_like 'Blocked URL' end context 'when localhost requests are allowed' do @@ -43,7 +60,7 @@ RSpec.describe Gitlab::Octokit::Middleware, feature_category: :importers do stub_application_setting(allow_local_requests_from_web_hooks_and_services: true) end - it_behaves_like 'Public URL' + it_behaves_like 'Allowed URL' end end @@ -55,7 +72,7 @@ RSpec.describe Gitlab::Octokit::Middleware, feature_category: :importers do stub_application_setting(allow_local_requests_from_web_hooks_and_services: false) end - it_behaves_like 'Local URL' + it_behaves_like 'Blocked URL' end context 'when local network requests are allowed' do @@ -63,7 +80,7 @@ RSpec.describe Gitlab::Octokit::Middleware, feature_category: :importers do stub_application_setting(allow_local_requests_from_web_hooks_and_services: true) end - it_behaves_like 'Public URL' + it_behaves_like 'Allowed URL' end end diff --git a/spec/lib/gitlab/redis/multi_store_spec.rb b/spec/lib/gitlab/redis/multi_store_spec.rb index baf2546fc5c..e45c29a9dd2 100644 --- a/spec/lib/gitlab/redis/multi_store_spec.rb +++ b/spec/lib/gitlab/redis/multi_store_spec.rb @@ -333,7 +333,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do stub_feature_flags(use_primary_store_as_default_for_test_store: false) end - it 'executes only on secondary redis store', :aggregate_errors do + it 'executes only on secondary redis store', :aggregate_failures do expect(secondary_store).to receive(name).with(*expected_args).and_call_original expect(primary_store).not_to receive(name).with(*expected_args).and_call_original @@ -342,7 +342,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do end context 'when using primary store as default' do - it 'executes only on primary redis store', :aggregate_errors do + it 'executes only on primary redis store', :aggregate_failures do expect(primary_store).to receive(name).with(*expected_args).and_call_original expect(secondary_store).not_to receive(name).with(*expected_args).and_call_original @@ -437,7 +437,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do end RSpec.shared_examples_for 'verify that store contains values' do |store| - it "#{store} redis store contains correct values", :aggregate_errors do + it "#{store} redis store contains correct values", :aggregate_failures do subject redis_store = multi_store.send(store) @@ -530,7 +530,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do end context 'when executing on primary instance is successful' do - it 'executes on both primary and secondary redis store', :aggregate_errors do + it 'executes on both primary and secondary redis store', :aggregate_failures do expect(primary_store).to receive(name).with(*expected_args).and_call_original expect(secondary_store).to receive(name).with(*expected_args).and_call_original @@ -551,7 +551,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do stub_feature_flags(use_primary_store_as_default_for_test_store: false) end - it 'executes only on secondary redis store', :aggregate_errors do + it 'executes only on secondary redis store', :aggregate_failures do expect(secondary_store).to receive(name).with(*expected_args).and_call_original expect(primary_store).not_to receive(name).with(*expected_args).and_call_original @@ -560,7 +560,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do end context 'when using primary store as default' do - it 'executes only on primary redis store', :aggregate_errors do + it 'executes only on primary redis store', :aggregate_failures do expect(primary_store).to receive(name).with(*expected_args).and_call_original expect(secondary_store).not_to receive(name).with(*expected_args).and_call_original @@ -575,7 +575,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do allow(Gitlab::ErrorTracking).to receive(:log_exception) end - it 'logs the exception and execute on secondary instance', :aggregate_errors do + it 'logs the exception and execute on secondary instance', :aggregate_failures do expect(Gitlab::ErrorTracking).to receive(:log_exception).with(an_instance_of(StandardError), hash_including(:multi_store_error_message, command_name: name, instance_name: instance_name)) expect(secondary_store).to receive(name).with(*expected_args).and_call_original @@ -593,7 +593,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do end end - it 'is executed only 1 time on each instance', :aggregate_errors do + it 'is executed only 1 time on each instance', :aggregate_failures do expect(primary_store).to receive(:pipelined).and_call_original expect_next_instance_of(Redis::PipelinedConnection) do |pipeline| expect(pipeline).to receive(name).with(*expected_args).once.and_call_original @@ -645,7 +645,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do end context 'when executing on primary instance is successful' do - it 'executes on both primary and secondary redis store', :aggregate_errors do + it 'executes on both primary and secondary redis store', :aggregate_failures do expect(primary_store).to receive(name).and_call_original expect(secondary_store).to receive(name).and_call_original @@ -662,7 +662,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do allow(Gitlab::ErrorTracking).to receive(:log_exception) end - it 'logs the exception and execute on secondary instance', :aggregate_errors do + it 'logs the exception and execute on secondary instance', :aggregate_failures do expect(Gitlab::ErrorTracking).to receive(:log_exception).with(an_instance_of(StandardError), hash_including(:multi_store_error_message, command_name: name)) expect(secondary_store).to receive(name).and_call_original @@ -760,7 +760,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do stub_feature_flags(use_primary_store_as_default_for_test_store: false) end - it 'executes on secondary store', :aggregate_errors do + it 'executes on secondary store', :aggregate_failures do expect(primary_store).not_to receive(:send).and_call_original expect(secondary_store).to receive(:send).and_call_original @@ -769,7 +769,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do end context 'when using primary store as default' do - it 'executes on primary store', :aggregate_errors do + it 'executes on primary store', :aggregate_failures do expect(secondary_store).not_to receive(:send).and_call_original expect(primary_store).to receive(:send).and_call_original @@ -930,7 +930,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do subject end - it 'fallback and executes only on the secondary store', :aggregate_errors do + it 'fallback and executes only on the secondary store', :aggregate_failures do expect(primary_store).to receive(:command).and_call_original expect(secondary_store).not_to receive(:command) @@ -955,7 +955,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do end context 'with feature flag :use_primary_store_as_default_for_test_store is enabled' do - it 'fallback and executes only on the secondary store', :aggregate_errors do + it 'fallback and executes only on the secondary store', :aggregate_failures do expect(primary_store).to receive(:command).and_call_original expect(secondary_store).not_to receive(:command) @@ -968,7 +968,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do stub_feature_flags(use_primary_store_as_default_for_test_store: false) end - it 'fallback and executes only on the secondary store', :aggregate_errors do + it 'fallback and executes only on the secondary store', :aggregate_failures do expect(secondary_store).to receive(:command).and_call_original expect(primary_store).not_to receive(:command) @@ -981,7 +981,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do multi_store.pipelined(&:command) end - it 'is executed only 1 time on each instance', :aggregate_errors do + it 'is executed only 1 time on each instance', :aggregate_failures do expect(primary_store).to receive(:pipelined).once.and_call_original expect(secondary_store).to receive(:pipelined).once.and_call_original diff --git a/spec/lib/gitlab/reference_extractor_spec.rb b/spec/lib/gitlab/reference_extractor_spec.rb index 4d608c07736..0c6a832a730 100644 --- a/spec/lib/gitlab/reference_extractor_spec.rb +++ b/spec/lib/gitlab/reference_extractor_spec.rb @@ -301,7 +301,7 @@ RSpec.describe Gitlab::ReferenceExtractor do describe 'referables prefixes' do def prefixes - described_class::REFERABLES.each_with_object({}) do |referable, result| + described_class.referrables.each_with_object({}) do |referable, result| class_name = referable.to_s.camelize klass = class_name.constantize if Object.const_defined?(class_name) @@ -314,7 +314,7 @@ RSpec.describe Gitlab::ReferenceExtractor do end it 'returns all supported prefixes' do - expect(prefixes.keys.uniq).to match_array(%w(@ # ~ % ! $ & [vulnerability: *iteration:)) + expect(prefixes.keys.uniq).to include(*%w(@ # ~ % ! $ & [vulnerability:)) end it 'does not allow one prefix for multiple referables if not allowed specificly' do diff --git a/spec/lib/gitlab/regex_spec.rb b/spec/lib/gitlab/regex_spec.rb index e51e62d5f0a..5e58282ff92 100644 --- a/spec/lib/gitlab/regex_spec.rb +++ b/spec/lib/gitlab/regex_spec.rb @@ -79,10 +79,10 @@ RSpec.describe Gitlab::Regex, feature_category: :tooling do it { is_expected - .to eq("cannot start with a non-alphanumeric character except for periods or underscores, " \ - "can contain only alphanumeric characters, forward slashes, periods, and underscores, " \ - "cannot end with a period or forward slash, and has a relative path structure " \ - "with no http protocol chars or leading or trailing forward slashes") + .to eq("must have a relative path structure with no HTTP " \ + "protocol characters, or leading or trailing forward slashes. Path segments must not start or " \ + "end with a special character, and must not contain consecutive special characters." + ) } end @@ -101,13 +101,14 @@ RSpec.describe Gitlab::Regex, feature_category: :tooling do it { is_expected.not_to match('good_for+you') } it { is_expected.not_to match('source/') } it { is_expected.not_to match('.source/full./path') } + it { is_expected.not_to match('.source/.full/.path') } + it { is_expected.not_to match('_source') } + it { is_expected.not_to match('.source') } it { is_expected.to match('source') } - it { is_expected.to match('.source') } - it { is_expected.to match('_source') } it { is_expected.to match('source/full') } it { is_expected.to match('source/full/path') } - it { is_expected.to match('.source/.full/.path') } + it { is_expected.to match('sou_rce/fu-ll/pa.th') } it { is_expected.to match('domain_namespace') } it { is_expected.to match('gitlab-migration-test') } it { is_expected.to match('1-project-path') } @@ -115,10 +116,22 @@ RSpec.describe Gitlab::Regex, feature_category: :tooling do it { is_expected.to match('') } # it is possible to pass an empty string for destination_namespace in bulk_import POST request end + describe '.bulk_import_source_full_path_regex_message' do + subject { described_class.bulk_import_source_full_path_regex_message } + + it { + is_expected + .to eq( + "must have a relative path structure with no HTTP " \ + "protocol characters, or leading or trailing forward slashes. Path segments must not start or " \ + "end with a special character, and must not contain consecutive special characters." + ) + } + end + describe '.bulk_import_source_full_path_regex' do subject { described_class.bulk_import_source_full_path_regex } - it { is_expected.not_to match('?gitlab') } it { is_expected.not_to match("Users's something") } it { is_expected.not_to match('/source') } it { is_expected.not_to match('http:') } @@ -126,20 +139,32 @@ RSpec.describe Gitlab::Regex, feature_category: :tooling do it { is_expected.not_to match('example.com/?stuff=true') } it { is_expected.not_to match('example.com:5000/?stuff=true') } it { is_expected.not_to match('http://gitlab.example/gitlab-org/manage/import/gitlab-migration-test') } - it { is_expected.not_to match('_good_for_me!') } - it { is_expected.not_to match('good_for+you') } it { is_expected.not_to match('source/') } - it { is_expected.not_to match('.source/full./path') } it { is_expected.not_to match('') } + it { is_expected.not_to match('.source/full./path') } + it { is_expected.not_to match('?gitlab') } + it { is_expected.not_to match('_good_for_me!') } + it { is_expected.not_to match('group/@*%_my_other-project-----') } + it { is_expected.not_to match('_foog-for-me!') } + it { is_expected.not_to match('.source/full/path.') } + it { is_expected.to match('good_for+you') } it { is_expected.to match('source') } it { is_expected.to match('.source') } it { is_expected.to match('_source') } it { is_expected.to match('source/full') } it { is_expected.to match('source/full/path') } - it { is_expected.to match('.source/.full/.path') } it { is_expected.to match('domain_namespace') } it { is_expected.to match('gitlab-migration-test') } + it { is_expected.to match('source/full/path-') } + it { is_expected.to match('.source/full/path') } + it { is_expected.to match('.source/.full/.path') } + it { is_expected.to match('source/full/.path') } + it { is_expected.to match('source/full/..path') } + it { is_expected.to match('source/full/---1path') } + it { is_expected.to match('source/full/-___path') } + it { is_expected.to match('source/full/path---') } + it { is_expected.to match('group/__my_other-project-----') } end describe '.group_path_regex' do @@ -1164,10 +1189,21 @@ RSpec.describe Gitlab::Regex, feature_category: :tooling do MARKDOWN end - it { is_expected.to match(%(<section>\nsomething\n</section>)) } - it { is_expected.not_to match(%(must start in first column <section>\nsomething\n</section>)) } - it { is_expected.not_to match(%(<section>must be multi-line</section>)) } - it { expect(subject.match(markdown)[:html]).to eq expected } + describe 'normal regular expression' do + it { is_expected.to match(%(<section>\nsomething\n</section>)) } + it { is_expected.not_to match(%(must start in first column <section>\nsomething\n</section>)) } + it { is_expected.not_to match(%(<section>must be multi-line</section>)) } + it { expect(subject.match(markdown)[:html]).to eq expected } + end + + describe 'untrusted regular expression' do + subject { Gitlab::UntrustedRegexp.new(described_class::MARKDOWN_HTML_BLOCK_REGEX_UNTRUSTED, multiline: true) } + + it { is_expected.to match(%(<section>\nsomething\n</section>)) } + it { is_expected.not_to match(%(must start in first column <section>\nsomething\n</section>)) } + it { is_expected.not_to match(%(<section>must be multi-line</section>)) } + it { expect(subject.match(markdown)[:html]).to eq expected } + end end context 'HTML comment lines' do diff --git a/spec/lib/gitlab/resource_events/assignment_event_recorder_spec.rb b/spec/lib/gitlab/resource_events/assignment_event_recorder_spec.rb new file mode 100644 index 00000000000..b15f95dbd9c --- /dev/null +++ b/spec/lib/gitlab/resource_events/assignment_event_recorder_spec.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::ResourceEvents::AssignmentEventRecorder, feature_category: :value_stream_management do + using RSpec::Parameterized::TableSyntax + + let_it_be(:user1) { create(:user) } + let_it_be(:user2) { create(:user) } + let_it_be(:user3) { create(:user) } + + let_it_be_with_refind(:issue_with_two_assignees) { create(:issue, assignees: [user1, user2]) } + let_it_be_with_refind(:mr_with_no_assignees) { create(:merge_request) } + let_it_be_with_refind(:mr_with_one_assignee) { create(:merge_request, assignee: [user3]) } + + let(:parent_records) do + { + issue_with_two_assignees: issue_with_two_assignees, + mr_with_no_assignees: mr_with_no_assignees, + mr_with_one_assignee: mr_with_one_assignee + } + end + + let(:user_records) do + { + user1: user1, + user2: user2, + user3: user3 + } + end + + where(:parent, :new_assignees, :assignee_history) do + :issue_with_two_assignees | [:user1, :user2, :user3] | [[:user3, :add]] + :issue_with_two_assignees | [:user1, :user3] | [[:user2, :remove], [:user3, :add]] + :issue_with_two_assignees | [:user1] | [[:user2, :remove]] + :issue_with_two_assignees | [] | [[:user1, :remove], [:user2, :remove]] + :mr_with_no_assignees | [:user1] | [[:user1, :add]] + :mr_with_no_assignees | [] | [] + :mr_with_one_assignee | [:user3] | [] + :mr_with_one_assignee | [:user1] | [[:user3, :remove], [:user1, :add]] + end + + with_them do + it 'records the assignment history corrently' do + parent_record = parent_records[parent] + old_assignees = parent_record.assignees.to_a + parent_record.assignees = new_assignees.map { |user_variable_name| user_records[user_variable_name] } + + described_class.new(parent: parent_record, old_assignees: old_assignees).record + + expected_records = assignee_history.map do |user_variable_name, action| + have_attributes({ + user_id: user_records[user_variable_name].id, + action: action.to_s + }) + end + + expect(parent_record.assignment_events).to match_array(expected_records) + end + end + + context 'when batching' do + it 'invokes multiple insert queries' do + stub_const('Gitlab::ResourceEvents::AssignmentEventRecorder::BATCH_SIZE', 1) + + expect(ResourceEvents::MergeRequestAssignmentEvent).to receive(:insert_all).twice + + described_class.new(parent: mr_with_one_assignee, old_assignees: [user1]).record # 1 assignment, 1 unassignment + end + end + + context 'when duplicated old assignees were given' do + it 'deduplicates the records' do + expect do + described_class.new(parent: mr_with_one_assignee, old_assignees: [user3, user2, user2]).record + end.to change { ResourceEvents::MergeRequestAssignmentEvent.count }.by(1) + end + end + + context 'when the record_issue_and_mr_assignee_events FF is off' do + before do + stub_feature_flags(record_issue_and_mr_assignee_events: false) + end + + it 'does nothing' do + expect do + described_class.new(parent: mr_with_one_assignee, old_assignees: [user2, user3]).record + end.not_to change { mr_with_one_assignee.assignment_events.count } + end + end +end diff --git a/spec/lib/gitlab/service_desk_spec.rb b/spec/lib/gitlab/service_desk_spec.rb index f554840ec78..d6725f37d39 100644 --- a/spec/lib/gitlab/service_desk_spec.rb +++ b/spec/lib/gitlab/service_desk_spec.rb @@ -4,8 +4,8 @@ require 'spec_helper' RSpec.describe Gitlab::ServiceDesk do before do - allow(Gitlab::IncomingEmail).to receive(:enabled?).and_return(true) - allow(Gitlab::IncomingEmail).to receive(:supports_wildcard?).and_return(true) + allow(Gitlab::Email::IncomingEmail).to receive(:enabled?).and_return(true) + allow(Gitlab::Email::IncomingEmail).to receive(:supports_wildcard?).and_return(true) end describe 'enabled?' do @@ -39,7 +39,7 @@ RSpec.describe Gitlab::ServiceDesk do context 'when incoming emails are disabled' do before do - allow(Gitlab::IncomingEmail).to receive(:enabled?).and_return(false) + allow(Gitlab::Email::IncomingEmail).to receive(:enabled?).and_return(false) end it { is_expected.to be_falsy } @@ -47,7 +47,7 @@ RSpec.describe Gitlab::ServiceDesk do context 'when email key is not supported' do before do - allow(Gitlab::IncomingEmail).to receive(:supports_wildcard?).and_return(false) + allow(Gitlab::Email::IncomingEmail).to receive(:supports_wildcard?).and_return(false) end it { is_expected.to be_falsy } diff --git a/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb b/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb index e3d9549a3c0..4b589dc43af 100644 --- a/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb +++ b/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb @@ -309,7 +309,7 @@ RSpec.describe Gitlab::SidekiqLogging::StructuredLogger do end shared_examples 'performs database queries' do - it 'logs the database time', :aggregate_errors do + it 'logs the database time', :aggregate_failures do expect(logger).to receive(:info).with(expected_start_payload).ordered expect(logger).to receive(:info).with(expected_end_payload_with_db).ordered @@ -318,7 +318,7 @@ RSpec.describe Gitlab::SidekiqLogging::StructuredLogger do end end - it 'prevents database time from leaking to the next job', :aggregate_errors do + it 'prevents database time from leaking to the next job', :aggregate_failures do expect(logger).to receive(:info).with(expected_start_payload).ordered expect(logger).to receive(:info).with(expected_end_payload_with_db).ordered expect(logger).to receive(:info).with(expected_start_payload).ordered diff --git a/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job_spec.rb b/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job_spec.rb index 6a515a2b8a5..31258c42b5f 100644 --- a/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job_spec.rb @@ -79,10 +79,10 @@ RSpec.describe Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob, :clean_gi context 'with Redis cookies' do def with_redis(&block) - Sidekiq.redis(&block) + Gitlab::Redis::Queues.with(&block) end - let(:cookie_key) { "#{idempotency_key}:cookie:v2" } + let(:cookie_key) { "#{Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE}:#{idempotency_key}:cookie:v2" } let(:cookie) { get_redis_msgpack(cookie_key) } describe '#check!' do diff --git a/spec/lib/gitlab/slug/environment_spec.rb b/spec/lib/gitlab/slug/environment_spec.rb index e8f0fba27b2..8e23ad118d4 100644 --- a/spec/lib/gitlab/slug/environment_spec.rb +++ b/spec/lib/gitlab/slug/environment_spec.rb @@ -1,38 +1,41 @@ # frozen_string_literal: true require 'fast_spec_helper' +require 'rspec-parameterized' -RSpec.describe Gitlab::Slug::Environment do +RSpec.describe Gitlab::Slug::Environment, feature_category: :environment_management do describe '#generate' do - { - "staging-12345678901234567" => "staging-123456789-q517sa", - "9-staging-123456789012345" => "env-9-staging-123-q517sa", - "staging-1234567890123456" => "staging-1234567890123456", - "staging-1234567890123456-" => "staging-123456789-q517sa", - "production" => "production", - "PRODUCTION" => "production-q517sa", - "review/1-foo" => "review-1-foo-q517sa", - "1-foo" => "env-1-foo-q517sa", - "1/foo" => "env-1-foo-q517sa", - "foo-" => "foo", - "foo--bar" => "foo-bar-q517sa", - "foo**bar" => "foo-bar-q517sa", - "*-foo" => "env-foo-q517sa", - "staging-12345678-" => "staging-12345678", - "staging-12345678-01234567" => "staging-12345678-q517sa", - "" => "env-q517sa", - nil => "env-q517sa" - }.each do |name, matcher| - before do - # ('a' * 64).to_i(16).to_s(36).last(6) gives 'q517sa' - allow(Digest::SHA2).to receive(:hexdigest).with(name).and_return('a' * 64) - end + using RSpec::Parameterized::TableSyntax - it "returns a slug matching #{matcher}, given #{name}" do - slug = described_class.new(name).generate + subject { described_class.new(name).generate } - expect(slug).to match(/\A#{matcher}\z/) - end + before do + # ('a' * 64).to_i(16).to_s(36).last(6) gives 'q517sa' + allow(Digest::SHA2).to receive(:hexdigest).with(name.to_s).and_return('a' * 64) + end + + where(:name, :slug) do + "staging-12345678901234567" | "staging-123456789-q517sa" + "9-staging-123456789012345" | "env-9-staging-123-q517sa" + "staging-1234567890123456" | "staging-1234567890123456" + "staging-1234567890123456-" | "staging-123456789-q517sa" + "production" | "production" + "PRODUCTION" | "production-q517sa" + "review/1-foo" | "review-1-foo-q517sa" + "1-foo" | "env-1-foo-q517sa" + "1/foo" | "env-1-foo-q517sa" + "foo-" | "foo" + "foo--bar" | "foo-bar-q517sa" + "foo**bar" | "foo-bar-q517sa" + "*-foo" | "env-foo-q517sa" + "staging-12345678-" | "staging-12345678" + "staging-12345678-01234567" | "staging-12345678-q517sa" + "" | "env-q517sa" + nil | "env-q517sa" + end + + with_them do + it { is_expected.to eq(slug) } end end end diff --git a/spec/lib/gitlab/subscription_portal_spec.rb b/spec/lib/gitlab/subscription_portal_spec.rb index f93eb6f96cc..52c7a68921b 100644 --- a/spec/lib/gitlab/subscription_portal_spec.rb +++ b/spec/lib/gitlab/subscription_portal_spec.rb @@ -64,6 +64,7 @@ RSpec.describe ::Gitlab::SubscriptionPortal do :subscriptions_more_minutes_url | "#{staging_customers_url}/buy_pipeline_minutes" :subscriptions_more_storage_url | "#{staging_customers_url}/buy_storage" :subscriptions_manage_url | "#{staging_customers_url}/subscriptions" + :subscriptions_legacy_sign_in_url | "#{staging_customers_url}/customers/sign_in?legacy=true" :subscriptions_instance_review_url | "#{staging_customers_url}/instance_review" :subscriptions_gitlab_plans_url | "#{staging_customers_url}/gitlab_plans" :edit_account_url | "#{staging_customers_url}/customers/edit" diff --git a/spec/lib/gitlab/template/finders/global_template_finder_spec.rb b/spec/lib/gitlab/template/finders/global_template_finder_spec.rb index 38ec28c2b9a..c1dfee3cccb 100644 --- a/spec/lib/gitlab/template/finders/global_template_finder_spec.rb +++ b/spec/lib/gitlab/template/finders/global_template_finder_spec.rb @@ -16,10 +16,12 @@ RSpec.describe Gitlab::Template::Finders::GlobalTemplateFinder do end subject(:finder) do - described_class.new(base_dir, '', - { 'General' => '', 'Bar' => 'Bar' }, - include_categories_for_file, - excluded_patterns: excluded_patterns) + described_class.new( + base_dir, '', + { 'General' => '', 'Bar' => 'Bar' }, + include_categories_for_file, + excluded_patterns: excluded_patterns + ) end let(:excluded_patterns) { [] } diff --git a/spec/lib/gitlab/tracking/destinations/database_events_snowplow_spec.rb b/spec/lib/gitlab/tracking/destinations/database_events_snowplow_spec.rb new file mode 100644 index 00000000000..0f2082c1f25 --- /dev/null +++ b/spec/lib/gitlab/tracking/destinations/database_events_snowplow_spec.rb @@ -0,0 +1,113 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Tracking::Destinations::DatabaseEventsSnowplow, :do_not_stub_snowplow_by_default, feature_category: :application_instrumentation do + let(:emitter) { SnowplowTracker::Emitter.new(endpoint: 'localhost', options: { buffer_size: 1 }) } + + let(:tracker) do + SnowplowTracker::Tracker + .new( + emitters: [emitter], + subject: SnowplowTracker::Subject.new, + namespace: 'namespace', + app_id: 'app_id' + ) + end + + before do + stub_application_setting(snowplow_app_id: '_abc123_') + end + + around do |example| + freeze_time { example.run } + end + + context 'when snowplow is enabled' do + before do + allow(SnowplowTracker::AsyncEmitter) + .to receive(:new) + .with(endpoint: 'localhost:9091', + options: + { + protocol: 'https', + on_success: subject.method(:increment_successful_events_emissions), + on_failure: subject.method(:failure_callback) + } + ).and_return(emitter) + + allow(SnowplowTracker::Tracker) + .to receive(:new) + .with( + emitters: [emitter], + subject: an_instance_of(SnowplowTracker::Subject), + namespace: described_class::SNOWPLOW_NAMESPACE, + app_id: '_abc123_' + ).and_return(tracker) + end + + describe '#event' do + it 'sends event to tracker' do + allow(tracker).to receive(:track_struct_event).and_call_original + + subject.event('category', 'action', label: 'label', property: 'property', value: 1.5) + + expect(tracker) + .to have_received(:track_struct_event) + .with(category: 'category', action: 'action', label: 'label', property: 'property', value: 1.5, context: nil, + tstamp: (Time.now.to_f * 1000).to_i) + end + + it 'increase total snowplow events counter' do + counter = double + + expect(counter).to receive(:increment) + expect(Gitlab::Metrics).to receive(:counter) + .with(:gitlab_db_events_snowplow_events_total, 'Number of Snowplow events') + .and_return(counter) + + subject.event('category', 'action', label: 'label', property: 'property', value: 1.5) + end + end + end + + context 'for callbacks' do + describe 'on success' do + it 'increase gitlab_successful_snowplow_events_total counter' do + counter = double + + expect(counter).to receive(:increment).with({}, 2) + expect(Gitlab::Metrics).to receive(:counter) + .with( + :gitlab_db_events_snowplow_successful_events_total, + 'Number of successful Snowplow events emissions').and_return(counter) + + subject.method(:increment_successful_events_emissions).call(2) + end + end + + describe 'on failure' do + it 'increase gitlab_failed_snowplow_events_total counter and logs failures', :aggregate_failures do + counter = double + error_message = "Issue database_event_update failed to be reported to collector at localhost:9091" + failures = [{ "e" => "se", + "se_ca" => "Issue", + "se_la" => "issues", + "se_ac" => "database_event_update" }] + allow(Gitlab::Metrics).to receive(:counter) + .with( + :gitlab_db_events_snowplow_successful_events_total, + 'Number of successful Snowplow events emissions').and_call_original + + expect(Gitlab::AppLogger).to receive(:error).with(error_message) + expect(counter).to receive(:increment).with({}, 1) + expect(Gitlab::Metrics).to receive(:counter) + .with( + :gitlab_db_events_snowplow_failed_events_total, + 'Number of failed Snowplow events emissions').and_return(counter) + + subject.method(:failure_callback).call(2, failures) + end + end + end +end diff --git a/spec/lib/gitlab/tracking_spec.rb b/spec/lib/gitlab/tracking_spec.rb index e79bb2ef129..56be80678e9 100644 --- a/spec/lib/gitlab/tracking_spec.rb +++ b/spec/lib/gitlab/tracking_spec.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true require 'spec_helper' -RSpec.describe Gitlab::Tracking do +RSpec.describe Gitlab::Tracking, feature_category: :application_instrumentation do include StubENV before do @@ -102,12 +102,28 @@ RSpec.describe Gitlab::Tracking do end end - describe '.event' do + context 'event tracking' do let(:namespace) { create(:namespace) } - shared_examples 'delegates to destination' do |klass| + shared_examples 'rescued error raised by destination class' do + it 'rescues error' do + error = StandardError.new("something went wrong") + allow_any_instance_of(destination_class).to receive(:event).and_raise(error) + + expect(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception) + .with( + error, + snowplow_category: category, + snowplow_action: action + ) + + expect { tracking_method }.not_to raise_error + end + end + + shared_examples 'delegates to destination' do |klass, method| before do - allow_any_instance_of(Gitlab::Tracking::Destinations::Snowplow).to receive(:event) + allow_any_instance_of(klass).to receive(:event) end it "delegates to #{klass} destination" do @@ -118,8 +134,8 @@ RSpec.describe Gitlab::Tracking do expect(Gitlab::Tracking::StandardContext) .to receive(:new) - .with(project: project, user: user, namespace: namespace, extra_key_1: 'extra value 1', extra_key_2: 'extra value 2') - .and_call_original + .with(project: project, user: user, namespace: namespace, extra_key_1: 'extra value 1', extra_key_2: 'extra value 2') + .and_call_original expect_any_instance_of(klass).to receive(:event) do |_, category, action, args| expect(category).to eq('category') @@ -132,7 +148,7 @@ RSpec.describe Gitlab::Tracking do expect(args[:context].last).to eq(other_context) end - described_class.event('category', 'action', + described_class.method(method).call('category', 'action', label: 'label', property: 'property', value: 1.5, @@ -141,44 +157,95 @@ RSpec.describe Gitlab::Tracking do user: user, namespace: namespace, extra_key_1: 'extra value 1', - extra_key_2: 'extra value 2') + extra_key_2: 'extra value 2' + ) end end - context 'when the action is not passed in as a string' do - it 'allows symbols' do - expect(Gitlab::ErrorTracking).not_to receive(:track_and_raise_for_dev_exception) + describe '.database_event' do + context 'when the action is not passed in as a string' do + it 'allows symbols' do + expect(Gitlab::ErrorTracking).not_to receive(:track_and_raise_for_dev_exception) - described_class.event('category', :some_action) - end + described_class.database_event('category', :some_action) + end + + it 'allows nil' do + expect(Gitlab::ErrorTracking).not_to receive(:track_and_raise_for_dev_exception) + + described_class.database_event('category', nil) + end - it 'allows nil' do - expect(Gitlab::ErrorTracking).not_to receive(:track_and_raise_for_dev_exception) + it 'allows integers' do + expect(Gitlab::ErrorTracking).not_to receive(:track_and_raise_for_dev_exception) - described_class.event('category', nil) + described_class.database_event('category', 1) + end end - it 'allows integers' do - expect(Gitlab::ErrorTracking).not_to receive(:track_and_raise_for_dev_exception) + it_behaves_like 'rescued error raised by destination class' do + let(:category) { 'Issue' } + let(:action) { 'created' } + let(:destination_class) { Gitlab::Tracking::Destinations::DatabaseEventsSnowplow } - described_class.event('category', 1) + subject(:tracking_method) { described_class.database_event(category, action) } end + + it_behaves_like 'delegates to destination', Gitlab::Tracking::Destinations::DatabaseEventsSnowplow, :database_event end - context 'when destination is Snowplow' do - before do - allow(Rails.env).to receive(:development?).and_return(true) + describe '.event' do + context 'when the action is not passed in as a string' do + it 'allows symbols' do + expect(Gitlab::ErrorTracking).not_to receive(:track_and_raise_for_dev_exception) + + described_class.event('category', :some_action) + end + + it 'allows nil' do + expect(Gitlab::ErrorTracking).not_to receive(:track_and_raise_for_dev_exception) + + described_class.event('category', nil) + end + + it 'allows integers' do + expect(Gitlab::ErrorTracking).not_to receive(:track_and_raise_for_dev_exception) + + described_class.event('category', 1) + end end - it_behaves_like 'delegates to destination', Gitlab::Tracking::Destinations::Snowplow - end + context 'when destination is Snowplow' do + before do + allow(Rails.env).to receive(:development?).and_return(true) + end - context 'when destination is SnowplowMicro' do - before do - allow(Rails.env).to receive(:development?).and_return(true) + it_behaves_like 'rescued error raised by destination class' do + let(:category) { 'category' } + let(:action) { 'action' } + let(:destination_class) { Gitlab::Tracking::Destinations::Snowplow } + + subject(:tracking_method) { described_class.event(category, action) } + end + + it_behaves_like 'delegates to destination', Gitlab::Tracking::Destinations::Snowplow, :event end - it_behaves_like 'delegates to destination', Gitlab::Tracking::Destinations::SnowplowMicro + context 'when destination is SnowplowMicro' do + before do + allow(Rails.env).to receive(:development?).and_return(true) + end + + it_behaves_like 'rescued error raised by destination class' do + let(:category) { 'category' } + let(:action) { 'action' } + let(:destination_class) { Gitlab::Tracking::Destinations::Snowplow } + + subject(:tracking_method) { described_class.event(category, action) } + end + + it_behaves_like 'delegates to destination', Gitlab::Tracking::Destinations::SnowplowMicro, :event + end end end diff --git a/spec/lib/gitlab/untrusted_regexp_spec.rb b/spec/lib/gitlab/untrusted_regexp_spec.rb index 66675b20107..232329a5a1b 100644 --- a/spec/lib/gitlab/untrusted_regexp_spec.rb +++ b/spec/lib/gitlab/untrusted_regexp_spec.rb @@ -3,7 +3,7 @@ require 'fast_spec_helper' require 'support/shared_examples/lib/gitlab/malicious_regexp_shared_examples' -RSpec.describe Gitlab::UntrustedRegexp do +RSpec.describe Gitlab::UntrustedRegexp, feature_category: :shared do describe '#initialize' do subject { described_class.new(pattern) } @@ -22,6 +22,39 @@ RSpec.describe Gitlab::UntrustedRegexp do end end + describe '#replace_gsub' do + let(:regex_str) { '(?P<scheme>(ftp))' } + let(:regex) { described_class.new(regex_str, multiline: true) } + + def result(regex, text) + regex.replace_gsub(text) do |match| + if match[:scheme] + "http|#{match[:scheme]}|rss" + else + match.to_s + end + end + end + + it 'replaces all instances of the match in a string' do + text = 'Use only https instead of ftp' + + expect(result(regex, text)).to eq('Use only https instead of http|ftp|rss') + end + + it 'replaces nothing when no match' do + text = 'Use only https instead of gopher' + + expect(result(regex, text)).to eq(text) + end + + it 'handles empty text' do + text = '' + + expect(result(regex, text)).to eq('') + end + end + describe '#replace' do it 'replaces the first instance of the match in a string' do result = described_class.new('foo').replace('foo bar foo', 'oof') diff --git a/spec/lib/gitlab/url_blocker_spec.rb b/spec/lib/gitlab/url_blocker_spec.rb index 912093be29f..7b6c89b5dd3 100644 --- a/spec/lib/gitlab/url_blocker_spec.rb +++ b/spec/lib/gitlab/url_blocker_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do +RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only, feature_category: :shared do include StubRequests let(:schemes) { %w[http https] } @@ -21,7 +21,7 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do end end - shared_context 'instance configured to deny all requests' do + shared_context 'when instance configured to deny all requests' do before do allow(Gitlab::CurrentSettings).to receive(:current_application_settings?).and_return(true) stub_application_setting(deny_all_requests_except_allowed: true) @@ -30,7 +30,7 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do shared_examples 'a URI denied by `deny_all_requests_except_allowed`' do context 'when instance setting is enabled' do - include_context 'instance configured to deny all requests' + include_context 'when instance configured to deny all requests' it 'blocks the request' do expect { subject }.to raise_error(described_class::BlockedUrlError) @@ -81,7 +81,7 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do end shared_examples 'a URI exempt from `deny_all_requests_except_allowed`' do - include_context 'instance configured to deny all requests' + include_context 'when instance configured to deny all requests' it 'does not block the request' do expect { subject }.not_to raise_error @@ -248,15 +248,30 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do context 'when domain cannot be resolved' do let(:import_url) { 'http://foobar.x' } - it 'raises an error' do + before do stub_env('RSPEC_ALLOW_INVALID_URLS', 'false') + end + it 'raises an error' do expect { subject }.to raise_error(described_class::BlockedUrlError) end + + context 'with HTTP_PROXY' do + let(:import_url) { 'http://foobar.x' } + + before do + allow(Gitlab).to receive(:http_proxy_env?).and_return(true) + end + + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { import_url } + let(:expected_hostname) { nil } + end + end end context 'when domain is too long' do - let(:import_url) { 'https://example' + 'a' * 1024 + '.com' } + let(:import_url) { "https://example#{'a' * 1024}.com" } it 'raises an error' do expect { subject }.to raise_error(described_class::BlockedUrlError) @@ -285,7 +300,7 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do end end - context 'DNS rebinding protection with IP allowed' do + context 'when DNS rebinding protection with IP allowed' do let(:import_url) { 'http://a.192.168.0.120.3times.127.0.0.1.1time.repeat.rebind.network:9121/scrape?target=unix:///var/opt/gitlab/redis/redis.socket&check-keys=*' } before do @@ -300,9 +315,31 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do end it_behaves_like 'a URI exempt from `deny_all_requests_except_allowed`' + + context 'with HTTP_PROXY' do + before do + allow(Gitlab).to receive(:http_proxy_env?).and_return(true) + end + + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { import_url } + let(:expected_hostname) { nil } + end + + context 'when domain is in no_proxy env' do + before do + stub_env('no_proxy', 'a.192.168.0.120.3times.127.0.0.1.1time.repeat.rebind.network') + end + + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { 'http://192.168.0.120:9121/scrape?target=unix:///var/opt/gitlab/redis/redis.socket&check-keys=*' } + let(:expected_hostname) { 'a.192.168.0.120.3times.127.0.0.1.1time.repeat.rebind.network' } + end + end + end end - context 'disabled DNS rebinding protection' do + context 'with disabled DNS rebinding protection' do let(:options) { { dns_rebind_protection: false, schemes: schemes } } context 'when URI is internal' do @@ -483,7 +520,7 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do expect(described_class.blocked_url?('https://gitlab.com/foo/foo.git', schemes: schemes)).to be false end - context 'when allow_local_network is' do + describe 'allow_local_network' do let(:shared_address_space_ips) { ['100.64.0.0', '100.64.127.127', '100.64.255.255'] } let(:local_ips) do @@ -564,11 +601,11 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do end end - context 'true (default)' do + context 'when true (default)' do it_behaves_like 'allows local requests', { allow_localhost: true, allow_local_network: true, schemes: %w[http https] } end - context 'false' do + context 'when false' do it 'blocks urls from private networks' do local_ips.each do |ip| stub_domain_resolv(fake_domain, ip) do @@ -721,14 +758,14 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do end end - context 'when dns_rebinding_setting is' do - context 'enabled' do + describe 'dns_rebinding_setting' do + context 'when enabled' do let(:dns_rebind_value) { true } it_behaves_like 'allowlists the domain' end - context 'disabled' do + context 'when disabled' do let(:dns_rebind_value) { false } it_behaves_like 'allowlists the domain' @@ -768,8 +805,8 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do end end - context 'when enforce_user is' do - context 'false (default)' do + describe 'enforce_user' do + context 'when false (default)' do it 'does not block urls with a non-alphanumeric username' do expect(described_class).not_to be_blocked_url('ssh://-oProxyCommand=whoami@example.com/a', schemes: ['ssh']) @@ -781,7 +818,7 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do end end - context 'true' do + context 'when true' do it 'blocks urls with a non-alphanumeric username' do aggregate_failures do expect(described_class).to be_blocked_url('ssh://-oProxyCommand=whoami@example.com/a', enforce_user: true, schemes: ['ssh']) @@ -849,7 +886,7 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do end end - def stub_domain_resolv(domain, ip, port = 80, &block) + def stub_domain_resolv(domain, ip, port = 80) address = instance_double(Addrinfo, ip_address: ip, ipv4_private?: true, diff --git a/spec/lib/gitlab/url_blockers/ip_allowlist_entry_spec.rb b/spec/lib/gitlab/url_blockers/ip_allowlist_entry_spec.rb index 8dcb402dfb2..c56e5ce4e7a 100644 --- a/spec/lib/gitlab/url_blockers/ip_allowlist_entry_spec.rb +++ b/spec/lib/gitlab/url_blockers/ip_allowlist_entry_spec.rb @@ -2,7 +2,7 @@ require 'fast_spec_helper' -RSpec.describe Gitlab::UrlBlockers::IpAllowlistEntry do +RSpec.describe Gitlab::UrlBlockers::IpAllowlistEntry, feature_category: :integrations do let(:ipv4) { IPAddr.new('192.168.1.1') } describe '#initialize' do @@ -65,11 +65,31 @@ RSpec.describe Gitlab::UrlBlockers::IpAllowlistEntry do end it 'matches IPv6 within IPv6 range' do - ipv6_range = IPAddr.new('fd84:6d02:f6d8:c89e::/124') + ipv6_range = IPAddr.new('::ffff:192.168.1.0/8') ip_allowlist_entry = described_class.new(ipv6_range) expect(ip_allowlist_entry).to be_match(ipv6_range.to_range.last.to_s, 8080) expect(ip_allowlist_entry).not_to be_match('fd84:6d02:f6d8:f::f', 8080) end + + it 'matches IPv4 to IPv6 mapped addresses in allow list' do + ipv6_range = IPAddr.new('::ffff:192.168.1.1') + ip_allowlist_entry = described_class.new(ipv6_range) + + expect(ip_allowlist_entry).to be_match(ipv4, 8080) + expect(ip_allowlist_entry).to be_match(ipv6_range.to_range.last.to_s, 8080) + expect(ip_allowlist_entry).not_to be_match('::ffff:192.168.1.0', 8080) + expect(ip_allowlist_entry).not_to be_match('::ffff:169.254.168.101', 8080) + end + + it 'matches IPv4 to IPv6 mapped addresses in requested IP' do + ipv4_range = IPAddr.new('192.168.1.1/24') + ip_allowlist_entry = described_class.new(ipv4_range) + + expect(ip_allowlist_entry).to be_match(ipv4, 8080) + expect(ip_allowlist_entry).to be_match('::ffff:192.168.1.0', 8080) + expect(ip_allowlist_entry).to be_match('::ffff:192.168.1.1', 8080) + expect(ip_allowlist_entry).not_to be_match('::ffff:169.254.170.100/8', 8080) + end end end diff --git a/spec/lib/gitlab/usage/metric_definition_spec.rb b/spec/lib/gitlab/usage/metric_definition_spec.rb index 4b835d11975..c336a4850d2 100644 --- a/spec/lib/gitlab/usage/metric_definition_spec.rb +++ b/spec/lib/gitlab/usage/metric_definition_spec.rb @@ -7,7 +7,6 @@ RSpec.describe Gitlab::Usage::MetricDefinition do { description: 'GitLab instance unique identifier', value_type: 'string', - product_category: 'collection', product_stage: 'growth', product_section: 'devops', status: 'active', @@ -263,7 +262,6 @@ RSpec.describe Gitlab::Usage::MetricDefinition do { description: 'Test metric definition', value_type: 'string', - product_category: 'collection', product_stage: 'growth', product_section: 'devops', status: 'active', diff --git a/spec/lib/gitlab/usage/metric_spec.rb b/spec/lib/gitlab/usage/metric_spec.rb index 8e0fce37e46..d0ea4e7aa16 100644 --- a/spec/lib/gitlab/usage/metric_spec.rb +++ b/spec/lib/gitlab/usage/metric_spec.rb @@ -13,7 +13,6 @@ RSpec.describe Gitlab::Usage::Metric do product_section: "dev", product_stage: "plan", product_group: "plan", - product_category: "issue_tracking", value_type: "number", status: "active", time_frame: "all", diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/database_mode_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/database_mode_spec.rb new file mode 100644 index 00000000000..a6128b4df1f --- /dev/null +++ b/spec/lib/gitlab/usage/metrics/instrumentations/database_mode_spec.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Usage::Metrics::Instrumentations::DatabaseMode, feature_category: :cell do + let(:expected_value) { Gitlab::Database.database_mode } + + it_behaves_like 'a correct instrumented metric value', { time_frame: 'none' } +end diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/incoming_email_encrypted_secrets_enabled_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/incoming_email_encrypted_secrets_enabled_metric_spec.rb index ed35b2c8cde..b1b193c8d04 100644 --- a/spec/lib/gitlab/usage/metrics/instrumentations/incoming_email_encrypted_secrets_enabled_metric_spec.rb +++ b/spec/lib/gitlab/usage/metrics/instrumentations/incoming_email_encrypted_secrets_enabled_metric_spec.rb @@ -5,6 +5,6 @@ require 'spec_helper' RSpec.describe Gitlab::Usage::Metrics::Instrumentations::IncomingEmailEncryptedSecretsEnabledMetric, feature_category: :service_ping do it_behaves_like 'a correct instrumented metric value', { time_frame: 'none', data_source: 'ruby' } do - let(:expected_value) { ::Gitlab::IncomingEmail.encrypted_secrets.active? } + let(:expected_value) { ::Gitlab::Email::IncomingEmail.encrypted_secrets.active? } end end diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/index_inconsistencies_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/index_inconsistencies_metric_spec.rb index afc9d610207..92a576d1a9f 100644 --- a/spec/lib/gitlab/usage/metrics/instrumentations/index_inconsistencies_metric_spec.rb +++ b/spec/lib/gitlab/usage/metrics/instrumentations/index_inconsistencies_metric_spec.rb @@ -13,7 +13,7 @@ RSpec.describe Gitlab::Usage::Metrics::Instrumentations::IndexInconsistenciesMet end let(:runner) { instance_double(Gitlab::Database::SchemaValidation::Runner, execute: inconsistencies) } - let(:inconsistency_class) { Gitlab::Database::SchemaValidation::Validators::BaseValidator::Inconsistency } + let(:inconsistency_class) { Gitlab::Database::SchemaValidation::Inconsistency } let(:inconsistencies) do [ diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/service_desk_email_encrypted_secrets_enabled_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/service_desk_email_encrypted_secrets_enabled_metric_spec.rb index d602eae3159..ea239e53d01 100644 --- a/spec/lib/gitlab/usage/metrics/instrumentations/service_desk_email_encrypted_secrets_enabled_metric_spec.rb +++ b/spec/lib/gitlab/usage/metrics/instrumentations/service_desk_email_encrypted_secrets_enabled_metric_spec.rb @@ -5,6 +5,6 @@ require 'spec_helper' RSpec.describe Gitlab::Usage::Metrics::Instrumentations::ServiceDeskEmailEncryptedSecretsEnabledMetric, feature_category: :service_ping do it_behaves_like 'a correct instrumented metric value', { time_frame: 'none', data_source: 'ruby' } do - let(:expected_value) { ::Gitlab::ServiceDeskEmail.encrypted_secrets.active? } + let(:expected_value) { ::Gitlab::Email::ServiceDeskEmail.encrypted_secrets.active? } end end diff --git a/spec/lib/gitlab/usage/metrics/names_suggestions/generator_spec.rb b/spec/lib/gitlab/usage/metrics/names_suggestions/generator_spec.rb index 4f647c2700a..271e9595703 100644 --- a/spec/lib/gitlab/usage/metrics/names_suggestions/generator_spec.rb +++ b/spec/lib/gitlab/usage/metrics/names_suggestions/generator_spec.rb @@ -75,7 +75,7 @@ RSpec.describe Gitlab::Usage::Metrics::NamesSuggestions::Generator, feature_cate end end - context 'for redis metrics' do + context 'for redis metrics', quarantine: 'https://gitlab.com/gitlab-org/gitlab/-/issues/399421' do it_behaves_like 'name suggestion' do let(:key_path) { 'usage_activity_by_stage_monthly.create.merge_requests_users' } let(:name_suggestion) { /<please fill metric name, suggested format is: {subject}_{verb}{ing|ed}_{object} eg: users_creating_epics or merge_requests_viewed_in_single_file_mode>/ } diff --git a/spec/lib/gitlab/usage/service_ping_report_spec.rb b/spec/lib/gitlab/usage/service_ping_report_spec.rb index 730c05b7dcb..f1ce48468fe 100644 --- a/spec/lib/gitlab/usage/service_ping_report_spec.rb +++ b/spec/lib/gitlab/usage/service_ping_report_spec.rb @@ -72,25 +72,34 @@ RSpec.describe Gitlab::Usage::ServicePingReport, :use_clean_rails_memory_store_c context 'when using cached' do let(:new_usage_data) { { 'uuid' => '1112' } } + let(:instrumented_payload) { { 'instrumented' => { 'metric' => 1 } } } + let(:full_payload) { usage_data.merge(instrumented_payload) } + let(:new_full_payload) { new_usage_data.merge(instrumented_payload) } + + before do + allow_next_instance_of(Gitlab::Usage::ServicePing::InstrumentedPayload) do |instance| + allow(instance).to receive(:build).and_return(instrumented_payload) + end + end context 'for cached: true' do it 'caches the values' do allow(Gitlab::UsageData).to receive(:data).and_return(usage_data, new_usage_data) - expect(described_class.for(output: :all_metrics_values)).to eq(usage_data) - expect(described_class.for(output: :all_metrics_values, cached: true)).to eq(usage_data) + expect(described_class.for(output: :all_metrics_values)).to eq(full_payload) + expect(described_class.for(output: :all_metrics_values, cached: true)).to eq(full_payload) - expect(Rails.cache.fetch('usage_data')).to eq(usage_data) + expect(Rails.cache.fetch('usage_data')).to eq(full_payload) end it 'writes to cache and returns fresh data' do allow(Gitlab::UsageData).to receive(:data).and_return(usage_data, new_usage_data) - expect(described_class.for(output: :all_metrics_values)).to eq(usage_data) - expect(described_class.for(output: :all_metrics_values)).to eq(new_usage_data) - expect(described_class.for(output: :all_metrics_values, cached: true)).to eq(new_usage_data) + expect(described_class.for(output: :all_metrics_values)).to eq(full_payload) + expect(described_class.for(output: :all_metrics_values)).to eq(new_full_payload) + expect(described_class.for(output: :all_metrics_values, cached: true)).to eq(new_full_payload) - expect(Rails.cache.fetch('usage_data')).to eq(new_usage_data) + expect(Rails.cache.fetch('usage_data')).to eq(new_full_payload) end end @@ -98,10 +107,10 @@ RSpec.describe Gitlab::Usage::ServicePingReport, :use_clean_rails_memory_store_c it 'returns fresh data' do allow(Gitlab::UsageData).to receive(:data).and_return(usage_data, new_usage_data) - expect(described_class.for(output: :all_metrics_values)).to eq(usage_data) - expect(described_class.for(output: :all_metrics_values)).to eq(new_usage_data) + expect(described_class.for(output: :all_metrics_values)).to eq(full_payload) + expect(described_class.for(output: :all_metrics_values)).to eq(new_full_payload) - expect(Rails.cache.fetch('usage_data')).to eq(new_usage_data) + expect(Rails.cache.fetch('usage_data')).to eq(new_full_payload) end end end diff --git a/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb b/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb index 8c497970555..aadd398e5fd 100644 --- a/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb +++ b/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb @@ -428,7 +428,7 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s described_class.track_event('event4', values: entity2, time: 2.days.ago) end - it 'calculates union of given events', :aggregate_failure do + it 'calculates union of given events', :aggregate_failures do expect(described_class.calculate_events_union(**time_range.merge(event_names: %w[event4]))).to eq 2 expect(described_class.calculate_events_union(**time_range.merge(event_names: %w[event1_slot event2_slot event3_slot]))).to eq 3 end diff --git a/spec/lib/gitlab/usage_data_counters/issue_activity_unique_counter_spec.rb b/spec/lib/gitlab/usage_data_counters/issue_activity_unique_counter_spec.rb index 383938b0324..d6a99b5ea8b 100644 --- a/spec/lib/gitlab/usage_data_counters/issue_activity_unique_counter_spec.rb +++ b/spec/lib/gitlab/usage_data_counters/issue_activity_unique_counter_spec.rb @@ -6,11 +6,12 @@ RSpec.describe Gitlab::UsageDataCounters::IssueActivityUniqueCounter, :clean_git let_it_be(:user1) { build(:user, id: 1) } let_it_be(:user2) { build(:user, id: 2) } let_it_be(:user3) { build(:user, id: 3) } - let_it_be(:project) { build(:project) } + let_it_be(:project) { create(:project) } let_it_be(:category) { Gitlab::UsageDataCounters::IssueActivityUniqueCounter::ISSUE_CATEGORY } let_it_be(:event_action) { Gitlab::UsageDataCounters::IssueActivityUniqueCounter::ISSUE_ACTION } let_it_be(:event_label) { Gitlab::UsageDataCounters::IssueActivityUniqueCounter::ISSUE_LABEL } + let(:original_params) { nil } let(:event_property) { action } let(:time) { Time.zone.now } @@ -67,6 +68,7 @@ RSpec.describe Gitlab::UsageDataCounters::IssueActivityUniqueCounter, :clean_git context 'for Issue created actions' do it_behaves_like 'daily tracked issuable snowplow and service ping events with project' do let(:action) { described_class::ISSUE_CREATED } + let(:original_params) { { namespace: project.project_namespace.reload } } def track_action(params) described_class.track_issue_created_action(**params) diff --git a/spec/lib/gitlab/usage_data_spec.rb b/spec/lib/gitlab/usage_data_spec.rb index d529319e6e9..f2b332501be 100644 --- a/spec/lib/gitlab/usage_data_spec.rb +++ b/spec/lib/gitlab/usage_data_spec.rb @@ -29,10 +29,8 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures, feature_category: :servic .to include(:configure, :create, :manage, :monitor, :plan, :release, :verify) expect(subject[:usage_activity_by_stage_monthly]) .to include(:configure, :create, :manage, :monitor, :plan, :release, :verify) - expect(subject[:usage_activity_by_stage][:create]) - .not_to include(:merge_requests_users) expect(subject[:usage_activity_by_stage_monthly][:create]) - .to include(:merge_requests_users) + .to include(:snippets) end it 'clears memoized values' do @@ -715,7 +713,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures, feature_category: :servic expect(subject[:ldap_enabled]).to eq(Gitlab.config.ldap.enabled) expect(subject[:gravatar_enabled]).to eq(Gitlab::CurrentSettings.gravatar_enabled?) expect(subject[:omniauth_enabled]).to eq(Gitlab::Auth.omniauth_enabled?) - expect(subject[:reply_by_email_enabled]).to eq(Gitlab::IncomingEmail.enabled?) + expect(subject[:reply_by_email_enabled]).to eq(Gitlab::Email::IncomingEmail.enabled?) expect(subject[:container_registry_enabled]).to eq(Gitlab.config.registry.enabled) expect(subject[:dependency_proxy_enabled]).to eq(Gitlab.config.dependency_proxy.enabled) expect(subject[:gitlab_shared_runners_enabled]).to eq(Gitlab.config.gitlab_ci.shared_runners_enabled) @@ -1021,24 +1019,6 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures, feature_category: :servic end end - describe '.merge_requests_users', :clean_gitlab_redis_shared_state do - let(:time_period) { { created_at: 2.days.ago..time } } - let(:time) { Time.current } - - before do - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(:merge_request_action, values: 1, time: time) - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(:merge_request_action, values: 1, time: time) - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(:merge_request_action, values: 2, time: time) - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(:merge_request_action, values: 3, time: time) - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(:merge_request_action, values: 4, time: time - 3.days) - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(:design_action, values: 5, time: time) - end - - it 'returns the distinct count of users using merge requests (via events table) within the specified time period' do - expect(described_class.merge_requests_users(time_period)).to eq(3) - end - end - def for_defined_days_back(days: [31, 3]) days.each do |n| travel_to(n.days.ago) do @@ -1067,7 +1047,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures, feature_category: :servic expect(result.duration).to be_an(Float) end - it 'records error and returns nil', :aggregated_errors do + it 'records error and returns nil', :aggregate_failures do allow(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception) result = described_class.with_metadata { raise } diff --git a/spec/lib/gitlab/utils/error_message_spec.rb b/spec/lib/gitlab/utils/error_message_spec.rb index 2c2d16656e8..17786f2c8ef 100644 --- a/spec/lib/gitlab/utils/error_message_spec.rb +++ b/spec/lib/gitlab/utils/error_message_spec.rb @@ -9,15 +9,20 @@ RSpec.describe Gitlab::Utils::ErrorMessage, feature_category: :error_tracking do end end - subject(:object) { klass.new } + let(:message) { 'Something went wrong' } - describe 'error message' do - subject { object.to_user_facing(string) } + subject(:object) { klass.new } - let(:string) { 'Error Message' } + describe '#to_user_facing' do + it 'returns a user-facing error message with the UF prefix' do + expect(described_class.to_user_facing(message)).to eq("UF: #{message}") + end + end - it "returns input prefixed with UF:" do - is_expected.to eq 'UF: Error Message' + describe '#prefixed_error_message' do + it 'returns a message with the given prefix' do + prefix = 'ERROR' + expect(described_class.prefixed_error_message(message, prefix)).to eq("#{prefix}: #{message}") end end end diff --git a/spec/lib/gitlab/utils/measuring_spec.rb b/spec/lib/gitlab/utils/measuring_spec.rb index 5dad79b1c5f..4d2791f771f 100644 --- a/spec/lib/gitlab/utils/measuring_spec.rb +++ b/spec/lib/gitlab/utils/measuring_spec.rb @@ -17,7 +17,7 @@ RSpec.describe Gitlab::Utils::Measuring do measurement.with_measuring { result } end - it 'measures and logs data', :aggregate_failure do + it 'measures and logs data', :aggregate_failures do expect(measurement).to receive(:with_measure_time).and_call_original expect(measurement).to receive(:with_count_queries).and_call_original expect(measurement).to receive(:with_gc_stats).and_call_original diff --git a/spec/lib/gitlab/utils/strong_memoize_spec.rb b/spec/lib/gitlab/utils/strong_memoize_spec.rb index 27bfe181ef6..ea8083e7d7f 100644 --- a/spec/lib/gitlab/utils/strong_memoize_spec.rb +++ b/spec/lib/gitlab/utils/strong_memoize_spec.rb @@ -3,12 +3,15 @@ require 'fast_spec_helper' require 'rspec-benchmark' require 'rspec-parameterized' +require 'active_support/testing/time_helpers' RSpec.configure do |config| config.include RSpec::Benchmark::Matchers end RSpec.describe Gitlab::Utils::StrongMemoize, feature_category: :shared do + include ActiveSupport::Testing::TimeHelpers + let(:klass) do strong_memoize_class = described_class @@ -30,6 +33,13 @@ RSpec.describe Gitlab::Utils::StrongMemoize, feature_category: :shared do end end + def method_name_with_expiration + strong_memoize_with_expiration(:method_name_with_expiration, 1) do + trace << value + value + end + end + def method_name_attr trace << value value @@ -142,6 +152,43 @@ RSpec.describe Gitlab::Utils::StrongMemoize, feature_category: :shared do end end + describe '#strong_memoize_with_expiration' do + [nil, false, true, 'value', 0, [0]].each do |value| + context "with value #{value}" do + let(:value) { value } + let(:method_name) { :method_name_with_expiration } + + it_behaves_like 'caching the value' + + it 'raises exception for invalid type as key' do + expect { object.strong_memoize_with_expiration(10, 1) { 20 } }.to raise_error /Invalid type of '10'/ + end + + it 'raises exception for invalid characters in key' do + expect { object.strong_memoize_with_expiration(:enabled?, 1) { 20 } } + .to raise_error /is not allowed as an instance variable name/ + end + end + end + + context 'value memoization test' do + let(:value) { 'value' } + + it 'caches the value for specified number of seconds' do + object.method_name_with_expiration + object.method_name_with_expiration + + expect(object.trace.count).to eq(1) + + travel_to(Time.current + 2.seconds) do + object.method_name_with_expiration + + expect(object.trace.count).to eq(2) + end + end + end + end + describe '#strong_memoize_with' do [nil, false, true, 'value', 0, [0]].each do |value| context "with value #{value}" do @@ -215,19 +262,21 @@ RSpec.describe Gitlab::Utils::StrongMemoize, feature_category: :shared do describe '.strong_memoize_attr' do [nil, false, true, 'value', 0, [0]].each do |value| - let(:value) { value } + context "with value '#{value}'" do + let(:value) { value } - context "memoized after method definition with value #{value}" do - let(:method_name) { :method_name_attr } + context 'memoized after method definition' do + let(:method_name) { :method_name_attr } - it_behaves_like 'caching the value' + it_behaves_like 'caching the value' - it 'calls the existing .method_added' do - expect(klass.method_added_list).to include(:method_name_attr) - end + it 'calls the existing .method_added' do + expect(klass.method_added_list).to include(:method_name_attr) + end - it 'retains method arity' do - expect(klass.instance_method(method_name).arity).to eq(0) + it 'retains method arity' do + expect(klass.instance_method(method_name).arity).to eq(0) + end end end end |