diff options
Diffstat (limited to 'spec/lib/gitlab')
253 files changed, 7336 insertions, 2468 deletions
diff --git a/spec/lib/gitlab/analytics/cycle_analytics/average_spec.rb b/spec/lib/gitlab/analytics/cycle_analytics/average_spec.rb index de325454b34..122a94a39c2 100644 --- a/spec/lib/gitlab/analytics/cycle_analytics/average_spec.rb +++ b/spec/lib/gitlab/analytics/cycle_analytics/average_spec.rb @@ -40,7 +40,7 @@ RSpec.describe Gitlab::Analytics::CycleAnalytics::Average do subject(:average_duration_in_seconds) { average.seconds } context 'when no results' do - let(:query) { Issue.none } + let(:query) { Issue.joins(:metrics).none } it { is_expected.to eq(nil) } end @@ -54,7 +54,7 @@ RSpec.describe Gitlab::Analytics::CycleAnalytics::Average do subject(:average_duration_in_days) { average.days } context 'when no results' do - let(:query) { Issue.none } + let(:query) { Issue.joins(:metrics).none } it { is_expected.to eq(nil) } end diff --git a/spec/lib/gitlab/analytics/cycle_analytics/request_params_spec.rb b/spec/lib/gitlab/analytics/cycle_analytics/request_params_spec.rb new file mode 100644 index 00000000000..3c171d684d6 --- /dev/null +++ b/spec/lib/gitlab/analytics/cycle_analytics/request_params_spec.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Analytics::CycleAnalytics::RequestParams, feature_category: :value_stream_management do + it_behaves_like 'unlicensed cycle analytics request params' do + let_it_be(:user) { create(:user) } + let_it_be(:root_group) { create(:group) } + let_it_be_with_refind(:project) { create(:project, group: root_group) } + + let(:namespace) { project.project_namespace } + + describe 'project-level data attributes' do + subject(:attributes) { described_class.new(params).to_data_attributes } + + it 'includes the namespace attribute' do + expect(attributes).to match(hash_including({ + namespace: { + name: project.name, + full_path: project.full_path + } + })) + end + end + end +end diff --git a/spec/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event_spec.rb b/spec/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event_spec.rb index 1e0034e386e..24248c557bd 100644 --- a/spec/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event_spec.rb +++ b/spec/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event_spec.rb @@ -1,8 +1,8 @@ # frozen_string_literal: true -require 'fast_spec_helper' +require 'spec_helper' -RSpec.describe Gitlab::Analytics::CycleAnalytics::StageEvents::StageEvent do +RSpec.describe Gitlab::Analytics::CycleAnalytics::StageEvents::StageEvent, feature_category: :product_analytics do let(:instance) { described_class.new({}) } it { expect(described_class).to respond_to(:name) } diff --git a/spec/lib/gitlab/api_authentication/token_resolver_spec.rb b/spec/lib/gitlab/api_authentication/token_resolver_spec.rb index c0c8e7aba63..48cae42dcd2 100644 --- a/spec/lib/gitlab/api_authentication/token_resolver_spec.rb +++ b/spec/lib/gitlab/api_authentication/token_resolver_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::APIAuthentication::TokenResolver, feature_category: :authentication_and_authorization do +RSpec.describe Gitlab::APIAuthentication::TokenResolver, feature_category: :system_access do let_it_be(:user) { create(:user) } let_it_be(:project, reload: true) { create(:project, :public) } let_it_be(:personal_access_token) { create(:personal_access_token, user: user) } diff --git a/spec/lib/gitlab/app_logger_spec.rb b/spec/lib/gitlab/app_logger_spec.rb index 85ca60d539f..e3415f4ad8c 100644 --- a/spec/lib/gitlab/app_logger_spec.rb +++ b/spec/lib/gitlab/app_logger_spec.rb @@ -5,22 +5,27 @@ require 'spec_helper' RSpec.describe Gitlab::AppLogger do subject { described_class } - it 'builds two Logger instances' do - expect(Gitlab::Logger).to receive(:new).and_call_original - expect(Gitlab::JsonLogger).to receive(:new).and_call_original + context 'when UNSTRUCTURED_RAILS_LOG is enabled' do + before do + stub_env('UNSTRUCTURED_RAILS_LOG', 'true') + end - subject.info('Hello World!') - end + it 'builds two Logger instances' do + expect(Gitlab::Logger).to receive(:new).and_call_original + expect(Gitlab::JsonLogger).to receive(:new).and_call_original - it 'logs info to AppLogger and AppJsonLogger' do - expect_any_instance_of(Gitlab::AppTextLogger).to receive(:info).and_call_original - expect_any_instance_of(Gitlab::AppJsonLogger).to receive(:info).and_call_original + subject.info('Hello World!') + end - subject.info('Hello World!') + it 'logs info to AppLogger and AppJsonLogger' do + expect_any_instance_of(Gitlab::AppTextLogger).to receive(:info).and_call_original + expect_any_instance_of(Gitlab::AppJsonLogger).to receive(:info).and_call_original + + subject.info('Hello World!') + end end it 'logs info to only the AppJsonLogger when unstructured logs are disabled' do - stub_env('UNSTRUCTURED_RAILS_LOG', 'false') expect_any_instance_of(Gitlab::AppTextLogger).not_to receive(:info).and_call_original expect_any_instance_of(Gitlab::AppJsonLogger).to receive(:info).and_call_original diff --git a/spec/lib/gitlab/asciidoc_spec.rb b/spec/lib/gitlab/asciidoc_spec.rb index cb9d1e9eae8..21cd87e89dc 100644 --- a/spec/lib/gitlab/asciidoc_spec.rb +++ b/spec/lib/gitlab/asciidoc_spec.rb @@ -369,7 +369,7 @@ module Gitlab <div> <div> <div class="gl-relative markdown-code-block js-markdown-code"> - <pre lang="javascript" class="code highlight js-syntax-highlight language-javascript" data-canonical-lang="js" v-pre="true"><code><span id="LC1" class="line" lang="javascript"><span class="nx">console</span><span class="p">.</span><span class="nx">log</span><span class="p">(</span><span class="dl">'</span><span class="s1">hello world</span><span class="dl">'</span><span class="p">)</span></span></code></pre> + <pre lang="javascript" class="code highlight js-syntax-highlight language-javascript" data-canonical-lang="js" v-pre="true"><code><span id="LC1" class="line" lang="javascript"><span class="nx">console</span><span class="p">.</span><span class="nf">log</span><span class="p">(</span><span class="dl">'</span><span class="s1">hello world</span><span class="dl">'</span><span class="p">)</span></span></code></pre> <copy-code></copy-code> </div> </div> diff --git a/spec/lib/gitlab/audit/auditor_spec.rb b/spec/lib/gitlab/audit/auditor_spec.rb index 4b16333d913..2b3c8506440 100644 --- a/spec/lib/gitlab/audit/auditor_spec.rb +++ b/spec/lib/gitlab/audit/auditor_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Audit::Auditor do +RSpec.describe Gitlab::Audit::Auditor, feature_category: :audit_events do let(:name) { 'audit_operation' } let(:author) { create(:user, :with_sign_ins) } let(:group) { create(:group) } @@ -22,9 +22,9 @@ RSpec.describe Gitlab::Audit::Auditor do subject(:auditor) { described_class } describe '.audit' do - context 'when authentication event' do - let(:audit!) { auditor.audit(context) } + let(:audit!) { auditor.audit(context) } + context 'when authentication event' do it 'creates an authentication event' do expect(AuthenticationEvent).to receive(:new).with( { @@ -210,19 +210,38 @@ RSpec.describe Gitlab::Audit::Auditor do end context 'when authentication event is false' do + let(:target) { group } let(:context) do { name: name, author: author, scope: group, - target: group, authentication_event: false, message: "sample message" } + target: target, authentication_event: false, message: "sample message" } end it 'does not create an authentication event' do expect { auditor.audit(context) }.not_to change(AuthenticationEvent, :count) end + + context 'with permitted target' do + { feature_flag: :operations_feature_flag }.each do |target_type, factory_name| + context "with #{target_type}" do + let(:target) { build_stubbed factory_name } + + it 'logs audit events to database', :aggregate_failures, :freeze_time do + audit! + audit_event = AuditEvent.last + + expect(audit_event.author_id).to eq(author.id) + expect(audit_event.entity_id).to eq(group.id) + expect(audit_event.entity_type).to eq(group.class.name) + expect(audit_event.created_at).to eq(Time.zone.now) + expect(audit_event.details[:target_id]).to eq(target.id) + expect(audit_event.details[:target_type]).to eq(target.class.name) + end + end + end + end end context 'when authentication event is invalid' do - let(:audit!) { auditor.audit(context) } - before do allow(AuthenticationEvent).to receive(:new).and_raise(ActiveRecord::RecordInvalid) allow(Gitlab::ErrorTracking).to receive(:track_exception) @@ -243,8 +262,6 @@ RSpec.describe Gitlab::Audit::Auditor do end context 'when audit events are invalid' do - let(:audit!) { auditor.audit(context) } - before do expect_next_instance_of(AuditEvent) do |instance| allow(instance).to receive(:save!).and_raise(ActiveRecord::RecordInvalid) diff --git a/spec/lib/gitlab/auth/auth_finders_spec.rb b/spec/lib/gitlab/auth/auth_finders_spec.rb index 6aedd0a0a23..ecc5c688228 100644 --- a/spec/lib/gitlab/auth/auth_finders_spec.rb +++ b/spec/lib/gitlab/auth/auth_finders_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Auth::AuthFinders, feature_category: :authentication_and_authorization do +RSpec.describe Gitlab::Auth::AuthFinders, feature_category: :system_access do include described_class include HttpBasicAuthHelpers diff --git a/spec/lib/gitlab/auth/o_auth/user_spec.rb b/spec/lib/gitlab/auth/o_auth/user_spec.rb index 04fbbff3559..8a86b306604 100644 --- a/spec/lib/gitlab/auth/o_auth/user_spec.rb +++ b/spec/lib/gitlab/auth/o_auth/user_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Auth::OAuth::User, feature_category: :authentication_and_authorization do +RSpec.describe Gitlab::Auth::OAuth::User, feature_category: :system_access do include LdapHelpers let(:oauth_user) { described_class.new(auth_hash) } diff --git a/spec/lib/gitlab/auth/otp/strategies/duo_auth/manual_otp_spec.rb b/spec/lib/gitlab/auth/otp/strategies/duo_auth/manual_otp_spec.rb new file mode 100644 index 00000000000..d04e0ad9fb4 --- /dev/null +++ b/spec/lib/gitlab/auth/otp/strategies/duo_auth/manual_otp_spec.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Auth::Otp::Strategies::DuoAuth::ManualOtp, feature_category: :system_access do + let_it_be(:user) { create(:user) } + + let_it_be(:otp_code) { 42 } + + let_it_be(:hostname) { 'duo_auth.example.com' } + let_it_be(:integration_key) { 'int3gr4t1on' } + let_it_be(:secret_key) { 's3cr3t' } + + let_it_be(:duo_response_builder) { Struct.new(:body) } + + let_it_be(:response_status) { 200 } + + let_it_be(:duo_auth_url) { "https://#{hostname}/auth/v2/auth/" } + let_it_be(:params) do + { username: user.username, + factor: "passcode", + passcode: otp_code } + end + + let_it_be(:manual_otp) { described_class.new(user) } + + subject(:response) { manual_otp.validate(otp_code) } + + before do + stub_duo_auth_config( + enabled: true, + hostname: hostname, + secret_key: secret_key, + integration_key: integration_key + ) + end + + context 'when successful validation' do + before do + allow(duo_client).to receive(:request) + .with("POST", "/auth/v2/auth", params) + .and_return(duo_response_builder.new('{ "response": { "result": "allow" }}')) + + allow(manual_otp).to receive(:duo_client).and_return(duo_client) + end + + it 'returns success' do + response + + expect(response[:status]).to eq(:success) + end + end + + context 'when unsuccessful validation' do + before do + allow(duo_client).to receive(:request) + .with("POST", "/auth/v2/auth", params) + .and_return(duo_response_builder.new('{ "response": { "result": "deny" }}')) + + allow(manual_otp).to receive(:duo_client).and_return(duo_client) + end + + it 'returns error' do + response + + expect(response[:status]).to eq(:error) + end + end + + context 'when unexpected error' do + before do + allow(duo_client).to receive(:request) + .with("POST", "/auth/v2/auth", params) + .and_return(duo_response_builder.new('aaa')) + + allow(manual_otp).to receive(:duo_client).and_return(duo_client) + end + + it 'returns error' do + response + + expect(response[:status]).to eq(:error) + expect(response[:message]).to match(/unexpected character/) + end + end + + def stub_duo_auth_config(duo_auth_settings) + allow(::Gitlab.config.duo_auth).to(receive_messages(duo_auth_settings)) + end + + def duo_client + manual_otp.send(:duo_client) + end +end diff --git a/spec/lib/gitlab/auth_spec.rb b/spec/lib/gitlab/auth_spec.rb index a5f46aa1f35..11e9ecdb878 100644 --- a/spec/lib/gitlab/auth_spec.rb +++ b/spec/lib/gitlab/auth_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching, feature_category: :authentication_and_authorization do +RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching, feature_category: :system_access do let_it_be(:project) { create(:project) } let(:auth_failure) { { actor: nil, project: nil, type: nil, authentication_abilities: nil } } diff --git a/spec/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens_spec.rb b/spec/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens_spec.rb index 7075d4694ae..d2da6867773 100644 --- a/spec/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::BackgroundMigration::BackfillAdminModeScopeForPersonalAccessTokens, - :migration, schema: 20221228103133, feature_category: :authentication_and_authorization do + :migration, schema: 20221228103133, feature_category: :system_access do let(:users) { table(:users) } let(:personal_access_tokens) { table(:personal_access_tokens) } diff --git a/spec/lib/gitlab/background_migration/backfill_prepared_at_merge_requests_spec.rb b/spec/lib/gitlab/background_migration/backfill_prepared_at_merge_requests_spec.rb new file mode 100644 index 00000000000..b33a1a31c40 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_prepared_at_merge_requests_spec.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillPreparedAtMergeRequests, :migration, + feature_category: :code_review_workflow, schema: 20230202135758 do + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:mr_table) { table(:merge_requests) } + + let(:namespace) { namespaces.create!(name: 'batchtest1', type: 'Group', path: 'space1') } + let(:proj_namespace) { namespaces.create!(name: 'proj1', path: 'proj1', type: 'Project', parent_id: namespace.id) } + let(:project) do + projects.create!(name: 'proj1', path: 'proj1', namespace_id: namespace.id, project_namespace_id: proj_namespace.id) + end + + let(:test_worker) do + described_class.new( + start_id: 1, + end_id: 100, + batch_table: :merge_requests, + batch_column: :id, + sub_batch_size: 10, + pause_ms: 0, + connection: ApplicationRecord.connection + ) + end + + it 'updates merge requests with prepared_at nil' do + time = Time.current + + mr_1 = mr_table.create!(target_project_id: project.id, source_branch: 'master', target_branch: 'feature', + prepared_at: nil, merge_status: 'checking') + mr_2 = mr_table.create!(target_project_id: project.id, source_branch: 'master', target_branch: 'feature', + prepared_at: nil, merge_status: 'preparing') + mr_3 = mr_table.create!(target_project_id: project.id, source_branch: 'master', target_branch: 'feature', + prepared_at: time) + mr_4 = mr_table.create!(target_project_id: project.id, source_branch: 'master', target_branch: 'feature', + prepared_at: time, merge_status: 'checking') + mr_5 = mr_table.create!(target_project_id: project.id, source_branch: 'master', target_branch: 'feature', + prepared_at: time, merge_status: 'preparing') + + expect(mr_1.prepared_at).to be_nil + expect(mr_2.prepared_at).to be_nil + expect(mr_3.prepared_at.to_i).to eq(time.to_i) + expect(mr_4.prepared_at.to_i).to eq(time.to_i) + expect(mr_5.prepared_at.to_i).to eq(time.to_i) + + test_worker.perform + + expect(mr_1.reload.prepared_at.to_i).to eq(mr_1.created_at.to_i) + expect(mr_2.reload.prepared_at).to be_nil + expect(mr_3.reload.prepared_at.to_i).to eq(time.to_i) + expect(mr_4.reload.prepared_at.to_i).to eq(time.to_i) + expect(mr_5.reload.prepared_at.to_i).to eq(time.to_i) + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_project_wiki_repositories_spec.rb b/spec/lib/gitlab/background_migration/backfill_project_wiki_repositories_spec.rb new file mode 100644 index 00000000000..e81bd0604e6 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_project_wiki_repositories_spec.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe( + Gitlab::BackgroundMigration::BackfillProjectWikiRepositories, + schema: 20230306195007, + feature_category: :geo_replication) do + let!(:namespaces) { table(:namespaces) } + let!(:projects) { table(:projects) } + let!(:project_wiki_repositories) { table(:project_wiki_repositories) } + + subject(:migration) do + described_class.new( + start_id: projects.minimum(:id), + end_id: projects.maximum(:id), + batch_table: :projects, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ) + end + + describe '#perform' do + it 'creates project_wiki_repositories entries for all projects in range' do + namespace1 = create_namespace('test1') + namespace2 = create_namespace('test2') + project1 = create_project(namespace1, 'test1') + project2 = create_project(namespace2, 'test2') + project_wiki_repositories.create!(project_id: project2.id) + + expect { migration.perform } + .to change { project_wiki_repositories.pluck(:project_id) } + .from([project2.id]) + .to match_array([project1.id, project2.id]) + end + + it 'does nothing if project_id already exist in project_wiki_repositories' do + namespace = create_namespace('test1') + project = create_project(namespace, 'test1') + project_wiki_repositories.create!(project_id: project.id) + + expect { migration.perform } + .not_to change { project_wiki_repositories.pluck(:project_id) } + end + + def create_namespace(name) + namespaces.create!( + name: name, + path: name, + type: 'Project' + ) + end + + def create_project(namespace, name) + projects.create!( + namespace_id: namespace.id, + project_namespace_id: namespace.id, + name: name, + path: name + ) + end + end +end diff --git a/spec/lib/gitlab/background_migration/batched_migration_job_spec.rb b/spec/lib/gitlab/background_migration/batched_migration_job_spec.rb index faaaccfdfaf..781bf93dd85 100644 --- a/spec/lib/gitlab/background_migration/batched_migration_job_spec.rb +++ b/spec/lib/gitlab/background_migration/batched_migration_job_spec.rb @@ -301,6 +301,28 @@ RSpec.describe Gitlab::BackgroundMigration::BatchedMigrationJob do perform_job end + context 'when using a sub batch exception for timeouts' do + let(:job_class) do + Class.new(described_class) do + operation_name :update + + def perform(*_) + each_sub_batch { raise ActiveRecord::StatementTimeout } # rubocop:disable Lint/UnreachableLoop + end + end + end + + let(:job_instance) do + job_class.new(start_id: 1, end_id: 10, batch_table: '_test_table', batch_column: 'id', + sub_batch_size: 2, pause_ms: 1000, connection: connection, + sub_batch_exception: StandardError) + end + + it 'raises the expected error type' do + expect { job_instance.perform }.to raise_error(StandardError) + end + end + context 'when batching_arguments are given' do it 'forwards them for batching' do expect(job_instance).to receive(:base_relation).and_return(test_table) diff --git a/spec/lib/gitlab/background_migration/delete_orphaned_packages_dependencies_spec.rb b/spec/lib/gitlab/background_migration/delete_orphaned_packages_dependencies_spec.rb new file mode 100644 index 00000000000..0d82717c7de --- /dev/null +++ b/spec/lib/gitlab/background_migration/delete_orphaned_packages_dependencies_spec.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::DeleteOrphanedPackagesDependencies, schema: 20230303105806, + feature_category: :package_registry do + let!(:migration_attrs) do + { + start_id: 1, + end_id: 1000, + batch_table: :packages_dependencies, + batch_column: :id, + sub_batch_size: 500, + pause_ms: 0, + connection: ApplicationRecord.connection + } + end + + let!(:migration) { described_class.new(**migration_attrs) } + + let(:packages_dependencies) { table(:packages_dependencies) } + + let!(:namespace) { table(:namespaces).create!(name: 'project', path: 'project', type: 'Project') } + let!(:project) do + table(:projects).create!(name: 'project', path: 'project', project_namespace_id: namespace.id, + namespace_id: namespace.id) + end + + let!(:package) do + table(:packages_packages).create!(name: 'test', version: '1.2.3', package_type: 2, project_id: project.id) + end + + let!(:orphan_dependency_1) { packages_dependencies.create!(name: 'dependency 1', version_pattern: '~0.0.1') } + let!(:orphan_dependency_2) { packages_dependencies.create!(name: 'dependency 2', version_pattern: '~0.0.2') } + let!(:orphan_dependency_3) { packages_dependencies.create!(name: 'dependency 3', version_pattern: '~0.0.3') } + let!(:linked_dependency) do + packages_dependencies.create!(name: 'dependency 4', version_pattern: '~0.0.4').tap do |dependency| + table(:packages_dependency_links).create!(package_id: package.id, dependency_id: dependency.id, + dependency_type: 'dependencies') + end + end + + subject(:perform_migration) { migration.perform } + + it 'executes 3 queries' do + queries = ActiveRecord::QueryRecorder.new do + perform_migration + end + + expect(queries.count).to eq(3) + end + + it 'deletes only orphaned dependencies' do + expect { perform_migration }.to change { packages_dependencies.count }.by(-3) + expect(packages_dependencies.all).to eq([linked_dependency]) + end +end diff --git a/spec/lib/gitlab/background_migration/fix_vulnerability_reads_has_issues_spec.rb b/spec/lib/gitlab/background_migration/fix_vulnerability_reads_has_issues_spec.rb new file mode 100644 index 00000000000..9f431c43f39 --- /dev/null +++ b/spec/lib/gitlab/background_migration/fix_vulnerability_reads_has_issues_spec.rb @@ -0,0 +1,100 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::FixVulnerabilityReadsHasIssues, schema: 20230302185739, feature_category: :vulnerability_management do # rubocop:disable Layout/LineLength + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:users) { table(:users) } + let(:scanners) { table(:vulnerability_scanners) } + let(:vulnerabilities) { table(:vulnerabilities) } + let(:vulnerability_reads) { table(:vulnerability_reads) } + let(:work_item_types) { table(:work_item_types) } + let(:issues) { table(:issues) } + let(:vulnerability_issue_links) { table(:vulnerability_issue_links) } + + let(:namespace) { namespaces.create!(name: 'user', path: 'user') } + let(:project) { projects.create!(namespace_id: namespace.id, project_namespace_id: namespace.id) } + let(:user) { users.create!(username: 'john_doe', email: 'johndoe@gitlab.com', projects_limit: 10) } + let(:scanner) { scanners.create!(project_id: project.id, external_id: 'external_id', name: 'Test Scanner') } + let(:work_item_type) { work_item_types.create!(name: 'test') } + + let(:vulnerability_records) do + Array.new(4).map do |_, n| + vulnerabilities.create!( + project_id: project.id, + author_id: user.id, + title: "vulnerability #{n}", + severity: 1, + confidence: 1, + report_type: 1 + ) + end + end + + let(:vulnerabilities_with_issues) { [vulnerability_records.first, vulnerability_records.third] } + let(:vulnerabilities_without_issues) { vulnerability_records - vulnerabilities_with_issues } + + let(:vulnerability_read_records) do + vulnerability_records.map do |vulnerability| + vulnerability_reads.create!( + project_id: project.id, + vulnerability_id: vulnerability.id, + scanner_id: scanner.id, + has_issues: false, + severity: 1, + report_type: 1, + state: 1, + uuid: SecureRandom.uuid + ) + end + end + + let!(:issue_links) do + vulnerabilities_with_issues.map do |vulnerability| + issue = issues.create!( + title: vulnerability.title, + author_id: user.id, + project_id: project.id, + confidential: true, + work_item_type_id: work_item_type.id, + namespace_id: namespace.id + ) + + vulnerability_issue_links.create!( + vulnerability_id: vulnerability.id, + issue_id: issue.id + ) + end + end + + def vulnerability_read_for(vulnerability) + vulnerability_read_records.find { |read| read.vulnerability_id == vulnerability.id } + end + + subject(:perform_migration) do + described_class.new( + start_id: issue_links.first.vulnerability_id, + end_id: issue_links.last.vulnerability_id, + batch_table: :vulnerability_issue_links, + batch_column: :vulnerability_id, + sub_batch_size: issue_links.size, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ).perform + end + + it 'only changes records with issue links' do + expect(vulnerability_read_records).to all(have_attributes(has_issues: false)) + + perform_migration + + vulnerabilities_with_issues.each do |vulnerability| + expect(vulnerability_read_for(vulnerability).reload.has_issues).to eq(true) + end + + vulnerabilities_without_issues.each do |vulnerability| + expect(vulnerability_read_for(vulnerability).reload.has_issues).to eq(false) + end + end +end diff --git a/spec/lib/gitlab/background_migration/issues_internal_id_scope_updater_spec.rb b/spec/lib/gitlab/background_migration/issues_internal_id_scope_updater_spec.rb new file mode 100644 index 00000000000..1adff322b41 --- /dev/null +++ b/spec/lib/gitlab/background_migration/issues_internal_id_scope_updater_spec.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +require 'spec_helper' +# this needs the schema to be before we introduce the not null constraint on routes#namespace_id +# rubocop:disable RSpec/MultipleMemoizedHelpers +RSpec.describe Gitlab::BackgroundMigration::IssuesInternalIdScopeUpdater, feature_category: :team_planning do + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:internal_ids) { table(:internal_ids) } + + let(:gr1) { namespaces.create!(name: 'batchtest1', type: 'Group', path: 'space1') } + let(:gr2) { namespaces.create!(name: 'batchtest2', type: 'Group', parent_id: gr1.id, path: 'space2') } + + let(:pr_nmsp1) { namespaces.create!(name: 'proj1', path: 'proj1', type: 'Project', parent_id: gr1.id) } + let(:pr_nmsp2) { namespaces.create!(name: 'proj2', path: 'proj2', type: 'Project', parent_id: gr1.id) } + let(:pr_nmsp3) { namespaces.create!(name: 'proj3', path: 'proj3', type: 'Project', parent_id: gr2.id) } + let(:pr_nmsp4) { namespaces.create!(name: 'proj4', path: 'proj4', type: 'Project', parent_id: gr2.id) } + let(:pr_nmsp5) { namespaces.create!(name: 'proj5', path: 'proj5', type: 'Project', parent_id: gr2.id) } + let(:pr_nmsp6) { namespaces.create!(name: 'proj6', path: 'proj6', type: 'Project', parent_id: gr2.id) } + + # rubocop:disable Layout/LineLength + let(:p1) { projects.create!(name: 'proj1', path: 'proj1', namespace_id: gr1.id, project_namespace_id: pr_nmsp1.id) } + let(:p2) { projects.create!(name: 'proj2', path: 'proj2', namespace_id: gr1.id, project_namespace_id: pr_nmsp2.id) } + let(:p3) { projects.create!(name: 'proj3', path: 'proj3', namespace_id: gr2.id, project_namespace_id: pr_nmsp3.id) } + let(:p4) { projects.create!(name: 'proj4', path: 'proj4', namespace_id: gr2.id, project_namespace_id: pr_nmsp4.id) } + let(:p5) { projects.create!(name: 'proj5', path: 'proj5', namespace_id: gr2.id, project_namespace_id: pr_nmsp5.id) } + let(:p6) { projects.create!(name: 'proj6', path: 'proj6', namespace_id: gr2.id, project_namespace_id: pr_nmsp6.id) } + # rubocop:enable Layout/LineLength + + # a project that already is covered by a record for its namespace. This should result in no new record added and + # project related record deleted + let!(:issues_internal_ids_p1) { internal_ids.create!(project_id: p1.id, usage: 0, last_value: 100) } + let!(:issues_internal_ids_pr_nmsp1) { internal_ids.create!(namespace_id: pr_nmsp1.id, usage: 0, last_value: 111) } + + # project records that do not have a corresponding namespace record. This should result 2 new records + # scoped to corresponding project namespaces being added and the project related records being deleted. + let!(:issues_internal_ids_p2) { internal_ids.create!(project_id: p2.id, usage: 0, last_value: 200) } + let!(:issues_internal_ids_p3) { internal_ids.create!(project_id: p3.id, usage: 0, last_value: 300) } + + # a project record on a different usage, should not be affected by the migration and + # no new record should be created for this case + let!(:issues_internal_ids_p4) { internal_ids.create!(project_id: p4.id, usage: 4, last_value: 400) } + + # a project namespace scoped record without a corresponding project record, should not affect anything. + let!(:issues_internal_ids_pr_nmsp5) { internal_ids.create!(namespace_id: pr_nmsp5.id, usage: 0, last_value: 500) } + + # a record scoped to a group, should not affect anything. + let!(:issues_internal_ids_gr1) { internal_ids.create!(namespace_id: gr1.id, usage: 0, last_value: 600) } + + # a project that is covered by a record for its namespace, but has a higher last_value, due to updates during rolling + # deploy for instance, see https://gitlab.com/gitlab-com/gl-infra/production/-/issues/8548 + let!(:issues_internal_ids_p6) { internal_ids.create!(project_id: p6.id, usage: 0, last_value: 111) } + let!(:issues_internal_ids_pr_nmsp6) { internal_ids.create!(namespace_id: pr_nmsp6.id, usage: 0, last_value: 100) } + + subject(:perform_migration) do + described_class.new( + start_id: internal_ids.minimum(:id), + end_id: internal_ids.maximum(:id), + batch_table: :internal_ids, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ).perform + end + + it 'backfills internal_ids records and removes related project records', :aggregate_failures do + perform_migration + + expected_recs = [pr_nmsp1.id, pr_nmsp2.id, pr_nmsp3.id, pr_nmsp5.id, gr1.id, pr_nmsp6.id] + + # all namespace scoped records for issues(0) usage + expect(internal_ids.where.not(namespace_id: nil).where(usage: 0).count).to eq(6) + # all namespace_ids for issues(0) usage + expect(internal_ids.where.not(namespace_id: nil).where(usage: 0).pluck(:namespace_id)).to match_array(expected_recs) + # this is the record with usage: 4 + expect(internal_ids.where.not(project_id: nil).count).to eq(1) + # no project scoped records for issues usage left + expect(internal_ids.where.not(project_id: nil).where(usage: 0).count).to eq(0) + + # the case when the project_id scoped record had the higher last_value, + # see `issues_internal_ids_p6` and issues_internal_ids_pr_nmsp6 definitions above + expect(internal_ids.where(namespace_id: pr_nmsp6.id).first.last_value).to eq(111) + + # the case when the namespace_id scoped record had the higher last_value, + # see `issues_internal_ids_p1` and issues_internal_ids_pr_nmsp1 definitions above. + expect(internal_ids.where(namespace_id: pr_nmsp1.id).first.last_value).to eq(111) + end +end +# rubocop:enable RSpec/MultipleMemoizedHelpers diff --git a/spec/lib/gitlab/background_migration/migrate_evidences_for_vulnerability_findings_spec.rb b/spec/lib/gitlab/background_migration/migrate_evidences_for_vulnerability_findings_spec.rb new file mode 100644 index 00000000000..b70044ab2a4 --- /dev/null +++ b/spec/lib/gitlab/background_migration/migrate_evidences_for_vulnerability_findings_spec.rb @@ -0,0 +1,136 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::MigrateEvidencesForVulnerabilityFindings, + feature_category: :vulnerability_management do + let(:vulnerability_occurrences) { table(:vulnerability_occurrences) } + let(:vulnerability_finding_evidences) { table(:vulnerability_finding_evidences) } + let(:evidence_hash) { { url: 'http://test.com' } } + let(:namespace1) { table(:namespaces).create!(name: 'namespace 1', path: 'namespace1') } + let(:project1) { table(:projects).create!(namespace_id: namespace1.id, project_namespace_id: namespace1.id) } + let(:user) { table(:users).create!(email: 'test1@example.com', projects_limit: 5) } + + let(:scanner1) do + table(:vulnerability_scanners).create!(project_id: project1.id, external_id: 'test 1', name: 'test scanner 1') + end + + let(:stating_id) { vulnerability_occurrences.pluck(:id).min } + let(:end_id) { vulnerability_occurrences.pluck(:id).max } + + let(:migration) do + described_class.new( + start_id: stating_id, + end_id: end_id, + batch_table: :vulnerability_occurrences, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 2, + connection: ApplicationRecord.connection + ) + end + + subject(:perform_migration) { migration.perform } + + context 'without the presence of evidence key' do + before do + create_finding!(project1.id, scanner1.id, { other_keys: 'test' }) + end + + it 'does not create any evidence' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_finding_evidences.count } + end + end + + context 'with evidence equals to nil' do + before do + create_finding!(project1.id, scanner1.id, { evidence: nil }) + end + + it 'does not create any evidence' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_finding_evidences.count } + end + end + + context 'with existing evidence within raw_metadata' do + let!(:finding1) { create_finding!(project1.id, scanner1.id, { evidence: evidence_hash }) } + let!(:finding2) { create_finding!(project1.id, scanner1.id, { evidence: evidence_hash }) } + + it 'creates new evidence for each finding' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.to change { vulnerability_finding_evidences.count }.by(2) + end + + context 'when create throws exception StandardError' do + before do + allow(migration).to receive(:create_evidences).and_raise(StandardError) + end + + it 'logs StandardError' do + expect(Gitlab::AppLogger).to receive(:error).with({ + class: described_class.name, message: StandardError.to_s + }) + expect { perform_migration }.not_to change { vulnerability_finding_evidences.count } + end + end + + context 'when parse throws exception JSON::ParserError' do + before do + allow(Gitlab::Json).to receive(:parse).and_raise(JSON::ParserError) + end + + it 'does not log this error nor create new records' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_finding_evidences.count } + end + end + end + + context 'with existing evidence records' do + let!(:finding) { create_finding!(project1.id, scanner1.id, { evidence: evidence_hash }) } + + before do + vulnerability_finding_evidences.create!(vulnerability_occurrence_id: finding.id, data: evidence_hash) + end + + it 'does not create new evidence' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_finding_evidences.count } + end + + context 'with non-existing evidence' do + let!(:finding3) { create_finding!(project1.id, scanner1.id, { evidence: { url: 'http://secondary.com' } }) } + + it 'creates a new evidence only to the non-existing evidence' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.to change { vulnerability_finding_evidences.count }.by(1) + end + end + end + + private + + def create_finding!(project_id, scanner_id, raw_metadata) + vulnerability = table(:vulnerabilities).create!(project_id: project_id, author_id: user.id, title: 'test', + severity: 4, confidence: 4, report_type: 0) + + identifier = table(:vulnerability_identifiers).create!(project_id: project_id, external_type: 'uuid-v5', + external_id: 'uuid-v5', fingerprint: OpenSSL::Digest::SHA256.hexdigest(vulnerability.id.to_s), + name: 'Identifier for UUIDv5 2 2') + + table(:vulnerability_occurrences).create!( + vulnerability_id: vulnerability.id, project_id: project_id, scanner_id: scanner_id, + primary_identifier_id: identifier.id, name: 'test', severity: 4, confidence: 4, report_type: 0, + uuid: SecureRandom.uuid, project_fingerprint: '123qweasdzxc', location: { "image" => "alpine:3.4" }, + location_fingerprint: 'test', metadata_version: 'test', + raw_metadata: raw_metadata.to_json) + end +end diff --git a/spec/lib/gitlab/background_migration/migrate_links_for_vulnerability_findings_spec.rb b/spec/lib/gitlab/background_migration/migrate_links_for_vulnerability_findings_spec.rb new file mode 100644 index 00000000000..fd2e3ffb670 --- /dev/null +++ b/spec/lib/gitlab/background_migration/migrate_links_for_vulnerability_findings_spec.rb @@ -0,0 +1,141 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::MigrateLinksForVulnerabilityFindings, + feature_category: :vulnerability_management do + let(:vulnerability_occurrences) { table(:vulnerability_occurrences) } + let(:vulnerability_finding_links) { table(:vulnerability_finding_links) } + let(:link_hash) { { url: 'http://test.com' } } + let(:namespace1) { table(:namespaces).create!(name: 'namespace 1', path: 'namespace1') } + let(:project1) { table(:projects).create!(namespace_id: namespace1.id, project_namespace_id: namespace1.id) } + let(:user) { table(:users).create!(email: 'test1@example.com', projects_limit: 5) } + + let(:scanner1) do + table(:vulnerability_scanners).create!(project_id: project1.id, external_id: 'test 1', name: 'test scanner 1') + end + + let(:stating_id) { vulnerability_occurrences.pluck(:id).min } + let(:end_id) { vulnerability_occurrences.pluck(:id).max } + + let(:migration) do + described_class.new( + start_id: stating_id, + end_id: end_id, + batch_table: :vulnerability_occurrences, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 2, + connection: ApplicationRecord.connection + ) + end + + subject(:perform_migration) { migration.perform } + + context 'without the presence of links key' do + before do + create_finding!(project1.id, scanner1.id, { other_keys: 'test' }) + end + + it 'does not create any link' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_finding_links.count } + end + end + + context 'with links equals to an array of nil element' do + before do + create_finding!(project1.id, scanner1.id, { links: [nil] }) + end + + it 'does not create any link' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_finding_links.count } + end + end + + context 'with links equals to an array of duplicated elements' do + let!(:finding) do + create_finding!(project1.id, scanner1.id, { links: [link_hash, link_hash] }) + end + + it 'creates one new link' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.to change { vulnerability_finding_links.count }.by(1) + end + end + + context 'with existing links within raw_metadata' do + let!(:finding1) { create_finding!(project1.id, scanner1.id, { links: [link_hash] }) } + let!(:finding2) { create_finding!(project1.id, scanner1.id, { links: [link_hash] }) } + + it 'creates new link for each finding' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.to change { vulnerability_finding_links.count }.by(2) + end + + context 'when create throws exception ActiveRecord::RecordNotUnique' do + before do + allow(migration).to receive(:create_links).and_raise(ActiveRecord::RecordNotUnique) + end + + it 'does not log this error nor create new records' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_finding_links.count } + end + end + + context 'when create throws exception StandardError' do + before do + allow(migration).to receive(:create_links).and_raise(StandardError) + end + + it 'logs StandardError' do + expect(Gitlab::AppLogger).to receive(:error).with({ + class: described_class.name, message: StandardError.to_s, model_id: finding1.id + }) + expect(Gitlab::AppLogger).to receive(:error).with({ + class: described_class.name, message: StandardError.to_s, model_id: finding2.id + }) + expect { perform_migration }.not_to change { vulnerability_finding_links.count } + end + end + end + + context 'with existing link records' do + let!(:finding) { create_finding!(project1.id, scanner1.id, { links: [link_hash] }) } + + before do + vulnerability_finding_links.create!(vulnerability_occurrence_id: finding.id, url: link_hash[:url]) + end + + it 'does not create new link' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_finding_links.count } + end + end + + private + + def create_finding!(project_id, scanner_id, raw_metadata) + vulnerability = table(:vulnerabilities).create!(project_id: project_id, author_id: user.id, title: 'test', + severity: 4, confidence: 4, report_type: 0) + + identifier = table(:vulnerability_identifiers).create!(project_id: project_id, external_type: 'uuid-v5', + external_id: 'uuid-v5', fingerprint: OpenSSL::Digest::SHA256.hexdigest(vulnerability.id.to_s), + name: 'Identifier for UUIDv5 2 2') + + table(:vulnerability_occurrences).create!( + vulnerability_id: vulnerability.id, project_id: project_id, scanner_id: scanner_id, + primary_identifier_id: identifier.id, name: 'test', severity: 4, confidence: 4, report_type: 0, + uuid: SecureRandom.uuid, project_fingerprint: '123qweasdzxc', location: { "image" => "alpine:3.4" }, + location_fingerprint: 'test', metadata_version: 'test', + raw_metadata: raw_metadata.to_json) + end +end diff --git a/spec/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings_spec.rb b/spec/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings_spec.rb new file mode 100644 index 00000000000..b75c0e61b19 --- /dev/null +++ b/spec/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings_spec.rb @@ -0,0 +1,173 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::MigrateRemediationsForVulnerabilityFindings, + feature_category: :vulnerability_management do + let(:vulnerability_occurrences) { table(:vulnerability_occurrences) } + let(:vulnerability_findings_remediations) { table(:vulnerability_findings_remediations) } + let(:vulnerability_remediations) { table(:vulnerability_remediations) } + let(:remediation_hash) { { summary: 'summary', diff: "ZGlmZiAtLWdp" } } + let(:namespace1) { table(:namespaces).create!(name: 'namespace 1', path: 'namespace1') } + let(:project1) { table(:projects).create!(namespace_id: namespace1.id, project_namespace_id: namespace1.id) } + let(:user) { table(:users).create!(email: 'test1@example.com', projects_limit: 5) } + + let(:scanner1) do + table(:vulnerability_scanners).create!(project_id: project1.id, external_id: 'test 1', name: 'test scanner 1') + end + + let(:stating_id) { vulnerability_occurrences.pluck(:id).min } + let(:end_id) { vulnerability_occurrences.pluck(:id).max } + + let(:migration) do + described_class.new( + start_id: stating_id, + end_id: end_id, + batch_table: :vulnerability_occurrences, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 2, + connection: ApplicationRecord.connection + ) + end + + subject(:perform_migration) { migration.perform } + + context 'without the presence of remediation key' do + before do + create_finding!(project1.id, scanner1.id, { other_keys: 'test' }) + end + + it 'does not create any remediation' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_remediations.count } + end + end + + context 'with remediation equals to an array of nil element' do + before do + create_finding!(project1.id, scanner1.id, { remediations: [nil] }) + end + + it 'does not create any remediation' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_remediations.count } + end + end + + context 'with remediation with empty string as the diff key' do + let!(:finding) do + create_finding!(project1.id, scanner1.id, { remediations: [{ summary: 'summary', diff: '' }] }) + end + + it 'does not create any remediation' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_remediations.count } + end + end + + context 'with remediation equals to an array of duplicated elements' do + let!(:finding) do + create_finding!(project1.id, scanner1.id, { remediations: [remediation_hash, remediation_hash] }) + end + + it 'creates new remediation' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.to change { vulnerability_remediations.count }.by(1) + expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding.id).length).to eq(1) + end + end + + context 'with existing remediations within raw_metadata' do + let!(:finding1) { create_finding!(project1.id, scanner1.id, { remediations: [remediation_hash] }) } + let!(:finding2) { create_finding!(project1.id, scanner1.id, { remediations: [remediation_hash] }) } + + it 'creates new remediation' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.to change { vulnerability_remediations.count }.by(1) + expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding1.id).length).to eq(1) + expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding2.id).length).to eq(1) + end + + context 'when create throws exception other than ActiveRecord::RecordNotUnique' do + before do + allow(migration).to receive(:create_finding_remediations).and_raise(StandardError) + end + + it 'rolls back all related transactions' do + expect(Gitlab::AppLogger).to receive(:error).with({ + class: described_class.name, message: StandardError.to_s, model_id: finding1.id + }) + expect(Gitlab::AppLogger).to receive(:error).with({ + class: described_class.name, message: StandardError.to_s, model_id: finding2.id + }) + expect { perform_migration }.not_to change { vulnerability_remediations.count } + expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding1.id).length).to eq(0) + expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding2.id).length).to eq(0) + end + end + end + + context 'with existing remediation records' do + let!(:finding) { create_finding!(project1.id, scanner1.id, { remediations: [remediation_hash] }) } + + before do + vulnerability_remediations.create!(project_id: project1.id, summary: remediation_hash[:summary], + checksum: checksum(remediation_hash[:diff]), file: Tempfile.new.path) + end + + it 'does not create new remediation' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_remediations.count } + expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding.id).length).to eq(1) + end + end + + context 'with same raw_metadata for different projects' do + let(:namespace2) { table(:namespaces).create!(name: 'namespace 2', path: 'namespace2') } + let(:project2) { table(:projects).create!(namespace_id: namespace2.id, project_namespace_id: namespace2.id) } + let(:scanner2) do + table(:vulnerability_scanners).create!(project_id: project2.id, external_id: 'test 2', name: 'test scanner 2') + end + + let!(:finding1) { create_finding!(project1.id, scanner1.id, { remediations: [remediation_hash] }) } + let!(:finding2) { create_finding!(project2.id, scanner2.id, { remediations: [remediation_hash] }) } + + it 'creates new remediation for each project' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.to change { vulnerability_remediations.count }.by(2) + expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding1.id).length).to eq(1) + expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding2.id).length).to eq(1) + end + end + + private + + def create_finding!(project_id, scanner_id, raw_metadata) + vulnerability = table(:vulnerabilities).create!(project_id: project_id, author_id: user.id, title: 'test', + severity: 4, confidence: 4, report_type: 0) + + identifier = table(:vulnerability_identifiers).create!(project_id: project_id, external_type: 'uuid-v5', + external_id: 'uuid-v5', fingerprint: OpenSSL::Digest::SHA256.hexdigest(vulnerability.id.to_s), + name: 'Identifier for UUIDv5 2 2') + + table(:vulnerability_occurrences).create!( + vulnerability_id: vulnerability.id, project_id: project_id, scanner_id: scanner_id, + primary_identifier_id: identifier.id, name: 'test', severity: 4, confidence: 4, report_type: 0, + uuid: SecureRandom.uuid, project_fingerprint: '123qweasdzxc', location: { "image" => "alpine:3.4" }, + location_fingerprint: 'test', metadata_version: 'test', + raw_metadata: raw_metadata.to_json) + end + + def checksum(value) + sha = Digest::SHA256.hexdigest(value) + Gitlab::Database::ShaAttribute.new.serialize(sha) + end +end diff --git a/spec/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects_spec.rb b/spec/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects_spec.rb index a8574411957..f671a673a08 100644 --- a/spec/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects_spec.rb +++ b/spec/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects_spec.rb @@ -2,7 +2,8 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::NullifyCreatorIdColumnOfOrphanedProjects, feature_category: :projects do +RSpec.describe Gitlab::BackgroundMigration::NullifyCreatorIdColumnOfOrphanedProjects, feature_category: :projects, + schema: 20230130073109 do let(:users) { table(:users) } let(:projects) { table(:projects) } let(:namespaces) { table(:namespaces) } diff --git a/spec/lib/gitlab/background_task_spec.rb b/spec/lib/gitlab/background_task_spec.rb index 102556b6b2f..da92fc9e765 100644 --- a/spec/lib/gitlab/background_task_spec.rb +++ b/spec/lib/gitlab/background_task_spec.rb @@ -1,10 +1,10 @@ # frozen_string_literal: true -require 'fast_spec_helper' +require 'spec_helper' # We need to capture task state from a closure, which requires instance variables. # rubocop: disable RSpec/InstanceVariable -RSpec.describe Gitlab::BackgroundTask do +RSpec.describe Gitlab::BackgroundTask, feature_category: :build do let(:options) { {} } let(:task) do proc do diff --git a/spec/lib/gitlab/bitbucket_import/importer_spec.rb b/spec/lib/gitlab/bitbucket_import/importer_spec.rb index 1526a1a9f2d..48ceda9e8d8 100644 --- a/spec/lib/gitlab/bitbucket_import/importer_spec.rb +++ b/spec/lib/gitlab/bitbucket_import/importer_spec.rb @@ -358,7 +358,7 @@ RSpec.describe Gitlab::BitbucketImport::Importer, feature_category: :integration describe 'issue import' do it 'allocates internal ids' do - expect(Issue).to receive(:track_project_iid!).with(project, 6) + expect(Issue).to receive(:track_namespace_iid!).with(project.project_namespace, 6) importer.execute end diff --git a/spec/lib/gitlab/cache/client_spec.rb b/spec/lib/gitlab/cache/client_spec.rb new file mode 100644 index 00000000000..ec22fcdee7e --- /dev/null +++ b/spec/lib/gitlab/cache/client_spec.rb @@ -0,0 +1,165 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Cache::Client, feature_category: :source_code_management do + subject(:client) { described_class.new(metadata, backend: backend) } + + let(:backend) { Rails.cache } + let(:metadata) do + Gitlab::Cache::Metadata.new( + caller_id: caller_id, + cache_identifier: cache_identifier, + feature_category: feature_category, + backing_resource: backing_resource + ) + end + + let(:caller_id) { 'caller-id' } + let(:cache_identifier) { 'MyClass#cache' } + let(:feature_category) { :source_code_management } + let(:backing_resource) { :cpu } + + let(:metadata_mock) do + Gitlab::Cache::Metadata.new( + cache_identifier: cache_identifier, + feature_category: feature_category + ) + end + + let(:metrics_mock) { Gitlab::Cache::Metrics.new(metadata_mock) } + + describe '.build_with_metadata' do + it 'builds a cache client with metrics support' do + attributes = { + caller_id: caller_id, + cache_identifier: cache_identifier, + feature_category: feature_category, + backing_resource: backing_resource + } + + instance = described_class.build_with_metadata(**attributes) + + expect(instance).to be_a(described_class) + expect(instance.metadata).to have_attributes(**attributes) + end + end + + describe 'Methods', :use_clean_rails_memory_store_caching do + let(:expected_key) { 'key' } + + before do + allow(Gitlab::Cache::Metrics).to receive(:new).and_return(metrics_mock) + end + + describe '#read' do + context 'when key does not exist' do + it 'returns nil' do + expect(client.read('key')).to be_nil + end + + it 'increments cache miss' do + expect(metrics_mock).to receive(:increment_cache_miss) + + client.read('key') + end + end + + context 'when key exists' do + before do + backend.write(expected_key, 'value') + end + + it 'returns key value' do + expect(client.read('key')).to eq('value') + end + + it 'increments cache hit' do + expect(metrics_mock).to receive(:increment_cache_hit) + + client.read('key') + end + end + end + + describe '#write' do + it 'calls backend "#write" method with the expected key' do + expect(backend).to receive(:write).with(expected_key, 'value') + + client.write('key', 'value') + end + end + + describe '#exist?' do + it 'calls backend "#exist?" method with the expected key' do + expect(backend).to receive(:exist?).with(expected_key) + + client.exist?('key') + end + end + + describe '#delete' do + it 'calls backend "#delete" method with the expected key' do + expect(backend).to receive(:delete).with(expected_key) + + client.delete('key') + end + end + + # rubocop:disable Style/RedundantFetchBlock + describe '#fetch' do + it 'creates key in the specific format' do + client.fetch('key') { 'value' } + + expect(backend.fetch(expected_key)).to eq('value') + end + + it 'yields the block once' do + expect { |b| client.fetch('key', &b) }.to yield_control.once + end + + context 'when key already exists' do + before do + backend.write(expected_key, 'value') + end + + it 'does not redefine the value' do + expect(client.fetch('key') { 'new-value' }).to eq('value') + end + + it 'increments a cache hit' do + expect(metrics_mock).to receive(:increment_cache_hit) + + client.fetch('key') + end + + it 'does not measure the cache generation time' do + expect(metrics_mock).not_to receive(:observe_cache_generation) + + client.fetch('key') { 'new-value' } + end + end + + context 'when key does not exist' do + it 'caches the key' do + expect(client.fetch('key') { 'value' }).to eq('value') + + expect(client.fetch('key')).to eq('value') + end + + it 'increments a cache miss' do + expect(metrics_mock).to receive(:increment_cache_miss) + + client.fetch('key') + end + + it 'measures the cache generation time' do + expect(metrics_mock).to receive(:observe_cache_generation) + + client.fetch('key') { 'value' } + end + end + end + end + # rubocop:enable Style/RedundantFetchBlock +end diff --git a/spec/lib/gitlab/changes_list_spec.rb b/spec/lib/gitlab/changes_list_spec.rb index 762a121340e..77deffe4b37 100644 --- a/spec/lib/gitlab/changes_list_spec.rb +++ b/spec/lib/gitlab/changes_list_spec.rb @@ -2,7 +2,7 @@ require 'fast_spec_helper' -RSpec.describe Gitlab::ChangesList do +RSpec.describe Gitlab::ChangesList, feature_category: :source_code_management do let(:valid_changes_string) { "\n000000 570e7b2 refs/heads/my_branch\nd14d6c 6fd24d refs/heads/master" } let(:invalid_changes) { 1 } diff --git a/spec/lib/gitlab/chat/responder_spec.rb b/spec/lib/gitlab/chat/responder_spec.rb index a9d290cb87c..15ca3427ae8 100644 --- a/spec/lib/gitlab/chat/responder_spec.rb +++ b/spec/lib/gitlab/chat/responder_spec.rb @@ -4,68 +4,32 @@ require 'spec_helper' RSpec.describe Gitlab::Chat::Responder, feature_category: :integrations do describe '.responder_for' do - context 'when the feature flag is disabled' do - before do - stub_feature_flags(use_response_url_for_chat_responder: false) - end - - context 'using a regular build' do - it 'returns nil' do - build = create(:ci_build) + context 'using a regular build' do + it 'returns nil' do + build = create(:ci_build) - expect(described_class.responder_for(build)).to be_nil - end - end - - context 'using a chat build' do - it 'returns the responder for the build' do - pipeline = create(:ci_pipeline) - build = create(:ci_build, pipeline: pipeline) - integration = double(:integration, chat_responder: Gitlab::Chat::Responder::Slack) - chat_name = double(:chat_name, integration: integration) - chat_data = double(:chat_data, chat_name: chat_name) - - allow(pipeline) - .to receive(:chat_data) - .and_return(chat_data) - - expect(described_class.responder_for(build)) - .to be_an_instance_of(Gitlab::Chat::Responder::Slack) - end + expect(described_class.responder_for(build)).to be_nil end end - context 'when the feature flag is enabled' do - before do - stub_feature_flags(use_response_url_for_chat_responder: true) - end - - context 'using a regular build' do - it 'returns nil' do - build = create(:ci_build) + context 'using a chat build' do + let_it_be(:pipeline) { create(:ci_pipeline) } + let_it_be(:build) { create(:ci_build, pipeline: pipeline) } - expect(described_class.responder_for(build)).to be_nil + context "when response_url starts with 'https://hooks.slack.com/'" do + before do + pipeline.build_chat_data(response_url: 'https://hooks.slack.com/services/12345', chat_name_id: 'U123') end + + it { expect(described_class.responder_for(build)).to be_an_instance_of(Gitlab::Chat::Responder::Slack) } end - context 'using a chat build' do - let(:chat_name) { create(:chat_name, chat_id: 'U123') } - let(:pipeline) do - pipeline = create(:ci_pipeline) - pipeline.create_chat_data!( - response_url: 'https://hooks.slack.com/services/12345', - chat_name_id: chat_name.id - ) - pipeline + context "when response_url does not start with 'https://hooks.slack.com/'" do + before do + pipeline.build_chat_data(response_url: 'https://mattermost.example.com/services/12345', chat_name_id: 'U123') end - let(:build) { create(:ci_build, pipeline: pipeline) } - let(:responder) { described_class.new(build) } - - it 'returns the responder for the build' do - expect(described_class.responder_for(build)) - .to be_an_instance_of(Gitlab::Chat::Responder::Slack) - end + it { expect(described_class.responder_for(build)).to be_an_instance_of(Gitlab::Chat::Responder::Mattermost) } end end end diff --git a/spec/lib/gitlab/checks/changes_access_spec.rb b/spec/lib/gitlab/checks/changes_access_spec.rb index 60118823b5a..552afcdb180 100644 --- a/spec/lib/gitlab/checks/changes_access_spec.rb +++ b/spec/lib/gitlab/checks/changes_access_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Checks::ChangesAccess do +RSpec.describe Gitlab::Checks::ChangesAccess, feature_category: :source_code_management do include_context 'changes access checks context' subject { changes_access } @@ -47,6 +47,16 @@ RSpec.describe Gitlab::Checks::ChangesAccess do expect(subject.commits).to match_array([]) end + context 'when change is for notes ref' do + let(:changes) do + [{ oldrev: oldrev, newrev: newrev, ref: 'refs/notes/commit' }] + end + + it 'does not return any commits' do + expect(subject.commits).to match_array([]) + end + end + context 'when changes contain empty revisions' do let(:expected_commit) { instance_double(Commit) } diff --git a/spec/lib/gitlab/checks/diff_check_spec.rb b/spec/lib/gitlab/checks/diff_check_spec.rb index 6b45b8d4628..0845c746545 100644 --- a/spec/lib/gitlab/checks/diff_check_spec.rb +++ b/spec/lib/gitlab/checks/diff_check_spec.rb @@ -2,10 +2,20 @@ require 'spec_helper' -RSpec.describe Gitlab::Checks::DiffCheck do +RSpec.describe Gitlab::Checks::DiffCheck, feature_category: :source_code_management do include_context 'change access checks context' describe '#validate!' do + context 'when ref is not tag or branch ref' do + let(:ref) { 'refs/notes/commit' } + + it 'does not call find_changed_paths' do + expect(project.repository).not_to receive(:find_changed_paths) + + subject.validate! + end + end + context 'when commits is empty' do it 'does not call find_changed_paths' do expect(project.repository).not_to receive(:find_changed_paths) diff --git a/spec/lib/gitlab/ci/badge/release/template_spec.rb b/spec/lib/gitlab/ci/badge/release/template_spec.rb index 2b66c296a94..6be0dcaae99 100644 --- a/spec/lib/gitlab/ci/badge/release/template_spec.rb +++ b/spec/lib/gitlab/ci/badge/release/template_spec.rb @@ -59,9 +59,30 @@ RSpec.describe Gitlab::Ci::Badge::Release::Template do end describe '#value_width' do - it 'has a fixed value width' do + it 'returns the default value width' do expect(template.value_width).to eq 54 end + + it 'returns custom value width' do + value_width = 100 + badge = Gitlab::Ci::Badge::Release::LatestRelease.new(project, user, opts: { value_width: value_width }) + + expect(described_class.new(badge).value_width).to eq value_width + end + + it 'returns VALUE_WIDTH_DEFAULT if the custom value_width supplied is greater than permissible limit' do + value_width = 250 + badge = Gitlab::Ci::Badge::Release::LatestRelease.new(project, user, opts: { value_width: value_width }) + + expect(described_class.new(badge).value_width).to eq 54 + end + + it 'returns VALUE_WIDTH_DEFAULT if value_width is not a number' do + value_width = "string" + badge = Gitlab::Ci::Badge::Release::LatestRelease.new(project, user, opts: { value_width: value_width }) + + expect(described_class.new(badge).value_width).to eq 54 + end end describe '#key_color' do diff --git a/spec/lib/gitlab/ci/build/auto_retry_spec.rb b/spec/lib/gitlab/ci/build/auto_retry_spec.rb index 314714c543b..0b275e7d564 100644 --- a/spec/lib/gitlab/ci/build/auto_retry_spec.rb +++ b/spec/lib/gitlab/ci/build/auto_retry_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Build::AutoRetry, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Build::AutoRetry, feature_category: :pipeline_composition do let(:auto_retry) { described_class.new(build) } describe '#allowed?' do diff --git a/spec/lib/gitlab/ci/build/context/build_spec.rb b/spec/lib/gitlab/ci/build/context/build_spec.rb index 74739a67be0..4fdeffb033a 100644 --- a/spec/lib/gitlab/ci/build/context/build_spec.rb +++ b/spec/lib/gitlab/ci/build/context/build_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Build::Context::Build, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Build::Context::Build, feature_category: :pipeline_composition do let(:pipeline) { create(:ci_pipeline) } let(:seed_attributes) { { 'name' => 'some-job' } } diff --git a/spec/lib/gitlab/ci/build/hook_spec.rb b/spec/lib/gitlab/ci/build/hook_spec.rb index 6ed40a44c97..6c9175b4260 100644 --- a/spec/lib/gitlab/ci/build/hook_spec.rb +++ b/spec/lib/gitlab/ci/build/hook_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Build::Hook, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Build::Hook, feature_category: :pipeline_composition do let_it_be(:build1) do FactoryBot.build(:ci_build, options: { hooks: { pre_get_sources_script: ["echo 'hello pre_get_sources_script'"] } }) diff --git a/spec/lib/gitlab/ci/components/header_spec.rb b/spec/lib/gitlab/ci/components/header_spec.rb new file mode 100644 index 00000000000..b1af4ca9238 --- /dev/null +++ b/spec/lib/gitlab/ci/components/header_spec.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +RSpec.describe Gitlab::Ci::Components::Header, feature_category: :pipeline_composition do + subject { described_class.new(spec) } + + context 'when spec is valid' do + let(:spec) do + { + spec: { + inputs: { + website: nil, + run: { + options: %w[opt1 opt2] + } + } + } + } + end + + it 'fabricates a spec from valid data' do + expect(subject).not_to be_empty + end + + describe '#inputs' do + it 'fabricates input data' do + input = subject.inputs({ website: 'https//gitlab.com', run: 'opt1' }) + + expect(input.count).to eq 2 + end + end + + describe '#context' do + it 'fabricates interpolation context' do + ctx = subject.context({ website: 'https//gitlab.com', run: 'opt1' }) + + expect(ctx).to be_valid + end + end + end + + context 'when spec is empty' do + let(:spec) { { spec: {} } } + + it 'returns an empty header' do + expect(subject).to be_empty + end + end +end diff --git a/spec/lib/gitlab/ci/components/instance_path_spec.rb b/spec/lib/gitlab/ci/components/instance_path_spec.rb index d9beae0555c..fbe5e0b9d42 100644 --- a/spec/lib/gitlab/ci/components/instance_path_spec.rb +++ b/spec/lib/gitlab/ci/components/instance_path_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Components::InstancePath, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Components::InstancePath, feature_category: :pipeline_composition do let_it_be(:user) { create(:user) } let(:path) { described_class.new(address: address, content_filename: 'template.yml') } diff --git a/spec/lib/gitlab/ci/config/entry/job_spec.rb b/spec/lib/gitlab/ci/config/entry/job_spec.rb index c1b9bd58d98..c8b4a8b8a0e 100644 --- a/spec/lib/gitlab/ci/config/entry/job_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/job_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::Entry::Job, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::Entry::Job, feature_category: :pipeline_composition do let(:entry) { described_class.new(config, name: :rspec) } it_behaves_like 'with inheritable CI config' do @@ -728,27 +728,6 @@ RSpec.describe Gitlab::Ci::Config::Entry::Job, feature_category: :pipeline_autho scheduling_type: :stage, id_tokens: { TEST_ID_TOKEN: { aud: 'https://gitlab.com' } }) end - - context 'when the FF ci_hooks_pre_get_sources_script is disabled' do - before do - stub_feature_flags(ci_hooks_pre_get_sources_script: false) - end - - it 'returns correct value' do - expect(entry.value) - .to eq(name: :rspec, - before_script: %w[ls pwd], - script: %w[rspec], - stage: 'test', - ignore: false, - after_script: %w[cleanup], - only: { refs: %w[branches tags] }, - job_variables: {}, - root_variables_inheritance: true, - scheduling_type: :stage, - id_tokens: { TEST_ID_TOKEN: { aud: 'https://gitlab.com' } }) - end - end end end diff --git a/spec/lib/gitlab/ci/config/entry/policy_spec.rb b/spec/lib/gitlab/ci/config/entry/policy_spec.rb index 378c0947e8a..7093a0a6edf 100644 --- a/spec/lib/gitlab/ci/config/entry/policy_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/policy_spec.rb @@ -1,8 +1,8 @@ # frozen_string_literal: true -require 'fast_spec_helper' +require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::Entry::Policy do +RSpec.describe Gitlab::Ci::Config::Entry::Policy, feature_category: :continuous_integration do let(:entry) { described_class.new(config) } context 'when using simplified policy' do diff --git a/spec/lib/gitlab/ci/config/entry/processable_spec.rb b/spec/lib/gitlab/ci/config/entry/processable_spec.rb index b28562ba2ea..4f13940d7e2 100644 --- a/spec/lib/gitlab/ci/config/entry/processable_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/processable_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::Entry::Processable, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::Entry::Processable, feature_category: :pipeline_composition do let(:node_class) do Class.new(::Gitlab::Config::Entry::Node) do include Gitlab::Ci::Config::Entry::Processable diff --git a/spec/lib/gitlab/ci/config/entry/pull_policy_spec.rb b/spec/lib/gitlab/ci/config/entry/pull_policy_spec.rb index c35355b10c6..40507a66c2d 100644 --- a/spec/lib/gitlab/ci/config/entry/pull_policy_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/pull_policy_spec.rb @@ -1,8 +1,8 @@ # frozen_string_literal: true -require 'fast_spec_helper' +require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::Entry::PullPolicy do +RSpec.describe Gitlab::Ci::Config::Entry::PullPolicy, feature_category: :continuous_integration do let(:entry) { described_class.new(config) } describe '#value' do diff --git a/spec/lib/gitlab/ci/config/entry/reports/coverage_report_spec.rb b/spec/lib/gitlab/ci/config/entry/reports/coverage_report_spec.rb index ccd6f6ab427..6f37dd72083 100644 --- a/spec/lib/gitlab/ci/config/entry/reports/coverage_report_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/reports/coverage_report_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::Entry::Reports::CoverageReport, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::Entry::Reports::CoverageReport, feature_category: :pipeline_composition do let(:entry) { described_class.new(config) } describe 'validations' do diff --git a/spec/lib/gitlab/ci/config/entry/reports_spec.rb b/spec/lib/gitlab/ci/config/entry/reports_spec.rb index 715cb18fb92..73bf2d422b7 100644 --- a/spec/lib/gitlab/ci/config/entry/reports_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/reports_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::Entry::Reports, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::Entry::Reports, feature_category: :pipeline_composition do let(:entry) { described_class.new(config) } describe 'validates ALLOWED_KEYS' do diff --git a/spec/lib/gitlab/ci/config/entry/trigger_spec.rb b/spec/lib/gitlab/ci/config/entry/trigger_spec.rb index f47923af45a..fdd598c2ab2 100644 --- a/spec/lib/gitlab/ci/config/entry/trigger_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/trigger_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::Entry::Trigger, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::Entry::Trigger, feature_category: :pipeline_composition do subject { described_class.new(config) } context 'when trigger config is a non-empty string' do diff --git a/spec/lib/gitlab/ci/config/external/context_spec.rb b/spec/lib/gitlab/ci/config/external/context_spec.rb index 1fd3cf3c99f..fc68d3d62a2 100644 --- a/spec/lib/gitlab/ci/config/external/context_spec.rb +++ b/spec/lib/gitlab/ci/config/external/context_spec.rb @@ -2,12 +2,21 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::External::Context, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::External::Context, feature_category: :pipeline_composition do let(:project) { build(:project) } let(:user) { double('User') } let(:sha) { '12345' } let(:variables) { Gitlab::Ci::Variables::Collection.new([{ 'key' => 'a', 'value' => 'b' }]) } - let(:attributes) { { project: project, user: user, sha: sha, variables: variables } } + let(:pipeline_config) { instance_double(Gitlab::Ci::ProjectConfig) } + let(:attributes) do + { + project: project, + user: user, + sha: sha, + variables: variables, + pipeline_config: pipeline_config + } + end subject(:subject) { described_class.new(**attributes) } @@ -15,11 +24,12 @@ RSpec.describe Gitlab::Ci::Config::External::Context, feature_category: :pipelin context 'with values' do it { is_expected.to have_attributes(**attributes) } it { expect(subject.expandset).to eq([]) } - it { expect(subject.max_includes).to eq(Gitlab::Ci::Config::External::Context::NEW_MAX_INCLUDES) } + it { expect(subject.max_includes).to eq(Gitlab::Ci::Config::External::Context::MAX_INCLUDES) } it { expect(subject.execution_deadline).to eq(0) } it { expect(subject.variables).to be_instance_of(Gitlab::Ci::Variables::Collection) } it { expect(subject.variables_hash).to be_instance_of(ActiveSupport::HashWithIndifferentAccess) } it { expect(subject.variables_hash).to include('a' => 'b') } + it { expect(subject.pipeline_config).to eq(pipeline_config) } end context 'without values' do @@ -27,37 +37,11 @@ RSpec.describe Gitlab::Ci::Config::External::Context, feature_category: :pipelin it { is_expected.to have_attributes(**attributes) } it { expect(subject.expandset).to eq([]) } - it { expect(subject.max_includes).to eq(Gitlab::Ci::Config::External::Context::NEW_MAX_INCLUDES) } + it { expect(subject.max_includes).to eq(Gitlab::Ci::Config::External::Context::MAX_INCLUDES) } it { expect(subject.execution_deadline).to eq(0) } it { expect(subject.variables).to be_instance_of(Gitlab::Ci::Variables::Collection) } it { expect(subject.variables_hash).to be_instance_of(ActiveSupport::HashWithIndifferentAccess) } - end - - context 'when FF ci_includes_count_duplicates is disabled' do - before do - stub_feature_flags(ci_includes_count_duplicates: false) - end - - context 'with values' do - it { is_expected.to have_attributes(**attributes) } - it { expect(subject.expandset).to eq(Set.new) } - it { expect(subject.max_includes).to eq(Gitlab::Ci::Config::External::Context::MAX_INCLUDES) } - it { expect(subject.execution_deadline).to eq(0) } - it { expect(subject.variables).to be_instance_of(Gitlab::Ci::Variables::Collection) } - it { expect(subject.variables_hash).to be_instance_of(ActiveSupport::HashWithIndifferentAccess) } - it { expect(subject.variables_hash).to include('a' => 'b') } - end - - context 'without values' do - let(:attributes) { { project: nil, user: nil, sha: nil } } - - it { is_expected.to have_attributes(**attributes) } - it { expect(subject.expandset).to eq(Set.new) } - it { expect(subject.max_includes).to eq(Gitlab::Ci::Config::External::Context::MAX_INCLUDES) } - it { expect(subject.execution_deadline).to eq(0) } - it { expect(subject.variables).to be_instance_of(Gitlab::Ci::Variables::Collection) } - it { expect(subject.variables_hash).to be_instance_of(ActiveSupport::HashWithIndifferentAccess) } - end + it { expect(subject.pipeline_config).to be_nil } end end @@ -170,4 +154,26 @@ RSpec.describe Gitlab::Ci::Config::External::Context, feature_category: :pipelin describe '#sentry_payload' do it { expect(subject.sentry_payload).to match(a_hash_including(:project, :user)) } end + + describe '#internal_include?' do + context 'when pipeline_config is provided' do + where(:value) { [true, false] } + + with_them do + it 'returns the value of .internal_include_prepended?' do + allow(pipeline_config).to receive(:internal_include_prepended?).and_return(value) + + expect(subject.internal_include?).to eq(value) + end + end + end + + context 'when pipeline_config is not provided' do + let(:pipeline_config) { nil } + + it 'returns false' do + expect(subject.internal_include?).to eq(false) + end + end + end end diff --git a/spec/lib/gitlab/ci/config/external/file/artifact_spec.rb b/spec/lib/gitlab/ci/config/external/file/artifact_spec.rb index 45a15fb5f36..52b8dcbcb44 100644 --- a/spec/lib/gitlab/ci/config/external/file/artifact_spec.rb +++ b/spec/lib/gitlab/ci/config/external/file/artifact_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::External::File::Artifact, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::External::File::Artifact, feature_category: :pipeline_composition do let(:parent_pipeline) { create(:ci_pipeline) } let(:variables) {} let(:context) do diff --git a/spec/lib/gitlab/ci/config/external/file/base_spec.rb b/spec/lib/gitlab/ci/config/external/file/base_spec.rb index 55d95d0c1f8..959dcdf31af 100644 --- a/spec/lib/gitlab/ci/config/external/file/base_spec.rb +++ b/spec/lib/gitlab/ci/config/external/file/base_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::External::File::Base, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::External::File::Base, feature_category: :pipeline_composition do let(:variables) {} let(:context_params) { { sha: 'HEAD', variables: variables } } let(:context) { Gitlab::Ci::Config::External::Context.new(**context_params) } diff --git a/spec/lib/gitlab/ci/config/external/file/component_spec.rb b/spec/lib/gitlab/ci/config/external/file/component_spec.rb index a162a1a8abf..1562e571060 100644 --- a/spec/lib/gitlab/ci/config/external/file/component_spec.rb +++ b/spec/lib/gitlab/ci/config/external/file/component_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::External::File::Component, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::External::File::Component, feature_category: :pipeline_composition do let_it_be(:context_project) { create(:project, :repository) } let_it_be(:project) { create(:project, :repository) } let_it_be(:user) { create(:user) } diff --git a/spec/lib/gitlab/ci/config/external/file/local_spec.rb b/spec/lib/gitlab/ci/config/external/file/local_spec.rb index b5895b4bc81..2bac8a6968b 100644 --- a/spec/lib/gitlab/ci/config/external/file/local_spec.rb +++ b/spec/lib/gitlab/ci/config/external/file/local_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::External::File::Local, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::External::File::Local, feature_category: :pipeline_composition do include RepoHelpers let_it_be(:project) { create(:project, :repository) } diff --git a/spec/lib/gitlab/ci/config/external/file/project_spec.rb b/spec/lib/gitlab/ci/config/external/file/project_spec.rb index abe38cdbc3e..0ef39a22932 100644 --- a/spec/lib/gitlab/ci/config/external/file/project_spec.rb +++ b/spec/lib/gitlab/ci/config/external/file/project_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::External::File::Project, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::External::File::Project, feature_category: :pipeline_composition do include RepoHelpers let_it_be(:context_project) { create(:project) } @@ -97,6 +97,36 @@ RSpec.describe Gitlab::Ci::Config::External::File::Project, feature_category: :p end end + context 'when a valid path is used in uppercase' do + let(:params) do + { project: project.full_path.upcase, file: '/file.yml' } + end + + around do |example| + create_and_delete_files(project, { '/file.yml' => 'image: image:1.0' }) do + example.run + end + end + + it { is_expected.to be_truthy } + end + + context 'when a valid different case path is used' do + let_it_be(:project) { create(:project, :repository, path: 'mY-teSt-proJect', name: 'My Test Project') } + + let(:params) do + { project: "#{project.namespace.full_path}/my-test-projecT", file: '/file.yml' } + end + + around do |example| + create_and_delete_files(project, { '/file.yml' => 'image: image:1.0' }) do + example.run + end + end + + it { is_expected.to be_truthy } + end + context 'when a valid path with custom ref is used' do let(:params) do { project: project.full_path, ref: 'master', file: '/file.yml' } diff --git a/spec/lib/gitlab/ci/config/external/file/remote_spec.rb b/spec/lib/gitlab/ci/config/external/file/remote_spec.rb index 27f401db76e..f8986e8fa10 100644 --- a/spec/lib/gitlab/ci/config/external/file/remote_spec.rb +++ b/spec/lib/gitlab/ci/config/external/file/remote_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::External::File::Remote, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::External::File::Remote, feature_category: :pipeline_composition do include StubRequests let(:variables) { Gitlab::Ci::Variables::Collection.new([{ 'key' => 'GITLAB_TOKEN', 'value' => 'secret_file', 'masked' => true }]) } diff --git a/spec/lib/gitlab/ci/config/external/file/template_spec.rb b/spec/lib/gitlab/ci/config/external/file/template_spec.rb index 83e98874118..79fd4203c3e 100644 --- a/spec/lib/gitlab/ci/config/external/file/template_spec.rb +++ b/spec/lib/gitlab/ci/config/external/file/template_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::External::File::Template, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::External::File::Template, feature_category: :pipeline_composition do let_it_be(:project) { create(:project) } let_it_be(:user) { create(:user) } diff --git a/spec/lib/gitlab/ci/config/external/mapper/base_spec.rb b/spec/lib/gitlab/ci/config/external/mapper/base_spec.rb index 0fdcc5e8ff7..ce8f3756cbc 100644 --- a/spec/lib/gitlab/ci/config/external/mapper/base_spec.rb +++ b/spec/lib/gitlab/ci/config/external/mapper/base_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::External::Mapper::Base, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::External::Mapper::Base, feature_category: :pipeline_composition do let(:test_class) do Class.new(described_class) do def self.name diff --git a/spec/lib/gitlab/ci/config/external/mapper/filter_spec.rb b/spec/lib/gitlab/ci/config/external/mapper/filter_spec.rb index df2a2f0fd01..5195567ebb4 100644 --- a/spec/lib/gitlab/ci/config/external/mapper/filter_spec.rb +++ b/spec/lib/gitlab/ci/config/external/mapper/filter_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::External::Mapper::Filter, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::External::Mapper::Filter, feature_category: :pipeline_composition do let_it_be(:variables) do Gitlab::Ci::Variables::Collection.new.tap do |variables| variables.append(key: 'VARIABLE1', value: 'hello') diff --git a/spec/lib/gitlab/ci/config/external/mapper/location_expander_spec.rb b/spec/lib/gitlab/ci/config/external/mapper/location_expander_spec.rb index b14b6b0ca29..1e490bf1d16 100644 --- a/spec/lib/gitlab/ci/config/external/mapper/location_expander_spec.rb +++ b/spec/lib/gitlab/ci/config/external/mapper/location_expander_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::External::Mapper::LocationExpander, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::External::Mapper::LocationExpander, feature_category: :pipeline_composition do include RepoHelpers let_it_be(:project) { create(:project, :repository) } diff --git a/spec/lib/gitlab/ci/config/external/mapper/matcher_spec.rb b/spec/lib/gitlab/ci/config/external/mapper/matcher_spec.rb index 11c79e19cff..6ca4fd24e61 100644 --- a/spec/lib/gitlab/ci/config/external/mapper/matcher_spec.rb +++ b/spec/lib/gitlab/ci/config/external/mapper/matcher_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::External::Mapper::Matcher, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::External::Mapper::Matcher, feature_category: :pipeline_composition do let_it_be(:variables) do Gitlab::Ci::Variables::Collection.new.tap do |variables| variables.append(key: 'A_MASKED_VAR', value: 'this-is-secret', masked: true) diff --git a/spec/lib/gitlab/ci/config/external/mapper/normalizer_spec.rb b/spec/lib/gitlab/ci/config/external/mapper/normalizer_spec.rb index 709c234253b..09212833d84 100644 --- a/spec/lib/gitlab/ci/config/external/mapper/normalizer_spec.rb +++ b/spec/lib/gitlab/ci/config/external/mapper/normalizer_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::External::Mapper::Normalizer, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::External::Mapper::Normalizer, feature_category: :pipeline_composition do let_it_be(:variables) do Gitlab::Ci::Variables::Collection.new.tap do |variables| variables.append(key: 'VARIABLE1', value: 'config') diff --git a/spec/lib/gitlab/ci/config/external/mapper/variables_expander_spec.rb b/spec/lib/gitlab/ci/config/external/mapper/variables_expander_spec.rb index f7454dcd4be..e27e8034faa 100644 --- a/spec/lib/gitlab/ci/config/external/mapper/variables_expander_spec.rb +++ b/spec/lib/gitlab/ci/config/external/mapper/variables_expander_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::External::Mapper::VariablesExpander, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::External::Mapper::VariablesExpander, feature_category: :pipeline_composition do let_it_be(:variables) do Gitlab::Ci::Variables::Collection.new.tap do |variables| variables.append(key: 'VARIABLE1', value: 'hello') diff --git a/spec/lib/gitlab/ci/config/external/mapper/verifier_spec.rb b/spec/lib/gitlab/ci/config/external/mapper/verifier_spec.rb index a219666f24e..ef5049158c1 100644 --- a/spec/lib/gitlab/ci/config/external/mapper/verifier_spec.rb +++ b/spec/lib/gitlab/ci/config/external/mapper/verifier_spec.rb @@ -2,11 +2,11 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::External::Mapper::Verifier, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::External::Mapper::Verifier, feature_category: :pipeline_composition do include RepoHelpers include StubRequests - let_it_be(:project) { create(:project, :repository) } + let_it_be(:project) { create(:project, :small_repo) } let_it_be(:user) { project.owner } let(:context) do @@ -38,7 +38,7 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper::Verifier, feature_category: } end - around(:all) do |example| + around do |example| create_and_delete_files(project, project_files) do example.run end @@ -84,42 +84,105 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper::Verifier, feature_category: end context 'when files are project files' do - let_it_be(:included_project) { create(:project, :repository, namespace: project.namespace, creator: user) } + let_it_be(:included_project1) { create(:project, :small_repo, namespace: project.namespace, creator: user) } + let_it_be(:included_project2) { create(:project, :small_repo, namespace: project.namespace, creator: user) } let(:files) do [ Gitlab::Ci::Config::External::File::Project.new( - { file: 'myfolder/file1.yml', project: included_project.full_path }, context + { file: 'myfolder/file1.yml', project: included_project1.full_path }, context ), Gitlab::Ci::Config::External::File::Project.new( - { file: 'myfolder/file2.yml', project: included_project.full_path }, context + { file: 'myfolder/file2.yml', project: included_project1.full_path }, context ), Gitlab::Ci::Config::External::File::Project.new( - { file: 'myfolder/file3.yml', project: included_project.full_path }, context + { file: 'myfolder/file3.yml', project: included_project1.full_path, ref: 'master' }, context + ), + Gitlab::Ci::Config::External::File::Project.new( + { file: 'myfolder/file1.yml', project: included_project2.full_path }, context + ), + Gitlab::Ci::Config::External::File::Project.new( + { file: 'myfolder/file2.yml', project: included_project2.full_path }, context ) ] end - around(:all) do |example| - create_and_delete_files(included_project, project_files) do - example.run + around do |example| + create_and_delete_files(included_project1, project_files) do + create_and_delete_files(included_project2, project_files) do + example.run + end end end - it 'returns an array of file objects' do + it 'returns an array of valid file objects' do expect(process.map(&:location)).to contain_exactly( - 'myfolder/file1.yml', 'myfolder/file2.yml', 'myfolder/file3.yml' + 'myfolder/file1.yml', 'myfolder/file2.yml', 'myfolder/file3.yml', 'myfolder/file1.yml', 'myfolder/file2.yml' ) + + expect(process.all?(&:valid?)).to be_truthy end it 'adds files to the expandset' do - expect { process }.to change { context.expandset.count }.by(3) + expect { process }.to change { context.expandset.count }.by(5) end it 'calls Gitaly only once for all files', :request_store do - # 1 for project.commit.id, 3 for the sha check, 1 for the files + files # calling this to load project creations and the `project.commit.id` call + + # 3 for the sha check, 2 for the files in batch expect { process }.to change { Gitlab::GitalyClient.get_request_count }.by(5) end + + it 'queries with batch', :use_sql_query_cache do + files # calling this to load project creations and the `project.commit.id` call + + queries = ActiveRecord::QueryRecorder.new(skip_cached: false) { process } + projects_queries = queries.occurrences_starting_with('SELECT "projects"') + access_check_queries = queries.occurrences_starting_with('SELECT MAX("project_authorizations"."access_level")') + + # We could not reduce the number of projects queries because we need to call project for + # the `can_access_local_content?` and `sha` BatchLoaders. + expect(projects_queries.values.sum).to eq(2) + expect(access_check_queries.values.sum).to eq(2) + end + + context 'when the FF ci_batch_project_includes_context is disabled' do + before do + stub_feature_flags(ci_batch_project_includes_context: false) + end + + it 'returns an array of file objects' do + expect(process.map(&:location)).to contain_exactly( + 'myfolder/file1.yml', 'myfolder/file2.yml', 'myfolder/file3.yml', + 'myfolder/file1.yml', 'myfolder/file2.yml' + ) + end + + it 'adds files to the expandset' do + expect { process }.to change { context.expandset.count }.by(5) + end + + it 'calls Gitaly for all files', :request_store do + files # calling this to load project creations and the `project.commit.id` call + + # 5 for the sha check, 2 for the files in batch + expect { process }.to change { Gitlab::GitalyClient.get_request_count }.by(7) + end + + it 'queries without batch', :use_sql_query_cache do + files # calling this to load project creations and the `project.commit.id` call + + queries = ActiveRecord::QueryRecorder.new(skip_cached: false) { process } + projects_queries = queries.occurrences_starting_with('SELECT "projects"') + access_check_queries = queries.occurrences_starting_with( + 'SELECT MAX("project_authorizations"."access_level")' + ) + + expect(projects_queries.values.sum).to eq(5) + expect(access_check_queries.values.sum).to eq(5) + end + end end context 'when a file includes other files' do @@ -150,7 +213,30 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper::Verifier, feature_category: end end - context 'when total file count exceeds max_includes' do + describe 'max includes detection' do + shared_examples 'verifies max includes' do + context 'when total file count is equal to max_includes' do + before do + allow(context).to receive(:max_includes).and_return(expected_total_file_count) + end + + it 'adds the expected number of files to expandset' do + expect { process }.not_to raise_error + expect(context.expandset.count).to eq(expected_total_file_count) + end + end + + context 'when total file count exceeds max_includes' do + before do + allow(context).to receive(:max_includes).and_return(expected_total_file_count - 1) + end + + it 'raises error' do + expect { process }.to raise_error(expected_error_class) + end + end + end + context 'when files are nested' do let(:files) do [ @@ -158,9 +244,21 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper::Verifier, feature_category: ] end - it 'raises Processor::IncludeError' do - allow(context).to receive(:max_includes).and_return(1) - expect { process }.to raise_error(Gitlab::Ci::Config::External::Processor::IncludeError) + let(:expected_total_file_count) { 4 } # Includes nested_configs.yml + 3 nested files + let(:expected_error_class) { Gitlab::Ci::Config::External::Processor::IncludeError } + + it_behaves_like 'verifies max includes' + + context 'when duplicate files are included' do + let(:expected_total_file_count) { 8 } # 2 x (Includes nested_configs.yml + 3 nested files) + let(:files) do + [ + Gitlab::Ci::Config::External::File::Local.new({ local: 'nested_configs.yml' }, context), + Gitlab::Ci::Config::External::File::Local.new({ local: 'nested_configs.yml' }, context) + ] + end + + it_behaves_like 'verifies max includes' end end @@ -172,34 +270,162 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper::Verifier, feature_category: ] end - it 'raises Mapper::TooManyIncludesError' do - allow(context).to receive(:max_includes).and_return(1) - expect { process }.to raise_error(Gitlab::Ci::Config::External::Mapper::TooManyIncludesError) + let(:expected_total_file_count) { files.count } + let(:expected_error_class) { Gitlab::Ci::Config::External::Mapper::TooManyIncludesError } + + it_behaves_like 'verifies max includes' + + context 'when duplicate files are included' do + let(:files) do + [ + Gitlab::Ci::Config::External::File::Local.new({ local: 'myfolder/file1.yml' }, context), + Gitlab::Ci::Config::External::File::Local.new({ local: 'myfolder/file2.yml' }, context), + Gitlab::Ci::Config::External::File::Local.new({ local: 'myfolder/file2.yml' }, context) + ] + end + + let(:expected_total_file_count) { files.count } + + it_behaves_like 'verifies max includes' end end - context 'when files are duplicates' do + context 'when there is a circular include' do + let(:project_files) do + { + 'myfolder/file1.yml' => <<~YAML + include: myfolder/file1.yml + YAML + } + end + let(:files) do [ - Gitlab::Ci::Config::External::File::Local.new({ local: 'myfolder/file1.yml' }, context), - Gitlab::Ci::Config::External::File::Local.new({ local: 'myfolder/file1.yml' }, context), Gitlab::Ci::Config::External::File::Local.new({ local: 'myfolder/file1.yml' }, context) ] end + before do + allow(context).to receive(:max_includes).and_return(10) + end + it 'raises error' do - allow(context).to receive(:max_includes).and_return(2) - expect { process }.to raise_error(Gitlab::Ci::Config::External::Mapper::TooManyIncludesError) + expect { process }.to raise_error(Gitlab::Ci::Config::External::Processor::IncludeError) end + end - context 'when FF ci_includes_count_duplicates is disabled' do - before do - stub_feature_flags(ci_includes_count_duplicates: false) + context 'when a file is an internal include' do + let(:project_files) do + { + 'myfolder/file1.yml' => <<~YAML, + my_build: + script: echo Hello World + YAML + '.internal-include.yml' => <<~YAML + include: + - local: myfolder/file1.yml + YAML + } + end + + let(:files) do + [ + Gitlab::Ci::Config::External::File::Local.new({ local: '.internal-include.yml' }, context) + ] + end + + let(:total_file_count) { 2 } # Includes .internal-include.yml + myfolder/file1.yml + let(:pipeline_config) { instance_double(Gitlab::Ci::ProjectConfig) } + + let(:context) do + Gitlab::Ci::Config::External::Context.new( + project: project, + user: user, + sha: project.commit.id, + pipeline_config: pipeline_config + ) + end + + before do + allow(pipeline_config).to receive(:internal_include_prepended?).and_return(true) + allow(context).to receive(:max_includes).and_return(1) + end + + context 'when total file count excluding internal include is equal to max_includes' do + it 'does not add the internal include to expandset' do + expect { process }.not_to raise_error + expect(context.expandset.count).to eq(total_file_count - 1) + expect(context.expandset.first.location).to eq('myfolder/file1.yml') end + end - it 'does not raise error' do + context 'when total file count excluding internal include exceeds max_includes' do + let(:project_files) do + { + 'myfolder/file1.yml' => <<~YAML, + my_build: + script: echo Hello World + YAML + '.internal-include.yml' => <<~YAML + include: + - local: myfolder/file1.yml + - local: myfolder/file1.yml + YAML + } + end + + it 'raises error' do + expect { process }.to raise_error(Gitlab::Ci::Config::External::Processor::IncludeError) + end + end + end + end + + context 'when FF ci_fix_max_includes is disabled' do + before do + stub_feature_flags(ci_fix_max_includes: false) + end + + context 'when total file count exceeds max_includes' do + context 'when files are nested' do + let(:files) do + [ + Gitlab::Ci::Config::External::File::Local.new({ local: 'nested_configs.yml' }, context) + ] + end + + it 'raises Processor::IncludeError' do + allow(context).to receive(:max_includes).and_return(1) + expect { process }.to raise_error(Gitlab::Ci::Config::External::Processor::IncludeError) + end + end + + context 'when files are not nested' do + let(:files) do + [ + Gitlab::Ci::Config::External::File::Local.new({ local: 'myfolder/file1.yml' }, context), + Gitlab::Ci::Config::External::File::Local.new({ local: 'myfolder/file2.yml' }, context) + ] + end + + it 'raises Mapper::TooManyIncludesError' do + allow(context).to receive(:max_includes).and_return(1) + expect { process }.to raise_error(Gitlab::Ci::Config::External::Mapper::TooManyIncludesError) + end + end + + context 'when files are duplicates' do + let(:files) do + [ + Gitlab::Ci::Config::External::File::Local.new({ local: 'myfolder/file1.yml' }, context), + Gitlab::Ci::Config::External::File::Local.new({ local: 'myfolder/file1.yml' }, context), + Gitlab::Ci::Config::External::File::Local.new({ local: 'myfolder/file1.yml' }, context) + ] + end + + it 'raises error' do allow(context).to receive(:max_includes).and_return(2) - expect { process }.not_to raise_error + expect { process }.to raise_error(Gitlab::Ci::Config::External::Mapper::TooManyIncludesError) end end end diff --git a/spec/lib/gitlab/ci/config/external/mapper_spec.rb b/spec/lib/gitlab/ci/config/external/mapper_spec.rb index b3115617084..56d1ddee4b8 100644 --- a/spec/lib/gitlab/ci/config/external/mapper_spec.rb +++ b/spec/lib/gitlab/ci/config/external/mapper_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::External::Mapper, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::External::Mapper, feature_category: :pipeline_composition do include StubRequests include RepoHelpers @@ -234,17 +234,6 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper, feature_category: :pipeline process expect(context.expandset.size).to eq(2) end - - context 'when FF ci_includes_count_duplicates is disabled' do - before do - stub_feature_flags(ci_includes_count_duplicates: false) - end - - it 'has expanset with one' do - process - expect(context.expandset.size).to eq(1) - end - end end context 'when passing max number of files' do diff --git a/spec/lib/gitlab/ci/config/external/processor_spec.rb b/spec/lib/gitlab/ci/config/external/processor_spec.rb index bb65c2ef10c..97f600baf25 100644 --- a/spec/lib/gitlab/ci/config/external/processor_spec.rb +++ b/spec/lib/gitlab/ci/config/external/processor_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::External::Processor, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::External::Processor, feature_category: :pipeline_composition do include StubRequests include RepoHelpers diff --git a/spec/lib/gitlab/ci/config/external/rules_spec.rb b/spec/lib/gitlab/ci/config/external/rules_spec.rb index 227b62d8ce8..cc73338b5a8 100644 --- a/spec/lib/gitlab/ci/config/external/rules_spec.rb +++ b/spec/lib/gitlab/ci/config/external/rules_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::External::Rules, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::External::Rules, feature_category: :pipeline_composition do let(:rule_hashes) {} subject(:rules) { described_class.new(rule_hashes) } diff --git a/spec/lib/gitlab/ci/config/header/input_spec.rb b/spec/lib/gitlab/ci/config/header/input_spec.rb new file mode 100644 index 00000000000..73b5b8f9497 --- /dev/null +++ b/spec/lib/gitlab/ci/config/header/input_spec.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Config::Header::Input, feature_category: :pipeline_composition do + let(:factory) do + Gitlab::Config::Entry::Factory + .new(described_class) + .value(input_hash) + .with(key: input_name) + end + + let(:input_name) { 'foo' } + + subject(:config) { factory.create!.tap(&:compose!) } + + shared_examples 'a valid input' do + let(:expected_hash) { input_hash } + + it 'passes validations' do + expect(config).to be_valid + expect(config.errors).to be_empty + end + + it 'returns the value' do + expect(config.value).to eq(expected_hash) + end + end + + shared_examples 'an invalid input' do + let(:expected_hash) { input_hash } + + it 'fails validations' do + expect(config).not_to be_valid + expect(config.errors).to eq(expected_errors) + end + + it 'returns the value' do + expect(config.value).to eq(expected_hash) + end + end + + context 'when has a default value' do + let(:input_hash) { { default: 'bar' } } + + it_behaves_like 'a valid input' + end + + context 'when is a required required input' do + let(:input_hash) { nil } + + it_behaves_like 'a valid input' + end + + context 'when contains unknown keywords' do + let(:input_hash) { { test: 123 } } + let(:expected_errors) { ['foo config contains unknown keys: test'] } + + it_behaves_like 'an invalid input' + end + + context 'when has invalid name' do + let(:input_name) { [123] } + let(:input_hash) { {} } + + let(:expected_errors) { ['123 key must be an alphanumeric string'] } + + it_behaves_like 'an invalid input' + end +end diff --git a/spec/lib/gitlab/ci/config/header/root_spec.rb b/spec/lib/gitlab/ci/config/header/root_spec.rb new file mode 100644 index 00000000000..55f77137619 --- /dev/null +++ b/spec/lib/gitlab/ci/config/header/root_spec.rb @@ -0,0 +1,133 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Config::Header::Root, feature_category: :pipeline_composition do + let(:factory) { Gitlab::Config::Entry::Factory.new(described_class).value(header_hash) } + + subject(:config) { factory.create!.tap(&:compose!) } + + shared_examples 'a valid header' do + let(:expected_hash) { header_hash } + + it 'passes validations' do + expect(config).to be_valid + expect(config.errors).to be_empty + end + + it 'returns the value' do + expect(config.value).to eq(expected_hash) + end + end + + shared_examples 'an invalid header' do + let(:expected_hash) { header_hash } + + it 'fails validations' do + expect(config).not_to be_valid + expect(config.errors).to eq(expected_errors) + end + + it 'returns the value' do + expect(config.value).to eq(expected_hash) + end + end + + context 'when header contains default and required values for inputs' do + let(:header_hash) do + { + spec: { + inputs: { + test: {}, + foo: { + default: 'bar' + } + } + } + } + end + + it_behaves_like 'a valid header' + end + + context 'when header contains minimal data' do + let(:header_hash) do + { + spec: { + inputs: nil + } + } + end + + it_behaves_like 'a valid header' do + let(:expected_hash) { { spec: {} } } + end + end + + context 'when header contains required inputs' do + let(:header_hash) do + { + spec: { + inputs: { foo: nil } + } + } + end + + it_behaves_like 'a valid header' do + let(:expected_hash) do + { + spec: { + inputs: { foo: {} } + } + } + end + end + end + + context 'when header contains unknown keywords' do + let(:header_hash) { { test: 123 } } + let(:expected_errors) { ['root config contains unknown keys: test'] } + + it_behaves_like 'an invalid header' + end + + context 'when header input entry has an unknown key' do + let(:header_hash) do + { + spec: { + inputs: { + foo: { + bad: 'value' + } + } + } + } + end + + let(:expected_errors) { ['spec:inputs:foo config contains unknown keys: bad'] } + + it_behaves_like 'an invalid header' + end + + describe '#inputs_value' do + let(:header_hash) do + { + spec: { + inputs: { + foo: nil, + bar: { + default: 'baz' + } + } + } + } + end + + it 'returns the inputs' do + expect(config.inputs_value).to eq({ + foo: {}, + bar: { default: 'baz' } + }) + end + end +end diff --git a/spec/lib/gitlab/ci/config/header/spec_spec.rb b/spec/lib/gitlab/ci/config/header/spec_spec.rb new file mode 100644 index 00000000000..cb4237f84ce --- /dev/null +++ b/spec/lib/gitlab/ci/config/header/spec_spec.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Config::Header::Spec, feature_category: :pipeline_composition do + let(:factory) { Gitlab::Config::Entry::Factory.new(described_class).value(spec_hash) } + + subject(:config) { factory.create!.tap(&:compose!) } + + context 'when spec contains default values for inputs' do + let(:spec_hash) do + { + inputs: { + foo: { + default: 'bar' + } + } + } + end + + it 'passes validations' do + expect(config).to be_valid + expect(config.errors).to be_empty + end + + it 'returns the value' do + expect(config.value).to eq(spec_hash) + end + end + + context 'when spec contains unknown keywords' do + let(:spec_hash) { { test: 123 } } + let(:expected_errors) { ['spec config contains unknown keys: test'] } + + it 'fails validations' do + expect(config).not_to be_valid + expect(config.errors).to eq(expected_errors) + end + + it 'returns the value' do + expect(config.value).to eq(spec_hash) + end + end +end diff --git a/spec/lib/gitlab/ci/config/yaml/result_spec.rb b/spec/lib/gitlab/ci/config/yaml/result_spec.rb new file mode 100644 index 00000000000..eda15ee9eb2 --- /dev/null +++ b/spec/lib/gitlab/ci/config/yaml/result_spec.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Config::Yaml::Result, feature_category: :pipeline_composition do + it 'does not have a header when config is a single hash' do + result = described_class.new(config: { a: 1, b: 2 }) + + expect(result).not_to have_header + end + + it 'has a header when config is an array of hashes' do + result = described_class.new(config: [{ a: 1 }, { b: 2 }]) + + expect(result).to have_header + expect(result.header).to eq({ a: 1 }) + end + + it 'raises an error when reading a header when there is none' do + result = described_class.new(config: { b: 2 }) + + expect { result.header }.to raise_error(ArgumentError) + end + + it 'stores an error / exception when initialized with it' do + result = described_class.new(error: ArgumentError.new('abc')) + + expect(result).not_to be_valid + expect(result.error).to be_a ArgumentError + end +end diff --git a/spec/lib/gitlab/ci/config/yaml_spec.rb b/spec/lib/gitlab/ci/config/yaml_spec.rb index 4b34553f55e..f4b70069bbe 100644 --- a/spec/lib/gitlab/ci/config/yaml_spec.rb +++ b/spec/lib/gitlab/ci/config/yaml_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::Yaml, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::Yaml, feature_category: :pipeline_composition do describe '.load!' do it 'loads a single-doc YAML file' do yaml = <<~YAML @@ -50,6 +50,15 @@ RSpec.describe Gitlab::Ci::Config::Yaml, feature_category: :pipeline_authoring d }) end + context 'when YAML is invalid' do + let(:yaml) { 'some: invalid: syntax' } + + it 'raises an error' do + expect { described_class.load!(yaml) } + .to raise_error ::Gitlab::Config::Loader::FormatError, /mapping values are not allowed in this context/ + end + end + context 'when ci_multi_doc_yaml is disabled' do before do stub_feature_flags(ci_multi_doc_yaml: false) @@ -102,4 +111,38 @@ RSpec.describe Gitlab::Ci::Config::Yaml, feature_category: :pipeline_authoring d end end end + + describe '.load_result!' do + context 'when syntax is invalid' do + let(:yaml) { 'some: invalid: syntax' } + + it 'returns an invalid result object' do + result = described_class.load_result!(yaml) + + expect(result).not_to be_valid + expect(result.error).to be_a ::Gitlab::Config::Loader::FormatError + end + end + + context 'when syntax is valid and contains a header document' do + let(:yaml) do + <<~YAML + a: 1 + --- + b: 2 + YAML + end + + let(:project) { create(:project) } + + it 'returns a result object' do + result = described_class.load_result!(yaml, project: project) + + expect(result).to be_valid + expect(result.error).to be_nil + expect(result.header).to eq({ a: 1 }) + expect(result.content).to eq({ b: 2 }) + end + end + end end diff --git a/spec/lib/gitlab/ci/config_spec.rb b/spec/lib/gitlab/ci/config_spec.rb index 5cdc9c21561..fdf152b3584 100644 --- a/spec/lib/gitlab/ci/config_spec.rb +++ b/spec/lib/gitlab/ci/config_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config, feature_category: :pipeline_composition do include StubRequests include RepoHelpers diff --git a/spec/lib/gitlab/ci/input/arguments/base_spec.rb b/spec/lib/gitlab/ci/input/arguments/base_spec.rb new file mode 100644 index 00000000000..ed8e99b7257 --- /dev/null +++ b/spec/lib/gitlab/ci/input/arguments/base_spec.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +RSpec.describe Gitlab::Ci::Input::Arguments::Base, feature_category: :pipeline_composition do + subject do + Class.new(described_class) do + def validate!; end + def to_value; end + end + end + + it 'fabricates an invalid input argument if unknown value is provided' do + argument = subject.new(:something, { spec: 123 }, [:a, :b]) + + expect(argument).not_to be_valid + expect(argument.errors.first).to eq 'unsupported value in input argument `something`' + end +end diff --git a/spec/lib/gitlab/ci/input/arguments/default_spec.rb b/spec/lib/gitlab/ci/input/arguments/default_spec.rb new file mode 100644 index 00000000000..6b5dd441eb7 --- /dev/null +++ b/spec/lib/gitlab/ci/input/arguments/default_spec.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +RSpec.describe Gitlab::Ci::Input::Arguments::Default, feature_category: :pipeline_composition do + it 'returns a user-provided value if it is present' do + argument = described_class.new(:website, { default: 'https://gitlab.com' }, 'https://example.gitlab.com') + + expect(argument).to be_valid + expect(argument.to_value).to eq 'https://example.gitlab.com' + expect(argument.to_hash).to eq({ website: 'https://example.gitlab.com' }) + end + + it 'returns an empty value if user-provider input is empty' do + argument = described_class.new(:website, { default: 'https://gitlab.com' }, '') + + expect(argument).to be_valid + expect(argument.to_value).to eq '' + expect(argument.to_hash).to eq({ website: '' }) + end + + it 'returns a default value if user-provider one is unknown' do + argument = described_class.new(:website, { default: 'https://gitlab.com' }, nil) + + expect(argument).to be_valid + expect(argument.to_value).to eq 'https://gitlab.com' + expect(argument.to_hash).to eq({ website: 'https://gitlab.com' }) + end + + it 'returns an error if the argument has not been fabricated correctly' do + argument = described_class.new(:website, { required: 'https://gitlab.com' }, 'https://example.gitlab.com') + + expect(argument).not_to be_valid + end + + describe '.matches?' do + it 'matches specs with default configuration' do + expect(described_class.matches?({ default: 'abc' })).to be true + end + + it 'does not match specs different configuration keyword' do + expect(described_class.matches?({ options: %w[a b] })).to be false + end + end +end diff --git a/spec/lib/gitlab/ci/input/arguments/options_spec.rb b/spec/lib/gitlab/ci/input/arguments/options_spec.rb new file mode 100644 index 00000000000..afa279ad48d --- /dev/null +++ b/spec/lib/gitlab/ci/input/arguments/options_spec.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +RSpec.describe Gitlab::Ci::Input::Arguments::Options, feature_category: :pipeline_composition do + it 'returns a user-provided value if it is an allowed one' do + argument = described_class.new(:run, { options: %w[opt1 opt2] }, 'opt1') + + expect(argument).to be_valid + expect(argument.to_value).to eq 'opt1' + expect(argument.to_hash).to eq({ run: 'opt1' }) + end + + it 'returns an error if user-provided value is not allowlisted' do + argument = described_class.new(:run, { options: %w[opt1 opt2] }, 'opt3') + + expect(argument).not_to be_valid + expect(argument.errors.first).to eq '`run` input: argument value opt3 not allowlisted' + end + + it 'returns an error if specification is not correct' do + argument = described_class.new(:website, { options: nil }, 'opt1') + + expect(argument).not_to be_valid + expect(argument.errors.first).to eq '`website` input: argument specification invalid' + end + + it 'returns an error if specification is using a hash' do + argument = described_class.new(:website, { options: { a: 1 } }, 'opt1') + + expect(argument).not_to be_valid + expect(argument.errors.first).to eq '`website` input: argument value opt1 not allowlisted' + end + + it 'returns an empty value if it is allowlisted' do + argument = described_class.new(:run, { options: ['opt1', ''] }, '') + + expect(argument).to be_valid + expect(argument.to_value).to be_empty + expect(argument.to_hash).to eq({ run: '' }) + end + + describe '.matches?' do + it 'matches specs with options configuration' do + expect(described_class.matches?({ options: %w[a b] })).to be true + end + + it 'does not match specs different configuration keyword' do + expect(described_class.matches?({ default: 'abc' })).to be false + end + end +end diff --git a/spec/lib/gitlab/ci/input/arguments/required_spec.rb b/spec/lib/gitlab/ci/input/arguments/required_spec.rb new file mode 100644 index 00000000000..0c2ffc282ea --- /dev/null +++ b/spec/lib/gitlab/ci/input/arguments/required_spec.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +RSpec.describe Gitlab::Ci::Input::Arguments::Required, feature_category: :pipeline_composition do + it 'returns a user-provided value if it is present' do + argument = described_class.new(:website, nil, 'https://example.gitlab.com') + + expect(argument).to be_valid + expect(argument.to_value).to eq 'https://example.gitlab.com' + expect(argument.to_hash).to eq({ website: 'https://example.gitlab.com' }) + end + + it 'returns an empty value if user-provider value is empty' do + argument = described_class.new(:website, nil, '') + + expect(argument).to be_valid + expect(argument.to_hash).to eq(website: '') + end + + it 'returns an error if user-provided value is unspecified' do + argument = described_class.new(:website, nil, nil) + + expect(argument).not_to be_valid + expect(argument.errors.first).to eq '`website` input: required value has not been provided' + end + + describe '.matches?' do + it 'matches specs without configuration' do + expect(described_class.matches?(nil)).to be true + end + + it 'matches specs with empty configuration' do + expect(described_class.matches?('')).to be true + end + + it 'does not match specs with configuration' do + expect(described_class.matches?({ options: %w[a b] })).to be false + end + end +end diff --git a/spec/lib/gitlab/ci/input/arguments/unknown_spec.rb b/spec/lib/gitlab/ci/input/arguments/unknown_spec.rb new file mode 100644 index 00000000000..1270423ac72 --- /dev/null +++ b/spec/lib/gitlab/ci/input/arguments/unknown_spec.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +RSpec.describe Gitlab::Ci::Input::Arguments::Unknown, feature_category: :pipeline_composition do + it 'raises an error when someone tries to evaluate the value' do + argument = described_class.new(:website, nil, 'https://example.gitlab.com') + + expect(argument).not_to be_valid + expect { argument.to_value }.to raise_error ArgumentError + end + + describe '.matches?' do + it 'always matches' do + expect(described_class.matches?('abc')).to be true + end + end +end diff --git a/spec/lib/gitlab/ci/input/inputs_spec.rb b/spec/lib/gitlab/ci/input/inputs_spec.rb new file mode 100644 index 00000000000..5d2d5192299 --- /dev/null +++ b/spec/lib/gitlab/ci/input/inputs_spec.rb @@ -0,0 +1,126 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +RSpec.describe Gitlab::Ci::Input::Inputs, feature_category: :pipeline_composition do + describe '#valid?' do + let(:spec) { { website: nil } } + + it 'describes user-provided inputs' do + inputs = described_class.new(spec, { website: 'http://example.gitlab.com' }) + + expect(inputs).to be_valid + end + end + + context 'when proper specification has been provided' do + let(:spec) do + { + website: nil, + env: { default: 'development' }, + run: { options: %w[tests spec e2e] } + } + end + + let(:args) { { website: 'https://gitlab.com', run: 'tests' } } + + it 'fabricates desired input arguments' do + inputs = described_class.new(spec, args) + + expect(inputs).to be_valid + expect(inputs.count).to eq 3 + expect(inputs.to_hash).to eq(args.merge(env: 'development')) + end + end + + context 'when inputs and args are empty' do + it 'is a valid use-case' do + inputs = described_class.new({}, {}) + + expect(inputs).to be_valid + expect(inputs.to_hash).to be_empty + end + end + + context 'when there are arguments recoincilation errors present' do + context 'when required argument is missing' do + let(:spec) { { website: nil } } + + it 'returns an error' do + inputs = described_class.new(spec, {}) + + expect(inputs).not_to be_valid + expect(inputs.errors.first).to eq '`website` input: required value has not been provided' + end + end + + context 'when argument is not present but configured as allowlist' do + let(:spec) do + { run: { options: %w[opt1 opt2] } } + end + + it 'returns an error' do + inputs = described_class.new(spec, {}) + + expect(inputs).not_to be_valid + expect(inputs.errors.first).to eq '`run` input: argument not provided' + end + end + end + + context 'when unknown specification argument has been used' do + let(:spec) do + { + website: nil, + env: { default: 'development' }, + run: { options: %w[tests spec e2e] }, + test: { unknown: 'something' } + } + end + + let(:args) { { website: 'https://gitlab.com', run: 'tests' } } + + it 'fabricates an unknown argument entry and returns an error' do + inputs = described_class.new(spec, args) + + expect(inputs).not_to be_valid + expect(inputs.count).to eq 4 + expect(inputs.errors.first).to eq '`test` input: unrecognized input argument specification: `unknown`' + end + end + + context 'when unknown arguments are being passed by a user' do + let(:spec) do + { env: { default: 'development' } } + end + + let(:args) { { website: 'https://gitlab.com', run: 'tests' } } + + it 'returns an error with a list of unknown arguments' do + inputs = described_class.new(spec, args) + + expect(inputs).not_to be_valid + expect(inputs.errors.first).to eq 'unknown input arguments: [:website, :run]' + end + end + + context 'when composite specification is being used' do + let(:spec) do + { + env: { + default: 'dev', + options: %w[test dev prod] + } + } + end + + let(:args) { { env: 'dev' } } + + it 'returns an error describing an unknown specification' do + inputs = described_class.new(spec, args) + + expect(inputs).not_to be_valid + expect(inputs.errors.first).to eq '`env` input: unrecognized input argument definition' + end + end +end diff --git a/spec/lib/gitlab/ci/interpolation/access_spec.rb b/spec/lib/gitlab/ci/interpolation/access_spec.rb index 9f6108a328d..f327377b7e3 100644 --- a/spec/lib/gitlab/ci/interpolation/access_spec.rb +++ b/spec/lib/gitlab/ci/interpolation/access_spec.rb @@ -2,7 +2,7 @@ require 'fast_spec_helper' -RSpec.describe Gitlab::Ci::Interpolation::Access, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Interpolation::Access, feature_category: :pipeline_composition do subject { described_class.new(access, ctx) } let(:access) do diff --git a/spec/lib/gitlab/ci/interpolation/block_spec.rb b/spec/lib/gitlab/ci/interpolation/block_spec.rb index 7f2be505d17..4a8709df3dc 100644 --- a/spec/lib/gitlab/ci/interpolation/block_spec.rb +++ b/spec/lib/gitlab/ci/interpolation/block_spec.rb @@ -2,7 +2,7 @@ require 'fast_spec_helper' -RSpec.describe Gitlab::Ci::Interpolation::Block, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Interpolation::Block, feature_category: :pipeline_composition do subject { described_class.new(block, data, ctx) } let(:data) do diff --git a/spec/lib/gitlab/ci/interpolation/config_spec.rb b/spec/lib/gitlab/ci/interpolation/config_spec.rb index e5987776e00..e745269d8c0 100644 --- a/spec/lib/gitlab/ci/interpolation/config_spec.rb +++ b/spec/lib/gitlab/ci/interpolation/config_spec.rb @@ -2,7 +2,7 @@ require 'fast_spec_helper' -RSpec.describe Gitlab::Ci::Interpolation::Config, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Interpolation::Config, feature_category: :pipeline_composition do subject { described_class.new(YAML.safe_load(config)) } let(:config) do diff --git a/spec/lib/gitlab/ci/interpolation/context_spec.rb b/spec/lib/gitlab/ci/interpolation/context_spec.rb index ada896f4980..2b126f4a8b3 100644 --- a/spec/lib/gitlab/ci/interpolation/context_spec.rb +++ b/spec/lib/gitlab/ci/interpolation/context_spec.rb @@ -2,7 +2,7 @@ require 'fast_spec_helper' -RSpec.describe Gitlab::Ci::Interpolation::Context, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Interpolation::Context, feature_category: :pipeline_composition do subject { described_class.new(ctx) } let(:ctx) do diff --git a/spec/lib/gitlab/ci/interpolation/template_spec.rb b/spec/lib/gitlab/ci/interpolation/template_spec.rb index 8a243b4db05..a3ef1bb4445 100644 --- a/spec/lib/gitlab/ci/interpolation/template_spec.rb +++ b/spec/lib/gitlab/ci/interpolation/template_spec.rb @@ -2,7 +2,7 @@ require 'fast_spec_helper' -RSpec.describe Gitlab::Ci::Interpolation::Template, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Interpolation::Template, feature_category: :pipeline_composition do subject { described_class.new(YAML.safe_load(config), ctx) } let(:config) do diff --git a/spec/lib/gitlab/ci/lint_spec.rb b/spec/lib/gitlab/ci/lint_spec.rb index b836ca395fa..b238e9161eb 100644 --- a/spec/lib/gitlab/ci/lint_spec.rb +++ b/spec/lib/gitlab/ci/lint_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Lint, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Lint, feature_category: :pipeline_composition do let_it_be(:project) { create(:project, :repository) } let_it_be(:user) { create(:user) } @@ -100,8 +100,8 @@ RSpec.describe Gitlab::Ci::Lint, feature_category: :pipeline_authoring do end it 'sets merged_config' do - root_config = YAML.safe_load(content, [Symbol]) - included_config = YAML.safe_load(included_content, [Symbol]) + root_config = YAML.safe_load(content, permitted_classes: [Symbol]) + included_config = YAML.safe_load(included_content, permitted_classes: [Symbol]) expected_config = included_config.merge(root_config).except(:include).deep_stringify_keys expect(subject.merged_yaml).to eq(expected_config.to_yaml) diff --git a/spec/lib/gitlab/ci/parsers/security/common_spec.rb b/spec/lib/gitlab/ci/parsers/security/common_spec.rb index 5d2d22c04fc..421aa29f860 100644 --- a/spec/lib/gitlab/ci/parsers/security/common_spec.rb +++ b/spec/lib/gitlab/ci/parsers/security/common_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Parsers::Security::Common do +RSpec.describe Gitlab::Ci::Parsers::Security::Common, feature_category: :vulnerability_management do describe '#parse!' do let_it_be(:scanner_data) do { @@ -410,6 +410,12 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Common do end end + describe 'setting the `found_by_pipeline` attribute' do + subject { report.findings.map(&:found_by_pipeline).uniq } + + it { is_expected.to eq([pipeline]) } + end + describe 'parsing tracking' do let(:finding) { report.findings.first } diff --git a/spec/lib/gitlab/ci/pipeline/chain/config/content_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/config/content_spec.rb index e0d656f456e..a9a52972294 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/config/content_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/config/content_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Pipeline::Chain::Config::Content do +RSpec.describe Gitlab::Ci::Pipeline::Chain::Config::Content, feature_category: :continuous_integration do let(:project) { create(:project, ci_config_path: ci_config_path) } let(:pipeline) { build(:ci_pipeline, project: project) } let(:content) { nil } @@ -26,6 +26,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Config::Content do expect(pipeline.config_source).to eq 'bridge_source' expect(command.config_content).to eq 'the-yaml' + expect(command.pipeline_config.internal_include_prepended?).to eq(false) end end @@ -52,6 +53,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Config::Content do expect(pipeline.config_source).to eq 'repository_source' expect(pipeline.pipeline_config.content).to eq(config_content_result) expect(command.config_content).to eq(config_content_result) + expect(command.pipeline_config.internal_include_prepended?).to eq(true) end end @@ -71,6 +73,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Config::Content do expect(pipeline.config_source).to eq 'remote_source' expect(pipeline.pipeline_config.content).to eq(config_content_result) expect(command.config_content).to eq(config_content_result) + expect(command.pipeline_config.internal_include_prepended?).to eq(true) end end @@ -91,6 +94,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Config::Content do expect(pipeline.config_source).to eq 'external_project_source' expect(pipeline.pipeline_config.content).to eq(config_content_result) expect(command.config_content).to eq(config_content_result) + expect(command.pipeline_config.internal_include_prepended?).to eq(true) end context 'when path specifies a refname' do @@ -111,6 +115,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Config::Content do expect(pipeline.config_source).to eq 'external_project_source' expect(pipeline.pipeline_config.content).to eq(config_content_result) expect(command.config_content).to eq(config_content_result) + expect(command.pipeline_config.internal_include_prepended?).to eq(true) end end end @@ -138,6 +143,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Config::Content do expect(pipeline.config_source).to eq 'repository_source' expect(pipeline.pipeline_config.content).to eq(config_content_result) expect(command.config_content).to eq(config_content_result) + expect(command.pipeline_config.internal_include_prepended?).to eq(true) end end @@ -161,6 +167,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Config::Content do expect(pipeline.config_source).to eq 'auto_devops_source' expect(pipeline.pipeline_config.content).to eq(config_content_result) expect(command.config_content).to eq(config_content_result) + expect(command.pipeline_config.internal_include_prepended?).to eq(true) end end @@ -181,6 +188,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Config::Content do expect(pipeline.config_source).to eq 'parameter_source' expect(pipeline.pipeline_config.content).to eq(content) expect(command.config_content).to eq(content) + expect(command.pipeline_config.internal_include_prepended?).to eq(false) end end @@ -197,6 +205,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Config::Content do expect(pipeline.config_source).to eq('unknown_source') expect(pipeline.pipeline_config).to be_nil expect(command.config_content).to be_nil + expect(command.pipeline_config).to be_nil expect(pipeline.errors.full_messages).to include('Missing CI config file') end end diff --git a/spec/lib/gitlab/ci/pipeline/duration_spec.rb b/spec/lib/gitlab/ci/pipeline/duration_spec.rb index 36714413da6..89c0ce46237 100644 --- a/spec/lib/gitlab/ci/pipeline/duration_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/duration_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Pipeline::Duration do +RSpec.describe Gitlab::Ci::Pipeline::Duration, feature_category: :continuous_integration do describe '.from_periods' do let(:calculated_duration) { calculate(data) } @@ -113,16 +113,17 @@ RSpec.describe Gitlab::Ci::Pipeline::Duration do described_class::Period.new(first, last) end - described_class.from_periods(periods.sort_by(&:first)) + described_class.send(:from_periods, periods.sort_by(&:first)) end end describe '.from_pipeline' do + let_it_be_with_reload(:pipeline) { create(:ci_pipeline) } + let_it_be(:start_time) { Time.current.change(usec: 0) } let_it_be(:current) { start_time + 1000 } - let_it_be(:pipeline) { create(:ci_pipeline) } - let_it_be(:success_build) { create_build(:success, started_at: start_time, finished_at: start_time + 60) } - let_it_be(:failed_build) { create_build(:failed, started_at: start_time + 60, finished_at: start_time + 120) } + let_it_be(:success_build) { create_build(:success, started_at: start_time, finished_at: start_time + 50) } + let_it_be(:failed_build) { create_build(:failed, started_at: start_time + 60, finished_at: start_time + 110) } let_it_be(:canceled_build) { create_build(:canceled, started_at: start_time + 120, finished_at: start_time + 180) } let_it_be(:skipped_build) { create_build(:skipped, started_at: start_time) } let_it_be(:pending_build) { create_build(:pending) } @@ -141,21 +142,55 @@ RSpec.describe Gitlab::Ci::Pipeline::Duration do end context 'when there is no running build' do - let(:running_build) { nil } + let!(:running_build) { nil } it 'returns the duration for all the builds' do travel_to(current) do - expect(described_class.from_pipeline(pipeline)).to eq 180.seconds + # 160 = success (50) + failed (50) + canceled (60) + expect(described_class.from_pipeline(pipeline)).to eq 160.seconds end end end - context 'when there are bridge jobs' do - let!(:success_bridge) { create_bridge(:success, started_at: start_time + 220, finished_at: start_time + 280) } - let!(:failed_bridge) { create_bridge(:failed, started_at: start_time + 180, finished_at: start_time + 240) } - let!(:skipped_bridge) { create_bridge(:skipped, started_at: start_time) } - let!(:created_bridge) { create_bridge(:created) } - let!(:manual_bridge) { create_bridge(:manual) } + context 'when there are direct bridge jobs' do + let_it_be(:success_bridge) do + create_bridge(:success, started_at: start_time + 220, finished_at: start_time + 280) + end + + let_it_be(:failed_bridge) { create_bridge(:failed, started_at: start_time + 180, finished_at: start_time + 240) } + # NOTE: bridge won't be `canceled` as it will be marked as failed when downstream pipeline is canceled + # @see Ci::Bridge#inherit_status_from_downstream + let_it_be(:canceled_bridge) do + create_bridge(:failed, started_at: start_time + 180, finished_at: start_time + 210) + end + + let_it_be(:skipped_bridge) { create_bridge(:skipped, started_at: start_time) } + let_it_be(:created_bridge) { create_bridge(:created) } + let_it_be(:manual_bridge) { create_bridge(:manual) } + + let_it_be(:success_bridge_pipeline) do + create(:ci_pipeline, :success, started_at: start_time + 230, finished_at: start_time + 280).tap do |p| + create(:ci_sources_pipeline, source_job: success_bridge, pipeline: p) + create_build(:success, pipeline: p, started_at: start_time + 235, finished_at: start_time + 280) + create_bridge(:success, pipeline: p, started_at: start_time + 240, finished_at: start_time + 280) + end + end + + let_it_be(:failed_bridge_pipeline) do + create(:ci_pipeline, :failed, started_at: start_time + 225, finished_at: start_time + 240).tap do |p| + create(:ci_sources_pipeline, source_job: failed_bridge, pipeline: p) + create_build(:failed, pipeline: p, started_at: start_time + 230, finished_at: start_time + 240) + create_bridge(:success, pipeline: p, started_at: start_time + 235, finished_at: start_time + 240) + end + end + + let_it_be(:canceled_bridge_pipeline) do + create(:ci_pipeline, :canceled, started_at: start_time + 190, finished_at: start_time + 210).tap do |p| + create(:ci_sources_pipeline, source_job: canceled_bridge, pipeline: p) + create_build(:canceled, pipeline: p, started_at: start_time + 200, finished_at: start_time + 210) + create_bridge(:success, pipeline: p, started_at: start_time + 205, finished_at: start_time + 210) + end + end it 'returns the duration of the running build' do travel_to(current) do @@ -166,12 +201,99 @@ RSpec.describe Gitlab::Ci::Pipeline::Duration do context 'when there is no running build' do let!(:running_build) { nil } - it 'returns the duration for all the builds and bridge jobs' do + it 'returns the duration for all the builds (including self and downstreams)' do travel_to(current) do - expect(described_class.from_pipeline(pipeline)).to eq 280.seconds + # 220 = 160 (see above) + # + success build (45) + failed (10) + canceled (10) - overlapping (success & failed) (5) + expect(described_class.from_pipeline(pipeline)).to eq 220.seconds end end end + + # rubocop:disable RSpec/MultipleMemoizedHelpers + context 'when there are downstream bridge jobs' do + let_it_be(:success_direct_bridge) do + create_bridge(:success, started_at: start_time + 280, finished_at: start_time + 400) + end + + let_it_be(:success_downstream_pipeline) do + create(:ci_pipeline, :success, started_at: start_time + 285, finished_at: start_time + 300).tap do |p| + create(:ci_sources_pipeline, source_job: success_direct_bridge, pipeline: p) + create_build(:success, pipeline: p, started_at: start_time + 290, finished_at: start_time + 296) + create_bridge(:success, pipeline: p, started_at: start_time + 285, finished_at: start_time + 288) + end + end + + let_it_be(:failed_downstream_pipeline) do + create(:ci_pipeline, :failed, started_at: start_time + 305, finished_at: start_time + 350).tap do |p| + create(:ci_sources_pipeline, source_job: success_direct_bridge, pipeline: p) + create_build(:failed, pipeline: p, started_at: start_time + 320, finished_at: start_time + 327) + create_bridge(:success, pipeline: p, started_at: start_time + 305, finished_at: start_time + 350) + end + end + + let_it_be(:canceled_downstream_pipeline) do + create(:ci_pipeline, :canceled, started_at: start_time + 360, finished_at: start_time + 400).tap do |p| + create(:ci_sources_pipeline, source_job: success_direct_bridge, pipeline: p) + create_build(:canceled, pipeline: p, started_at: start_time + 390, finished_at: start_time + 398) + create_bridge(:success, pipeline: p, started_at: start_time + 360, finished_at: start_time + 378) + end + end + + it 'returns the duration of the running build' do + travel_to(current) do + expect(described_class.from_pipeline(pipeline)).to eq 1000.seconds + end + end + + context 'when there is no running build' do + let!(:running_build) { nil } + + it 'returns the duration for all the builds (including self and downstreams)' do + travel_to(current) do + # 241 = 220 (see above) + # + success downstream build (6) + failed (7) + canceled (8) + expect(described_class.from_pipeline(pipeline)).to eq 241.seconds + end + end + end + end + # rubocop:enable RSpec/MultipleMemoizedHelpers + end + + it 'does not generate N+1 queries if more builds are added' do + travel_to(current) do + expect do + described_class.from_pipeline(pipeline) + end.not_to exceed_query_limit(1) + + create_list(:ci_build, 2, :success, pipeline: pipeline, started_at: start_time, finished_at: start_time + 50) + + expect do + described_class.from_pipeline(pipeline) + end.not_to exceed_query_limit(1) + end + end + + it 'does not generate N+1 queries if more bridges and their pipeline builds are added' do + travel_to(current) do + expect do + described_class.from_pipeline(pipeline) + end.not_to exceed_query_limit(1) + + create_list( + :ci_bridge, 2, :success, + pipeline: pipeline, started_at: start_time + 220, finished_at: start_time + 280).each do |bridge| + create(:ci_pipeline, :success, started_at: start_time + 235, finished_at: start_time + 280).tap do |p| + create(:ci_sources_pipeline, source_job: bridge, pipeline: p) + create_builds(3, :success) + end + end + + expect do + described_class.from_pipeline(pipeline) + end.not_to exceed_query_limit(1) + end end private @@ -180,6 +302,10 @@ RSpec.describe Gitlab::Ci::Pipeline::Duration do create(:ci_build, trait, pipeline: pipeline, **opts) end + def create_builds(counts, trait, **opts) + create_list(:ci_build, counts, trait, pipeline: pipeline, **opts) + end + def create_bridge(trait, **opts) create(:ci_bridge, trait, pipeline: pipeline, **opts) end diff --git a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb index 3043d7f5381..ce68e741d00 100644 --- a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Pipeline::Seed::Build, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Pipeline::Seed::Build, feature_category: :pipeline_composition do let_it_be_with_reload(:project) { create(:project, :repository) } let_it_be(:head_sha) { project.repository.head_commit.id } diff --git a/spec/lib/gitlab/ci/pipeline/seed/stage_spec.rb b/spec/lib/gitlab/ci/pipeline/seed/stage_spec.rb index 288ac3f3854..ae40626510f 100644 --- a/spec/lib/gitlab/ci/pipeline/seed/stage_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/seed/stage_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Pipeline::Seed::Stage, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Pipeline::Seed::Stage, feature_category: :pipeline_composition do let(:project) { create(:project, :repository) } let(:pipeline) { create(:ci_empty_pipeline, project: project) } let(:previous_stages) { [] } diff --git a/spec/lib/gitlab/ci/project_config/repository_spec.rb b/spec/lib/gitlab/ci/project_config/repository_spec.rb index 2105b691d9e..e8a997a7e43 100644 --- a/spec/lib/gitlab/ci/project_config/repository_spec.rb +++ b/spec/lib/gitlab/ci/project_config/repository_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::ProjectConfig::Repository do +RSpec.describe Gitlab::Ci::ProjectConfig::Repository, feature_category: :continuous_integration do let(:project) { create(:project, :custom_repo, files: files) } let(:sha) { project.repository.head_commit.sha } let(:files) { { 'README.md' => 'hello' } } @@ -44,4 +44,10 @@ RSpec.describe Gitlab::Ci::ProjectConfig::Repository do it { is_expected.to eq(:repository_source) } end + + describe '#internal_include_prepended?' do + subject { config.internal_include_prepended? } + + it { is_expected.to eq(true) } + end end diff --git a/spec/lib/gitlab/ci/project_config/source_spec.rb b/spec/lib/gitlab/ci/project_config/source_spec.rb index dda5c7cdce8..eefabe1babb 100644 --- a/spec/lib/gitlab/ci/project_config/source_spec.rb +++ b/spec/lib/gitlab/ci/project_config/source_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::ProjectConfig::Source do +RSpec.describe Gitlab::Ci::ProjectConfig::Source, feature_category: :continuous_integration do let_it_be(:custom_config_class) { Class.new(described_class) } let_it_be(:project) { build_stubbed(:project) } let_it_be(:sha) { '123456' } @@ -20,4 +20,10 @@ RSpec.describe Gitlab::Ci::ProjectConfig::Source do it { expect { source }.to raise_error(NotImplementedError) } end + + describe '#internal_include_prepended?' do + subject(:internal_include_prepended) { custom_config.internal_include_prepended? } + + it { expect(internal_include_prepended).to eq(false) } + end end diff --git a/spec/lib/gitlab/ci/reports/security/vulnerability_reports_comparer_spec.rb b/spec/lib/gitlab/ci/reports/security/vulnerability_reports_comparer_spec.rb deleted file mode 100644 index 6f75e2c55e8..00000000000 --- a/spec/lib/gitlab/ci/reports/security/vulnerability_reports_comparer_spec.rb +++ /dev/null @@ -1,163 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Ci::Reports::Security::VulnerabilityReportsComparer do - let(:identifier) { build(:ci_reports_security_identifier) } - - let_it_be(:project) { create(:project, :repository) } - - let(:location_param) { build(:ci_reports_security_locations_sast, :dynamic) } - let(:vulnerability_params) { vuln_params(project.id, [identifier], confidence: :low, severity: :critical) } - let(:base_vulnerability) { build(:ci_reports_security_finding, location: location_param, **vulnerability_params) } - let(:base_report) { build(:ci_reports_security_aggregated_reports, findings: [base_vulnerability]) } - - let(:head_vulnerability) { build(:ci_reports_security_finding, location: location_param, uuid: base_vulnerability.uuid, **vulnerability_params) } - let(:head_report) { build(:ci_reports_security_aggregated_reports, findings: [head_vulnerability]) } - - shared_context 'comparing reports' do - let(:vul_params) { vuln_params(project.id, [identifier]) } - let(:base_vulnerability) { build(:ci_reports_security_finding, :dynamic, **vul_params) } - let(:head_vulnerability) { build(:ci_reports_security_finding, :dynamic, **vul_params) } - let(:head_vul_findings) { [head_vulnerability, vuln] } - end - - subject { described_class.new(project, base_report, head_report) } - - where(vulnerability_finding_signatures: [true, false]) - - with_them do - before do - stub_licensed_features(vulnerability_finding_signatures: vulnerability_finding_signatures) - end - - describe '#base_report_out_of_date' do - context 'no base report' do - let(:base_report) { build(:ci_reports_security_aggregated_reports, reports: [], findings: []) } - - it 'is not out of date' do - expect(subject.base_report_out_of_date).to be false - end - end - - context 'base report older than one week' do - let(:report) { build(:ci_reports_security_report, created_at: 1.week.ago - 60.seconds) } - let(:base_report) { build(:ci_reports_security_aggregated_reports, reports: [report]) } - - it 'is not out of date' do - expect(subject.base_report_out_of_date).to be true - end - end - - context 'base report less than one week old' do - let(:report) { build(:ci_reports_security_report, created_at: 1.week.ago + 60.seconds) } - let(:base_report) { build(:ci_reports_security_aggregated_reports, reports: [report]) } - - it 'is not out of date' do - expect(subject.base_report_out_of_date).to be false - end - end - end - - describe '#added' do - let(:new_location) { build(:ci_reports_security_locations_sast, :dynamic) } - let(:vul_params) { vuln_params(project.id, [identifier], confidence: :high) } - let(:vuln) { build(:ci_reports_security_finding, severity: Enums::Vulnerability.severity_levels[:critical], location: new_location, **vul_params) } - let(:low_vuln) { build(:ci_reports_security_finding, severity: Enums::Vulnerability.severity_levels[:low], location: new_location, **vul_params) } - - context 'with new vulnerability' do - let(:head_report) { build(:ci_reports_security_aggregated_reports, findings: [head_vulnerability, vuln]) } - - it 'points to source tree' do - expect(subject.added).to eq([vuln]) - end - end - - context 'when comparing reports with different fingerprints' do - include_context 'comparing reports' - - let(:head_report) { build(:ci_reports_security_aggregated_reports, findings: head_vul_findings) } - - it 'does not find any overlap' do - expect(subject.added).to eq(head_vul_findings) - end - end - - context 'order' do - let(:head_report) { build(:ci_reports_security_aggregated_reports, findings: [head_vulnerability, vuln, low_vuln]) } - - it 'does not change' do - expect(subject.added).to eq([vuln, low_vuln]) - end - end - end - - describe '#fixed' do - let(:vul_params) { vuln_params(project.id, [identifier]) } - let(:vuln) { build(:ci_reports_security_finding, :dynamic, **vul_params ) } - let(:medium_vuln) { build(:ci_reports_security_finding, confidence: ::Enums::Vulnerability.confidence_levels[:high], severity: Enums::Vulnerability.severity_levels[:medium], uuid: vuln.uuid, **vul_params) } - - context 'with fixed vulnerability' do - let(:base_report) { build(:ci_reports_security_aggregated_reports, findings: [base_vulnerability, vuln]) } - - it 'points to base tree' do - expect(subject.fixed).to eq([vuln]) - end - end - - context 'when comparing reports with different fingerprints' do - include_context 'comparing reports' - - let(:base_report) { build(:ci_reports_security_aggregated_reports, findings: [base_vulnerability, vuln]) } - - it 'does not find any overlap' do - expect(subject.fixed).to eq([base_vulnerability, vuln]) - end - end - - context 'order' do - let(:vul_findings) { [vuln, medium_vuln] } - let(:base_report) { build(:ci_reports_security_aggregated_reports, findings: [*vul_findings, base_vulnerability]) } - - it 'does not change' do - expect(subject.fixed).to eq(vul_findings) - end - end - end - - describe 'with empty vulnerabilities' do - let(:empty_report) { build(:ci_reports_security_aggregated_reports, reports: [], findings: []) } - - it 'returns empty array when reports are not present' do - comparer = described_class.new(project, empty_report, empty_report) - - expect(comparer.fixed).to eq([]) - expect(comparer.added).to eq([]) - end - - it 'returns added vulnerability when base is empty and head is not empty' do - comparer = described_class.new(project, empty_report, head_report) - - expect(comparer.fixed).to eq([]) - expect(comparer.added).to eq([head_vulnerability]) - end - - it 'returns fixed vulnerability when head is empty and base is not empty' do - comparer = described_class.new(project, base_report, empty_report) - - expect(comparer.fixed).to eq([base_vulnerability]) - expect(comparer.added).to eq([]) - end - end - end - - def vuln_params(project_id, identifiers, confidence: :high, severity: :critical) - { - project_id: project_id, - report_type: :sast, - identifiers: identifiers, - confidence: ::Enums::Vulnerability.confidence_levels[confidence], - severity: ::Enums::Vulnerability.severity_levels[severity] - } - end -end diff --git a/spec/lib/gitlab/ci/runner_releases_spec.rb b/spec/lib/gitlab/ci/runner_releases_spec.rb index 14f3c95ec79..9e211327dee 100644 --- a/spec/lib/gitlab/ci/runner_releases_spec.rb +++ b/spec/lib/gitlab/ci/runner_releases_spec.rb @@ -177,6 +177,16 @@ RSpec.describe Gitlab::Ci::RunnerReleases, feature_category: :runner_fleet do it 'returns parsed and sorted Gitlab::VersionInfo objects' do expect(releases).to eq(expected_result) end + + context 'when fetching runner releases is disabled' do + before do + stub_application_setting(update_runner_versions_enabled: false) + end + + it 'returns nil' do + expect(releases).to be_nil + end + end end context 'when response contains unexpected input type' do @@ -218,6 +228,16 @@ RSpec.describe Gitlab::Ci::RunnerReleases, feature_category: :runner_fleet do it 'returns parsed and grouped Gitlab::VersionInfo objects' do expect(releases_by_minor).to eq(expected_result) end + + context 'when fetching runner releases is disabled' do + before do + stub_application_setting(update_runner_versions_enabled: false) + end + + it 'returns nil' do + expect(releases_by_minor).to be_nil + end + end end context 'when response contains unexpected input type' do @@ -233,6 +253,18 @@ RSpec.describe Gitlab::Ci::RunnerReleases, feature_category: :runner_fleet do end end + describe '#enabled?' do + it { is_expected.to be_enabled } + + context 'when fetching runner releases is disabled' do + before do + stub_application_setting(update_runner_versions_enabled: false) + end + + it { is_expected.not_to be_enabled } + end + end + def mock_http_response(response) http_response = instance_double(HTTParty::Response) diff --git a/spec/lib/gitlab/ci/templates/Jobs/build_gitlab_ci_yaml_spec.rb b/spec/lib/gitlab/ci/templates/Jobs/build_gitlab_ci_yaml_spec.rb index 07cfa939623..995922b6922 100644 --- a/spec/lib/gitlab/ci/templates/Jobs/build_gitlab_ci_yaml_spec.rb +++ b/spec/lib/gitlab/ci/templates/Jobs/build_gitlab_ci_yaml_spec.rb @@ -10,7 +10,7 @@ RSpec.describe 'Jobs/Build.gitlab-ci.yml' do describe 'AUTO_BUILD_IMAGE_VERSION' do it 'corresponds to a published image in the registry' do registry = "https://#{template_registry_host}" - repository = "gitlab-org/cluster-integration/auto-build-image" + repository = auto_build_image_repository reference = YAML.safe_load(template.content).dig('variables', 'AUTO_BUILD_IMAGE_VERSION') expect(public_image_exist?(registry, repository, reference)).to be true diff --git a/spec/lib/gitlab/ci/templates/Jobs/sast_iac_latest_gitlab_ci_yaml_spec.rb b/spec/lib/gitlab/ci/templates/Jobs/sast_iac_latest_gitlab_ci_yaml_spec.rb index 039a6a739dd..2b9213ea921 100644 --- a/spec/lib/gitlab/ci/templates/Jobs/sast_iac_latest_gitlab_ci_yaml_spec.rb +++ b/spec/lib/gitlab/ci/templates/Jobs/sast_iac_latest_gitlab_ci_yaml_spec.rb @@ -23,27 +23,33 @@ RSpec.describe 'Jobs/SAST-IaC.latest.gitlab-ci.yml', feature_category: :continuo allow(project).to receive(:default_branch).and_return(default_branch) end - context 'on feature branch' do - let(:pipeline_ref) { 'feature' } + context 'when SAST_DISABLED="false"' do + before do + create(:ci_variable, key: 'SAST_DISABLED', value: 'false', project: project) + end + + context 'on feature branch' do + let(:pipeline_ref) { 'feature' } - it 'creates the kics-iac-sast job' do - expect(build_names).to contain_exactly('kics-iac-sast') + it 'creates the kics-iac-sast job' do + expect(build_names).to contain_exactly('kics-iac-sast') + end end - end - context 'on merge request' do - let(:service) { MergeRequests::CreatePipelineService.new(project: project, current_user: user) } - let(:merge_request) { create(:merge_request, :simple, source_project: project) } - let(:pipeline) { service.execute(merge_request).payload } + context 'on merge request' do + let(:service) { MergeRequests::CreatePipelineService.new(project: project, current_user: user) } + let(:merge_request) { create(:merge_request, :simple, source_project: project) } + let(:pipeline) { service.execute(merge_request).payload } - it 'creates a pipeline with the expected jobs' do - expect(pipeline).to be_merge_request_event - expect(pipeline.errors.full_messages).to be_empty - expect(build_names).to match_array(%w(kics-iac-sast)) + it 'creates a pipeline with the expected jobs' do + expect(pipeline).to be_merge_request_event + expect(pipeline.errors.full_messages).to be_empty + expect(build_names).to match_array(%w(kics-iac-sast)) + end end end - context 'SAST_DISABLED is set' do + context 'when SAST_DISABLED="true"' do before do create(:ci_variable, key: 'SAST_DISABLED', value: 'true', project: project) end diff --git a/spec/lib/gitlab/ci/variables/builder/pipeline_spec.rb b/spec/lib/gitlab/ci/variables/builder/pipeline_spec.rb index a5365ae53b8..f8770457083 100644 --- a/spec/lib/gitlab/ci/variables/builder/pipeline_spec.rb +++ b/spec/lib/gitlab/ci/variables/builder/pipeline_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Variables::Builder::Pipeline, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Variables::Builder::Pipeline, feature_category: :pipeline_composition do let_it_be(:project) { create_default(:project, :repository, create_tag: 'test').freeze } let_it_be(:user) { create(:user) } diff --git a/spec/lib/gitlab/ci/variables/builder_spec.rb b/spec/lib/gitlab/ci/variables/builder_spec.rb index bbd3dc54e6a..215b18ea614 100644 --- a/spec/lib/gitlab/ci/variables/builder_spec.rb +++ b/spec/lib/gitlab/ci/variables/builder_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Variables::Builder, :clean_gitlab_redis_cache, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Variables::Builder, :clean_gitlab_redis_cache, feature_category: :pipeline_composition do include Ci::TemplateHelpers let_it_be(:group) { create(:group) } let_it_be(:project) { create(:project, :repository, namespace: group) } diff --git a/spec/lib/gitlab/ci/variables/collection_spec.rb b/spec/lib/gitlab/ci/variables/collection_spec.rb index 4ee122cc607..668f1173675 100644 --- a/spec/lib/gitlab/ci/variables/collection_spec.rb +++ b/spec/lib/gitlab/ci/variables/collection_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Variables::Collection, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Variables::Collection, feature_category: :pipeline_composition do describe '.new' do it 'can be initialized with an array' do variable = { key: 'VAR', value: 'value', public: true, masked: false } diff --git a/spec/lib/gitlab/ci/yaml_processor/result_spec.rb b/spec/lib/gitlab/ci/yaml_processor/result_spec.rb index 5c9f156e054..36ada9050b2 100644 --- a/spec/lib/gitlab/ci/yaml_processor/result_spec.rb +++ b/spec/lib/gitlab/ci/yaml_processor/result_spec.rb @@ -47,8 +47,8 @@ module Gitlab end it 'returns expanded yaml config' do - expanded_config = YAML.safe_load(config_metadata[:merged_yaml], [Symbol]) - included_config = YAML.safe_load(included_yml, [Symbol]) + expanded_config = YAML.safe_load(config_metadata[:merged_yaml], permitted_classes: [Symbol]) + included_config = YAML.safe_load(included_yml, permitted_classes: [Symbol]) expect(expanded_config).to include(*included_config.keys) end diff --git a/spec/lib/gitlab/ci/yaml_processor_spec.rb b/spec/lib/gitlab/ci/yaml_processor_spec.rb index 360686ce65c..b00d9b46bc7 100644 --- a/spec/lib/gitlab/ci/yaml_processor_spec.rb +++ b/spec/lib/gitlab/ci/yaml_processor_spec.rb @@ -4,7 +4,7 @@ require 'spec_helper' module Gitlab module Ci - RSpec.describe YamlProcessor, feature_category: :pipeline_authoring do + RSpec.describe YamlProcessor, feature_category: :pipeline_composition do include StubRequests include RepoHelpers diff --git a/spec/lib/gitlab/color_schemes_spec.rb b/spec/lib/gitlab/color_schemes_spec.rb index feb5648ff2d..bc69c8beeda 100644 --- a/spec/lib/gitlab/color_schemes_spec.rb +++ b/spec/lib/gitlab/color_schemes_spec.rb @@ -21,8 +21,9 @@ RSpec.describe Gitlab::ColorSchemes do end describe '.default' do - it 'returns the default scheme' do - expect(described_class.default.id).to eq 1 + it 'use config default' do + stub_application_setting(default_syntax_highlighting_theme: 2) + expect(described_class.default.id).to eq 2 end end @@ -36,7 +37,8 @@ RSpec.describe Gitlab::ColorSchemes do describe '.for_user' do it 'returns default when user is nil' do - expect(described_class.for_user(nil).id).to eq 1 + stub_application_setting(default_syntax_highlighting_theme: 2) + expect(described_class.for_user(nil).id).to eq 2 end it "returns user's preferred color scheme" do diff --git a/spec/lib/gitlab/config/entry/validators_spec.rb b/spec/lib/gitlab/config/entry/validators_spec.rb index 54a2adbefd2..abf3dbacb3d 100644 --- a/spec/lib/gitlab/config/entry/validators_spec.rb +++ b/spec/lib/gitlab/config/entry/validators_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Config::Entry::Validators, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Config::Entry::Validators, feature_category: :pipeline_composition do let(:klass) do Class.new do include ActiveModel::Validations diff --git a/spec/lib/gitlab/config/loader/multi_doc_yaml_spec.rb b/spec/lib/gitlab/config/loader/multi_doc_yaml_spec.rb index bae98f9bc35..f63aacecce6 100644 --- a/spec/lib/gitlab/config/loader/multi_doc_yaml_spec.rb +++ b/spec/lib/gitlab/config/loader/multi_doc_yaml_spec.rb @@ -2,26 +2,121 @@ require 'spec_helper' -RSpec.describe Gitlab::Config::Loader::MultiDocYaml, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Config::Loader::MultiDocYaml, feature_category: :pipeline_composition do let(:loader) { described_class.new(yml, max_documents: 2) } describe '#load!' do - let(:yml) do - <<~YAML - spec: - inputs: - test_input: - --- - test_job: - script: echo "$[[ inputs.test_input ]]" - YAML + context 'when a simple single delimiter is being used' do + let(:yml) do + <<~YAML + spec: + inputs: + env: + --- + test: + script: echo "$[[ inputs.env ]]" + YAML + end + + it 'returns the loaded YAML with all keys as symbols' do + expect(loader.load!).to contain_exactly( + { spec: { inputs: { env: nil } } }, + { test: { script: 'echo "$[[ inputs.env ]]"' } } + ) + end + end + + context 'when the delimiter has a trailing configuration' do + let(:yml) do + <<~YAML + spec: + inputs: + test_input: + --- !test/content + test_job: + script: echo "$[[ inputs.test_input ]]" + YAML + end + + it 'returns the loaded YAML with all keys as symbols' do + expect(loader.load!).to contain_exactly( + { spec: { inputs: { test_input: nil } } }, + { test_job: { script: 'echo "$[[ inputs.test_input ]]"' } } + ) + end + end + + context 'when the YAML file has a leading delimiter' do + let(:yml) do + <<~YAML + --- + spec: + inputs: + test_input: + --- !test/content + test_job: + script: echo "$[[ inputs.test_input ]]" + YAML + end + + it 'returns the loaded YAML with all keys as symbols' do + expect(loader.load!).to contain_exactly( + { spec: { inputs: { test_input: nil } } }, + { test_job: { script: 'echo "$[[ inputs.test_input ]]"' } } + ) + end + end + + context 'when the delimiter is followed by content on the same line' do + let(:yml) do + <<~YAML + --- a: 1 + --- b: 2 + YAML + end + + it 'loads the content as part of the document' do + expect(loader.load!).to contain_exactly({ a: 1 }, { b: 2 }) + end end - it 'returns the loaded YAML with all keys as symbols' do - expect(loader.load!).to eq([ - { spec: { inputs: { test_input: nil } } }, - { test_job: { script: 'echo "$[[ inputs.test_input ]]"' } } - ]) + context 'when the delimiter does not have trailing whitespace' do + let(:yml) do + <<~YAML + --- a: 1 + ---b: 2 + YAML + end + + it 'is not a valid delimiter' do + expect(loader.load!).to contain_exactly({ :'---b' => 2, a: 1 }) # rubocop:disable Style/HashSyntax + end + end + + context 'when the YAML file has whitespace preceding the content' do + let(:yml) do + <<-EOYML + variables: + SUPPORTED: "parsed" + + workflow: + rules: + - if: $VAR == "value" + + hello: + script: echo world + EOYML + end + + it 'loads everything correctly' do + expect(loader.load!).to contain_exactly( + { + variables: { SUPPORTED: 'parsed' }, + workflow: { rules: [{ if: '$VAR == "value"' }] }, + hello: { script: 'echo world' } + } + ) + end end context 'when the YAML file is empty' do @@ -32,67 +127,68 @@ RSpec.describe Gitlab::Config::Loader::MultiDocYaml, feature_category: :pipeline end end - context 'when the parsed YAML is too big' do + context 'when there are more than the maximum number of documents' do let(:yml) do <<~YAML - a: &a ["lol","lol","lol","lol","lol","lol","lol","lol","lol"] - b: &b [*a,*a,*a,*a,*a,*a,*a,*a,*a] - c: &c [*b,*b,*b,*b,*b,*b,*b,*b,*b] - d: &d [*c,*c,*c,*c,*c,*c,*c,*c,*c] - e: &e [*d,*d,*d,*d,*d,*d,*d,*d,*d] - f: &f [*e,*e,*e,*e,*e,*e,*e,*e,*e] - g: &g [*f,*f,*f,*f,*f,*f,*f,*f,*f] - h: &h [*g,*g,*g,*g,*g,*g,*g,*g,*g] - i: &i [*h,*h,*h,*h,*h,*h,*h,*h,*h] - --- - a: &a ["lol","lol","lol","lol","lol","lol","lol","lol","lol"] - b: &b [*a,*a,*a,*a,*a,*a,*a,*a,*a] - c: &c [*b,*b,*b,*b,*b,*b,*b,*b,*b] - d: &d [*c,*c,*c,*c,*c,*c,*c,*c,*c] - e: &e [*d,*d,*d,*d,*d,*d,*d,*d,*d] - f: &f [*e,*e,*e,*e,*e,*e,*e,*e,*e] - g: &g [*f,*f,*f,*f,*f,*f,*f,*f,*f] - h: &h [*g,*g,*g,*g,*g,*g,*g,*g,*g] - i: &i [*h,*h,*h,*h,*h,*h,*h,*h,*h] + --- a: 1 + --- b: 2 + --- c: 3 + --- d: 4 YAML end - it 'raises a DataTooLargeError' do - expect { loader.load! }.to raise_error(described_class::DataTooLargeError, 'The parsed YAML is too big') + it 'stops splitting documents after the maximum number' do + expect(loader.load!).to contain_exactly({ a: 1 }, { b: 2 }) end end + end + + describe '#load_raw!' do + let(:yml) do + <<~YAML + spec: + inputs: + test_input: + --- !test/content + test_job: + script: echo "$[[ inputs.test_input ]]" + YAML + end + + it 'returns the loaded YAML with all keys as strings' do + expect(loader.load_raw!).to contain_exactly( + { 'spec' => { 'inputs' => { 'test_input' => nil } } }, + { 'test_job' => { 'script' => 'echo "$[[ inputs.test_input ]]"' } } + ) + end + end - context 'when a document is not a hash' do + describe '#valid?' do + context 'when a document is invalid' do let(:yml) do <<~YAML - not_a_hash + a: b --- - test_job: - script: echo "$[[ inputs.test_input ]]" + c YAML end - it 'raises a NotHashError' do - expect { loader.load! }.to raise_error(described_class::NotHashError, 'Invalid configuration format') + it 'returns false' do + expect(loader).not_to be_valid end end - context 'when there are too many documents' do + context 'when the number of documents is below the maximum and all documents are valid' do let(:yml) do <<~YAML a: b --- c: d - --- - e: f YAML end - it 'raises a TooManyDocumentsError' do - expect { loader.load! }.to raise_error( - described_class::TooManyDocumentsError, - 'The parsed YAML has too many documents' - ) + it 'returns true' do + expect(loader).to be_valid end end end diff --git a/spec/lib/gitlab/config/loader/yaml_spec.rb b/spec/lib/gitlab/config/loader/yaml_spec.rb index 346424d1681..bba66f33718 100644 --- a/spec/lib/gitlab/config/loader/yaml_spec.rb +++ b/spec/lib/gitlab/config/loader/yaml_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Config::Loader::Yaml, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Config::Loader::Yaml, feature_category: :pipeline_composition do let(:loader) { described_class.new(yml) } let(:yml) do @@ -182,4 +182,30 @@ RSpec.describe Gitlab::Config::Loader::Yaml, feature_category: :pipeline_authori ) end end + + describe '#blank?' do + context 'when the loaded YAML is empty' do + let(:yml) do + <<~YAML + # only comments here + YAML + end + + it 'returns true' do + expect(loader).to be_blank + end + end + + context 'when the loaded YAML has content' do + let(:yml) do + <<~YAML + test: value + YAML + end + + it 'returns false' do + expect(loader).not_to be_blank + end + end + end end diff --git a/spec/lib/gitlab/console_spec.rb b/spec/lib/gitlab/console_spec.rb index f043433b4c5..5723a4421f6 100644 --- a/spec/lib/gitlab/console_spec.rb +++ b/spec/lib/gitlab/console_spec.rb @@ -1,8 +1,8 @@ # frozen_string_literal: true -require 'fast_spec_helper' +require 'spec_helper' -RSpec.describe Gitlab::Console do +RSpec.describe Gitlab::Console, feature_category: :application_instrumentation do describe '.welcome!' do context 'when running in the Rails console' do before do diff --git a/spec/lib/gitlab/content_security_policy/config_loader_spec.rb b/spec/lib/gitlab/content_security_policy/config_loader_spec.rb index f298890623f..ffb651fe23c 100644 --- a/spec/lib/gitlab/content_security_policy/config_loader_spec.rb +++ b/spec/lib/gitlab/content_security_policy/config_loader_spec.rb @@ -102,11 +102,7 @@ RSpec.describe Gitlab::ContentSecurityPolicy::ConfigLoader do end describe 'Zuora directives' do - context 'when is Gitlab.com?' do - before do - allow(::Gitlab).to receive(:com?).and_return(true) - end - + context 'when on SaaS', :saas do it 'adds Zuora host to CSP' do expect(directives['frame_src']).to include('https://*.zuora.com/apps/PublicHostedPageLite.do') end @@ -182,6 +178,53 @@ RSpec.describe Gitlab::ContentSecurityPolicy::ConfigLoader do end end + context 'when KAS is configured' do + before do + stub_config_setting(host: 'gitlab.example.com') + allow(::Gitlab::Kas).to receive(:enabled?).and_return true + end + + context 'when user access feature flag is disabled' do + before do + stub_feature_flags(kas_user_access: false) + end + + it 'does not add KAS url to CSP' do + expect(directives['connect_src']).not_to eq("'self' ws://gitlab.example.com #{::Gitlab::Kas.tunnel_url}") + end + end + + context 'when user access feature flag is enabled' do + before do + stub_feature_flags(kas_user_access: true) + end + + context 'when KAS is on same domain as rails' do + let_it_be(:kas_tunnel_url) { "ws://gitlab.example.com/-/k8s-proxy/" } + + before do + allow(::Gitlab::Kas).to receive(:tunnel_url).and_return(kas_tunnel_url) + end + + it 'does not add KAS url to CSP' do + expect(directives['connect_src']).not_to eq("'self' ws://gitlab.example.com #{::Gitlab::Kas.tunnel_url}") + end + end + + context 'when KAS is on subdomain' do + let_it_be(:kas_tunnel_url) { "ws://kas.gitlab.example.com/k8s-proxy/" } + + before do + allow(::Gitlab::Kas).to receive(:tunnel_url).and_return(kas_tunnel_url) + end + + it 'does add KAS url to CSP' do + expect(directives['connect_src']).to eq("'self' ws://gitlab.example.com #{kas_tunnel_url}") + end + end + end + end + context 'when CUSTOMER_PORTAL_URL is set' do let(:customer_portal_url) { 'https://customers.example.com' } diff --git a/spec/lib/gitlab/database/async_constraints/migration_helpers_spec.rb b/spec/lib/gitlab/database/async_constraints/migration_helpers_spec.rb new file mode 100644 index 00000000000..4dd510499ab --- /dev/null +++ b/spec/lib/gitlab/database/async_constraints/migration_helpers_spec.rb @@ -0,0 +1,288 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::AsyncConstraints::MigrationHelpers, feature_category: :database do + let(:migration) { Gitlab::Database::Migration[2.1].new } + let(:connection) { ApplicationRecord.connection } + let(:constraint_model) { Gitlab::Database::AsyncConstraints::PostgresAsyncConstraintValidation } + let(:table_name) { '_test_async_fks' } + let(:column_name) { 'parent_id' } + let(:fk_name) { nil } + + context 'with async FK validation on regular tables' do + before do + allow(migration).to receive(:puts) + allow(migration.connection).to receive(:transaction_open?).and_return(false) + + connection.create_table(table_name) do |t| + t.integer column_name + end + + migration.add_concurrent_foreign_key( + table_name, table_name, + column: column_name, validate: false, name: fk_name) + end + + describe '#prepare_async_foreign_key_validation' do + it 'creates the record for the async FK validation' do + expect do + migration.prepare_async_foreign_key_validation(table_name, column_name) + end.to change { constraint_model.where(table_name: table_name).count }.by(1) + + record = constraint_model.find_by(table_name: table_name) + + expect(record.name).to start_with('fk_') + expect(record).to be_foreign_key + end + + context 'when an explicit name is given' do + let(:fk_name) { 'my_fk_name' } + + it 'creates the record with the given name' do + expect do + migration.prepare_async_foreign_key_validation(table_name, name: fk_name) + end.to change { constraint_model.where(name: fk_name).count }.by(1) + + record = constraint_model.find_by(name: fk_name) + + expect(record.table_name).to eq(table_name) + expect(record).to be_foreign_key + end + end + + context 'when the FK does not exist' do + it 'returns an error' do + expect do + migration.prepare_async_foreign_key_validation(table_name, name: 'no_fk') + end.to raise_error RuntimeError, /Could not find foreign key "no_fk" on table "_test_async_fks"/ + end + end + + context 'when the record already exists' do + let(:fk_name) { 'my_fk_name' } + + it 'does attempt to create the record' do + create(:postgres_async_constraint_validation, table_name: table_name, name: fk_name) + + expect do + migration.prepare_async_foreign_key_validation(table_name, name: fk_name) + end.not_to change { constraint_model.where(name: fk_name).count } + end + end + + context 'when the async FK validation table does not exist' do + it 'does not raise an error' do + connection.drop_table(constraint_model.table_name) + + expect(constraint_model).not_to receive(:safe_find_or_create_by!) + + expect { migration.prepare_async_foreign_key_validation(table_name, column_name) }.not_to raise_error + end + end + end + + describe '#unprepare_async_foreign_key_validation' do + context 'with foreign keys' do + before do + migration.prepare_async_foreign_key_validation(table_name, column_name, name: fk_name) + end + + it 'destroys the record' do + expect do + migration.unprepare_async_foreign_key_validation(table_name, column_name) + end.to change { constraint_model.where(table_name: table_name).count }.by(-1) + end + + context 'when an explicit name is given' do + let(:fk_name) { 'my_test_async_fk' } + + it 'destroys the record' do + expect do + migration.unprepare_async_foreign_key_validation(table_name, name: fk_name) + end.to change { constraint_model.where(name: fk_name).count }.by(-1) + end + end + + context 'when the async fk validation table does not exist' do + it 'does not raise an error' do + connection.drop_table(constraint_model.table_name) + + expect(constraint_model).not_to receive(:find_by) + + expect { migration.unprepare_async_foreign_key_validation(table_name, column_name) }.not_to raise_error + end + end + end + + context 'with other types of constraints' do + let(:name) { 'my_test_async_constraint' } + let(:constraint) { create(:postgres_async_constraint_validation, table_name: table_name, name: name) } + + it 'does not destroy the record' do + constraint.update_column(:constraint_type, 99) + + expect do + migration.unprepare_async_foreign_key_validation(table_name, name: name) + end.not_to change { constraint_model.where(name: name).count } + + expect(constraint).to be_present + end + end + end + end + + context 'with partitioned tables' do + let(:partition_schema) { 'gitlab_partitions_dynamic' } + let(:partition1_name) { "#{partition_schema}.#{table_name}_202001" } + let(:partition2_name) { "#{partition_schema}.#{table_name}_202002" } + let(:fk_name) { 'my_partitioned_fk_name' } + + before do + connection.execute(<<~SQL) + CREATE TABLE #{table_name} ( + id serial NOT NULL, + #{column_name} int NOT NULL, + created_at timestamptz NOT NULL, + PRIMARY KEY (id, created_at) + ) PARTITION BY RANGE (created_at); + + CREATE TABLE #{partition1_name} PARTITION OF #{table_name} + FOR VALUES FROM ('2020-01-01') TO ('2020-02-01'); + + CREATE TABLE #{partition2_name} PARTITION OF #{table_name} + FOR VALUES FROM ('2020-02-01') TO ('2020-03-01'); + SQL + end + + describe '#prepare_partitioned_async_foreign_key_validation' do + it 'delegates to prepare_async_foreign_key_validation for each partition' do + expect(migration) + .to receive(:prepare_async_foreign_key_validation) + .with(partition1_name, column_name, name: fk_name) + + expect(migration) + .to receive(:prepare_async_foreign_key_validation) + .with(partition2_name, column_name, name: fk_name) + + migration.prepare_partitioned_async_foreign_key_validation(table_name, column_name, name: fk_name) + end + end + + describe '#unprepare_partitioned_async_foreign_key_validation' do + it 'delegates to unprepare_async_foreign_key_validation for each partition' do + expect(migration) + .to receive(:unprepare_async_foreign_key_validation) + .with(partition1_name, column_name, name: fk_name) + + expect(migration) + .to receive(:unprepare_async_foreign_key_validation) + .with(partition2_name, column_name, name: fk_name) + + migration.unprepare_partitioned_async_foreign_key_validation(table_name, column_name, name: fk_name) + end + end + end + + context 'with async check constraint validations' do + let(:table_name) { '_test_async_check_constraints' } + let(:check_name) { 'partitioning_constraint' } + + before do + allow(migration).to receive(:puts) + allow(migration.connection).to receive(:transaction_open?).and_return(false) + + connection.create_table(table_name) do |t| + t.integer column_name + end + + migration.add_check_constraint( + table_name, "#{column_name} = 1", + check_name, validate: false) + end + + describe '#prepare_async_check_constraint_validation' do + it 'creates the record for async validation' do + expect do + migration.prepare_async_check_constraint_validation(table_name, name: check_name) + end.to change { constraint_model.where(name: check_name).count }.by(1) + + record = constraint_model.find_by(name: check_name) + + expect(record.table_name).to eq(table_name) + expect(record).to be_check_constraint + end + + context 'when the check constraint does not exist' do + it 'returns an error' do + expect do + migration.prepare_async_check_constraint_validation(table_name, name: 'missing') + end.to raise_error RuntimeError, /Could not find check constraint "missing" on table "#{table_name}"/ + end + end + + context 'when the record already exists' do + it 'does attempt to create the record' do + create(:postgres_async_constraint_validation, + table_name: table_name, + name: check_name, + constraint_type: :check_constraint) + + expect do + migration.prepare_async_check_constraint_validation(table_name, name: check_name) + end.not_to change { constraint_model.where(name: check_name).count } + end + end + + context 'when the async validation table does not exist' do + it 'does not raise an error' do + connection.drop_table(constraint_model.table_name) + + expect(constraint_model).not_to receive(:safe_find_or_create_by!) + + expect { migration.prepare_async_check_constraint_validation(table_name, name: check_name) } + .not_to raise_error + end + end + end + + describe '#unprepare_async_check_constraint_validation' do + context 'with check constraints' do + before do + migration.prepare_async_check_constraint_validation(table_name, name: check_name) + end + + it 'destroys the record' do + expect do + migration.unprepare_async_check_constraint_validation(table_name, name: check_name) + end.to change { constraint_model.where(name: check_name).count }.by(-1) + end + + context 'when the async validation table does not exist' do + it 'does not raise an error' do + connection.drop_table(constraint_model.table_name) + + expect(constraint_model).not_to receive(:find_by) + + expect { migration.unprepare_async_check_constraint_validation(table_name, name: check_name) } + .not_to raise_error + end + end + end + + context 'with other types of constraints' do + let(:constraint) { create(:postgres_async_constraint_validation, table_name: table_name, name: check_name) } + + it 'does not destroy the record' do + constraint.update_column(:constraint_type, 99) + + expect do + migration.unprepare_async_check_constraint_validation(table_name, name: check_name) + end.not_to change { constraint_model.where(name: check_name).count } + + expect(constraint).to be_present + end + end + end + end +end diff --git a/spec/lib/gitlab/database/async_constraints/postgres_async_constraint_validation_spec.rb b/spec/lib/gitlab/database/async_constraints/postgres_async_constraint_validation_spec.rb new file mode 100644 index 00000000000..52fbf6d2f9b --- /dev/null +++ b/spec/lib/gitlab/database/async_constraints/postgres_async_constraint_validation_spec.rb @@ -0,0 +1,109 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::AsyncConstraints::PostgresAsyncConstraintValidation, type: :model, + feature_category: :database do + it { is_expected.to be_a Gitlab::Database::SharedModel } + + describe 'validations' do + let_it_be(:constraint_validation) { create(:postgres_async_constraint_validation) } + let(:identifier_limit) { described_class::MAX_IDENTIFIER_LENGTH } + let(:last_error_limit) { described_class::MAX_LAST_ERROR_LENGTH } + + subject { constraint_validation } + + it { is_expected.to validate_presence_of(:name) } + it { is_expected.to validate_uniqueness_of(:name).scoped_to(:table_name) } + it { is_expected.to validate_length_of(:name).is_at_most(identifier_limit) } + it { is_expected.to validate_presence_of(:table_name) } + it { is_expected.to validate_length_of(:table_name).is_at_most(identifier_limit) } + it { is_expected.to validate_length_of(:last_error).is_at_most(last_error_limit) } + end + + describe 'scopes' do + let!(:failed_validation) { create(:postgres_async_constraint_validation, attempts: 1) } + let!(:new_validation) { create(:postgres_async_constraint_validation) } + + describe '.ordered' do + subject { described_class.ordered } + + it { is_expected.to eq([new_validation, failed_validation]) } + end + + describe '.foreign_key_type' do + before do + new_validation.update_column(:constraint_type, 99) + end + + subject { described_class.foreign_key_type } + + it { is_expected.to eq([failed_validation]) } + + it 'does not apply the filter if the column is not present' do + expect(described_class) + .to receive(:constraint_type_exists?) + .and_return(false) + + is_expected.to match_array([failed_validation, new_validation]) + end + end + + describe '.check_constraint_type' do + before do + new_validation.update!(constraint_type: :check_constraint) + end + + subject { described_class.check_constraint_type } + + it { is_expected.to eq([new_validation]) } + end + end + + describe '.table_available?' do + subject { described_class.table_available? } + + it { is_expected.to be_truthy } + + context 'when the table does not exist' do + before do + described_class + .connection + .drop_table(described_class.table_name) + end + + it { is_expected.to be_falsy } + end + end + + describe '.constraint_type_exists?' do + it { expect(described_class.constraint_type_exists?).to be_truthy } + + it 'always asks the database' do + control = ActiveRecord::QueryRecorder.new(skip_schema_queries: false) do + described_class.constraint_type_exists? + end + + expect(control.count).to be >= 1 + expect { described_class.constraint_type_exists? }.to issue_same_number_of_queries_as(control) + end + end + + describe '#handle_exception!' do + let_it_be_with_reload(:constraint_validation) { create(:postgres_async_constraint_validation) } + + let(:error) { instance_double(StandardError, message: 'Oups', backtrace: %w[this that]) } + + subject { constraint_validation.handle_exception!(error) } + + it 'increases the attempts number' do + expect { subject }.to change { constraint_validation.reload.attempts }.by(1) + end + + it 'saves error details' do + subject + + expect(constraint_validation.reload.last_error).to eq("Oups\nthis\nthat") + end + end +end diff --git a/spec/lib/gitlab/database/async_constraints/validators/check_constraint_spec.rb b/spec/lib/gitlab/database/async_constraints/validators/check_constraint_spec.rb new file mode 100644 index 00000000000..7622b39feb1 --- /dev/null +++ b/spec/lib/gitlab/database/async_constraints/validators/check_constraint_spec.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::AsyncConstraints::Validators::CheckConstraint, feature_category: :database do + it_behaves_like 'async constraints validation' do + let(:constraint_type) { :check_constraint } + + before do + connection.create_table(table_name) do |t| + t.integer :parent_id + end + + connection.execute(<<~SQL.squish) + ALTER TABLE #{table_name} ADD CONSTRAINT #{constraint_name} + CHECK ( parent_id = 101 ) NOT VALID; + SQL + end + end +end diff --git a/spec/lib/gitlab/database/async_constraints/validators/foreign_key_spec.rb b/spec/lib/gitlab/database/async_constraints/validators/foreign_key_spec.rb new file mode 100644 index 00000000000..0e345e0e9ae --- /dev/null +++ b/spec/lib/gitlab/database/async_constraints/validators/foreign_key_spec.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::AsyncConstraints::Validators::ForeignKey, feature_category: :database do + it_behaves_like 'async constraints validation' do + let(:constraint_type) { :foreign_key } + + before do + connection.create_table(table_name) do |t| + t.references :parent, foreign_key: { to_table: table_name, validate: false, name: constraint_name } + end + end + + context 'with fully qualified table names' do + let(:validation) do + create(:postgres_async_constraint_validation, + table_name: "public.#{table_name}", + name: constraint_name, + constraint_type: constraint_type + ) + end + + it 'validates the constraint' do + allow(connection).to receive(:execute).and_call_original + + expect(connection).to receive(:execute) + .with(/ALTER TABLE "public"."#{table_name}" VALIDATE CONSTRAINT "#{constraint_name}";/) + .ordered.and_call_original + + subject.perform + end + end + end +end diff --git a/spec/lib/gitlab/database/async_constraints/validators_spec.rb b/spec/lib/gitlab/database/async_constraints/validators_spec.rb new file mode 100644 index 00000000000..e903b79dd1b --- /dev/null +++ b/spec/lib/gitlab/database/async_constraints/validators_spec.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::AsyncConstraints::Validators, feature_category: :database do + describe '.for' do + subject { described_class.for(record) } + + context 'with foreign keys validations' do + let(:record) { build(:postgres_async_constraint_validation, :foreign_key) } + + it { is_expected.to be_a(described_class::ForeignKey) } + end + + context 'with check constraint validations' do + let(:record) { build(:postgres_async_constraint_validation, :check_constraint) } + + it { is_expected.to be_a(described_class::CheckConstraint) } + end + end +end diff --git a/spec/lib/gitlab/database/async_constraints_spec.rb b/spec/lib/gitlab/database/async_constraints_spec.rb new file mode 100644 index 00000000000..e5cf782485f --- /dev/null +++ b/spec/lib/gitlab/database/async_constraints_spec.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::AsyncConstraints, feature_category: :database do + describe '.validate_pending_entries!' do + subject { described_class.validate_pending_entries! } + + let!(:fk_validation) do + create(:postgres_async_constraint_validation, :foreign_key, attempts: 2) + end + + let(:check_validation) do + create(:postgres_async_constraint_validation, :check_constraint, attempts: 1) + end + + it 'executes pending validations' do + expect_next_instance_of(described_class::Validators::ForeignKey, fk_validation) do |validator| + expect(validator).to receive(:perform) + end + + expect_next_instance_of(described_class::Validators::CheckConstraint, check_validation) do |validator| + expect(validator).to receive(:perform) + end + + subject + end + end +end diff --git a/spec/lib/gitlab/database/async_foreign_keys/foreign_key_validator_spec.rb b/spec/lib/gitlab/database/async_foreign_keys/foreign_key_validator_spec.rb deleted file mode 100644 index 90137e259f5..00000000000 --- a/spec/lib/gitlab/database/async_foreign_keys/foreign_key_validator_spec.rb +++ /dev/null @@ -1,152 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Database::AsyncForeignKeys::ForeignKeyValidator, feature_category: :database do - include ExclusiveLeaseHelpers - - describe '#perform' do - let!(:lease) { stub_exclusive_lease(lease_key, :uuid, timeout: lease_timeout) } - let(:lease_key) { "gitlab/database/asyncddl/actions/#{Gitlab::Database::PRIMARY_DATABASE_NAME}" } - let(:lease_timeout) { described_class::TIMEOUT_PER_ACTION } - - let(:fk_model) { Gitlab::Database::AsyncForeignKeys::PostgresAsyncForeignKeyValidation } - let(:table_name) { '_test_async_fks' } - let(:fk_name) { 'fk_parent_id' } - let(:validation) { create(:postgres_async_foreign_key_validation, table_name: table_name, name: fk_name) } - let(:connection) { validation.connection } - - subject { described_class.new(validation) } - - before do - connection.create_table(table_name) do |t| - t.references :parent, foreign_key: { to_table: table_name, validate: false, name: fk_name } - end - end - - it 'validates the FK while controlling statement timeout' do - allow(connection).to receive(:execute).and_call_original - expect(connection).to receive(:execute) - .with("SET statement_timeout TO '43200s'").ordered.and_call_original - expect(connection).to receive(:execute) - .with('ALTER TABLE "_test_async_fks" VALIDATE CONSTRAINT "fk_parent_id";').ordered.and_call_original - expect(connection).to receive(:execute) - .with("RESET statement_timeout").ordered.and_call_original - - subject.perform - end - - context 'with fully qualified table names' do - let(:validation) do - create(:postgres_async_foreign_key_validation, - table_name: "public.#{table_name}", - name: fk_name - ) - end - - it 'validates the FK' do - allow(connection).to receive(:execute).and_call_original - - expect(connection).to receive(:execute) - .with('ALTER TABLE "public"."_test_async_fks" VALIDATE CONSTRAINT "fk_parent_id";').ordered.and_call_original - - subject.perform - end - end - - it 'removes the FK validation record from table' do - expect(validation).to receive(:destroy!).and_call_original - - expect { subject.perform }.to change { fk_model.count }.by(-1) - end - - it 'skips logic if not able to acquire exclusive lease' do - expect(lease).to receive(:try_obtain).ordered.and_return(false) - expect(connection).not_to receive(:execute).with(/ALTER TABLE/) - expect(validation).not_to receive(:destroy!) - - expect { subject.perform }.not_to change { fk_model.count } - end - - it 'logs messages around execution' do - allow(Gitlab::AppLogger).to receive(:info).and_call_original - - subject.perform - - expect(Gitlab::AppLogger) - .to have_received(:info) - .with(a_hash_including(message: 'Starting to validate foreign key')) - - expect(Gitlab::AppLogger) - .to have_received(:info) - .with(a_hash_including(message: 'Finished validating foreign key')) - end - - context 'when the FK does not exist' do - before do - connection.create_table(table_name, force: true) - end - - it 'skips validation and removes the record' do - expect(connection).not_to receive(:execute).with(/ALTER TABLE/) - - expect { subject.perform }.to change { fk_model.count }.by(-1) - end - - it 'logs an appropriate message' do - expected_message = "Skipping #{fk_name} validation since it does not exist. The queuing entry will be deleted" - - allow(Gitlab::AppLogger).to receive(:info).and_call_original - - subject.perform - - expect(Gitlab::AppLogger) - .to have_received(:info) - .with(a_hash_including(message: expected_message)) - end - end - - context 'with error handling' do - before do - allow(connection).to receive(:execute).and_call_original - - allow(connection).to receive(:execute) - .with('ALTER TABLE "_test_async_fks" VALIDATE CONSTRAINT "fk_parent_id";') - .and_raise(ActiveRecord::StatementInvalid) - end - - context 'on production' do - before do - allow(Gitlab::ErrorTracking).to receive(:should_raise_for_dev?).and_return(false) - end - - it 'increases execution attempts' do - expect { subject.perform }.to change { validation.attempts }.by(1) - - expect(validation.last_error).to be_present - expect(validation).not_to be_destroyed - end - - it 'logs an error message including the fk_name' do - expect(Gitlab::AppLogger) - .to receive(:error) - .with(a_hash_including(:message, :fk_name)) - .and_call_original - - subject.perform - end - end - - context 'on development' do - it 'also raises errors' do - expect { subject.perform } - .to raise_error(ActiveRecord::StatementInvalid) - .and change { validation.attempts }.by(1) - - expect(validation.last_error).to be_present - expect(validation).not_to be_destroyed - end - end - end - end -end diff --git a/spec/lib/gitlab/database/async_foreign_keys/migration_helpers_spec.rb b/spec/lib/gitlab/database/async_foreign_keys/migration_helpers_spec.rb deleted file mode 100644 index 0bd0e8045ff..00000000000 --- a/spec/lib/gitlab/database/async_foreign_keys/migration_helpers_spec.rb +++ /dev/null @@ -1,167 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Database::AsyncForeignKeys::MigrationHelpers, feature_category: :database do - let(:migration) { Gitlab::Database::Migration[2.1].new } - let(:connection) { ApplicationRecord.connection } - let(:fk_model) { Gitlab::Database::AsyncForeignKeys::PostgresAsyncForeignKeyValidation } - let(:table_name) { '_test_async_fks' } - let(:column_name) { 'parent_id' } - let(:fk_name) { nil } - - context 'with regular tables' do - before do - allow(migration).to receive(:puts) - allow(migration.connection).to receive(:transaction_open?).and_return(false) - - connection.create_table(table_name) do |t| - t.integer column_name - end - - migration.add_concurrent_foreign_key( - table_name, table_name, - column: column_name, validate: false, name: fk_name) - end - - describe '#prepare_async_foreign_key_validation' do - it 'creates the record for the async FK validation' do - expect do - migration.prepare_async_foreign_key_validation(table_name, column_name) - end.to change { fk_model.where(table_name: table_name).count }.by(1) - - record = fk_model.find_by(table_name: table_name) - - expect(record.name).to start_with('fk_') - end - - context 'when an explicit name is given' do - let(:fk_name) { 'my_fk_name' } - - it 'creates the record with the given name' do - expect do - migration.prepare_async_foreign_key_validation(table_name, name: fk_name) - end.to change { fk_model.where(name: fk_name).count }.by(1) - - record = fk_model.find_by(name: fk_name) - - expect(record.table_name).to eq(table_name) - end - end - - context 'when the FK does not exist' do - it 'returns an error' do - expect do - migration.prepare_async_foreign_key_validation(table_name, name: 'no_fk') - end.to raise_error RuntimeError, /Could not find foreign key "no_fk" on table "_test_async_fks"/ - end - end - - context 'when the record already exists' do - let(:fk_name) { 'my_fk_name' } - - it 'does attempt to create the record' do - create(:postgres_async_foreign_key_validation, table_name: table_name, name: fk_name) - - expect do - migration.prepare_async_foreign_key_validation(table_name, name: fk_name) - end.not_to change { fk_model.where(name: fk_name).count } - end - end - - context 'when the async FK validation table does not exist' do - it 'does not raise an error' do - connection.drop_table(:postgres_async_foreign_key_validations) - - expect(fk_model).not_to receive(:safe_find_or_create_by!) - - expect { migration.prepare_async_foreign_key_validation(table_name, column_name) }.not_to raise_error - end - end - end - - describe '#unprepare_async_foreign_key_validation' do - before do - migration.prepare_async_foreign_key_validation(table_name, column_name, name: fk_name) - end - - it 'destroys the record' do - expect do - migration.unprepare_async_foreign_key_validation(table_name, column_name) - end.to change { fk_model.where(table_name: table_name).count }.by(-1) - end - - context 'when an explicit name is given' do - let(:fk_name) { 'my_test_async_fk' } - - it 'destroys the record' do - expect do - migration.unprepare_async_foreign_key_validation(table_name, name: fk_name) - end.to change { fk_model.where(name: fk_name).count }.by(-1) - end - end - - context 'when the async fk validation table does not exist' do - it 'does not raise an error' do - connection.drop_table(:postgres_async_foreign_key_validations) - - expect(fk_model).not_to receive(:find_by) - - expect { migration.unprepare_async_foreign_key_validation(table_name, column_name) }.not_to raise_error - end - end - end - end - - context 'with partitioned tables' do - let(:partition_schema) { 'gitlab_partitions_dynamic' } - let(:partition1_name) { "#{partition_schema}.#{table_name}_202001" } - let(:partition2_name) { "#{partition_schema}.#{table_name}_202002" } - let(:fk_name) { 'my_partitioned_fk_name' } - - before do - connection.execute(<<~SQL) - CREATE TABLE #{table_name} ( - id serial NOT NULL, - #{column_name} int NOT NULL, - created_at timestamptz NOT NULL, - PRIMARY KEY (id, created_at) - ) PARTITION BY RANGE (created_at); - - CREATE TABLE #{partition1_name} PARTITION OF #{table_name} - FOR VALUES FROM ('2020-01-01') TO ('2020-02-01'); - - CREATE TABLE #{partition2_name} PARTITION OF #{table_name} - FOR VALUES FROM ('2020-02-01') TO ('2020-03-01'); - SQL - end - - describe '#prepare_partitioned_async_foreign_key_validation' do - it 'delegates to prepare_async_foreign_key_validation for each partition' do - expect(migration) - .to receive(:prepare_async_foreign_key_validation) - .with(partition1_name, column_name, name: fk_name) - - expect(migration) - .to receive(:prepare_async_foreign_key_validation) - .with(partition2_name, column_name, name: fk_name) - - migration.prepare_partitioned_async_foreign_key_validation(table_name, column_name, name: fk_name) - end - end - - describe '#unprepare_partitioned_async_foreign_key_validation' do - it 'delegates to unprepare_async_foreign_key_validation for each partition' do - expect(migration) - .to receive(:unprepare_async_foreign_key_validation) - .with(partition1_name, column_name, name: fk_name) - - expect(migration) - .to receive(:unprepare_async_foreign_key_validation) - .with(partition2_name, column_name, name: fk_name) - - migration.unprepare_partitioned_async_foreign_key_validation(table_name, column_name, name: fk_name) - end - end - end -end diff --git a/spec/lib/gitlab/database/async_foreign_keys/postgres_async_foreign_key_validation_spec.rb b/spec/lib/gitlab/database/async_foreign_keys/postgres_async_foreign_key_validation_spec.rb deleted file mode 100644 index ba201d93f52..00000000000 --- a/spec/lib/gitlab/database/async_foreign_keys/postgres_async_foreign_key_validation_spec.rb +++ /dev/null @@ -1,52 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Database::AsyncForeignKeys::PostgresAsyncForeignKeyValidation, type: :model, - feature_category: :database do - it { is_expected.to be_a Gitlab::Database::SharedModel } - - describe 'validations' do - let_it_be(:fk_validation) { create(:postgres_async_foreign_key_validation) } - let(:identifier_limit) { described_class::MAX_IDENTIFIER_LENGTH } - let(:last_error_limit) { described_class::MAX_LAST_ERROR_LENGTH } - - subject { fk_validation } - - it { is_expected.to validate_presence_of(:name) } - it { is_expected.to validate_uniqueness_of(:name) } - it { is_expected.to validate_length_of(:name).is_at_most(identifier_limit) } - it { is_expected.to validate_presence_of(:table_name) } - it { is_expected.to validate_length_of(:table_name).is_at_most(identifier_limit) } - it { is_expected.to validate_length_of(:last_error).is_at_most(last_error_limit) } - end - - describe 'scopes' do - let!(:failed_validation) { create(:postgres_async_foreign_key_validation, attempts: 1) } - let!(:new_validation) { create(:postgres_async_foreign_key_validation) } - - describe '.ordered' do - subject { described_class.ordered } - - it { is_expected.to eq([new_validation, failed_validation]) } - end - end - - describe '#handle_exception!' do - let_it_be_with_reload(:fk_validation) { create(:postgres_async_foreign_key_validation) } - - let(:error) { instance_double(StandardError, message: 'Oups', backtrace: %w[this that]) } - - subject { fk_validation.handle_exception!(error) } - - it 'increases the attempts number' do - expect { subject }.to change { fk_validation.reload.attempts }.by(1) - end - - it 'saves error details' do - subject - - expect(fk_validation.reload.last_error).to eq("Oups\nthis\nthat") - end - end -end diff --git a/spec/lib/gitlab/database/async_foreign_keys_spec.rb b/spec/lib/gitlab/database/async_foreign_keys_spec.rb deleted file mode 100644 index f15eb364929..00000000000 --- a/spec/lib/gitlab/database/async_foreign_keys_spec.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Database::AsyncForeignKeys, feature_category: :database do - describe '.validate_pending_entries!' do - subject { described_class.validate_pending_entries! } - - before do - create_list(:postgres_async_foreign_key_validation, 3) - end - - it 'takes 2 pending FK validations and executes them' do - validations = described_class::PostgresAsyncForeignKeyValidation.ordered.limit(2).to_a - - expect_next_instances_of(described_class::ForeignKeyValidator, 2, validations) do |validator| - expect(validator).to receive(:perform) - end - - subject - end - end -end diff --git a/spec/lib/gitlab/database/background_migration/batch_optimizer_spec.rb b/spec/lib/gitlab/database/background_migration/batch_optimizer_spec.rb index c367f4a4493..fb9b16d46d6 100644 --- a/spec/lib/gitlab/database/background_migration/batch_optimizer_spec.rb +++ b/spec/lib/gitlab/database/background_migration/batch_optimizer_spec.rb @@ -113,5 +113,14 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchOptimizer do expect { subject }.to change { migration.reload.batch_size }.to(1_000) end end + + context 'when migration max_batch_size is less than MIN_BATCH_SIZE' do + let(:migration_params) { { max_batch_size: 900 } } + + it 'does not raise an error' do + mock_efficiency(0.7) + expect { subject }.not_to raise_error + end + end end end diff --git a/spec/lib/gitlab/database/background_migration/batched_job_spec.rb b/spec/lib/gitlab/database/background_migration/batched_job_spec.rb index cc9f3d5b7f1..073a30e7839 100644 --- a/spec/lib/gitlab/database/background_migration/batched_job_spec.rb +++ b/spec/lib/gitlab/database/background_migration/batched_job_spec.rb @@ -184,6 +184,35 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedJob, type: :model d expect(transition_log.exception_message).to eq('RuntimeError') end end + + context 'when job fails during sub batch processing' do + let(:args) { { error: ActiveRecord::StatementTimeout.new, from_sub_batch: true } } + let(:attempts) { 0 } + let(:failure) { job.failure!(**args) } + let(:job) do + create(:batched_background_migration_job, :running, batch_size: 20, sub_batch_size: 10, attempts: attempts) + end + + context 'when sub batch size can be reduced in 25%' do + it { expect { failure }.to change { job.sub_batch_size }.to 7 } + end + + context 'when retries exceeds 2 attempts' do + let(:attempts) { 3 } + + before do + allow(job).to receive(:split_and_retry!) + end + + it 'calls split_and_retry! once sub_batch_size cannot be decreased anymore' do + failure + + expect(job).to have_received(:split_and_retry!).once + end + + it { expect { failure }.not_to change { job.sub_batch_size } } + end + end end describe 'scopes' do @@ -271,6 +300,24 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedJob, type: :model d end end + describe '.extract_transition_options' do + let(:perform) { subject.class.extract_transition_options(args) } + + where(:args, :expected_result) do + [ + [[], []], + [[{ error: StandardError }], [StandardError, nil]], + [[{ error: StandardError, from_sub_batch: true }], [StandardError, true]] + ] + end + + with_them do + it 'matches expected keys and result' do + expect(perform).to match_array(expected_result) + end + end + end + describe '#can_split?' do subject { job.can_split?(exception) } @@ -327,6 +374,48 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedJob, type: :model d end end + describe '#can_reduce_sub_batch_size?' do + let(:attempts) { 0 } + let(:batch_size) { 10 } + let(:sub_batch_size) { 6 } + let(:feature_flag) { :reduce_sub_batch_size_on_timeouts } + let(:job) do + create(:batched_background_migration_job, attempts: attempts, + batch_size: batch_size, sub_batch_size: sub_batch_size) + end + + where(:feature_flag_state, :within_boundaries, :outside_boundaries, :limit_reached) do + [ + [true, true, false, false], + [false, false, false, false] + ] + end + + with_them do + before do + stub_feature_flags(feature_flag => feature_flag_state) + end + + context 'when the number of attempts is lower than the limit and batch size are within boundaries' do + let(:attempts) { 1 } + + it { expect(job.can_reduce_sub_batch_size?).to be(within_boundaries) } + end + + context 'when the number of attempts is lower than the limit and batch size are outside boundaries' do + let(:batch_size) { 1 } + + it { expect(job.can_reduce_sub_batch_size?).to be(outside_boundaries) } + end + + context 'when the number of attempts is greater than the limit and batch size are within boundaries' do + let(:attempts) { 3 } + + it { expect(job.can_reduce_sub_batch_size?).to be(limit_reached) } + end + end + end + describe '#time_efficiency' do subject { job.time_efficiency } @@ -465,4 +554,80 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedJob, type: :model d end end end + + describe '#reduce_sub_batch_size!' do + let(:migration_batch_size) { 20 } + let(:migration_sub_batch_size) { 10 } + let(:job_batch_size) { 20 } + let(:job_sub_batch_size) { 10 } + let(:status) { :failed } + + let(:migration) do + create(:batched_background_migration, :active, batch_size: migration_batch_size, + sub_batch_size: migration_sub_batch_size) + end + + let(:job) do + create(:batched_background_migration_job, status, sub_batch_size: job_sub_batch_size, + batch_size: job_batch_size, batched_migration: migration) + end + + context 'when the job sub batch size can be reduced' do + let(:expected_sub_batch_size) { 7 } + + it 'reduces sub batch size in 25%' do + expect { job.reduce_sub_batch_size! }.to change { job.sub_batch_size }.to(expected_sub_batch_size) + end + + it 'log the changes' do + expect(Gitlab::AppLogger).to receive(:warn).with( + message: 'Sub batch size reduced due to timeout', + batched_job_id: job.id, + sub_batch_size: job_sub_batch_size, + reduced_sub_batch_size: expected_sub_batch_size, + attempts: job.attempts, + batched_migration_id: migration.id, + job_class_name: job.migration_job_class_name, + job_arguments: job.migration_job_arguments + ) + + job.reduce_sub_batch_size! + end + end + + context 'when reduced sub_batch_size is greater than sub_batch' do + let(:job_batch_size) { 5 } + + it "doesn't allow sub_batch_size to greater than sub_batch" do + expect { job.reduce_sub_batch_size! }.to change { job.sub_batch_size }.to 5 + end + end + + context 'when sub_batch_size is already 1' do + let(:job_sub_batch_size) { 1 } + + it "updates sub_batch_size to it's minimum value" do + expect { job.reduce_sub_batch_size! }.not_to change { job.sub_batch_size } + end + end + + context 'when job has not failed' do + let(:status) { :succeeded } + let(:error) { Gitlab::Database::BackgroundMigration::ReduceSubBatchSizeError } + + it 'raises an exception' do + expect { job.reduce_sub_batch_size! }.to raise_error(error) + end + end + + context 'when the amount to be reduced exceeds the threshold' do + let(:migration_batch_size) { 150 } + let(:migration_sub_batch_size) { 100 } + let(:job_sub_batch_size) { 30 } + + it 'prevents sub batch size to be reduced' do + expect { job.reduce_sub_batch_size! }.not_to change { job.sub_batch_size } + end + end + end end diff --git a/spec/lib/gitlab/database/background_migration/batched_migration_wrapper_spec.rb b/spec/lib/gitlab/database/background_migration/batched_migration_wrapper_spec.rb index f3a292abbae..8d74d16f4e5 100644 --- a/spec/lib/gitlab/database/background_migration/batched_migration_wrapper_spec.rb +++ b/spec/lib/gitlab/database/background_migration/batched_migration_wrapper_spec.rb @@ -8,6 +8,7 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper, ' let(:connection) { Gitlab::Database.database_base_models[:main].connection } let(:metrics_tracker) { instance_double('::Gitlab::Database::BackgroundMigration::PrometheusMetrics', track: nil) } let(:job_class) { Class.new(Gitlab::BackgroundMigration::BatchedMigrationJob) } + let(:sub_batch_exception) { Gitlab::Database::BackgroundMigration::SubBatchTimeoutError } let_it_be(:pause_ms) { 250 } let_it_be(:active_migration) { create(:batched_background_migration, :active, job_arguments: [:id, :other_id]) } @@ -39,7 +40,8 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper, ' sub_batch_size: 1, pause_ms: pause_ms, job_arguments: active_migration.job_arguments, - connection: connection) + connection: connection, + sub_batch_exception: sub_batch_exception) .and_return(job_instance) expect(job_instance).to receive(:perform).with(no_args) @@ -119,12 +121,14 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper, ' end context 'when the migration job raises an error' do - shared_examples 'an error is raised' do |error_class| + shared_examples 'an error is raised' do |error_class, cause| + let(:expected_to_raise) { cause || error_class } + it 'marks the tracking record as failed' do expect(job_instance).to receive(:perform).with(no_args).and_raise(error_class) freeze_time do - expect { perform }.to raise_error(error_class) + expect { perform }.to raise_error(expected_to_raise) reloaded_job_record = job_record.reload @@ -137,13 +141,16 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper, ' expect(job_instance).to receive(:perform).with(no_args).and_raise(error_class) expect(metrics_tracker).to receive(:track).with(job_record) - expect { perform }.to raise_error(error_class) + expect { perform }.to raise_error(expected_to_raise) end end it_behaves_like 'an error is raised', RuntimeError.new('Something broke!') it_behaves_like 'an error is raised', SignalException.new('SIGTERM') it_behaves_like 'an error is raised', ActiveRecord::StatementTimeout.new('Timeout!') + + error = StandardError.new + it_behaves_like('an error is raised', Gitlab::Database::BackgroundMigration::SubBatchTimeoutError.new(error), error) end context 'when the batched background migration does not inherit from BatchedMigrationJob' do diff --git a/spec/lib/gitlab/database/gitlab_schema_spec.rb b/spec/lib/gitlab/database/gitlab_schema_spec.rb index 28a087d5401..b187b29c270 100644 --- a/spec/lib/gitlab/database/gitlab_schema_spec.rb +++ b/spec/lib/gitlab/database/gitlab_schema_spec.rb @@ -16,19 +16,21 @@ RSpec.shared_examples 'validate schema data' do |tables_and_views| end end -RSpec.describe Gitlab::Database::GitlabSchema do +RSpec.describe Gitlab::Database::GitlabSchema, feature_category: :database do shared_examples 'maps table name to table schema' do using RSpec::Parameterized::TableSyntax where(:name, :classification) do - 'ci_builds' | :gitlab_ci - 'my_schema.ci_builds' | :gitlab_ci - 'information_schema.columns' | :gitlab_internal - 'audit_events_part_5fc467ac26' | :gitlab_main - '_test_gitlab_main_table' | :gitlab_main - '_test_gitlab_ci_table' | :gitlab_ci - '_test_my_table' | :gitlab_shared - 'pg_attribute' | :gitlab_internal + 'ci_builds' | :gitlab_ci + 'my_schema.ci_builds' | :gitlab_ci + 'my_schema.ci_runner_machine_builds_100' | :gitlab_ci + 'my_schema._test_gitlab_main_table' | :gitlab_main + 'information_schema.columns' | :gitlab_internal + 'audit_events_part_5fc467ac26' | :gitlab_main + '_test_gitlab_main_table' | :gitlab_main + '_test_gitlab_ci_table' | :gitlab_ci + '_test_my_table' | :gitlab_shared + 'pg_attribute' | :gitlab_internal end with_them do diff --git a/spec/lib/gitlab/database/migration_helpers/convert_to_bigint_spec.rb b/spec/lib/gitlab/database/migration_helpers/convert_to_bigint_spec.rb new file mode 100644 index 00000000000..b1971977e7c --- /dev/null +++ b/spec/lib/gitlab/database/migration_helpers/convert_to_bigint_spec.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::MigrationHelpers::ConvertToBigint, feature_category: :database do + describe 'com_or_dev_or_test_but_not_jh?' do + using RSpec::Parameterized::TableSyntax + + where(:dot_com, :dev_or_test, :jh, :expectation) do + true | true | true | false + true | false | true | false + false | true | true | false + false | false | true | false + true | true | false | true + true | false | false | true + false | true | false | true + false | false | false | false + end + + with_them do + it 'returns true for GitLab.com (but not JH), dev, or test' do + allow(Gitlab).to receive(:com?).and_return(dot_com) + allow(Gitlab).to receive(:dev_or_test_env?).and_return(dev_or_test) + allow(Gitlab).to receive(:jh?).and_return(jh) + + migration = Class + .new + .include(Gitlab::Database::MigrationHelpers::ConvertToBigint) + .new + + expect(migration.com_or_dev_or_test_but_not_jh?).to eq(expectation) + end + end + end +end diff --git a/spec/lib/gitlab/database/migration_helpers_spec.rb b/spec/lib/gitlab/database/migration_helpers_spec.rb index 9df23776be8..3f6528558b1 100644 --- a/spec/lib/gitlab/database/migration_helpers_spec.rb +++ b/spec/lib/gitlab/database/migration_helpers_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::MigrationHelpers do +RSpec.describe Gitlab::Database::MigrationHelpers, feature_category: :database do include Database::TableSchemaHelpers include Database::TriggerHelpers @@ -928,13 +928,13 @@ RSpec.describe Gitlab::Database::MigrationHelpers do it 'references the custom target columns when provided', :aggregate_failures do expect(model).to receive(:with_lock_retries).and_yield expect(model).to receive(:execute).with( - "ALTER TABLE projects\n" \ - "ADD CONSTRAINT fk_multiple_columns\n" \ - "FOREIGN KEY \(partition_number, user_id\)\n" \ - "REFERENCES users \(partition_number, id\)\n" \ - "ON UPDATE CASCADE\n" \ - "ON DELETE CASCADE\n" \ - "NOT VALID;\n" + "ALTER TABLE projects " \ + "ADD CONSTRAINT fk_multiple_columns " \ + "FOREIGN KEY \(partition_number, user_id\) " \ + "REFERENCES users \(partition_number, id\) " \ + "ON UPDATE CASCADE " \ + "ON DELETE CASCADE " \ + "NOT VALID;" ) model.add_concurrent_foreign_key( @@ -979,6 +979,80 @@ RSpec.describe Gitlab::Database::MigrationHelpers do end end end + + context 'when creating foreign key on a partitioned table' do + let(:source) { :_test_source_partitioned_table } + let(:dest) { :_test_dest_partitioned_table } + let(:args) { [source, dest] } + let(:options) { { column: [:partition_id, :owner_id], target_column: [:partition_id, :id] } } + + before do + model.execute(<<~SQL) + CREATE TABLE public.#{source} ( + id serial NOT NULL, + partition_id serial NOT NULL, + owner_id bigint NOT NULL, + PRIMARY KEY (id, partition_id) + ) PARTITION BY LIST(partition_id); + + CREATE TABLE #{source}_1 + PARTITION OF public.#{source} + FOR VALUES IN (1); + + CREATE TABLE public.#{dest} ( + id serial NOT NULL, + partition_id serial NOT NULL, + PRIMARY KEY (id, partition_id) + ); + SQL + end + + it 'creates the FK without using NOT VALID', :aggregate_failures do + allow(model).to receive(:execute).and_call_original + + expect(model).to receive(:with_lock_retries).and_yield + + expect(model).to receive(:execute).with( + "ALTER TABLE #{source} " \ + "ADD CONSTRAINT fk_multiple_columns " \ + "FOREIGN KEY \(partition_id, owner_id\) " \ + "REFERENCES #{dest} \(partition_id, id\) " \ + "ON UPDATE CASCADE ON DELETE CASCADE ;" + ) + + model.add_concurrent_foreign_key( + *args, + name: :fk_multiple_columns, + on_update: :cascade, + allow_partitioned: true, + **options + ) + end + + it 'raises an error if allow_partitioned is not set' do + expect(model).not_to receive(:with_lock_retries).and_yield + expect(model).not_to receive(:execute).with(/FOREIGN KEY/) + + expect { model.add_concurrent_foreign_key(*args, **options) } + .to raise_error ArgumentError, /use add_concurrent_partitioned_foreign_key/ + end + + context 'when the reverse_lock_order flag is set' do + it 'explicitly locks the tables in target-source order', :aggregate_failures do + expect(model).to receive(:with_lock_retries).and_call_original + expect(model).to receive(:disable_statement_timeout).and_call_original + expect(model).to receive(:statement_timeout_disabled?).and_return(false) + expect(model).to receive(:execute).with(/SET statement_timeout TO/) + expect(model).to receive(:execute).ordered.with(/VALIDATE CONSTRAINT/) + expect(model).to receive(:execute).ordered.with(/RESET statement_timeout/) + + expect(model).to receive(:execute).with("LOCK TABLE #{dest}, #{source} IN ACCESS EXCLUSIVE MODE") + expect(model).to receive(:execute).with(/REFERENCES #{dest} \(partition_id, id\)/) + + model.add_concurrent_foreign_key(*args, reverse_lock_order: true, allow_partitioned: true, **options) + end + end + end end end @@ -1049,6 +1123,8 @@ RSpec.describe Gitlab::Database::MigrationHelpers do describe '#foreign_key_exists?' do let(:referenced_table_name) { '_test_gitlab_main_referenced' } let(:referencing_table_name) { '_test_gitlab_main_referencing' } + let(:schema) { 'public' } + let(:identifier) { "#{schema}.#{referencing_table_name}" } before do model.connection.execute(<<~SQL) @@ -1085,6 +1161,10 @@ RSpec.describe Gitlab::Database::MigrationHelpers do expect(model.foreign_key_exists?(referencing_table_name, target_table)).to be_truthy end + it 'finds existing foreign_keys by identifier' do + expect(model.foreign_key_exists?(identifier, target_table)).to be_truthy + end + it 'compares by column name if given' do expect(model.foreign_key_exists?(referencing_table_name, target_table, column: :user_id)).to be_falsey end @@ -2890,4 +2970,18 @@ RSpec.describe Gitlab::Database::MigrationHelpers do it { is_expected.to be_falsey } end end + + describe "#table_partitioned?" do + subject { model.table_partitioned?(table_name) } + + let(:table_name) { 'p_ci_builds_metadata' } + + it { is_expected.to be_truthy } + + context 'with a non-partitioned table' do + let(:table_name) { 'users' } + + it { is_expected.to be_falsey } + end + end end diff --git a/spec/lib/gitlab/database/migrations/batched_background_migration_helpers_spec.rb b/spec/lib/gitlab/database/migrations/batched_background_migration_helpers_spec.rb index 3e249b14f2e..f5ce207773f 100644 --- a/spec/lib/gitlab/database/migrations/batched_background_migration_helpers_spec.rb +++ b/spec/lib/gitlab/database/migrations/batched_background_migration_helpers_spec.rb @@ -482,16 +482,46 @@ RSpec.describe Gitlab::Database::Migrations::BatchedBackgroundMigrationHelpers d .not_to raise_error end - it 'logs a warning when migration does not exist' do - expect(Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas).to receive(:require_dml_mode!) + context 'when specified migration does not exist' do + let(:lab_key) { 'DBLAB_ENVIRONMENT' } - create(:batched_background_migration, :active, migration_attributes.merge(gitlab_schema: :gitlab_something_else)) + context 'when DBLAB_ENVIRONMENT is not set' do + it 'logs a warning' do + stub_env(lab_key, nil) + expect(Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas).to receive(:require_dml_mode!) - expect(Gitlab::AppLogger).to receive(:warn) - .with("Could not find batched background migration for the given configuration: #{configuration}") + create(:batched_background_migration, :active, migration_attributes.merge(gitlab_schema: :gitlab_something_else)) - expect { ensure_batched_background_migration_is_finished } - .not_to raise_error + expect(Gitlab::AppLogger).to receive(:warn) + .with("Could not find batched background migration for the given configuration: #{configuration}") + + expect { ensure_batched_background_migration_is_finished } + .not_to raise_error + end + end + + context 'when DBLAB_ENVIRONMENT is set' do + it 'raises an error' do + stub_env(lab_key, 'foo') + expect(Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas).to receive(:require_dml_mode!) + + create(:batched_background_migration, :active, migration_attributes.merge(gitlab_schema: :gitlab_something_else)) + + expect { ensure_batched_background_migration_is_finished } + .to raise_error(Gitlab::Database::Migrations::BatchedBackgroundMigrationHelpers::NonExistentMigrationError) + end + end + end + + context 'when within transaction' do + before do + allow(migration).to receive(:transaction_open?).and_return(true) + end + + it 'does raise an exception' do + expect { ensure_batched_background_migration_is_finished } + .to raise_error /`ensure_batched_background_migration_is_finished` cannot be run inside a transaction./ + end end it 'finalizes the migration' do diff --git a/spec/lib/gitlab/database/migrations/constraints_helpers_spec.rb b/spec/lib/gitlab/database/migrations/constraints_helpers_spec.rb index 6848fc85aa1..07d913cf5cc 100644 --- a/spec/lib/gitlab/database/migrations/constraints_helpers_spec.rb +++ b/spec/lib/gitlab/database/migrations/constraints_helpers_spec.rb @@ -23,43 +23,46 @@ RSpec.describe Gitlab::Database::Migrations::ConstraintsHelpers do end end - describe '#check_constraint_exists?' do + describe '#check_constraint_exists?', :aggregate_failures do before do - ActiveRecord::Migration.connection.execute( - 'ALTER TABLE projects ADD CONSTRAINT check_1 CHECK (char_length(path) <= 5) NOT VALID' - ) - - ActiveRecord::Migration.connection.execute( - 'CREATE SCHEMA new_test_schema' - ) - - ActiveRecord::Migration.connection.execute( - 'CREATE TABLE new_test_schema.projects (id integer, name character varying)' - ) - - ActiveRecord::Migration.connection.execute( - 'ALTER TABLE new_test_schema.projects ADD CONSTRAINT check_2 CHECK (char_length(name) <= 5)' - ) + ActiveRecord::Migration.connection.execute(<<~SQL) + ALTER TABLE projects ADD CONSTRAINT check_1 CHECK (char_length(path) <= 5) NOT VALID; + CREATE SCHEMA new_test_schema; + CREATE TABLE new_test_schema.projects (id integer, name character varying); + ALTER TABLE new_test_schema.projects ADD CONSTRAINT check_2 CHECK (char_length(name) <= 5); + SQL end it 'returns true if a constraint exists' do expect(model) .to be_check_constraint_exists(:projects, 'check_1') + + expect(described_class) + .to be_check_constraint_exists(:projects, 'check_1', connection: model.connection) end it 'returns false if a constraint does not exist' do expect(model) .not_to be_check_constraint_exists(:projects, 'this_does_not_exist') + + expect(described_class) + .not_to be_check_constraint_exists(:projects, 'this_does_not_exist', connection: model.connection) end it 'returns false if a constraint with the same name exists in another table' do expect(model) .not_to be_check_constraint_exists(:users, 'check_1') + + expect(described_class) + .not_to be_check_constraint_exists(:users, 'check_1', connection: model.connection) end it 'returns false if a constraint with the same name exists for the same table in another schema' do expect(model) .not_to be_check_constraint_exists(:projects, 'check_2') + + expect(described_class) + .not_to be_check_constraint_exists(:projects, 'check_2', connection: model.connection) end end diff --git a/spec/lib/gitlab/database/migrations/test_batched_background_runner_spec.rb b/spec/lib/gitlab/database/migrations/test_batched_background_runner_spec.rb index 57c5011590c..6bcefa455cf 100644 --- a/spec/lib/gitlab/database/migrations/test_batched_background_runner_spec.rb +++ b/spec/lib/gitlab/database/migrations/test_batched_background_runner_spec.rb @@ -48,6 +48,7 @@ RSpec.describe Gitlab::Database::Migrations::TestBatchedBackgroundRunner, :freez let(:result_dir) { Pathname.new(Dir.mktmpdir) } let(:connection) { base_model.connection } let(:table_name) { "_test_column_copying" } + let(:num_rows_in_table) { 1000 } let(:from_id) { 0 } after do @@ -61,7 +62,7 @@ RSpec.describe Gitlab::Database::Migrations::TestBatchedBackgroundRunner, :freez data bigint default 0 ); - insert into #{table_name} (id) select i from generate_series(1, 1000) g(i); + insert into #{table_name} (id) select i from generate_series(1, #{num_rows_in_table}) g(i); SQL end @@ -134,6 +135,24 @@ RSpec.describe Gitlab::Database::Migrations::TestBatchedBackgroundRunner, :freez expect(calls).not_to be_empty end + it 'samples 1 job with a batch size higher than the table size' do + calls = [] + define_background_migration(migration_name) do |*args| + travel 1.minute + calls << args + end + + queue_migration(migration_name, table_name, :id, + job_interval: 5.minutes, + batch_size: num_rows_in_table * 2, + sub_batch_size: num_rows_in_table * 2) + + described_class.new(result_dir: result_dir, connection: connection, + from_id: from_id).run_jobs(for_duration: 3.minutes) + + expect(calls.size).to eq(1) + end + context 'with multiple jobs to run' do let(:last_id) do Gitlab::Database::SharedModel.using_connection(connection) do diff --git a/spec/lib/gitlab/database/partitioning/ci_sliding_list_strategy_spec.rb b/spec/lib/gitlab/database/partitioning/ci_sliding_list_strategy_spec.rb new file mode 100644 index 00000000000..f415e892818 --- /dev/null +++ b/spec/lib/gitlab/database/partitioning/ci_sliding_list_strategy_spec.rb @@ -0,0 +1,178 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::Partitioning::CiSlidingListStrategy, feature_category: :database do + let(:connection) { ActiveRecord::Base.connection } + let(:table_name) { :_test_gitlab_ci_partitioned_test } + let(:model) { class_double(ApplicationRecord, table_name: table_name, connection: connection) } + let(:next_partition_if) { nil } + let(:detach_partition_if) { nil } + + subject(:strategy) do + described_class.new(model, :partition, + next_partition_if: next_partition_if, + detach_partition_if: detach_partition_if) + end + + before do + next if table_name.to_s.starts_with?('p_') + + connection.execute(<<~SQL) + create table #{table_name} + ( + id serial not null, + partition_id bigint not null, + created_at timestamptz not null, + primary key (id, partition_id) + ) + partition by list(partition_id); + + create table #{table_name}_100 + partition of #{table_name} for values in (100); + + create table #{table_name}_101 + partition of #{table_name} for values in (101); + SQL + end + + describe '#current_partitions' do + it 'detects both partitions' do + expect(strategy.current_partitions).to eq( + [ + Gitlab::Database::Partitioning::SingleNumericListPartition.new( + table_name, 100, partition_name: "#{table_name}_100" + ), + Gitlab::Database::Partitioning::SingleNumericListPartition.new( + table_name, 101, partition_name: "#{table_name}_101" + ) + ]) + end + end + + describe '#validate_and_fix' do + it 'does not call change_column_default' do + expect(strategy.model.connection).not_to receive(:change_column_default) + + strategy.validate_and_fix + end + end + + describe '#active_partition' do + it 'is the partition with the largest value' do + expect(strategy.active_partition.value).to eq(101) + end + end + + describe '#missing_partitions' do + context 'when next_partition_if returns true' do + let(:next_partition_if) { proc { true } } + + it 'is a partition definition for the next partition in the series' do + extra = strategy.missing_partitions + + expect(extra.length).to eq(1) + expect(extra.first.value).to eq(102) + end + end + + context 'when next_partition_if returns false' do + let(:next_partition_if) { proc { false } } + + it 'is empty' do + expect(strategy.missing_partitions).to be_empty + end + end + + context 'when there are no partitions for the table' do + it 'returns a partition for value 1' do + connection.execute("drop table #{table_name}_100; drop table #{table_name}_101;") + + missing_partitions = strategy.missing_partitions + + expect(missing_partitions.size).to eq(1) + missing_partition = missing_partitions.first + + expect(missing_partition.value).to eq(100) + end + end + end + + describe '#extra_partitions' do + context 'when all partitions are true for detach_partition_if' do + let(:detach_partition_if) { ->(_p) { true } } + + it { expect(strategy.extra_partitions).to be_empty } + end + + context 'when all partitions are false for detach_partition_if' do + let(:detach_partition_if) { proc { false } } + + it { expect(strategy.extra_partitions).to be_empty } + end + end + + describe '#initial_partition' do + it 'starts with the value 100', :aggregate_failures do + initial_partition = strategy.initial_partition + expect(initial_partition.value).to eq(100) + expect(initial_partition.table).to eq(strategy.table_name) + expect(initial_partition.partition_name).to eq("#{strategy.table_name}_100") + end + + context 'with routing tables' do + let(:table_name) { :p_test_gitlab_ci_partitioned_test } + + it 'removes the prefix', :aggregate_failures do + initial_partition = strategy.initial_partition + + expect(initial_partition.value).to eq(100) + expect(initial_partition.table).to eq(strategy.table_name) + expect(initial_partition.partition_name).to eq('test_gitlab_ci_partitioned_test_100') + end + end + end + + describe '#next_partition' do + before do + allow(strategy) + .to receive(:active_partition) + .and_return(instance_double(Gitlab::Database::Partitioning::SingleNumericListPartition, value: 105)) + end + + it 'is one after the active partition', :aggregate_failures do + next_partition = strategy.next_partition + + expect(next_partition.value).to eq(106) + expect(next_partition.table).to eq(strategy.table_name) + expect(next_partition.partition_name).to eq("#{strategy.table_name}_106") + end + + context 'with routing tables' do + let(:table_name) { :p_test_gitlab_ci_partitioned_test } + + it 'removes the prefix', :aggregate_failures do + next_partition = strategy.next_partition + + expect(next_partition.value).to eq(106) + expect(next_partition.table).to eq(strategy.table_name) + expect(next_partition.partition_name).to eq('test_gitlab_ci_partitioned_test_106') + end + end + end + + describe '#ensure_partitioning_column_ignored_or_readonly!' do + it 'does not raise when the column is not ignored' do + expect do + Class.new(ApplicationRecord) do + include PartitionedTable + + partitioned_by :partition_id, + strategy: :ci_sliding_list, + next_partition_if: proc { false }, + detach_partition_if: proc { false } + end + end.not_to raise_error + end + end +end diff --git a/spec/lib/gitlab/database/partitioning/partition_manager_spec.rb b/spec/lib/gitlab/database/partitioning/partition_manager_spec.rb index 2212cb09888..ac54c307108 100644 --- a/spec/lib/gitlab/database/partitioning/partition_manager_spec.rb +++ b/spec/lib/gitlab/database/partitioning/partition_manager_spec.rb @@ -45,7 +45,7 @@ RSpec.describe Gitlab::Database::Partitioning::PartitionManager do sync_partitions end - context 'with eplicitly provided connection' do + context 'with explicitly provided connection' do let(:connection) { Ci::ApplicationRecord.connection } it 'uses the explicitly provided connection when any' do @@ -59,6 +59,14 @@ RSpec.describe Gitlab::Database::Partitioning::PartitionManager do end end + context 'when an ArgumentError occurs during partition management' do + it 'raises error' do + expect(partitioning_strategy).to receive(:missing_partitions).and_raise(ArgumentError) + + expect { sync_partitions }.to raise_error(ArgumentError) + end + end + context 'when an error occurs during partition management' do it 'does not raise an error' do expect(partitioning_strategy).to receive(:missing_partitions).and_raise('this should never happen (tm)') @@ -230,23 +238,20 @@ RSpec.describe Gitlab::Database::Partitioning::PartitionManager do expect(pending_drop.drop_after).to eq(Time.current + described_class::RETAIN_DETACHED_PARTITIONS_FOR) end - # Postgres 11 does not support foreign keys to partitioned tables - if ApplicationRecord.database.version.to_f >= 12 - context 'when the model is the target of a foreign key' do - before do - connection.execute(<<~SQL) + context 'when the model is the target of a foreign key' do + before do + connection.execute(<<~SQL) create unique index idx_for_fk ON #{partitioned_table_name}(created_at); create table _test_gitlab_main_referencing_table ( id bigserial primary key not null, referencing_created_at timestamptz references #{partitioned_table_name}(created_at) ); - SQL - end + SQL + end - it 'does not detach partitions with a referenced foreign key' do - expect { subject }.not_to change { find_partitions(my_model.table_name).size } - end + it 'does not detach partitions with a referenced foreign key' do + expect { subject }.not_to change { find_partitions(my_model.table_name).size } end end end diff --git a/spec/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers_spec.rb b/spec/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers_spec.rb index f0e34476cf2..d5f4afd7ba4 100644 --- a/spec/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers_spec.rb +++ b/spec/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::ForeignKeyHelpers do +RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::ForeignKeyHelpers, feature_category: :database do include Database::TableSchemaHelpers let(:migration) do @@ -16,15 +16,23 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::ForeignKeyHelpers let(:partition_schema) { 'gitlab_partitions_dynamic' } let(:partition1_name) { "#{partition_schema}.#{source_table_name}_202001" } let(:partition2_name) { "#{partition_schema}.#{source_table_name}_202002" } + let(:validate) { true } let(:options) do { column: column_name, name: foreign_key_name, on_delete: :cascade, - validate: true + on_update: nil, + primary_key: :id } end + let(:create_options) do + options + .except(:primary_key) + .merge!(reverse_lock_order: false, target_column: :id, validate: validate) + end + before do allow(migration).to receive(:puts) allow(migration).to receive(:transaction_open?).and_return(false) @@ -67,12 +75,11 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::ForeignKeyHelpers expect(migration).to receive(:concurrent_partitioned_foreign_key_name).and_return(foreign_key_name) - expect_add_concurrent_fk_and_call_original(partition1_name, target_table_name, **options) - expect_add_concurrent_fk_and_call_original(partition2_name, target_table_name, **options) + expect_add_concurrent_fk_and_call_original(partition1_name, target_table_name, **create_options) + expect_add_concurrent_fk_and_call_original(partition2_name, target_table_name, **create_options) - expect(migration).to receive(:with_lock_retries).ordered.and_yield - expect(migration).to receive(:add_foreign_key) - .with(source_table_name, target_table_name, **options) + expect(migration).to receive(:add_concurrent_foreign_key) + .with(source_table_name, target_table_name, allow_partitioned: true, **create_options) .ordered .and_call_original @@ -81,6 +88,39 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::ForeignKeyHelpers expect_foreign_key_to_exist(source_table_name, foreign_key_name) end + context 'with validate: false option' do + let(:validate) { false } + let(:options) do + { + column: column_name, + name: foreign_key_name, + on_delete: :cascade, + on_update: nil, + primary_key: :id + } + end + + it 'creates the foreign key only on partitions' do + expect(migration).to receive(:foreign_key_exists?) + .with(source_table_name, target_table_name, **options) + .and_return(false) + + expect(migration).to receive(:concurrent_partitioned_foreign_key_name).and_return(foreign_key_name) + + expect_add_concurrent_fk_and_call_original(partition1_name, target_table_name, **create_options) + expect_add_concurrent_fk_and_call_original(partition2_name, target_table_name, **create_options) + + expect(migration).not_to receive(:add_concurrent_foreign_key) + .with(source_table_name, target_table_name, **create_options) + + migration.add_concurrent_partitioned_foreign_key( + source_table_name, target_table_name, + column: column_name, validate: false) + + expect_foreign_key_not_to_exist(source_table_name, foreign_key_name) + end + end + def expect_add_concurrent_fk_and_call_original(source_table_name, target_table_name, options) expect(migration).to receive(:add_concurrent_foreign_key) .ordered @@ -100,8 +140,6 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::ForeignKeyHelpers .and_return(true) expect(migration).not_to receive(:add_concurrent_foreign_key) - expect(migration).not_to receive(:with_lock_retries) - expect(migration).not_to receive(:add_foreign_key) migration.add_concurrent_partitioned_foreign_key(source_table_name, target_table_name, column: column_name) @@ -110,30 +148,43 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::ForeignKeyHelpers end context 'when additional foreign key options are given' do - let(:options) do + let(:exits_options) do { column: column_name, name: '_my_fk_name', on_delete: :restrict, - validate: true + on_update: nil, + primary_key: :id } end + let(:create_options) do + exits_options + .except(:primary_key) + .merge!(reverse_lock_order: false, target_column: :id, validate: true) + end + it 'forwards them to the foreign key helper methods' do expect(migration).to receive(:foreign_key_exists?) - .with(source_table_name, target_table_name, **options) + .with(source_table_name, target_table_name, **exits_options) .and_return(false) expect(migration).not_to receive(:concurrent_partitioned_foreign_key_name) - expect_add_concurrent_fk(partition1_name, target_table_name, **options) - expect_add_concurrent_fk(partition2_name, target_table_name, **options) + expect_add_concurrent_fk(partition1_name, target_table_name, **create_options) + expect_add_concurrent_fk(partition2_name, target_table_name, **create_options) - expect(migration).to receive(:with_lock_retries).ordered.and_yield - expect(migration).to receive(:add_foreign_key).with(source_table_name, target_table_name, **options).ordered + expect(migration).to receive(:add_concurrent_foreign_key) + .with(source_table_name, target_table_name, allow_partitioned: true, **create_options) + .ordered - migration.add_concurrent_partitioned_foreign_key(source_table_name, target_table_name, - column: column_name, name: '_my_fk_name', on_delete: :restrict) + migration.add_concurrent_partitioned_foreign_key( + source_table_name, + target_table_name, + column: column_name, + name: '_my_fk_name', + on_delete: :restrict + ) end def expect_add_concurrent_fk(source_table_name, target_table_name, options) @@ -153,4 +204,39 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::ForeignKeyHelpers end end end + + describe '#validate_partitioned_foreign_key' do + context 'when run inside a transaction block' do + it 'raises an error' do + expect(migration).to receive(:transaction_open?).and_return(true) + + expect do + migration.validate_partitioned_foreign_key(source_table_name, column_name, name: '_my_fk_name') + end.to raise_error(/can not be run inside a transaction/) + end + end + + context 'when run outside a transaction block' do + before do + migration.add_concurrent_partitioned_foreign_key( + source_table_name, + target_table_name, + column: column_name, + name: foreign_key_name, + validate: false + ) + end + + it 'validates FK for each partition' do + expect(migration).to receive(:execute).with(/SET statement_timeout TO 0/).twice + expect(migration).to receive(:execute).with(/RESET statement_timeout/).twice + expect(migration).to receive(:execute) + .with(/ALTER TABLE #{partition1_name} VALIDATE CONSTRAINT #{foreign_key_name}/).ordered + expect(migration).to receive(:execute) + .with(/ALTER TABLE #{partition2_name} VALIDATE CONSTRAINT #{foreign_key_name}/).ordered + + migration.validate_partitioned_foreign_key(source_table_name, column_name, name: foreign_key_name) + end + end + end end diff --git a/spec/lib/gitlab/database/partitioning_spec.rb b/spec/lib/gitlab/database/partitioning_spec.rb index ae74ee60a4b..4c0fde46b2f 100644 --- a/spec/lib/gitlab/database/partitioning_spec.rb +++ b/spec/lib/gitlab/database/partitioning_spec.rb @@ -67,14 +67,19 @@ RSpec.describe Gitlab::Database::Partitioning do let(:ci_connection) { Ci::ApplicationRecord.connection } let(:table_names) { %w[partitioning_test1 partitioning_test2] } let(:models) do - table_names.map do |table_name| + [ Class.new(ApplicationRecord) do include PartitionedTable - self.table_name = table_name + self.table_name = 'partitioning_test1' partitioned_by :created_at, strategy: :monthly + end, + Class.new(Gitlab::Database::Partitioning::TableWithoutModel).tap do |klass| + klass.table_name = 'partitioning_test2' + klass.partitioned_by(:created_at, strategy: :monthly) + klass.limit_connection_names = %i[main] end - end + ] end before do diff --git a/spec/lib/gitlab/database/postgres_foreign_key_spec.rb b/spec/lib/gitlab/database/postgres_foreign_key_spec.rb index ae56f66737d..c128c56c708 100644 --- a/spec/lib/gitlab/database/postgres_foreign_key_spec.rb +++ b/spec/lib/gitlab/database/postgres_foreign_key_spec.rb @@ -70,13 +70,29 @@ RSpec.describe Gitlab::Database::PostgresForeignKey, type: :model, feature_categ end describe '#by_constrained_table_name' do - it 'finds the foreign keys for the constrained table' do - expected = described_class.where(name: %w[fk_constrained_to_referenced fk_constrained_to_other_referenced]).to_a + let(:expected) { described_class.where(name: %w[fk_constrained_to_referenced fk_constrained_to_other_referenced]).to_a } + it 'finds the foreign keys for the constrained table' do expect(described_class.by_constrained_table_name(table_name("constrained_table"))).to match_array(expected) end end + describe '#by_constrained_table_name_or_identifier' do + let(:expected) { described_class.where(name: %w[fk_constrained_to_referenced fk_constrained_to_other_referenced]).to_a } + + context 'when using table name' do + it 'finds the foreign keys for the constrained table' do + expect(described_class.by_constrained_table_name_or_identifier(table_name("constrained_table"))).to match_array(expected) + end + end + + context 'when using identifier' do + it 'finds the foreign keys for the constrained table' do + expect(described_class.by_constrained_table_name_or_identifier(schema_table_name('constrained_table'))).to match_array(expected) + end + end + end + describe '#by_name' do it 'finds foreign keys by name' do expect(described_class.by_name('fk_constrained_to_referenced').pluck(:name)).to contain_exactly('fk_constrained_to_referenced') @@ -187,10 +203,8 @@ RSpec.describe Gitlab::Database::PostgresForeignKey, type: :model, feature_categ end end - context 'when supporting foreign keys to inherited tables in postgres 12' do + context 'when supporting foreign keys to inherited tables' do before do - skip('not supported before postgres 12') if ApplicationRecord.database.version.to_f < 12 - ApplicationRecord.connection.execute(<<~SQL) create table #{schema_table_name('parent')} ( id bigserial primary key not null diff --git a/spec/lib/gitlab/database/postgres_partition_spec.rb b/spec/lib/gitlab/database/postgres_partition_spec.rb index 14a4d405621..48dbdbc7757 100644 --- a/spec/lib/gitlab/database/postgres_partition_spec.rb +++ b/spec/lib/gitlab/database/postgres_partition_spec.rb @@ -2,7 +2,8 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::PostgresPartition, type: :model do +RSpec.describe Gitlab::Database::PostgresPartition, type: :model, feature_category: :database do + let(:current_schema) { ActiveRecord::Base.connection.select_value("SELECT current_schema()") } let(:schema) { 'gitlab_partitions_dynamic' } let(:name) { '_test_partition_01' } let(:identifier) { "#{schema}.#{name}" } @@ -56,9 +57,20 @@ RSpec.describe Gitlab::Database::PostgresPartition, type: :model do expect(partitions.pluck(:name)).to eq([name, second_name]) end + it 'returns the partitions if the parent table schema is included in the table name' do + partitions = described_class.for_parent_table("#{current_schema}._test_partitioned_table") + + expect(partitions.count).to eq(2) + expect(partitions.pluck(:name)).to eq([name, second_name]) + end + it 'does not return partitions for tables not in the current schema' do expect(described_class.for_parent_table('_test_other_table').count).to eq(0) end + + it 'does not return partitions for tables if the schema is not the current' do + expect(described_class.for_parent_table('foo_bar._test_partitioned_table').count).to eq(0) + end end describe '#parent_identifier' do diff --git a/spec/lib/gitlab/database/reindexing_spec.rb b/spec/lib/gitlab/database/reindexing_spec.rb index a8af9bb5a38..4d0e58b0937 100644 --- a/spec/lib/gitlab/database/reindexing_spec.rb +++ b/spec/lib/gitlab/database/reindexing_spec.rb @@ -71,7 +71,7 @@ RSpec.describe Gitlab::Database::Reindexing, feature_category: :database, time_t context 'when async FK validation is enabled' do it 'executes FK validation for each database prior to any reindexing actions' do - expect(Gitlab::Database::AsyncForeignKeys).to receive(:validate_pending_entries!).ordered.exactly(databases_count).times + expect(Gitlab::Database::AsyncConstraints).to receive(:validate_pending_entries!).ordered.exactly(databases_count).times expect(described_class).to receive(:automatic_reindexing).ordered.exactly(databases_count).times described_class.invoke @@ -82,7 +82,7 @@ RSpec.describe Gitlab::Database::Reindexing, feature_category: :database, time_t it 'does not execute FK validation' do stub_feature_flags(database_async_foreign_key_validation: false) - expect(Gitlab::Database::AsyncForeignKeys).not_to receive(:validate_pending_entries!) + expect(Gitlab::Database::AsyncConstraints).not_to receive(:validate_pending_entries!) described_class.invoke end diff --git a/spec/lib/gitlab/database/schema_validation/database_spec.rb b/spec/lib/gitlab/database/schema_validation/database_spec.rb index c0026f91b46..eadaf683a29 100644 --- a/spec/lib/gitlab/database/schema_validation/database_spec.rb +++ b/spec/lib/gitlab/database/schema_validation/database_spec.rb @@ -3,43 +3,108 @@ require 'spec_helper' RSpec.describe Gitlab::Database::SchemaValidation::Database, feature_category: :database do - let(:database_name) { 'main' } - let(:database_indexes) do - [['index', 'CREATE UNIQUE INDEX "index" ON public.achievements USING btree (namespace_id, lower(name))']] - end + subject(:database) { described_class.new(connection) } - let(:query_result) { instance_double('ActiveRecord::Result', rows: database_indexes) } - let(:database_model) { Gitlab::Database.database_base_models[database_name] } + let(:database_model) { Gitlab::Database.database_base_models['main'] } let(:connection) { database_model.connection } - subject(:database) { described_class.new(connection) } + context 'when having indexes' do + let(:schema_object) { Gitlab::Database::SchemaValidation::SchemaObjects::Index } + let(:results) do + [['index', 'CREATE UNIQUE INDEX "index" ON public.achievements USING btree (namespace_id, lower(name))']] + end - before do - allow(connection).to receive(:exec_query).and_return(query_result) - end + before do + allow(connection).to receive(:select_rows).and_return(results) + end + + describe '#fetch_index_by_name' do + context 'when index does not exist' do + it 'returns nil' do + index = database.fetch_index_by_name('non_existing_index') + + expect(index).to be_nil + end + end + + it 'returns index by name' do + index = database.fetch_index_by_name('index') + + expect(index.name).to eq('index') + end + end + + describe '#index_exists?' do + context 'when index exists' do + it 'returns true' do + index_exists = database.index_exists?('index') + + expect(index_exists).to be_truthy + end + end - describe '#fetch_index_by_name' do - context 'when index does not exist' do - it 'returns nil' do - index = database.fetch_index_by_name('non_existing_index') + context 'when index does not exist' do + it 'returns false' do + index_exists = database.index_exists?('non_existing_index') - expect(index).to be_nil + expect(index_exists).to be_falsey + end end end - it 'returns index by name' do - index = database.fetch_index_by_name('index') + describe '#indexes' do + it 'returns indexes' do + indexes = database.indexes - expect(index.name).to eq('index') + expect(indexes).to all(be_a(schema_object)) + expect(indexes.map(&:name)).to eq(['index']) + end end end - describe '#indexes' do - it 'returns indexes' do - indexes = database.indexes + context 'when having triggers' do + let(:schema_object) { Gitlab::Database::SchemaValidation::SchemaObjects::Trigger } + let(:results) do + { 'my_trigger' => 'CREATE TRIGGER my_trigger BEFORE INSERT ON todos FOR EACH ROW EXECUTE FUNCTION trigger()' } + end + + before do + allow(database).to receive(:fetch_triggers).and_return(results) + end + + describe '#fetch_trigger_by_name' do + context 'when trigger does not exist' do + it 'returns nil' do + expect(database.fetch_trigger_by_name('non_existing_trigger')).to be_nil + end + end + + it 'returns trigger by name' do + expect(database.fetch_trigger_by_name('my_trigger').name).to eq('my_trigger') + end + end + + describe '#trigger_exists?' do + context 'when trigger exists' do + it 'returns true' do + expect(database.trigger_exists?('my_trigger')).to be_truthy + end + end + + context 'when trigger does not exist' do + it 'returns false' do + expect(database.trigger_exists?('non_existing_trigger')).to be_falsey + end + end + end - expect(indexes).to all(be_a(Gitlab::Database::SchemaValidation::Index)) - expect(indexes.map(&:name)).to eq(['index']) + describe '#triggers' do + it 'returns triggers' do + triggers = database.triggers + + expect(triggers).to all(be_a(schema_object)) + expect(triggers.map(&:name)).to eq(['my_trigger']) + end end end end diff --git a/spec/lib/gitlab/database/schema_validation/index_spec.rb b/spec/lib/gitlab/database/schema_validation/index_spec.rb deleted file mode 100644 index 297211d79ed..00000000000 --- a/spec/lib/gitlab/database/schema_validation/index_spec.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true -require 'spec_helper' - -RSpec.describe Gitlab::Database::SchemaValidation::Index, feature_category: :database do - let(:index_statement) { 'CREATE INDEX index_name ON public.achievements USING btree (namespace_id)' } - - let(:stmt) { PgQuery.parse(index_statement).tree.stmts.first.stmt.index_stmt } - - let(:index) { described_class.new(stmt) } - - describe '#name' do - it 'returns index name' do - expect(index.name).to eq('index_name') - end - end - - describe '#statement' do - it 'returns index statement' do - expect(index.statement).to eq(index_statement) - end - end -end diff --git a/spec/lib/gitlab/database/schema_validation/indexes_spec.rb b/spec/lib/gitlab/database/schema_validation/indexes_spec.rb deleted file mode 100644 index 4351031a4b4..00000000000 --- a/spec/lib/gitlab/database/schema_validation/indexes_spec.rb +++ /dev/null @@ -1,56 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Database::SchemaValidation::Indexes, feature_category: :database do - let(:structure_file_path) { Rails.root.join('spec/fixtures/structure.sql') } - let(:database_indexes) do - [ - ['wrong_index', 'CREATE UNIQUE INDEX wrong_index ON public.table_name (column_name)'], - ['extra_index', 'CREATE INDEX extra_index ON public.table_name (column_name)'], - ['index', 'CREATE UNIQUE INDEX "index" ON public.achievements USING btree (namespace_id, lower(name))'] - ] - end - - let(:database_name) { 'main' } - - let(:database_model) { Gitlab::Database.database_base_models[database_name] } - - let(:connection) { database_model.connection } - - let(:query_result) { instance_double('ActiveRecord::Result', rows: database_indexes) } - - let(:database) { Gitlab::Database::SchemaValidation::Database.new(connection) } - let(:structure_file) { Gitlab::Database::SchemaValidation::StructureSql.new(structure_file_path) } - - subject(:schema_validation) { described_class.new(structure_file, database) } - - before do - allow(connection).to receive(:exec_query).and_return(query_result) - end - - describe '#missing_indexes' do - it 'returns missing indexes' do - missing_indexes = %w[ - missing_index - index_namespaces_public_groups_name_id - index_on_deploy_keys_id_and_type_and_public - index_users_on_public_email_excluding_null_and_empty - ] - - expect(schema_validation.missing_indexes).to match_array(missing_indexes) - end - end - - describe '#extra_indexes' do - it 'returns extra indexes' do - expect(schema_validation.extra_indexes).to match_array(['extra_index']) - end - end - - describe '#wrong_indexes' do - it 'returns wrong indexes' do - expect(schema_validation.wrong_indexes).to match_array(['wrong_index']) - end - end -end diff --git a/spec/lib/gitlab/database/schema_validation/runner_spec.rb b/spec/lib/gitlab/database/schema_validation/runner_spec.rb new file mode 100644 index 00000000000..ddbdedcd8b4 --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/runner_spec.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::Runner, feature_category: :database do + let(:structure_file_path) { Rails.root.join('spec/fixtures/structure.sql') } + let(:connection) { ActiveRecord::Base.connection } + + let(:database) { Gitlab::Database::SchemaValidation::Database.new(connection) } + let(:structure_sql) { Gitlab::Database::SchemaValidation::StructureSql.new(structure_file_path, 'public') } + + describe '#execute' do + subject(:inconsistencies) { described_class.new(structure_sql, database).execute } + + it 'returns inconsistencies' do + expect(inconsistencies).not_to be_empty + end + + it 'execute all validators' do + all_validators = Gitlab::Database::SchemaValidation::Validators::BaseValidator.all_validators + + expect(all_validators).to all(receive(:new).with(structure_sql, database).and_call_original) + + inconsistencies + end + + context 'when validators are passed' do + subject(:inconsistencies) { described_class.new(structure_sql, database, validators: validators).execute } + + let(:class_name) { 'Gitlab::Database::SchemaValidation::Validators::ExtraIndexes' } + let(:inconsistency_class_name) { 'Gitlab::Database::SchemaValidation::Validators::BaseValidator::Inconsistency' } + + let(:extra_indexes) { class_double(class_name) } + let(:instace_extra_index) { instance_double(class_name, execute: [inconsistency]) } + let(:inconsistency) { instance_double(inconsistency_class_name, object_name: 'test') } + + let(:validators) { [extra_indexes] } + + it 'only execute the validators passed' do + expect(extra_indexes).to receive(:new).with(structure_sql, database).and_return(instace_extra_index) + + Gitlab::Database::SchemaValidation::Validators::BaseValidator.all_validators.each do |validator| + expect(validator).not_to receive(:new).with(structure_sql, database) + end + + expect(inconsistencies.map(&:object_name)).to eql ['test'] + end + end + end +end diff --git a/spec/lib/gitlab/database/schema_validation/schema_objects/index_spec.rb b/spec/lib/gitlab/database/schema_validation/schema_objects/index_spec.rb new file mode 100644 index 00000000000..1aaa994e3bb --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/schema_objects/index_spec.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::SchemaObjects::Index, feature_category: :database do + let(:statement) { 'CREATE INDEX index_name ON public.achievements USING btree (namespace_id)' } + let(:name) { 'index_name' } + + include_examples 'schema objects assertions for', 'index_stmt' +end diff --git a/spec/lib/gitlab/database/schema_validation/schema_objects/trigger_spec.rb b/spec/lib/gitlab/database/schema_validation/schema_objects/trigger_spec.rb new file mode 100644 index 00000000000..8000a54ee27 --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/schema_objects/trigger_spec.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::SchemaObjects::Trigger, feature_category: :database do + let(:statement) { 'CREATE TRIGGER my_trigger BEFORE INSERT ON todos FOR EACH ROW EXECUTE FUNCTION trigger()' } + let(:name) { 'my_trigger' } + + include_examples 'schema objects assertions for', 'create_trig_stmt' +end diff --git a/spec/lib/gitlab/database/schema_validation/structure_sql_spec.rb b/spec/lib/gitlab/database/schema_validation/structure_sql_spec.rb new file mode 100644 index 00000000000..cc0bd4125ef --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/structure_sql_spec.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::StructureSql, feature_category: :database do + let(:structure_file_path) { Rails.root.join('spec/fixtures/structure.sql') } + let(:schema_name) { 'public' } + + subject(:structure_sql) { described_class.new(structure_file_path, schema_name) } + + context 'when having indexes' do + describe '#index_exists?' do + subject(:index_exists) { structure_sql.index_exists?(index_name) } + + context 'when the index does not exist' do + let(:index_name) { 'non-existent-index' } + + it 'returns false' do + expect(index_exists).to be_falsey + end + end + + context 'when the index exists' do + let(:index_name) { 'index' } + + it 'returns true' do + expect(index_exists).to be_truthy + end + end + end + + describe '#indexes' do + it 'returns indexes' do + indexes = structure_sql.indexes + + expected_indexes = %w[ + missing_index + wrong_index + index + index_namespaces_public_groups_name_id + index_on_deploy_keys_id_and_type_and_public + index_users_on_public_email_excluding_null_and_empty + ] + + expect(indexes).to all(be_a(Gitlab::Database::SchemaValidation::SchemaObjects::Index)) + expect(indexes.map(&:name)).to eq(expected_indexes) + end + end + end + + context 'when having triggers' do + describe '#trigger_exists?' do + subject(:trigger_exists) { structure_sql.trigger_exists?(name) } + + context 'when the trigger does not exist' do + let(:name) { 'non-existent-trigger' } + + it 'returns false' do + expect(trigger_exists).to be_falsey + end + end + + context 'when the trigger exists' do + let(:name) { 'trigger' } + + it 'returns true' do + expect(trigger_exists).to be_truthy + end + end + end + + describe '#triggers' do + it 'returns triggers' do + triggers = structure_sql.triggers + expected_triggers = %w[trigger wrong_trigger missing_trigger_1 projects_loose_fk_trigger] + + expect(triggers).to all(be_a(Gitlab::Database::SchemaValidation::SchemaObjects::Trigger)) + expect(triggers.map(&:name)).to eq(expected_triggers) + end + end + end +end diff --git a/spec/lib/gitlab/database/schema_validation/validators/base_validator_spec.rb b/spec/lib/gitlab/database/schema_validation/validators/base_validator_spec.rb new file mode 100644 index 00000000000..2f38c25cf68 --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/validators/base_validator_spec.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::Validators::BaseValidator, feature_category: :database do + describe '.all_validators' do + subject(:all_validators) { described_class.all_validators } + + it 'returns an array of all validators' do + expect(all_validators).to eq([ + Gitlab::Database::SchemaValidation::Validators::ExtraIndexes, + Gitlab::Database::SchemaValidation::Validators::ExtraTriggers, + Gitlab::Database::SchemaValidation::Validators::MissingIndexes, + Gitlab::Database::SchemaValidation::Validators::MissingTriggers, + Gitlab::Database::SchemaValidation::Validators::DifferentDefinitionIndexes, + Gitlab::Database::SchemaValidation::Validators::DifferentDefinitionTriggers + ]) + end + end + + describe '#execute' do + let(:structure_sql) { instance_double(Gitlab::Database::SchemaValidation::StructureSql) } + let(:database) { instance_double(Gitlab::Database::SchemaValidation::Database) } + + subject(:inconsistencies) { described_class.new(structure_sql, database).execute } + + it 'raises an exception' do + expect { inconsistencies }.to raise_error(NoMethodError) + end + end +end diff --git a/spec/lib/gitlab/database/schema_validation/validators/different_definition_indexes_spec.rb b/spec/lib/gitlab/database/schema_validation/validators/different_definition_indexes_spec.rb new file mode 100644 index 00000000000..b9744c86b80 --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/validators/different_definition_indexes_spec.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::Validators::DifferentDefinitionIndexes, + feature_category: :database do + include_examples 'index validators', described_class, ['wrong_index'] +end diff --git a/spec/lib/gitlab/database/schema_validation/validators/different_definition_triggers_spec.rb b/spec/lib/gitlab/database/schema_validation/validators/different_definition_triggers_spec.rb new file mode 100644 index 00000000000..4d065929708 --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/validators/different_definition_triggers_spec.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::Validators::DifferentDefinitionTriggers, + feature_category: :database do + include_examples 'trigger validators', described_class, ['wrong_trigger'] +end diff --git a/spec/lib/gitlab/database/schema_validation/validators/extra_indexes_spec.rb b/spec/lib/gitlab/database/schema_validation/validators/extra_indexes_spec.rb new file mode 100644 index 00000000000..842dbb42120 --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/validators/extra_indexes_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::Validators::ExtraIndexes, feature_category: :database do + include_examples 'index validators', described_class, ['extra_index'] +end diff --git a/spec/lib/gitlab/database/schema_validation/validators/extra_triggers_spec.rb b/spec/lib/gitlab/database/schema_validation/validators/extra_triggers_spec.rb new file mode 100644 index 00000000000..d2e1c18a1ab --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/validators/extra_triggers_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::Validators::ExtraTriggers, feature_category: :database do + include_examples 'trigger validators', described_class, ['extra_trigger'] +end diff --git a/spec/lib/gitlab/database/schema_validation/validators/missing_indexes_spec.rb b/spec/lib/gitlab/database/schema_validation/validators/missing_indexes_spec.rb new file mode 100644 index 00000000000..c402c3a2fa7 --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/validators/missing_indexes_spec.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::Validators::MissingIndexes, feature_category: :database do + missing_indexes = %w[ + missing_index + index_namespaces_public_groups_name_id + index_on_deploy_keys_id_and_type_and_public + index_users_on_public_email_excluding_null_and_empty + ] + + include_examples 'index validators', described_class, missing_indexes +end diff --git a/spec/lib/gitlab/database/schema_validation/validators/missing_triggers_spec.rb b/spec/lib/gitlab/database/schema_validation/validators/missing_triggers_spec.rb new file mode 100644 index 00000000000..87bc3ded808 --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/validators/missing_triggers_spec.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::Validators::MissingTriggers, feature_category: :database do + missing_triggers = %w[missing_trigger_1 projects_loose_fk_trigger] + + include_examples 'trigger validators', described_class, missing_triggers +end diff --git a/spec/lib/gitlab/database/tables_locker_spec.rb b/spec/lib/gitlab/database/tables_locker_spec.rb index d74f455eaad..30f0f9376c8 100644 --- a/spec/lib/gitlab/database/tables_locker_spec.rb +++ b/spec/lib/gitlab/database/tables_locker_spec.rb @@ -2,20 +2,38 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::TablesLocker, :reestablished_active_record_base, :delete, :silence_stdout, - :suppress_gitlab_schemas_validate_connection, feature_category: :pods do - let(:detached_partition_table) { '_test_gitlab_main_part_20220101' } - let(:lock_writes_manager) do +RSpec.describe Gitlab::Database::TablesLocker, :suppress_gitlab_schemas_validate_connection, :silence_stdout, + feature_category: :pods do + let(:default_lock_writes_manager) do instance_double(Gitlab::Database::LockWritesManager, lock_writes: nil, unlock_writes: nil) end before do - allow(Gitlab::Database::LockWritesManager).to receive(:new).with(any_args).and_return(lock_writes_manager) + allow(Gitlab::Database::LockWritesManager).to receive(:new).with(any_args).and_return(default_lock_writes_manager) + # Limiting the scope of the tests to a subset of the database tables + allow(Gitlab::Database::GitlabSchema).to receive(:tables_to_schema).and_return({ + 'application_setttings' => :gitlab_main_clusterwide, + 'projects' => :gitlab_main, + 'security_findings' => :gitlab_main, + 'ci_builds' => :gitlab_ci, + 'ci_jobs' => :gitlab_ci, + 'loose_foreign_keys_deleted_records' => :gitlab_shared, + 'ar_internal_metadata' => :gitlab_internal + }) end before(:all) do + create_partition_sql = <<~SQL + CREATE TABLE IF NOT EXISTS #{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.security_findings_test_partition + PARTITION OF security_findings + FOR VALUES IN (0) + SQL + + ApplicationRecord.connection.execute(create_partition_sql) + Ci::ApplicationRecord.connection.execute(create_partition_sql) + create_detached_partition_sql = <<~SQL - CREATE TABLE IF NOT EXISTS gitlab_partitions_dynamic._test_gitlab_main_part_20220101 ( + CREATE TABLE IF NOT EXISTS #{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}._test_gitlab_main_part_202201 ( id bigserial primary key not null ) SQL @@ -29,35 +47,81 @@ RSpec.describe Gitlab::Database::TablesLocker, :reestablished_active_record_base drop_after: Time.current ) end + Gitlab::Database::SharedModel.using_connection(Ci::ApplicationRecord.connection) do + Postgresql::DetachedPartition.create!( + table_name: '_test_gitlab_main_part_20220101', + drop_after: Time.current + ) + end end - after(:all) do - drop_detached_partition_sql = <<~SQL - DROP TABLE IF EXISTS gitlab_partitions_dynamic._test_gitlab_main_part_20220101 - SQL + shared_examples "lock tables" do |gitlab_schema, database_name| + let(:connection) { Gitlab::Database.database_base_models[database_name].connection } + let(:tables_to_lock) do + Gitlab::Database::GitlabSchema + .tables_to_schema.filter_map { |table_name, schema| table_name if schema == gitlab_schema } + end - ApplicationRecord.connection.execute(drop_detached_partition_sql) - Ci::ApplicationRecord.connection.execute(drop_detached_partition_sql) + it "locks table in schema #{gitlab_schema} and database #{database_name}" do + expect(tables_to_lock).not_to be_empty - Gitlab::Database::SharedModel.using_connection(ApplicationRecord.connection) do - Postgresql::DetachedPartition.delete_all + tables_to_lock.each do |table_name| + lock_writes_manager = instance_double(Gitlab::Database::LockWritesManager, lock_writes: nil) + + expect(Gitlab::Database::LockWritesManager).to receive(:new).with( + table_name: table_name, + connection: connection, + database_name: database_name, + with_retries: true, + logger: anything, + dry_run: anything + ).once.and_return(lock_writes_manager) + expect(lock_writes_manager).to receive(:lock_writes).once + end + + subject end end - shared_examples "lock tables" do |table_schema, database_name| - let(:table_name) do + shared_examples "unlock tables" do |gitlab_schema, database_name| + let(:connection) { Gitlab::Database.database_base_models[database_name].connection } + + let(:tables_to_unlock) do Gitlab::Database::GitlabSchema - .tables_to_schema.filter_map { |table_name, schema| table_name if schema == table_schema } - .first + .tables_to_schema.filter_map { |table_name, schema| table_name if schema == gitlab_schema } + end + + it "unlocks table in schema #{gitlab_schema} and database #{database_name}" do + expect(tables_to_unlock).not_to be_empty + + tables_to_unlock.each do |table_name| + lock_writes_manager = instance_double(Gitlab::Database::LockWritesManager, unlock_writes: nil) + + expect(Gitlab::Database::LockWritesManager).to receive(:new).with( + table_name: table_name, + connection: anything, + database_name: database_name, + with_retries: true, + logger: anything, + dry_run: anything + ).once.and_return(lock_writes_manager) + expect(lock_writes_manager).to receive(:unlock_writes) + end + + subject end + end + + shared_examples "lock partitions" do |partition_identifier, database_name| + let(:connection) { Gitlab::Database.database_base_models[database_name].connection } - let(:database) { database_name } + it 'locks the partition' do + lock_writes_manager = instance_double(Gitlab::Database::LockWritesManager, lock_writes: nil) - it "locks table in schema #{table_schema} and database #{database_name}" do expect(Gitlab::Database::LockWritesManager).to receive(:new).with( - table_name: table_name, - connection: anything, - database_name: database, + table_name: partition_identifier, + connection: connection, + database_name: database_name, with_retries: true, logger: anything, dry_run: anything @@ -68,20 +132,16 @@ RSpec.describe Gitlab::Database::TablesLocker, :reestablished_active_record_base end end - shared_examples "unlock tables" do |table_schema, database_name| - let(:table_name) do - Gitlab::Database::GitlabSchema - .tables_to_schema.filter_map { |table_name, schema| table_name if schema == table_schema } - .first - end + shared_examples "unlock partitions" do |partition_identifier, database_name| + let(:connection) { Gitlab::Database.database_base_models[database_name].connection } - let(:database) { database_name } + it 'unlocks the partition' do + lock_writes_manager = instance_double(Gitlab::Database::LockWritesManager, unlock_writes: nil) - it "unlocks table in schema #{table_schema} and database #{database_name}" do expect(Gitlab::Database::LockWritesManager).to receive(:new).with( - table_name: table_name, - connection: anything, - database_name: database, + table_name: partition_identifier, + connection: connection, + database_name: database_name, with_retries: true, logger: anything, dry_run: anything @@ -100,25 +160,29 @@ RSpec.describe Gitlab::Database::TablesLocker, :reestablished_active_record_base describe '#lock_writes' do subject { described_class.new.lock_writes } - it 'does not call Gitlab::Database::LockWritesManager.lock_writes' do - expect(Gitlab::Database::LockWritesManager).to receive(:new).with(any_args).and_return(lock_writes_manager) - expect(lock_writes_manager).not_to receive(:lock_writes) + it 'does not lock any table' do + expect(Gitlab::Database::LockWritesManager).to receive(:new) + .with(any_args).and_return(default_lock_writes_manager) + expect(default_lock_writes_manager).not_to receive(:lock_writes) subject end - include_examples "unlock tables", :gitlab_main, 'main' - include_examples "unlock tables", :gitlab_ci, 'ci' - include_examples "unlock tables", :gitlab_shared, 'main' - include_examples "unlock tables", :gitlab_internal, 'main' + it_behaves_like 'unlock tables', :gitlab_main, 'main' + it_behaves_like 'unlock tables', :gitlab_ci, 'main' + it_behaves_like 'unlock tables', :gitlab_main_clusterwide, 'main' + it_behaves_like 'unlock tables', :gitlab_shared, 'main' + it_behaves_like 'unlock tables', :gitlab_internal, 'main' end describe '#unlock_writes' do subject { described_class.new.lock_writes } it 'does call Gitlab::Database::LockWritesManager.unlock_writes' do - expect(Gitlab::Database::LockWritesManager).to receive(:new).with(any_args).and_return(lock_writes_manager) - expect(lock_writes_manager).to receive(:unlock_writes) + expect(Gitlab::Database::LockWritesManager).to receive(:new) + .with(any_args).and_return(default_lock_writes_manager) + expect(default_lock_writes_manager).to receive(:unlock_writes) + expect(default_lock_writes_manager).not_to receive(:lock_writes) subject end @@ -133,43 +197,61 @@ RSpec.describe Gitlab::Database::TablesLocker, :reestablished_active_record_base describe '#lock_writes' do subject { described_class.new.lock_writes } - include_examples "lock tables", :gitlab_ci, 'main' - include_examples "lock tables", :gitlab_main, 'ci' - - include_examples "unlock tables", :gitlab_main, 'main' - include_examples "unlock tables", :gitlab_ci, 'ci' - include_examples "unlock tables", :gitlab_shared, 'main' - include_examples "unlock tables", :gitlab_shared, 'ci' - include_examples "unlock tables", :gitlab_internal, 'main' - include_examples "unlock tables", :gitlab_internal, 'ci' + it_behaves_like 'lock tables', :gitlab_ci, 'main' + it_behaves_like 'lock tables', :gitlab_main, 'ci' + it_behaves_like 'lock tables', :gitlab_main_clusterwide, 'ci' + + it_behaves_like 'unlock tables', :gitlab_main_clusterwide, 'main' + it_behaves_like 'unlock tables', :gitlab_main, 'main' + it_behaves_like 'unlock tables', :gitlab_ci, 'ci' + it_behaves_like 'unlock tables', :gitlab_shared, 'main' + it_behaves_like 'unlock tables', :gitlab_shared, 'ci' + it_behaves_like 'unlock tables', :gitlab_internal, 'main' + it_behaves_like 'unlock tables', :gitlab_internal, 'ci' + + gitlab_main_partition = "#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.security_findings_test_partition" + it_behaves_like 'unlock partitions', gitlab_main_partition, 'main' + it_behaves_like 'lock partitions', gitlab_main_partition, 'ci' + + gitlab_main_detached_partition = "#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}._test_gitlab_main_part_20220101" + it_behaves_like 'unlock partitions', gitlab_main_detached_partition, 'main' + it_behaves_like 'lock partitions', gitlab_main_detached_partition, 'ci' end describe '#unlock_writes' do subject { described_class.new.unlock_writes } - include_examples "unlock tables", :gitlab_ci, 'main' - include_examples "unlock tables", :gitlab_main, 'ci' - include_examples "unlock tables", :gitlab_main, 'main' - include_examples "unlock tables", :gitlab_ci, 'ci' - include_examples "unlock tables", :gitlab_shared, 'main' - include_examples "unlock tables", :gitlab_shared, 'ci' - include_examples "unlock tables", :gitlab_internal, 'main' - include_examples "unlock tables", :gitlab_internal, 'ci' + it_behaves_like "unlock tables", :gitlab_ci, 'main' + it_behaves_like "unlock tables", :gitlab_main, 'ci' + it_behaves_like "unlock tables", :gitlab_main, 'main' + it_behaves_like "unlock tables", :gitlab_ci, 'ci' + it_behaves_like "unlock tables", :gitlab_shared, 'main' + it_behaves_like "unlock tables", :gitlab_shared, 'ci' + it_behaves_like "unlock tables", :gitlab_internal, 'main' + it_behaves_like "unlock tables", :gitlab_internal, 'ci' + + gitlab_main_partition = "#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.security_findings_test_partition" + it_behaves_like 'unlock partitions', gitlab_main_partition, 'main' + it_behaves_like 'unlock partitions', gitlab_main_partition, 'ci' + + gitlab_main_detached_partition = "#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}._test_gitlab_main_part_20220101" + it_behaves_like 'unlock partitions', gitlab_main_detached_partition, 'main' + it_behaves_like 'unlock partitions', gitlab_main_detached_partition, 'ci' end context 'when running in dry_run mode' do subject { described_class.new(dry_run: true).lock_writes } - it 'passes dry_run flag to LockManger' do + it 'passes dry_run flag to LockWritesManager' do expect(Gitlab::Database::LockWritesManager).to receive(:new).with( - table_name: 'users', + table_name: 'security_findings', connection: anything, database_name: 'ci', with_retries: true, logger: anything, dry_run: true - ).and_return(lock_writes_manager) - expect(lock_writes_manager).to receive(:lock_writes) + ).and_return(default_lock_writes_manager) + expect(default_lock_writes_manager).to receive(:lock_writes) subject end @@ -185,8 +267,9 @@ RSpec.describe Gitlab::Database::TablesLocker, :reestablished_active_record_base end it 'does not lock any tables if the ci database is shared with main database' do - expect(Gitlab::Database::LockWritesManager).to receive(:new).with(any_args).and_return(lock_writes_manager) - expect(lock_writes_manager).not_to receive(:lock_writes) + expect(Gitlab::Database::LockWritesManager).to receive(:new) + .with(any_args).and_return(default_lock_writes_manager) + expect(default_lock_writes_manager).not_to receive(:lock_writes) subject end @@ -220,7 +303,3 @@ RSpec.describe Gitlab::Database::TablesLocker, :reestablished_active_record_base end end end - -def number_of_triggers(connection) - connection.select_value("SELECT count(*) FROM information_schema.triggers") -end diff --git a/spec/lib/gitlab/doorkeeper_secret_storing/secret/pbkdf2_sha512_spec.rb b/spec/lib/gitlab/doorkeeper_secret_storing/secret/pbkdf2_sha512_spec.rb index df17d92bb0c..fb433923db5 100644 --- a/spec/lib/gitlab/doorkeeper_secret_storing/secret/pbkdf2_sha512_spec.rb +++ b/spec/lib/gitlab/doorkeeper_secret_storing/secret/pbkdf2_sha512_spec.rb @@ -10,16 +10,6 @@ RSpec.describe Gitlab::DoorkeeperSecretStoring::Secret::Pbkdf2Sha512 do expect(described_class.transform_secret(plaintext_secret)) .to eq("$pbkdf2-sha512$20000$$.c0G5XJVEew1TyeJk5TrkvB0VyOaTmDzPrsdNRED9vVeZlSyuG3G90F0ow23zUCiWKAVwmNnR/ceh.nJG3MdpQ") # rubocop:disable Layout/LineLength end - - context 'when hash_oauth_secrets is disabled' do - before do - stub_feature_flags(hash_oauth_secrets: false) - end - - it 'returns a plaintext secret' do - expect(described_class.transform_secret(plaintext_secret)).to eq(plaintext_secret) - end - end end describe 'STRETCHES' do @@ -36,7 +26,6 @@ RSpec.describe Gitlab::DoorkeeperSecretStoring::Secret::Pbkdf2Sha512 do describe '.secret_matches?' do it "match by hashing the input if the stored value is hashed" do - stub_feature_flags(hash_oauth_secrets: false) plain_secret = 'plain_secret' stored_value = '$pbkdf2-sha512$20000$$/BwQRdwSpL16xkQhstavh7nvA5avCP7.4n9LLKe9AupgJDeA7M5xOAvG3N3E5XbRyGWWBbbr.BsojPVWzd1Sqg' # rubocop:disable Layout/LineLength expect(described_class.secret_matches?(plain_secret, stored_value)).to be true diff --git a/spec/lib/gitlab/email/handler/create_issue_handler_spec.rb b/spec/lib/gitlab/email/handler/create_issue_handler_spec.rb index 8ff8de2379a..369d7e994d2 100644 --- a/spec/lib/gitlab/email/handler/create_issue_handler_spec.rb +++ b/spec/lib/gitlab/email/handler/create_issue_handler_spec.rb @@ -116,7 +116,7 @@ RSpec.describe Gitlab::Email::Handler::CreateIssueHandler do context "when the issue could not be saved" do before do allow_any_instance_of(Issue).to receive(:persisted?).and_return(false) - allow_any_instance_of(Issue).to receive(:ensure_metrics).and_return(nil) + allow_any_instance_of(Issue).to receive(:ensure_metrics!).and_return(nil) end it "raises an InvalidIssueError" do diff --git a/spec/lib/gitlab/email/html_to_markdown_parser_spec.rb b/spec/lib/gitlab/email/html_to_markdown_parser_spec.rb index fe585d47d59..59c488739dc 100644 --- a/spec/lib/gitlab/email/html_to_markdown_parser_spec.rb +++ b/spec/lib/gitlab/email/html_to_markdown_parser_spec.rb @@ -1,17 +1,21 @@ # frozen_string_literal: true -require 'spec_helper' +require 'kramdown' +require 'html2text' +require 'fast_spec_helper' +require 'support/helpers/fixture_helpers' RSpec.describe Gitlab::Email::HtmlToMarkdownParser, feature_category: :service_desk do + include FixtureHelpers + subject { described_class.convert(html) } describe '.convert' do let(:html) { fixture_file("lib/gitlab/email/basic.html") } it 'parses html correctly' do - expect(subject) - .to eq( - <<-BODY.strip_heredoc.chomp + expect(subject).to eq( + <<~BODY.chomp Hello, World! This is some e-mail content. Even though it has whitespace and newlines, the e-mail converter will handle it correctly. *Even* mismatched tags. diff --git a/spec/lib/gitlab/email/message/build_ios_app_guide_spec.rb b/spec/lib/gitlab/email/message/build_ios_app_guide_spec.rb index 3089f955252..4b77b2f7192 100644 --- a/spec/lib/gitlab/email/message/build_ios_app_guide_spec.rb +++ b/spec/lib/gitlab/email/message/build_ios_app_guide_spec.rb @@ -2,13 +2,9 @@ require 'spec_helper' -RSpec.describe Gitlab::Email::Message::BuildIosAppGuide do +RSpec.describe Gitlab::Email::Message::BuildIosAppGuide, :saas do subject(:message) { described_class.new } - before do - allow(Gitlab).to receive(:com?) { true } - end - it 'contains the correct message', :aggregate_failures do expect(message.subject_line).to eq 'Get set up to build for iOS' expect(message.title).to eq "Building for iOS? We've got you covered." diff --git a/spec/lib/gitlab/email/message/in_product_marketing/helper_spec.rb b/spec/lib/gitlab/email/message/in_product_marketing/helper_spec.rb index 3c0d83d0f9e..a3c2d1b428e 100644 --- a/spec/lib/gitlab/email/message/in_product_marketing/helper_spec.rb +++ b/spec/lib/gitlab/email/message/in_product_marketing/helper_spec.rb @@ -27,11 +27,7 @@ RSpec.describe Gitlab::Email::Message::InProductMarketing::Helper do subject(:class_with_helper) { dummy_class_with_helper.new(format) } - context 'gitlab.com' do - before do - allow(Gitlab).to receive(:com?) { true } - end - + context 'for SaaS', :saas do context 'format is HTML' do it 'returns the correct HTML' do message = "If you no longer wish to receive marketing emails from us, " \ diff --git a/spec/lib/gitlab/endpoint_attributes_spec.rb b/spec/lib/gitlab/endpoint_attributes_spec.rb index 53f5b302f05..a623070c3eb 100644 --- a/spec/lib/gitlab/endpoint_attributes_spec.rb +++ b/spec/lib/gitlab/endpoint_attributes_spec.rb @@ -1,11 +1,8 @@ # frozen_string_literal: true -require 'fast_spec_helper' -require_relative '../../support/matchers/be_request_urgency' -require_relative '../../../lib/gitlab/endpoint_attributes/config' -require_relative '../../../lib/gitlab/endpoint_attributes' +require 'spec_helper' -RSpec.describe Gitlab::EndpointAttributes do +RSpec.describe Gitlab::EndpointAttributes, feature_category: :api do let(:base_controller) do Class.new do include ::Gitlab::EndpointAttributes diff --git a/spec/lib/gitlab/etag_caching/middleware_spec.rb b/spec/lib/gitlab/etag_caching/middleware_spec.rb index fa0b3d1c6dd..d25511843ff 100644 --- a/spec/lib/gitlab/etag_caching/middleware_spec.rb +++ b/spec/lib/gitlab/etag_caching/middleware_spec.rb @@ -145,8 +145,11 @@ RSpec.describe Gitlab::EtagCaching::Middleware, :clean_gitlab_redis_shared_state expect(payload[:headers].env['HTTP_IF_NONE_MATCH']).to eq('W/"123"') end - it 'log subscriber processes action' do - expect_any_instance_of(ActionController::LogSubscriber).to receive(:process_action) + it "publishes process_action.action_controller event to be picked up by lograge's subscriber" do + # Lograge unhooks the default Rails subscriber (ActionController::LogSubscriber) + # and replaces with its own (Lograge::LogSubscribers::ActionController). + # When `lograge.keep_original_rails_log = true`, ActionController::LogSubscriber is kept. + expect_any_instance_of(Lograge::LogSubscribers::ActionController).to receive(:process_action) .with(instance_of(ActiveSupport::Notifications::Event)) .and_call_original diff --git a/spec/lib/gitlab/exception_log_formatter_spec.rb b/spec/lib/gitlab/exception_log_formatter_spec.rb index 7dda56f0bf5..82166971603 100644 --- a/spec/lib/gitlab/exception_log_formatter_spec.rb +++ b/spec/lib/gitlab/exception_log_formatter_spec.rb @@ -45,6 +45,12 @@ RSpec.describe Gitlab::ExceptionLogFormatter do allow(exception).to receive(:cause).and_return(ActiveRecord::StatementInvalid.new(sql: 'SELECT "users".* FROM "users" WHERE "users"."id" = 1 AND "users"."foo" = $1')) end + it 'adds the cause_class to payload' do + described_class.format!(exception, payload) + + expect(payload['exception.cause_class']).to eq('ActiveRecord::StatementInvalid') + end + it 'adds the normalized SQL query to payload' do described_class.format!(exception, payload) diff --git a/spec/lib/gitlab/external_authorization/config_spec.rb b/spec/lib/gitlab/external_authorization/config_spec.rb index 4231b0d3747..f1daa9249f4 100644 --- a/spec/lib/gitlab/external_authorization/config_spec.rb +++ b/spec/lib/gitlab/external_authorization/config_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::ExternalAuthorization::Config, feature_category: :authentication_and_authorization do +RSpec.describe Gitlab::ExternalAuthorization::Config, feature_category: :system_access do it 'allows deploy tokens and keys when external authorization is disabled' do stub_application_setting(external_authorization_service_enabled: false) expect(described_class.allow_deploy_tokens_and_deploy_keys?).to be_eql(true) diff --git a/spec/lib/gitlab/file_finder_spec.rb b/spec/lib/gitlab/file_finder_spec.rb index 27750f10e87..8afaec3c381 100644 --- a/spec/lib/gitlab/file_finder_spec.rb +++ b/spec/lib/gitlab/file_finder_spec.rb @@ -13,124 +13,58 @@ RSpec.describe Gitlab::FileFinder, feature_category: :global_search do let(:expected_file_by_content) { 'CHANGELOG' } end - context 'when code_basic_search_files_by_regexp is enabled' do - before do - stub_feature_flags(code_basic_search_files_by_regexp: true) - end - - context 'with inclusive filters' do - it 'filters by filename' do - results = subject.find('files filename:wm.svg') - - expect(results.count).to eq(1) - end - - it 'filters by path' do - results = subject.find('white path:images') - - expect(results.count).to eq(2) - end - - it 'filters by extension' do - results = subject.find('files extension:md') - - expect(results.count).to eq(4) - end - end - - context 'with exclusive filters' do - it 'filters by filename' do - results = subject.find('files -filename:wm.svg') - - expect(results.count).to eq(26) - end - - it 'filters by path' do - results = subject.find('white -path:images') - - expect(results.count).to eq(5) - end - - it 'filters by extension' do - results = subject.find('files -extension:md') + context 'with inclusive filters' do + it 'filters by filename' do + results = subject.find('files filename:wm.svg') - expect(results.count).to eq(23) - end + expect(results.count).to eq(1) end - context 'with white space in the path' do - it 'filters by path correctly' do - results = subject.find('directory path:"with space/README.md"') + it 'filters by path' do + results = subject.find('white path:images') - expect(results.count).to eq(1) - end + expect(results.count).to eq(2) end - it 'does not cause N+1 query' do - expect(Gitlab::GitalyClient).to receive(:call).at_most(10).times.and_call_original + it 'filters by extension' do + results = subject.find('files extension:md') - subject.find(': filename:wm.svg') + expect(results.count).to eq(4) end end - context 'when code_basic_search_files_by_regexp is disabled' do - before do - stub_feature_flags(code_basic_search_files_by_regexp: false) - end - - context 'with inclusive filters' do - it 'filters by filename' do - results = subject.find('files filename:wm.svg') - - expect(results.count).to eq(1) - end - - it 'filters by path' do - results = subject.find('white path:images') - - expect(results.count).to eq(1) - end - - it 'filters by extension' do - results = subject.find('files extension:md') + context 'with exclusive filters' do + it 'filters by filename' do + results = subject.find('files -filename:wm.svg') - expect(results.count).to eq(4) - end + expect(results.count).to eq(26) end - context 'with exclusive filters' do - it 'filters by filename' do - results = subject.find('files -filename:wm.svg') + it 'filters by path' do + results = subject.find('white -path:images') - expect(results.count).to eq(26) - end - - it 'filters by path' do - results = subject.find('white -path:images') - - expect(results.count).to eq(4) - end + expect(results.count).to eq(5) + end - it 'filters by extension' do - results = subject.find('files -extension:md') + it 'filters by extension' do + results = subject.find('files -extension:md') - expect(results.count).to eq(23) - end + expect(results.count).to eq(23) end + end - context 'with white space in the path' do - it 'filters by path correctly' do - results = subject.find('directory path:"with space/README.md"') + context 'with white space in the path' do + it 'filters by path correctly' do + results = subject.find('directory path:"with space/README.md"') - expect(results.count).to eq(1) - end + expect(results.count).to eq(1) end + end - it 'does not cause N+1 query' do - expect(Gitlab::GitalyClient).to receive(:call).at_most(10).times.and_call_original + it 'does not cause N+1 query' do + expect(Gitlab::GitalyClient).to receive(:call).at_most(10).times.and_call_original - subject.find(': filename:wm.svg') - end + subject.find(': filename:wm.svg') end end end diff --git a/spec/lib/gitlab/git/commit_spec.rb b/spec/lib/gitlab/git/commit_spec.rb index d873151421d..26af9d5d5b8 100644 --- a/spec/lib/gitlab/git/commit_spec.rb +++ b/spec/lib/gitlab/git/commit_spec.rb @@ -660,7 +660,8 @@ RSpec.describe Gitlab::Git::Commit do id: SeedRepo::Commit::ID, message: "tree css fixes", parent_ids: ["874797c3a73b60d2187ed6e2fcabd289ff75171e"], - trailers: {} + trailers: {}, + referenced_by: [] } end end diff --git a/spec/lib/gitlab/git/diff_collection_spec.rb b/spec/lib/gitlab/git/diff_collection_spec.rb index 7fa5bd8a92b..5fa0447091c 100644 --- a/spec/lib/gitlab/git/diff_collection_spec.rb +++ b/spec/lib/gitlab/git/diff_collection_spec.rb @@ -777,6 +777,26 @@ RSpec.describe Gitlab::Git::DiffCollection do end end + describe '.limits' do + let(:options) { {} } + + subject { described_class.limits(options) } + + context 'when options do not include max_patch_bytes_for_file_extension' do + it 'sets max_patch_bytes_for_file_extension as empty' do + expect(subject[:max_patch_bytes_for_file_extension]).to eq({}) + end + end + + context 'when options include max_patch_bytes_for_file_extension' do + let(:options) { { max_patch_bytes_for_file_extension: { '.file' => 1 } } } + + it 'sets value for max_patch_bytes_for_file_extension' do + expect(subject[:max_patch_bytes_for_file_extension]).to eq({ '.file' => 1 }) + end + end + end + def fake_diff(line_length, line_count) { 'diff' => "#{'a' * line_length}\n" * line_count } end diff --git a/spec/lib/gitlab/git/repository_spec.rb b/spec/lib/gitlab/git/repository_spec.rb index 72043ba2a21..483140052f0 100644 --- a/spec/lib/gitlab/git/repository_spec.rb +++ b/spec/lib/gitlab/git/repository_spec.rb @@ -1794,47 +1794,37 @@ RSpec.describe Gitlab::Git::Repository, feature_category: :source_code_managemen end describe '#license' do - where(from_gitaly: [true, false]) - with_them do - subject(:license) { repository.license(from_gitaly) } - - context 'when no license file can be found' do - let_it_be(:project) { create(:project, :repository) } - let(:repository) { project.repository.raw_repository } + subject(:license) { repository.license } - before do - project.repository.delete_file(project.owner, 'LICENSE', message: 'remove license', branch_name: 'master') - end + context 'when no license file can be found' do + let_it_be(:project) { create(:project, :repository) } + let(:repository) { project.repository.raw_repository } - it { is_expected.to be_nil } + before do + project.repository.delete_file(project.owner, 'LICENSE', message: 'remove license', branch_name: 'master') end - context 'when an mit license is found' do - it { is_expected.to have_attributes(key: 'mit') } - end + it { is_expected.to be_nil } + end - context 'when license is not recognized ' do - let_it_be(:project) { create(:project, :repository) } - let(:repository) { project.repository.raw_repository } + context 'when an mit license is found' do + it { is_expected.to have_attributes(key: 'mit') } + end - before do - project.repository.update_file( - project.owner, - 'LICENSE', - 'This software is licensed under the Dummy license.', - message: 'Update license', - branch_name: 'master') - end + context 'when license is not recognized ' do + let_it_be(:project) { create(:project, :repository) } + let(:repository) { project.repository.raw_repository } - it { is_expected.to have_attributes(key: 'other', nickname: 'LICENSE') } + before do + project.repository.update_file( + project.owner, + 'LICENSE', + 'This software is licensed under the Dummy license.', + message: 'Update license', + branch_name: 'master') end - end - - it 'does not crash when license is invalid' do - expect(Licensee::License).to receive(:new) - .and_raise(Licensee::InvalidLicense) - expect(repository.license(false)).to be_nil + it { is_expected.to have_attributes(key: 'other', nickname: 'LICENSE') } end end diff --git a/spec/lib/gitlab/git_access_spec.rb b/spec/lib/gitlab/git_access_spec.rb index ea2c239df07..13e9aeb4c53 100644 --- a/spec/lib/gitlab/git_access_spec.rb +++ b/spec/lib/gitlab/git_access_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::GitAccess, :aggregate_failures, feature_category: :authentication_and_authorization do +RSpec.describe Gitlab::GitAccess, :aggregate_failures, feature_category: :system_access do include TermsHelper include AdminModeHelper include ExternalAuthorizationServiceHelpers diff --git a/spec/lib/gitlab/gitaly_client/ref_service_spec.rb b/spec/lib/gitlab/gitaly_client/ref_service_spec.rb index 09d8ea3cc0a..7bdfa8922d3 100644 --- a/spec/lib/gitlab/gitaly_client/ref_service_spec.rb +++ b/spec/lib/gitlab/gitaly_client/ref_service_spec.rb @@ -213,8 +213,13 @@ RSpec.describe Gitlab::GitalyClient::RefService, feature_category: :gitaly do client.local_branches(sort_by: 'name_asc') end - it 'raises an argument error if an invalid sort_by parameter is passed' do - expect { client.local_branches(sort_by: 'invalid_sort') }.to raise_error(ArgumentError) + it 'uses default sort by name' do + expect_any_instance_of(Gitaly::RefService::Stub) + .to receive(:find_local_branches) + .with(gitaly_request_with_params(sort_by: :NAME), kind_of(Hash)) + .and_return([]) + + client.local_branches(sort_by: 'invalid') end end @@ -270,6 +275,17 @@ RSpec.describe Gitlab::GitalyClient::RefService, feature_category: :gitaly do client.tags(sort_by: 'version_asc') end end + + context 'when sorting option is invalid' do + it 'uses default sort by name' do + expect_any_instance_of(Gitaly::RefService::Stub) + .to receive(:find_all_tags) + .with(gitaly_request_with_params(sort_by: nil), kind_of(Hash)) + .and_return([]) + + client.tags(sort_by: 'invalid') + end + end end context 'with pagination option' do diff --git a/spec/lib/gitlab/gitaly_client/repository_service_spec.rb b/spec/lib/gitlab/gitaly_client/repository_service_spec.rb index 434550186c1..f457ba06074 100644 --- a/spec/lib/gitlab/gitaly_client/repository_service_spec.rb +++ b/spec/lib/gitlab/gitaly_client/repository_service_spec.rb @@ -275,7 +275,8 @@ RSpec.describe Gitlab::GitalyClient::RepositoryService do it 'sends a create_repository message without arguments' do expect_any_instance_of(Gitaly::RepositoryService::Stub) .to receive(:create_repository) - .with(gitaly_request_with_path(storage_name, relative_path).and(gitaly_request_with_params(default_branch: '')), kind_of(Hash)) + .with(gitaly_request_with_path(storage_name, relative_path) + .and(gitaly_request_with_params(default_branch: '')), kind_of(Hash)) .and_return(double) client.create_repository @@ -284,11 +285,23 @@ RSpec.describe Gitlab::GitalyClient::RepositoryService do it 'sends a create_repository message with default branch' do expect_any_instance_of(Gitaly::RepositoryService::Stub) .to receive(:create_repository) - .with(gitaly_request_with_path(storage_name, relative_path).and(gitaly_request_with_params(default_branch: 'default-branch-name')), kind_of(Hash)) + .with(gitaly_request_with_path(storage_name, relative_path) + .and(gitaly_request_with_params(default_branch: 'default-branch-name')), kind_of(Hash)) .and_return(double) client.create_repository('default-branch-name') end + + it 'sends a create_repository message with default branch containing non ascii chars' do + expect_any_instance_of(Gitaly::RepositoryService::Stub) + .to receive(:create_repository) + .with(gitaly_request_with_path(storage_name, relative_path) + .and(gitaly_request_with_params( + default_branch: Gitlab::EncodingHelper.encode_binary('feature/新機能'))), kind_of(Hash) + ).and_return(double) + + client.create_repository('feature/新機能') + end end describe '#create_from_snapshot' do @@ -314,17 +327,31 @@ RSpec.describe Gitlab::GitalyClient::RepositoryService do end describe '#search_files_by_regexp' do - subject(:result) { client.search_files_by_regexp('master', '.*') } + subject(:result) { client.search_files_by_regexp(ref, '.*') } before do expect_any_instance_of(Gitaly::RepositoryService::Stub) .to receive(:search_files_by_name) - .with(gitaly_request_with_path(storage_name, relative_path), kind_of(Hash)) - .and_return([double(files: ['file1.txt']), double(files: ['file2.txt'])]) + .with(gitaly_request_with_path(storage_name, relative_path), kind_of(Hash)) + .and_return([double(files: ['file1.txt']), double(files: ['file2.txt'])]) end - it 'sends a search_files_by_name message and returns a flatten array' do - expect(result).to contain_exactly('file1.txt', 'file2.txt') + shared_examples 'a search for files by regexp' do + it 'sends a search_files_by_name message and returns a flatten array' do + expect(result).to contain_exactly('file1.txt', 'file2.txt') + end + end + + context 'with ASCII ref' do + let(:ref) { 'master' } + + it_behaves_like 'a search for files by regexp' + end + + context 'with non-ASCII ref' do + let(:ref) { 'ref-ñéüçæøß-val' } + + it_behaves_like 'a search for files by regexp' end end diff --git a/spec/lib/gitlab/github_import/client_spec.rb b/spec/lib/gitlab/github_import/client_spec.rb index e93d585bc3c..e37b27aeaef 100644 --- a/spec/lib/gitlab/github_import/client_spec.rb +++ b/spec/lib/gitlab/github_import/client_spec.rb @@ -600,7 +600,8 @@ RSpec.describe Gitlab::GithubImport::Client, feature_category: :importers do endCursor hasNextPage hasPreviousPage - } + }, + repositoryCount } } TEXT @@ -707,44 +708,30 @@ RSpec.describe Gitlab::GithubImport::Client, feature_category: :importers do end end - describe '#search_repos_by_name' do - let(:expected_query) { 'test in:name is:public,private user:user repo:repo1 repo:repo2 org:org1 org:org2' } - - it 'searches for repositories based on name' do - expect(client.octokit).to receive(:search_repositories).with(expected_query, {}) + describe '#count_repos_by_relation_type_graphql' do + relation_types = { + 'owned' => ' in:name is:public,private user:user', + 'collaborated' => ' in:name is:public,private repo:repo1 repo:repo2', + 'organization' => 'org:org1 org:org2' + } - client.search_repos_by_name('test') - end + relation_types.each do |relation_type, expected_query| + expected_graphql_params = "type: REPOSITORY, query: \"#{expected_query}\"" + expected_graphql = + <<-TEXT + { + search(#{expected_graphql_params}) { + repositoryCount + } + } + TEXT - context 'when pagination options present' do - it 'searches for repositories via expected query' do - expect(client.octokit).to receive(:search_repositories).with( - expected_query, { page: 2, per_page: 25 } + it 'returns count by relation_type' do + expect(client.octokit).to receive(:post).with( + '/graphql', { query: expected_graphql }.to_json ) - client.search_repos_by_name('test', { page: 2, per_page: 25 }) - end - end - - context 'when Faraday error received from octokit', :aggregate_failures do - let(:error_class) { described_class::CLIENT_CONNECTION_ERROR } - let(:info_params) { { 'error.class': error_class } } - - it 'retries on error and succeeds' do - allow_retry(:search_repositories) - - expect(Gitlab::Import::Logger).to receive(:info).with(hash_including(info_params)).once - - expect(client.search_repos_by_name('test')).to eq({}) - end - - it 'retries and does not succeed' do - allow(client.octokit) - .to receive(:search_repositories) - .with(expected_query, {}) - .and_raise(error_class, 'execution expired') - - expect { client.search_repos_by_name('test') }.to raise_error(error_class, 'execution expired') + client.count_repos_by_relation_type_graphql(relation_type: relation_type) end end end diff --git a/spec/lib/gitlab/github_import/clients/proxy_spec.rb b/spec/lib/gitlab/github_import/clients/proxy_spec.rb index 0baff7bafcb..7b2a8fa9d74 100644 --- a/spec/lib/gitlab/github_import/clients/proxy_spec.rb +++ b/spec/lib/gitlab/github_import/clients/proxy_spec.rb @@ -8,6 +8,10 @@ RSpec.describe Gitlab::GithubImport::Clients::Proxy, :manage, feature_category: let(:access_token) { 'test_token' } let(:client_options) { { foo: :bar } } + it { expect(client).to delegate_method(:each_object).to(:client) } + it { expect(client).to delegate_method(:user).to(:client) } + it { expect(client).to delegate_method(:octokit).to(:client) } + describe '#repos' do let(:search_text) { 'search text' } let(:pagination_options) { { limit: 10 } } @@ -15,54 +19,32 @@ RSpec.describe Gitlab::GithubImport::Clients::Proxy, :manage, feature_category: context 'when remove_legacy_github_client FF is enabled' do let(:client_stub) { instance_double(Gitlab::GithubImport::Client) } - context 'with github_client_fetch_repos_via_graphql FF enabled' do - let(:client_response) do - { - data: { - search: { - nodes: [{ name: 'foo' }, { name: 'bar' }], - pageInfo: { startCursor: 'foo', endCursor: 'bar' } - } + let(:client_response) do + { + data: { + search: { + nodes: [{ name: 'foo' }, { name: 'bar' }], + pageInfo: { startCursor: 'foo', endCursor: 'bar' }, + repositoryCount: 2 } } - end - - it 'fetches repos with Gitlab::GithubImport::Client (GraphQL API)' do - expect(Gitlab::GithubImport::Client) - .to receive(:new).with(access_token).and_return(client_stub) - expect(client_stub) - .to receive(:search_repos_by_name_graphql) - .with(search_text, pagination_options).and_return(client_response) - - expect(client.repos(search_text, pagination_options)).to eq( - { - repos: [{ name: 'foo' }, { name: 'bar' }], - page_info: { startCursor: 'foo', endCursor: 'bar' } - } - ) - end + } end - context 'with github_client_fetch_repos_via_graphql FF disabled' do - let(:client_response) do - { items: [{ name: 'foo' }, { name: 'bar' }] } - end - - before do - stub_feature_flags(github_client_fetch_repos_via_graphql: false) - end - - it 'fetches repos with Gitlab::GithubImport::Client (REST API)' do - expect(Gitlab::GithubImport::Client) - .to receive(:new).with(access_token).and_return(client_stub) - expect(client_stub) - .to receive(:search_repos_by_name) - .with(search_text, pagination_options).and_return(client_response) + it 'fetches repos with Gitlab::GithubImport::Client (GraphQL API)' do + expect(Gitlab::GithubImport::Client) + .to receive(:new).with(access_token).and_return(client_stub) + expect(client_stub) + .to receive(:search_repos_by_name_graphql) + .with(search_text, pagination_options).and_return(client_response) - expect(client.repos(search_text, pagination_options)).to eq( - { repos: [{ name: 'foo' }, { name: 'bar' }] } - ) - end + expect(client.repos(search_text, pagination_options)).to eq( + { + repos: [{ name: 'foo' }, { name: 'bar' }], + page_info: { startCursor: 'foo', endCursor: 'bar' }, + count: 2 + } + ) end end @@ -99,4 +81,59 @@ RSpec.describe Gitlab::GithubImport::Clients::Proxy, :manage, feature_category: end end end + + describe '#count_by', :clean_gitlab_redis_cache do + context 'when remove_legacy_github_client FF is enabled' do + let(:client_stub) { instance_double(Gitlab::GithubImport::Client) } + let(:client_response) { { data: { search: { repositoryCount: 1 } } } } + + before do + stub_feature_flags(remove_legacy_github_client: true) + end + + context 'when value is cached' do + before do + Gitlab::Cache::Import::Caching.write('github-importer/provider-repo-count/owned/user_id', 3) + end + + it 'returns repository count from cache' do + expect(Gitlab::GithubImport::Client) + .to receive(:new).with(access_token).and_return(client_stub) + expect(client_stub) + .not_to receive(:count_repos_by_relation_type_graphql) + .with({ relation_type: 'owned' }) + expect(client.count_repos_by('owned', 'user_id')).to eq(3) + end + end + + context 'when value is not cached' do + it 'returns repository count' do + expect(Gitlab::GithubImport::Client) + .to receive(:new).with(access_token).and_return(client_stub) + expect(client_stub) + .to receive(:count_repos_by_relation_type_graphql) + .with({ relation_type: 'owned' }).and_return(client_response) + expect(Gitlab::Cache::Import::Caching) + .to receive(:write) + .with('github-importer/provider-repo-count/owned/user_id', 1, timeout: 5.minutes) + .and_call_original + expect(client.count_repos_by('owned', 'user_id')).to eq(1) + end + end + end + + context 'when remove_legacy_github_client FF is disabled' do + let(:client_stub) { instance_double(Gitlab::LegacyGithubImport::Client) } + + before do + stub_feature_flags(remove_legacy_github_client: false) + end + + it 'returns nil' do + expect(Gitlab::LegacyGithubImport::Client) + .to receive(:new).with(access_token, client_options).and_return(client_stub) + expect(client.count_repos_by('owned', 'user_id')).to be_nil + end + end + end end diff --git a/spec/lib/gitlab/github_import/importer/collaborator_importer_spec.rb b/spec/lib/gitlab/github_import/importer/collaborator_importer_spec.rb new file mode 100644 index 00000000000..07c10fe57f0 --- /dev/null +++ b/spec/lib/gitlab/github_import/importer/collaborator_importer_spec.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::GithubImport::Importer::CollaboratorImporter, feature_category: :importers do + subject(:importer) { described_class.new(collaborator, project, client) } + + let_it_be(:group) { create(:group) } + let_it_be(:project) { create(:project, :repository, group: group) } + let_it_be(:user) { create(:user) } + + let(:client) { instance_double(Gitlab::GithubImport::Client) } + let(:github_user_id) { rand(1000) } + let(:collaborator) do + Gitlab::GithubImport::Representation::Collaborator.from_json_hash( + 'id' => github_user_id, + 'login' => user.username, + 'role_name' => github_role_name + ) + end + + let(:basic_member_attrs) do + { + source: project, + user_id: user.id, + member_namespace_id: project.project_namespace_id, + created_by_id: project.creator_id + }.stringify_keys + end + + describe '#execute' do + before do + allow_next_instance_of(Gitlab::GithubImport::UserFinder) do |finder| + allow(finder).to receive(:find).with(github_user_id, user.username).and_return(user.id) + end + end + + shared_examples 'role mapping' do |collaborator_role, member_access_level| + let(:github_role_name) { collaborator_role } + + it 'creates expected member' do + expect { importer.execute }.to change { project.members.count } + .from(0).to(1) + + expected_member_attrs = basic_member_attrs.merge(access_level: member_access_level) + expect(project.members.last).to have_attributes(expected_member_attrs) + end + end + + it_behaves_like 'role mapping', 'read', Gitlab::Access::GUEST + it_behaves_like 'role mapping', 'triage', Gitlab::Access::REPORTER + it_behaves_like 'role mapping', 'write', Gitlab::Access::DEVELOPER + it_behaves_like 'role mapping', 'maintain', Gitlab::Access::MAINTAINER + it_behaves_like 'role mapping', 'admin', Gitlab::Access::OWNER + + context 'when role name is unknown (custom role)' do + let(:github_role_name) { 'custom_role' } + + it 'raises expected error' do + expect { importer.execute }.to raise_exception( + ::Gitlab::GithubImport::ObjectImporter::NotRetriableError + ).with_message("Unknown GitHub role: #{github_role_name}") + end + end + + context 'when user has lower role in a project group' do + before do + create(:group_member, group: group, user: user, access_level: Gitlab::Access::DEVELOPER) + end + + it_behaves_like 'role mapping', 'maintain', Gitlab::Access::MAINTAINER + end + + context 'when user has higher role in a project group' do + let(:github_role_name) { 'write' } + + before do + create(:group_member, group: group, user: user, access_level: Gitlab::Access::MAINTAINER) + end + + it 'skips creating member for the project' do + expect { importer.execute }.not_to change { project.members.count } + end + end + end +end diff --git a/spec/lib/gitlab/github_import/importer/collaborators_importer_spec.rb b/spec/lib/gitlab/github_import/importer/collaborators_importer_spec.rb new file mode 100644 index 00000000000..e48b562279e --- /dev/null +++ b/spec/lib/gitlab/github_import/importer/collaborators_importer_spec.rb @@ -0,0 +1,119 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::GithubImport::Importer::CollaboratorsImporter, feature_category: :importers do + subject(:importer) { described_class.new(project, client, parallel: parallel) } + + let(:parallel) { true } + let(:project) { instance_double(Project, id: 4, import_source: 'foo/bar', import_state: nil) } + let(:client) { instance_double(Gitlab::GithubImport::Client) } + + let(:github_collaborator) do + { + id: 100500, + login: 'bob', + role_name: 'maintainer' + } + end + + describe '#parallel?' do + context 'when parallel option is true' do + it { expect(importer).to be_parallel } + end + + context 'when parallel option is false' do + let(:parallel) { false } + + it { expect(importer).not_to be_parallel } + end + end + + describe '#execute' do + context 'when running in parallel mode' do + it 'imports collaborators in parallel' do + expect(importer).to receive(:parallel_import) + importer.execute + end + end + + context 'when running in sequential mode' do + let(:parallel) { false } + + it 'imports collaborators in sequence' do + expect(importer).to receive(:sequential_import) + importer.execute + end + end + end + + describe '#sequential_import' do + let(:parallel) { false } + + it 'imports each collaborator in sequence' do + collaborator_importer = instance_double(Gitlab::GithubImport::Importer::CollaboratorImporter) + + allow(importer) + .to receive(:each_object_to_import) + .and_yield(github_collaborator) + + expect(Gitlab::GithubImport::Importer::CollaboratorImporter) + .to receive(:new) + .with( + an_instance_of(Gitlab::GithubImport::Representation::Collaborator), + project, + client + ) + .and_return(collaborator_importer) + + expect(collaborator_importer).to receive(:execute) + + importer.sequential_import + end + end + + describe '#parallel_import', :clean_gitlab_redis_cache do + let(:page_struct) { Struct.new(:objects, :number, keyword_init: true) } + + before do + allow(client).to receive(:each_page) + .with(:collaborators, project.import_source, { page: 1 }) + .and_yield(page_struct.new(number: 1, objects: [github_collaborator])) + end + + it 'imports each collaborator in parallel' do + expect(Gitlab::GithubImport::ImportCollaboratorWorker).to receive(:perform_in) + .with(1.second, project.id, an_instance_of(Hash), an_instance_of(String)) + + waiter = importer.parallel_import + + expect(waiter).to be_an_instance_of(Gitlab::JobWaiter) + expect(waiter.jobs_remaining).to eq(1) + end + + context 'when collaborator is already imported' do + before do + Gitlab::Cache::Import::Caching.set_add( + "github-importer/already-imported/#{project.id}/collaborators", + github_collaborator[:id] + ) + end + + it "doesn't run importer on it" do + expect(Gitlab::GithubImport::ImportCollaboratorWorker).not_to receive(:perform_in) + + waiter = importer.parallel_import + + expect(waiter).to be_an_instance_of(Gitlab::JobWaiter) + expect(waiter.jobs_remaining).to eq(0) + end + end + end + + describe '#id_for_already_imported_cache' do + it 'returns the ID of the given note' do + expect(importer.id_for_already_imported_cache(github_collaborator)) + .to eq(100500) + end + end +end diff --git a/spec/lib/gitlab/github_import/importer/label_links_importer_spec.rb b/spec/lib/gitlab/github_import/importer/label_links_importer_spec.rb index e005d8eda84..16816dfbcea 100644 --- a/spec/lib/gitlab/github_import/importer/label_links_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/label_links_importer_spec.rb @@ -44,6 +44,10 @@ RSpec.describe Gitlab::GithubImport::Importer::LabelLinksImporter do end it 'does not insert label links for non-existing labels' do + expect(importer) + .to receive(:find_target_id) + .and_return(4) + expect(importer.label_finder) .to receive(:id_for) .with('bug') @@ -55,6 +59,20 @@ RSpec.describe Gitlab::GithubImport::Importer::LabelLinksImporter do importer.create_labels end + + it 'does not insert label links for non-existing targets' do + expect(importer) + .to receive(:find_target_id) + .and_return(nil) + + expect(importer.label_finder) + .not_to receive(:id_for) + + expect(LabelLink) + .not_to receive(:bulk_insert!) + + importer.create_labels + end end describe '#find_target_id' do diff --git a/spec/lib/gitlab/github_import/importer/note_attachments_importer_spec.rb b/spec/lib/gitlab/github_import/importer/note_attachments_importer_spec.rb index 7d4e3c3bcce..450ebe9a719 100644 --- a/spec/lib/gitlab/github_import/importer/note_attachments_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/note_attachments_importer_spec.rb @@ -2,10 +2,10 @@ require 'spec_helper' -RSpec.describe Gitlab::GithubImport::Importer::NoteAttachmentsImporter do +RSpec.describe Gitlab::GithubImport::Importer::NoteAttachmentsImporter, feature_category: :importers do subject(:importer) { described_class.new(note_text, project, client) } - let_it_be(:project) { create(:project) } + let_it_be(:project) { create(:project, import_source: 'nickname/public-test-repo') } let(:note_text) { Gitlab::GithubImport::Representation::NoteText.from_db_record(record) } let(:client) { instance_double('Gitlab::GithubImport::Client') } @@ -13,6 +13,8 @@ RSpec.describe Gitlab::GithubImport::Importer::NoteAttachmentsImporter do let(:doc_url) { 'https://github.com/nickname/public-test-repo/files/9020437/git-cheat-sheet.txt' } let(:image_url) { 'https://user-images.githubusercontent.com/6833842/0cf366b61ef2.jpeg' } let(:image_tag_url) { 'https://user-images.githubusercontent.com/6833842/0cf366b61ea5.jpeg' } + let(:project_blob_url) { 'https://github.com/nickname/public-test-repo/blob/main/example.md' } + let(:other_project_blob_url) { 'https://github.com/nickname/other-repo/blob/main/README.md' } let(:text) do <<-TEXT.split("\n").map(&:strip).join("\n") Some text... @@ -20,11 +22,14 @@ RSpec.describe Gitlab::GithubImport::Importer::NoteAttachmentsImporter do [special-doc](#{doc_url})  <img width=\"248\" alt=\"tag-image\" src="#{image_tag_url}"> + + [link to project blob file](#{project_blob_url}) + [link to other project blob file](#{other_project_blob_url}) TEXT end shared_examples 'updates record description' do - it do + it 'changes attachment links' do importer.execute record.reload @@ -32,6 +37,22 @@ RSpec.describe Gitlab::GithubImport::Importer::NoteAttachmentsImporter do expect(record.description).to include(' expect(record.description).to include('<img width="248" alt="tag-image" src="/uploads') end + + it 'changes link to project blob files' do + importer.execute + + record.reload + expected_blob_link = "[link to project blob file](http://localhost/#{project.full_path}/-/blob/main/example.md)" + expect(record.description).not_to include("[link to project blob file](#{project_blob_url})") + expect(record.description).to include(expected_blob_link) + end + + it "doesn't change links to other projects" do + importer.execute + + record.reload + expect(record.description).to include("[link to other project blob file](#{other_project_blob_url})") + end end describe '#execute' do @@ -72,7 +93,7 @@ RSpec.describe Gitlab::GithubImport::Importer::NoteAttachmentsImporter do context 'when importing note attachments' do let(:record) { create(:note, project: project, note: text) } - it 'updates note text with new attachment urls' do + it 'changes note text with new attachment urls' do importer.execute record.reload @@ -80,6 +101,22 @@ RSpec.describe Gitlab::GithubImport::Importer::NoteAttachmentsImporter do expect(record.note).to include(' expect(record.note).to include('<img width="248" alt="tag-image" src="/uploads') end + + it 'changes note links to project blob files' do + importer.execute + + record.reload + expected_blob_link = "[link to project blob file](http://localhost/#{project.full_path}/-/blob/main/example.md)" + expect(record.note).not_to include("[link to project blob file](#{project_blob_url})") + expect(record.note).to include(expected_blob_link) + end + + it "doesn't change note links to other projects" do + importer.execute + + record.reload + expect(record.note).to include("[link to other project blob file](#{other_project_blob_url})") + end end end end diff --git a/spec/lib/gitlab/github_import/markdown/attachment_spec.rb b/spec/lib/gitlab/github_import/markdown/attachment_spec.rb index 588a3076f59..84b0886ebcc 100644 --- a/spec/lib/gitlab/github_import/markdown/attachment_spec.rb +++ b/spec/lib/gitlab/github_import/markdown/attachment_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::GithubImport::Markdown::Attachment do +RSpec.describe Gitlab::GithubImport::Markdown::Attachment, feature_category: :importers do let(:name) { FFaker::Lorem.word } let(:url) { FFaker::Internet.uri('https') } @@ -101,6 +101,62 @@ RSpec.describe Gitlab::GithubImport::Markdown::Attachment do end end + describe '#part_of_project_blob?' do + let(:attachment) { described_class.new('test', url) } + let(:import_source) { 'nickname/public-test-repo' } + + context 'when url is a part of project blob' do + let(:url) { "https://github.com/#{import_source}/blob/main/example.md" } + + it { expect(attachment.part_of_project_blob?(import_source)).to eq true } + end + + context 'when url is not a part of project blob' do + let(:url) { "https://github.com/#{import_source}/files/9020437/git-cheat-sheet.txt" } + + it { expect(attachment.part_of_project_blob?(import_source)).to eq false } + end + end + + describe '#doc_belongs_to_project?' do + let(:attachment) { described_class.new('test', url) } + let(:import_source) { 'nickname/public-test-repo' } + + context 'when url relates to this project' do + let(:url) { "https://github.com/#{import_source}/files/9020437/git-cheat-sheet.txt" } + + it { expect(attachment.doc_belongs_to_project?(import_source)).to eq true } + end + + context 'when url is not related to this project' do + let(:url) { 'https://github.com/nickname/other-repo/files/9020437/git-cheat-sheet.txt' } + + it { expect(attachment.doc_belongs_to_project?(import_source)).to eq false } + end + + context 'when url is a part of project blob' do + let(:url) { "https://github.com/#{import_source}/blob/main/example.md" } + + it { expect(attachment.doc_belongs_to_project?(import_source)).to eq false } + end + end + + describe '#media?' do + let(:attachment) { described_class.new('test', url) } + + context 'when it is a media link' do + let(:url) { 'https://user-images.githubusercontent.com/6833842/0cf366b61ef2.jpeg' } + + it { expect(attachment.media?).to eq true } + end + + context 'when it is not a media link' do + let(:url) { 'https://github.com/nickname/public-test-repo/files/9020437/git-cheat-sheet.txt' } + + it { expect(attachment.media?).to eq false } + end + end + describe '#inspect' do it 'returns attachment basic info' do attachment = described_class.new(name, url) diff --git a/spec/lib/gitlab/github_import/parallel_scheduling_spec.rb b/spec/lib/gitlab/github_import/parallel_scheduling_spec.rb index c351ead91eb..9de39a3ff7e 100644 --- a/spec/lib/gitlab/github_import/parallel_scheduling_spec.rb +++ b/spec/lib/gitlab/github_import/parallel_scheduling_spec.rb @@ -289,77 +289,52 @@ RSpec.describe Gitlab::GithubImport::ParallelScheduling, feature_category: :impo .and_return({ title: 'One' }, { title: 'Two' }, { title: 'Three' }) end - context 'with multiple objects' do - before do - stub_feature_flags(improved_spread_parallel_import: false) - - expect(importer).to receive(:each_object_to_import).and_yield(object).and_yield(object).and_yield(object) - end - - it 'imports data in parallel batches with delays' do - expect(worker_class).to receive(:bulk_perform_in) - .with(1.second, [ - [project.id, { title: 'One' }, an_instance_of(String)], - [project.id, { title: 'Two' }, an_instance_of(String)], - [project.id, { title: 'Three' }, an_instance_of(String)] - ], batch_size: batch_size, batch_delay: batch_delay) - - importer.parallel_import - end + it 'imports data in parallel with delays respecting parallel_import_batch definition and return job waiter' do + allow(::Gitlab::JobWaiter).to receive(:generate_key).and_return('waiter-key') + allow(importer).to receive(:parallel_import_batch).and_return({ size: 2, delay: 1.minute }) + + expect(importer).to receive(:each_object_to_import) + .and_yield(object).and_yield(object).and_yield(object) + expect(worker_class).to receive(:perform_in) + .with(1.second, project.id, { title: 'One' }, 'waiter-key').ordered + expect(worker_class).to receive(:perform_in) + .with(1.second, project.id, { title: 'Two' }, 'waiter-key').ordered + expect(worker_class).to receive(:perform_in) + .with(1.minute + 1.second, project.id, { title: 'Three' }, 'waiter-key').ordered + + job_waiter = importer.parallel_import + + expect(job_waiter.key).to eq('waiter-key') + expect(job_waiter.jobs_remaining).to eq(3) end - context 'when the feature flag `improved_spread_parallel_import` is enabled' do + context 'when job restarts due to API rate limit or Sidekiq interruption' do before do - stub_feature_flags(improved_spread_parallel_import: true) + cache_key = format(described_class::JOB_WAITER_CACHE_KEY, + project: project.id, collection: importer.collection_method) + Gitlab::Cache::Import::Caching.write(cache_key, 'waiter-key') + + cache_key = format(described_class::JOB_WAITER_REMAINING_CACHE_KEY, + project: project.id, collection: importer.collection_method) + Gitlab::Cache::Import::Caching.write(cache_key, 3) end - it 'imports data in parallel with delays respecting parallel_import_batch definition and return job waiter' do - allow(::Gitlab::JobWaiter).to receive(:generate_key).and_return('waiter-key') - allow(importer).to receive(:parallel_import_batch).and_return({ size: 2, delay: 1.minute }) + it "restores job waiter's key and jobs_remaining" do + allow(importer).to receive(:parallel_import_batch).and_return({ size: 1, delay: 1.minute }) + + expect(importer).to receive(:each_object_to_import).and_yield(object).and_yield(object).and_yield(object) - expect(importer).to receive(:each_object_to_import) - .and_yield(object).and_yield(object).and_yield(object) expect(worker_class).to receive(:perform_in) .with(1.second, project.id, { title: 'One' }, 'waiter-key').ordered expect(worker_class).to receive(:perform_in) - .with(1.second, project.id, { title: 'Two' }, 'waiter-key').ordered + .with(1.minute + 1.second, project.id, { title: 'Two' }, 'waiter-key').ordered expect(worker_class).to receive(:perform_in) - .with(1.minute + 1.second, project.id, { title: 'Three' }, 'waiter-key').ordered + .with(2.minutes + 1.second, project.id, { title: 'Three' }, 'waiter-key').ordered job_waiter = importer.parallel_import expect(job_waiter.key).to eq('waiter-key') - expect(job_waiter.jobs_remaining).to eq(3) - end - - context 'when job restarts due to API rate limit or Sidekiq interruption' do - before do - cache_key = format(described_class::JOB_WAITER_CACHE_KEY, - project: project.id, collection: importer.collection_method) - Gitlab::Cache::Import::Caching.write(cache_key, 'waiter-key') - - cache_key = format(described_class::JOB_WAITER_REMAINING_CACHE_KEY, - project: project.id, collection: importer.collection_method) - Gitlab::Cache::Import::Caching.write(cache_key, 3) - end - - it "restores job waiter's key and jobs_remaining" do - allow(importer).to receive(:parallel_import_batch).and_return({ size: 1, delay: 1.minute }) - - expect(importer).to receive(:each_object_to_import).and_yield(object).and_yield(object).and_yield(object) - - expect(worker_class).to receive(:perform_in) - .with(1.second, project.id, { title: 'One' }, 'waiter-key').ordered - expect(worker_class).to receive(:perform_in) - .with(1.minute + 1.second, project.id, { title: 'Two' }, 'waiter-key').ordered - expect(worker_class).to receive(:perform_in) - .with(2.minutes + 1.second, project.id, { title: 'Three' }, 'waiter-key').ordered - - job_waiter = importer.parallel_import - - expect(job_waiter.key).to eq('waiter-key') - expect(job_waiter.jobs_remaining).to eq(6) - end + expect(job_waiter.jobs_remaining).to eq(6) end end end diff --git a/spec/lib/gitlab/github_import/project_relation_type_spec.rb b/spec/lib/gitlab/github_import/project_relation_type_spec.rb new file mode 100644 index 00000000000..419cb6de121 --- /dev/null +++ b/spec/lib/gitlab/github_import/project_relation_type_spec.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::GithubImport::ProjectRelationType, :manage, feature_category: :importers do + subject(:project_relation_type) { described_class.new(client) } + + let(:octokit) { instance_double(Octokit::Client) } + let(:client) do + instance_double(Gitlab::GithubImport::Clients::Proxy, octokit: octokit, user: { login: 'nickname' }) + end + + describe '#for', :use_clean_rails_redis_caching do + before do + allow(client).to receive(:each_object).with(:organizations).and_yield({ login: 'great-org' }) + allow(octokit).to receive(:access_token).and_return('stub') + end + + context "when it's user owned repo" do + let(:import_source) { 'nickname/repo_name' } + + it { expect(project_relation_type.for(import_source)).to eq 'owned' } + end + + context "when it's organization repo" do + let(:import_source) { 'great-org/repo_name' } + + it { expect(project_relation_type.for(import_source)).to eq 'organization' } + end + + context "when it's user collaborated repo" do + let(:import_source) { 'some-another-namespace/repo_name' } + + it { expect(project_relation_type.for(import_source)).to eq 'collaborated' } + end + + context 'with cache' do + let(:import_source) { 'some-another-namespace/repo_name' } + + it 'calls client only once during 5 minutes timeframe', :request_store do + expect(project_relation_type.for(import_source)).to eq 'collaborated' + expect(project_relation_type.for('another/repo')).to eq 'collaborated' + + expect(client).to have_received(:each_object).once + expect(client).to have_received(:user).once + end + end + end +end diff --git a/spec/lib/gitlab/github_import/representation/collaborator_spec.rb b/spec/lib/gitlab/github_import/representation/collaborator_spec.rb new file mode 100644 index 00000000000..d5952f9459b --- /dev/null +++ b/spec/lib/gitlab/github_import/representation/collaborator_spec.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +RSpec.describe Gitlab::GithubImport::Representation::Collaborator, feature_category: :importers do + shared_examples 'a Collaborator' do + it 'returns an instance of Collaborator' do + expect(collaborator).to be_an_instance_of(described_class) + end + + context 'with Collaborator' do + it 'includes the user ID' do + expect(collaborator.id).to eq(42) + end + + it 'includes the username' do + expect(collaborator.login).to eq('alice') + end + + it 'includes the role' do + expect(collaborator.role_name).to eq('maintainer') + end + end + end + + describe '.from_api_response' do + it_behaves_like 'a Collaborator' do + let(:response) { { id: 42, login: 'alice', role_name: 'maintainer' } } + let(:collaborator) { described_class.from_api_response(response) } + end + end + + describe '.from_json_hash' do + it_behaves_like 'a Collaborator' do + let(:hash) { { 'id' => 42, 'login' => 'alice', role_name: 'maintainer' } } + let(:collaborator) { described_class.from_json_hash(hash) } + end + end +end diff --git a/spec/lib/gitlab/i18n/pluralization_spec.rb b/spec/lib/gitlab/i18n/pluralization_spec.rb new file mode 100644 index 00000000000..857562d549c --- /dev/null +++ b/spec/lib/gitlab/i18n/pluralization_spec.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' +require 'rspec-parameterized' +require 'gettext_i18n_rails' + +RSpec.describe Gitlab::I18n::Pluralization, feature_category: :internationalization do + describe '.call' do + subject(:rule) { described_class.call(1) } + + context 'with available locales' do + around do |example| + Gitlab::I18n.with_locale(locale, &example) + end + + where(:locale) do + Gitlab::I18n.available_locales + end + + with_them do + it 'supports pluralization' do + expect(rule).not_to be_nil + end + end + + context 'with missing rules' do + let(:locale) { "pl_PL" } + + before do + stub_const("#{described_class}::MAP", described_class::MAP.except(locale)) + end + + it 'raises an ArgumentError' do + expect { rule }.to raise_error(ArgumentError, + /Missing pluralization rule for locale "#{locale}"/ + ) + end + end + end + end + + describe '.install_on' do + let(:mod) { Module.new } + + before do + described_class.install_on(mod) + end + + it 'adds pluralisation_rule method' do + expect(mod.pluralisation_rule).to eq(described_class) + end + end +end diff --git a/spec/lib/gitlab/i18n_spec.rb b/spec/lib/gitlab/i18n_spec.rb index b752d89bf0d..ee92831922d 100644 --- a/spec/lib/gitlab/i18n_spec.rb +++ b/spec/lib/gitlab/i18n_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::I18n do +RSpec.describe Gitlab::I18n, feature_category: :internationalization do let(:user) { create(:user, preferred_language: :es) } describe '.selectable_locales' do @@ -47,4 +47,19 @@ RSpec.describe Gitlab::I18n do expect(::I18n.locale).to eq(:en) end end + + describe '.pluralisation_rule' do + context 'when overridden' do + before do + # Internally, FastGettext sets + # Thread.current[:fast_gettext_pluralisation_rule]. + # Our patch patches `FastGettext.pluralisation_rule` instead. + FastGettext.pluralisation_rule = :something + end + + it 'returns custom definition regardless' do + expect(FastGettext.pluralisation_rule).to eq(Gitlab::I18n::Pluralization) + end + end + end end diff --git a/spec/lib/gitlab/import/errors_spec.rb b/spec/lib/gitlab/import/errors_spec.rb new file mode 100644 index 00000000000..f89cb36bbb4 --- /dev/null +++ b/spec/lib/gitlab/import/errors_spec.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Import::Errors, feature_category: :importers do + let_it_be(:project) { create(:project) } + + describe '.merge_nested_errors' do + it 'merges nested collection errors' do + issue = project.issues.new( + title: 'test', + notes: [ + Note.new( + award_emoji: [AwardEmoji.new(name: 'test')] + ) + ], + sentry_issue: SentryIssue.new + ) + + issue.validate + + expect(issue.errors.full_messages) + .to contain_exactly( + "Author can't be blank", + "Notes is invalid", + "Sentry issue sentry issue identifier can't be blank" + ) + + described_class.merge_nested_errors(issue) + + expect(issue.errors.full_messages) + .to contain_exactly( + "Notes is invalid", + "Author can't be blank", + "Sentry issue sentry issue identifier can't be blank", + "Award emoji is invalid", + "Note can't be blank", + "Project can't be blank", + "Noteable can't be blank", + "Author can't be blank", + "Project does not match noteable project", + "User can't be blank", + "Awardable can't be blank", + "Name is not a valid emoji name" + ) + end + end +end diff --git a/spec/lib/gitlab/import/metrics_spec.rb b/spec/lib/gitlab/import/metrics_spec.rb index 9b8b58d00f3..1a988af0dbd 100644 --- a/spec/lib/gitlab/import/metrics_spec.rb +++ b/spec/lib/gitlab/import/metrics_spec.rb @@ -11,7 +11,6 @@ RSpec.describe Gitlab::Import::Metrics, :aggregate_failures do subject { described_class.new(importer, project) } before do - allow(Gitlab::Metrics).to receive(:counter) { counter } allow(counter).to receive(:increment) allow(histogram).to receive(:observe) end @@ -42,6 +41,13 @@ RSpec.describe Gitlab::Import::Metrics, :aggregate_failures do context 'when project is not a github import' do it 'does not emit importer metrics' do expect(subject).not_to receive(:track_usage_event) + expect_no_snowplow_event( + category: :test_importer, + action: 'create', + label: 'github_import_project_state', + project: project, + extra: { import_type: 'github', state: 'failed' } + ) subject.track_failed_import end @@ -50,39 +56,81 @@ RSpec.describe Gitlab::Import::Metrics, :aggregate_failures do context 'when project is a github import' do before do project.import_type = 'github' + allow(project).to receive(:import_status).and_return('failed') end it 'emits importer metrics' do expect(subject).to receive(:track_usage_event).with(:github_import_project_failure, project.id) subject.track_failed_import + + expect_snowplow_event( + category: :test_importer, + action: 'create', + label: 'github_import_project_state', + project: project, + extra: { import_type: 'github', state: 'failed' } + ) end end end describe '#track_finished_import' do - before do - allow(Gitlab::Metrics).to receive(:histogram) { histogram } - end + context 'when project is a github import' do + before do + project.import_type = 'github' + allow(Gitlab::Metrics).to receive(:counter) { counter } + allow(Gitlab::Metrics).to receive(:histogram) { histogram } + allow(project).to receive(:beautified_import_status_name).and_return('completed') + end - it 'emits importer metrics' do - expect(Gitlab::Metrics).to receive(:counter).with( - :test_importer_imported_projects_total, - 'The number of imported projects' - ) + it 'emits importer metrics' do + expect(Gitlab::Metrics).to receive(:counter).with( + :test_importer_imported_projects_total, + 'The number of imported projects' + ) - expect(Gitlab::Metrics).to receive(:histogram).with( - :test_importer_total_duration_seconds, - 'Total time spent importing projects, in seconds', - {}, - described_class::IMPORT_DURATION_BUCKETS - ) + expect(Gitlab::Metrics).to receive(:histogram).with( + :test_importer_total_duration_seconds, + 'Total time spent importing projects, in seconds', + {}, + described_class::IMPORT_DURATION_BUCKETS + ) + + expect(counter).to receive(:increment) - expect(counter).to receive(:increment) + subject.track_finished_import - subject.track_finished_import + expect_snowplow_event( + category: :test_importer, + action: 'create', + label: 'github_import_project_state', + project: project, + extra: { import_type: 'github', state: 'completed' } + ) + + expect(subject.duration).not_to be_nil + end - expect(subject.duration).not_to be_nil + context 'when import is partially completed' do + before do + allow(project).to receive(:beautified_import_status_name).and_return('partially completed') + end + + it 'emits snowplow metrics' do + expect(subject).to receive(:track_usage_event).with(:github_import_project_partially_completed, project.id) + + subject.track_finished_import + + expect_snowplow_event( + category: :test_importer, + action: 'create', + label: 'github_import_project_state', + project: project, + extra: { import_type: 'github', state: 'partially completed' } + ) + end + end end context 'when project is not a github import' do @@ -91,7 +139,51 @@ RSpec.describe Gitlab::Import::Metrics, :aggregate_failures do subject.track_finished_import - expect(histogram).to have_received(:observe).with({ importer: :test_importer }, anything) + expect_no_snowplow_event( + category: :test_importer, + action: 'create', + label: 'github_import_project_state', + project: project, + extra: { import_type: 'github', state: 'completed' } + ) + end + end + end + + describe '#track_cancelled_import' do + context 'when project is not a github import' do + it 'does not emit importer metrics' do + expect(subject).not_to receive(:track_usage_event) + expect_no_snowplow_event( + category: :test_importer, + action: 'create', + label: 'github_import_project_state', + project: project, + extra: { import_type: 'github', state: 'canceled' } + ) + + subject.track_canceled_import + end + end + + context 'when project is a github import' do + before do + project.import_type = 'github' + allow(project).to receive(:import_status).and_return('canceled') + end + + it 'emits importer metrics' do + expect(subject).to receive(:track_usage_event).with(:github_import_project_cancelled, project.id) + + subject.track_canceled_import + + expect_snowplow_event( + category: :test_importer, + action: 'create', + label: 'github_import_project_state', + project: project, + extra: { import_type: 'github', state: 'canceled' } + ) end end end diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index 0c2c3ffc664..f6d6a791e8c 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -92,6 +92,20 @@ notes: - suggestions - diff_note_positions - review +commit_notes: +- award_emoji +- noteable +- author +- updated_by +- last_edited_by +- resolved_by +- todos +- events +- system_note_metadata +- note_diff_file +- suggestions +- diff_note_positions +- review label_links: - target - label @@ -283,6 +297,7 @@ ci_pipelines: - job_artifacts - vulnerabilities_finding_pipelines - vulnerability_findings +- vulnerability_state_transitions - pipeline_config - security_scans - security_findings @@ -317,6 +332,7 @@ stages: - processables - builds - bridges +- generic_commit_statuses - latest_statuses - retried_statuses statuses: @@ -327,6 +343,92 @@ statuses: - auto_canceled_by - needs - ci_stage +builds: +- user +- auto_canceled_by +- ci_stage +- needs +- resource +- pipeline +- sourced_pipeline +- resource_group +- metadata +- runner +- trigger_request +- erased_by +- deployment +- pending_state +- queuing_entry +- runtime_metadata +- trace_chunks +- report_results +- namespace +- job_artifacts +- job_variables +- sourced_pipelines +- pages_deployments +- job_artifacts_archive +- job_artifacts_metadata +- job_artifacts_trace +- job_artifacts_junit +- job_artifacts_sast +- job_artifacts_dependency_scanning +- job_artifacts_container_scanning +- job_artifacts_dast +- job_artifacts_codequality +- job_artifacts_license_scanning +- job_artifacts_performance +- job_artifacts_metrics +- job_artifacts_metrics_referee +- job_artifacts_network_referee +- job_artifacts_lsif +- job_artifacts_dotenv +- job_artifacts_cobertura +- job_artifacts_terraform +- job_artifacts_accessibility +- job_artifacts_cluster_applications +- job_artifacts_secret_detection +- job_artifacts_requirements +- job_artifacts_coverage_fuzzing +- job_artifacts_browser_performance +- job_artifacts_load_performance +- job_artifacts_api_fuzzing +- job_artifacts_cluster_image_scanning +- job_artifacts_cyclonedx +- job_artifacts_requirements_v2 +- runner_machine +- runner_machine_build +- runner_session +- trace_metadata +- terraform_state_versions +- taggings +- base_tags +- tag_taggings +- tags +- security_scans +- dast_site_profiles_build +- dast_site_profile +- dast_scanner_profiles_build +- dast_scanner_profile +bridges: +- user +- pipeline +- auto_canceled_by +- ci_stage +- needs +- resource +- sourced_pipeline +- resource_group +- metadata +- trigger_request +- downstream_pipeline +- upstream_pipeline +generic_commit_statuses: +- user +- pipeline +- auto_canceled_by +- ci_stage +- needs variables: - project triggers: @@ -408,6 +510,7 @@ project: - project_namespace - management_clusters - boards +- application_setting - last_event - integrations - push_hooks_integrations @@ -432,6 +535,7 @@ project: - discord_integration - drone_ci_integration - emails_on_push_integration +- google_play_integration - pipelines_email_integration - mattermost_slash_commands_integration - shimo_integration @@ -466,6 +570,7 @@ project: - external_wiki_integration - mock_ci_integration - mock_monitoring_integration +- squash_tm_integration - forked_to_members - forked_from_project - forks @@ -600,7 +705,7 @@ project: - project_registry - packages - package_files -- repository_files +- rpm_repository_files - packages_cleanup_policy - alerting_setting - project_setting @@ -618,6 +723,7 @@ project: - upstream_project_subscriptions - downstream_project_subscriptions - service_desk_setting +- service_desk_custom_email_verification - security_setting - import_failures - container_expiration_policy @@ -859,6 +965,8 @@ bulk_import_export: - group service_desk_setting: - file_template_project +service_desk_custom_email_verification: + - triggerer approvals: - user - merge_request diff --git a/spec/lib/gitlab/import_export/attribute_configuration_spec.rb b/spec/lib/gitlab/import_export/attribute_configuration_spec.rb index 572f809e43b..1d84cba3825 100644 --- a/spec/lib/gitlab/import_export/attribute_configuration_spec.rb +++ b/spec/lib/gitlab/import_export/attribute_configuration_spec.rb @@ -9,7 +9,7 @@ require 'spec_helper' # to be included as part of the export, or blacklist them using the import_export.yml configuration file. # Likewise, new models added to import_export.yml, will need to be added with their correspondent attributes # to this spec. -RSpec.describe 'Import/Export attribute configuration' do +RSpec.describe 'Import/Export attribute configuration', feature_category: :importers do include ConfigurationHelper let(:safe_attributes_file) { 'spec/lib/gitlab/import_export/safe_model_attributes.yml' } diff --git a/spec/lib/gitlab/import_export/attributes_finder_spec.rb b/spec/lib/gitlab/import_export/attributes_finder_spec.rb index 6536b895b2f..767b7a3c84e 100644 --- a/spec/lib/gitlab/import_export/attributes_finder_spec.rb +++ b/spec/lib/gitlab/import_export/attributes_finder_spec.rb @@ -2,7 +2,7 @@ require 'fast_spec_helper' -RSpec.describe Gitlab::ImportExport::AttributesFinder do +RSpec.describe Gitlab::ImportExport::AttributesFinder, feature_category: :importers do describe '#find_root' do subject { described_class.new(config: config).find_root(model_key) } @@ -207,6 +207,19 @@ RSpec.describe Gitlab::ImportExport::AttributesFinder do it { is_expected.to be_nil } end + + context 'when include_import_only_tree is true' do + subject { described_class.new(config: config).find_relations_tree(model_key, include_import_only_tree: true) } + + let(:config) do + { + tree: { project: { ci_pipelines: { stages: { builds: nil } } } }, + import_only_tree: { project: { ci_pipelines: { stages: { statuses: nil } } } } + } + end + + it { is_expected.to eq({ ci_pipelines: { stages: { builds: nil, statuses: nil } } }) } + end end describe '#find_excluded_keys' do diff --git a/spec/lib/gitlab/import_export/attributes_permitter_spec.rb b/spec/lib/gitlab/import_export/attributes_permitter_spec.rb index c748f966463..8089b40cae8 100644 --- a/spec/lib/gitlab/import_export/attributes_permitter_spec.rb +++ b/spec/lib/gitlab/import_export/attributes_permitter_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::ImportExport::AttributesPermitter do +RSpec.describe Gitlab::ImportExport::AttributesPermitter, feature_category: :importers do let(:yml_config) do <<-EOF tree: @@ -12,6 +12,15 @@ RSpec.describe Gitlab::ImportExport::AttributesPermitter do - milestones: - events: - :push_event_payload + - ci_pipelines: + - stages: + - :builds + + import_only_tree: + project: + - ci_pipelines: + - stages: + - :statuses included_attributes: labels: @@ -43,12 +52,16 @@ RSpec.describe Gitlab::ImportExport::AttributesPermitter do it 'builds permitted attributes hash' do expect(subject.permitted_attributes).to match( a_hash_including( - project: [:labels, :milestones], + project: [:labels, :milestones, :ci_pipelines], labels: [:priorities, :title, :description, :type], events: [:push_event_payload], milestones: [:events], priorities: [], - push_event_payload: [] + push_event_payload: [], + ci_pipelines: [:stages], + stages: [:builds, :statuses], + statuses: [], + builds: [] ) ) end @@ -129,6 +142,9 @@ RSpec.describe Gitlab::ImportExport::AttributesPermitter do :external_pull_request | true :external_pull_requests | true :statuses | true + :builds | true + :generic_commit_statuses | true + :bridges | true :ci_pipelines | true :stages | true :actions | true diff --git a/spec/lib/gitlab/import_export/base/relation_object_saver_spec.rb b/spec/lib/gitlab/import_export/base/relation_object_saver_spec.rb index a8b4b9a6f05..e42a1d0ff8b 100644 --- a/spec/lib/gitlab/import_export/base/relation_object_saver_spec.rb +++ b/spec/lib/gitlab/import_export/base/relation_object_saver_spec.rb @@ -82,24 +82,13 @@ RSpec.describe Gitlab::ImportExport::Base::RelationObjectSaver, feature_category it 'saves valid subrelations and logs invalid subrelation' do expect(relation_object.notes).to receive(:<<).twice.and_call_original expect(relation_object).to receive(:save).and_call_original - expect(Gitlab::Import::Logger) - .to receive(:info) - .with( - message: '[Project/Group Import] Invalid subrelation', - project_id: project.id, - relation_key: 'issues', - error_messages: "Project does not match noteable project" - ) saver.execute issue = project.issues.last - import_failure = project.import_failures.last expect(invalid_note.persisted?).to eq(false) expect(issue.notes.count).to eq(5) - expect(import_failure.source).to eq('RelationObjectSaver#save!') - expect(import_failure.exception_message).to eq('Project does not match noteable project') end context 'when invalid subrelation can still be persisted' do @@ -111,7 +100,6 @@ RSpec.describe Gitlab::ImportExport::Base::RelationObjectSaver, feature_category it 'saves the subrelation' do expect(approval_1.valid?).to eq(false) - expect(Gitlab::Import::Logger).not_to receive(:info) saver.execute @@ -128,24 +116,10 @@ RSpec.describe Gitlab::ImportExport::Base::RelationObjectSaver, feature_category let(:invalid_priority) { build(:label_priority, priority: -1) } let(:relation_object) { build(:group_label, group: importable, title: 'test', priorities: valid_priorities + [invalid_priority]) } - it 'logs invalid subrelation for a group' do - expect(Gitlab::Import::Logger) - .to receive(:info) - .with( - message: '[Project/Group Import] Invalid subrelation', - group_id: importable.id, - relation_key: 'labels', - error_messages: 'Priority must be greater than or equal to 0' - ) - + it 'saves relation without invalid subrelations' do saver.execute - label = importable.labels.last - import_failure = importable.import_failures.last - - expect(label.priorities.count).to eq(5) - expect(import_failure.source).to eq('RelationObjectSaver#save!') - expect(import_failure.exception_message).to eq('Priority must be greater than or equal to 0') + expect(importable.labels.last.priorities.count).to eq(5) end end end diff --git a/spec/lib/gitlab/import_export/command_line_util_spec.rb b/spec/lib/gitlab/import_export/command_line_util_spec.rb index f47f1ab58a8..91cfab1688a 100644 --- a/spec/lib/gitlab/import_export/command_line_util_spec.rb +++ b/spec/lib/gitlab/import_export/command_line_util_spec.rb @@ -2,13 +2,14 @@ require 'spec_helper' -RSpec.describe Gitlab::ImportExport::CommandLineUtil do +RSpec.describe Gitlab::ImportExport::CommandLineUtil, feature_category: :importers do include ExportFileHelper let(:path) { "#{Dir.tmpdir}/symlink_test" } let(:archive) { 'spec/fixtures/symlink_export.tar.gz' } let(:shared) { Gitlab::ImportExport::Shared.new(nil) } let(:tmpdir) { Dir.mktmpdir } + let(:archive_dir) { Dir.mktmpdir } subject do Class.new do @@ -25,20 +26,38 @@ RSpec.describe Gitlab::ImportExport::CommandLineUtil do before do FileUtils.mkdir_p(path) - subject.untar_zxf(archive: archive, dir: path) end after do FileUtils.rm_rf(path) + FileUtils.rm_rf(archive_dir) FileUtils.remove_entry(tmpdir) end - it 'has the right mask for project.json' do - expect(file_permissions("#{path}/project.json")).to eq(0755) # originally 777 - end - - it 'has the right mask for uploads' do - expect(file_permissions("#{path}/uploads")).to eq(0755) # originally 555 + shared_examples 'deletes symlinks' do |compression, decompression| + it 'deletes the symlinks', :aggregate_failures do + Dir.mkdir("#{tmpdir}/.git") + Dir.mkdir("#{tmpdir}/folder") + FileUtils.touch("#{tmpdir}/file.txt") + FileUtils.touch("#{tmpdir}/folder/file.txt") + FileUtils.touch("#{tmpdir}/.gitignore") + FileUtils.touch("#{tmpdir}/.git/config") + File.symlink('file.txt', "#{tmpdir}/.symlink") + File.symlink('file.txt', "#{tmpdir}/.git/.symlink") + File.symlink('file.txt', "#{tmpdir}/folder/.symlink") + archive = File.join(archive_dir, 'archive') + subject.public_send(compression, archive: archive, dir: tmpdir) + + subject.public_send(decompression, archive: archive, dir: archive_dir) + + expect(File.exist?("#{archive_dir}/file.txt")).to eq(true) + expect(File.exist?("#{archive_dir}/folder/file.txt")).to eq(true) + expect(File.exist?("#{archive_dir}/.gitignore")).to eq(true) + expect(File.exist?("#{archive_dir}/.git/config")).to eq(true) + expect(File.exist?("#{archive_dir}/.symlink")).to eq(false) + expect(File.exist?("#{archive_dir}/.git/.symlink")).to eq(false) + expect(File.exist?("#{archive_dir}/folder/.symlink")).to eq(false) + end end describe '#download_or_copy_upload' do @@ -228,12 +247,6 @@ RSpec.describe Gitlab::ImportExport::CommandLineUtil do end describe '#tar_cf' do - let(:archive_dir) { Dir.mktmpdir } - - after do - FileUtils.remove_entry(archive_dir) - end - it 'archives a folder without compression' do archive_file = File.join(archive_dir, 'archive.tar') @@ -256,12 +269,24 @@ RSpec.describe Gitlab::ImportExport::CommandLineUtil do end end - describe '#untar_xf' do - let(:archive_dir) { Dir.mktmpdir } + describe '#untar_zxf' do + it_behaves_like 'deletes symlinks', :tar_czf, :untar_zxf - after do - FileUtils.remove_entry(archive_dir) + it 'has the right mask for project.json' do + subject.untar_zxf(archive: archive, dir: path) + + expect(file_permissions("#{path}/project.json")).to eq(0755) # originally 777 + end + + it 'has the right mask for uploads' do + subject.untar_zxf(archive: archive, dir: path) + + expect(file_permissions("#{path}/uploads")).to eq(0755) # originally 555 end + end + + describe '#untar_xf' do + it_behaves_like 'deletes symlinks', :tar_cf, :untar_xf it 'extracts archive without decompression' do filename = 'archive.tar.gz' diff --git a/spec/lib/gitlab/import_export/config_spec.rb b/spec/lib/gitlab/import_export/config_spec.rb index 8f848af8bd3..2a52a0a2ff2 100644 --- a/spec/lib/gitlab/import_export/config_spec.rb +++ b/spec/lib/gitlab/import_export/config_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::ImportExport::Config do +RSpec.describe Gitlab::ImportExport::Config, feature_category: :importers do let(:yaml_file) { described_class.new } describe '#to_h' do @@ -21,7 +21,9 @@ RSpec.describe Gitlab::ImportExport::Config do end it 'parses default config' do - expected_keys = [:tree, :excluded_attributes, :included_attributes, :methods, :preloads, :export_reorders] + expected_keys = [ + :tree, :import_only_tree, :excluded_attributes, :included_attributes, :methods, :preloads, :export_reorders + ] expected_keys << :include_if_exportable if ee expect { subject }.not_to raise_error @@ -82,7 +84,7 @@ RSpec.describe Gitlab::ImportExport::Config do EOF end - let(:config_hash) { YAML.safe_load(config, [Symbol]) } + let(:config_hash) { YAML.safe_load(config, permitted_classes: [Symbol]) } before do allow_any_instance_of(described_class).to receive(:parse_yaml) do @@ -110,6 +112,7 @@ RSpec.describe Gitlab::ImportExport::Config do } } }, + import_only_tree: {}, included_attributes: { user: [:id] }, @@ -153,6 +156,7 @@ RSpec.describe Gitlab::ImportExport::Config do } } }, + import_only_tree: {}, included_attributes: { user: [:id, :name_ee] }, diff --git a/spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb b/spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb index f18d9e64f52..02419267f0e 100644 --- a/spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb +++ b/spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::ImportExport::FastHashSerializer, :with_license do +RSpec.describe Gitlab::ImportExport::FastHashSerializer, :with_license, feature_category: :importers do # FastHashSerializer#execute generates the hash which is not easily accessible # and includes `JSONBatchRelation` items which are serialized at this point. # Wrapping the result into JSON generating/parsing is for making @@ -125,13 +125,13 @@ RSpec.describe Gitlab::ImportExport::FastHashSerializer, :with_license do expect(subject.dig('ci_pipelines', 0, 'stages')).not_to be_empty end - it 'has pipeline statuses' do - expect(subject.dig('ci_pipelines', 0, 'stages', 0, 'statuses')).not_to be_empty + it 'has pipeline builds' do + expect(subject.dig('ci_pipelines', 0, 'stages', 0, 'builds')).not_to be_empty end it 'has pipeline builds' do builds_count = subject - .dig('ci_pipelines', 0, 'stages', 0, 'statuses') + .dig('ci_pipelines', 0, 'stages', 0, 'builds') .count { |hash| hash['type'] == 'Ci::Build' } expect(builds_count).to eq(1) @@ -141,8 +141,8 @@ RSpec.describe Gitlab::ImportExport::FastHashSerializer, :with_license do expect(subject['ci_pipelines']).not_to be_empty end - it 'has ci pipeline notes' do - expect(subject['ci_pipelines'].first['notes']).not_to be_empty + it 'has commit notes' do + expect(subject['commit_notes']).not_to be_empty end it 'has labels with no associations' do diff --git a/spec/lib/gitlab/import_export/group/relation_tree_restorer_spec.rb b/spec/lib/gitlab/import_export/group/relation_tree_restorer_spec.rb index 5e84284a060..07971d6271c 100644 --- a/spec/lib/gitlab/import_export/group/relation_tree_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/group/relation_tree_restorer_spec.rb @@ -9,7 +9,7 @@ require 'spec_helper' -RSpec.describe Gitlab::ImportExport::Group::RelationTreeRestorer do +RSpec.describe Gitlab::ImportExport::Group::RelationTreeRestorer, feature_category: :importers do let(:group) { create(:group).tap { |g| g.add_owner(user) } } let(:importable) { create(:group, parent: group) } @@ -60,4 +60,81 @@ RSpec.describe Gitlab::ImportExport::Group::RelationTreeRestorer do subject end + + describe 'relation object saving' do + let(:importable) { create(:group) } + let(:relation_reader) do + Gitlab::ImportExport::Json::LegacyReader::File.new( + path, + relation_names: [:labels]) + end + + before do + allow(shared.logger).to receive(:info).and_call_original + allow(relation_reader).to receive(:consume_relation).and_call_original + + allow(relation_reader) + .to receive(:consume_relation) + .with(nil, 'labels') + .and_return([[label, 0]]) + end + + context 'when relation object is new' do + context 'when relation object has invalid subrelations' do + let(:label) do + { + 'title' => 'test', + 'priorities' => [LabelPriority.new, LabelPriority.new], + 'type' => 'GroupLabel' + } + end + + it 'logs invalid subrelations' do + expect(shared.logger) + .to receive(:info) + .with( + message: '[Project/Group Import] Invalid subrelation', + group_id: importable.id, + relation_key: 'labels', + error_messages: "Project can't be blank, Priority can't be blank, and Priority is not a number" + ) + + subject + + label = importable.labels.first + failure = importable.import_failures.first + + expect(importable.import_failures.count).to eq(2) + expect(label.title).to eq('test') + expect(failure.exception_class).to eq('ActiveRecord::RecordInvalid') + expect(failure.source).to eq('RelationTreeRestorer#save_relation_object') + expect(failure.exception_message) + .to eq("Project can't be blank, Priority can't be blank, and Priority is not a number") + end + end + end + + context 'when relation object is persisted' do + context 'when relation object is invalid' do + let(:label) { create(:group_label, group: group, title: 'test') } + + it 'saves import failure with nested errors' do + label.priorities << [LabelPriority.new, LabelPriority.new] + + subject + + failure = importable.import_failures.first + + expect(importable.labels.count).to eq(0) + expect(importable.import_failures.count).to eq(1) + expect(failure.exception_class).to eq('ActiveRecord::RecordInvalid') + expect(failure.source).to eq('process_relation_item!') + expect(failure.exception_message) + .to eq("Validation failed: Priorities is invalid, Project can't be blank, Priority can't be blank, " \ + "Priority is not a number, Project can't be blank, Priority can't be blank, " \ + "Priority is not a number") + end + end + end + end end diff --git a/spec/lib/gitlab/import_export/import_failure_service_spec.rb b/spec/lib/gitlab/import_export/import_failure_service_spec.rb index 51f1fc9c6a2..30d16347828 100644 --- a/spec/lib/gitlab/import_export/import_failure_service_spec.rb +++ b/spec/lib/gitlab/import_export/import_failure_service_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::ImportExport::ImportFailureService do +RSpec.describe Gitlab::ImportExport::ImportFailureService, feature_category: :importers do let(:importable) { create(:project, :builds_enabled, :issues_disabled, name: 'project', path: 'project') } let(:label) { create(:label) } let(:subject) { described_class.new(importable) } diff --git a/spec/lib/gitlab/import_export/json/legacy_writer_spec.rb b/spec/lib/gitlab/import_export/json/legacy_writer_spec.rb index e8ecd98b1e1..2c0f023ad2c 100644 --- a/spec/lib/gitlab/import_export/json/legacy_writer_spec.rb +++ b/spec/lib/gitlab/import_export/json/legacy_writer_spec.rb @@ -1,8 +1,9 @@ # frozen_string_literal: true require 'fast_spec_helper' +require 'tmpdir' -RSpec.describe Gitlab::ImportExport::Json::LegacyWriter do +RSpec.describe Gitlab::ImportExport::Json::LegacyWriter, feature_category: :importers do let(:path) { "#{Dir.tmpdir}/legacy_writer_spec/test.json" } subject do diff --git a/spec/lib/gitlab/import_export/model_configuration_spec.rb b/spec/lib/gitlab/import_export/model_configuration_spec.rb index 4f01f470ce7..ce965a05a32 100644 --- a/spec/lib/gitlab/import_export/model_configuration_spec.rb +++ b/spec/lib/gitlab/import_export/model_configuration_spec.rb @@ -5,7 +5,7 @@ require 'spec_helper' # Part of the test security suite for the Import/Export feature # Finds if a new model has been added that can potentially be part of the Import/Export # If it finds a new model, it will show a +failure_message+ with the options available. -RSpec.describe 'Import/Export model configuration' do +RSpec.describe 'Import/Export model configuration', feature_category: :importers do include ConfigurationHelper let(:all_models_yml) { 'spec/lib/gitlab/import_export/all_models.yml' } diff --git a/spec/lib/gitlab/import_export/project/import_task_spec.rb b/spec/lib/gitlab/import_export/project/import_task_spec.rb index c847224cb9b..693f1984ce8 100644 --- a/spec/lib/gitlab/import_export/project/import_task_spec.rb +++ b/spec/lib/gitlab/import_export/project/import_task_spec.rb @@ -2,7 +2,7 @@ require 'rake_helper' -RSpec.describe Gitlab::ImportExport::Project::ImportTask, :request_store, :silence_stdout do +RSpec.describe Gitlab::ImportExport::Project::ImportTask, :request_store, :silence_stdout, feature_category: :importers do let(:username) { 'root' } let(:namespace_path) { username } let!(:user) { create(:user, username: username) } diff --git a/spec/lib/gitlab/import_export/project/object_builder_spec.rb b/spec/lib/gitlab/import_export/project/object_builder_spec.rb index 189b798c2e8..5fa8590e8fd 100644 --- a/spec/lib/gitlab/import_export/project/object_builder_spec.rb +++ b/spec/lib/gitlab/import_export/project/object_builder_spec.rb @@ -86,13 +86,16 @@ RSpec.describe Gitlab::ImportExport::Project::ObjectBuilder do 'group' => group)).to eq(group_label) end - it 'creates a new label' do + it 'creates a new project label' do label = described_class.build(Label, 'title' => 'group label', 'project' => project, - 'group' => project.group) + 'group' => project.group, + 'group_id' => project.group.id) expect(label.persisted?).to be true + expect(label).to be_an_instance_of(ProjectLabel) + expect(label.group_id).to be_nil end end diff --git a/spec/lib/gitlab/import_export/project/relation_tree_restorer_spec.rb b/spec/lib/gitlab/import_export/project/relation_tree_restorer_spec.rb index 6053df8ba97..75012aa80ec 100644 --- a/spec/lib/gitlab/import_export/project/relation_tree_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/project/relation_tree_restorer_spec.rb @@ -50,6 +50,7 @@ RSpec.describe Gitlab::ImportExport::Project::RelationTreeRestorer, feature_cate expect(project.custom_attributes.count).to eq(2) expect(project.project_badges.count).to eq(2) expect(project.snippets.count).to eq(1) + expect(project.commit_notes.count).to eq(3) end end end diff --git a/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb b/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb index 125d1736b9b..a07fe4fd29c 100644 --- a/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb @@ -295,6 +295,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i it 'has project labels' do expect(ProjectLabel.count).to eq(3) + expect(ProjectLabel.pluck(:group_id).compact).to be_empty end it 'has merge request approvals' do @@ -528,7 +529,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i it 'has the correct number of pipelines and statuses' do expect(@project.ci_pipelines.size).to eq(7) - @project.ci_pipelines.order(:id).zip([2, 0, 2, 2, 2, 2, 0]) + @project.ci_pipelines.order(:id).zip([2, 0, 2, 3, 2, 2, 0]) .each do |(pipeline, expected_status_size)| expect(pipeline.statuses.size).to eq(expected_status_size) end @@ -548,8 +549,16 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i expect(Ci::Stage.all).to all(have_attributes(pipeline_id: a_value > 0)) end - it 'restores statuses' do - expect(CommitStatus.all.count).to be 10 + it 'restores builds' do + expect(Ci::Build.all.count).to be 7 + end + + it 'restores bridges' do + expect(Ci::Bridge.all.count).to be 1 + end + + it 'restores generic commit statuses' do + expect(GenericCommitStatus.all.count).to be 1 end it 'correctly restores association between a stage and a job' do @@ -574,6 +583,10 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i expect(@project.import_failures.size).to eq 0 end end + + it 'restores commit notes' do + expect(@project.commit_notes.count).to eq(3) + end end end diff --git a/spec/lib/gitlab/import_export/project/tree_saver_spec.rb b/spec/lib/gitlab/import_export/project/tree_saver_spec.rb index 74b6e039601..b87992c4594 100644 --- a/spec/lib/gitlab/import_export/project/tree_saver_spec.rb +++ b/spec/lib/gitlab/import_export/project/tree_saver_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::ImportExport::Project::TreeSaver, :with_license do +RSpec.describe Gitlab::ImportExport::Project::TreeSaver, :with_license, feature_category: :importers do let_it_be(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" } let_it_be(:exportable_path) { 'project' } let_it_be(:user) { create(:user) } @@ -223,22 +223,31 @@ RSpec.describe Gitlab::ImportExport::Project::TreeSaver, :with_license do expect(subject.dig(0, 'stages')).not_to be_empty end - it 'has pipeline statuses' do - expect(subject.dig(0, 'stages', 0, 'statuses')).not_to be_empty + it 'has pipeline builds' do + count = subject.dig(0, 'stages', 0, 'builds').count + + expect(count).to eq(1) end - it 'has pipeline builds' do - builds_count = subject.dig(0, 'stages', 0, 'statuses') - .count { |hash| hash['type'] == 'Ci::Build' } + it 'has pipeline generic_commit_statuses' do + count = subject.dig(0, 'stages', 0, 'generic_commit_statuses').count - expect(builds_count).to eq(1) + expect(count).to eq(1) end - it 'has ci pipeline notes' do - expect(subject.first['notes']).not_to be_empty + it 'has pipeline bridges' do + count = subject.dig(0, 'stages', 0, 'bridges').count + + expect(count).to eq(1) end end + context 'with commit_notes' do + let(:relation_name) { :commit_notes } + + it { is_expected.not_to be_empty } + end + context 'with labels' do let(:relation_name) { :labels } @@ -468,6 +477,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeSaver, :with_license do end end + # rubocop: disable Metrics/AbcSize def setup_project release = create(:release) @@ -496,6 +506,8 @@ RSpec.describe Gitlab::ImportExport::Project::TreeSaver, :with_license do ci_build = create(:ci_build, project: project, when: nil) ci_build.pipeline.update!(project: project) create(:commit_status, project: project, pipeline: ci_build.pipeline) + create(:generic_commit_status, pipeline: ci_build.pipeline, ci_stage: ci_build.ci_stage, project: project) + create(:ci_bridge, pipeline: ci_build.pipeline, ci_stage: ci_build.ci_stage, project: project) create(:milestone, project: project) discussion_note = create(:discussion_note, noteable: issue, project: project) @@ -528,4 +540,5 @@ RSpec.describe Gitlab::ImportExport::Project::TreeSaver, :with_license do project end + # rubocop: enable Metrics/AbcSize end diff --git a/spec/lib/gitlab/import_export/references_configuration_spec.rb b/spec/lib/gitlab/import_export/references_configuration_spec.rb index ad165790b77..84c5b564cb1 100644 --- a/spec/lib/gitlab/import_export/references_configuration_spec.rb +++ b/spec/lib/gitlab/import_export/references_configuration_spec.rb @@ -9,7 +9,7 @@ require 'spec_helper' # or to be blacklisted by using the import_export.yml configuration file. # Likewise, new models added to import_export.yml, will need to be added with their correspondent relations # to this spec. -RSpec.describe 'Import/Export Project configuration' do +RSpec.describe 'Import/Export Project configuration', feature_category: :importers do include ConfigurationHelper where(:relation_path, :relation_name) do diff --git a/spec/lib/gitlab/import_export/safe_model_attributes.yml b/spec/lib/gitlab/import_export/safe_model_attributes.yml index e14e929faf3..2384baabb6b 100644 --- a/spec/lib/gitlab/import_export/safe_model_attributes.yml +++ b/spec/lib/gitlab/import_export/safe_model_attributes.yml @@ -86,6 +86,7 @@ Note: - original_discussion_id - confidential - last_edited_at +- internal LabelLink: - id - target_type @@ -347,7 +348,111 @@ Ci::Stage: - pipeline_id - created_at - updated_at -CommitStatus: +Ci::Build: +- id +- project_id +- status +- finished_at +- trace +- created_at +- updated_at +- started_at +- runner_id +- coverage +- commit_id +- commands +- job_id +- name +- deploy +- options +- allow_failure +- stage +- trigger_request_id +- stage_idx +- stage_id +- tag +- ref +- user_id +- type +- target_url +- description +- artifacts_file +- artifacts_file_store +- artifacts_metadata +- artifacts_metadata_store +- erased_by_id +- erased_at +- artifacts_expire_at +- environment +- artifacts_size +- when +- yaml_variables +- queued_at +- token +- lock_version +- coverage_regex +- auto_canceled_by_id +- retried +- protected +- failure_reason +- scheduled_at +- upstream_pipeline_id +- interruptible +- processed +- scheduling_type +Ci::Bridge: +- id +- project_id +- status +- finished_at +- trace +- created_at +- updated_at +- started_at +- runner_id +- coverage +- commit_id +- commands +- job_id +- name +- deploy +- options +- allow_failure +- stage +- trigger_request_id +- stage_idx +- stage_id +- tag +- ref +- user_id +- type +- target_url +- description +- artifacts_file +- artifacts_file_store +- artifacts_metadata +- artifacts_metadata_store +- erased_by_id +- erased_at +- artifacts_expire_at +- environment +- artifacts_size +- when +- yaml_variables +- queued_at +- token +- lock_version +- coverage_regex +- auto_canceled_by_id +- retried +- protected +- failure_reason +- scheduled_at +- upstream_pipeline_id +- interruptible +- processed +- scheduling_type +GenericCommitStatus: - id - project_id - status diff --git a/spec/lib/gitlab/instrumentation/redis_base_spec.rb b/spec/lib/gitlab/instrumentation/redis_base_spec.rb index 656e6ffba05..426997f6e86 100644 --- a/spec/lib/gitlab/instrumentation/redis_base_spec.rb +++ b/spec/lib/gitlab/instrumentation/redis_base_spec.rb @@ -210,4 +210,16 @@ RSpec.describe Gitlab::Instrumentation::RedisBase, :request_store do end end end + + describe '.log_exception' do + it 'logs exception with storage details' do + expect(::Gitlab::ErrorTracking).to receive(:log_exception) + .with( + an_instance_of(StandardError), + storage: instrumentation_class_a.storage_key + ) + + instrumentation_class_a.log_exception(StandardError.new) + end + end end diff --git a/spec/lib/gitlab/instrumentation/redis_interceptor_spec.rb b/spec/lib/gitlab/instrumentation/redis_interceptor_spec.rb index 187a6ff1739..18301f01a30 100644 --- a/spec/lib/gitlab/instrumentation/redis_interceptor_spec.rb +++ b/spec/lib/gitlab/instrumentation/redis_interceptor_spec.rb @@ -64,16 +64,34 @@ RSpec.describe Gitlab::Instrumentation::RedisInterceptor, :clean_gitlab_redis_sh end end - it 'counts exceptions' do - expect(instrumentation_class).to receive(:instance_count_exception) - .with(instance_of(Redis::CommandError)).and_call_original - expect(instrumentation_class).to receive(:instance_count_request).and_call_original + context 'when encountering exceptions' do + where(:case_name, :exception, :exception_counter) do + 'generic exception' | Redis::CommandError | :instance_count_exception + 'moved redirection' | Redis::CommandError.new("MOVED 123 127.0.0.1:6380") | :instance_count_cluster_redirection + 'ask redirection' | Redis::CommandError.new("ASK 123 127.0.0.1:6380") | :instance_count_cluster_redirection + end - expect do - Gitlab::Redis::SharedState.with do |redis| - redis.call(:auth, 'foo', 'bar') + with_them do + before do + Gitlab::Redis::SharedState.with do |redis| + # We need to go 1 layer deeper to stub _client as we monkey-patch Redis::Client + # with the interceptor. Stubbing `redis` will skip the instrumentation_class. + allow(redis._client).to receive(:process).and_raise(exception) + end end - end.to raise_exception(Redis::CommandError) + + it 'counts exception' do + expect(instrumentation_class).to receive(exception_counter) + .with(instance_of(Redis::CommandError)).and_call_original + expect(instrumentation_class).to receive(:log_exception) + .with(instance_of(Redis::CommandError)).and_call_original + expect(instrumentation_class).to receive(:instance_count_request).and_call_original + + expect do + Gitlab::Redis::SharedState.with { |redis| redis.call(:auth, 'foo', 'bar') } + end.to raise_exception(Redis::CommandError) + end + end end context 'in production environment' do diff --git a/spec/lib/gitlab/internal_post_receive/response_spec.rb b/spec/lib/gitlab/internal_post_receive/response_spec.rb index 23ea5191486..2792cf49d06 100644 --- a/spec/lib/gitlab/internal_post_receive/response_spec.rb +++ b/spec/lib/gitlab/internal_post_receive/response_spec.rb @@ -76,7 +76,7 @@ RSpec.describe Gitlab::InternalPostReceive::Response do describe '#add_alert_message' do context 'when text is present' do - it 'adds a alert message' do + it 'adds an alert message' do subject.add_alert_message('hello') expect(subject.messages.first.message).to eq('hello') diff --git a/spec/lib/gitlab/issuable_sorter_spec.rb b/spec/lib/gitlab/issuable_sorter_spec.rb index b8d0c7b0609..0d9940bab6f 100644 --- a/spec/lib/gitlab/issuable_sorter_spec.rb +++ b/spec/lib/gitlab/issuable_sorter_spec.rb @@ -4,16 +4,42 @@ require 'spec_helper' RSpec.describe Gitlab::IssuableSorter do let(:namespace1) { build_stubbed(:namespace, id: 1) } - let(:project1) { build_stubbed(:project, id: 1, namespace: namespace1) } - - let(:project2) { build_stubbed(:project, id: 2, path: "a", namespace: project1.namespace) } - let(:project3) { build_stubbed(:project, id: 3, path: "b", namespace: project1.namespace) } - let(:namespace2) { build_stubbed(:namespace, id: 2, path: "a") } let(:namespace3) { build_stubbed(:namespace, id: 3, path: "b") } - let(:project4) { build_stubbed(:project, id: 4, path: "a", namespace: namespace2) } - let(:project5) { build_stubbed(:project, id: 5, path: "b", namespace: namespace2) } - let(:project6) { build_stubbed(:project, id: 6, path: "a", namespace: namespace3) } + + let(:project1) do + build_stubbed(:project, id: 1, namespace: namespace1, project_namespace: build_stubbed(:project_namespace)) + end + + let(:project2) do + build_stubbed( + :project, id: 2, path: "a", namespace: project1.namespace, project_namespace: build_stubbed(:project_namespace) + ) + end + + let(:project3) do + build_stubbed( + :project, id: 3, path: "b", namespace: project1.namespace, project_namespace: build_stubbed(:project_namespace) + ) + end + + let(:project4) do + build_stubbed( + :project, id: 4, path: "a", namespace: namespace2, project_namespace: build_stubbed(:project_namespace) + ) + end + + let(:project5) do + build_stubbed( + :project, id: 5, path: "b", namespace: namespace2, project_namespace: build_stubbed(:project_namespace) + ) + end + + let(:project6) do + build_stubbed( + :project, id: 6, path: "a", namespace: namespace3, project_namespace: build_stubbed(:project_namespace) + ) + end let(:unsorted) { [sorted[2], sorted[3], sorted[0], sorted[1]] } diff --git a/spec/lib/gitlab/jira_import/issues_importer_spec.rb b/spec/lib/gitlab/jira_import/issues_importer_spec.rb index 9f654bbcd15..36135c56dd9 100644 --- a/spec/lib/gitlab/jira_import/issues_importer_spec.rb +++ b/spec/lib/gitlab/jira_import/issues_importer_spec.rb @@ -44,7 +44,7 @@ RSpec.describe Gitlab::JiraImport::IssuesImporter do def mock_issue_serializer(count, raise_exception_on_even_mocks: false) serializer = instance_double(Gitlab::JiraImport::IssueSerializer, execute: { key: 'data' }) - allow(Issue).to receive(:with_project_iid_supply).and_return('issue_iid') + allow(Issue).to receive(:with_namespace_iid_supply).and_return('issue_iid') count.times do |i| if raise_exception_on_even_mocks && i.even? diff --git a/spec/lib/gitlab/kas/user_access_spec.rb b/spec/lib/gitlab/kas/user_access_spec.rb new file mode 100644 index 00000000000..8795ad565d0 --- /dev/null +++ b/spec/lib/gitlab/kas/user_access_spec.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Kas::UserAccess, feature_category: :kubernetes_management do + describe '.enabled?' do + subject { described_class.enabled? } + + before do + allow(::Gitlab::Kas).to receive(:enabled?).and_return true + end + + it { is_expected.to be true } + + context 'when flag kas_user_access is disabled' do + before do + stub_feature_flags(kas_user_access: false) + end + + it { is_expected.to be false } + end + end + + describe '.enabled_for?' do + subject { described_class.enabled_for?(agent) } + + let(:agent) { build(:cluster_agent) } + + before do + allow(::Gitlab::Kas).to receive(:enabled?).and_return true + end + + it { is_expected.to be true } + + context 'when flag kas_user_access is disabled' do + before do + stub_feature_flags(kas_user_access: false) + end + + it { is_expected.to be false } + end + + context 'when flag kas_user_access_project is disabled' do + before do + stub_feature_flags(kas_user_access_project: false) + end + + it { is_expected.to be false } + end + end + + describe '.{encrypt,decrypt}_public_session_id' do + let(:data) { 'the data' } + let(:encrypted) { described_class.encrypt_public_session_id(data) } + let(:decrypted) { described_class.decrypt_public_session_id(encrypted) } + + it { expect(encrypted).not_to include data } + it { expect(decrypted).to eq data } + end + + describe '.cookie_data' do + subject(:cookie_data) { described_class.cookie_data(public_session_id) } + + let(:public_session_id) { 'the-public-session-id' } + let(:external_k8s_proxy_url) { 'https://example.com:1234' } + + before do + stub_config( + gitlab: { host: 'example.com', https: true }, + gitlab_kas: { external_k8s_proxy_url: external_k8s_proxy_url } + ) + end + + it 'is encrypted, secure, httponly', :aggregate_failures do + expect(cookie_data[:value]).not_to include public_session_id + expect(cookie_data).to include(httponly: true, secure: true, path: '/') + expect(cookie_data).not_to have_key(:domain) + end + + context 'when on non-root path' do + let(:external_k8s_proxy_url) { 'https://example.com/k8s-proxy' } + + it 'sets :path' do + expect(cookie_data).to include(httponly: true, secure: true, path: '/k8s-proxy') + end + end + + context 'when on subdomain' do + let(:external_k8s_proxy_url) { 'https://k8s-proxy.example.com' } + + it 'sets :domain' do + expect(cookie_data[:domain]).to eq "example.com" + end + end + end +end diff --git a/spec/lib/gitlab/kroki_spec.rb b/spec/lib/gitlab/kroki_spec.rb index 3d6ecf20377..6d8e6ecbf54 100644 --- a/spec/lib/gitlab/kroki_spec.rb +++ b/spec/lib/gitlab/kroki_spec.rb @@ -6,7 +6,7 @@ RSpec.describe Gitlab::Kroki do describe '.formats' do def default_formats - %w[bytefield c4plantuml ditaa erd graphviz nomnoml pikchr plantuml + %w[bytefield c4plantuml d2 dbml diagramsnet ditaa erd graphviz nomnoml pikchr plantuml structurizr svgbob umlet vega vegalite wavedrom].freeze end diff --git a/spec/lib/gitlab/kubernetes/config_map_spec.rb b/spec/lib/gitlab/kubernetes/config_map_spec.rb index 2d0d205ffb1..ebc2202921b 100644 --- a/spec/lib/gitlab/kubernetes/config_map_spec.rb +++ b/spec/lib/gitlab/kubernetes/config_map_spec.rb @@ -4,20 +4,23 @@ require 'spec_helper' RSpec.describe Gitlab::Kubernetes::ConfigMap do let(:kubeclient) { double('kubernetes client') } - let(:application) { create(:clusters_applications_prometheus) } - let(:config_map) { described_class.new(application.name, application.files) } + let(:name) { 'my-name' } + let(:files) { [] } + let(:config_map) { described_class.new(name, files) } let(:namespace) { Gitlab::Kubernetes::Helm::NAMESPACE } let(:metadata) do { - name: "values-content-configuration-#{application.name}", + name: "values-content-configuration-#{name}", namespace: namespace, - labels: { name: "values-content-configuration-#{application.name}" } + labels: { name: "values-content-configuration-#{name}" } } end describe '#generate' do - let(:resource) { ::Kubeclient::Resource.new(metadata: metadata, data: application.files) } + let(:resource) do + ::Kubeclient::Resource.new(metadata: metadata, data: files) + end subject { config_map.generate } @@ -28,7 +31,8 @@ RSpec.describe Gitlab::Kubernetes::ConfigMap do describe '#config_map_name' do it 'returns the config_map name' do - expect(config_map.config_map_name).to eq("values-content-configuration-#{application.name}") + expect(config_map.config_map_name) + .to eq("values-content-configuration-#{name}") end end end diff --git a/spec/lib/gitlab/kubernetes/helm/pod_spec.rb b/spec/lib/gitlab/kubernetes/helm/pod_spec.rb index e3763977add..8aa755bffce 100644 --- a/spec/lib/gitlab/kubernetes/helm/pod_spec.rb +++ b/spec/lib/gitlab/kubernetes/helm/pod_spec.rb @@ -13,7 +13,7 @@ RSpec.describe Gitlab::Kubernetes::Helm::Pod do with_them do let(:cluster) { create(:cluster, helm_major_version: helm_major_version) } - let(:app) { create(:clusters_applications_prometheus, cluster: cluster) } + let(:app) { create(:clusters_applications_knative, cluster: cluster) } let(:command) { app.install_command } let(:namespace) { Gitlab::Kubernetes::Helm::NAMESPACE } let(:service_account_name) { nil } diff --git a/spec/lib/gitlab/legacy_github_import/importer_spec.rb b/spec/lib/gitlab/legacy_github_import/importer_spec.rb index cd66b93eb8b..bb38f4b1bca 100644 --- a/spec/lib/gitlab/legacy_github_import/importer_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/importer_spec.rb @@ -2,7 +2,9 @@ require 'spec_helper' -RSpec.describe Gitlab::LegacyGithubImport::Importer do +RSpec.describe Gitlab::LegacyGithubImport::Importer, feature_category: :importers do + subject(:importer) { described_class.new(project) } + shared_examples 'Gitlab::LegacyGithubImport::Importer#execute' do let(:expected_not_called) { [] } @@ -11,8 +13,6 @@ RSpec.describe Gitlab::LegacyGithubImport::Importer do end it 'calls import methods' do - importer = described_class.new(project) - expected_called = [ :import_labels, :import_milestones, :import_pull_requests, :import_issues, :import_wiki, :import_releases, :handle_errors, @@ -51,11 +51,13 @@ RSpec.describe Gitlab::LegacyGithubImport::Importer do allow_any_instance_of(Octokit::Client).to receive(:labels).and_return([label1, label2]) allow_any_instance_of(Octokit::Client).to receive(:milestones).and_return([milestone, milestone]) allow_any_instance_of(Octokit::Client).to receive(:issues).and_return([issue1, issue2]) - allow_any_instance_of(Octokit::Client).to receive(:pull_requests).and_return([pull_request, pull_request]) + allow_any_instance_of(Octokit::Client).to receive(:pull_requests).and_return([pull_request, pull_request_missing_source_branch]) allow_any_instance_of(Octokit::Client).to receive(:issues_comments).and_raise(Octokit::NotFound) allow_any_instance_of(Octokit::Client).to receive(:pull_requests_comments).and_return([]) allow_any_instance_of(Octokit::Client).to receive(:last_response).and_return(double(rels: { next: nil })) allow_any_instance_of(Octokit::Client).to receive(:releases).and_return([release1, release2]) + + allow(importer).to receive(:restore_source_branch).and_raise(StandardError, 'Some error') end let(:label1) do @@ -153,8 +155,6 @@ RSpec.describe Gitlab::LegacyGithubImport::Importer do } end - subject { described_class.new(project) } - it 'returns true' do expect(subject.execute).to eq true end @@ -163,18 +163,19 @@ RSpec.describe Gitlab::LegacyGithubImport::Importer do expect { subject.execute }.not_to raise_error end - it 'stores error messages' do + it 'stores error messages', :unlimited_max_formatted_output_length do error = { message: 'The remote data could not be fully imported.', errors: [ { type: :label, url: "#{api_root}/repos/octocat/Hello-World/labels/bug", errors: "Validation failed: Title can't be blank, Title is invalid" }, + { type: :pull_request, url: "#{api_root}/repos/octocat/Hello-World/pulls/1347", errors: 'Some error' }, { type: :issue, url: "#{api_root}/repos/octocat/Hello-World/issues/1348", errors: "Validation failed: Title can't be blank" }, { type: :issues_comments, errors: 'Octokit::NotFound' }, { type: :wiki, errors: "Gitlab::Git::CommandError" } ] } - described_class.new(project).execute + importer.execute expect(project.import_state.last_error).to eq error.to_json end @@ -182,8 +183,6 @@ RSpec.describe Gitlab::LegacyGithubImport::Importer do shared_examples 'Gitlab::LegacyGithubImport unit-testing' do describe '#clean_up_restored_branches' do - subject { described_class.new(project) } - before do allow(gh_pull_request).to receive(:source_branch_exists?).at_least(:once) { false } allow(gh_pull_request).to receive(:target_branch_exists?).at_least(:once) { false } @@ -240,6 +239,16 @@ RSpec.describe Gitlab::LegacyGithubImport::Importer do } end + let(:pull_request_missing_source_branch) do + pull_request.merge( + head: { + ref: 'missing', + repo: repository, + sha: RepoHelpers.another_sample_commit + } + ) + end + let(:closed_pull_request) do { number: 1347, @@ -264,8 +273,6 @@ RSpec.describe Gitlab::LegacyGithubImport::Importer do let(:api_root) { 'https://try.gitea.io/api/v1' } let(:repo_root) { 'https://try.gitea.io' } - subject { described_class.new(project) } - before do project.update!(import_type: 'gitea', import_url: "#{repo_root}/foo/group/project.git") end diff --git a/spec/lib/gitlab/loggable_spec.rb b/spec/lib/gitlab/loggable_spec.rb new file mode 100644 index 00000000000..8238e47014b --- /dev/null +++ b/spec/lib/gitlab/loggable_spec.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Loggable, feature_category: :logging do + subject(:klass_instance) do + Class.new do + include Gitlab::Loggable + + def self.name + 'MyTestClass' + end + end.new + end + + describe '#build_structured_payload' do + it 'adds class and returns formatted json' do + expected = { + 'class' => 'MyTestClass', + 'message' => 'test' + } + + expect(klass_instance.build_structured_payload(message: 'test')).to eq(expected) + end + + it 'appends additional params and returns formatted json' do + expected = { + 'class' => 'MyTestClass', + 'message' => 'test', + 'extra_param' => 1 + } + + expect(klass_instance.build_structured_payload(message: 'test', extra_param: 1)).to eq(expected) + end + + it 'does not raise an error in loggers when passed non-symbols' do + expected = { + 'class' => 'MyTestClass', + 'message' => 'test', + '["hello", "thing"]' => :world + } + + payload = klass_instance.build_structured_payload(message: 'test', %w[hello thing] => :world) + expect(payload).to eq(expected) + expect { Gitlab::Export::Logger.info(payload) }.not_to raise_error + end + + it 'handles anonymous classes' do + anonymous_klass_instance = Class.new { include Gitlab::Loggable }.new + + expected = { + 'class' => '<Anonymous>', + 'message' => 'test' + } + + expect(anonymous_klass_instance.build_structured_payload(message: 'test')).to eq(expected) + end + + it 'handles duplicate keys' do + expected = { + 'class' => 'MyTestClass', + 'message' => 'test2' + } + + expect(klass_instance.build_structured_payload(message: 'test', 'message' => 'test2')).to eq(expected) + end + end +end diff --git a/spec/lib/gitlab/memory/watchdog/monitor/rss_memory_limit_spec.rb b/spec/lib/gitlab/memory/watchdog/monitor/rss_memory_limit_spec.rb index 4780b1eba53..67d185fd2f1 100644 --- a/spec/lib/gitlab/memory/watchdog/monitor/rss_memory_limit_spec.rb +++ b/spec/lib/gitlab/memory/watchdog/monitor/rss_memory_limit_spec.rb @@ -1,9 +1,10 @@ # frozen_string_literal: true require 'fast_spec_helper' +require 'prometheus/client' require 'support/shared_examples/lib/gitlab/memory/watchdog/monitor_result_shared_examples' -RSpec.describe Gitlab::Memory::Watchdog::Monitor::RssMemoryLimit do +RSpec.describe Gitlab::Memory::Watchdog::Monitor::RssMemoryLimit, feature_category: :application_performance do let(:max_rss_limit_gauge) { instance_double(::Prometheus::Client::Gauge) } let(:memory_limit_bytes) { 2_097_152_000 } let(:worker_memory_bytes) { 1_048_576_000 } diff --git a/spec/lib/gitlab/metrics/boot_time_tracker_spec.rb b/spec/lib/gitlab/metrics/boot_time_tracker_spec.rb index 8a17fa8dd2e..3175c0a6b32 100644 --- a/spec/lib/gitlab/metrics/boot_time_tracker_spec.rb +++ b/spec/lib/gitlab/metrics/boot_time_tracker_spec.rb @@ -1,8 +1,8 @@ # frozen_string_literal: true -require 'fast_spec_helper' +require 'spec_helper' -RSpec.describe Gitlab::Metrics::BootTimeTracker do +RSpec.describe Gitlab::Metrics::BootTimeTracker, feature_category: :metrics do let(:logger) { double('logger') } let(:gauge) { double('gauge') } diff --git a/spec/lib/gitlab/metrics/subscribers/rack_attack_spec.rb b/spec/lib/gitlab/metrics/subscribers/rack_attack_spec.rb index 9f939d0d7d6..13965bf1244 100644 --- a/spec/lib/gitlab/metrics/subscribers/rack_attack_spec.rb +++ b/spec/lib/gitlab/metrics/subscribers/rack_attack_spec.rb @@ -32,33 +32,6 @@ RSpec.describe Gitlab::Metrics::Subscribers::RackAttack, :request_store do end end - describe '#redis' do - it 'accumulates per-request RackAttack cache usage' do - freeze_time do - subscriber.redis( - ActiveSupport::Notifications::Event.new( - 'redis.rack_attack', Time.current, Time.current + 1.second, '1', { operation: 'fetch' } - ) - ) - subscriber.redis( - ActiveSupport::Notifications::Event.new( - 'redis.rack_attack', Time.current, Time.current + 2.seconds, '1', { operation: 'write' } - ) - ) - subscriber.redis( - ActiveSupport::Notifications::Event.new( - 'redis.rack_attack', Time.current, Time.current + 3.seconds, '1', { operation: 'read' } - ) - ) - end - - expect(Gitlab::SafeRequestStore[:rack_attack_instrumentation]).to eql( - rack_attack_redis_count: 3, - rack_attack_redis_duration_s: 6.0 - ) - end - end - shared_examples 'log into auth logger' do context 'when matched throttle does not require user information' do let(:event) do diff --git a/spec/lib/gitlab/metrics/subscribers/rails_cache_spec.rb b/spec/lib/gitlab/metrics/subscribers/rails_cache_spec.rb index 59bfe2042fa..1731da9b752 100644 --- a/spec/lib/gitlab/metrics/subscribers/rails_cache_spec.rb +++ b/spec/lib/gitlab/metrics/subscribers/rails_cache_spec.rb @@ -6,13 +6,14 @@ RSpec.describe Gitlab::Metrics::Subscribers::RailsCache do let(:env) { {} } let(:transaction) { Gitlab::Metrics::WebTransaction.new(env) } let(:subscriber) { described_class.new } - - let(:event) { double(:event, duration: 15.2, payload: { key: %w[a b c] }) } + let(:store) { 'Gitlab::CustomStore' } + let(:store_label) { 'CustomStore' } + let(:event) { double(:event, duration: 15.2, payload: { key: %w[a b c], store: store }) } describe '#cache_read' do it 'increments the cache_read duration' do expect(subscriber).to receive(:observe) - .with(:read, event.duration) + .with(:read, event) subscriber.cache_read(event) end @@ -27,7 +28,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::RailsCache do let(:event) { double(:event, duration: 15.2, payload: { hit: true }) } context 'when super operation is fetch' do - let(:event) { double(:event, duration: 15.2, payload: { hit: true, super_operation: :fetch }) } + let(:event) { double(:event, duration: 15.2, payload: { hit: true, super_operation: :fetch, store: store }) } it 'does not increment cache read miss total' do expect(transaction).not_to receive(:increment) @@ -39,7 +40,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::RailsCache do end context 'with miss event' do - let(:event) { double(:event, duration: 15.2, payload: { hit: false }) } + let(:event) { double(:event, duration: 15.2, payload: { hit: false, store: store }) } it 'increments the cache_read_miss total' do expect(transaction).to receive(:increment) @@ -51,7 +52,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::RailsCache do end context 'when super operation is fetch' do - let(:event) { double(:event, duration: 15.2, payload: { hit: false, super_operation: :fetch }) } + let(:event) { double(:event, duration: 15.2, payload: { hit: false, super_operation: :fetch, store: store }) } it 'does not increment cache read miss total' do expect(transaction).not_to receive(:increment) @@ -92,7 +93,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::RailsCache do it 'observes read_multi duration' do expect(subscriber).to receive(:observe) - .with(:read_multi, event.duration) + .with(:read_multi, event) subject end @@ -101,7 +102,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::RailsCache do describe '#cache_write' do it 'observes write duration' do expect(subscriber).to receive(:observe) - .with(:write, event.duration) + .with(:write, event) subscriber.cache_write(event) end @@ -110,7 +111,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::RailsCache do describe '#cache_delete' do it 'observes delete duration' do expect(subscriber).to receive(:observe) - .with(:delete, event.duration) + .with(:delete, event) subscriber.cache_delete(event) end @@ -119,7 +120,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::RailsCache do describe '#cache_exist?' do it 'observes the exists duration' do expect(subscriber).to receive(:observe) - .with(:exists, event.duration) + .with(:exists, event) subscriber.cache_exist?(event) end @@ -179,7 +180,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::RailsCache do it 'returns' do expect(transaction).not_to receive(:increment) - subscriber.observe(:foo, 15.2) + subscriber.observe(:foo, event) end end @@ -192,17 +193,17 @@ RSpec.describe Gitlab::Metrics::Subscribers::RailsCache do it 'observes cache metric' do expect(subscriber.send(:metric_cache_operation_duration_seconds)) .to receive(:observe) - .with({ operation: :delete }, event.duration / 1000.0) + .with({ operation: :delete, store: store_label }, event.duration / 1000.0) - subscriber.observe(:delete, event.duration) + subscriber.observe(:delete, event) end it 'increments the operations total' do expect(transaction) .to receive(:increment) - .with(:gitlab_cache_operations_total, 1, { operation: :delete }) + .with(:gitlab_cache_operations_total, 1, { operation: :delete, store: store_label }) - subscriber.observe(:delete, event.duration) + subscriber.observe(:delete, event) end end end diff --git a/spec/lib/gitlab/monitor/demo_projects_spec.rb b/spec/lib/gitlab/monitor/demo_projects_spec.rb index 262c78eb62e..6b0f855e38d 100644 --- a/spec/lib/gitlab/monitor/demo_projects_spec.rb +++ b/spec/lib/gitlab/monitor/demo_projects_spec.rb @@ -6,15 +6,13 @@ RSpec.describe Gitlab::Monitor::DemoProjects do describe '#primary_keys' do subject { described_class.primary_keys } - it 'fetches primary_keys when on gitlab.com' do - allow(Gitlab).to receive(:com?).and_return(true) + it 'fetches primary_keys when on SaaS', :saas do allow(Gitlab).to receive(:staging?).and_return(false) expect(subject).to eq(Gitlab::Monitor::DemoProjects::DOT_COM_IDS) end - it 'fetches primary_keys when on staging' do - allow(Gitlab).to receive(:com?).and_return(true) + it 'fetches primary_keys when on staging', :saas do allow(Gitlab).to receive(:staging?).and_return(true) expect(subject).to eq(Gitlab::Monitor::DemoProjects::STAGING_IDS) diff --git a/spec/lib/gitlab/multi_collection_paginator_spec.rb b/spec/lib/gitlab/multi_collection_paginator_spec.rb index 080b3382684..25baa8913bf 100644 --- a/spec/lib/gitlab/multi_collection_paginator_spec.rb +++ b/spec/lib/gitlab/multi_collection_paginator_spec.rb @@ -5,6 +5,13 @@ require 'spec_helper' RSpec.describe Gitlab::MultiCollectionPaginator do subject(:paginator) { described_class.new(Project.all.order(:id), Group.all.order(:id), per_page: 3) } + it 'raises an error for invalid page size' do + expect { described_class.new(Project.all.order(:id), Group.all.order(:id), per_page: 0) } + .to raise_error(ArgumentError) + expect { described_class.new(Project.all.order(:id), Group.all.order(:id), per_page: -1) } + .to raise_error(ArgumentError) + end + it 'combines both collections' do project = create(:project) group = create(:group) diff --git a/spec/lib/gitlab/nav/top_nav_menu_item_spec.rb b/spec/lib/gitlab/nav/top_nav_menu_item_spec.rb index 6632a8106ca..1d3452a004a 100644 --- a/spec/lib/gitlab/nav/top_nav_menu_item_spec.rb +++ b/spec/lib/gitlab/nav/top_nav_menu_item_spec.rb @@ -14,7 +14,8 @@ RSpec.describe ::Gitlab::Nav::TopNavMenuItem, feature_category: :navigation do view: 'view', css_class: 'css_class', data: {}, - emoji: 'smile' + partial: 'groups/some_view_partial_file', + component: '_some_component_used_as_a_trigger_for_frontend_dropdown_item_render_' } expect(described_class.build(**item)).to eq(item.merge(type: :item)) diff --git a/spec/lib/gitlab/net_http_adapter_spec.rb b/spec/lib/gitlab/net_http_adapter_spec.rb index fdaf35be31e..cfb90578a4b 100644 --- a/spec/lib/gitlab/net_http_adapter_spec.rb +++ b/spec/lib/gitlab/net_http_adapter_spec.rb @@ -1,8 +1,9 @@ # frozen_string_literal: true require 'fast_spec_helper' +require 'net/http' -RSpec.describe Gitlab::NetHttpAdapter do +RSpec.describe Gitlab::NetHttpAdapter, feature_category: :api do describe '#connect' do let(:url) { 'https://example.org' } let(:net_http_adapter) { described_class.new(url) } diff --git a/spec/lib/gitlab/observability_spec.rb b/spec/lib/gitlab/observability_spec.rb index 8068d2f8ec9..5082d193197 100644 --- a/spec/lib/gitlab/observability_spec.rb +++ b/spec/lib/gitlab/observability_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Observability do +RSpec.describe Gitlab::Observability, feature_category: :error_tracking do describe '.observability_url' do let(:gitlab_url) { 'https://example.com' } @@ -31,29 +31,189 @@ RSpec.describe Gitlab::Observability do end end - describe '.observability_enabled?' do - let_it_be(:group) { build(:user) } - let_it_be(:user) { build(:group) } + describe '.build_full_url' do + let_it_be(:group) { build_stubbed(:group, id: 123) } + let(:observability_url) { described_class.observability_url } + + it 'returns the full observability url for the given params' do + url = described_class.build_full_url(group, '/foo?bar=baz', '/') + expect(url).to eq("https://observe.gitlab.com/-/123/foo?bar=baz") + end + + it 'handles missing / from observability_path' do + url = described_class.build_full_url(group, 'foo?bar=baz', '/') + expect(url).to eq("https://observe.gitlab.com/-/123/foo?bar=baz") + end + + it 'sanitises observability_path' do + url = described_class.build_full_url(group, "/test?groupId=<script>alert('attack!')</script>", '/') + expect(url).to eq("https://observe.gitlab.com/-/123/test?groupId=alert('attack!')") + end + + context 'when observability_path is missing' do + it 'builds the url with the fallback_path' do + url = described_class.build_full_url(group, nil, '/fallback') + expect(url).to eq("https://observe.gitlab.com/-/123/fallback") + end + + it 'defaults to / if fallback_path is also missing' do + url = described_class.build_full_url(group, nil, nil) + expect(url).to eq("https://observe.gitlab.com/-/123/") + end + end + end + + describe '.embeddable_url' do + before do + stub_config_setting(url: "https://www.gitlab.com") + # Can't use build/build_stubbed as we want the routes to be generated as well + create(:group, path: 'test-path', id: 123) + end + + context 'when URL is valid' do + where(:input, :expected) do + [ + [ + "https://www.gitlab.com/groups/test-path/-/observability/explore?observability_path=%2Fexplore%3FgroupId%3D14485840%26left%3D%255B%2522now-1h%2522,%2522now%2522,%2522new-sentry.gitlab.net%2522,%257B%257D%255D", + "https://observe.gitlab.com/-/123/explore?groupId=14485840&left=%5B%22now-1h%22,%22now%22,%22new-sentry.gitlab.net%22,%7B%7D%5D" + ], + [ + "https://www.gitlab.com/groups/test-path/-/observability/explore?observability_path=/goto/foo", + "https://observe.gitlab.com/-/123/goto/foo" + ] + ] + end + + with_them do + it 'returns an embeddable observability url' do + expect(described_class.embeddable_url(input)).to eq(expected) + end + end + end + + context 'when URL is invalid' do + where(:input) do + [ + # direct links to observe.gitlab.com + "https://observe.gitlab.com/-/123/explore", + 'https://observe.gitlab.com/v1/auth/start', + + # invalid GitLab URL + "not a link", + "https://foo.bar/groups/test-path/-/observability/explore?observability_path=/explore", + "http://www.gitlab.com/groups/test-path/-/observability/explore?observability_path=/explore", + "https://www.gitlab.com:123/groups/test-path/-/observability/explore?observability_path=/explore", + "https://www.gitlab.com@example.com/groups/test-path/-/observability/explore?observability_path=/explore", + "https://www.gitlab.com/groups/test-path/-/observability/explore?observability_path=@example.com", + + # invalid group/controller/actions + "https://www.gitlab.com/groups/INVALID_GROUP/-/observability/explore?observability_path=/explore", + "https://www.gitlab.com/groups/test-path/-/INVALID_CONTROLLER/explore?observability_path=/explore", + "https://www.gitlab.com/groups/test-path/-/observability/INVALID_ACTION?observability_path=/explore", + + # invalid observablity path + "https://www.gitlab.com/groups/test-path/-/observability/explore", + "https://www.gitlab.com/groups/test-path/-/observability/explore?missing_observability_path=/explore", + "https://www.gitlab.com/groups/test-path/-/observability/explore?observability_path=/not_embeddable", + "https://www.gitlab.com/groups/test-path/-/observability/explore?observability_path=/datasources", + "https://www.gitlab.com/groups/test-path/-/observability/explore?observability_path=not a valid path" + ] + end + + with_them do + it 'returns nil' do + expect(described_class.embeddable_url(input)).to be_nil + end + end + + it 'returns nil if the path detection throws an error' do + test_url = "https://www.gitlab.com/groups/test-path/-/observability/explore" + allow(Rails.application.routes).to receive(:recognize_path).with(test_url) { + raise ActionController::RoutingError, 'test' + } + expect(described_class.embeddable_url(test_url)).to be_nil + end + + it 'returns nil if parsing observaboility path throws an error' do + observability_path = 'some-path' + test_url = "https://www.gitlab.com/groups/test-path/-/observability/explore?observability_path=#{observability_path}" + + allow(URI).to receive(:parse).and_call_original + allow(URI).to receive(:parse).with(observability_path) { + raise URI::InvalidURIError, 'test' + } + + expect(described_class.embeddable_url(test_url)).to be_nil + end + end + end + + describe '.allowed_for_action?' do + let(:group) { build_stubbed(:group) } + let(:user) { build_stubbed(:user) } + + before do + allow(described_class).to receive(:allowed?).and_call_original + end + + it 'returns false if action is nil' do + expect(described_class.allowed_for_action?(user, group, nil)).to eq(false) + end + + describe 'allowed? calls' do + using RSpec::Parameterized::TableSyntax + + where(:action, :permission) do + :foo | :admin_observability + :explore | :read_observability + :datasources | :admin_observability + :manage | :admin_observability + :dashboards | :read_observability + end + + with_them do + it "calls allowed? with #{params[:permission]} when actions is #{params[:action]}" do + described_class.allowed_for_action?(user, group, action) + expect(described_class).to have_received(:allowed?).with(user, group, permission) + end + end + end + end + + describe '.allowed?' do + let(:user) { build_stubbed(:user) } + let(:group) { build_stubbed(:group) } + let(:test_permission) { :read_observability } + + before do + allow(Ability).to receive(:allowed?).and_return(false) + end subject do - described_class.observability_enabled?(user, group) + described_class.allowed?(user, group, test_permission) end - it 'checks if read_observability ability is allowed for the given user and group' do + it 'checks if ability is allowed for the given user and group' do allow(Ability).to receive(:allowed?).and_return(true) subject - expect(Ability).to have_received(:allowed?).with(user, :read_observability, group) + expect(Ability).to have_received(:allowed?).with(user, test_permission, group) end - it 'returns true if the read_observability ability is allowed' do + it 'checks for admin_observability if permission is missing' do + described_class.allowed?(user, group) + + expect(Ability).to have_received(:allowed?).with(user, :admin_observability, group) + end + + it 'returns true if the ability is allowed' do allow(Ability).to receive(:allowed?).and_return(true) expect(subject).to eq(true) end - it 'returns false if the read_observability ability is not allowed' do + it 'returns false if the ability is not allowed' do allow(Ability).to receive(:allowed?).and_return(false) expect(subject).to eq(false) @@ -64,5 +224,13 @@ RSpec.describe Gitlab::Observability do expect(subject).to eq(false) end + + it 'returns false if group is missing' do + expect(described_class.allowed?(user, nil, :read_observability)).to eq(false) + end + + it 'returns false if user is missing' do + expect(described_class.allowed?(nil, group, :read_observability)).to eq(false) + end end end diff --git a/spec/lib/gitlab/optimistic_locking_spec.rb b/spec/lib/gitlab/optimistic_locking_spec.rb index 1d669573b74..34f197b5ddb 100644 --- a/spec/lib/gitlab/optimistic_locking_spec.rb +++ b/spec/lib/gitlab/optimistic_locking_spec.rb @@ -16,6 +16,19 @@ RSpec.describe Gitlab::OptimisticLocking do describe '#retry_lock' do let(:name) { 'optimistic_locking_spec' } + it 'does not change current_scope', :aggregate_failures do + instance = Class.new { include Gitlab::OptimisticLocking }.new + relation = pipeline.cancelable_statuses + + expected_scope = Ci::Build.current_scope&.to_sql + + instance.send(:retry_lock, relation, name: :test) do + expect(Ci::Build.current_scope&.to_sql).to eq(expected_scope) + end + + expect(Ci::Build.current_scope&.to_sql).to eq(expected_scope) + end + context 'when state changed successfully without retries' do subject do described_class.retry_lock(pipeline, name: name) do |lock_subject| diff --git a/spec/lib/gitlab/pages/random_domain_spec.rb b/spec/lib/gitlab/pages/random_domain_spec.rb new file mode 100644 index 00000000000..978412bb72c --- /dev/null +++ b/spec/lib/gitlab/pages/random_domain_spec.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Pages::RandomDomain, feature_category: :pages do + let(:namespace_path) { 'namespace' } + + subject(:generator) do + described_class.new(project_path: project_path, namespace_path: namespace_path) + end + + RSpec.shared_examples 'random domain' do |domain| + it do + expect(SecureRandom) + .to receive(:hex) + .and_wrap_original do |_, size, _| + ('h' * size) + end + + generated = generator.generate + + expect(generated).to eq(domain) + expect(generated.length).to eq(63) + end + end + + context 'when project path is less than 48 chars' do + let(:project_path) { 'p' } + + it_behaves_like 'random domain', 'p-namespace-hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh' + end + + context 'when project path is close to 48 chars' do + let(:project_path) { 'p' * 45 } + + it_behaves_like 'random domain', 'ppppppppppppppppppppppppppppppppppppppppppppp-na-hhhhhhhhhhhhhh' + end + + context 'when project path is larger than 48 chars' do + let(:project_path) { 'p' * 49 } + + it_behaves_like 'random domain', 'pppppppppppppppppppppppppppppppppppppppppppppppp-hhhhhhhhhhhhhh' + end +end diff --git a/spec/lib/gitlab/pages/virtual_host_finder_spec.rb b/spec/lib/gitlab/pages/virtual_host_finder_spec.rb new file mode 100644 index 00000000000..4b584a45503 --- /dev/null +++ b/spec/lib/gitlab/pages/virtual_host_finder_spec.rb @@ -0,0 +1,214 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Pages::VirtualHostFinder, feature_category: :pages do + let_it_be(:project) { create(:project) } + + before_all do + project.update_pages_deployment!(create(:pages_deployment, project: project)) + end + + it 'returns nil when host is empty' do + expect(described_class.new(nil).execute).to be_nil + expect(described_class.new('').execute).to be_nil + end + + context 'when host is a pages custom domain host' do + let_it_be(:pages_domain) { create(:pages_domain, project: project) } + + subject(:virtual_domain) { described_class.new(pages_domain.domain).execute } + + context 'when there are no pages deployed for the project' do + before_all do + project.mark_pages_as_not_deployed + end + + it 'returns nil' do + expect(virtual_domain).to be_nil + end + end + + context 'when there are pages deployed for the project' do + before_all do + project.mark_pages_as_deployed + end + + it 'returns the virual domain when there are pages deployed for the project' do + expect(virtual_domain).to be_an_instance_of(Pages::VirtualDomain) + expect(virtual_domain.cache_key).to match(/pages_domain_for_domain_#{pages_domain.id}_/) + expect(virtual_domain.lookup_paths.length).to eq(1) + expect(virtual_domain.lookup_paths.first.project_id).to eq(project.id) + end + + context 'when :cache_pages_domain_api is disabled' do + before do + stub_feature_flags(cache_pages_domain_api: false) + end + + it 'returns the virual domain when there are pages deployed for the project' do + expect(virtual_domain).to be_an_instance_of(Pages::VirtualDomain) + expect(virtual_domain.cache_key).to be_nil + expect(virtual_domain.lookup_paths.length).to eq(1) + expect(virtual_domain.lookup_paths.first.project_id).to eq(project.id) + end + end + end + end + + context 'when host is a namespace domain' do + context 'when there are no pages deployed for the project' do + before_all do + project.mark_pages_as_not_deployed + end + + it 'returns no result if the provided host is not subdomain of the Pages host' do + virtual_domain = described_class.new("#{project.namespace.path}.something.io").execute + + expect(virtual_domain).to eq(nil) + end + + it 'returns the virual domain with no lookup_paths' do + virtual_domain = described_class.new("#{project.namespace.path}.#{Settings.pages.host}").execute + + expect(virtual_domain).to be_an_instance_of(Pages::VirtualDomain) + expect(virtual_domain.cache_key).to match(/pages_domain_for_namespace_#{project.namespace.id}_/) + expect(virtual_domain.lookup_paths.length).to eq(0) + end + + context 'when :cache_pages_domain_api is disabled' do + before do + stub_feature_flags(cache_pages_domain_api: false) + end + + it 'returns the virual domain with no lookup_paths' do + virtual_domain = described_class.new("#{project.namespace.path}.#{Settings.pages.host}".downcase).execute + + expect(virtual_domain).to be_an_instance_of(Pages::VirtualDomain) + expect(virtual_domain.cache_key).to be_nil + expect(virtual_domain.lookup_paths.length).to eq(0) + end + end + end + + context 'when there are pages deployed for the project' do + before_all do + project.mark_pages_as_deployed + project.namespace.update!(path: 'topNAMEspace') + end + + it 'returns no result if the provided host is not subdomain of the Pages host' do + virtual_domain = described_class.new("#{project.namespace.path}.something.io").execute + + expect(virtual_domain).to eq(nil) + end + + it 'returns the virual domain when there are pages deployed for the project' do + virtual_domain = described_class.new("#{project.namespace.path}.#{Settings.pages.host}").execute + + expect(virtual_domain).to be_an_instance_of(Pages::VirtualDomain) + expect(virtual_domain.cache_key).to match(/pages_domain_for_namespace_#{project.namespace.id}_/) + expect(virtual_domain.lookup_paths.length).to eq(1) + expect(virtual_domain.lookup_paths.first.project_id).to eq(project.id) + end + + it 'finds domain with case-insensitive' do + virtual_domain = described_class.new("#{project.namespace.path}.#{Settings.pages.host.upcase}").execute + + expect(virtual_domain).to be_an_instance_of(Pages::VirtualDomain) + expect(virtual_domain.cache_key).to match(/pages_domain_for_namespace_#{project.namespace.id}_/) + expect(virtual_domain.lookup_paths.length).to eq(1) + expect(virtual_domain.lookup_paths.first.project_id).to eq(project.id) + end + + context 'when :cache_pages_domain_api is disabled' do + before_all do + stub_feature_flags(cache_pages_domain_api: false) + end + + it 'returns the virual domain when there are pages deployed for the project' do + virtual_domain = described_class.new("#{project.namespace.path}.#{Settings.pages.host}").execute + + expect(virtual_domain).to be_an_instance_of(Pages::VirtualDomain) + expect(virtual_domain.cache_key).to be_nil + expect(virtual_domain.lookup_paths.length).to eq(1) + expect(virtual_domain.lookup_paths.first.project_id).to eq(project.id) + end + end + end + end + + context 'when host is a unique domain' do + before_all do + project.project_setting.update!(pages_unique_domain: 'unique-domain') + end + + subject(:virtual_domain) { described_class.new("unique-domain.#{Settings.pages.host.upcase}").execute } + + context 'when pages unique domain is enabled' do + before_all do + project.project_setting.update!(pages_unique_domain_enabled: true) + end + + context 'when there are no pages deployed for the project' do + before_all do + project.mark_pages_as_not_deployed + end + + it 'returns nil' do + expect(virtual_domain).to be_nil + end + end + + context 'when there are pages deployed for the project' do + before_all do + project.mark_pages_as_deployed + end + + it 'returns the virual domain when there are pages deployed for the project' do + expect(virtual_domain).to be_an_instance_of(Pages::VirtualDomain) + expect(virtual_domain.lookup_paths.length).to eq(1) + expect(virtual_domain.lookup_paths.first.project_id).to eq(project.id) + end + + context 'when :cache_pages_domain_api is disabled' do + before do + stub_feature_flags(cache_pages_domain_api: false) + end + + it 'returns the virual domain when there are pages deployed for the project' do + expect(virtual_domain).to be_an_instance_of(Pages::VirtualDomain) + expect(virtual_domain.lookup_paths.length).to eq(1) + expect(virtual_domain.lookup_paths.first.project_id).to eq(project.id) + end + end + end + end + + context 'when pages unique domain is disabled' do + before_all do + project.project_setting.update!(pages_unique_domain_enabled: false) + end + + context 'when there are no pages deployed for the project' do + before_all do + project.mark_pages_as_not_deployed + end + + it 'returns nil' do + expect(virtual_domain).to be_nil + end + end + + context 'when there are pages deployed for the project' do + before_all do + project.mark_pages_as_deployed + end + + it 'returns nil' do + expect(virtual_domain).to be_nil + end + end + end + end +end diff --git a/spec/lib/gitlab/patch/node_loader_spec.rb b/spec/lib/gitlab/patch/node_loader_spec.rb new file mode 100644 index 00000000000..000083fc6d0 --- /dev/null +++ b/spec/lib/gitlab/patch/node_loader_spec.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Patch::NodeLoader, feature_category: :redis do + using RSpec::Parameterized::TableSyntax + + describe '#fetch_node_info' do + let(:redis) { double(:redis) } # rubocop:disable RSpec/VerifiedDoubles + + # rubocop:disable Naming/InclusiveLanguage + where(:case_name, :args, :value) do + [ + [ + 'when only ip address is present', + "07c37df 127.0.0.1:30004@31004 slave e7d1eec 0 1426238317239 4 connected +67ed2db 127.0.0.1:30002@31002 master - 0 1426238316232 2 connected 5461-10922 +292f8b3 127.0.0.1:30003@31003 master - 0 1426238318243 3 connected 10923-16383 +6ec2392 127.0.0.1:30005@31005 slave 67ed2db 0 1426238316232 5 connected +824fe11 127.0.0.1:30006@31006 slave 292f8b3 0 1426238317741 6 connected +e7d1eec 127.0.0.1:30001@31001 myself,master - 0 0 1 connected 0-5460", + { + '127.0.0.1:30004' => 'slave', '127.0.0.1:30002' => 'master', '127.0.0.1:30003' => 'master', + '127.0.0.1:30005' => 'slave', '127.0.0.1:30006' => 'slave', '127.0.0.1:30001' => 'master' + } + ], + [ + 'when hostname is present', + "07c37df 127.0.0.1:30004@31004,host1 slave e7d1eec 0 1426238317239 4 connected +67ed2db 127.0.0.1:30002@31002,host2 master - 0 1426238316232 2 connected 5461-10922 +292f8b3 127.0.0.1:30003@31003,host3 master - 0 1426238318243 3 connected 10923-16383 +6ec2392 127.0.0.1:30005@31005,host4 slave 67ed2db 0 1426238316232 5 connected +824fe11 127.0.0.1:30006@31006,host5 slave 292f8b3 0 1426238317741 6 connected +e7d1eec 127.0.0.1:30001@31001,host6 myself,master - 0 0 1 connected 0-5460", + { + 'host1:30004' => 'slave', 'host2:30002' => 'master', 'host3:30003' => 'master', + 'host4:30005' => 'slave', 'host5:30006' => 'slave', 'host6:30001' => 'master' + } + ], + [ + 'when auxiliary fields are present', + "07c37df 127.0.0.1:30004@31004,,shard-id=69bc slave e7d1eec 0 1426238317239 4 connected +67ed2db 127.0.0.1:30002@31002,,shard-id=114f master - 0 1426238316232 2 connected 5461-10922 +292f8b3 127.0.0.1:30003@31003,,shard-id=fdb3 master - 0 1426238318243 3 connected 10923-16383 +6ec2392 127.0.0.1:30005@31005,,shard-id=114f slave 67ed2db 0 1426238316232 5 connected +824fe11 127.0.0.1:30006@31006,,shard-id=fdb3 slave 292f8b3 0 1426238317741 6 connected +e7d1eec 127.0.0.1:30001@31001,,shard-id=69bc myself,master - 0 0 1 connected 0-5460", + { + '127.0.0.1:30004' => 'slave', '127.0.0.1:30002' => 'master', '127.0.0.1:30003' => 'master', + '127.0.0.1:30005' => 'slave', '127.0.0.1:30006' => 'slave', '127.0.0.1:30001' => 'master' + } + ], + [ + 'when hostname and auxiliary fields are present', + "07c37df 127.0.0.1:30004@31004,host1,shard-id=69bc slave e7d1eec 0 1426238317239 4 connected +67ed2db 127.0.0.1:30002@31002,host2,shard-id=114f master - 0 1426238316232 2 connected 5461-10922 +292f8b3 127.0.0.1:30003@31003,host3,shard-id=fdb3 master - 0 1426238318243 3 connected 10923-16383 +6ec2392 127.0.0.1:30005@31005,host4,shard-id=114f slave 67ed2db 0 1426238316232 5 connected +824fe11 127.0.0.1:30006@31006,host5,shard-id=fdb3 slave 292f8b3 0 1426238317741 6 connected +e7d1eec 127.0.0.1:30001@31001,host6,shard-id=69bc myself,master - 0 0 1 connected 0-5460", + { + 'host1:30004' => 'slave', 'host2:30002' => 'master', 'host3:30003' => 'master', + 'host4:30005' => 'slave', 'host5:30006' => 'slave', 'host6:30001' => 'master' + } + ] + ] + end + # rubocop:enable Naming/InclusiveLanguage + + with_them do + before do + allow(redis).to receive(:call).with([:cluster, :nodes]).and_return(args) + end + + it do + expect(Redis::Cluster::NodeLoader.load_flags([redis])).to eq(value) + end + end + end +end diff --git a/spec/lib/gitlab/prometheus/queries/knative_invocation_query_spec.rb b/spec/lib/gitlab/prometheus/queries/knative_invocation_query_spec.rb deleted file mode 100644 index ff48b9ada90..00000000000 --- a/spec/lib/gitlab/prometheus/queries/knative_invocation_query_spec.rb +++ /dev/null @@ -1,31 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Prometheus::Queries::KnativeInvocationQuery do - include PrometheusHelpers - - let(:project) { create(:project) } - let(:serverless_func) { ::Serverless::Function.new(project, 'test-name', 'test-ns') } - let(:client) { double('prometheus_client') } - - subject { described_class.new(client) } - - context 'verify queries' do - before do - create(:prometheus_metric, - :common, - identifier: :system_metrics_knative_function_invocation_count, - query: 'sum(ceil(rate(istio_requests_total{destination_service_namespace="%{kube_namespace}", destination_service=~"%{function_name}.*"}[1m])*60))') - end - - it 'has the query, but no data' do - expect(client).to receive(:query_range).with( - 'sum(ceil(rate(istio_requests_total{destination_service_namespace="test-ns", destination_service=~"test-name.*"}[1m])*60))', - hash_including(:start_time, :end_time) - ) - - subject.query(serverless_func.id) - end - end -end diff --git a/spec/lib/gitlab/rack_attack/instrumented_cache_store_spec.rb b/spec/lib/gitlab/rack_attack/instrumented_cache_store_spec.rb deleted file mode 100644 index 8151519ddec..00000000000 --- a/spec/lib/gitlab/rack_attack/instrumented_cache_store_spec.rb +++ /dev/null @@ -1,89 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::RackAttack::InstrumentedCacheStore do - using RSpec::Parameterized::TableSyntax - - let(:store) { ::ActiveSupport::Cache::NullStore.new } - - subject { described_class.new(upstream_store: store) } - - where(:operation, :params, :test_proc) do - :fetch | [:key] | ->(s) { s.fetch(:key) } - :read | [:key] | ->(s) { s.read(:key) } - :read_multi | [:key_1, :key_2, :key_3] | ->(s) { s.read_multi(:key_1, :key_2, :key_3) } - :write_multi | [{ key_1: 1, key_2: 2, key_3: 3 }] | ->(s) { s.write_multi(key_1: 1, key_2: 2, key_3: 3) } - :fetch_multi | [:key_1, :key_2, :key_3] | ->(s) { s.fetch_multi(:key_1, :key_2, :key_3) {} } - :write | [:key, :value, { option_1: 1 }] | ->(s) { s.write(:key, :value, option_1: 1) } - :delete | [:key] | ->(s) { s.delete(:key) } - :exist? | [:key, { option_1: 1 }] | ->(s) { s.exist?(:key, option_1: 1) } - :delete_matched | [/^key$/, { option_1: 1 }] | ->(s) { s.delete_matched(/^key$/, option_1: 1 ) } - :increment | [:key, 1] | ->(s) { s.increment(:key, 1) } - :decrement | [:key, 1] | ->(s) { s.decrement(:key, 1) } - :cleanup | [] | ->(s) { s.cleanup } - :clear | [] | ->(s) { s.clear } - end - - with_them do - it 'publishes a notification' do - event = nil - - begin - subscriber = ActiveSupport::Notifications.subscribe("redis.rack_attack") do |*args| - event = ActiveSupport::Notifications::Event.new(*args) - end - - test_proc.call(subject) - ensure - ActiveSupport::Notifications.unsubscribe(subscriber) if subscriber - end - - expect(event).not_to be_nil - expect(event.name).to eq("redis.rack_attack") - expect(event.duration).to be_a(Float).and(be > 0.0) - expect(event.payload[:operation]).to eql(operation) - end - - it 'publishes a notification even if the cache store returns an error' do - allow(store).to receive(operation).and_raise('Something went wrong') - - event = nil - exception = nil - - begin - subscriber = ActiveSupport::Notifications.subscribe("redis.rack_attack") do |*args| - event = ActiveSupport::Notifications::Event.new(*args) - end - - begin - test_proc.call(subject) - rescue StandardError => e - exception = e - end - ensure - ActiveSupport::Notifications.unsubscribe(subscriber) if subscriber - end - - expect(event).not_to be_nil - expect(event.name).to eq("redis.rack_attack") - expect(event.duration).to be_a(Float).and(be > 0.0) - expect(event.payload[:operation]).to eql(operation) - - expect(exception).not_to be_nil - expect(exception.message).to eql('Something went wrong') - end - - it 'delegates to the upstream store' do - allow(store).to receive(operation).and_call_original - - if params.empty? - expect(store).to receive(operation).with(no_args) - else - expect(store).to receive(operation).with(*params) - end - - test_proc.call(subject) - end - end -end diff --git a/spec/lib/gitlab/rack_attack/store_spec.rb b/spec/lib/gitlab/rack_attack/store_spec.rb new file mode 100644 index 00000000000..19b3f239d91 --- /dev/null +++ b/spec/lib/gitlab/rack_attack/store_spec.rb @@ -0,0 +1,113 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::RackAttack::Store, :clean_gitlab_redis_rate_limiting, feature_category: :scalability do + let(:store) { described_class.new } + let(:key) { 'foobar' } + let(:namespaced_key) { "cache:gitlab:#{key}" } + + def with_redis(&block) + Gitlab::Redis::RateLimiting.with(&block) + end + + describe '#increment' do + it 'increments without expiry' do + 5.times do |i| + expect(store.increment(key, 1)).to eq(i + 1) + + with_redis do |redis| + expect(redis.get(namespaced_key).to_i).to eq(i + 1) + expect(redis.ttl(namespaced_key)).to eq(-1) + end + end + end + + it 'rejects amounts other than 1' do + expect { store.increment(key, 2) }.to raise_exception(described_class::InvalidAmount) + end + + context 'with expiry' do + it 'increments and sets expiry' do + 5.times do |i| + expect(store.increment(key, 1, expires_in: 456)).to eq(i + 1) + + with_redis do |redis| + expect(redis.get(namespaced_key).to_i).to eq(i + 1) + expect(redis.ttl(namespaced_key)).to be_within(10).of(456) + end + end + end + end + end + + describe '#read' do + subject { store.read(key) } + + it 'reads the namespaced key' do + with_redis { |r| r.set(namespaced_key, '123') } + + expect(subject).to eq('123') + end + end + + describe '#write' do + subject { store.write(key, '123', options) } + + let(:options) { {} } + + it 'sets the key' do + subject + + with_redis do |redis| + expect(redis.get(namespaced_key)).to eq('123') + expect(redis.ttl(namespaced_key)).to eq(-1) + end + end + + context 'with expiry' do + let(:options) { { expires_in: 456 } } + + it 'sets the key with expiry' do + subject + + with_redis do |redis| + expect(redis.get(namespaced_key)).to eq('123') + expect(redis.ttl(namespaced_key)).to be_within(10).of(456) + end + end + end + end + + describe '#delete' do + subject { store.delete(key) } + + it { expect(subject).to eq(0) } + + context 'when the key exists' do + before do + with_redis { |r| r.set(namespaced_key, '123') } + end + + it { expect(subject).to eq(1) } + end + end + + describe '#with' do + subject { store.send(:with, &:ping) } + + it { expect(subject).to eq('PONG') } + + context 'when redis is unavailable' do + before do + broken_redis = Redis.new( + url: 'redis://127.0.0.0:0', + instrumentation_class: Gitlab::Redis::RateLimiting.instrumentation_class + ) + allow(Gitlab::Redis::RateLimiting).to receive(:with).and_yield(broken_redis) + end + + it { expect(subject).to eq(nil) } + end + end +end diff --git a/spec/lib/gitlab/redis/cache_spec.rb b/spec/lib/gitlab/redis/cache_spec.rb index 64615c4d9ad..82ff8a26199 100644 --- a/spec/lib/gitlab/redis/cache_spec.rb +++ b/spec/lib/gitlab/redis/cache_spec.rb @@ -26,22 +26,5 @@ RSpec.describe Gitlab::Redis::Cache do expect(described_class.active_support_config[:expires_in]).to eq(1.day) end - - context 'when encountering an error' do - let(:cache) { ActiveSupport::Cache::RedisCacheStore.new(**described_class.active_support_config) } - - subject { cache.read('x') } - - before do - described_class.with do |redis| - allow(redis).to receive(:get).and_raise(::Redis::CommandError) - end - end - - it 'logs error' do - expect(::Gitlab::ErrorTracking).to receive(:log_exception) - subject - end - end end end diff --git a/spec/lib/gitlab/redis/multi_store_spec.rb b/spec/lib/gitlab/redis/multi_store_spec.rb index 423a7e80ead..baf2546fc5c 100644 --- a/spec/lib/gitlab/redis/multi_store_spec.rb +++ b/spec/lib/gitlab/redis/multi_store_spec.rb @@ -210,47 +210,6 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do end end - RSpec.shared_examples_for 'fallback read from the non-default store' do - let(:counter) { Gitlab::Metrics::NullMetric.instance } - - before do - allow(Gitlab::Metrics).to receive(:counter).and_return(counter) - end - - it 'fallback and execute on secondary instance' do - expect(multi_store.fallback_store).to receive(name).with(*expected_args).and_call_original - - subject - end - - it 'logs the ReadFromPrimaryError' do - expect(Gitlab::ErrorTracking).to receive(:log_exception).with( - an_instance_of(Gitlab::Redis::MultiStore::ReadFromPrimaryError), - hash_including(command_name: name, instance_name: instance_name) - ) - - subject - end - - it 'increment read fallback count metrics' do - expect(counter).to receive(:increment).with(command: name, instance_name: instance_name) - - subject - end - - include_examples 'reads correct value' - - context 'when fallback read from the secondary instance raises an exception' do - before do - allow(multi_store.fallback_store).to receive(name).with(*expected_args).and_raise(StandardError) - end - - it 'fails with exception' do - expect { subject }.to raise_error(StandardError) - end - end - end - RSpec.shared_examples_for 'secondary store' do it 'execute on the secondary instance' do expect(secondary_store).to receive(name).with(*expected_args).and_call_original @@ -283,31 +242,21 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do subject end - unless params[:block] - it 'does not execute on the secondary store' do - expect(secondary_store).not_to receive(name) - - subject - end - end - include_examples 'reads correct value' end - context 'when reading from primary instance is raising an exception' do + context 'when reading from default instance is raising an exception' do before do allow(multi_store.default_store).to receive(name).with(*expected_args).and_raise(StandardError) allow(Gitlab::ErrorTracking).to receive(:log_exception) end - it 'logs the exception' do + it 'logs the exception and re-raises the error' do expect(Gitlab::ErrorTracking).to receive(:log_exception).with(an_instance_of(StandardError), hash_including(:multi_store_error_message, instance_name: instance_name, command_name: name)) - subject + expect { subject }.to raise_error(an_instance_of(StandardError)) end - - include_examples 'fallback read from the non-default store' end context 'when reading from empty default instance' do @@ -316,7 +265,9 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do multi_store.default_store.flushdb end - include_examples 'fallback read from the non-default store' + it 'does not call the fallback store' do + expect(multi_store.fallback_store).not_to receive(name) + end end context 'when the command is executed within pipelined block' do @@ -346,16 +297,16 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do end context 'when block is provided' do - it 'both stores yields to the block' do + it 'only default store yields to the block' do expect(primary_store).to receive(name).and_yield(value) - expect(secondary_store).to receive(name).and_yield(value) + expect(secondary_store).not_to receive(name).and_yield(value) subject end - it 'both stores to execute' do + it 'only default store to execute' do expect(primary_store).to receive(name).with(*expected_args).and_call_original - expect(secondary_store).to receive(name).with(*expected_args).and_call_original + expect(secondary_store).not_to receive(name).with(*expected_args).and_call_original subject end @@ -421,27 +372,19 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do subject do multi_store.mget(values) do |v| multi_store.sadd(skey, v) - multi_store.scard(skey) - end - end - - RSpec.shared_examples_for 'primary instance executes block' do - it 'ensures primary instance is executing the block' do - expect(primary_store).to receive(:send).with(:mget, values).and_call_original - expect(primary_store).to receive(:send).with(:sadd, skey, %w[1 2 3]).and_call_original - expect(primary_store).to receive(:send).with(:scard, skey).and_call_original - - expect(secondary_store).to receive(:send).with(:mget, values).and_call_original - expect(secondary_store).to receive(:send).with(:sadd, skey, %w[10 20 30]).and_call_original - expect(secondary_store).to receive(:send).with(:scard, skey).and_call_original - - subject end end context 'when using both stores' do context 'when primary instance is default store' do - it_behaves_like 'primary instance executes block' + it 'ensures primary instance is executing the block' do + expect(primary_store).to receive(:send).with(:mget, values).and_call_original + expect(primary_store).to receive(:send).with(:sadd, skey, %w[1 2 3]).and_call_original + + expect(secondary_store).not_to receive(:send) + + subject + end end context 'when secondary instance is default store' do @@ -449,8 +392,14 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do stub_feature_flags(use_primary_store_as_default_for_test_store: false) end - # multistore read still favours the primary store - it_behaves_like 'primary instance executes block' + it 'ensures secondary instance is executing the block' do + expect(primary_store).not_to receive(:send) + + expect(secondary_store).to receive(:send).with(:mget, values).and_call_original + expect(secondary_store).to receive(:send).with(:sadd, skey, %w[10 20 30]).and_call_original + + subject + end end end @@ -465,7 +414,6 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do expect(primary_store).to receive(:send).with(:mget, values).and_call_original expect(primary_store).to receive(:send).with(:sadd, skey, %w[1 2 3]).and_call_original - expect(primary_store).to receive(:send).with(:scard, skey).and_call_original subject end @@ -479,7 +427,6 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do it 'ensures only secondary instance is executing the block' do expect(secondary_store).to receive(:send).with(:mget, values).and_call_original expect(secondary_store).to receive(:send).with(:sadd, skey, %w[10 20 30]).and_call_original - expect(secondary_store).to receive(:send).with(:scard, skey).and_call_original expect(primary_store).not_to receive(:send) @@ -668,120 +615,6 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do end # rubocop:enable RSpec/MultipleMemoizedHelpers - context 'with ENUMERATOR_COMMANDS redis commands' do - let_it_be(:hkey) { "redis:hash" } - let_it_be(:skey) { "redis:set" } - let_it_be(:zkey) { "redis:sortedset" } - let_it_be(:rvalue) { "value1" } - let_it_be(:scan_kwargs) { { match: 'redis:hash' } } - - where(:case_name, :name, :args, :kwargs) do - 'execute :scan_each command' | :scan_each | nil | ref(:scan_kwargs) - 'execute :sscan_each command' | :sscan_each | ref(:skey) | {} - 'execute :hscan_each command' | :hscan_each | ref(:hkey) | {} - 'execute :zscan_each command' | :zscan_each | ref(:zkey) | {} - end - - before(:all) do - primary_store.hset(hkey, rvalue, 1) - primary_store.sadd?(skey, rvalue) - primary_store.zadd(zkey, 1, rvalue) - - secondary_store.hset(hkey, rvalue, 1) - secondary_store.sadd?(skey, rvalue) - secondary_store.zadd(zkey, 1, rvalue) - end - - RSpec.shared_examples_for 'enumerator commands execution' do |both_stores, default_primary| - context 'without block passed in' do - subject do - multi_store.send(name, *args, **kwargs) - end - - it 'returns an enumerator' do - expect(subject).to be_instance_of(Enumerator) - end - end - - context 'with block passed in' do - subject do - multi_store.send(name, *args, **kwargs) { |key| multi_store.incr(rvalue) } - end - - it 'returns nil' do - expect(subject).to eq(nil) - end - - it 'runs block on correct Redis instance' do - if both_stores - expect(primary_store).to receive(name).with(*expected_args).and_call_original - expect(secondary_store).to receive(name).with(*expected_args).and_call_original - - expect(primary_store).to receive(:incr).with(rvalue) - expect(secondary_store).to receive(:incr).with(rvalue) - elsif default_primary - expect(primary_store).to receive(name).with(*expected_args).and_call_original - expect(primary_store).to receive(:incr).with(rvalue) - - expect(secondary_store).not_to receive(name) - expect(secondary_store).not_to receive(:incr).with(rvalue) - else - expect(secondary_store).to receive(name).with(*expected_args).and_call_original - expect(secondary_store).to receive(:incr).with(rvalue) - - expect(primary_store).not_to receive(name) - expect(primary_store).not_to receive(:incr).with(rvalue) - end - - subject - end - end - end - - with_them do - describe name.to_s do - let(:expected_args) { kwargs.present? ? [*args, { **kwargs }] : Array(args) } - - before do - allow(primary_store).to receive(name).and_call_original - allow(secondary_store).to receive(name).and_call_original - end - - context 'when only using 1 store' do - before do - stub_feature_flags(use_primary_and_secondary_stores_for_test_store: false) - end - - context 'when using secondary store as default' do - before do - stub_feature_flags(use_primary_store_as_default_for_test_store: false) - end - - it_behaves_like 'enumerator commands execution', false, false - end - - context 'when using primary store as default' do - it_behaves_like 'enumerator commands execution', false, true - end - end - - context 'when using both stores' do - context 'when using secondary store as default' do - before do - stub_feature_flags(use_primary_store_as_default_for_test_store: false) - end - - it_behaves_like 'enumerator commands execution', true, false - end - - context 'when using primary store as default' do - it_behaves_like 'enumerator commands execution', true, true - end - end - end - end - end - RSpec.shared_examples_for 'pipelined command' do |name| let_it_be(:key1) { "redis:{1}:key_a" } let_it_be(:value1) { "redis_value1" } diff --git a/spec/lib/gitlab/redis/rate_limiting_spec.rb b/spec/lib/gitlab/redis/rate_limiting_spec.rb index d82228426f0..0bea7f8bcb2 100644 --- a/spec/lib/gitlab/redis/rate_limiting_spec.rb +++ b/spec/lib/gitlab/redis/rate_limiting_spec.rb @@ -6,19 +6,8 @@ RSpec.describe Gitlab::Redis::RateLimiting do include_examples "redis_new_instance_shared_examples", 'rate_limiting', Gitlab::Redis::Cache describe '.cache_store' do - context 'when encountering an error' do - subject { described_class.cache_store.read('x') } - - before do - described_class.with do |redis| - allow(redis).to receive(:get).and_raise(::Redis::CommandError) - end - end - - it 'logs error' do - expect(::Gitlab::ErrorTracking).to receive(:log_exception) - subject - end + it 'uses the CACHE_NAMESPACE namespace' do + expect(described_class.cache_store.options[:namespace]).to eq(Gitlab::Redis::Cache::CACHE_NAMESPACE) end end end diff --git a/spec/lib/gitlab/redis/repository_cache_spec.rb b/spec/lib/gitlab/redis/repository_cache_spec.rb index 2c167a6eb62..8cdc4580f9e 100644 --- a/spec/lib/gitlab/redis/repository_cache_spec.rb +++ b/spec/lib/gitlab/redis/repository_cache_spec.rb @@ -17,20 +17,5 @@ RSpec.describe Gitlab::Redis::RepositoryCache, feature_category: :scalability do it 'has a default ttl of 8 hours' do expect(described_class.cache_store.options[:expires_in]).to eq(8.hours) end - - context 'when encountering an error' do - subject { described_class.cache_store.read('x') } - - before do - described_class.with do |redis| - allow(redis).to receive(:get).and_raise(::Redis::CommandError) - end - end - - it 'logs error' do - expect(::Gitlab::ErrorTracking).to receive(:log_exception) - subject - end - end end end diff --git a/spec/lib/gitlab/regex_spec.rb b/spec/lib/gitlab/regex_spec.rb index 31de4068bc5..d885051b93b 100644 --- a/spec/lib/gitlab/regex_spec.rb +++ b/spec/lib/gitlab/regex_spec.rb @@ -110,6 +110,8 @@ RSpec.describe Gitlab::Regex, feature_category: :tooling do it { is_expected.to match('.source/.full/.path') } it { is_expected.to match('domain_namespace') } it { is_expected.to match('gitlab-migration-test') } + it { is_expected.to match('1-project-path') } + it { is_expected.to match('e-project-path') } it { is_expected.to match('') } # it is possible to pass an empty string for destination_namespace in bulk_import POST request end @@ -710,6 +712,7 @@ RSpec.describe Gitlab::Regex, feature_category: :tooling do it { is_expected.to match('libsample0_1.2.3~alpha2_amd64.deb') } it { is_expected.to match('sample-dev_1.2.3~binary_amd64.deb') } it { is_expected.to match('sample-udeb_1.2.3~alpha2_amd64.udeb') } + it { is_expected.to match('sample-ddeb_1.2.3~alpha2_amd64.ddeb') } it { is_expected.not_to match('sample_1.2.3~alpha2_amd64.buildinfo') } it { is_expected.not_to match('sample_1.2.3~alpha2_amd64.changes') } @@ -1015,6 +1018,34 @@ RSpec.describe Gitlab::Regex, feature_category: :tooling do it { is_expected.not_to match('/api/v4/groups/1234/packages/debian/pool/compon/a/pkg/file.name') } end + describe 'Packages::MAVEN_SNAPSHOT_DYNAMIC_PARTS' do + subject { described_class::Packages::MAVEN_SNAPSHOT_DYNAMIC_PARTS } + + it { is_expected.to match('test-2.11-20230303.163304-1.jar') } + it { is_expected.to match('test-2.11-20230303.163304-1-javadoc.jar') } + it { is_expected.to match('test-2.11-20230303.163304-1-sources.jar') } + it { is_expected.to match('test-2.11-20230303.163304-1-20230303.163304-1.jar') } + it { is_expected.to match('test-2.11-20230303.163304-1-20230303.163304-1-javadoc.jar') } + it { is_expected.to match('test-2.11-20230303.163304-1-20230303.163304-1-sources.jar') } + it { is_expected.to match("#{'a' * 500}-20230303.163304-1-sources.jar") } + it { is_expected.to match("test-2.11-20230303.163304-1-#{'a' * 500}.jar") } + it { is_expected.to match("#{'a' * 500}-20230303.163304-1-#{'a' * 500}.jar") } + + it { is_expected.not_to match('') } + it { is_expected.not_to match(nil) } + it { is_expected.not_to match('test') } + it { is_expected.not_to match('1.2.3') } + it { is_expected.not_to match('1.2.3-javadoc.jar') } + it { is_expected.not_to match('-202303039.163304-1.jar') } + it { is_expected.not_to match('test-2.11-202303039.163304-1.jar') } + it { is_expected.not_to match('test-2.11-20230303.16330-1.jar') } + it { is_expected.not_to match('test-2.11-202303039.163304.jar') } + it { is_expected.not_to match('test-2.11-202303039.163304-.jar') } + it { is_expected.not_to match("#{'a' * 2000}-20230303.163304-1-sources.jar") } + it { is_expected.not_to match("test-2.11-20230303.163304-1-#{'a' * 2000}.jar") } + it { is_expected.not_to match("#{'a' * 2000}-20230303.163304-1-#{'a' * 2000}.jar") } + end + describe '.composer_package_version_regex' do subject { described_class.composer_package_version_regex } diff --git a/spec/lib/gitlab/safe_device_detector_spec.rb b/spec/lib/gitlab/safe_device_detector_spec.rb index c37dc1e1c7e..56ba084c435 100644 --- a/spec/lib/gitlab/safe_device_detector_spec.rb +++ b/spec/lib/gitlab/safe_device_detector_spec.rb @@ -4,7 +4,7 @@ require 'fast_spec_helper' require 'device_detector' require_relative '../../../lib/gitlab/safe_device_detector' -RSpec.describe Gitlab::SafeDeviceDetector, feature_category: :authentication_and_authorization do +RSpec.describe Gitlab::SafeDeviceDetector, feature_category: :system_access do it 'retains the behavior for normal user agents' do chrome_user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 \ (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36" diff --git a/spec/lib/gitlab/sanitizers/exception_message_spec.rb b/spec/lib/gitlab/sanitizers/exception_message_spec.rb index 8b54b353235..c2c4a5de32d 100644 --- a/spec/lib/gitlab/sanitizers/exception_message_spec.rb +++ b/spec/lib/gitlab/sanitizers/exception_message_spec.rb @@ -1,9 +1,10 @@ # frozen_string_literal: true require 'fast_spec_helper' +require 'addressable' require 'rspec-parameterized' -RSpec.describe Gitlab::Sanitizers::ExceptionMessage do +RSpec.describe Gitlab::Sanitizers::ExceptionMessage, feature_category: :compliance_management do describe '.clean' do let(:exception_name) { exception.class.name } let(:exception_message) { exception.message } diff --git a/spec/lib/gitlab/seeders/ci/runner/runner_fleet_seeder_spec.rb b/spec/lib/gitlab/seeders/ci/runner/runner_fleet_seeder_spec.rb index fe52b586d49..4597cc6b315 100644 --- a/spec/lib/gitlab/seeders/ci/runner/runner_fleet_seeder_spec.rb +++ b/spec/lib/gitlab/seeders/ci/runner/runner_fleet_seeder_spec.rb @@ -67,5 +67,29 @@ RSpec.describe ::Gitlab::Seeders::Ci::Runner::RunnerFleetSeeder, feature_categor expect(::Ci::Build.where(runner_id: project[:runner_ids])).to be_empty end end + + context 'when number of group runners exceeds plan limit' do + before do + create(:plan_limits, :default_plan, ci_registered_group_runners: 1) + end + + it { is_expected.to be_nil } + + it 'does not change runner count' do + expect { seed }.not_to change { Ci::Runner.count } + end + end + + context 'when number of project runners exceeds plan limit' do + before do + create(:plan_limits, :default_plan, ci_registered_project_runners: 1) + end + + it { is_expected.to be_nil } + + it 'does not change runner count' do + expect { seed }.not_to change { Ci::Runner.count } + end + end end end diff --git a/spec/lib/gitlab/serverless/service_spec.rb b/spec/lib/gitlab/serverless/service_spec.rb deleted file mode 100644 index 3400be5b48e..00000000000 --- a/spec/lib/gitlab/serverless/service_spec.rb +++ /dev/null @@ -1,136 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Serverless::Service do - let(:cluster) { create(:cluster) } - let(:environment) { create(:environment) } - let(:attributes) do - { - 'apiVersion' => 'serving.knative.dev/v1alpha1', - 'kind' => 'Service', - 'metadata' => { - 'creationTimestamp' => '2019-10-22T21:19:13Z', - 'name' => 'kubetest', - 'namespace' => 'project1-1-environment1' - }, - 'spec' => { - 'runLatest' => { - 'configuration' => { - 'build' => { - 'template' => { - 'name' => 'some-image' - } - } - } - } - }, - 'environment_scope' => '*', - 'cluster' => cluster, - 'environment' => environment, - 'podcount' => 0 - } - end - - it 'exposes methods extracting data from the attributes hash' do - service = Gitlab::Serverless::Service.new(attributes) - - expect(service.name).to eq('kubetest') - expect(service.namespace).to eq('project1-1-environment1') - expect(service.environment_scope).to eq('*') - expect(service.podcount).to eq(0) - expect(service.created_at).to eq(DateTime.parse('2019-10-22T21:19:13Z')) - expect(service.image).to eq('some-image') - expect(service.cluster).to eq(cluster) - expect(service.environment).to eq(environment) - end - - it 'returns nil for missing attributes' do - service = Gitlab::Serverless::Service.new({}) - - [:name, :namespace, :environment_scope, :cluster, :podcount, :created_at, :image, :description, :url, :environment].each do |method| - expect(service.send(method)).to be_nil - end - end - - describe '#description' do - it 'extracts the description in knative 7 format if available' do - attributes = { - 'spec' => { - 'template' => { - 'metadata' => { - 'annotations' => { - 'Description' => 'some description' - } - } - } - } - } - service = Gitlab::Serverless::Service.new(attributes) - - expect(service.description).to eq('some description') - end - - it 'extracts the description in knative 5/6 format if 7 is not available' do - attributes = { - 'spec' => { - 'runLatest' => { - 'configuration' => { - 'revisionTemplate' => { - 'metadata' => { - 'annotations' => { - 'Description' => 'some description' - } - } - } - } - } - } - } - service = Gitlab::Serverless::Service.new(attributes) - - expect(service.description).to eq('some description') - end - end - - describe '#url' do - let(:serverless_domain) { instance_double(::Serverless::Domain, uri: URI('https://proxy.example.com')) } - - it 'returns proxy URL if cluster has serverless domain' do - # cluster = create(:cluster) - knative = create(:clusters_applications_knative, :installed, cluster: cluster) - create(:serverless_domain_cluster, clusters_applications_knative_id: knative.id) - service = Gitlab::Serverless::Service.new(attributes.merge('cluster' => cluster)) - - expect(::Serverless::Domain).to receive(:new).with( - function_name: service.name, - serverless_domain_cluster: service.cluster.serverless_domain, - environment: service.environment - ).and_return(serverless_domain) - - expect(service.url).to eq('https://proxy.example.com') - end - - it 'returns the URL from the knative 6/7 format' do - attributes = { - 'status' => { - 'url' => 'https://example.com' - } - } - service = Gitlab::Serverless::Service.new(attributes) - - expect(service.url).to eq('https://example.com') - end - - it 'returns the URL from the knative 5 format' do - attributes = { - 'status' => { - 'domain' => 'example.com' - } - } - service = Gitlab::Serverless::Service.new(attributes) - - expect(service.url).to eq('http://example.com') - end - end -end diff --git a/spec/lib/gitlab/sidekiq_middleware/server_metrics_spec.rb b/spec/lib/gitlab/sidekiq_middleware/server_metrics_spec.rb index f7cee6beb58..80c1af1b913 100644 --- a/spec/lib/gitlab/sidekiq_middleware/server_metrics_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware/server_metrics_spec.rb @@ -331,7 +331,7 @@ RSpec.describe Gitlab::SidekiqMiddleware::ServerMetrics do include_context 'server metrics call' context 'when a worker has a feature category' do - let(:worker_category) { 'authentication_and_authorization' } + let(:worker_category) { 'system_access' } it 'uses that category for metrics' do expect(completion_seconds_metric).to receive(:observe).with(a_hash_including(feature_category: worker_category), anything) diff --git a/spec/lib/gitlab/sidekiq_middleware/worker_context/client_spec.rb b/spec/lib/gitlab/sidekiq_middleware/worker_context/client_spec.rb index 1b6cd7ac5fb..4fbc64a45d6 100644 --- a/spec/lib/gitlab/sidekiq_middleware/worker_context/client_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware/worker_context/client_spec.rb @@ -123,7 +123,7 @@ RSpec.describe Gitlab::SidekiqMiddleware::WorkerContext::Client do context 'when the feature category is already set in the surrounding block' do it 'takes the feature category from the worker, not the caller' do - Gitlab::ApplicationContext.with_context(feature_category: 'authentication_and_authorization') do + Gitlab::ApplicationContext.with_context(feature_category: 'system_access') do TestWithContextWorker.bulk_perform_async_with_contexts( %w(job1 job2), arguments_proc: -> (name) { [name, 1, 2, 3] }, @@ -139,7 +139,7 @@ RSpec.describe Gitlab::SidekiqMiddleware::WorkerContext::Client do end it 'takes the feature category from the caller if the worker is not owned' do - Gitlab::ApplicationContext.with_context(feature_category: 'authentication_and_authorization') do + Gitlab::ApplicationContext.with_context(feature_category: 'system_access') do TestNotOwnedWithContextWorker.bulk_perform_async_with_contexts( %w(job1 job2), arguments_proc: -> (name) { [name, 1, 2, 3] }, @@ -150,8 +150,8 @@ RSpec.describe Gitlab::SidekiqMiddleware::WorkerContext::Client do job1 = TestNotOwnedWithContextWorker.job_for_args(['job1', 1, 2, 3]) job2 = TestNotOwnedWithContextWorker.job_for_args(['job2', 1, 2, 3]) - expect(job1['meta.feature_category']).to eq('authentication_and_authorization') - expect(job2['meta.feature_category']).to eq('authentication_and_authorization') + expect(job1['meta.feature_category']).to eq('system_access') + expect(job2['meta.feature_category']).to eq('system_access') end end end diff --git a/spec/lib/gitlab/sidekiq_middleware/worker_context/server_spec.rb b/spec/lib/gitlab/sidekiq_middleware/worker_context/server_spec.rb index 2deab3064eb..eb077a0371c 100644 --- a/spec/lib/gitlab/sidekiq_middleware/worker_context/server_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware/worker_context/server_spec.rb @@ -69,7 +69,7 @@ RSpec.describe Gitlab::SidekiqMiddleware::WorkerContext::Server do context 'feature category' do it 'takes the feature category from the worker' do - Gitlab::ApplicationContext.with_context(feature_category: 'authentication_and_authorization') do + Gitlab::ApplicationContext.with_context(feature_category: 'system_access') do TestWorker.perform_async('identifier', 1) end @@ -78,11 +78,11 @@ RSpec.describe Gitlab::SidekiqMiddleware::WorkerContext::Server do context 'when the worker is not owned' do it 'takes the feature category from the surrounding context' do - Gitlab::ApplicationContext.with_context(feature_category: 'authentication_and_authorization') do + Gitlab::ApplicationContext.with_context(feature_category: 'system_access') do NotOwnedWorker.perform_async('identifier', 1) end - expect(NotOwnedWorker.contexts['identifier']).to include('meta.feature_category' => 'authentication_and_authorization') + expect(NotOwnedWorker.contexts['identifier']).to include('meta.feature_category' => 'system_access') end end end diff --git a/spec/lib/gitlab/sidekiq_queue_spec.rb b/spec/lib/gitlab/sidekiq_queue_spec.rb index 5e91282612e..93632848788 100644 --- a/spec/lib/gitlab/sidekiq_queue_spec.rb +++ b/spec/lib/gitlab/sidekiq_queue_spec.rb @@ -4,15 +4,15 @@ require 'spec_helper' RSpec.describe Gitlab::SidekiqQueue, :clean_gitlab_redis_queues do around do |example| - Sidekiq::Queue.new('default').clear + Sidekiq::Queue.new('foobar').clear Sidekiq::Testing.disable!(&example) - Sidekiq::Queue.new('default').clear + Sidekiq::Queue.new('foobar').clear end def add_job(args, user:, klass: 'AuthorizedProjectsWorker') Sidekiq::Client.push( 'class' => klass, - 'queue' => 'default', + 'queue' => 'foobar', 'args' => args, 'meta.user' => user.username ) @@ -20,7 +20,7 @@ RSpec.describe Gitlab::SidekiqQueue, :clean_gitlab_redis_queues do describe '#drop_jobs!' do shared_examples 'queue processing' do - let(:sidekiq_queue) { described_class.new('default') } + let(:sidekiq_queue) { described_class.new('foobar') } let_it_be(:sidekiq_queue_user) { create(:user) } before do @@ -80,7 +80,7 @@ RSpec.describe Gitlab::SidekiqQueue, :clean_gitlab_redis_queues do it 'raises NoMetadataError' do add_job([1], user: create(:user)) - expect { described_class.new('default').drop_jobs!({ username: 'sidekiq_queue_user' }, timeout: 1) } + expect { described_class.new('foobar').drop_jobs!({ username: 'sidekiq_queue_user' }, timeout: 1) } .to raise_error(described_class::NoMetadataError) end end diff --git a/spec/lib/gitlab/slug/path_spec.rb b/spec/lib/gitlab/slug/path_spec.rb index 9a7067e40a2..bbc2a05713d 100644 --- a/spec/lib/gitlab/slug/path_spec.rb +++ b/spec/lib/gitlab/slug/path_spec.rb @@ -2,7 +2,7 @@ require 'fast_spec_helper' -RSpec.describe Gitlab::Slug::Path, feature_category: :not_owned do +RSpec.describe Gitlab::Slug::Path, feature_category: :shared do describe '#generate' do { 'name': 'name', diff --git a/spec/lib/gitlab/url_blocker_spec.rb b/spec/lib/gitlab/url_blocker_spec.rb index 05f7af7606d..912093be29f 100644 --- a/spec/lib/gitlab/url_blocker_spec.rb +++ b/spec/lib/gitlab/url_blocker_spec.rb @@ -8,7 +8,9 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do let(:schemes) { %w[http https] } describe '#validate!' do - subject { described_class.validate!(import_url, schemes: schemes) } + let(:options) { { schemes: schemes } } + + subject { described_class.validate!(import_url, **options) } shared_examples 'validates URI and hostname' do it 'runs the url validations' do @@ -19,6 +21,73 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do end end + shared_context 'instance configured to deny all requests' do + before do + allow(Gitlab::CurrentSettings).to receive(:current_application_settings?).and_return(true) + stub_application_setting(deny_all_requests_except_allowed: true) + end + end + + shared_examples 'a URI denied by `deny_all_requests_except_allowed`' do + context 'when instance setting is enabled' do + include_context 'instance configured to deny all requests' + + it 'blocks the request' do + expect { subject }.to raise_error(described_class::BlockedUrlError) + end + end + + context 'when instance setting is not enabled' do + it 'does not block the request' do + expect { subject }.not_to raise_error + end + end + + context 'when passed as an argument' do + let(:options) { super().merge(deny_all_requests_except_allowed: arg_value) } + + context 'when argument is a proc that evaluates to true' do + let(:arg_value) { proc { true } } + + it 'blocks the request' do + expect { subject }.to raise_error(described_class::BlockedUrlError) + end + end + + context 'when argument is a proc that evaluates to false' do + let(:arg_value) { proc { false } } + + it 'does not block the request' do + expect { subject }.not_to raise_error + end + end + + context 'when argument is true' do + let(:arg_value) { true } + + it 'blocks the request' do + expect { subject }.to raise_error(described_class::BlockedUrlError) + end + end + + context 'when argument is false' do + let(:arg_value) { false } + + it 'does not block the request' do + expect { subject }.not_to raise_error + end + end + end + end + + shared_examples 'a URI exempt from `deny_all_requests_except_allowed`' do + include_context 'instance configured to deny all requests' + + it 'does not block the request' do + expect { subject }.not_to raise_error + end + end + context 'when URI is nil' do let(:import_url) { nil } @@ -26,6 +95,8 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do let(:expected_uri) { nil } let(:expected_hostname) { nil } end + + it_behaves_like 'a URI exempt from `deny_all_requests_except_allowed`' end context 'when URI is internal' do @@ -39,6 +110,8 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do let(:expected_uri) { 'http://127.0.0.1' } let(:expected_hostname) { 'localhost' } end + + it_behaves_like 'a URI exempt from `deny_all_requests_except_allowed`' end context 'when URI is for a local object storage' do @@ -61,7 +134,7 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do end context 'when allow_object_storage is true' do - subject { described_class.validate!(import_url, allow_object_storage: true, schemes: schemes) } + let(:options) { { allow_object_storage: true, schemes: schemes } } context 'with a local domain name' do let(:host) { 'http://review-minio-svc.svc:9000' } @@ -74,6 +147,8 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do let(:expected_uri) { 'http://127.0.0.1:9000/external-diffs/merge_request_diffs/mr-1/diff-1' } let(:expected_hostname) { 'review-minio-svc.svc' } end + + it_behaves_like 'a URI exempt from `deny_all_requests_except_allowed`' end context 'with an IP address' do @@ -83,6 +158,8 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do let(:expected_uri) { 'http://127.0.0.1:9000/external-diffs/merge_request_diffs/mr-1/diff-1' } let(:expected_hostname) { nil } end + + it_behaves_like 'a URI exempt from `deny_all_requests_except_allowed`' end context 'when LFS object storage is enabled' do @@ -164,6 +241,8 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do let(:expected_uri) { 'https://93.184.216.34' } let(:expected_hostname) { 'example.org' } end + + it_behaves_like 'a URI denied by `deny_all_requests_except_allowed`' end context 'when domain cannot be resolved' do @@ -193,6 +272,8 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do let(:expected_hostname) { nil } end + it_behaves_like 'a URI denied by `deny_all_requests_except_allowed`' + context 'when the address is invalid' do let(:import_url) { 'http://1.1.1.1.1' } @@ -217,10 +298,12 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do let(:expected_uri) { 'http://192.168.0.120:9121/scrape?target=unix:///var/opt/gitlab/redis/redis.socket&check-keys=*' } let(:expected_hostname) { 'a.192.168.0.120.3times.127.0.0.1.1time.repeat.rebind.network' } end + + it_behaves_like 'a URI exempt from `deny_all_requests_except_allowed`' end context 'disabled DNS rebinding protection' do - subject { described_class.validate!(import_url, dns_rebind_protection: false, schemes: schemes) } + let(:options) { { dns_rebind_protection: false, schemes: schemes } } context 'when URI is internal' do let(:import_url) { 'http://localhost' } @@ -229,6 +312,8 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do let(:expected_uri) { import_url } let(:expected_hostname) { nil } end + + it_behaves_like 'a URI exempt from `deny_all_requests_except_allowed`' end context 'when the URL hostname is a domain' do @@ -243,6 +328,8 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do let(:expected_uri) { import_url } let(:expected_hostname) { nil } end + + it_behaves_like 'a URI denied by `deny_all_requests_except_allowed`' end context 'when domain cannot be resolved' do @@ -252,6 +339,8 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do let(:expected_uri) { import_url } let(:expected_hostname) { nil } end + + it_behaves_like 'a URI denied by `deny_all_requests_except_allowed`' end end @@ -263,6 +352,8 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do let(:expected_hostname) { nil } end + it_behaves_like 'a URI denied by `deny_all_requests_except_allowed`' + context 'when it is invalid' do let(:import_url) { 'http://1.1.1.1.1' } @@ -270,6 +361,8 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do let(:expected_uri) { import_url } let(:expected_hostname) { nil } end + + it_behaves_like 'a URI denied by `deny_all_requests_except_allowed`' end end end diff --git a/spec/lib/gitlab/url_builder_spec.rb b/spec/lib/gitlab/url_builder_spec.rb index 2e9a444bd24..08a25666ae9 100644 --- a/spec/lib/gitlab/url_builder_spec.rb +++ b/spec/lib/gitlab/url_builder_spec.rb @@ -227,27 +227,5 @@ RSpec.describe Gitlab::UrlBuilder do expect(subject.build(object, only_path: true)).to eq("/#{project.full_path}") end end - - context 'when use_iid_in_work_items_path feature flag is disabled' do - before do - stub_feature_flags(use_iid_in_work_items_path: false) - end - - context 'when a task issue is passed' do - it 'returns a path using the work item\'s ID and no query params' do - task = create(:issue, :task) - - expect(subject.build(task, only_path: true)).to eq("/#{task.project.full_path}/-/work_items/#{task.id}") - end - end - - context 'when a work item is passed' do - it 'returns a path using the work item\'s ID and no query params' do - work_item = create(:work_item) - - expect(subject.build(work_item, only_path: true)).to eq("/#{work_item.project.full_path}/-/work_items/#{work_item.id}") - end - end - end end end diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric_spec.rb index ce15d44b1e1..317929f77e6 100644 --- a/spec/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric_spec.rb +++ b/spec/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Usage::Metrics::Instrumentations::CountBulkImportsEntitiesMetric do +RSpec.describe Gitlab::Usage::Metrics::Instrumentations::CountBulkImportsEntitiesMetric, feature_category: :importers do let_it_be(:user) { create(:user) } let_it_be(:bulk_import_projects) do create_list(:bulk_import_entity, 2, source_type: 'project_entity', created_at: 3.weeks.ago, status: 2) @@ -163,4 +163,121 @@ RSpec.describe Gitlab::Usage::Metrics::Instrumentations::CountBulkImportsEntitie options: { status: 2, source_type: 'project_entity' } end end + + context 'with has_failures: true' do + before(:all) do + create_list(:bulk_import_entity, 3, :project_entity, :finished, created_at: 3.weeks.ago, has_failures: true) + create_list(:bulk_import_entity, 2, :project_entity, :finished, created_at: 2.months.ago, has_failures: true) + create_list(:bulk_import_entity, 3, :group_entity, :finished, created_at: 3.weeks.ago, has_failures: true) + create_list(:bulk_import_entity, 2, :group_entity, :finished, created_at: 2.months.ago, has_failures: true) + end + + context 'with all time frame' do + context 'with project entity' do + let(:expected_value) { 5 } + let(:expected_query) do + "SELECT COUNT(\"bulk_import_entities\".\"id\") FROM \"bulk_import_entities\" " \ + "WHERE \"bulk_import_entities\".\"source_type\" = 1 AND \"bulk_import_entities\".\"status\" = 2 " \ + "AND \"bulk_import_entities\".\"has_failures\" = TRUE" + end + + it_behaves_like 'a correct instrumented metric value and query', + time_frame: 'all', + options: { status: 2, source_type: 'project_entity', has_failures: true } + end + + context 'with group entity' do + let(:expected_value) { 5 } + let(:expected_query) do + "SELECT COUNT(\"bulk_import_entities\".\"id\") FROM \"bulk_import_entities\" " \ + "WHERE \"bulk_import_entities\".\"source_type\" = 0 AND \"bulk_import_entities\".\"status\" = 2 " \ + "AND \"bulk_import_entities\".\"has_failures\" = TRUE" + end + + it_behaves_like 'a correct instrumented metric value and query', + time_frame: 'all', + options: { status: 2, source_type: 'group_entity', has_failures: true } + end + end + + context 'for 28d time frame' do + let(:expected_value) { 3 } + let(:start) { 30.days.ago.to_s(:db) } + let(:finish) { 2.days.ago.to_s(:db) } + let(:expected_query) do + "SELECT COUNT(\"bulk_import_entities\".\"id\") FROM \"bulk_import_entities\" " \ + "WHERE \"bulk_import_entities\".\"created_at\" BETWEEN '#{start}' AND '#{finish}' " \ + "AND \"bulk_import_entities\".\"source_type\" = 1 AND \"bulk_import_entities\".\"status\" = 2 " \ + "AND \"bulk_import_entities\".\"has_failures\" = TRUE" + end + + it_behaves_like 'a correct instrumented metric value and query', + time_frame: '28d', + options: { status: 2, source_type: 'project_entity', has_failures: true } + end + end + + context 'with has_failures: false' do + context 'with all time frame' do + context 'with project entity' do + let(:expected_value) { 3 } + let(:expected_query) do + "SELECT COUNT(\"bulk_import_entities\".\"id\") FROM \"bulk_import_entities\" " \ + "WHERE \"bulk_import_entities\".\"source_type\" = 1 AND \"bulk_import_entities\".\"status\" = 2 " \ + "AND \"bulk_import_entities\".\"has_failures\" = FALSE" + end + + it_behaves_like 'a correct instrumented metric value and query', + time_frame: 'all', + options: { status: 2, source_type: 'project_entity', has_failures: false } + end + + context 'with group entity' do + let(:expected_value) { 2 } + let(:expected_query) do + "SELECT COUNT(\"bulk_import_entities\".\"id\") FROM \"bulk_import_entities\" " \ + "WHERE \"bulk_import_entities\".\"source_type\" = 0 AND \"bulk_import_entities\".\"status\" = 2 " \ + "AND \"bulk_import_entities\".\"has_failures\" = FALSE" + end + + it_behaves_like 'a correct instrumented metric value and query', + time_frame: 'all', + options: { status: 2, source_type: 'group_entity', has_failures: false } + end + end + + context 'for 28d time frame' do + context 'with project entity' do + let(:expected_value) { 2 } + let(:start) { 30.days.ago.to_s(:db) } + let(:finish) { 2.days.ago.to_s(:db) } + let(:expected_query) do + "SELECT COUNT(\"bulk_import_entities\".\"id\") FROM \"bulk_import_entities\" " \ + "WHERE \"bulk_import_entities\".\"created_at\" BETWEEN '#{start}' AND '#{finish}' " \ + "AND \"bulk_import_entities\".\"source_type\" = 1 AND \"bulk_import_entities\".\"status\" = 2 " \ + "AND \"bulk_import_entities\".\"has_failures\" = FALSE" + end + + it_behaves_like 'a correct instrumented metric value and query', + time_frame: '28d', + options: { status: 2, source_type: 'project_entity', has_failures: false } + end + + context 'with group entity' do + let(:expected_value) { 2 } + let(:start) { 30.days.ago.to_s(:db) } + let(:finish) { 2.days.ago.to_s(:db) } + let(:expected_query) do + "SELECT COUNT(\"bulk_import_entities\".\"id\") FROM \"bulk_import_entities\" " \ + "WHERE \"bulk_import_entities\".\"created_at\" BETWEEN '#{start}' AND '#{finish}' " \ + "AND \"bulk_import_entities\".\"source_type\" = 0 AND \"bulk_import_entities\".\"status\" = 2 " \ + "AND \"bulk_import_entities\".\"has_failures\" = FALSE" + end + + it_behaves_like 'a correct instrumented metric value and query', + time_frame: '28d', + options: { status: 2, source_type: 'group_entity', has_failures: false } + end + end + end end diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/count_ci_internal_pipelines_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/count_ci_internal_pipelines_metric_spec.rb index afd8fccd56c..77c49d448d7 100644 --- a/spec/lib/gitlab/usage/metrics/instrumentations/count_ci_internal_pipelines_metric_spec.rb +++ b/spec/lib/gitlab/usage/metrics/instrumentations/count_ci_internal_pipelines_metric_spec.rb @@ -4,25 +4,23 @@ require 'spec_helper' RSpec.describe Gitlab::Usage::Metrics::Instrumentations::CountCiInternalPipelinesMetric, feature_category: :service_ping do - let_it_be(:ci_pipeline_1) { create(:ci_pipeline, source: :external) } - let_it_be(:ci_pipeline_2) { create(:ci_pipeline, source: :push) } - - let(:expected_value) { 1 } - let(:expected_query) do - 'SELECT COUNT("ci_pipelines"."id") FROM "ci_pipelines" ' \ - 'WHERE ("ci_pipelines"."source" IN (1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 12, 13, 14, 15) ' \ - 'OR "ci_pipelines"."source" IS NULL)' - end + let_it_be(:ci_pipeline_1) { create(:ci_pipeline, source: :external, created_at: 3.days.ago) } + let_it_be(:ci_pipeline_2) { create(:ci_pipeline, source: :push, created_at: 3.days.ago) } + let_it_be(:old_pipeline) { create(:ci_pipeline, source: :push, created_at: 2.months.ago) } + let_it_be(:expected_value) { 2 } it_behaves_like 'a correct instrumented metric value', { time_frame: 'all', data_source: 'database' } - context 'on Gitlab.com' do - before do - allow(Gitlab).to receive(:com?).and_return(true) - end + context 'for monthly counts' do + let_it_be(:expected_value) { 1 } + + it_behaves_like 'a correct instrumented metric value', { time_frame: '28d', data_source: 'database' } + end - let(:expected_value) { -1 } + context 'on SaaS', :saas do + let_it_be(:expected_value) { -1 } it_behaves_like 'a correct instrumented metric value', { time_frame: 'all', data_source: 'database' } + it_behaves_like 'a correct instrumented metric value', { time_frame: '28d', data_source: 'database' } end end diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/count_issues_created_manually_from_alerts_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/count_issues_created_manually_from_alerts_metric_spec.rb index 86f54c48666..65e514bf345 100644 --- a/spec/lib/gitlab/usage/metrics/instrumentations/count_issues_created_manually_from_alerts_metric_spec.rb +++ b/spec/lib/gitlab/usage/metrics/instrumentations/count_issues_created_manually_from_alerts_metric_spec.rb @@ -16,11 +16,7 @@ feature_category: :service_ping do it_behaves_like 'a correct instrumented metric value', { time_frame: 'all', data_source: 'database' } - context 'on Gitlab.com' do - before do - allow(Gitlab).to receive(:com?).and_return(true) - end - + context 'on SaaS', :saas do let(:expected_value) { -1 } it_behaves_like 'a correct instrumented metric value', { time_frame: 'all', data_source: 'database' } diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/gitlab_dedicated_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/gitlab_dedicated_metric_spec.rb new file mode 100644 index 00000000000..a35022ec2c4 --- /dev/null +++ b/spec/lib/gitlab/usage/metrics/instrumentations/gitlab_dedicated_metric_spec.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Usage::Metrics::Instrumentations::GitlabDedicatedMetric, feature_category: :service_ping do + let(:expected_value) { Gitlab::CurrentSettings.gitlab_dedicated_instance } + + it_behaves_like 'a correct instrumented metric value', { time_frame: 'none' } +end diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/index_inconsistencies_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/index_inconsistencies_metric_spec.rb new file mode 100644 index 00000000000..afc9d610207 --- /dev/null +++ b/spec/lib/gitlab/usage/metrics/instrumentations/index_inconsistencies_metric_spec.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Usage::Metrics::Instrumentations::IndexInconsistenciesMetric, feature_category: :database do + it_behaves_like 'a correct instrumented metric value', { time_frame: 'all' } do + let(:expected_value) do + [ + { inconsistency_type: 'wrong_indexes', object_name: 'index_name_1' }, + { inconsistency_type: 'missing_indexes', object_name: 'index_name_2' }, + { inconsistency_type: 'extra_indexes', object_name: 'index_name_3' } + ] + end + + let(:runner) { instance_double(Gitlab::Database::SchemaValidation::Runner, execute: inconsistencies) } + let(:inconsistency_class) { Gitlab::Database::SchemaValidation::Validators::BaseValidator::Inconsistency } + + let(:inconsistencies) do + [ + instance_double(inconsistency_class, object_name: 'index_name_1', type: 'wrong_indexes'), + instance_double(inconsistency_class, object_name: 'index_name_2', type: 'missing_indexes'), + instance_double(inconsistency_class, object_name: 'index_name_3', type: 'extra_indexes') + ] + end + + before do + allow(Gitlab::Database::SchemaValidation::Runner).to receive(:new).and_return(runner) + end + end +end diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/installation_creation_date_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/installation_creation_date_metric_spec.rb new file mode 100644 index 00000000000..ff6be56c13f --- /dev/null +++ b/spec/lib/gitlab/usage/metrics/instrumentations/installation_creation_date_metric_spec.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Usage::Metrics::Instrumentations::InstallationCreationDateMetric, + feature_category: :service_ping do + context 'with a root user' do + let_it_be(:root) { create(:user, id: 1) } + let_it_be(:expected_value) { root.reload.created_at } # reloading to get the timestamp from the database + + it_behaves_like 'a correct instrumented metric value', { time_frame: 'all', data_source: 'database' } + end + + context 'without a root user' do + let_it_be(:another_user) { create(:user, id: 2) } + let_it_be(:expected_value) { nil } + + it_behaves_like 'a correct instrumented metric value', { time_frame: 'all', data_source: 'database' } + end +end diff --git a/spec/lib/gitlab/usage_data_counters/code_review_events_spec.rb b/spec/lib/gitlab/usage_data_counters/code_review_events_spec.rb index 63a1da490ed..8da86e4fae5 100644 --- a/spec/lib/gitlab/usage_data_counters/code_review_events_spec.rb +++ b/spec/lib/gitlab/usage_data_counters/code_review_events_spec.rb @@ -6,17 +6,23 @@ require 'spec_helper' # NOTE: ONLY user related metrics to be added to the aggregates - otherwise add it to the exception list RSpec.describe 'Code review events' do it 'the aggregated metrics contain all the code review metrics' do - code_review_events = Gitlab::UsageDataCounters::HLLRedisCounter.events_for_category("code_review") + mr_related_events = %w[i_code_review_create_mr i_code_review_mr_diffs i_code_review_mr_with_invalid_approvers i_code_review_mr_single_file_diffs i_code_review_total_suggestions_applied i_code_review_total_suggestions_added i_code_review_create_note_in_ipynb_diff i_code_review_create_note_in_ipynb_diff_mr i_code_review_create_note_in_ipynb_diff_commit i_code_review_merge_request_widget_license_compliance_warning] + + all_code_review_events = Gitlab::Usage::MetricDefinition.all.flat_map do |definition| + next [] unless definition.attributes[:key_path].include?('.code_review.') && + definition.attributes[:status] == 'active' && + definition.attributes[:instrumentation_class] != 'AggregatedMetric' + + definition.attributes.dig(:options, :events) + end.uniq.compact + code_review_aggregated_events = Gitlab::Usage::MetricDefinition.all.flat_map do |definition| next [] unless code_review_aggregated_metric?(definition.attributes) definition.attributes.dig(:options, :events) end.uniq - exceptions = %w[i_code_review_create_mr i_code_review_mr_diffs i_code_review_mr_with_invalid_approvers i_code_review_mr_single_file_diffs i_code_review_total_suggestions_applied i_code_review_total_suggestions_added i_code_review_create_note_in_ipynb_diff i_code_review_create_note_in_ipynb_diff_mr i_code_review_create_note_in_ipynb_diff_commit] - code_review_aggregated_events += exceptions - - expect(code_review_events - code_review_aggregated_events).to be_empty + expect(all_code_review_events - (code_review_aggregated_events + mr_related_events)).to be_empty end def code_review_aggregated_metric?(attributes) diff --git a/spec/lib/gitlab/usage_data_counters/container_registry_event_counter_spec.rb b/spec/lib/gitlab/usage_data_counters/container_registry_event_counter_spec.rb new file mode 100644 index 00000000000..052735db96b --- /dev/null +++ b/spec/lib/gitlab/usage_data_counters/container_registry_event_counter_spec.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::UsageDataCounters::ContainerRegistryEventCounter, :clean_gitlab_redis_shared_state, + feature_category: :container_registry do + described_class::KNOWN_EVENTS.each do |event| + it_behaves_like 'a redis usage counter', 'ContainerRegistryEvent', event + it_behaves_like 'a redis usage counter with totals', :container_registry_events, "#{event}": 5 + end +end diff --git a/spec/lib/gitlab/usage_data_counters/editor_unique_counter_spec.rb b/spec/lib/gitlab/usage_data_counters/editor_unique_counter_spec.rb index f8a4603c1f8..c16d31cd8ef 100644 --- a/spec/lib/gitlab/usage_data_counters/editor_unique_counter_spec.rb +++ b/spec/lib/gitlab/usage_data_counters/editor_unique_counter_spec.rb @@ -25,12 +25,29 @@ RSpec.describe Gitlab::UsageDataCounters::EditorUniqueCounter, :clean_gitlab_red end end + it 'track snowplow event' do + track_action(author: user1, project: project) + + expect_snowplow_event( + category: described_class.name, + action: 'ide_edit', + label: 'usage_activity_by_stage_monthly.create.action_monthly_active_users_ide_edit', + namespace: project.namespace, + property: event_name, + project: project, + user: user1, + context: [Gitlab::Tracking::ServicePingContext.new(data_source: :redis_hll, event: event_name).to_h] + ) + end + it 'does not track edit actions if author is not present' do expect(track_action(author: nil, project: project)).to be_nil end end context 'for web IDE edit actions' do + let(:event_name) { described_class::EDIT_BY_WEB_IDE } + it_behaves_like 'tracks and counts action' do def track_action(params) described_class.track_web_ide_edit_action(**params) @@ -43,6 +60,8 @@ RSpec.describe Gitlab::UsageDataCounters::EditorUniqueCounter, :clean_gitlab_red end context 'for SFE edit actions' do + let(:event_name) { described_class::EDIT_BY_SFE } + it_behaves_like 'tracks and counts action' do def track_action(params) described_class.track_sfe_edit_action(**params) @@ -55,6 +74,8 @@ RSpec.describe Gitlab::UsageDataCounters::EditorUniqueCounter, :clean_gitlab_red end context 'for snippet editor edit actions' do + let(:event_name) { described_class::EDIT_BY_SNIPPET_EDITOR } + it_behaves_like 'tracks and counts action' do def track_action(params) described_class.track_snippet_editor_edit_action(**params) diff --git a/spec/lib/gitlab/usage_data_counters/gitlab_cli_activity_unique_counter_spec.rb b/spec/lib/gitlab/usage_data_counters/gitlab_cli_activity_unique_counter_spec.rb index d6eb67e5c35..9cbac835a6f 100644 --- a/spec/lib/gitlab/usage_data_counters/gitlab_cli_activity_unique_counter_spec.rb +++ b/spec/lib/gitlab/usage_data_counters/gitlab_cli_activity_unique_counter_spec.rb @@ -7,9 +7,18 @@ RSpec.describe Gitlab::UsageDataCounters::GitLabCliActivityUniqueCounter, :clean let(:user2) { build(:user, id: 2) } let(:time) { Time.current } let(:action) { described_class::GITLAB_CLI_API_REQUEST_ACTION } - let(:user_agent) { { user_agent: 'GLab - GitLab CLI' } } context 'when tracking a gitlab cli request' do - it_behaves_like 'a request from an extension' + context 'with the old UserAgent' do + let(:user_agent) { { user_agent: 'GLab - GitLab CLI' } } + + it_behaves_like 'a request from an extension' + end + + context 'with the current UserAgent' do + let(:user_agent) { { user_agent: 'glab/v1.25.3-27-g7ec258fb (built 2023-02-16), darwin' } } + + it_behaves_like 'a request from an extension' + end end end diff --git a/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb b/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb index f955fd265e5..8c497970555 100644 --- a/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb +++ b/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb @@ -23,47 +23,12 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s described_class.clear_memoization(:known_events) end - describe '.categories' do - it 'gets CE unique category names' do - expect(described_class.categories).to include( - 'analytics', - 'ci_templates', - 'ci_users', - 'code_review', - 'deploy_token_packages', - 'ecosystem', - 'environments', - 'error_tracking', - 'geo', - 'ide_edit', - 'importer', - 'incident_management_alerts', - 'incident_management', - 'issues_edit', - 'kubernetes_agent', - 'manage', - 'pipeline_authoring', - 'quickactions', - 'search', - 'secure', - 'snippets', - 'source_code', - 'terraform', - 'testing', - 'user_packages', - 'work_items' - ) - end - end - describe '.known_events' do let(:ce_temp_dir) { Dir.mktmpdir } let(:ce_temp_file) { Tempfile.new(%w[common .yml], ce_temp_dir) } let(:ce_event) do { "name" => "ce_event", - "redis_slot" => "analytics", - "category" => "analytics", "aggregation" => "weekly" } end @@ -84,8 +49,6 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s end describe 'known_events' do - let(:feature) { 'test_hll_redis_counter_ff_check' } - let(:weekly_event) { 'g_analytics_contribution' } let(:daily_event) { 'g_analytics_search' } let(:analytics_slot_event) { 'g_analytics_contribution' } @@ -105,13 +68,13 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s let(:known_events) do [ - { name: weekly_event, redis_slot: "analytics", category: analytics_category, aggregation: "weekly", feature_flag: feature }, - { name: daily_event, redis_slot: "analytics", category: analytics_category, aggregation: "daily" }, - { name: category_productivity_event, redis_slot: "analytics", category: productivity_category, aggregation: "weekly" }, - { name: compliance_slot_event, redis_slot: "compliance", category: compliance_category, aggregation: "weekly" }, - { name: no_slot, category: global_category, aggregation: "daily" }, - { name: different_aggregation, category: global_category, aggregation: "monthly" }, - { name: context_event, category: other_category, aggregation: 'weekly' } + { name: weekly_event, aggregation: "weekly" }, + { name: daily_event, aggregation: "daily" }, + { name: category_productivity_event, aggregation: "weekly" }, + { name: compliance_slot_event, aggregation: "weekly" }, + { name: no_slot, aggregation: "daily" }, + { name: different_aggregation, aggregation: "monthly" }, + { name: context_event, aggregation: 'weekly' } ].map(&:with_indifferent_access) end @@ -121,12 +84,6 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s allow(described_class).to receive(:known_events).and_return(known_events) end - describe '.events_for_category' do - it 'gets the event names for given category' do - expect(described_class.events_for_category(:analytics)).to contain_exactly(weekly_event, daily_event) - end - end - describe '.track_event' do context 'with redis_hll_tracking' do it 'tracks the event when feature enabled' do @@ -146,32 +103,6 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s end end - context 'with event feature flag set' do - it 'tracks the event when feature enabled' do - stub_feature_flags(feature => true) - - expect(Gitlab::Redis::HLL).to receive(:add) - - described_class.track_event(weekly_event, values: 1) - end - - it 'does not track the event with feature flag disabled' do - stub_feature_flags(feature => false) - - expect(Gitlab::Redis::HLL).not_to receive(:add) - - described_class.track_event(weekly_event, values: 1) - end - end - - context 'with no event feature flag set' do - it 'tracks the event' do - expect(Gitlab::Redis::HLL).to receive(:add) - - described_class.track_event(daily_event, values: 1) - end - end - context 'when usage_ping is disabled' do it 'does not track the event' do allow(::ServicePing::ServicePingSettings).to receive(:enabled?).and_return(false) @@ -195,7 +126,7 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s it 'tracks events with multiple values' do values = [entity1, entity2] - expect(Gitlab::Redis::HLL).to receive(:add).with(key: /g_{analytics}_contribution/, value: values, + expect(Gitlab::Redis::HLL).to receive(:add).with(key: /g_analytics_contribution/, value: values, expiry: described_class::DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH) described_class.track_event(:g_analytics_contribution, values: values) @@ -237,7 +168,7 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s described_class.track_event("g_compliance_dashboard", values: entity1) Gitlab::Redis::SharedState.with do |redis| - keys = redis.scan_each(match: "g_{compliance}_dashboard-*").to_a + keys = redis.scan_each(match: "{#{described_class::REDIS_SLOT}}_g_compliance_dashboard-*").to_a expect(keys).not_to be_empty keys.each do |key| @@ -252,7 +183,7 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s described_class.track_event("no_slot", values: entity1) Gitlab::Redis::SharedState.with do |redis| - keys = redis.scan_each(match: "*-{no_slot}").to_a + keys = redis.scan_each(match: "*_no_slot").to_a expect(keys).not_to be_empty keys.each do |key| @@ -276,7 +207,7 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s it 'tracks events with multiple values' do values = [entity1, entity2] - expect(Gitlab::Redis::HLL).to receive(:add).with(key: /g_{analytics}_contribution/, + expect(Gitlab::Redis::HLL).to receive(:add).with(key: /g_analytics_contribution/, value: values, expiry: described_class::DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH) @@ -340,18 +271,6 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s expect(described_class.unique_events(event_names: [weekly_event], start_date: Date.current, end_date: 4.weeks.ago)).to eq(-1) end - it 'raise error if metrics are not in the same slot' do - expect do - described_class.unique_events(event_names: [compliance_slot_event, analytics_slot_event], start_date: 4.weeks.ago, end_date: Date.current) - end.to raise_error(Gitlab::UsageDataCounters::HLLRedisCounter::SlotMismatch) - end - - it 'raise error if metrics are not in the same category' do - expect do - described_class.unique_events(event_names: [category_analytics_event, category_productivity_event], start_date: 4.weeks.ago, end_date: Date.current) - end.to raise_error(Gitlab::UsageDataCounters::HLLRedisCounter::CategoryMismatch) - end - it "raise error if metrics don't have same aggregation" do expect do described_class.unique_events(event_names: [daily_event, weekly_event], start_date: 4.weeks.ago, end_date: Date.current) @@ -398,6 +317,10 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s let(:weekly_event) { 'i_search_total' } let(:redis_event) { described_class.send(:event_for, weekly_event) } + let(:week_one) { "{#{described_class::REDIS_SLOT}}_i_search_total-2020-52" } + let(:week_two) { "{#{described_class::REDIS_SLOT}}_i_search_total-2020-53" } + let(:week_three) { "{#{described_class::REDIS_SLOT}}_i_search_total-2021-01" } + let(:week_four) { "{#{described_class::REDIS_SLOT}}_i_search_total-2021-02" } subject(:weekly_redis_keys) { described_class.send(:weekly_redis_keys, events: [redis_event], start_date: DateTime.parse(start_date), end_date: DateTime.parse(end_date)) } @@ -406,13 +329,13 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s '2020-12-21' | '2020-12-20' | [] '2020-12-21' | '2020-11-21' | [] '2021-01-01' | '2020-12-28' | [] - '2020-12-21' | '2020-12-28' | ['i_{search}_total-2020-52'] - '2020-12-21' | '2021-01-01' | ['i_{search}_total-2020-52'] - '2020-12-27' | '2021-01-01' | ['i_{search}_total-2020-52'] - '2020-12-26' | '2021-01-04' | ['i_{search}_total-2020-52', 'i_{search}_total-2020-53'] - '2020-12-26' | '2021-01-11' | ['i_{search}_total-2020-52', 'i_{search}_total-2020-53', 'i_{search}_total-2021-01'] - '2020-12-26' | '2021-01-17' | ['i_{search}_total-2020-52', 'i_{search}_total-2020-53', 'i_{search}_total-2021-01'] - '2020-12-26' | '2021-01-18' | ['i_{search}_total-2020-52', 'i_{search}_total-2020-53', 'i_{search}_total-2021-01', 'i_{search}_total-2021-02'] + '2020-12-21' | '2020-12-28' | lazy { [week_one] } + '2020-12-21' | '2021-01-01' | lazy { [week_one] } + '2020-12-27' | '2021-01-01' | lazy { [week_one] } + '2020-12-26' | '2021-01-04' | lazy { [week_one, week_two] } + '2020-12-26' | '2021-01-11' | lazy { [week_one, week_two, week_three] } + '2020-12-26' | '2021-01-17' | lazy { [week_one, week_two, week_three] } + '2020-12-26' | '2021-01-18' | lazy { [week_one, week_two, week_three, week_four] } end with_them do @@ -435,9 +358,9 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s let(:known_events) do [ - { name: 'event_name_1', redis_slot: 'event', category: 'category1', aggregation: "weekly" }, - { name: 'event_name_2', redis_slot: 'event', category: 'category1', aggregation: "weekly" }, - { name: 'event_name_3', redis_slot: 'event', category: 'category1', aggregation: "weekly" } + { name: 'event_name_1', aggregation: "weekly" }, + { name: 'event_name_2', aggregation: "weekly" }, + { name: 'event_name_3', aggregation: "weekly" } ].map(&:with_indifferent_access) end @@ -476,11 +399,11 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s let(:time_range) { { start_date: 7.days.ago, end_date: DateTime.current } } let(:known_events) do [ - { name: 'event1_slot', redis_slot: "slot", category: 'category1', aggregation: "weekly" }, - { name: 'event2_slot', redis_slot: "slot", category: 'category2', aggregation: "weekly" }, - { name: 'event3_slot', redis_slot: "slot", category: 'category3', aggregation: "weekly" }, - { name: 'event5_slot', redis_slot: "slot", category: 'category4', aggregation: "daily" }, - { name: 'event4', category: 'category2', aggregation: "weekly" } + { name: 'event1_slot', aggregation: "weekly" }, + { name: 'event2_slot', aggregation: "weekly" }, + { name: 'event3_slot', aggregation: "weekly" }, + { name: 'event5_slot', aggregation: "daily" }, + { name: 'event4', aggregation: "weekly" } ].map(&:with_indifferent_access) end @@ -510,11 +433,6 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s expect(described_class.calculate_events_union(**time_range.merge(event_names: %w[event1_slot event2_slot event3_slot]))).to eq 3 end - it 'validates and raise exception if events has mismatched slot or aggregation', :aggregate_failure do - expect { described_class.calculate_events_union(**time_range.merge(event_names: %w[event1_slot event4])) }.to raise_error described_class::SlotMismatch - expect { described_class.calculate_events_union(**time_range.merge(event_names: %w[event5_slot event3_slot])) }.to raise_error described_class::AggregationMismatch - end - it 'returns 0 if there are no keys for given events' do expect(Gitlab::Redis::HLL).not_to receive(:count) expect(described_class.calculate_events_union(event_names: %w[event1_slot event2_slot event3_slot], start_date: Date.current, end_date: 4.weeks.ago)).to eq(-1) diff --git a/spec/lib/gitlab/usage_data_counters/issue_activity_unique_counter_spec.rb b/spec/lib/gitlab/usage_data_counters/issue_activity_unique_counter_spec.rb index 33e0d446fca..383938b0324 100644 --- a/spec/lib/gitlab/usage_data_counters/issue_activity_unique_counter_spec.rb +++ b/spec/lib/gitlab/usage_data_counters/issue_activity_unique_counter_spec.rb @@ -306,7 +306,7 @@ RSpec.describe Gitlab::UsageDataCounters::IssueActivityUniqueCounter, :clean_git described_class.track_issue_assignee_changed_action(author: user3, project: project) end - events = Gitlab::UsageDataCounters::HLLRedisCounter.events_for_category(described_class::ISSUE_CATEGORY) + events = [described_class::ISSUE_TITLE_CHANGED, described_class::ISSUE_DESCRIPTION_CHANGED, described_class::ISSUE_ASSIGNEE_CHANGED] today_count = Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: events, start_date: time, end_date: time) week_count = Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: events, start_date: time - 5.days, end_date: 1.day.since(time)) diff --git a/spec/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter_spec.rb b/spec/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter_spec.rb index 42aa84c2c3e..e41da6d9ea2 100644 --- a/spec/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter_spec.rb +++ b/spec/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter_spec.rb @@ -69,7 +69,6 @@ RSpec.describe Gitlab::UsageDataCounters::MergeRequestActivityUniqueCounter, :cl let(:project) { target_project } let(:namespace) { project.namespace.reload } let(:user) { project.creator } - let(:feature_flag_name) { :route_hll_to_snowplow_phase2 } let(:label) { 'redis_hll_counters.code_review.i_code_review_user_create_mr_monthly' } let(:property) { described_class::MR_USER_CREATE_ACTION } end @@ -118,7 +117,6 @@ RSpec.describe Gitlab::UsageDataCounters::MergeRequestActivityUniqueCounter, :cl let(:project) { target_project } let(:namespace) { project.namespace.reload } let(:user) { project.creator } - let(:feature_flag_name) { :route_hll_to_snowplow_phase2 } let(:label) { 'redis_hll_counters.code_review.i_code_review_user_approve_mr_monthly' } let(:property) { described_class::MR_APPROVE_ACTION } end diff --git a/spec/lib/gitlab/usage_data_counters/track_unique_events_spec.rb b/spec/lib/gitlab/usage_data_counters/track_unique_events_spec.rb deleted file mode 100644 index d1144dd0bc5..00000000000 --- a/spec/lib/gitlab/usage_data_counters/track_unique_events_spec.rb +++ /dev/null @@ -1,72 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::UsageDataCounters::TrackUniqueEvents, :clean_gitlab_redis_shared_state do - subject(:track_unique_events) { described_class } - - let(:time) { Time.zone.now } - - def track_event(params) - track_unique_events.track_event(**params) - end - - def count_unique(params) - track_unique_events.count_unique_events(**params) - end - - context 'tracking an event' do - context 'when tracking successfully' do - context 'when the application setting is enabled' do - context 'when the target and the action is valid' do - before do - stub_application_setting(usage_ping_enabled: true) - end - - it 'tracks and counts the events as expected' do - project = Event::TARGET_TYPES[:project] - design = Event::TARGET_TYPES[:design] - wiki = Event::TARGET_TYPES[:wiki] - - expect(track_event(event_action: :pushed, event_target: project, author_id: 1)).to be_truthy - expect(track_event(event_action: :pushed, event_target: project, author_id: 1)).to be_truthy - expect(track_event(event_action: :pushed, event_target: project, author_id: 2)).to be_truthy - expect(track_event(event_action: :pushed, event_target: project, author_id: 3)).to be_truthy - expect(track_event(event_action: :pushed, event_target: project, author_id: 4, time: time - 3.days)).to be_truthy - - expect(track_event(event_action: :destroyed, event_target: design, author_id: 3)).to be_truthy - expect(track_event(event_action: :created, event_target: design, author_id: 4)).to be_truthy - expect(track_event(event_action: :updated, event_target: design, author_id: 5)).to be_truthy - - expect(track_event(event_action: :destroyed, event_target: wiki, author_id: 5)).to be_truthy - expect(track_event(event_action: :created, event_target: wiki, author_id: 3)).to be_truthy - expect(track_event(event_action: :updated, event_target: wiki, author_id: 4)).to be_truthy - - expect(count_unique(event_action: described_class::PUSH_ACTION, date_from: time, date_to: Date.today)).to eq(3) - expect(count_unique(event_action: described_class::PUSH_ACTION, date_from: time - 5.days, date_to: Date.tomorrow)).to eq(4) - expect(count_unique(event_action: described_class::DESIGN_ACTION, date_from: time - 5.days, date_to: Date.today)).to eq(3) - expect(count_unique(event_action: described_class::WIKI_ACTION, date_from: time - 5.days, date_to: Date.today)).to eq(3) - expect(count_unique(event_action: described_class::PUSH_ACTION, date_from: time - 5.days, date_to: time - 2.days)).to eq(1) - end - end - end - end - - context 'when tracking unsuccessfully' do - using RSpec::Parameterized::TableSyntax - - where(:target, :action) do - Project | :invalid_action - :invalid_target | :pushed - Project | :created - end - - with_them do - it 'returns the expected values' do - expect(track_event(event_action: action, event_target: target, author_id: 2)).to be_nil - expect(count_unique(event_action: described_class::PUSH_ACTION, date_from: time, date_to: Date.today)).to eq(0) - end - end - end - end -end diff --git a/spec/lib/gitlab/usage_data_spec.rb b/spec/lib/gitlab/usage_data_spec.rb index 5325ef5b5dd..d529319e6e9 100644 --- a/spec/lib/gitlab/usage_data_spec.rb +++ b/spec/lib/gitlab/usage_data_spec.rb @@ -529,8 +529,6 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures, feature_category: :servic expect(count_data[:projects_prometheus_active]).to eq(1) expect(count_data[:projects_jenkins_active]).to eq(1) expect(count_data[:projects_jira_active]).to eq(4) - expect(count_data[:projects_jira_server_active]).to eq(2) - expect(count_data[:projects_jira_cloud_active]).to eq(2) expect(count_data[:jira_imports_projects_count]).to eq(2) expect(count_data[:jira_imports_total_imported_count]).to eq(3) expect(count_data[:jira_imports_total_imported_issues_count]).to eq(13) @@ -614,14 +612,6 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures, feature_category: :servic it 'raises an error' do expect { subject }.to raise_error(ActiveRecord::StatementInvalid) end - - context 'when metric calls find_in_batches' do - let(:metric_method) { :find_in_batches } - - it 'raises an error for jira_usage' do - expect { described_class.jira_usage }.to raise_error(ActiveRecord::StatementInvalid) - end - end end context 'with should_raise_for_dev? false' do @@ -630,14 +620,6 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures, feature_category: :servic it 'does not raise an error' do expect { subject }.not_to raise_error end - - context 'when metric calls find_in_batches' do - let(:metric_method) { :find_in_batches } - - it 'does not raise an error for jira_usage' do - expect { described_class.jira_usage }.not_to raise_error - end - end end end @@ -1044,16 +1026,12 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures, feature_category: :servic let(:time) { Time.current } before do - counter = Gitlab::UsageDataCounters::TrackUniqueEvents - merge_request = Event::TARGET_TYPES[:merge_request] - design = Event::TARGET_TYPES[:design] - - counter.track_event(event_action: :commented, event_target: merge_request, author_id: 1, time: time) - counter.track_event(event_action: :opened, event_target: merge_request, author_id: 1, time: time) - counter.track_event(event_action: :merged, event_target: merge_request, author_id: 2, time: time) - counter.track_event(event_action: :closed, event_target: merge_request, author_id: 3, time: time) - counter.track_event(event_action: :opened, event_target: merge_request, author_id: 4, time: time - 3.days) - counter.track_event(event_action: :created, event_target: design, author_id: 5, time: time) + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(:merge_request_action, values: 1, time: time) + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(:merge_request_action, values: 1, time: time) + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(:merge_request_action, values: 2, time: time) + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(:merge_request_action, values: 3, time: time) + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(:merge_request_action, values: 4, time: time - 3.days) + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(:design_action, values: 5, time: time) end it 'returns the distinct count of users using merge requests (via events table) within the specified time period' do @@ -1069,42 +1047,6 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures, feature_category: :servic end end - describe '#action_monthly_active_users', :clean_gitlab_redis_shared_state do - let(:time_period) { { created_at: 2.days.ago..time } } - let(:time) { Time.zone.now } - let(:user1) { build(:user, id: 1) } - let(:user2) { build(:user, id: 2) } - let(:user3) { build(:user, id: 3) } - let(:user4) { build(:user, id: 4) } - let(:project) { build(:project) } - - before do - counter = Gitlab::UsageDataCounters::EditorUniqueCounter - - counter.track_web_ide_edit_action(author: user1, project: project) - counter.track_web_ide_edit_action(author: user1, project: project) - counter.track_sfe_edit_action(author: user1, project: project) - counter.track_snippet_editor_edit_action(author: user1, project: project) - counter.track_snippet_editor_edit_action(author: user1, time: time - 3.days, project: project) - - counter.track_web_ide_edit_action(author: user2, project: project) - counter.track_sfe_edit_action(author: user2, project: project) - - counter.track_web_ide_edit_action(author: user3, time: time - 3.days, project: project) - counter.track_snippet_editor_edit_action(author: user3, project: project) - end - - it 'returns the distinct count of user actions within the specified time period' do - expect(described_class.action_monthly_active_users(time_period)).to eq( - { - action_monthly_active_users_web_ide_edit: 2, - action_monthly_active_users_sfe_edit: 2, - action_monthly_active_users_snippet_editor_edit: 2 - } - ) - end - end - describe '.service_desk_counts' do subject { described_class.send(:service_desk_counts) } diff --git a/spec/lib/gitlab/utils/error_message_spec.rb b/spec/lib/gitlab/utils/error_message_spec.rb new file mode 100644 index 00000000000..2c2d16656e8 --- /dev/null +++ b/spec/lib/gitlab/utils/error_message_spec.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +RSpec.describe Gitlab::Utils::ErrorMessage, feature_category: :error_tracking do + let(:klass) do + Class.new do + include Gitlab::Utils::ErrorMessage + end + end + + subject(:object) { klass.new } + + describe 'error message' do + subject { object.to_user_facing(string) } + + let(:string) { 'Error Message' } + + it "returns input prefixed with UF:" do + is_expected.to eq 'UF: Error Message' + end + end +end diff --git a/spec/lib/gitlab/utils/strong_memoize_spec.rb b/spec/lib/gitlab/utils/strong_memoize_spec.rb index 71f2502b91c..27bfe181ef6 100644 --- a/spec/lib/gitlab/utils/strong_memoize_spec.rb +++ b/spec/lib/gitlab/utils/strong_memoize_spec.rb @@ -8,7 +8,7 @@ RSpec.configure do |config| config.include RSpec::Benchmark::Matchers end -RSpec.describe Gitlab::Utils::StrongMemoize, feature_category: :not_owned do +RSpec.describe Gitlab::Utils::StrongMemoize, feature_category: :shared do let(:klass) do strong_memoize_class = described_class diff --git a/spec/lib/gitlab/utils/uniquify_spec.rb b/spec/lib/gitlab/utils/uniquify_spec.rb new file mode 100644 index 00000000000..df02fbe8c82 --- /dev/null +++ b/spec/lib/gitlab/utils/uniquify_spec.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +RSpec.describe Gitlab::Utils::Uniquify, feature_category: :shared do + subject(:uniquify) { described_class.new } + + describe "#string" do + it 'returns the given string if it does not exist' do + result = uniquify.string('test_string') { |_s| false } + + expect(result).to eq('test_string') + end + + it 'returns the given string with a counter attached if the string exists' do + result = uniquify.string('test_string') { |s| s == 'test_string' } + + expect(result).to eq('test_string1') + end + + it 'increments the counter for each candidate string that also exists' do + result = uniquify.string('test_string') { |s| s == 'test_string' || s == 'test_string1' } + + expect(result).to eq('test_string2') + end + + it 'allows to pass an initial value for the counter' do + start_counting_from = 2 + uniquify = described_class.new(start_counting_from) + + result = uniquify.string('test_string') { |s| s == 'test_string' } + + expect(result).to eq('test_string2') + end + + it 'allows passing in a base function that defines the location of the counter' do + result = uniquify.string(->(counter) { "test_#{counter}_string" }) do |s| + s == 'test__string' + end + + expect(result).to eq('test_1_string') + end + end +end diff --git a/spec/lib/gitlab/utils/usage_data_spec.rb b/spec/lib/gitlab/utils/usage_data_spec.rb index 2925ceef256..586ee04a835 100644 --- a/spec/lib/gitlab/utils/usage_data_spec.rb +++ b/spec/lib/gitlab/utils/usage_data_spec.rb @@ -487,12 +487,12 @@ RSpec.describe Gitlab::Utils::UsageData do end context 'when Redis HLL raises any error' do - subject { described_class.redis_usage_data { raise Gitlab::UsageDataCounters::HLLRedisCounter::CategoryMismatch } } + subject { described_class.redis_usage_data { raise Gitlab::UsageDataCounters::HLLRedisCounter::EventError } } let(:fallback) { 15 } let(:failing_class) { nil } - it_behaves_like 'failing hardening method', Gitlab::UsageDataCounters::HLLRedisCounter::CategoryMismatch + it_behaves_like 'failing hardening method', Gitlab::UsageDataCounters::HLLRedisCounter::EventError end it 'returns the evaluated block when given' do diff --git a/spec/lib/gitlab/utils/username_and_email_generator_spec.rb b/spec/lib/gitlab/utils/username_and_email_generator_spec.rb new file mode 100644 index 00000000000..45df8f08055 --- /dev/null +++ b/spec/lib/gitlab/utils/username_and_email_generator_spec.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Utils::UsernameAndEmailGenerator, feature_category: :system_access do + let(:username_prefix) { 'username_prefix' } + let(:email_domain) { 'example.com' } + + subject { described_class.new(username_prefix: username_prefix, email_domain: email_domain) } + + describe 'email domain' do + it 'defaults to `Gitlab.config.gitlab.host`' do + expect(described_class.new(username_prefix: username_prefix).email).to end_with("@#{Gitlab.config.gitlab.host}") + end + + context 'when specified' do + it 'uses the specified email domain' do + expect(subject.email).to end_with("@#{email_domain}") + end + end + end + + include_examples 'username and email pair is generated by Gitlab::Utils::UsernameAndEmailGenerator' +end |