diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-10-20 11:43:02 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-10-20 11:43:02 +0300 |
commit | d9ab72d6080f594d0b3cae15f14b3ef2c6c638cb (patch) | |
tree | 2341ef426af70ad1e289c38036737e04b0aa5007 /spec/lib/gitlab | |
parent | d6e514dd13db8947884cd58fe2a9c2a063400a9b (diff) |
Add latest changes from gitlab-org/gitlab@14-4-stable-eev14.4.0-rc42
Diffstat (limited to 'spec/lib/gitlab')
168 files changed, 4415 insertions, 2088 deletions
diff --git a/spec/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start_spec.rb b/spec/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start_spec.rb index b6f9c8106c9..2e96fd09602 100644 --- a/spec/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start_spec.rb +++ b/spec/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start_spec.rb @@ -19,4 +19,16 @@ RSpec.describe Gitlab::Analytics::CycleAnalytics::StageEvents::CodeStageStart do expect(records).to eq([merge_request]) expect(records).not_to include(other_merge_request) end + + it_behaves_like 'LEFT JOIN-able value stream analytics event' do + let_it_be(:record_with_data) do + mr_closing_issue = FactoryBot.create(:merge_requests_closing_issues) + issue = mr_closing_issue.issue + issue.metrics.update!(first_mentioned_in_commit_at: Time.current) + + mr_closing_issue.merge_request + end + + let_it_be(:record_without_data) { create(:merge_request) } + end end diff --git a/spec/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created_spec.rb b/spec/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created_spec.rb index 224a18653ed..3f50dd38322 100644 --- a/spec/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created_spec.rb +++ b/spec/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created_spec.rb @@ -4,4 +4,8 @@ require 'spec_helper' RSpec.describe Gitlab::Analytics::CycleAnalytics::StageEvents::IssueCreated do it_behaves_like 'value stream analytics event' + + it_behaves_like 'LEFT JOIN-able value stream analytics event' do + let_it_be(:record_with_data) { create(:issue) } + end end diff --git a/spec/lib/gitlab/analytics/cycle_analytics/stage_events/issue_deployed_to_production_spec.rb b/spec/lib/gitlab/analytics/cycle_analytics/stage_events/issue_deployed_to_production_spec.rb index 93e588675d3..e807565ecb5 100644 --- a/spec/lib/gitlab/analytics/cycle_analytics/stage_events/issue_deployed_to_production_spec.rb +++ b/spec/lib/gitlab/analytics/cycle_analytics/stage_events/issue_deployed_to_production_spec.rb @@ -4,4 +4,16 @@ require 'spec_helper' RSpec.describe Gitlab::Analytics::CycleAnalytics::StageEvents::IssueDeployedToProduction do it_behaves_like 'value stream analytics event' + + it_behaves_like 'LEFT JOIN-able value stream analytics event' do + let_it_be(:record_with_data) do + mr_closing_issue = FactoryBot.create(:merge_requests_closing_issues) + mr = mr_closing_issue.merge_request + mr.metrics.update!(first_deployed_to_production_at: Time.current) + + mr_closing_issue.issue + end + + let_it_be(:record_without_data) { create(:issue) } + end end diff --git a/spec/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit_spec.rb b/spec/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit_spec.rb index bc0e388cf53..9bb023f9fdc 100644 --- a/spec/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit_spec.rb +++ b/spec/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit_spec.rb @@ -4,4 +4,9 @@ require 'spec_helper' RSpec.describe Gitlab::Analytics::CycleAnalytics::StageEvents::IssueFirstMentionedInCommit do it_behaves_like 'value stream analytics event' + + it_behaves_like 'LEFT JOIN-able value stream analytics event' do + let_it_be(:record_with_data) { create(:issue).tap { |i| i.metrics.update!(first_mentioned_in_commit_at: Time.current) } } + let_it_be(:record_without_data) { create(:issue) } + end end diff --git a/spec/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end_spec.rb b/spec/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end_spec.rb index ddc5f015a8c..7b46a86cbe2 100644 --- a/spec/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end_spec.rb +++ b/spec/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end_spec.rb @@ -4,4 +4,9 @@ require 'spec_helper' RSpec.describe Gitlab::Analytics::CycleAnalytics::StageEvents::IssueStageEnd do it_behaves_like 'value stream analytics event' + + it_behaves_like 'LEFT JOIN-able value stream analytics event' do + let_it_be(:record_with_data) { create(:issue).tap { |i| i.metrics.update!(first_added_to_board_at: Time.current) } } + let_it_be(:record_without_data) { create(:issue) } + end end diff --git a/spec/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created_spec.rb b/spec/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created_spec.rb index 281cc31c9e0..1139f9099cb 100644 --- a/spec/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created_spec.rb +++ b/spec/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created_spec.rb @@ -4,4 +4,8 @@ require 'spec_helper' RSpec.describe Gitlab::Analytics::CycleAnalytics::StageEvents::MergeRequestCreated do it_behaves_like 'value stream analytics event' + + it_behaves_like 'LEFT JOIN-able value stream analytics event' do + let_it_be(:record_with_data) { create(:merge_request) } + end end diff --git a/spec/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production_spec.rb b/spec/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production_spec.rb index e1dd2e56e2b..a62facb6974 100644 --- a/spec/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production_spec.rb +++ b/spec/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production_spec.rb @@ -4,4 +4,9 @@ require 'spec_helper' RSpec.describe Gitlab::Analytics::CycleAnalytics::StageEvents::MergeRequestFirstDeployedToProduction do it_behaves_like 'value stream analytics event' + + it_behaves_like 'LEFT JOIN-able value stream analytics event' do + let_it_be(:record_with_data) { create(:merge_request).tap { |mr| mr.metrics.update!(first_deployed_to_production_at: Time.current) } } + let_it_be(:record_without_data) { create(:merge_request) } + end end diff --git a/spec/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished_spec.rb b/spec/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished_spec.rb index 51324966f26..c5cfe43895e 100644 --- a/spec/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished_spec.rb +++ b/spec/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished_spec.rb @@ -4,4 +4,9 @@ require 'spec_helper' RSpec.describe Gitlab::Analytics::CycleAnalytics::StageEvents::MergeRequestLastBuildFinished do it_behaves_like 'value stream analytics event' + + it_behaves_like 'LEFT JOIN-able value stream analytics event' do + let_it_be(:record_with_data) { create(:merge_request).tap { |mr| mr.metrics.update!(latest_build_finished_at: Time.current) } } + let_it_be(:record_without_data) { create(:merge_request) } + end end diff --git a/spec/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started_spec.rb b/spec/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started_spec.rb index 10dcaf23b81..6f8a82a9ae5 100644 --- a/spec/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started_spec.rb +++ b/spec/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started_spec.rb @@ -4,4 +4,9 @@ require 'spec_helper' RSpec.describe Gitlab::Analytics::CycleAnalytics::StageEvents::MergeRequestLastBuildStarted do it_behaves_like 'value stream analytics event' + + it_behaves_like 'LEFT JOIN-able value stream analytics event' do + let_it_be(:record_with_data) { create(:merge_request).tap { |mr| mr.metrics.update!(latest_build_started_at: Time.current) } } + let_it_be(:record_without_data) { create(:merge_request) } + end end diff --git a/spec/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged_spec.rb b/spec/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged_spec.rb index 6e20eb73ed9..0060ed0fd48 100644 --- a/spec/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged_spec.rb +++ b/spec/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged_spec.rb @@ -4,4 +4,9 @@ require 'spec_helper' RSpec.describe Gitlab::Analytics::CycleAnalytics::StageEvents::MergeRequestMerged do it_behaves_like 'value stream analytics event' + + it_behaves_like 'LEFT JOIN-able value stream analytics event' do + let_it_be(:record_with_data) { create(:merge_request).tap { |mr| mr.metrics.update!(merged_at: Time.current) } } + let_it_be(:record_without_data) { create(:merge_request) } + end end diff --git a/spec/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start_spec.rb b/spec/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start_spec.rb index b8c68003127..379d59e4c5e 100644 --- a/spec/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start_spec.rb +++ b/spec/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start_spec.rb @@ -21,4 +21,9 @@ RSpec.describe Gitlab::Analytics::CycleAnalytics::StageEvents::PlanStageStart do expect(records).to match_array([issue1, issue2]) expect(records).not_to include(issue_without_metrics) end + + it_behaves_like 'LEFT JOIN-able value stream analytics event' do + let_it_be(:record_with_data) { create(:issue).tap { |i| i.metrics.update!(first_added_to_board_at: Time.current) } } + let_it_be(:record_without_data) { create(:issue) } + end end diff --git a/spec/lib/gitlab/application_rate_limiter_spec.rb b/spec/lib/gitlab/application_rate_limiter_spec.rb index 2525b1ce41e..0fb99688d27 100644 --- a/spec/lib/gitlab/application_rate_limiter_spec.rb +++ b/spec/lib/gitlab/application_rate_limiter_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::ApplicationRateLimiter, :clean_gitlab_redis_cache do +RSpec.describe Gitlab::ApplicationRateLimiter do let(:redis) { double('redis') } let(:user) { create(:user) } let(:project) { create(:project) } @@ -20,7 +20,7 @@ RSpec.describe Gitlab::ApplicationRateLimiter, :clean_gitlab_redis_cache do subject { described_class } before do - allow(Gitlab::Redis::Cache).to receive(:with).and_yield(redis) + allow(Gitlab::Redis::RateLimiting).to receive(:with).and_yield(redis) allow(described_class).to receive(:rate_limits).and_return(rate_limits) end @@ -106,9 +106,9 @@ RSpec.describe Gitlab::ApplicationRateLimiter, :clean_gitlab_redis_cache do let(:attributes) do base_attributes.merge({ - user_id: current_user.id, - username: current_user.username - }) + user_id: current_user.id, + username: current_user.username + }) end it 'logs information to auth.log' do diff --git a/spec/lib/gitlab/auth/request_authenticator_spec.rb b/spec/lib/gitlab/auth/request_authenticator_spec.rb index 2543eb3a5e9..6f3d6187076 100644 --- a/spec/lib/gitlab/auth/request_authenticator_spec.rb +++ b/spec/lib/gitlab/auth/request_authenticator_spec.rb @@ -3,6 +3,8 @@ require 'spec_helper' RSpec.describe Gitlab::Auth::RequestAuthenticator do + include DependencyProxyHelpers + let(:env) do { 'rack.input' => '', @@ -15,8 +17,8 @@ RSpec.describe Gitlab::Auth::RequestAuthenticator do subject { described_class.new(request) } describe '#user' do - let!(:sessionless_user) { build(:user) } - let!(:session_user) { build(:user) } + let_it_be(:sessionless_user) { build(:user) } + let_it_be(:session_user) { build(:user) } it 'returns sessionless user first' do allow_any_instance_of(described_class).to receive(:find_sessionless_user).and_return(sessionless_user) @@ -41,15 +43,25 @@ RSpec.describe Gitlab::Auth::RequestAuthenticator do end describe '#find_sessionless_user' do - let!(:access_token_user) { build(:user) } - let!(:feed_token_user) { build(:user) } - let!(:static_object_token_user) { build(:user) } - let!(:job_token_user) { build(:user) } - let!(:lfs_token_user) { build(:user) } - let!(:basic_auth_access_token_user) { build(:user) } - let!(:basic_auth_password_user) { build(:user) } - - it 'returns access_token user first' do + let_it_be(:dependency_proxy_user) { build(:user) } + let_it_be(:access_token_user) { build(:user) } + let_it_be(:feed_token_user) { build(:user) } + let_it_be(:static_object_token_user) { build(:user) } + let_it_be(:job_token_user) { build(:user) } + let_it_be(:lfs_token_user) { build(:user) } + let_it_be(:basic_auth_access_token_user) { build(:user) } + let_it_be(:basic_auth_password_user) { build(:user) } + + it 'returns dependency_proxy user first' do + allow_any_instance_of(described_class).to receive(:find_user_from_dependency_proxy_token) + .and_return(dependency_proxy_user) + + allow_any_instance_of(described_class).to receive(:find_user_from_web_access_token).and_return(access_token_user) + + expect(subject.find_sessionless_user(:api)).to eq dependency_proxy_user + end + + it 'returns access_token user if no dependency_proxy user found' do allow_any_instance_of(described_class).to receive(:find_user_from_web_access_token) .with(anything, scopes: [:api, :read_api]) .and_return(access_token_user) @@ -154,6 +166,75 @@ RSpec.describe Gitlab::Auth::RequestAuthenticator do expect(subject.find_sessionless_user(:api)).to be_blank end + + context 'dependency proxy' do + let_it_be(:dependency_proxy_user) { create(:user) } + + let(:token) { build_jwt(dependency_proxy_user).encoded } + let(:authenticator) { described_class.new(request) } + + subject { authenticator.find_sessionless_user(:api) } + + before do + env['SCRIPT_NAME'] = accessed_path + env['HTTP_AUTHORIZATION'] = "Bearer #{token}" + end + + shared_examples 'identifying dependency proxy urls properly with' do |user_type| + context 'with pulling a manifest' do + let(:accessed_path) { '/v2/group1/dependency_proxy/containers/alpine/manifests/latest' } + + it { is_expected.to eq(dependency_proxy_user) } if user_type == :user + it { is_expected.to eq(nil) } if user_type == :no_user + end + + context 'with pulling a blob' do + let(:accessed_path) { '/v2/group1/dependency_proxy/containers/alpine/blobs/sha256:a0d0a0d46f8b52473982a3c466318f479767577551a53ffc9074c9fa7035982e' } + + it { is_expected.to eq(dependency_proxy_user) } if user_type == :user + it { is_expected.to eq(nil) } if user_type == :no_user + end + + context 'with any other path' do + let(:accessed_path) { '/foo/bar' } + + it { is_expected.to eq(nil) } + end + end + + context 'with a user' do + it_behaves_like 'identifying dependency proxy urls properly with', :user + + context 'with an invalid id' do + let(:token) { build_jwt { |jwt| jwt['user_id'] = 'this_is_not_a_user' } } + + it_behaves_like 'identifying dependency proxy urls properly with', :no_user + end + end + + context 'with a deploy token' do + let_it_be(:dependency_proxy_user) { create(:deploy_token) } + + it_behaves_like 'identifying dependency proxy urls properly with', :no_user + end + + context 'with no jwt token' do + let(:token) { nil } + + it_behaves_like 'identifying dependency proxy urls properly with', :no_user + end + + context 'with an expired jwt token' do + let(:token) { build_jwt(dependency_proxy_user).encoded } + let(:accessed_path) { '/v2/group1/dependency_proxy/containers/alpine/manifests/latest' } + + it 'returns nil' do + travel_to(Time.zone.now + Auth::DependencyProxyAuthenticationService.token_expire_at + 1.minute) do + expect(subject).to eq(nil) + end + end + end + end end describe '#find_personal_access_token_from_http_basic_auth' do @@ -201,8 +282,8 @@ RSpec.describe Gitlab::Auth::RequestAuthenticator do end describe '#find_user_from_job_token' do - let!(:user) { build(:user) } - let!(:job) { build(:ci_build, user: user, status: :running) } + let_it_be(:user) { build(:user) } + let_it_be(:job) { build(:ci_build, user: user, status: :running) } before do env[Gitlab::Auth::AuthFinders::JOB_TOKEN_HEADER] = 'token' @@ -239,7 +320,7 @@ RSpec.describe Gitlab::Auth::RequestAuthenticator do end describe '#runner' do - let!(:runner) { build(:ci_runner) } + let_it_be(:runner) { build(:ci_runner) } it 'returns the runner using #find_runner_from_token' do expect_any_instance_of(described_class) diff --git a/spec/lib/gitlab/background_migration/fix_first_mentioned_in_commit_at_spec.rb b/spec/lib/gitlab/background_migration/fix_first_mentioned_in_commit_at_spec.rb new file mode 100644 index 00000000000..d2bfa86f0d1 --- /dev/null +++ b/spec/lib/gitlab/background_migration/fix_first_mentioned_in_commit_at_spec.rb @@ -0,0 +1,140 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::FixFirstMentionedInCommitAt, :migration, schema: 20211004110500 do + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:users) { table(:users) } + let(:merge_requests) { table(:merge_requests) } + let(:issues) { table(:issues) } + let(:issue_metrics) { table(:issue_metrics) } + let(:merge_requests_closing_issues) { table(:merge_requests_closing_issues) } + let(:diffs) { table(:merge_request_diffs) } + let(:ten_days_ago) { 10.days.ago } + let(:commits) do + table(:merge_request_diff_commits).tap do |t| + t.extend(SuppressCompositePrimaryKeyWarning) + end + end + + let(:namespace) { namespaces.create!(name: 'ns', path: 'ns') } + let(:project) { projects.create!(namespace_id: namespace.id) } + + let!(:issue1) do + issues.create!( + title: 'issue', + description: 'description', + project_id: project.id + ) + end + + let!(:issue2) do + issues.create!( + title: 'issue', + description: 'description', + project_id: project.id + ) + end + + let!(:merge_request1) do + merge_requests.create!( + source_branch: 'a', + target_branch: 'master', + target_project_id: project.id + ) + end + + let!(:merge_request2) do + merge_requests.create!( + source_branch: 'b', + target_branch: 'master', + target_project_id: project.id + ) + end + + let!(:merge_request_closing_issue1) do + merge_requests_closing_issues.create!(issue_id: issue1.id, merge_request_id: merge_request1.id) + end + + let!(:merge_request_closing_issue2) do + merge_requests_closing_issues.create!(issue_id: issue2.id, merge_request_id: merge_request2.id) + end + + let!(:diff1) { diffs.create!(merge_request_id: merge_request1.id) } + let!(:diff2) { diffs.create!(merge_request_id: merge_request1.id) } + + let!(:other_diff) { diffs.create!(merge_request_id: merge_request2.id) } + + let!(:commit1) do + commits.create!( + merge_request_diff_id: diff2.id, + relative_order: 0, + sha: Gitlab::Database::ShaAttribute.serialize('aaa'), + authored_date: 5.days.ago + ) + end + + let!(:commit2) do + commits.create!( + merge_request_diff_id: diff2.id, + relative_order: 1, + sha: Gitlab::Database::ShaAttribute.serialize('aaa'), + authored_date: 10.days.ago + ) + end + + let!(:commit3) do + commits.create!( + merge_request_diff_id: other_diff.id, + relative_order: 1, + sha: Gitlab::Database::ShaAttribute.serialize('aaa'), + authored_date: 5.days.ago + ) + end + + def run_migration + described_class + .new + .perform(issue_metrics.minimum(:issue_id), issue_metrics.maximum(:issue_id)) + end + + it "marks successful slices as completed" do + min_issue_id = issue_metrics.minimum(:issue_id) + max_issue_id = issue_metrics.maximum(:issue_id) + + expect(subject).to receive(:mark_job_as_succeeded).with(min_issue_id, max_issue_id) + + subject.perform(min_issue_id, max_issue_id) + end + + context 'when the persisted first_mentioned_in_commit_at is later than the first commit authored_date' do + it 'updates the issue_metrics record' do + record1 = issue_metrics.create!(issue_id: issue1.id, first_mentioned_in_commit_at: Time.current) + record2 = issue_metrics.create!(issue_id: issue2.id, first_mentioned_in_commit_at: Time.current) + + run_migration + record1.reload + record2.reload + + expect(record1.first_mentioned_in_commit_at).to be_within(2.seconds).of(commit2.authored_date) + expect(record2.first_mentioned_in_commit_at).to be_within(2.seconds).of(commit3.authored_date) + end + end + + context 'when the persisted first_mentioned_in_commit_at is earlier than the first commit authored_date' do + it 'does not update the issue_metrics record' do + record = issue_metrics.create!(issue_id: issue1.id, first_mentioned_in_commit_at: 20.days.ago) + + expect { run_migration }.not_to change { record.reload.first_mentioned_in_commit_at } + end + end + + context 'when the first_mentioned_in_commit_at is null' do + it 'does nothing' do + record = issue_metrics.create!(issue_id: issue1.id, first_mentioned_in_commit_at: nil) + + expect { run_migration }.not_to change { record.reload.first_mentioned_in_commit_at } + end + end +end diff --git a/spec/lib/gitlab/background_migration/fix_promoted_epics_discussion_ids_spec.rb b/spec/lib/gitlab/background_migration/fix_promoted_epics_discussion_ids_spec.rb index 452fc962c7b..35ec8be691a 100644 --- a/spec/lib/gitlab/background_migration/fix_promoted_epics_discussion_ids_spec.rb +++ b/spec/lib/gitlab/background_migration/fix_promoted_epics_discussion_ids_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::FixPromotedEpicsDiscussionIds, schema: 20190715193142 do +RSpec.describe Gitlab::BackgroundMigration::FixPromotedEpicsDiscussionIds, schema: 20181228175414 do let(:namespaces) { table(:namespaces) } let(:users) { table(:users) } let(:epics) { table(:epics) } diff --git a/spec/lib/gitlab/background_migration/fix_user_namespace_names_spec.rb b/spec/lib/gitlab/background_migration/fix_user_namespace_names_spec.rb index 0d0ad2cc39e..95509f9b897 100644 --- a/spec/lib/gitlab/background_migration/fix_user_namespace_names_spec.rb +++ b/spec/lib/gitlab/background_migration/fix_user_namespace_names_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::FixUserNamespaceNames, schema: 20190620112608 do +RSpec.describe Gitlab::BackgroundMigration::FixUserNamespaceNames, schema: 20181228175414 do let(:namespaces) { table(:namespaces) } let(:users) { table(:users) } let(:user) { users.create!(name: "The user's full name", projects_limit: 10, username: 'not-null', email: '1') } diff --git a/spec/lib/gitlab/background_migration/fix_user_project_route_names_spec.rb b/spec/lib/gitlab/background_migration/fix_user_project_route_names_spec.rb index 211693d917b..b4444df674e 100644 --- a/spec/lib/gitlab/background_migration/fix_user_project_route_names_spec.rb +++ b/spec/lib/gitlab/background_migration/fix_user_project_route_names_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::FixUserProjectRouteNames, schema: 20190620112608 do +RSpec.describe Gitlab::BackgroundMigration::FixUserProjectRouteNames, schema: 20181228175414 do let(:namespaces) { table(:namespaces) } let(:users) { table(:users) } let(:routes) { table(:routes) } diff --git a/spec/lib/gitlab/background_migration/migrate_null_private_profile_to_false_spec.rb b/spec/lib/gitlab/background_migration/migrate_null_private_profile_to_false_spec.rb deleted file mode 100644 index 6ff1157cb86..00000000000 --- a/spec/lib/gitlab/background_migration/migrate_null_private_profile_to_false_spec.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::MigrateNullPrivateProfileToFalse, schema: 20190620105427 do - let(:users) { table(:users) } - - it 'correctly migrates nil private_profile to false' do - private_profile_true = users.create!(private_profile: true, projects_limit: 1, email: 'a@b.com') - private_profile_false = users.create!(private_profile: false, projects_limit: 1, email: 'b@c.com') - private_profile_nil = users.create!(private_profile: nil, projects_limit: 1, email: 'c@d.com') - - described_class.new.perform(private_profile_true.id, private_profile_nil.id) - - private_profile_true.reload - private_profile_false.reload - private_profile_nil.reload - - expect(private_profile_true.private_profile).to eq(true) - expect(private_profile_false.private_profile).to eq(false) - expect(private_profile_nil.private_profile).to eq(false) - end -end diff --git a/spec/lib/gitlab/background_migration/migrate_pages_metadata_spec.rb b/spec/lib/gitlab/background_migration/migrate_pages_metadata_spec.rb index 815dc2e73e5..b6d93b9ff54 100644 --- a/spec/lib/gitlab/background_migration/migrate_pages_metadata_spec.rb +++ b/spec/lib/gitlab/background_migration/migrate_pages_metadata_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::MigratePagesMetadata, schema: 20190919040324 do +RSpec.describe Gitlab::BackgroundMigration::MigratePagesMetadata, schema: 20181228175414 do let(:projects) { table(:projects) } subject(:migrate_pages_metadata) { described_class.new } diff --git a/spec/lib/gitlab/background_migration/populate_merge_request_assignees_table_spec.rb b/spec/lib/gitlab/background_migration/populate_merge_request_assignees_table_spec.rb index 4e7872a9a1b..1d8eed53553 100644 --- a/spec/lib/gitlab/background_migration/populate_merge_request_assignees_table_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_merge_request_assignees_table_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::PopulateMergeRequestAssigneesTable, schema: 20190315191339 do +RSpec.describe Gitlab::BackgroundMigration::PopulateMergeRequestAssigneesTable, schema: 20181228175414 do let(:namespaces) { table(:namespaces) } let(:projects) { table(:projects) } let(:users) { table(:users) } diff --git a/spec/lib/gitlab/background_migration/populate_topics_total_projects_count_cache_spec.rb b/spec/lib/gitlab/background_migration/populate_topics_total_projects_count_cache_spec.rb new file mode 100644 index 00000000000..8e07b43f5b9 --- /dev/null +++ b/spec/lib/gitlab/background_migration/populate_topics_total_projects_count_cache_spec.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::PopulateTopicsTotalProjectsCountCache, schema: 20211006060436 do + it 'correctly populates total projects count cache' do + namespaces = table(:namespaces) + projects = table(:projects) + topics = table(:topics) + project_topics = table(:project_topics) + + group = namespaces.create!(name: 'group', path: 'group') + project_1 = projects.create!(namespace_id: group.id) + project_2 = projects.create!(namespace_id: group.id) + project_3 = projects.create!(namespace_id: group.id) + topic_1 = topics.create!(name: 'Topic1') + topic_2 = topics.create!(name: 'Topic2') + topic_3 = topics.create!(name: 'Topic3') + topic_4 = topics.create!(name: 'Topic4') + + project_topics.create!(project_id: project_1.id, topic_id: topic_1.id) + project_topics.create!(project_id: project_1.id, topic_id: topic_3.id) + project_topics.create!(project_id: project_2.id, topic_id: topic_3.id) + project_topics.create!(project_id: project_1.id, topic_id: topic_4.id) + project_topics.create!(project_id: project_2.id, topic_id: topic_4.id) + project_topics.create!(project_id: project_3.id, topic_id: topic_4.id) + + subject.perform(topic_1.id, topic_4.id) + + expect(topic_1.reload.total_projects_count).to eq(1) + expect(topic_2.reload.total_projects_count).to eq(0) + expect(topic_3.reload.total_projects_count).to eq(2) + expect(topic_4.reload.total_projects_count).to eq(3) + end +end diff --git a/spec/lib/gitlab/backtrace_cleaner_spec.rb b/spec/lib/gitlab/backtrace_cleaner_spec.rb index 51d99bf5f74..e46a90e8606 100644 --- a/spec/lib/gitlab/backtrace_cleaner_spec.rb +++ b/spec/lib/gitlab/backtrace_cleaner_spec.rb @@ -22,9 +22,6 @@ RSpec.describe Gitlab::BacktraceCleaner do "lib/gitlab/git/repository.rb:1451:in `gitaly_migrate'", "lib/gitlab/git/commit.rb:66:in `find'", "app/models/repository.rb:1047:in `find_commit'", - "lib/gitlab/metrics/instrumentation.rb:159:in `block in find_commit'", - "lib/gitlab/metrics/method_call.rb:36:in `measure'", - "lib/gitlab/metrics/instrumentation.rb:159:in `find_commit'", "app/models/repository.rb:113:in `commit'", "lib/gitlab/i18n.rb:50:in `with_locale'", "lib/gitlab/middleware/multipart.rb:95:in `call'", diff --git a/spec/lib/gitlab/cache/import/caching_spec.rb b/spec/lib/gitlab/cache/import/caching_spec.rb index f770960e27a..946a7c604a1 100644 --- a/spec/lib/gitlab/cache/import/caching_spec.rb +++ b/spec/lib/gitlab/cache/import/caching_spec.rb @@ -58,6 +58,16 @@ RSpec.describe Gitlab::Cache::Import::Caching, :clean_gitlab_redis_cache do end end + describe '.increment' do + it 'increment a key and returns the current value' do + expect(described_class.increment('foo')).to eq(1) + + value = Gitlab::Redis::Cache.with { |r| r.get(described_class.cache_key_for('foo')) } + + expect(value.to_i).to eq(1) + end + end + describe '.set_add' do it 'adds a value to a set' do described_class.set_add('foo', 10) diff --git a/spec/lib/gitlab/chat/command_spec.rb b/spec/lib/gitlab/chat/command_spec.rb index d99c07d1fa3..c8b4b3f73b2 100644 --- a/spec/lib/gitlab/chat/command_spec.rb +++ b/spec/lib/gitlab/chat/command_spec.rb @@ -72,6 +72,7 @@ RSpec.describe Gitlab::Chat::Command do expect(vars['CHAT_INPUT']).to eq('foo') expect(vars['CHAT_CHANNEL']).to eq('123') + expect(vars['CHAT_USER_ID']).to eq(chat_name.chat_id) end end end diff --git a/spec/lib/gitlab/checks/matching_merge_request_spec.rb b/spec/lib/gitlab/checks/matching_merge_request_spec.rb index 2e562a5a350..c65a1e4d656 100644 --- a/spec/lib/gitlab/checks/matching_merge_request_spec.rb +++ b/spec/lib/gitlab/checks/matching_merge_request_spec.rb @@ -31,33 +31,30 @@ RSpec.describe Gitlab::Checks::MatchingMergeRequest do expect(matcher.match?).to be false end - context 'with load balancing disabled', :request_store, :redis do - before do - expect(::Gitlab::Database::LoadBalancing).to receive(:enable?).at_least(:once).and_return(false) - expect(::Gitlab::Database::LoadBalancing::Sticking).not_to receive(:unstick_or_continue_sticking) - expect(::Gitlab::Database::LoadBalancing::Sticking).not_to receive(:select_valid_replicas) - end - - it 'does not attempt to stick to primary' do - expect(subject.match?).to be true - end - - it 'increments no counters' do - expect { subject.match? } - .to change { total_counter.get }.by(0) - .and change { stale_counter.get }.by(0) - end - end - - context 'with load balancing enabled', :db_load_balancing do + context 'with load balancing enabled' do let(:session) { ::Gitlab::Database::LoadBalancing::Session.current } let(:all_caught_up) { true } before do - allow(::Gitlab::Database::LoadBalancing::Sticking).to receive(:all_caught_up?).and_return(all_caught_up) + Gitlab::Database::LoadBalancing::Session.clear_session + + allow(::ApplicationRecord.sticking) + .to receive(:all_caught_up?) + .and_return(all_caught_up) + + expect(::ApplicationRecord.sticking) + .to receive(:select_valid_host) + .with(:project, project.id) + .and_call_original + + allow(::ApplicationRecord.sticking) + .to receive(:select_caught_up_replicas) + .with(:project, project.id) + .and_return(all_caught_up) + end - expect(::Gitlab::Database::LoadBalancing::Sticking).to receive(:select_valid_host).with(:project, project.id).and_call_original - allow(::Gitlab::Database::LoadBalancing::Sticking).to receive(:select_caught_up_replicas).with(:project, project.id).and_return(all_caught_up) + after do + Gitlab::Database::LoadBalancing::Session.clear_session end shared_examples 'secondary that has caught up to a primary' do diff --git a/spec/lib/gitlab/ci/build/auto_retry_spec.rb b/spec/lib/gitlab/ci/build/auto_retry_spec.rb index e83e1326206..fc5999d59ac 100644 --- a/spec/lib/gitlab/ci/build/auto_retry_spec.rb +++ b/spec/lib/gitlab/ci/build/auto_retry_spec.rb @@ -24,6 +24,7 @@ RSpec.describe Gitlab::Ci::Build::AutoRetry do "default for scheduler failure" | 1 | {} | :scheduler_failure | true "quota is exceeded" | 0 | { max: 2 } | :ci_quota_exceeded | false "no matching runner" | 0 | { max: 2 } | :no_matching_runner | false + "missing dependencies" | 0 | { max: 2 } | :missing_dependency_failure | false end with_them do diff --git a/spec/lib/gitlab/ci/config/entry/retry_spec.rb b/spec/lib/gitlab/ci/config/entry/retry_spec.rb index b38387a437e..84ef5344a8b 100644 --- a/spec/lib/gitlab/ci/config/entry/retry_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/retry_spec.rb @@ -101,7 +101,6 @@ RSpec.describe Gitlab::Ci::Config::Entry::Retry do api_failure stuck_or_timeout_failure runner_system_failure - missing_dependency_failure runner_unsupported stale_schedule job_execution_timeout diff --git a/spec/lib/gitlab/ci/config/external/mapper_spec.rb b/spec/lib/gitlab/ci/config/external/mapper_spec.rb index a471997e43a..cebe8984741 100644 --- a/spec/lib/gitlab/ci/config/external/mapper_spec.rb +++ b/spec/lib/gitlab/ci/config/external/mapper_spec.rb @@ -363,17 +363,6 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper do expect(subject).to contain_exactly(an_instance_of(Gitlab::Ci::Config::External::File::Remote), an_instance_of(Gitlab::Ci::Config::External::File::Local)) end - - context 'when the FF ci_include_rules is disabled' do - before do - stub_feature_flags(ci_include_rules: false) - end - - it 'includes the file' do - expect(subject).to contain_exactly(an_instance_of(Gitlab::Ci::Config::External::File::Remote), - an_instance_of(Gitlab::Ci::Config::External::File::Local)) - end - end end context 'when the rules does not match' do @@ -382,17 +371,6 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper do it 'does not include the file' do expect(subject).to contain_exactly(an_instance_of(Gitlab::Ci::Config::External::File::Remote)) end - - context 'when the FF ci_include_rules is disabled' do - before do - stub_feature_flags(ci_include_rules: false) - end - - it 'includes the file' do - expect(subject).to contain_exactly(an_instance_of(Gitlab::Ci::Config::External::File::Remote), - an_instance_of(Gitlab::Ci::Config::External::File::Local)) - end - end end end end diff --git a/spec/lib/gitlab/ci/config/external/processor_spec.rb b/spec/lib/gitlab/ci/config/external/processor_spec.rb index e032d372ecb..c2f28253f54 100644 --- a/spec/lib/gitlab/ci/config/external/processor_spec.rb +++ b/spec/lib/gitlab/ci/config/external/processor_spec.rb @@ -402,5 +402,17 @@ RSpec.describe Gitlab::Ci::Config::External::Processor do expect(output.keys).to match_array([:image, :my_build, :my_test]) end end + + context 'when rules defined' do + context 'when a rule is invalid' do + let(:values) do + { include: [{ local: 'builds.yml', rules: [{ exists: ['$MY_VAR'] }] }] } + end + + it 'raises IncludeError' do + expect { subject }.to raise_error(described_class::IncludeError, /invalid include rule/) + end + end + end end end diff --git a/spec/lib/gitlab/ci/config/external/rules_spec.rb b/spec/lib/gitlab/ci/config/external/rules_spec.rb index 89ea13d710d..9a5c29befa2 100644 --- a/spec/lib/gitlab/ci/config/external/rules_spec.rb +++ b/spec/lib/gitlab/ci/config/external/rules_spec.rb @@ -16,7 +16,7 @@ RSpec.describe Gitlab::Ci::Config::External::Rules do it { is_expected.to eq(true) } end - context 'when there is a rule' do + context 'when there is a rule with if' do let(:rule_hashes) { [{ if: '$MY_VAR == "hello"' }] } context 'when the rule matches' do @@ -31,5 +31,23 @@ RSpec.describe Gitlab::Ci::Config::External::Rules do it { is_expected.to eq(false) } end end + + context 'when there is a rule with if and when' do + let(:rule_hashes) { [{ if: '$MY_VAR == "hello"', when: 'on_success' }] } + + it 'raises an error' do + expect { result }.to raise_error(described_class::InvalidIncludeRulesError, + 'invalid include rule: {:if=>"$MY_VAR == \"hello\"", :when=>"on_success"}') + end + end + + context 'when there is a rule with exists' do + let(:rule_hashes) { [{ exists: ['$MY_VAR'] }] } + + it 'raises an error' do + expect { result }.to raise_error(described_class::InvalidIncludeRulesError, + 'invalid include rule: {:exists=>["$MY_VAR"]}') + end + end end end diff --git a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb index 0c28515b574..3aa6b2e3c05 100644 --- a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb @@ -440,17 +440,30 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build do context 'when the environment name is invalid' do let(:attributes) { { name: 'deploy', ref: 'master', environment: '!!!' } } - it_behaves_like 'non-deployment job' - it_behaves_like 'ensures environment inexistence' + it 'fails the job with a failure reason and does not create an environment' do + expect(subject).to be_failed + expect(subject).to be_environment_creation_failure + expect(subject.metadata.expanded_environment_name).to be_nil + expect(Environment.exists?(name: expected_environment_name)).to eq(false) + end + + context 'when surface_environment_creation_failure feature flag is disabled' do + before do + stub_feature_flags(surface_environment_creation_failure: false) + end - it 'tracks an exception' do - expect(Gitlab::ErrorTracking).to receive(:track_exception) - .with(an_instance_of(described_class::EnvironmentCreationFailure), - project_id: project.id, - reason: %q{Name can contain only letters, digits, '-', '_', '/', '$', '{', '}', '.', and spaces, but it cannot start or end with '/'}) - .once + it_behaves_like 'non-deployment job' + it_behaves_like 'ensures environment inexistence' - subject + it 'tracks an exception' do + expect(Gitlab::ErrorTracking).to receive(:track_exception) + .with(an_instance_of(described_class::EnvironmentCreationFailure), + project_id: project.id, + reason: %q{Name can contain only letters, digits, '-', '_', '/', '$', '{', '}', '.', and spaces, but it cannot start or end with '/'}) + .once + + subject + end end end end diff --git a/spec/lib/gitlab/ci/reports/security/flag_spec.rb b/spec/lib/gitlab/ci/reports/security/flag_spec.rb index 27f83694ac2..d677425a8da 100644 --- a/spec/lib/gitlab/ci/reports/security/flag_spec.rb +++ b/spec/lib/gitlab/ci/reports/security/flag_spec.rb @@ -18,9 +18,9 @@ RSpec.describe Gitlab::Ci::Reports::Security::Flag do end end - describe '#to_hash' do + describe '#to_h' do it 'returns expected hash' do - expect(security_flag.to_hash).to eq( + expect(security_flag.to_h).to eq( { flag_type: :false_positive, origin: 'post analyzer X', diff --git a/spec/lib/gitlab/ci/templates/templates_spec.rb b/spec/lib/gitlab/ci/templates/templates_spec.rb index 81fc66c4a11..cdda7e953d0 100644 --- a/spec/lib/gitlab/ci/templates/templates_spec.rb +++ b/spec/lib/gitlab/ci/templates/templates_spec.rb @@ -13,13 +13,6 @@ RSpec.describe 'CI YML Templates' do excluded + ["Terraform.gitlab-ci.yml"] end - before do - stub_feature_flags( - redirect_to_latest_template_terraform: false, - redirect_to_latest_template_security_api_fuzzing: false, - redirect_to_latest_template_security_dast: false) - end - shared_examples 'require default stages to be included' do it 'require default stages to be included' do expect(subject.stages).to include(*Gitlab::Ci::Config::Entry::Stages.default) diff --git a/spec/lib/gitlab/ci/trace/archive_spec.rb b/spec/lib/gitlab/ci/trace/archive_spec.rb new file mode 100644 index 00000000000..c9fc4e720c4 --- /dev/null +++ b/spec/lib/gitlab/ci/trace/archive_spec.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Trace::Archive do + let_it_be(:job) { create(:ci_build, :success, :trace_live) } + let_it_be_with_reload(:trace_metadata) { create(:ci_build_trace_metadata, build: job) } + let_it_be(:src_checksum) do + job.trace.read { |stream| Digest::MD5.hexdigest(stream.raw) } + end + + let(:metrics) { spy('metrics') } + + describe '#execute' do + subject { described_class.new(job, trace_metadata, metrics) } + + it 'computes and assigns checksum' do + Gitlab::Ci::Trace::ChunkedIO.new(job) do |stream| + expect { subject.execute!(stream) }.to change { Ci::JobArtifact.count }.by(1) + end + + expect(trace_metadata.checksum).to eq(src_checksum) + expect(trace_metadata.trace_artifact).to eq(job.job_artifacts_trace) + end + + context 'validating artifact checksum' do + let(:trace) { 'abc' } + let(:stream) { StringIO.new(trace, 'rb') } + let(:src_checksum) { Digest::MD5.hexdigest(trace) } + + context 'when the object store is disabled' do + before do + stub_artifacts_object_storage(enabled: false) + end + + it 'skips validation' do + subject.execute!(stream) + + expect(trace_metadata.checksum).to eq(src_checksum) + expect(trace_metadata.remote_checksum).to be_nil + expect(metrics) + .not_to have_received(:increment_error_counter) + .with(type: :archive_invalid_checksum) + end + end + + context 'with background_upload enabled' do + before do + stub_artifacts_object_storage(background_upload: true) + end + + it 'skips validation' do + subject.execute!(stream) + + expect(trace_metadata.checksum).to eq(src_checksum) + expect(trace_metadata.remote_checksum).to be_nil + expect(metrics) + .not_to have_received(:increment_error_counter) + .with(type: :archive_invalid_checksum) + end + end + + context 'with direct_upload enabled' do + before do + stub_artifacts_object_storage(direct_upload: true) + end + + it 'validates the archived trace' do + subject.execute!(stream) + + expect(trace_metadata.checksum).to eq(src_checksum) + expect(trace_metadata.remote_checksum).to eq(src_checksum) + expect(metrics) + .not_to have_received(:increment_error_counter) + .with(type: :archive_invalid_checksum) + end + + context 'when the checksum does not match' do + let(:invalid_remote_checksum) { SecureRandom.hex } + + before do + expect(::Gitlab::Ci::Trace::RemoteChecksum) + .to receive(:new) + .with(an_instance_of(Ci::JobArtifact)) + .and_return(double(md5_checksum: invalid_remote_checksum)) + end + + it 'validates the archived trace' do + subject.execute!(stream) + + expect(trace_metadata.checksum).to eq(src_checksum) + expect(trace_metadata.remote_checksum).to eq(invalid_remote_checksum) + expect(metrics) + .to have_received(:increment_error_counter) + .with(type: :archive_invalid_checksum) + end + end + end + end + end +end diff --git a/spec/lib/gitlab/ci/trace/metrics_spec.rb b/spec/lib/gitlab/ci/trace/metrics_spec.rb index 6518d0ab075..53e55a57973 100644 --- a/spec/lib/gitlab/ci/trace/metrics_spec.rb +++ b/spec/lib/gitlab/ci/trace/metrics_spec.rb @@ -15,4 +15,27 @@ RSpec.describe Gitlab::Ci::Trace::Metrics, :prometheus do end end end + + describe '#increment_error_counter' do + context 'when the operation type is known' do + it 'increments the counter' do + subject.increment_error_counter(type: :chunks_invalid_size) + subject.increment_error_counter(type: :chunks_invalid_checksum) + subject.increment_error_counter(type: :archive_invalid_checksum) + + expect(described_class.trace_errors_counter.get(type: :chunks_invalid_size)).to eq 1 + expect(described_class.trace_errors_counter.get(type: :chunks_invalid_checksum)).to eq 1 + expect(described_class.trace_errors_counter.get(type: :archive_invalid_checksum)).to eq 1 + + expect(described_class.trace_errors_counter.values.count).to eq 3 + end + end + + context 'when the operation type is known' do + it 'raises an exception' do + expect { subject.increment_error_counter(type: :invalid_type) } + .to raise_error(ArgumentError) + end + end + end end diff --git a/spec/lib/gitlab/ci/trace/remote_checksum_spec.rb b/spec/lib/gitlab/ci/trace/remote_checksum_spec.rb new file mode 100644 index 00000000000..8837ebc3652 --- /dev/null +++ b/spec/lib/gitlab/ci/trace/remote_checksum_spec.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Trace::RemoteChecksum do + let_it_be(:job) { create(:ci_build, :success) } + + let(:file_store) { JobArtifactUploader::Store::LOCAL } + let(:trace_artifact) { create(:ci_job_artifact, :trace, job: job, file_store: file_store) } + let(:checksum) { Digest::MD5.hexdigest(trace_artifact.file.read) } + let(:base64checksum) { Digest::MD5.base64digest(trace_artifact.file.read) } + let(:fetcher) { described_class.new(trace_artifact) } + + describe '#md5_checksum' do + subject { fetcher.md5_checksum } + + context 'when the file is stored locally' do + it { is_expected.to be_nil } + end + + context 'when object store is enabled' do + before do + stub_artifacts_object_storage + end + + context 'with local files' do + it { is_expected.to be_nil } + end + + context 'with remote files' do + let(:file_store) { JobArtifactUploader::Store::REMOTE } + + context 'when the feature flag is disabled' do + before do + stub_feature_flags(ci_archived_build_trace_checksum: false) + end + + it { is_expected.to be_nil } + end + + context 'with AWS as provider' do + it { is_expected.to eq(checksum) } + end + + context 'with Google as provider' do + before do + spy_file = spy(:file) + expect(fetcher).to receive(:provider_google?) { true } + expect(fetcher).not_to receive(:provider_aws?) { false } + allow(spy_file).to receive(:attributes).and_return(metadata) + + allow_next_found_instance_of(Ci::JobArtifact) do |trace_artifact| + allow(trace_artifact.file).to receive(:file) { spy_file } + end + end + + context 'when the response does not include :content_md5' do + let(:metadata) {{}} + + it 'raises an exception' do + expect { subject }.to raise_error KeyError, /content_md5/ + end + end + + context 'when the response include :content_md5' do + let(:metadata) {{ content_md5: base64checksum }} + + it { is_expected.to eq(checksum) } + end + end + + context 'with unsupported providers' do + let(:file_store) { JobArtifactUploader::Store::REMOTE } + + before do + expect(fetcher).to receive(:provider_aws?) { false } + expect(fetcher).to receive(:provider_google?) { false } + end + + it { is_expected.to be_nil } + end + end + end + end +end diff --git a/spec/lib/gitlab/content_security_policy/config_loader_spec.rb b/spec/lib/gitlab/content_security_policy/config_loader_spec.rb index 239eff11bf3..3ec332dace5 100644 --- a/spec/lib/gitlab/content_security_policy/config_loader_spec.rb +++ b/spec/lib/gitlab/content_security_policy/config_loader_spec.rb @@ -11,6 +11,7 @@ RSpec.describe Gitlab::ContentSecurityPolicy::ConfigLoader do directives: { base_uri: 'http://example.com', child_src: "'self' https://child.example.com", + connect_src: "'self' ws://example.com", default_src: "'self' https://other.example.com", script_src: "'self' https://script.exammple.com ", worker_src: "data: https://worker.example.com", @@ -52,6 +53,28 @@ RSpec.describe Gitlab::ContentSecurityPolicy::ConfigLoader do expect(directives['child_src']).to eq(directives['frame_src']) end + context 'adds all websocket origins to support Safari' do + it 'with insecure domain' do + stub_config_setting(host: 'example.com', https: false) + expect(directives['connect_src']).to eq("'self' ws://example.com") + end + + it 'with secure domain' do + stub_config_setting(host: 'example.com', https: true) + expect(directives['connect_src']).to eq("'self' wss://example.com") + end + + it 'with custom port' do + stub_config_setting(host: 'example.com', port: '1234') + expect(directives['connect_src']).to eq("'self' ws://example.com:1234") + end + + it 'with custom port and secure domain' do + stub_config_setting(host: 'example.com', https: true, port: '1234') + expect(directives['connect_src']).to eq("'self' wss://example.com:1234") + end + end + context 'when CDN host is defined' do before do stub_config_setting(cdn_host: 'https://example.com') @@ -67,10 +90,11 @@ RSpec.describe Gitlab::ContentSecurityPolicy::ConfigLoader do context 'when sentry is configured' do before do stub_sentry_settings + stub_config_setting(host: 'example.com') end it 'adds sentry path to CSP without user' do - expect(directives['connect_src']).to eq("'self' dummy://example.com/43") + expect(directives['connect_src']).to eq("'self' ws://example.com dummy://example.com/43") end end @@ -113,6 +137,7 @@ RSpec.describe Gitlab::ContentSecurityPolicy::ConfigLoader do expect(policy.directives['base-uri']).to eq([csp_config[:directives][:base_uri]]) expect(policy.directives['default-src']).to eq(expected_config(:default_src)) + expect(policy.directives['connect-src']).to eq(expected_config(:connect_src)) expect(policy.directives['child-src']).to eq(expected_config(:child_src)) expect(policy.directives['worker-src']).to eq(expected_config(:worker_src)) expect(policy.directives['report-uri']).to eq(expected_config(:report_uri)) diff --git a/spec/lib/gitlab/database/bulk_update_spec.rb b/spec/lib/gitlab/database/bulk_update_spec.rb index dbafada26ca..9a6463c99fa 100644 --- a/spec/lib/gitlab/database/bulk_update_spec.rb +++ b/spec/lib/gitlab/database/bulk_update_spec.rb @@ -91,45 +91,38 @@ RSpec.describe Gitlab::Database::BulkUpdate do .to eq(['MR a', 'Issue a', 'Issue b']) end - shared_examples 'basic functionality' do - it 'sets multiple values' do - create_default(:user) - create_default(:project) - - i_a, i_b = create_list(:issue, 2) + context 'validates prepared_statements support', :reestablished_active_record_base do + using RSpec::Parameterized::TableSyntax - mapping = { - i_a => { title: 'Issue a' }, - i_b => { title: 'Issue b' } - } + where(:prepared_statements) do + [false, true] + end - described_class.execute(%i[title], mapping) + before do + configuration_hash = ActiveRecord::Base.connection_db_config.configuration_hash - expect([i_a, i_b].map { |x| x.reset.title }) - .to eq(['Issue a', 'Issue b']) + ActiveRecord::Base.establish_connection( + configuration_hash.merge(prepared_statements: prepared_statements) + ) end - end - include_examples 'basic functionality' + with_them do + it 'sets multiple values' do + create_default(:user) + create_default(:project) - context 'when prepared statements are configured differently to the normal test environment' do - before do - klass = Class.new(ActiveRecord::Base) do - def self.abstract_class? - true # So it gets its own connection - end - end + i_a, i_b = create_list(:issue, 2) - stub_const('ActiveRecordBasePreparedStatementsInverted', klass) + mapping = { + i_a => { title: 'Issue a' }, + i_b => { title: 'Issue b' } + } - c = ActiveRecord::Base.connection.instance_variable_get(:@config) - inverted = c.merge(prepared_statements: !ActiveRecord::Base.connection.prepared_statements) - ActiveRecordBasePreparedStatementsInverted.establish_connection(inverted) + described_class.execute(%i[title], mapping) - allow(ActiveRecord::Base).to receive(:connection_specification_name) - .and_return(ActiveRecordBasePreparedStatementsInverted.connection_specification_name) + expect([i_a, i_b].map { |x| x.reset.title }) + .to eq(['Issue a', 'Issue b']) + end end - - include_examples 'basic functionality' end end diff --git a/spec/lib/gitlab/database/connection_spec.rb b/spec/lib/gitlab/database/connection_spec.rb index 7f94d7af4a9..ee1df141cd6 100644 --- a/spec/lib/gitlab/database/connection_spec.rb +++ b/spec/lib/gitlab/database/connection_spec.rb @@ -126,15 +126,7 @@ RSpec.describe Gitlab::Database::Connection do end end - describe '#disable_prepared_statements' do - around do |example| - original_config = connection.scope.connection.pool.db_config - - example.run - - connection.scope.establish_connection(original_config) - end - + describe '#disable_prepared_statements', :reestablished_active_record_base do it 'disables prepared statements' do connection.scope.establish_connection( ::Gitlab::Database.main.config.merge(prepared_statements: true) diff --git a/spec/lib/gitlab/database/consistency_spec.rb b/spec/lib/gitlab/database/consistency_spec.rb index 35fa65512ae..5055be81c88 100644 --- a/spec/lib/gitlab/database/consistency_spec.rb +++ b/spec/lib/gitlab/database/consistency_spec.rb @@ -7,6 +7,14 @@ RSpec.describe Gitlab::Database::Consistency do Gitlab::Database::LoadBalancing::Session.current end + before do + Gitlab::Database::LoadBalancing::Session.clear_session + end + + after do + Gitlab::Database::LoadBalancing::Session.clear_session + end + describe '.with_read_consistency' do it 'sticks to primary database' do expect(session).not_to be_using_primary diff --git a/spec/lib/gitlab/database/count_spec.rb b/spec/lib/gitlab/database/count_spec.rb index d65413c2a00..e712ad09927 100644 --- a/spec/lib/gitlab/database/count_spec.rb +++ b/spec/lib/gitlab/database/count_spec.rb @@ -46,5 +46,49 @@ RSpec.describe Gitlab::Database::Count do subject end end + + context 'default strategies' do + subject { described_class.approximate_counts(models) } + + context 'with a read-only database' do + before do + allow(Gitlab::Database).to receive(:read_only?).and_return(true) + end + + it 'only uses the ExactCountStrategy' do + allow_next_instance_of(Gitlab::Database::Count::TablesampleCountStrategy) do |instance| + expect(instance).not_to receive(:count) + end + allow_next_instance_of(Gitlab::Database::Count::ReltuplesCountStrategy) do |instance| + expect(instance).not_to receive(:count) + end + expect_next_instance_of(Gitlab::Database::Count::ExactCountStrategy) do |instance| + expect(instance).to receive(:count).and_return({}) + end + + subject + end + end + + context 'with a read-write database' do + before do + allow(Gitlab::Database).to receive(:read_only?).and_return(false) + end + + it 'uses the available strategies' do + [ + Gitlab::Database::Count::TablesampleCountStrategy, + Gitlab::Database::Count::ReltuplesCountStrategy, + Gitlab::Database::Count::ExactCountStrategy + ].each do |strategy_klass| + expect_next_instance_of(strategy_klass) do |instance| + expect(instance).to receive(:count).and_return({}) + end + end + + subject + end + end + end end end diff --git a/spec/lib/gitlab/database/load_balancing/action_cable_callbacks_spec.rb b/spec/lib/gitlab/database/load_balancing/action_cable_callbacks_spec.rb index ebbbafb855f..768855464c1 100644 --- a/spec/lib/gitlab/database/load_balancing/action_cable_callbacks_spec.rb +++ b/spec/lib/gitlab/database/load_balancing/action_cable_callbacks_spec.rb @@ -5,7 +5,7 @@ require 'spec_helper' RSpec.describe Gitlab::Database::LoadBalancing::ActionCableCallbacks, :request_store do describe '.wrapper' do it 'uses primary and then releases the connection and clears the session' do - expect(Gitlab::Database::LoadBalancing).to receive_message_chain(:proxy, :load_balancer, :release_host) + expect(Gitlab::Database::LoadBalancing).to receive(:release_hosts) expect(Gitlab::Database::LoadBalancing::Session).to receive(:clear_session) described_class.wrapper.call( @@ -18,7 +18,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::ActionCableCallbacks, :request_s context 'with an exception' do it 'releases the connection and clears the session' do - expect(Gitlab::Database::LoadBalancing).to receive_message_chain(:proxy, :load_balancer, :release_host) + expect(Gitlab::Database::LoadBalancing).to receive(:release_hosts) expect(Gitlab::Database::LoadBalancing::Session).to receive(:clear_session) expect do diff --git a/spec/lib/gitlab/database/load_balancing/active_record_proxy_spec.rb b/spec/lib/gitlab/database/load_balancing/active_record_proxy_spec.rb deleted file mode 100644 index 8886ce9756d..00000000000 --- a/spec/lib/gitlab/database/load_balancing/active_record_proxy_spec.rb +++ /dev/null @@ -1,20 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Database::LoadBalancing::ActiveRecordProxy do - describe '#connection' do - it 'returns a connection proxy' do - dummy = Class.new do - include Gitlab::Database::LoadBalancing::ActiveRecordProxy - end - - proxy = double(:proxy) - - expect(Gitlab::Database::LoadBalancing).to receive(:proxy) - .and_return(proxy) - - expect(dummy.new.connection).to eq(proxy) - end - end -end diff --git a/spec/lib/gitlab/database/load_balancing/configuration_spec.rb b/spec/lib/gitlab/database/load_balancing/configuration_spec.rb index 6621e6276a5..3e5249a3dea 100644 --- a/spec/lib/gitlab/database/load_balancing/configuration_spec.rb +++ b/spec/lib/gitlab/database/load_balancing/configuration_spec.rb @@ -108,6 +108,14 @@ RSpec.describe Gitlab::Database::LoadBalancing::Configuration do end describe '#load_balancing_enabled?' do + it 'returns false when running inside a Rake task' do + config = described_class.new(ActiveRecord::Base, %w[foo bar]) + + allow(Gitlab::Runtime).to receive(:rake?).and_return(true) + + expect(config.load_balancing_enabled?).to eq(false) + end + it 'returns true when hosts are configured' do config = described_class.new(ActiveRecord::Base, %w[foo bar]) diff --git a/spec/lib/gitlab/database/load_balancing/host_spec.rb b/spec/lib/gitlab/database/load_balancing/host_spec.rb index e2011692228..b040c7a76bd 100644 --- a/spec/lib/gitlab/database/load_balancing/host_spec.rb +++ b/spec/lib/gitlab/database/load_balancing/host_spec.rb @@ -172,6 +172,14 @@ RSpec.describe Gitlab::Database::LoadBalancing::Host do expect(host).not_to be_online end + + it 'returns false when ActiveRecord::ConnectionNotEstablished is raised' do + allow(host) + .to receive(:check_replica_status?) + .and_raise(ActiveRecord::ConnectionNotEstablished) + + expect(host).not_to be_online + end end end diff --git a/spec/lib/gitlab/database/load_balancing/load_balancer_spec.rb b/spec/lib/gitlab/database/load_balancing/load_balancer_spec.rb index 86fae14b961..f3ce5563e38 100644 --- a/spec/lib/gitlab/database/load_balancing/load_balancer_spec.rb +++ b/spec/lib/gitlab/database/load_balancing/load_balancer_spec.rb @@ -47,16 +47,27 @@ RSpec.describe Gitlab::Database::LoadBalancing::LoadBalancer, :request_store do end describe '#initialize' do - it 'ignores the hosts when the primary_only option is enabled' do + it 'ignores the hosts when load balancing is disabled' do config = Gitlab::Database::LoadBalancing::Configuration .new(ActiveRecord::Base, [db_host]) - lb = described_class.new(config, primary_only: true) + + allow(config).to receive(:load_balancing_enabled?).and_return(false) + + lb = described_class.new(config) hosts = lb.host_list.hosts expect(hosts.length).to eq(1) expect(hosts.first) .to be_instance_of(Gitlab::Database::LoadBalancing::PrimaryHost) end + + it 'sets the name of the connection that is used' do + config = + Gitlab::Database::LoadBalancing::Configuration.new(ActiveRecord::Base) + lb = described_class.new(config) + + expect(lb.name).to eq(:main) + end end describe '#read' do @@ -140,10 +151,13 @@ RSpec.describe Gitlab::Database::LoadBalancing::LoadBalancer, :request_store do .to yield_with_args(ActiveRecord::Base.retrieve_connection) end - it 'uses the primary when the primary_only option is enabled' do + it 'uses the primary when load balancing is disabled' do config = Gitlab::Database::LoadBalancing::Configuration .new(ActiveRecord::Base) - lb = described_class.new(config, primary_only: true) + + allow(config).to receive(:load_balancing_enabled?).and_return(false) + + lb = described_class.new(config) # When no hosts are configured, we don't want to produce any warnings, as # they aren't useful/too noisy. @@ -274,34 +288,43 @@ RSpec.describe Gitlab::Database::LoadBalancing::LoadBalancer, :request_store do expect { lb.retry_with_backoff { raise } }.to raise_error(RuntimeError) end - end - describe '#connection_error?' do - before do - stub_const('Gitlab::Database::LoadBalancing::LoadBalancer::CONNECTION_ERRORS', - [NotImplementedError]) + it 'skips retries when only the primary is used' do + allow(lb).to receive(:primary_only?).and_return(true) + + expect(lb).not_to receive(:sleep) + + expect { lb.retry_with_backoff { raise } }.to raise_error(RuntimeError) end + end + describe '#connection_error?' do it 'returns true for a connection error' do - error = NotImplementedError.new + error = ActiveRecord::ConnectionNotEstablished.new expect(lb.connection_error?(error)).to eq(true) end + it 'returns false for a missing database error' do + error = ActiveRecord::NoDatabaseError.new + + expect(lb.connection_error?(error)).to eq(false) + end + it 'returns true for a wrapped connection error' do - wrapped = wrapped_exception(ActiveRecord::StatementInvalid, NotImplementedError) + wrapped = wrapped_exception(ActiveRecord::StatementInvalid, ActiveRecord::ConnectionNotEstablished) expect(lb.connection_error?(wrapped)).to eq(true) end it 'returns true for a wrapped connection error from a view' do - wrapped = wrapped_exception(ActionView::Template::Error, NotImplementedError) + wrapped = wrapped_exception(ActionView::Template::Error, ActiveRecord::ConnectionNotEstablished) expect(lb.connection_error?(wrapped)).to eq(true) end it 'returns true for deeply wrapped/nested errors' do - top = twice_wrapped_exception(ActionView::Template::Error, ActiveRecord::StatementInvalid, NotImplementedError) + top = twice_wrapped_exception(ActionView::Template::Error, ActiveRecord::StatementInvalid, ActiveRecord::ConnectionNotEstablished) expect(lb.connection_error?(top)).to eq(true) end diff --git a/spec/lib/gitlab/database/load_balancing/primary_host_spec.rb b/spec/lib/gitlab/database/load_balancing/primary_host_spec.rb index a0e63a7ee4e..45d81808971 100644 --- a/spec/lib/gitlab/database/load_balancing/primary_host_spec.rb +++ b/spec/lib/gitlab/database/load_balancing/primary_host_spec.rb @@ -63,9 +63,8 @@ RSpec.describe Gitlab::Database::LoadBalancing::PrimaryHost do end describe '#primary_write_location' do - it 'returns the write location of the primary' do - expect(host.primary_write_location).to be_an_instance_of(String) - expect(host.primary_write_location).not_to be_empty + it 'raises NotImplementedError' do + expect { host.primary_write_location }.to raise_error(NotImplementedError) end end @@ -76,51 +75,8 @@ RSpec.describe Gitlab::Database::LoadBalancing::PrimaryHost do end describe '#database_replica_location' do - let(:connection) { double(:connection) } - - it 'returns the write ahead location of the replica', :aggregate_failures do - expect(host) - .to receive(:query_and_release) - .and_return({ 'location' => '0/D525E3A8' }) - - expect(host.database_replica_location).to be_an_instance_of(String) - end - - it 'returns nil when the database query returned no rows' do - expect(host).to receive(:query_and_release).and_return({}) - - expect(host.database_replica_location).to be_nil - end - - it 'returns nil when the database connection fails' do - allow(host).to receive(:connection).and_raise(PG::Error) - - expect(host.database_replica_location).to be_nil - end - end - - describe '#query_and_release' do - it 'executes a SQL query' do - results = host.query_and_release('SELECT 10 AS number') - - expect(results).to be_an_instance_of(Hash) - expect(results['number'].to_i).to eq(10) - end - - it 'releases the connection after running the query' do - expect(host) - .to receive(:release_connection) - .once - - host.query_and_release('SELECT 10 AS number') - end - - it 'returns an empty Hash in the event of an error' do - expect(host.connection) - .to receive(:select_all) - .and_raise(RuntimeError, 'kittens') - - expect(host.query_and_release('SELECT 10 AS number')).to eq({}) + it 'raises NotImplementedError' do + expect { host.database_replica_location }.to raise_error(NotImplementedError) end end end diff --git a/spec/lib/gitlab/database/load_balancing/rack_middleware_spec.rb b/spec/lib/gitlab/database/load_balancing/rack_middleware_spec.rb index ea0c7f781fd..af7e2a4b167 100644 --- a/spec/lib/gitlab/database/load_balancing/rack_middleware_spec.rb +++ b/spec/lib/gitlab/database/load_balancing/rack_middleware_spec.rb @@ -6,12 +6,12 @@ RSpec.describe Gitlab::Database::LoadBalancing::RackMiddleware, :redis do let(:app) { double(:app) } let(:middleware) { described_class.new(app) } let(:warden_user) { double(:warden, user: double(:user, id: 42)) } - let(:single_sticking_object) { Set.new([[:user, 42]]) } + let(:single_sticking_object) { Set.new([[ActiveRecord::Base, :user, 42]]) } let(:multiple_sticking_objects) do Set.new([ - [:user, 42], - [:runner, '123456789'], - [:runner, '1234'] + [ActiveRecord::Base, :user, 42], + [ActiveRecord::Base, :runner, '123456789'], + [ActiveRecord::Base, :runner, '1234'] ]) end @@ -19,47 +19,6 @@ RSpec.describe Gitlab::Database::LoadBalancing::RackMiddleware, :redis do Gitlab::Database::LoadBalancing::Session.clear_session end - describe '.stick_or_unstick' do - before do - allow(Gitlab::Database::LoadBalancing).to receive(:enable?) - .and_return(true) - end - - it 'sticks or unsticks a single object and updates the Rack environment' do - expect(Gitlab::Database::LoadBalancing::Sticking) - .to receive(:unstick_or_continue_sticking) - .with(:user, 42) - - env = {} - - described_class.stick_or_unstick(env, :user, 42) - - expect(env[described_class::STICK_OBJECT].to_a).to eq([[:user, 42]]) - end - - it 'sticks or unsticks multiple objects and updates the Rack environment' do - expect(Gitlab::Database::LoadBalancing::Sticking) - .to receive(:unstick_or_continue_sticking) - .with(:user, 42) - .ordered - - expect(Gitlab::Database::LoadBalancing::Sticking) - .to receive(:unstick_or_continue_sticking) - .with(:runner, '123456789') - .ordered - - env = {} - - described_class.stick_or_unstick(env, :user, 42) - described_class.stick_or_unstick(env, :runner, '123456789') - - expect(env[described_class::STICK_OBJECT].to_a).to eq([ - [:user, 42], - [:runner, '123456789'] - ]) - end - end - describe '#call' do it 'handles a request' do env = {} @@ -82,7 +41,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::RackMiddleware, :redis do describe '#unstick_or_continue_sticking' do it 'does not stick if no namespace and identifier could be found' do - expect(Gitlab::Database::LoadBalancing::Sticking) + expect(ApplicationRecord.sticking) .not_to receive(:unstick_or_continue_sticking) middleware.unstick_or_continue_sticking({}) @@ -91,9 +50,11 @@ RSpec.describe Gitlab::Database::LoadBalancing::RackMiddleware, :redis do it 'sticks to the primary if a warden user is found' do env = { 'warden' => warden_user } - expect(Gitlab::Database::LoadBalancing::Sticking) - .to receive(:unstick_or_continue_sticking) - .with(:user, 42) + Gitlab::Database::LoadBalancing.base_models.each do |model| + expect(model.sticking) + .to receive(:unstick_or_continue_sticking) + .with(:user, 42) + end middleware.unstick_or_continue_sticking(env) end @@ -101,7 +62,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::RackMiddleware, :redis do it 'sticks to the primary if a sticking namespace and identifier is found' do env = { described_class::STICK_OBJECT => single_sticking_object } - expect(Gitlab::Database::LoadBalancing::Sticking) + expect(ApplicationRecord.sticking) .to receive(:unstick_or_continue_sticking) .with(:user, 42) @@ -111,17 +72,17 @@ RSpec.describe Gitlab::Database::LoadBalancing::RackMiddleware, :redis do it 'sticks to the primary if multiple sticking namespaces and identifiers were found' do env = { described_class::STICK_OBJECT => multiple_sticking_objects } - expect(Gitlab::Database::LoadBalancing::Sticking) + expect(ApplicationRecord.sticking) .to receive(:unstick_or_continue_sticking) .with(:user, 42) .ordered - expect(Gitlab::Database::LoadBalancing::Sticking) + expect(ApplicationRecord.sticking) .to receive(:unstick_or_continue_sticking) .with(:runner, '123456789') .ordered - expect(Gitlab::Database::LoadBalancing::Sticking) + expect(ApplicationRecord.sticking) .to receive(:unstick_or_continue_sticking) .with(:runner, '1234') .ordered @@ -132,7 +93,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::RackMiddleware, :redis do describe '#stick_if_necessary' do it 'does not stick to the primary if not necessary' do - expect(Gitlab::Database::LoadBalancing::Sticking) + expect(ApplicationRecord.sticking) .not_to receive(:stick_if_necessary) middleware.stick_if_necessary({}) @@ -141,9 +102,11 @@ RSpec.describe Gitlab::Database::LoadBalancing::RackMiddleware, :redis do it 'sticks to the primary if a warden user is found' do env = { 'warden' => warden_user } - expect(Gitlab::Database::LoadBalancing::Sticking) - .to receive(:stick_if_necessary) - .with(:user, 42) + Gitlab::Database::LoadBalancing.base_models.each do |model| + expect(model.sticking) + .to receive(:stick_if_necessary) + .with(:user, 42) + end middleware.stick_if_necessary(env) end @@ -151,7 +114,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::RackMiddleware, :redis do it 'sticks to the primary if a a single sticking object is found' do env = { described_class::STICK_OBJECT => single_sticking_object } - expect(Gitlab::Database::LoadBalancing::Sticking) + expect(ApplicationRecord.sticking) .to receive(:stick_if_necessary) .with(:user, 42) @@ -161,17 +124,17 @@ RSpec.describe Gitlab::Database::LoadBalancing::RackMiddleware, :redis do it 'sticks to the primary if multiple sticking namespaces and identifiers were found' do env = { described_class::STICK_OBJECT => multiple_sticking_objects } - expect(Gitlab::Database::LoadBalancing::Sticking) + expect(ApplicationRecord.sticking) .to receive(:stick_if_necessary) .with(:user, 42) .ordered - expect(Gitlab::Database::LoadBalancing::Sticking) + expect(ApplicationRecord.sticking) .to receive(:stick_if_necessary) .with(:runner, '123456789') .ordered - expect(Gitlab::Database::LoadBalancing::Sticking) + expect(ApplicationRecord.sticking) .to receive(:stick_if_necessary) .with(:runner, '1234') .ordered @@ -182,47 +145,34 @@ RSpec.describe Gitlab::Database::LoadBalancing::RackMiddleware, :redis do describe '#clear' do it 'clears the currently used host and session' do - lb = double(:lb) session = spy(:session) - allow(middleware).to receive(:load_balancer).and_return(lb) - - expect(lb).to receive(:release_host) - stub_const('Gitlab::Database::LoadBalancing::Session', session) + expect(Gitlab::Database::LoadBalancing).to receive(:release_hosts) + middleware.clear expect(session).to have_received(:clear_session) end end - describe '.load_balancer' do - it 'returns a the load balancer' do - proxy = double(:proxy) - - expect(Gitlab::Database::LoadBalancing).to receive(:proxy) - .and_return(proxy) - - expect(proxy).to receive(:load_balancer) - - middleware.load_balancer - end - end - - describe '#sticking_namespaces_and_ids' do + describe '#sticking_namespaces' do context 'using a Warden request' do it 'returns the warden user if present' do env = { 'warden' => warden_user } + ids = Gitlab::Database::LoadBalancing.base_models.map do |model| + [model, :user, 42] + end - expect(middleware.sticking_namespaces_and_ids(env)).to eq([[:user, 42]]) + expect(middleware.sticking_namespaces(env)).to eq(ids) end it 'returns an empty Array if no user was present' do warden = double(:warden, user: nil) env = { 'warden' => warden } - expect(middleware.sticking_namespaces_and_ids(env)).to eq([]) + expect(middleware.sticking_namespaces(env)).to eq([]) end end @@ -230,17 +180,17 @@ RSpec.describe Gitlab::Database::LoadBalancing::RackMiddleware, :redis do it 'returns the sticking object' do env = { described_class::STICK_OBJECT => multiple_sticking_objects } - expect(middleware.sticking_namespaces_and_ids(env)).to eq([ - [:user, 42], - [:runner, '123456789'], - [:runner, '1234'] + expect(middleware.sticking_namespaces(env)).to eq([ + [ActiveRecord::Base, :user, 42], + [ActiveRecord::Base, :runner, '123456789'], + [ActiveRecord::Base, :runner, '1234'] ]) end end context 'using a regular request' do it 'returns an empty Array' do - expect(middleware.sticking_namespaces_and_ids({})).to eq([]) + expect(middleware.sticking_namespaces({})).to eq([]) end end end diff --git a/spec/lib/gitlab/database/load_balancing/setup_spec.rb b/spec/lib/gitlab/database/load_balancing/setup_spec.rb new file mode 100644 index 00000000000..01646bc76ef --- /dev/null +++ b/spec/lib/gitlab/database/load_balancing/setup_spec.rb @@ -0,0 +1,119 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::LoadBalancing::Setup do + describe '#setup' do + it 'sets up the load balancer' do + setup = described_class.new(ActiveRecord::Base) + + expect(setup).to receive(:disable_prepared_statements) + expect(setup).to receive(:setup_load_balancer) + expect(setup).to receive(:setup_service_discovery) + + setup.setup + end + end + + describe '#disable_prepared_statements' do + it 'disables prepared statements and reconnects to the database' do + config = double( + :config, + configuration_hash: { host: 'localhost' }, + env_name: 'test', + name: 'main' + ) + model = double(:model, connection_db_config: config) + + expect(ActiveRecord::DatabaseConfigurations::HashConfig) + .to receive(:new) + .with('test', 'main', { host: 'localhost', prepared_statements: false }) + .and_call_original + + # HashConfig doesn't implement its own #==, so we can't directly compare + # the expected value with a pre-defined one. + expect(model) + .to receive(:establish_connection) + .with(an_instance_of(ActiveRecord::DatabaseConfigurations::HashConfig)) + + described_class.new(model).disable_prepared_statements + end + end + + describe '#setup_load_balancer' do + it 'sets up the load balancer' do + model = Class.new(ActiveRecord::Base) + setup = described_class.new(model) + config = Gitlab::Database::LoadBalancing::Configuration.new(model) + lb = instance_spy(Gitlab::Database::LoadBalancing::LoadBalancer) + + allow(lb).to receive(:configuration).and_return(config) + + expect(Gitlab::Database::LoadBalancing::LoadBalancer) + .to receive(:new) + .with(setup.configuration) + .and_return(lb) + + setup.setup_load_balancer + + expect(model.connection.load_balancer).to eq(lb) + expect(model.sticking) + .to be_an_instance_of(Gitlab::Database::LoadBalancing::Sticking) + end + end + + describe '#setup_service_discovery' do + context 'when service discovery is disabled' do + it 'does nothing' do + expect(Gitlab::Database::LoadBalancing::ServiceDiscovery) + .not_to receive(:new) + + described_class.new(ActiveRecord::Base).setup_service_discovery + end + end + + context 'when service discovery is enabled' do + it 'immediately performs service discovery' do + model = ActiveRecord::Base + setup = described_class.new(model) + sv = instance_spy(Gitlab::Database::LoadBalancing::ServiceDiscovery) + lb = model.connection.load_balancer + + allow(setup.configuration) + .to receive(:service_discovery_enabled?) + .and_return(true) + + allow(Gitlab::Database::LoadBalancing::ServiceDiscovery) + .to receive(:new) + .with(lb, setup.configuration.service_discovery) + .and_return(sv) + + expect(sv).to receive(:perform_service_discovery) + expect(sv).not_to receive(:start) + + setup.setup_service_discovery + end + + it 'starts service discovery if needed' do + model = ActiveRecord::Base + setup = described_class.new(model, start_service_discovery: true) + sv = instance_spy(Gitlab::Database::LoadBalancing::ServiceDiscovery) + lb = model.connection.load_balancer + + allow(setup.configuration) + .to receive(:service_discovery_enabled?) + .and_return(true) + + allow(Gitlab::Database::LoadBalancing::ServiceDiscovery) + .to receive(:new) + .with(lb, setup.configuration.service_discovery) + .and_return(sv) + + expect(sv).to receive(:perform_service_discovery) + expect(sv).to receive(:start) + + setup.setup_service_discovery + end + end + end +end diff --git a/spec/lib/gitlab/database/load_balancing/sidekiq_client_middleware_spec.rb b/spec/lib/gitlab/database/load_balancing/sidekiq_client_middleware_spec.rb index f683ade978a..08dd6a0a788 100644 --- a/spec/lib/gitlab/database/load_balancing/sidekiq_client_middleware_spec.rb +++ b/spec/lib/gitlab/database/load_balancing/sidekiq_client_middleware_spec.rb @@ -5,14 +5,12 @@ require 'spec_helper' RSpec.describe Gitlab::Database::LoadBalancing::SidekiqClientMiddleware do let(:middleware) { described_class.new } - let(:load_balancer) { double.as_null_object } let(:worker_class) { 'TestDataConsistencyWorker' } let(:job) { { "job_id" => "a180b47c-3fd6-41b8-81e9-34da61c3400e" } } before do skip_feature_flags_yaml_validation skip_default_enabled_yaml_check - allow(::Gitlab::Database::LoadBalancing).to receive_message_chain(:proxy, :load_balancer).and_return(load_balancer) end after do @@ -23,7 +21,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqClientMiddleware do middleware.call(worker_class, job, nil, nil) {} end - describe '#call' do + describe '#call', :database_replica do shared_context 'data consistency worker class' do |data_consistency, feature_flag| let(:expected_consistency) { data_consistency } let(:worker_class) do @@ -85,9 +83,15 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqClientMiddleware do end it 'passes database_replica_location' do - expected_location = { Gitlab::Database::MAIN_DATABASE_NAME.to_sym => location } + expected_location = {} - expect(load_balancer).to receive_message_chain(:host, "database_replica_location").and_return(location) + Gitlab::Database::LoadBalancing.each_load_balancer do |lb| + expect(lb.host) + .to receive(:database_replica_location) + .and_return(location) + + expected_location[lb.name] = location + end run_middleware @@ -103,9 +107,15 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqClientMiddleware do end it 'passes primary write location', :aggregate_failures do - expected_location = { Gitlab::Database::MAIN_DATABASE_NAME.to_sym => location } + expected_location = {} - expect(load_balancer).to receive(:primary_write_location).and_return(location) + Gitlab::Database::LoadBalancing.each_load_balancer do |lb| + expect(lb) + .to receive(:primary_write_location) + .and_return(location) + + expected_location[lb.name] = location + end run_middleware @@ -137,8 +147,10 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqClientMiddleware do let(:job) { { "job_id" => "a180b47c-3fd6-41b8-81e9-34da61c3400e", 'wal_locations' => wal_locations } } before do - allow(load_balancer).to receive(:primary_write_location).and_return(new_location) - allow(load_balancer).to receive(:database_replica_location).and_return(new_location) + Gitlab::Database::LoadBalancing.each_load_balancer do |lb| + allow(lb).to receive(:primary_write_location).and_return(new_location) + allow(lb).to receive(:database_replica_location).and_return(new_location) + end end shared_examples_for 'does not set database location again' do |use_primary| diff --git a/spec/lib/gitlab/database/load_balancing/sidekiq_server_middleware_spec.rb b/spec/lib/gitlab/database/load_balancing/sidekiq_server_middleware_spec.rb index 9f23eb0094f..06efdcd8f99 100644 --- a/spec/lib/gitlab/database/load_balancing/sidekiq_server_middleware_spec.rb +++ b/spec/lib/gitlab/database/load_balancing/sidekiq_server_middleware_spec.rb @@ -2,20 +2,17 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware do +RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware, :clean_gitlab_redis_queues do let(:middleware) { described_class.new } - - let(:load_balancer) { double.as_null_object } - let(:worker) { worker_class.new } let(:job) { { "retry" => 3, "job_id" => "a180b47c-3fd6-41b8-81e9-34da61c3400e", 'database_replica_location' => '0/D525E3A8' } } before do skip_feature_flags_yaml_validation skip_default_enabled_yaml_check - allow(::Gitlab::Database::LoadBalancing).to receive_message_chain(:proxy, :load_balancer).and_return(load_balancer) replication_lag!(false) + Gitlab::Database::LoadBalancing::Session.clear_session end after do @@ -67,7 +64,10 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware do let(:wal_locations) { { Gitlab::Database::MAIN_DATABASE_NAME.to_sym => location } } it 'does not stick to the primary', :aggregate_failures do - expect(load_balancer).to receive(:select_up_to_date_host).with(location).and_return(true) + expect(ActiveRecord::Base.connection.load_balancer) + .to receive(:select_up_to_date_host) + .with(location) + .and_return(true) run_middleware do expect(Gitlab::Database::LoadBalancing::Session.current.use_primary?).not_to be_truthy @@ -92,7 +92,12 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware do let(:job) { { 'job_id' => 'a180b47c-3fd6-41b8-81e9-34da61c3400e', 'wal_locations' => wal_locations } } before do - allow(load_balancer).to receive(:select_up_to_date_host).with(location).and_return(true) + Gitlab::Database::LoadBalancing.each_load_balancer do |lb| + allow(lb) + .to receive(:select_up_to_date_host) + .with(location) + .and_return(true) + end end it_behaves_like 'replica is up to date', 'replica' @@ -102,7 +107,10 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware do let(:job) { { 'job_id' => 'a180b47c-3fd6-41b8-81e9-34da61c3400e', 'dedup_wal_locations' => wal_locations } } before do - allow(load_balancer).to receive(:select_up_to_date_host).with(wal_locations[:main]).and_return(true) + allow(ActiveRecord::Base.connection.load_balancer) + .to receive(:select_up_to_date_host) + .with(wal_locations[:main]) + .and_return(true) end it_behaves_like 'replica is up to date', 'replica' @@ -112,7 +120,10 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware do let(:job) { { 'job_id' => 'a180b47c-3fd6-41b8-81e9-34da61c3400e', 'database_write_location' => '0/D525E3A8' } } before do - allow(load_balancer).to receive(:select_up_to_date_host).with('0/D525E3A8').and_return(true) + allow(ActiveRecord::Base.connection.load_balancer) + .to receive(:select_up_to_date_host) + .with('0/D525E3A8') + .and_return(true) end it_behaves_like 'replica is up to date', 'replica' @@ -158,18 +169,15 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware do process_job(job) end.to raise_error(Sidekiq::JobRetry::Skip) - expect(job['error_class']).to eq('Gitlab::Database::LoadBalancing::SidekiqServerMiddleware::JobReplicaNotUpToDate') + job_for_retry = Sidekiq::RetrySet.new.first + expect(job_for_retry['error_class']).to eq('Gitlab::Database::LoadBalancing::SidekiqServerMiddleware::JobReplicaNotUpToDate') end include_examples 'load balancing strategy', 'retry' end context 'when job is retried' do - before do - expect do - process_job(job) - end.to raise_error(Sidekiq::JobRetry::Skip) - end + let(:job) { { "retry" => 3, "job_id" => "a180b47c-3fd6-41b8-81e9-34da61c3400e", 'database_replica_location' => '0/D525E3A8', 'retry_count' => 0 } } context 'and replica still lagging behind' do include_examples 'stick to the primary', 'primary' @@ -191,7 +199,9 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware do context 'when replica is not up to date' do before do - allow(load_balancer).to receive(:select_up_to_date_host).and_return(false) + Gitlab::Database::LoadBalancing.each_load_balancer do |lb| + allow(lb).to receive(:select_up_to_date_host).and_return(false) + end end include_examples 'stick to the primary', 'primary' @@ -199,8 +209,47 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware do end end + describe '#databases_in_sync?' do + it 'treats load balancers without WAL entries as in sync' do + expect(middleware.send(:databases_in_sync?, {})) + .to eq(true) + end + + it 'returns true when all load balancers are in sync' do + locations = {} + + Gitlab::Database::LoadBalancing.each_load_balancer do |lb| + locations[lb.name] = 'foo' + + expect(lb) + .to receive(:select_up_to_date_host) + .with('foo') + .and_return(true) + end + + expect(middleware.send(:databases_in_sync?, locations)) + .to eq(true) + end + + it 'returns false when the load balancers are not in sync' do + locations = {} + + Gitlab::Database::LoadBalancing.each_load_balancer do |lb| + locations[lb.name] = 'foo' + + allow(lb) + .to receive(:select_up_to_date_host) + .with('foo') + .and_return(false) + end + + expect(middleware.send(:databases_in_sync?, locations)) + .to eq(false) + end + end + def process_job(job) - Sidekiq::JobRetry.new.local(worker_class, job, 'default') do + Sidekiq::JobRetry.new.local(worker_class, job.to_json, 'default') do worker_class.process_job(job) end end @@ -212,6 +261,8 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware do end def replication_lag!(exists) - allow(load_balancer).to receive(:select_up_to_date_host).and_return(!exists) + Gitlab::Database::LoadBalancing.each_load_balancer do |lb| + allow(lb).to receive(:select_up_to_date_host).and_return(!exists) + end end end diff --git a/spec/lib/gitlab/database/load_balancing/sticking_spec.rb b/spec/lib/gitlab/database/load_balancing/sticking_spec.rb index cf52e59db3a..8ceda52ee85 100644 --- a/spec/lib/gitlab/database/load_balancing/sticking_spec.rb +++ b/spec/lib/gitlab/database/load_balancing/sticking_spec.rb @@ -3,55 +3,82 @@ require 'spec_helper' RSpec.describe Gitlab::Database::LoadBalancing::Sticking, :redis do + let(:sticking) do + described_class.new(ActiveRecord::Base.connection.load_balancer) + end + after do Gitlab::Database::LoadBalancing::Session.clear_session end - describe '.stick_if_necessary' do - context 'when sticking is disabled' do - it 'does not perform any sticking' do - expect(described_class).not_to receive(:stick) + describe '#stick_or_unstick_request' do + it 'sticks or unsticks a single object and updates the Rack environment' do + expect(sticking) + .to receive(:unstick_or_continue_sticking) + .with(:user, 42) - described_class.stick_if_necessary(:user, 42) - end + env = {} + + sticking.stick_or_unstick_request(env, :user, 42) + + expect(env[Gitlab::Database::LoadBalancing::RackMiddleware::STICK_OBJECT].to_a) + .to eq([[ActiveRecord::Base, :user, 42]]) end - context 'when sticking is enabled' do - before do - allow(Gitlab::Database::LoadBalancing).to receive(:enable?) - .and_return(true) - end + it 'sticks or unsticks multiple objects and updates the Rack environment' do + expect(sticking) + .to receive(:unstick_or_continue_sticking) + .with(:user, 42) + .ordered - it 'does not stick if no write was performed' do - allow(Gitlab::Database::LoadBalancing::Session.current) - .to receive(:performed_write?) - .and_return(false) + expect(sticking) + .to receive(:unstick_or_continue_sticking) + .with(:runner, '123456789') + .ordered - expect(described_class).not_to receive(:stick) + env = {} - described_class.stick_if_necessary(:user, 42) - end + sticking.stick_or_unstick_request(env, :user, 42) + sticking.stick_or_unstick_request(env, :runner, '123456789') - it 'sticks to the primary if a write was performed' do - allow(Gitlab::Database::LoadBalancing::Session.current) - .to receive(:performed_write?) - .and_return(true) + expect(env[Gitlab::Database::LoadBalancing::RackMiddleware::STICK_OBJECT].to_a).to eq([ + [ActiveRecord::Base, :user, 42], + [ActiveRecord::Base, :runner, '123456789'] + ]) + end + end - expect(described_class).to receive(:stick).with(:user, 42) + describe '#stick_if_necessary' do + it 'does not stick if no write was performed' do + allow(Gitlab::Database::LoadBalancing::Session.current) + .to receive(:performed_write?) + .and_return(false) - described_class.stick_if_necessary(:user, 42) - end + expect(sticking).not_to receive(:stick) + + sticking.stick_if_necessary(:user, 42) + end + + it 'sticks to the primary if a write was performed' do + allow(Gitlab::Database::LoadBalancing::Session.current) + .to receive(:performed_write?) + .and_return(true) + + expect(sticking) + .to receive(:stick) + .with(:user, 42) + + sticking.stick_if_necessary(:user, 42) end end - describe '.all_caught_up?' do - let(:lb) { double(:lb) } + describe '#all_caught_up?' do + let(:lb) { ActiveRecord::Base.connection.load_balancer } let(:last_write_location) { 'foo' } before do - allow(described_class).to receive(:load_balancer).and_return(lb) - - allow(described_class).to receive(:last_write_location_for) + allow(sticking) + .to receive(:last_write_location_for) .with(:user, 42) .and_return(last_write_location) end @@ -60,13 +87,9 @@ RSpec.describe Gitlab::Database::LoadBalancing::Sticking, :redis do let(:last_write_location) { nil } it 'returns true' do - allow(described_class).to receive(:last_write_location_for) - .with(:user, 42) - .and_return(nil) - expect(lb).not_to receive(:select_up_to_date_host) - expect(described_class.all_caught_up?(:user, 42)).to eq(true) + expect(sticking.all_caught_up?(:user, 42)).to eq(true) end end @@ -76,9 +99,11 @@ RSpec.describe Gitlab::Database::LoadBalancing::Sticking, :redis do end it 'returns true, and unsticks' do - expect(described_class).to receive(:unstick).with(:user, 42) + expect(sticking) + .to receive(:unstick) + .with(:user, 42) - expect(described_class.all_caught_up?(:user, 42)).to eq(true) + expect(sticking.all_caught_up?(:user, 42)).to eq(true) end it 'notifies with the proper event payload' do @@ -87,7 +112,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::Sticking, :redis do .with('caught_up_replica_pick.load_balancing', { result: true }) .and_call_original - described_class.all_caught_up?(:user, 42) + sticking.all_caught_up?(:user, 42) end end @@ -97,7 +122,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::Sticking, :redis do end it 'returns false' do - expect(described_class.all_caught_up?(:user, 42)).to eq(false) + expect(sticking.all_caught_up?(:user, 42)).to eq(false) end it 'notifies with the proper event payload' do @@ -106,42 +131,43 @@ RSpec.describe Gitlab::Database::LoadBalancing::Sticking, :redis do .with('caught_up_replica_pick.load_balancing', { result: false }) .and_call_original - described_class.all_caught_up?(:user, 42) + sticking.all_caught_up?(:user, 42) end end end - describe '.unstick_or_continue_sticking' do - let(:lb) { double(:lb) } - - before do - allow(described_class).to receive(:load_balancer).and_return(lb) - end + describe '#unstick_or_continue_sticking' do + let(:lb) { ActiveRecord::Base.connection.load_balancer } it 'simply returns if no write location could be found' do - allow(described_class).to receive(:last_write_location_for) + allow(sticking) + .to receive(:last_write_location_for) .with(:user, 42) .and_return(nil) expect(lb).not_to receive(:select_up_to_date_host) - described_class.unstick_or_continue_sticking(:user, 42) + sticking.unstick_or_continue_sticking(:user, 42) end it 'unsticks if all secondaries have caught up' do - allow(described_class).to receive(:last_write_location_for) + allow(sticking) + .to receive(:last_write_location_for) .with(:user, 42) .and_return('foo') allow(lb).to receive(:select_up_to_date_host).with('foo').and_return(true) - expect(described_class).to receive(:unstick).with(:user, 42) + expect(sticking) + .to receive(:unstick) + .with(:user, 42) - described_class.unstick_or_continue_sticking(:user, 42) + sticking.unstick_or_continue_sticking(:user, 42) end it 'continues using the primary if the secondaries have not yet caught up' do - allow(described_class).to receive(:last_write_location_for) + allow(sticking) + .to receive(:last_write_location_for) .with(:user, 42) .and_return('foo') @@ -150,184 +176,151 @@ RSpec.describe Gitlab::Database::LoadBalancing::Sticking, :redis do expect(Gitlab::Database::LoadBalancing::Session.current) .to receive(:use_primary!) - described_class.unstick_or_continue_sticking(:user, 42) + sticking.unstick_or_continue_sticking(:user, 42) end end RSpec.shared_examples 'sticking' do - context 'when sticking is disabled' do - it 'does not perform any sticking', :aggregate_failures do - expect(described_class).not_to receive(:set_write_location_for) - expect(Gitlab::Database::LoadBalancing::Session.current).not_to receive(:use_primary!) - - described_class.bulk_stick(:user, ids) - end + before do + allow(ActiveRecord::Base.connection.load_balancer) + .to receive(:primary_write_location) + .and_return('foo') end - context 'when sticking is enabled' do - before do - allow(Gitlab::Database::LoadBalancing).to receive(:configured?).and_return(true) - - lb = double(:lb, primary_write_location: 'foo') + it 'sticks an entity to the primary', :aggregate_failures do + allow(ActiveRecord::Base.connection.load_balancer) + .to receive(:primary_only?) + .and_return(false) - allow(described_class).to receive(:load_balancer).and_return(lb) + ids.each do |id| + expect(sticking) + .to receive(:set_write_location_for) + .with(:user, id, 'foo') end - it 'sticks an entity to the primary', :aggregate_failures do - ids.each do |id| - expect(described_class).to receive(:set_write_location_for) - .with(:user, id, 'foo') - end + expect(Gitlab::Database::LoadBalancing::Session.current) + .to receive(:use_primary!) - expect(Gitlab::Database::LoadBalancing::Session.current) - .to receive(:use_primary!) + subject + end - subject - end + it 'does not update the write location when no replicas are used' do + expect(sticking).not_to receive(:set_write_location_for) + + subject end end - describe '.stick' do + describe '#stick' do it_behaves_like 'sticking' do let(:ids) { [42] } - subject { described_class.stick(:user, ids.first) } + subject { sticking.stick(:user, ids.first) } end end - describe '.bulk_stick' do + describe '#bulk_stick' do it_behaves_like 'sticking' do let(:ids) { [42, 43] } - subject { described_class.bulk_stick(:user, ids) } + subject { sticking.bulk_stick(:user, ids) } end end - describe '.mark_primary_write_location' do - context 'when enabled' do - before do - allow(Gitlab::Database::LoadBalancing).to receive(:enable?).and_return(true) - allow(Gitlab::Database::LoadBalancing).to receive(:configured?).and_return(true) - end - - it 'updates the write location with the load balancer' do - lb = double(:lb, primary_write_location: 'foo') + describe '#mark_primary_write_location' do + it 'updates the write location with the load balancer' do + allow(ActiveRecord::Base.connection.load_balancer) + .to receive(:primary_write_location) + .and_return('foo') - allow(described_class).to receive(:load_balancer).and_return(lb) + allow(ActiveRecord::Base.connection.load_balancer) + .to receive(:primary_only?) + .and_return(false) - expect(described_class).to receive(:set_write_location_for) - .with(:user, 42, 'foo') + expect(sticking) + .to receive(:set_write_location_for) + .with(:user, 42, 'foo') - described_class.mark_primary_write_location(:user, 42) - end + sticking.mark_primary_write_location(:user, 42) end - context 'when load balancing is configured but not enabled' do - before do - allow(Gitlab::Database::LoadBalancing).to receive(:enable?).and_return(false) - allow(Gitlab::Database::LoadBalancing).to receive(:configured?).and_return(true) - end - - it 'updates the write location with the main ActiveRecord connection' do - allow(described_class).to receive(:load_balancer).and_return(nil) - expect(ActiveRecord::Base).to receive(:connection).and_call_original - expect(described_class).to receive(:set_write_location_for) - .with(:user, 42, anything) + it 'does nothing when no replicas are used' do + expect(sticking).not_to receive(:set_write_location_for) - described_class.mark_primary_write_location(:user, 42) - end - - context 'when write location is nil' do - before do - allow(Gitlab::Database.main).to receive(:get_write_location).and_return(nil) - end + sticking.mark_primary_write_location(:user, 42) + end + end - it 'does not update the write location' do - expect(described_class).not_to receive(:set_write_location_for) + describe '#unstick' do + it 'removes the sticking data from Redis' do + sticking.set_write_location_for(:user, 4, 'foo') + sticking.unstick(:user, 4) - described_class.mark_primary_write_location(:user, 42) - end - end + expect(sticking.last_write_location_for(:user, 4)).to be_nil end - context 'when load balancing is disabled' do - before do - allow(Gitlab::Database::LoadBalancing).to receive(:enable?).and_return(false) - allow(Gitlab::Database::LoadBalancing).to receive(:configured?).and_return(false) + it 'removes the old key' do + Gitlab::Redis::SharedState.with do |redis| + redis.set(sticking.send(:old_redis_key_for, :user, 4), 'foo', ex: 30) end - it 'updates the write location with the main ActiveRecord connection' do - expect(described_class).not_to receive(:set_write_location_for) - - described_class.mark_primary_write_location(:user, 42) - end + sticking.unstick(:user, 4) + expect(sticking.last_write_location_for(:user, 4)).to be_nil end end - describe '.unstick' do - it 'removes the sticking data from Redis' do - described_class.set_write_location_for(:user, 4, 'foo') - described_class.unstick(:user, 4) + describe '#last_write_location_for' do + it 'returns the last WAL write location for a user' do + sticking.set_write_location_for(:user, 4, 'foo') - expect(described_class.last_write_location_for(:user, 4)).to be_nil + expect(sticking.last_write_location_for(:user, 4)).to eq('foo') end - end - describe '.last_write_location_for' do - it 'returns the last WAL write location for a user' do - described_class.set_write_location_for(:user, 4, 'foo') + it 'falls back to reading the old key' do + Gitlab::Redis::SharedState.with do |redis| + redis.set(sticking.send(:old_redis_key_for, :user, 4), 'foo', ex: 30) + end - expect(described_class.last_write_location_for(:user, 4)).to eq('foo') + expect(sticking.last_write_location_for(:user, 4)).to eq('foo') end end - describe '.redis_key_for' do + describe '#redis_key_for' do it 'returns a String' do - expect(described_class.redis_key_for(:user, 42)) - .to eq('database-load-balancing/write-location/user/42') + expect(sticking.redis_key_for(:user, 42)) + .to eq('database-load-balancing/write-location/main/user/42') end end - describe '.load_balancer' do - it 'returns a the load balancer' do - proxy = double(:proxy) - - expect(Gitlab::Database::LoadBalancing).to receive(:proxy) - .and_return(proxy) - - expect(proxy).to receive(:load_balancer) - - described_class.load_balancer - end - end - - describe '.select_caught_up_replicas' do - let(:lb) { double(:lb) } - - before do - allow(described_class).to receive(:load_balancer).and_return(lb) - end + describe '#select_caught_up_replicas' do + let(:lb) { ActiveRecord::Base.connection.load_balancer } context 'with no write location' do before do - allow(described_class).to receive(:last_write_location_for) - .with(:project, 42).and_return(nil) + allow(sticking) + .to receive(:last_write_location_for) + .with(:project, 42) + .and_return(nil) end it 'returns false and does not try to find caught up hosts' do expect(lb).not_to receive(:select_up_to_date_host) - expect(described_class.select_caught_up_replicas(:project, 42)).to be false + expect(sticking.select_caught_up_replicas(:project, 42)).to be false end end context 'with write location' do before do - allow(described_class).to receive(:last_write_location_for) - .with(:project, 42).and_return('foo') + allow(sticking) + .to receive(:last_write_location_for) + .with(:project, 42) + .and_return('foo') end it 'returns true, selects hosts, and unsticks if any secondary has caught up' do expect(lb).to receive(:select_up_to_date_host).and_return(true) - expect(described_class).to receive(:unstick).with(:project, 42) - expect(described_class.select_caught_up_replicas(:project, 42)).to be true + expect(sticking) + .to receive(:unstick) + .with(:project, 42) + expect(sticking.select_caught_up_replicas(:project, 42)).to be true end end end diff --git a/spec/lib/gitlab/database/load_balancing_spec.rb b/spec/lib/gitlab/database/load_balancing_spec.rb index f40ad444081..bf5314e2c34 100644 --- a/spec/lib/gitlab/database/load_balancing_spec.rb +++ b/spec/lib/gitlab/database/load_balancing_spec.rb @@ -3,203 +3,52 @@ require 'spec_helper' RSpec.describe Gitlab::Database::LoadBalancing do - describe '.proxy' do - before do - @previous_proxy = ActiveRecord::Base.load_balancing_proxy + describe '.base_models' do + it 'returns the models to apply load balancing to' do + models = described_class.base_models - ActiveRecord::Base.load_balancing_proxy = connection_proxy - end - - after do - ActiveRecord::Base.load_balancing_proxy = @previous_proxy - end - - context 'when configured' do - let(:connection_proxy) { double(:connection_proxy) } - - it 'returns the connection proxy' do - expect(subject.proxy).to eq(connection_proxy) - end - end - - context 'when not configured' do - let(:connection_proxy) { nil } + expect(models).to include(ActiveRecord::Base) - it 'returns nil' do - expect(subject.proxy).to be_nil + if Gitlab::Database.has_config?(:ci) + expect(models).to include(Ci::CiDatabaseRecord) end - - it 'tracks an error to sentry' do - expect(Gitlab::ErrorTracking).to receive(:track_exception).with( - an_instance_of(subject::ProxyNotConfiguredError) - ) - - subject.proxy - end - end - end - - describe '.configuration' do - it 'returns the configuration for the load balancer' do - raw = ActiveRecord::Base.connection_db_config.configuration_hash - cfg = described_class.configuration - - # There isn't much to test here as the load balancing settings might not - # (and likely aren't) set when running tests. - expect(cfg.pool_size).to eq(raw[:pool]) - end - end - - describe '.enable?' do - before do - allow(described_class.configuration) - .to receive(:hosts) - .and_return(%w(foo)) - end - - it 'returns false when no hosts are specified' do - allow(described_class.configuration).to receive(:hosts).and_return([]) - - expect(described_class.enable?).to eq(false) - end - - it 'returns true when Sidekiq is being used' do - allow(Gitlab::Runtime).to receive(:sidekiq?).and_return(true) - - expect(described_class.enable?).to eq(true) - end - - it 'returns false when running inside a Rake task' do - allow(Gitlab::Runtime).to receive(:rake?).and_return(true) - - expect(described_class.enable?).to eq(false) - end - - it 'returns true when load balancing should be enabled' do - allow(Gitlab::Runtime).to receive(:sidekiq?).and_return(false) - - expect(described_class.enable?).to eq(true) end - it 'returns true when service discovery is enabled' do - allow(described_class.configuration).to receive(:hosts).and_return([]) - allow(Gitlab::Runtime).to receive(:sidekiq?).and_return(false) - - allow(described_class.configuration) - .to receive(:service_discovery_enabled?) - .and_return(true) - - expect(described_class.enable?).to eq(true) + it 'returns the models as a frozen array' do + expect(described_class.base_models).to be_frozen end end - describe '.configured?' do - it 'returns true when hosts are configured' do - allow(described_class.configuration) - .to receive(:hosts) - .and_return(%w[foo]) - - expect(described_class.configured?).to eq(true) - end + describe '.each_load_balancer' do + it 'yields every load balancer to the supplied block' do + lbs = [] - it 'returns true when service discovery is enabled' do - allow(described_class.configuration).to receive(:hosts).and_return([]) - allow(described_class.configuration) - .to receive(:service_discovery_enabled?) - .and_return(true) + described_class.each_load_balancer do |lb| + lbs << lb + end - expect(described_class.configured?).to eq(true) + expect(lbs.length).to eq(described_class.base_models.length) end - it 'returns false when neither service discovery nor hosts are configured' do - allow(described_class.configuration).to receive(:hosts).and_return([]) - allow(described_class.configuration) - .to receive(:service_discovery_enabled?) - .and_return(false) + it 'returns an Enumerator when no block is given' do + res = described_class.each_load_balancer - expect(described_class.configured?).to eq(false) + expect(res.next) + .to be_an_instance_of(Gitlab::Database::LoadBalancing::LoadBalancer) end end - describe '.configure_proxy' do - before do - allow(ActiveRecord::Base).to receive(:load_balancing_proxy=) - end - - it 'configures the connection proxy' do - described_class.configure_proxy - - expect(ActiveRecord::Base).to have_received(:load_balancing_proxy=) - .with(Gitlab::Database::LoadBalancing::ConnectionProxy) - end - - context 'when service discovery is enabled' do - it 'runs initial service discovery when configuring the connection proxy' do - discover = instance_spy(Gitlab::Database::LoadBalancing::ServiceDiscovery) - - allow(described_class.configuration) - .to receive(:service_discovery) - .and_return({ record: 'foo' }) - - expect(Gitlab::Database::LoadBalancing::ServiceDiscovery) - .to receive(:new) - .with( - an_instance_of(Gitlab::Database::LoadBalancing::LoadBalancer), - an_instance_of(Hash) - ) - .and_return(discover) - - expect(discover).to receive(:perform_service_discovery) - - described_class.configure_proxy + describe '.release_hosts' do + it 'releases the host of every load balancer' do + described_class.each_load_balancer do |lb| + expect(lb).to receive(:release_host) end - end - end - - describe '.start_service_discovery' do - it 'does not start if service discovery is disabled' do - expect(Gitlab::Database::LoadBalancing::ServiceDiscovery) - .not_to receive(:new) - described_class.start_service_discovery - end - - it 'starts service discovery if enabled' do - allow(described_class.configuration) - .to receive(:service_discovery_enabled?) - .and_return(true) - - instance = double(:instance) - config = Gitlab::Database::LoadBalancing::Configuration - .new(ActiveRecord::Base) - lb = Gitlab::Database::LoadBalancing::LoadBalancer.new(config) - proxy = Gitlab::Database::LoadBalancing::ConnectionProxy.new(lb) - - allow(described_class) - .to receive(:proxy) - .and_return(proxy) - - expect(Gitlab::Database::LoadBalancing::ServiceDiscovery) - .to receive(:new) - .with(lb, an_instance_of(Hash)) - .and_return(instance) - - expect(instance) - .to receive(:start) - - described_class.start_service_discovery + described_class.release_hosts end end describe '.db_role_for_connection' do - context 'when the load balancing is not configured' do - let(:connection) { ActiveRecord::Base.connection } - - it 'returns primary' do - expect(described_class.db_role_for_connection(connection)).to eq(:primary) - end - end - context 'when the NullPool is used for connection' do let(:pool) { ActiveRecord::ConnectionAdapters::NullPool.new } let(:connection) { double(:connection, pool: pool) } @@ -253,7 +102,7 @@ RSpec.describe Gitlab::Database::LoadBalancing do # - In each test, we listen to the SQL queries (via sql.active_record # instrumentation) while triggering real queries from the defined model. # - We assert the desinations (replica/primary) of the queries in order. - describe 'LoadBalancing integration tests', :db_load_balancing, :delete do + describe 'LoadBalancing integration tests', :database_replica, :delete do before(:all) do ActiveRecord::Schema.define do create_table :load_balancing_test, force: true do |t| @@ -274,10 +123,6 @@ RSpec.describe Gitlab::Database::LoadBalancing do end end - before do - model.singleton_class.prepend ::Gitlab::Database::LoadBalancing::ActiveRecordProxy - end - where(:queries, :include_transaction, :expected_results) do [ # Read methods diff --git a/spec/lib/gitlab/database/migration_helpers/loose_foreign_key_helpers_spec.rb b/spec/lib/gitlab/database/migration_helpers/loose_foreign_key_helpers_spec.rb index 708d1be6e00..54b3ad22faf 100644 --- a/spec/lib/gitlab/database/migration_helpers/loose_foreign_key_helpers_spec.rb +++ b/spec/lib/gitlab/database/migration_helpers/loose_foreign_key_helpers_spec.rb @@ -19,6 +19,10 @@ RSpec.describe Gitlab::Database::MigrationHelpers::LooseForeignKeyHelpers do end end + after(:all) do + migration.drop_table :loose_fk_test_table + end + before do 3.times { model.create! } end @@ -45,8 +49,9 @@ RSpec.describe Gitlab::Database::MigrationHelpers::LooseForeignKeyHelpers do expect(LooseForeignKeys::DeletedRecord.count).to eq(1) deleted_record = LooseForeignKeys::DeletedRecord.all.first - expect(deleted_record.deleted_table_primary_key_value).to eq(record_to_be_deleted.id) - expect(deleted_record.deleted_table_name).to eq('loose_fk_test_table') + expect(deleted_record.primary_key_value).to eq(record_to_be_deleted.id) + expect(deleted_record.fully_qualified_table_name).to eq('public.loose_fk_test_table') + expect(deleted_record.partition).to eq(1) end it 'stores multiple record deletions' do diff --git a/spec/lib/gitlab/database/migration_helpers_spec.rb b/spec/lib/gitlab/database/migration_helpers_spec.rb index 006f8a39f9c..d89af1521a2 100644 --- a/spec/lib/gitlab/database/migration_helpers_spec.rb +++ b/spec/lib/gitlab/database/migration_helpers_spec.rb @@ -1631,10 +1631,19 @@ RSpec.describe Gitlab::Database::MigrationHelpers do let(:worker) do Class.new do include Sidekiq::Worker + sidekiq_options queue: 'test' + + def self.name + 'WorkerClass' + end end end + before do + stub_const(worker.name, worker) + end + describe '#sidekiq_queue_length' do context 'when queue is empty' do it 'returns zero' do diff --git a/spec/lib/gitlab/database/migrations/instrumentation_spec.rb b/spec/lib/gitlab/database/migrations/instrumentation_spec.rb index 5945e5a2039..841d2a98a16 100644 --- a/spec/lib/gitlab/database/migrations/instrumentation_spec.rb +++ b/spec/lib/gitlab/database/migrations/instrumentation_spec.rb @@ -2,8 +2,13 @@ require 'spec_helper' RSpec.describe Gitlab::Database::Migrations::Instrumentation do + let(:result_dir) { Dir.mktmpdir } + + after do + FileUtils.rm_rf(result_dir) + end describe '#observe' do - subject { described_class.new } + subject { described_class.new(result_dir: result_dir) } let(:migration_name) { 'test' } let(:migration_version) { '12345' } @@ -13,7 +18,7 @@ RSpec.describe Gitlab::Database::Migrations::Instrumentation do end context 'behavior with observers' do - subject { described_class.new([Gitlab::Database::Migrations::Observers::MigrationObserver]).observe(version: migration_version, name: migration_name) {} } + subject { described_class.new(observer_classes: [Gitlab::Database::Migrations::Observers::MigrationObserver], result_dir: result_dir).observe(version: migration_version, name: migration_name) {} } let(:observer) { instance_double('Gitlab::Database::Migrations::Observers::MigrationObserver', before: nil, after: nil, record: nil) } @@ -24,7 +29,7 @@ RSpec.describe Gitlab::Database::Migrations::Instrumentation do it 'instantiates observer with observation' do expect(Gitlab::Database::Migrations::Observers::MigrationObserver) .to receive(:new) - .with(instance_of(Gitlab::Database::Migrations::Observation)) { |observation| expect(observation.version).to eq(migration_version) } + .with(instance_of(Gitlab::Database::Migrations::Observation), anything) { |observation| expect(observation.version).to eq(migration_version) } .and_return(observer) subject @@ -58,7 +63,7 @@ RSpec.describe Gitlab::Database::Migrations::Instrumentation do end context 'on successful execution' do - subject { described_class.new.observe(version: migration_version, name: migration_name) {} } + subject { described_class.new(result_dir: result_dir).observe(version: migration_version, name: migration_name) {} } it 'records walltime' do expect(subject.walltime).not_to be_nil @@ -78,7 +83,7 @@ RSpec.describe Gitlab::Database::Migrations::Instrumentation do end context 'upon failure' do - subject { described_class.new.observe(version: migration_version, name: migration_name) { raise 'something went wrong' } } + subject { described_class.new(result_dir: result_dir).observe(version: migration_version, name: migration_name) { raise 'something went wrong' } } it 'raises the exception' do expect { subject }.to raise_error(/something went wrong/) @@ -93,7 +98,7 @@ RSpec.describe Gitlab::Database::Migrations::Instrumentation do # ignore end - let(:instance) { described_class.new } + let(:instance) { described_class.new(result_dir: result_dir) } it 'records walltime' do expect(subject.walltime).not_to be_nil @@ -114,7 +119,7 @@ RSpec.describe Gitlab::Database::Migrations::Instrumentation do end context 'sequence of migrations with failures' do - subject { described_class.new } + subject { described_class.new(result_dir: result_dir) } let(:migration1) { double('migration1', call: nil) } let(:migration2) { double('migration2', call: nil) } diff --git a/spec/lib/gitlab/database/migrations/observers/query_details_spec.rb b/spec/lib/gitlab/database/migrations/observers/query_details_spec.rb index 36885a1594f..191ac29e3b3 100644 --- a/spec/lib/gitlab/database/migrations/observers/query_details_spec.rb +++ b/spec/lib/gitlab/database/migrations/observers/query_details_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' RSpec.describe Gitlab::Database::Migrations::Observers::QueryDetails do - subject { described_class.new(observation) } + subject { described_class.new(observation, directory_path) } let(:observation) { Gitlab::Database::Migrations::Observation.new(migration_version, migration_name) } let(:connection) { ActiveRecord::Base.connection } @@ -14,10 +14,6 @@ RSpec.describe Gitlab::Database::Migrations::Observers::QueryDetails do let(:migration_version) { 20210422152437 } let(:migration_name) { 'test' } - before do - stub_const('Gitlab::Database::Migrations::Instrumentation::RESULT_DIR', directory_path) - end - after do FileUtils.remove_entry(directory_path) end diff --git a/spec/lib/gitlab/database/migrations/observers/query_log_spec.rb b/spec/lib/gitlab/database/migrations/observers/query_log_spec.rb index 2a49d8e8b73..2e70a85fd5b 100644 --- a/spec/lib/gitlab/database/migrations/observers/query_log_spec.rb +++ b/spec/lib/gitlab/database/migrations/observers/query_log_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' RSpec.describe Gitlab::Database::Migrations::Observers::QueryLog do - subject { described_class.new(observation) } + subject { described_class.new(observation, directory_path) } let(:observation) { Gitlab::Database::Migrations::Observation.new(migration_version, migration_name) } let(:connection) { ActiveRecord::Base.connection } @@ -11,10 +11,6 @@ RSpec.describe Gitlab::Database::Migrations::Observers::QueryLog do let(:migration_version) { 20210422152437 } let(:migration_name) { 'test' } - before do - stub_const('Gitlab::Database::Migrations::Instrumentation::RESULT_DIR', directory_path) - end - after do FileUtils.remove_entry(directory_path) end diff --git a/spec/lib/gitlab/database/migrations/observers/query_statistics_spec.rb b/spec/lib/gitlab/database/migrations/observers/query_statistics_spec.rb index 32a25fdaa28..9727a215d71 100644 --- a/spec/lib/gitlab/database/migrations/observers/query_statistics_spec.rb +++ b/spec/lib/gitlab/database/migrations/observers/query_statistics_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' RSpec.describe Gitlab::Database::Migrations::Observers::QueryStatistics do - subject { described_class.new(observation) } + subject { described_class.new(observation, double("unused path")) } let(:observation) { Gitlab::Database::Migrations::Observation.new } let(:connection) { ActiveRecord::Base.connection } diff --git a/spec/lib/gitlab/database/migrations/observers/total_database_size_change_spec.rb b/spec/lib/gitlab/database/migrations/observers/total_database_size_change_spec.rb index 61e28003e66..e689759c574 100644 --- a/spec/lib/gitlab/database/migrations/observers/total_database_size_change_spec.rb +++ b/spec/lib/gitlab/database/migrations/observers/total_database_size_change_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' RSpec.describe Gitlab::Database::Migrations::Observers::TotalDatabaseSizeChange do - subject { described_class.new(observation) } + subject { described_class.new(observation, double('unused path')) } let(:observation) { Gitlab::Database::Migrations::Observation.new } let(:connection) { ActiveRecord::Base.connection } diff --git a/spec/lib/gitlab/database/migrations/runner_spec.rb b/spec/lib/gitlab/database/migrations/runner_spec.rb new file mode 100644 index 00000000000..52fb5ec2ba8 --- /dev/null +++ b/spec/lib/gitlab/database/migrations/runner_spec.rb @@ -0,0 +1,109 @@ +# frozen_string_literal: true +require 'spec_helper' + +RSpec.describe Gitlab::Database::Migrations::Runner do + let(:result_dir) { Pathname.new(Dir.mktmpdir) } + + let(:migration_runs) { [] } # This list gets populated as the runner tries to run migrations + + # Tests depend on all of these lists being sorted in the order migrations would be applied + let(:applied_migrations_other_branches) { [double(ActiveRecord::Migration, version: 1, name: 'migration_complete_other_branch')] } + + let(:applied_migrations_this_branch) do + [ + double(ActiveRecord::Migration, version: 2, name: 'older_migration_complete_this_branch'), + double(ActiveRecord::Migration, version: 3, name: 'newer_migration_complete_this_branch') + ].sort_by(&:version) + end + + let(:pending_migrations) do + [ + double(ActiveRecord::Migration, version: 4, name: 'older_migration_pending'), + double(ActiveRecord::Migration, version: 5, name: 'newer_migration_pending') + ].sort_by(&:version) + end + + before do + stub_const('Gitlab::Database::Migrations::Runner::BASE_RESULT_DIR', result_dir) + allow(ActiveRecord::Migrator).to receive(:new) do |dir, _all_migrations, _schema_migration_class, version_to_migrate| + migrator = double(ActiveRecord::Migrator) + expect(migrator).to receive(:run) do + migration_runs << OpenStruct.new(dir: dir, version_to_migrate: version_to_migrate) + end + migrator + end + + all_versions = (applied_migrations_other_branches + applied_migrations_this_branch).map(&:version) + migrations = applied_migrations_other_branches + applied_migrations_this_branch + pending_migrations + ctx = double(ActiveRecord::MigrationContext, get_all_versions: all_versions, migrations: migrations, schema_migration: ActiveRecord::SchemaMigration) + + allow(described_class).to receive(:migration_context).and_return(ctx) + + names_this_branch = (applied_migrations_this_branch + pending_migrations).map { |m| "db/migrate/#{m.version}_#{m.name}.rb"} + allow(described_class).to receive(:migration_file_names_this_branch).and_return(names_this_branch) + end + + after do + FileUtils.rm_rf(result_dir) + end + + it 'creates the results dir when one does not exist' do + FileUtils.rm_rf(result_dir) + + expect do + described_class.new(direction: :up, migrations: [], result_dir: result_dir).run + end.to change { Dir.exist?(result_dir) }.from(false).to(true) + end + + describe '.up' do + context 'result directory' do + it 'uses the /up subdirectory' do + expect(described_class.up.result_dir).to eq(result_dir.join('up')) + end + end + + context 'migrations to run' do + subject(:up) { described_class.up } + + it 'is the list of pending migrations' do + expect(up.migrations).to eq(pending_migrations) + end + end + + context 'running migrations' do + subject(:up) { described_class.up } + + it 'runs the unapplied migrations in version order', :aggregate_failures do + up.run + + expect(migration_runs.map(&:dir)).to eq([:up, :up]) + expect(migration_runs.map(&:version_to_migrate)).to eq(pending_migrations.map(&:version)) + end + end + end + + describe '.down' do + subject(:down) { described_class.down } + + context 'result directory' do + it 'is the /down subdirectory' do + expect(down.result_dir).to eq(result_dir.join('down')) + end + end + + context 'migrations to run' do + it 'is the list of migrations that are up and on this branch' do + expect(down.migrations).to eq(applied_migrations_this_branch) + end + end + + context 'running migrations' do + it 'runs the applied migrations for the current branch in reverse order', :aggregate_failures do + down.run + + expect(migration_runs.map(&:dir)).to eq([:down, :down]) + expect(migration_runs.map(&:version_to_migrate)).to eq(applied_migrations_this_branch.reverse.map(&:version)) + end + end + end +end diff --git a/spec/lib/gitlab/database/partitioning/detached_partition_dropper_spec.rb b/spec/lib/gitlab/database/partitioning/detached_partition_dropper_spec.rb index 8523b7104f0..8c406c90e36 100644 --- a/spec/lib/gitlab/database/partitioning/detached_partition_dropper_spec.rb +++ b/spec/lib/gitlab/database/partitioning/detached_partition_dropper_spec.rb @@ -84,6 +84,7 @@ RSpec.describe Gitlab::Database::Partitioning::DetachedPartitionDropper do before do stub_feature_flags(drop_detached_partitions: false) end + it 'does not drop the partition' do subject.perform @@ -162,8 +163,8 @@ RSpec.describe Gitlab::Database::Partitioning::DetachedPartitionDropper do context 'when the first drop returns an error' do it 'still drops the second partition' do - expect(subject).to receive(:drop_one).ordered.and_raise('injected error') - expect(subject).to receive(:drop_one).ordered.and_call_original + expect(subject).to receive(:drop_detached_partition).ordered.and_raise('injected error') + expect(subject).to receive(:drop_detached_partition).ordered.and_call_original subject.perform diff --git a/spec/lib/gitlab/database/partitioning/multi_database_partition_dropper_spec.rb b/spec/lib/gitlab/database/partitioning/multi_database_partition_dropper_spec.rb new file mode 100644 index 00000000000..56d6ebb7aff --- /dev/null +++ b/spec/lib/gitlab/database/partitioning/multi_database_partition_dropper_spec.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::Partitioning::MultiDatabasePartitionDropper, '#drop_detached_partitions' do + subject(:drop_detached_partitions) { multi_db_dropper.drop_detached_partitions } + + let(:multi_db_dropper) { described_class.new } + + let(:connection_wrapper1) { double(scope: scope1) } + let(:connection_wrapper2) { double(scope: scope2) } + + let(:scope1) { double(connection: connection1) } + let(:scope2) { double(connection: connection2) } + + let(:connection1) { double('connection') } + let(:connection2) { double('connection') } + + let(:dropper_class) { Gitlab::Database::Partitioning::DetachedPartitionDropper } + let(:dropper1) { double('partition dropper') } + let(:dropper2) { double('partition dropper') } + + before do + allow(multi_db_dropper).to receive(:databases).and_return({ db1: connection_wrapper1, db2: connection_wrapper2 }) + end + + it 'drops detached partitions for each database' do + expect(Gitlab::Database::SharedModel).to receive(:using_connection).with(connection1).and_yield.ordered + expect(dropper_class).to receive(:new).and_return(dropper1).ordered + expect(dropper1).to receive(:perform) + + expect(Gitlab::Database::SharedModel).to receive(:using_connection).with(connection2).and_yield.ordered + expect(dropper_class).to receive(:new).and_return(dropper2).ordered + expect(dropper2).to receive(:perform) + + drop_detached_partitions + end +end diff --git a/spec/lib/gitlab/database/partitioning/partition_manager_spec.rb b/spec/lib/gitlab/database/partitioning/partition_manager_spec.rb index 8f1f5b5ba1b..7c4cfcfb3a9 100644 --- a/spec/lib/gitlab/database/partitioning/partition_manager_spec.rb +++ b/spec/lib/gitlab/database/partitioning/partition_manager_spec.rb @@ -176,7 +176,7 @@ RSpec.describe Gitlab::Database::Partitioning::PartitionManager do end it 'detaches exactly one partition' do - expect { subject }.to change { find_partitions(my_model.table_name, schema: Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA).size }.from(9).to(8) + expect { subject }.to change { find_partitions(my_model.table_name).size }.from(9).to(8) end it 'detaches the old partition' do diff --git a/spec/lib/gitlab/database/partitioning_spec.rb b/spec/lib/gitlab/database/partitioning_spec.rb index f163b45e01e..486af9413e8 100644 --- a/spec/lib/gitlab/database/partitioning_spec.rb +++ b/spec/lib/gitlab/database/partitioning_spec.rb @@ -33,4 +33,22 @@ RSpec.describe Gitlab::Database::Partitioning do end end end + + describe '.drop_detached_partitions' do + let(:partition_dropper_class) { described_class::MultiDatabasePartitionDropper } + + it 'delegates to the partition dropper' do + expect_next_instance_of(partition_dropper_class) do |partition_dropper| + expect(partition_dropper).to receive(:drop_detached_partitions) + end + + described_class.drop_detached_partitions + end + end + + context 'ensure that the registered models have partitioning strategy' do + it 'fails when partitioning_strategy is not specified for the model' do + expect(described_class.registered_models).to all(respond_to(:partitioning_strategy)) + end + end end diff --git a/spec/lib/gitlab/database/postgresql_adapter/force_disconnectable_mixin_spec.rb b/spec/lib/gitlab/database/postgresql_adapter/force_disconnectable_mixin_spec.rb index 2a1f91b5b21..399fcae2fa0 100644 --- a/spec/lib/gitlab/database/postgresql_adapter/force_disconnectable_mixin_spec.rb +++ b/spec/lib/gitlab/database/postgresql_adapter/force_disconnectable_mixin_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::PostgresqlAdapter::ForceDisconnectableMixin do +RSpec.describe Gitlab::Database::PostgresqlAdapter::ForceDisconnectableMixin, :reestablished_active_record_base do describe 'checking in a connection to the pool' do let(:model) do Class.new(ActiveRecord::Base) do diff --git a/spec/lib/gitlab/database/schema_migrations/context_spec.rb b/spec/lib/gitlab/database/schema_migrations/context_spec.rb index a79e6706149..0323fa22b78 100644 --- a/spec/lib/gitlab/database/schema_migrations/context_spec.rb +++ b/spec/lib/gitlab/database/schema_migrations/context_spec.rb @@ -23,19 +23,7 @@ RSpec.describe Gitlab::Database::SchemaMigrations::Context do end end - context 'multiple databases' do - let(:connection_class) do - Class.new(::ApplicationRecord) do - self.abstract_class = true - - def self.name - 'Gitlab::Database::SchemaMigrations::Context::TestConnection' - end - end - end - - let(:configuration_overrides) { {} } - + context 'multiple databases', :reestablished_active_record_base do before do connection_class.establish_connection( ActiveRecord::Base @@ -46,10 +34,6 @@ RSpec.describe Gitlab::Database::SchemaMigrations::Context do ) end - after do - connection_class.remove_connection - end - context 'when `schema_migrations_path` is configured as string' do let(:configuration_overrides) do { "schema_migrations_path" => "db/ci_schema_migrations" } diff --git a/spec/lib/gitlab/database/with_lock_retries_spec.rb b/spec/lib/gitlab/database/with_lock_retries_spec.rb index 0b960830d89..c2c818aa106 100644 --- a/spec/lib/gitlab/database/with_lock_retries_spec.rb +++ b/spec/lib/gitlab/database/with_lock_retries_spec.rb @@ -7,7 +7,7 @@ RSpec.describe Gitlab::Database::WithLockRetries do let(:logger) { Gitlab::Database::WithLockRetries::NULL_LOGGER } let(:subject) { described_class.new(env: env, logger: logger, allow_savepoints: allow_savepoints, timing_configuration: timing_configuration) } let(:allow_savepoints) { true } - let(:connection) { ActiveRecord::Base.connection } + let(:connection) { ActiveRecord::Base.retrieve_connection } let(:timing_configuration) do [ diff --git a/spec/lib/gitlab/database_spec.rb b/spec/lib/gitlab/database_spec.rb index a9a8d5e6314..a2e7b6d27b9 100644 --- a/spec/lib/gitlab/database_spec.rb +++ b/spec/lib/gitlab/database_spec.rb @@ -15,6 +15,13 @@ RSpec.describe Gitlab::Database do end end + describe '.databases' do + it 'stores connections as a HashWithIndifferentAccess' do + expect(described_class.databases.has_key?('main')).to be true + expect(described_class.databases.has_key?(:main)).to be true + end + end + describe '.default_pool_size' do before do allow(Gitlab::Runtime).to receive(:max_threads).and_return(7) @@ -185,10 +192,23 @@ RSpec.describe Gitlab::Database do describe '.db_config_name' do it 'returns the db_config name for the connection' do - connection = ActiveRecord::Base.connection + model = ActiveRecord::Base + + # This is a ConnectionProxy + expect(described_class.db_config_name(model.connection)) + .to eq('unknown') - expect(described_class.db_config_name(connection)).to be_a(String) - expect(described_class.db_config_name(connection)).to eq(connection.pool.db_config.name) + # This is an actual connection + expect(described_class.db_config_name(model.retrieve_connection)) + .to eq('main') + end + + context 'when replicas are configured', :database_replica do + it 'returns the name for a replica' do + replica = ActiveRecord::Base.connection.load_balancer.host + + expect(described_class.db_config_name(replica)).to eq('main_replica') + end end end @@ -279,7 +299,7 @@ RSpec.describe Gitlab::Database do expect(event).not_to be_nil expect(event.duration).to be > 0.0 expect(event.payload).to a_hash_including( - connection: be_a(ActiveRecord::ConnectionAdapters::AbstractAdapter) + connection: be_a(Gitlab::Database::LoadBalancing::ConnectionProxy) ) end end @@ -296,7 +316,7 @@ RSpec.describe Gitlab::Database do expect(event).not_to be_nil expect(event.duration).to be > 0.0 expect(event.payload).to a_hash_including( - connection: be_a(ActiveRecord::ConnectionAdapters::AbstractAdapter) + connection: be_a(Gitlab::Database::LoadBalancing::ConnectionProxy) ) end end @@ -319,7 +339,7 @@ RSpec.describe Gitlab::Database do expect(event).not_to be_nil expect(event.duration).to be > 0.0 expect(event.payload).to a_hash_including( - connection: be_a(ActiveRecord::ConnectionAdapters::AbstractAdapter) + connection: be_a(Gitlab::Database::LoadBalancing::ConnectionProxy) ) end end @@ -340,7 +360,7 @@ RSpec.describe Gitlab::Database do expect(event).not_to be_nil expect(event.duration).to be > 0.0 expect(event.payload).to a_hash_including( - connection: be_a(ActiveRecord::ConnectionAdapters::AbstractAdapter) + connection: be_a(Gitlab::Database::LoadBalancing::ConnectionProxy) ) end end diff --git a/spec/lib/gitlab/doctor/secrets_spec.rb b/spec/lib/gitlab/doctor/secrets_spec.rb index b9e054ce14f..f95a7eb1492 100644 --- a/spec/lib/gitlab/doctor/secrets_spec.rb +++ b/spec/lib/gitlab/doctor/secrets_spec.rb @@ -5,6 +5,8 @@ require 'spec_helper' RSpec.describe Gitlab::Doctor::Secrets do let!(:user) { create(:user, otp_secret: "test") } let!(:group) { create(:group, runners_token: "test") } + let!(:project) { create(:project) } + let!(:grafana_integration) { create(:grafana_integration, project: project, token: "test") } let(:logger) { double(:logger).as_null_object } subject { described_class.new(logger).run! } @@ -39,4 +41,12 @@ RSpec.describe Gitlab::Doctor::Secrets do subject end end + + context 'when GrafanaIntegration token is set via private method' do + it 'can access GrafanaIntegration token value' do + expect(logger).to receive(:info).with(/GrafanaIntegration failures: 0/) + + subject + end + end end diff --git a/spec/lib/gitlab/email/handler/create_issue_handler_spec.rb b/spec/lib/gitlab/email/handler/create_issue_handler_spec.rb index dd230140b30..bd4f1d164a8 100644 --- a/spec/lib/gitlab/email/handler/create_issue_handler_spec.rb +++ b/spec/lib/gitlab/email/handler/create_issue_handler_spec.rb @@ -136,6 +136,36 @@ RSpec.describe Gitlab::Email::Handler::CreateIssueHandler do expect { handler.execute }.to raise_error(Gitlab::Email::ProjectNotFound) end end + + context 'rate limiting' do + let(:rate_limited_service_feature_enabled) { nil } + + before do + stub_feature_flags(rate_limited_service_issues_create: rate_limited_service_feature_enabled) + end + + context 'when :rate_limited_service Feature is disabled' do + let(:rate_limited_service_feature_enabled) { false } + + it 'does not attempt to throttle' do + expect(::Gitlab::ApplicationRateLimiter).not_to receive(:throttled?) + + setup_attachment + receiver.execute + end + end + + context 'when :rate_limited_service Feature is enabled' do + let(:rate_limited_service_feature_enabled) { true } + + it 'raises a RateLimitedService::RateLimitedError' do + allow(::Gitlab::ApplicationRateLimiter).to receive(:throttled?).and_return(true) + + setup_attachment + expect { receiver.execute }.to raise_error(RateLimitedService::RateLimitedError, _('This endpoint has been requested too many times. Try again later.')) + end + end + end end def email_fixture(path) diff --git a/spec/lib/gitlab/email/handler/service_desk_handler_spec.rb b/spec/lib/gitlab/email/handler/service_desk_handler_spec.rb index 2916e65528f..8cb1ccc065b 100644 --- a/spec/lib/gitlab/email/handler/service_desk_handler_spec.rb +++ b/spec/lib/gitlab/email/handler/service_desk_handler_spec.rb @@ -243,6 +243,15 @@ RSpec.describe Gitlab::Email::Handler::ServiceDeskHandler do end end end + + context 'when rate limiting is in effect' do + it 'allows unlimited new issue creation' do + stub_application_setting(issues_create_limit: 1) + setup_attachment + + expect { 2.times { receiver.execute } }.to change { Issue.count }.by(2) + end + end end describe '#can_handle?' do diff --git a/spec/lib/gitlab/email/hook/smime_signature_interceptor_spec.rb b/spec/lib/gitlab/email/hook/smime_signature_interceptor_spec.rb index 56cf58dcf92..0a1f04ed793 100644 --- a/spec/lib/gitlab/email/hook/smime_signature_interceptor_spec.rb +++ b/spec/lib/gitlab/email/hook/smime_signature_interceptor_spec.rb @@ -14,15 +14,15 @@ RSpec.describe Gitlab::Email::Hook::SmimeSignatureInterceptor do end let(:root_certificate) do - Gitlab::Email::Smime::Certificate.new(@root_ca[:key], @root_ca[:cert]) + Gitlab::X509::Certificate.new(@root_ca[:key], @root_ca[:cert]) end let(:intermediate_certificate) do - Gitlab::Email::Smime::Certificate.new(@intermediate_ca[:key], @intermediate_ca[:cert]) + Gitlab::X509::Certificate.new(@intermediate_ca[:key], @intermediate_ca[:cert]) end let(:certificate) do - Gitlab::Email::Smime::Certificate.new(@cert[:key], @cert[:cert], [intermediate_certificate.cert]) + Gitlab::X509::Certificate.new(@cert[:key], @cert[:cert], [intermediate_certificate.cert]) end let(:mail_body) { "signed hello with Unicode €áø and\r\n newlines\r\n" } @@ -36,7 +36,7 @@ RSpec.describe Gitlab::Email::Hook::SmimeSignatureInterceptor do end before do - allow(Gitlab::Email::Smime::Certificate).to receive_messages(from_files: certificate) + allow(Gitlab::X509::Certificate).to receive_messages(from_files: certificate) Mail.register_interceptor(described_class) mail.deliver_now diff --git a/spec/lib/gitlab/endpoint_attributes_spec.rb b/spec/lib/gitlab/endpoint_attributes_spec.rb new file mode 100644 index 00000000000..4d4cfed57fa --- /dev/null +++ b/spec/lib/gitlab/endpoint_attributes_spec.rb @@ -0,0 +1,133 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' +require_relative "../../support/matchers/be_request_urgency" +require_relative "../../../lib/gitlab/endpoint_attributes" + +RSpec.describe Gitlab::EndpointAttributes do + let(:base_controller) do + Class.new do + include ::Gitlab::EndpointAttributes + end + end + + let(:controller) do + Class.new(base_controller) do + feature_category :foo, %w(update edit) + feature_category :bar, %w(index show) + feature_category :quux, %w(destroy) + + urgency :high, %w(do_a) + urgency :low, %w(do_b do_c) + end + end + + let(:subclass) do + Class.new(controller) do + feature_category :baz, %w(subclass_index) + urgency :high, %w(superclass_do_something) + end + end + + it "is nil when nothing was defined" do + expect(base_controller.feature_category_for_action("hello")).to be_nil + end + + it "returns the expected category", :aggregate_failures do + expect(controller.feature_category_for_action("update")).to eq(:foo) + expect(controller.feature_category_for_action("index")).to eq(:bar) + expect(controller.feature_category_for_action("destroy")).to eq(:quux) + end + + it "falls back to default when urgency was not defined", :aggregate_failures do + expect(base_controller.urgency_for_action("hello")).to be_request_urgency(:default) + expect(controller.urgency_for_action("update")).to be_request_urgency(:default) + expect(controller.urgency_for_action("index")).to be_request_urgency(:default) + expect(controller.urgency_for_action("destroy")).to be_request_urgency(:default) + end + + it "returns the expected urgency", :aggregate_failures do + expect(controller.urgency_for_action("do_a")).to be_request_urgency(:high) + expect(controller.urgency_for_action("do_b")).to be_request_urgency(:low) + expect(controller.urgency_for_action("do_c")).to be_request_urgency(:low) + end + + it "returns feature category for an implied action if not specify actions" do + klass = Class.new(base_controller) do + feature_category :foo + end + expect(klass.feature_category_for_action("index")).to eq(:foo) + expect(klass.feature_category_for_action("show")).to eq(:foo) + end + + it "returns expected duration for an implied action if not specify actions" do + klass = Class.new(base_controller) do + feature_category :foo + urgency :low + end + expect(klass.urgency_for_action("index")).to be_request_urgency(:low) + expect(klass.urgency_for_action("show")).to be_request_urgency(:low) + end + + it "returns the expected category for categories defined in subclasses" do + expect(subclass.feature_category_for_action("subclass_index")).to eq(:baz) + end + + it "falls back to superclass's feature category" do + expect(subclass.feature_category_for_action("update")).to eq(:foo) + end + + it "returns the expected urgency for categories defined in subclasses" do + expect(subclass.urgency_for_action("superclass_do_something")).to be_request_urgency(:high) + end + + it "falls back to superclass's expected duration" do + expect(subclass.urgency_for_action("do_a")).to be_request_urgency(:high) + end + + it "raises an error when defining for the controller and for individual actions" do + expect do + Class.new(base_controller) do + feature_category :hello + feature_category :goodbye, [:world] + end + end.to raise_error(ArgumentError, "feature_category are already defined for all actions, but re-defined for world") + end + + it "raises an error when multiple calls define the same action" do + expect do + Class.new(base_controller) do + feature_category :hello, [:world] + feature_category :goodbye, ["world"] + end + end.to raise_error(ArgumentError, "Attributes re-defined for action world: feature_category") + end + + it "raises an error when multiple calls define the same action" do + expect do + Class.new(base_controller) do + urgency :high, [:world] + urgency :low, ["world"] + end + end.to raise_error(ArgumentError, "Attributes re-defined for action world: urgency") + end + + it "does not raise an error when multiple calls define the same action and configs" do + expect do + Class.new(base_controller) do + feature_category :hello, [:world] + feature_category :hello, ["world"] + urgency :medium, [:moon] + urgency :medium, ["moon"] + end + end.not_to raise_error + end + + it "raises an error if the expected duration is not supported" do + expect do + Class.new(base_controller) do + urgency :super_slow + end + end.to raise_error(ArgumentError, "Urgency not supported: super_slow") + end +end diff --git a/spec/lib/gitlab/etag_caching/router/graphql_spec.rb b/spec/lib/gitlab/etag_caching/router/graphql_spec.rb index d151dcba413..9a6787e3640 100644 --- a/spec/lib/gitlab/etag_caching/router/graphql_spec.rb +++ b/spec/lib/gitlab/etag_caching/router/graphql_spec.rb @@ -11,7 +11,7 @@ RSpec.describe Gitlab::EtagCaching::Router::Graphql do end it 'has a valid feature category for every route', :aggregate_failures do - feature_categories = YAML.load_file(Rails.root.join('config', 'feature_categories.yml')).to_set + feature_categories = Gitlab::FeatureCategories.default.categories described_class::ROUTES.each do |route| expect(feature_categories).to include(route.feature_category), "#{route.name} has a category of #{route.feature_category}, which is not valid" diff --git a/spec/lib/gitlab/etag_caching/router/restful_spec.rb b/spec/lib/gitlab/etag_caching/router/restful_spec.rb index 1f5cac09b6d..a0fc480369c 100644 --- a/spec/lib/gitlab/etag_caching/router/restful_spec.rb +++ b/spec/lib/gitlab/etag_caching/router/restful_spec.rb @@ -107,7 +107,7 @@ RSpec.describe Gitlab::EtagCaching::Router::Restful do end it 'has a valid feature category for every route', :aggregate_failures do - feature_categories = YAML.load_file(Rails.root.join('config', 'feature_categories.yml')).to_set + feature_categories = Gitlab::FeatureCategories.default.categories described_class::ROUTES.each do |route| expect(feature_categories).to include(route.feature_category), "#{route.name} has a category of #{route.feature_category}, which is not valid" diff --git a/spec/lib/gitlab/feature_categories_spec.rb b/spec/lib/gitlab/feature_categories_spec.rb new file mode 100644 index 00000000000..daced154a69 --- /dev/null +++ b/spec/lib/gitlab/feature_categories_spec.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +RSpec.describe Gitlab::FeatureCategories do + let(:fake_categories) { %w(foo bar) } + + subject { described_class.new(fake_categories) } + + describe "#valid?" do + it "returns true if category is known", :aggregate_failures do + expect(subject.valid?('foo')).to be(true) + expect(subject.valid?('zzz')).to be(false) + end + end + + describe "#from_request" do + let(:request_env) { {} } + let(:verified) { true } + + def fake_request(request_feature_category) + double('request', env: request_env, headers: { "HTTP_X_GITLAB_FEATURE_CATEGORY" => request_feature_category }) + end + + before do + allow(::Gitlab::RequestForgeryProtection).to receive(:verified?).with(request_env).and_return(verified) + end + + it "returns category from request when valid, otherwise returns nil", :aggregate_failures do + expect(subject.from_request(fake_request("foo"))).to be("foo") + expect(subject.from_request(fake_request("zzz"))).to be_nil + end + + context "when request is not verified" do + let(:verified) { false } + + it "returns nil" do + expect(subject.from_request(fake_request("foo"))).to be_nil + end + end + end + + describe "#categories" do + it "returns a set of the given categories" do + expect(subject.categories).to be_a(Set) + expect(subject.categories).to contain_exactly(*fake_categories) + end + end + + describe ".load_from_yaml" do + subject { described_class.load_from_yaml } + + it "creates FeatureCategories from feature_categories.yml file" do + contents = YAML.load_file(Rails.root.join('config', 'feature_categories.yml')) + + expect(subject.categories).to contain_exactly(*contents) + end + end + + describe ".default" do + it "returns a memoization of load_from_yaml", :aggregate_failures do + # FeatureCategories.default could have been referenced in another spec, so we need to clean it up here + described_class.instance_variable_set(:@default, nil) + + expect(described_class).to receive(:load_from_yaml).once.and_call_original + + 2.times { described_class.default } + + # Uses reference equality to verify memoization + expect(described_class.default).to equal(described_class.default) + expect(described_class.default).to be_a(described_class) + end + end +end diff --git a/spec/lib/gitlab/form_builders/gitlab_ui_form_builder_spec.rb b/spec/lib/gitlab/form_builders/gitlab_ui_form_builder_spec.rb index a46846e9820..e160e88487b 100644 --- a/spec/lib/gitlab/form_builders/gitlab_ui_form_builder_spec.rb +++ b/spec/lib/gitlab/form_builders/gitlab_ui_form_builder_spec.rb @@ -75,7 +75,68 @@ RSpec.describe Gitlab::FormBuilders::GitlabUiFormBuilder do checkbox_html - expect(fake_template).to have_received(:label).with(:user, :view_diffs_file_by_file, { class: %w(custom-control-label label-foo-bar), object: user }) + expect(fake_template).to have_received(:label).with(:user, :view_diffs_file_by_file, { class: %w(custom-control-label label-foo-bar), object: user, value: nil }) + end + end + end + + describe '#gitlab_ui_radio_component' do + let(:optional_args) { {} } + + subject(:radio_html) { form_builder.gitlab_ui_radio_component(:access_level, :admin, "Access Level", **optional_args) } + + context 'without optional arguments' do + it 'renders correct html' do + expected_html = <<~EOS + <div class="gl-form-radio custom-control custom-radio"> + <input class="custom-control-input" type="radio" value="admin" name="user[access_level]" id="user_access_level_admin" /> + <label class="custom-control-label" for="user_access_level_admin"> + Access Level + </label> + </div> + EOS + + expect(radio_html).to eq(html_strip_whitespace(expected_html)) + end + end + + context 'with optional arguments' do + let(:optional_args) do + { + help_text: 'Administrators have access to all groups, projects, and users and can manage all features in this installation', + radio_options: { class: 'radio-foo-bar' }, + label_options: { class: 'label-foo-bar' } + } + end + + it 'renders help text' do + expected_html = <<~EOS + <div class="gl-form-radio custom-control custom-radio"> + <input class="custom-control-input radio-foo-bar" type="radio" value="admin" name="user[access_level]" id="user_access_level_admin" /> + <label class="custom-control-label label-foo-bar" for="user_access_level_admin"> + <span>Access Level</span> + <p class="help-text">Administrators have access to all groups, projects, and users and can manage all features in this installation</p> + </label> + </div> + EOS + + expect(radio_html).to eq(html_strip_whitespace(expected_html)) + end + + it 'passes arguments to `radio_button` method' do + allow(fake_template).to receive(:radio_button).and_return('') + + radio_html + + expect(fake_template).to have_received(:radio_button).with(:user, :access_level, :admin, { class: %w(custom-control-input radio-foo-bar), object: user }) + end + + it 'passes arguments to `label` method' do + allow(fake_template).to receive(:label).and_return('') + + radio_html + + expect(fake_template).to have_received(:label).with(:user, :access_level, { class: %w(custom-control-label label-foo-bar), object: user, value: :admin }) end end end diff --git a/spec/lib/gitlab/git/repository_spec.rb b/spec/lib/gitlab/git/repository_spec.rb index 9ecd281cce0..c7b68ff3e28 100644 --- a/spec/lib/gitlab/git/repository_spec.rb +++ b/spec/lib/gitlab/git/repository_spec.rb @@ -2238,7 +2238,6 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do end describe '#squash' do - let(:squash_id) { '1' } let(:branch_name) { 'fix' } let(:start_sha) { '4b4918a572fa86f9771e5ba40fbd48e1eb03e2c6' } let(:end_sha) { '12d65c8dd2b2676fa3ac47d955accc085a37a9c1' } @@ -2252,7 +2251,7 @@ RSpec.describe Gitlab::Git::Repository, :seed_helper do message: 'Squash commit message' } - repository.squash(user, squash_id, opts) + repository.squash(user, opts) end # Should be ported to gitaly-ruby rspec suite https://gitlab.com/gitlab-org/gitaly/issues/1234 diff --git a/spec/lib/gitlab/git/wraps_gitaly_errors_spec.rb b/spec/lib/gitlab/git/wraps_gitaly_errors_spec.rb index 2c9da0f6606..e551dfaa1c5 100644 --- a/spec/lib/gitlab/git/wraps_gitaly_errors_spec.rb +++ b/spec/lib/gitlab/git/wraps_gitaly_errors_spec.rb @@ -12,6 +12,7 @@ RSpec.describe Gitlab::Git::WrapsGitalyErrors do mapping = { GRPC::NotFound => Gitlab::Git::Repository::NoRepository, GRPC::InvalidArgument => ArgumentError, + GRPC::DeadlineExceeded => Gitlab::Git::CommandTimedOut, GRPC::BadStatus => Gitlab::Git::CommandError } diff --git a/spec/lib/gitlab/gitaly_client/operation_service_spec.rb b/spec/lib/gitlab/gitaly_client/operation_service_spec.rb index 3789bc76a94..27e7d446770 100644 --- a/spec/lib/gitlab/gitaly_client/operation_service_spec.rb +++ b/spec/lib/gitlab/gitaly_client/operation_service_spec.rb @@ -169,6 +169,56 @@ RSpec.describe Gitlab::GitalyClient::OperationService do end end + describe '#user_merge_branch' do + let(:target_branch) { 'master' } + let(:source_sha) { '5937ac0a7beb003549fc5fd26fc247adbce4a52e' } + let(:message) { 'Merge a branch' } + + subject { client.user_merge_branch(user, source_sha, target_branch, message) {} } + + it 'sends a user_merge_branch message' do + expect(subject).to be_a(Gitlab::Git::OperationService::BranchUpdate) + expect(subject.newrev).to be_present + expect(subject.repo_created).to be(false) + expect(subject.branch_created).to be(false) + end + + context 'with an exception with the UserMergeBranchError' do + let(:permission_error) do + GRPC::PermissionDenied.new( + "GitLab: You are not allowed to push code to this project.", + { "grpc-status-details-bin" => + "\b\a\x129GitLab: You are not allowed to push code to this project.\x1A\xDE\x01\n/type.googleapis.com/gitaly.UserMergeBranchError\x12\xAA\x01\n\xA7\x01\n1You are not allowed to push code to this project.\x12\x03web\x1A\auser-15\"df15b32277d2c55c6c595845a87109b09c913c556 5d6e0f935ad9240655f64e883cd98fad6f9a17ee refs/heads/master\n" } + ) + end + + it 'raises PreRecieveError with the error message' do + expect_any_instance_of(Gitaly::OperationService::Stub) + .to receive(:user_merge_branch).with(kind_of(Enumerator), kind_of(Hash)) + .and_raise(permission_error) + + expect { subject }.to raise_error do |error| + expect(error).to be_a(Gitlab::Git::PreReceiveError) + expect(error.message).to eq("You are not allowed to push code to this project.") + end + end + end + + context 'with an exception without the detailed error' do + let(:permission_error) do + GRPC::PermissionDenied.new + end + + it 'raises PermissionDenied' do + expect_any_instance_of(Gitaly::OperationService::Stub) + .to receive(:user_merge_branch).with(kind_of(Enumerator), kind_of(Hash)) + .and_raise(permission_error) + + expect { subject }.to raise_error(GRPC::PermissionDenied) + end + end + end + describe '#user_ff_branch' do let(:target_branch) { 'my-branch' } let(:source_sha) { 'cfe32cf61b73a0d5e9f13e774abde7ff789b1660' } @@ -308,7 +358,6 @@ RSpec.describe Gitlab::GitalyClient::OperationService do end describe '#user_squash' do - let(:squash_id) { '1' } let(:start_sha) { 'b83d6e391c22777fca1ed3012fce84f633d7fed0' } let(:end_sha) { '54cec5282aa9f21856362fe321c800c236a61615' } let(:commit_message) { 'Squash message' } @@ -321,7 +370,6 @@ RSpec.describe Gitlab::GitalyClient::OperationService do Gitaly::UserSquashRequest.new( repository: repository.gitaly_repository, user: gitaly_user, - squash_id: squash_id.to_s, start_sha: start_sha, end_sha: end_sha, author: gitaly_user, @@ -334,7 +382,7 @@ RSpec.describe Gitlab::GitalyClient::OperationService do let(:response) { Gitaly::UserSquashResponse.new(squash_sha: squash_sha) } subject do - client.user_squash(user, squash_id, start_sha, end_sha, user, commit_message, time) + client.user_squash(user, start_sha, end_sha, user, commit_message, time) end it 'sends a user_squash message and returns the squash sha' do diff --git a/spec/lib/gitlab/github_import/importer/diff_notes_importer_spec.rb b/spec/lib/gitlab/github_import/importer/diff_notes_importer_spec.rb index 46b9959ff64..be4fc3cbf16 100644 --- a/spec/lib/gitlab/github_import/importer/diff_notes_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/diff_notes_importer_spec.rb @@ -15,10 +15,18 @@ RSpec.describe Gitlab::GithubImport::Importer::DiffNotesImporter do original_commit_id: 'original123abc', diff_hunk: "@@ -1 +1 @@\n-Hello\n+Hello world", user: double(:user, id: 4, login: 'alice'), - body: 'Hello world', created_at: Time.zone.now, updated_at: Time.zone.now, - id: 1 + line: 23, + start_line: nil, + id: 1, + body: <<~BODY + Hello World + + ```suggestion + sug1 + ``` + BODY ) end diff --git a/spec/lib/gitlab/github_import/parallel_importer_spec.rb b/spec/lib/gitlab/github_import/parallel_importer_spec.rb index 06304bf84ca..c7b300ff043 100644 --- a/spec/lib/gitlab/github_import/parallel_importer_spec.rb +++ b/spec/lib/gitlab/github_import/parallel_importer_spec.rb @@ -9,6 +9,18 @@ RSpec.describe Gitlab::GithubImport::ParallelImporter do end end + describe '.track_start_import' do + it 'tracks the start of import' do + project = double(:project) + metrics = double(:metrics) + + expect(Gitlab::Import::Metrics).to receive(:new).with(:github_importer, project).and_return(metrics) + expect(metrics).to receive(:track_start_import) + + described_class.track_start_import(project) + end + end + describe '#execute', :clean_gitlab_redis_shared_state do let(:project) { create(:project) } let(:importer) { described_class.new(project) } diff --git a/spec/lib/gitlab/github_import/parallel_scheduling_spec.rb b/spec/lib/gitlab/github_import/parallel_scheduling_spec.rb index 1fc7d3c887f..f375e84e0fd 100644 --- a/spec/lib/gitlab/github_import/parallel_scheduling_spec.rb +++ b/spec/lib/gitlab/github_import/parallel_scheduling_spec.rb @@ -130,7 +130,8 @@ RSpec.describe Gitlab::GithubImport::ParallelScheduling do project_id: project.id, exception: exception, error_source: 'MyImporter', - fail_import: false + fail_import: false, + metrics: true ).and_call_original expect { importer.execute } @@ -195,7 +196,8 @@ RSpec.describe Gitlab::GithubImport::ParallelScheduling do project_id: project.id, exception: exception, error_source: 'MyImporter', - fail_import: true + fail_import: true, + metrics: true ).and_call_original expect { importer.execute } diff --git a/spec/lib/gitlab/github_import/representation/diff_note_spec.rb b/spec/lib/gitlab/github_import/representation/diff_note_spec.rb index 7c24cd0a5db..81722c0eba7 100644 --- a/spec/lib/gitlab/github_import/representation/diff_note_spec.rb +++ b/spec/lib/gitlab/github_import/representation/diff_note_spec.rb @@ -51,7 +51,7 @@ RSpec.describe Gitlab::GithubImport::Representation::DiffNote do end it 'includes the GitHub ID' do - expect(note.github_id).to eq(1) + expect(note.note_id).to eq(1) end it 'returns the noteable type' do @@ -73,6 +73,8 @@ RSpec.describe Gitlab::GithubImport::Representation::DiffNote do body: 'Hello world', created_at: created_at, updated_at: updated_at, + line: 23, + start_line: nil, id: 1 ) end @@ -90,47 +92,70 @@ RSpec.describe Gitlab::GithubImport::Representation::DiffNote do expect(note.author).to be_nil end - end - describe '.from_json_hash' do - it_behaves_like 'a DiffNote' do - let(:hash) do - { - 'noteable_type' => 'MergeRequest', - 'noteable_id' => 42, - 'file_path' => 'README.md', - 'commit_id' => '123abc', - 'original_commit_id' => 'original123abc', - 'diff_hunk' => hunk, - 'author' => { 'id' => 4, 'login' => 'alice' }, - 'note' => 'Hello world', - 'created_at' => created_at.to_s, - 'updated_at' => updated_at.to_s, - 'github_id' => 1 - } - end + it 'formats a suggestion in the note body' do + allow(response) + .to receive(:body) + .and_return <<~BODY + ```suggestion + Hello World + ``` + BODY - let(:note) { described_class.from_json_hash(hash) } + note = described_class.from_api_response(response) + + expect(note.note).to eq <<~BODY + ```suggestion:-0+0 + Hello World + ``` + BODY end + end - it 'does not convert the author if it was not specified' do - hash = { + describe '.from_json_hash' do + let(:hash) do + { 'noteable_type' => 'MergeRequest', 'noteable_id' => 42, 'file_path' => 'README.md', 'commit_id' => '123abc', 'original_commit_id' => 'original123abc', 'diff_hunk' => hunk, + 'author' => { 'id' => 4, 'login' => 'alice' }, 'note' => 'Hello world', 'created_at' => created_at.to_s, 'updated_at' => updated_at.to_s, - 'github_id' => 1 + 'note_id' => 1 } + end + + it_behaves_like 'a DiffNote' do + let(:note) { described_class.from_json_hash(hash) } + end + + it 'does not convert the author if it was not specified' do + hash.delete('author') note = described_class.from_json_hash(hash) expect(note.author).to be_nil end + + it 'formats a suggestion in the note body' do + hash['note'] = <<~BODY + ```suggestion + Hello World + ``` + BODY + + note = described_class.from_json_hash(hash) + + expect(note.note).to eq <<~BODY + ```suggestion:-0+0 + Hello World + ``` + BODY + end end describe '#line_code' do @@ -154,7 +179,7 @@ RSpec.describe Gitlab::GithubImport::Representation::DiffNote do 'note' => 'Hello world', 'created_at' => created_at.to_s, 'updated_at' => updated_at.to_s, - 'github_id' => 1 + 'note_id' => 1 ) expect(note.diff_hash).to eq( @@ -167,4 +192,68 @@ RSpec.describe Gitlab::GithubImport::Representation::DiffNote do ) end end + + describe '#github_identifiers' do + it 'returns a hash with needed identifiers' do + github_identifiers = { + noteable_id: 42, + noteable_type: 'MergeRequest', + note_id: 1 + } + other_attributes = { something_else: '_something_else_' } + note = described_class.new(github_identifiers.merge(other_attributes)) + + expect(note.github_identifiers).to eq(github_identifiers) + end + end + + describe '#note' do + it 'returns the given note' do + hash = { + 'note': 'simple text' + } + + note = described_class.new(hash) + + expect(note.note).to eq 'simple text' + end + + it 'returns the suggestion formatted in the note' do + hash = { + 'note': <<~BODY + ```suggestion + Hello World + ``` + BODY + } + + note = described_class.new(hash) + + expect(note.note).to eq <<~BODY + ```suggestion:-0+0 + Hello World + ``` + BODY + end + + it 'returns the multi-line suggestion formatted in the note' do + hash = { + 'start_line': 20, + 'end_line': 23, + 'note': <<~BODY + ```suggestion + Hello World + ``` + BODY + } + + note = described_class.new(hash) + + expect(note.note).to eq <<~BODY + ```suggestion:-3+0 + Hello World + ``` + BODY + end + end end diff --git a/spec/lib/gitlab/github_import/representation/diff_notes/suggestion_formatter_spec.rb b/spec/lib/gitlab/github_import/representation/diff_notes/suggestion_formatter_spec.rb new file mode 100644 index 00000000000..2ffd5f50d3b --- /dev/null +++ b/spec/lib/gitlab/github_import/representation/diff_notes/suggestion_formatter_spec.rb @@ -0,0 +1,164 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::GithubImport::Representation::DiffNotes::SuggestionFormatter do + it 'does nothing when there is any text before the suggestion tag' do + note = <<~BODY + looks like```suggestion but it isn't + ``` + BODY + + expect(described_class.formatted_note_for(note: note)).to eq(note) + end + + it 'handles nil value for note' do + note = nil + + expect(described_class.formatted_note_for(note: note)).to eq(note) + end + + it 'does not allow over 3 leading spaces for valid suggestion' do + note = <<~BODY + Single-line suggestion + ```suggestion + sug1 + ``` + BODY + + expect(described_class.formatted_note_for(note: note)).to eq(note) + end + + it 'allows up to 3 leading spaces' do + note = <<~BODY + Single-line suggestion + ```suggestion + sug1 + ``` + BODY + + expected = <<~BODY + Single-line suggestion + ```suggestion:-0+0 + sug1 + ``` + BODY + + expect(described_class.formatted_note_for(note: note)).to eq(expected) + end + + it 'does nothing when there is any text without space after the suggestion tag' do + note = <<~BODY + ```suggestionbut it isn't + ``` + BODY + + expect(described_class.formatted_note_for(note: note)).to eq(note) + end + + it 'formats single-line suggestions' do + note = <<~BODY + Single-line suggestion + ```suggestion + sug1 + ``` + BODY + + expected = <<~BODY + Single-line suggestion + ```suggestion:-0+0 + sug1 + ``` + BODY + + expect(described_class.formatted_note_for(note: note)).to eq(expected) + end + + it 'ignores text after suggestion tag on the same line' do + note = <<~BODY + looks like + ```suggestion text to be ignored + suggestion + ``` + BODY + + expected = <<~BODY + looks like + ```suggestion:-0+0 + suggestion + ``` + BODY + + expect(described_class.formatted_note_for(note: note)).to eq(expected) + end + + it 'formats multiple single-line suggestions' do + note = <<~BODY + Single-line suggestion + ```suggestion + sug1 + ``` + OR + ```suggestion + sug2 + ``` + BODY + + expected = <<~BODY + Single-line suggestion + ```suggestion:-0+0 + sug1 + ``` + OR + ```suggestion:-0+0 + sug2 + ``` + BODY + + expect(described_class.formatted_note_for(note: note)).to eq(expected) + end + + it 'formats multi-line suggestions' do + note = <<~BODY + Multi-line suggestion + ```suggestion + sug1 + ``` + BODY + + expected = <<~BODY + Multi-line suggestion + ```suggestion:-2+0 + sug1 + ``` + BODY + + expect(described_class.formatted_note_for(note: note, start_line: 6, end_line: 8)).to eq(expected) + end + + it 'formats multiple multi-line suggestions' do + note = <<~BODY + Multi-line suggestion + ```suggestion + sug1 + ``` + OR + ```suggestion + sug2 + ``` + BODY + + expected = <<~BODY + Multi-line suggestion + ```suggestion:-2+0 + sug1 + ``` + OR + ```suggestion:-2+0 + sug2 + ``` + BODY + + expect(described_class.formatted_note_for(note: note, start_line: 6, end_line: 8)).to eq(expected) + end +end diff --git a/spec/lib/gitlab/github_import/representation/issue_spec.rb b/spec/lib/gitlab/github_import/representation/issue_spec.rb index 3d306a4a3a3..f3052efea70 100644 --- a/spec/lib/gitlab/github_import/representation/issue_spec.rb +++ b/spec/lib/gitlab/github_import/representation/issue_spec.rb @@ -181,4 +181,17 @@ RSpec.describe Gitlab::GithubImport::Representation::Issue do expect(object.truncated_title).to eq('foo') end end + + describe '#github_identifiers' do + it 'returns a hash with needed identifiers' do + github_identifiers = { + iid: 42, + issuable_type: 'MergeRequest' + } + other_attributes = { pull_request: true, something_else: '_something_else_' } + issue = described_class.new(github_identifiers.merge(other_attributes)) + + expect(issue.github_identifiers).to eq(github_identifiers) + end + end end diff --git a/spec/lib/gitlab/github_import/representation/lfs_object_spec.rb b/spec/lib/gitlab/github_import/representation/lfs_object_spec.rb new file mode 100644 index 00000000000..b59ea513436 --- /dev/null +++ b/spec/lib/gitlab/github_import/representation/lfs_object_spec.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::GithubImport::Representation::LfsObject do + describe '#github_identifiers' do + it 'returns a hash with needed identifiers' do + github_identifiers = { + oid: 42 + } + other_attributes = { something_else: '_something_else_' } + lfs_object = described_class.new(github_identifiers.merge(other_attributes)) + + expect(lfs_object.github_identifiers).to eq(github_identifiers) + end + end +end diff --git a/spec/lib/gitlab/github_import/representation/note_spec.rb b/spec/lib/gitlab/github_import/representation/note_spec.rb index 112bb7eb908..97addcc1c98 100644 --- a/spec/lib/gitlab/github_import/representation/note_spec.rb +++ b/spec/lib/gitlab/github_import/representation/note_spec.rb @@ -40,8 +40,8 @@ RSpec.describe Gitlab::GithubImport::Representation::Note do expect(note.updated_at).to eq(updated_at) end - it 'includes the GitHub ID' do - expect(note.github_id).to eq(1) + it 'includes the note ID' do + expect(note.note_id).to eq(1) end end end @@ -84,7 +84,7 @@ RSpec.describe Gitlab::GithubImport::Representation::Note do 'note' => 'Hello world', 'created_at' => created_at.to_s, 'updated_at' => updated_at.to_s, - 'github_id' => 1 + 'note_id' => 1 } end @@ -98,7 +98,7 @@ RSpec.describe Gitlab::GithubImport::Representation::Note do 'note' => 'Hello world', 'created_at' => created_at.to_s, 'updated_at' => updated_at.to_s, - 'github_id' => 1 + 'note_id' => 1 } note = described_class.from_json_hash(hash) @@ -106,4 +106,18 @@ RSpec.describe Gitlab::GithubImport::Representation::Note do expect(note.author).to be_nil end end + + describe '#github_identifiers' do + it 'returns a hash with needed identifiers' do + github_identifiers = { + noteable_id: 42, + noteable_type: 'Issue', + note_id: 1 + } + other_attributes = { something_else: '_something_else_' } + note = described_class.new(github_identifiers.merge(other_attributes)) + + expect(note.github_identifiers).to eq(github_identifiers) + end + end end diff --git a/spec/lib/gitlab/github_import/representation/pull_request_review_spec.rb b/spec/lib/gitlab/github_import/representation/pull_request_review_spec.rb index cad9b13774e..f812fd85fbc 100644 --- a/spec/lib/gitlab/github_import/representation/pull_request_review_spec.rb +++ b/spec/lib/gitlab/github_import/representation/pull_request_review_spec.rb @@ -14,7 +14,7 @@ RSpec.describe Gitlab::GithubImport::Representation::PullRequestReview do expect(review.note).to eq('note') expect(review.review_type).to eq('APPROVED') expect(review.submitted_at).to eq(submitted_at) - expect(review.github_id).to eq(999) + expect(review.review_id).to eq(999) expect(review.merge_request_id).to eq(42) end end @@ -50,7 +50,7 @@ RSpec.describe Gitlab::GithubImport::Representation::PullRequestReview do describe '.from_json_hash' do let(:hash) do { - 'github_id' => 999, + 'review_id' => 999, 'merge_request_id' => 42, 'note' => 'note', 'review_type' => 'APPROVED', @@ -75,4 +75,17 @@ RSpec.describe Gitlab::GithubImport::Representation::PullRequestReview do expect(review.submitted_at).to be_nil end end + + describe '#github_identifiers' do + it 'returns a hash with needed identifiers' do + github_identifiers = { + review_id: 999, + merge_request_id: 42 + } + other_attributes = { something_else: '_something_else_' } + review = described_class.new(github_identifiers.merge(other_attributes)) + + expect(review.github_identifiers).to eq(github_identifiers) + end + end end diff --git a/spec/lib/gitlab/github_import/representation/pull_request_spec.rb b/spec/lib/gitlab/github_import/representation/pull_request_spec.rb index 27a82951b01..925dba5b5a7 100644 --- a/spec/lib/gitlab/github_import/representation/pull_request_spec.rb +++ b/spec/lib/gitlab/github_import/representation/pull_request_spec.rb @@ -288,4 +288,16 @@ RSpec.describe Gitlab::GithubImport::Representation::PullRequest do expect(object.truncated_title).to eq('foo') end end + + describe '#github_identifiers' do + it 'returns a hash with needed identifiers' do + github_identifiers = { + iid: 1 + } + other_attributes = { something_else: '_something_else_' } + pr = described_class.new(github_identifiers.merge(other_attributes)) + + expect(pr.github_identifiers).to eq(github_identifiers.merge(issuable_type: 'MergeRequest')) + end + end end diff --git a/spec/lib/gitlab/github_import/sequential_importer_spec.rb b/spec/lib/gitlab/github_import/sequential_importer_spec.rb index 3c3f8ff59d0..2b76f0e27c9 100644 --- a/spec/lib/gitlab/github_import/sequential_importer_spec.rb +++ b/spec/lib/gitlab/github_import/sequential_importer_spec.rb @@ -4,10 +4,17 @@ require 'spec_helper' RSpec.describe Gitlab::GithubImport::SequentialImporter do describe '#execute' do + let_it_be(:project) do + create(:project, import_url: 'http://t0ken@github.another-domain.com/repo-org/repo.git', import_type: 'github') + end + + subject(:importer) { described_class.new(project, token: 'foo') } + it 'imports a project in sequence' do - repository = double(:repository) - project = double(:project, id: 1, repository: repository, import_url: 'http://t0ken@github.another-domain.com/repo-org/repo.git', group: nil) - importer = described_class.new(project, token: 'foo') + expect_next_instance_of(Gitlab::Import::Metrics) do |instance| + expect(instance).to receive(:track_start_import) + expect(instance).to receive(:track_finished_import) + end expect_next_instance_of(Gitlab::GithubImport::Importer::RepositoryImporter) do |instance| expect(instance).to receive(:execute) @@ -35,5 +42,23 @@ RSpec.describe Gitlab::GithubImport::SequentialImporter do expect(importer.execute).to eq(true) end + + it 'raises an error' do + exception = StandardError.new('_some_error_') + + expect_next_instance_of(Gitlab::GithubImport::Importer::RepositoryImporter) do |importer| + expect(importer).to receive(:execute).and_raise(exception) + end + expect(Gitlab::Import::ImportFailureService).to receive(:track) + .with( + project_id: project.id, + exception: exception, + error_source: described_class.name, + fail_import: true, + metrics: true + ).and_call_original + + expect { importer.execute }.to raise_error(StandardError) + end end end diff --git a/spec/lib/gitlab/health_checks/probes/collection_spec.rb b/spec/lib/gitlab/health_checks/probes/collection_spec.rb index 69828c143db..741c45d953c 100644 --- a/spec/lib/gitlab/health_checks/probes/collection_spec.rb +++ b/spec/lib/gitlab/health_checks/probes/collection_spec.rb @@ -16,6 +16,9 @@ RSpec.describe Gitlab::HealthChecks::Probes::Collection do Gitlab::HealthChecks::Redis::CacheCheck, Gitlab::HealthChecks::Redis::QueuesCheck, Gitlab::HealthChecks::Redis::SharedStateCheck, + Gitlab::HealthChecks::Redis::TraceChunksCheck, + Gitlab::HealthChecks::Redis::RateLimitingCheck, + Gitlab::HealthChecks::Redis::SessionsCheck, Gitlab::HealthChecks::GitalyCheck ] end diff --git a/spec/lib/gitlab/health_checks/redis/rate_limiting_check_spec.rb b/spec/lib/gitlab/health_checks/redis/rate_limiting_check_spec.rb new file mode 100644 index 00000000000..1521fc99cde --- /dev/null +++ b/spec/lib/gitlab/health_checks/redis/rate_limiting_check_spec.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_relative '../simple_check_shared' + +RSpec.describe Gitlab::HealthChecks::Redis::RateLimitingCheck do + include_examples 'simple_check', 'redis_rate_limiting_ping', 'RedisRateLimiting', 'PONG' +end diff --git a/spec/lib/gitlab/health_checks/redis/sessions_check_spec.rb b/spec/lib/gitlab/health_checks/redis/sessions_check_spec.rb new file mode 100644 index 00000000000..82b3b33ec0a --- /dev/null +++ b/spec/lib/gitlab/health_checks/redis/sessions_check_spec.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_relative '../simple_check_shared' + +RSpec.describe Gitlab::HealthChecks::Redis::SessionsCheck do + include_examples 'simple_check', 'redis_sessions_ping', 'RedisSessions', 'PONG' +end diff --git a/spec/lib/gitlab/import/import_failure_service_spec.rb b/spec/lib/gitlab/import/import_failure_service_spec.rb index 50b32d634ad..c16d4a7c804 100644 --- a/spec/lib/gitlab/import/import_failure_service_spec.rb +++ b/spec/lib/gitlab/import/import_failure_service_spec.rb @@ -2,135 +2,171 @@ require 'spec_helper' -RSpec.describe Gitlab::Import::ImportFailureService do +RSpec.describe Gitlab::Import::ImportFailureService, :aggregate_failures do let_it_be(:import_type) { 'import_type' } + let_it_be(:project) { create(:project, :import_started, import_type: import_type) } - let_it_be(:project) do - create( - :project, - :import_started, - import_type: import_type - ) - end - - let(:import_state) { project.import_state } let(:exception) { StandardError.new('some error') } + let(:arguments) { { project_id: project.id } } + let(:base_arguments) { { error_source: 'SomeImporter', exception: exception }.merge(arguments) } + let(:exe_arguments) { { fail_import: false, metrics: false } } + + describe '.track' do + context 'with all arguments provided' do + let(:instance) { double(:failure_service) } + let(:instance_arguments) do + { + exception: exception, + import_state: '_import_state_', + project_id: '_project_id_', + error_source: '_error_source_' + } + end - shared_examples 'logs the exception and fails the import' do - it 'when the failure does not abort the import' do - expect(Gitlab::ErrorTracking) - .to receive(:track_exception) - .with( - exception, - project_id: project.id, - import_type: import_type, - source: 'SomeImporter' - ) - - expect(Gitlab::Import::Logger) - .to receive(:error) - .with( - message: 'importer failed', - 'error.message': 'some error', - project_id: project.id, - import_type: import_type, - source: 'SomeImporter' - ) - - described_class.track(**arguments) - - expect(project.import_state.reload.status).to eq('failed') - - expect(project.import_failures).not_to be_empty - expect(project.import_failures.last.exception_class).to eq('StandardError') - expect(project.import_failures.last.exception_message).to eq('some error') - end - end + let(:exe_arguments) do + { + fail_import: '_fail_import_', + metrics: '_metrics_' + } + end - shared_examples 'logs the exception and does not fail the import' do - it 'when the failure does not abort the import' do - expect(Gitlab::ErrorTracking) - .to receive(:track_exception) - .with( - exception, - project_id: project.id, - import_type: import_type, - source: 'SomeImporter' - ) - - expect(Gitlab::Import::Logger) - .to receive(:error) - .with( - message: 'importer failed', - 'error.message': 'some error', - project_id: project.id, - import_type: import_type, - source: 'SomeImporter' - ) - - described_class.track(**arguments) - - expect(project.import_state.reload.status).to eq('started') - - expect(project.import_failures).not_to be_empty - expect(project.import_failures.last.exception_class).to eq('StandardError') - expect(project.import_failures.last.exception_message).to eq('some error') + it 'invokes a new instance and executes' do + expect(described_class).to receive(:new).with(**instance_arguments).and_return(instance) + expect(instance).to receive(:execute).with(**exe_arguments) + + described_class.track(**instance_arguments.merge(exe_arguments)) + end end - end - context 'when using the project as reference' do - context 'when it fails the import' do - let(:arguments) do + context 'with only necessary arguments utilizing defaults' do + let(:instance) { double(:failure_service) } + let(:instance_arguments) do { - project_id: project.id, exception: exception, - error_source: 'SomeImporter', - fail_import: true + import_state: nil, + project_id: nil, + error_source: nil } end - it_behaves_like 'logs the exception and fails the import' - end - - context 'when it does not fail the import' do - let(:arguments) do + let(:exe_arguments) do { - project_id: project.id, - exception: exception, - error_source: 'SomeImporter', - fail_import: false + fail_import: false, + metrics: false } end - it_behaves_like 'logs the exception and does not fail the import' + it 'invokes a new instance and executes' do + expect(described_class).to receive(:new).with(**instance_arguments).and_return(instance) + expect(instance).to receive(:execute).with(**exe_arguments) + + described_class.track(exception: exception) + end end end - context 'when using the import_state as reference' do - context 'when it fails the import' do - let(:arguments) do - { - import_state: import_state, - exception: exception, - error_source: 'SomeImporter', - fail_import: true - } + describe '#execute' do + subject(:service) { described_class.new(**base_arguments) } + + shared_examples 'logs the exception and fails the import' do + it 'when the failure does not abort the import' do + expect(Gitlab::ErrorTracking) + .to receive(:track_exception) + .with( + exception, + project_id: project.id, + import_type: import_type, + source: 'SomeImporter' + ) + + expect(Gitlab::Import::Logger) + .to receive(:error) + .with( + message: 'importer failed', + 'error.message': 'some error', + project_id: project.id, + import_type: import_type, + source: 'SomeImporter' + ) + + service.execute(**exe_arguments) + + expect(project.import_state.reload.status).to eq('failed') + + expect(project.import_failures).not_to be_empty + expect(project.import_failures.last.exception_class).to eq('StandardError') + expect(project.import_failures.last.exception_message).to eq('some error') end + end - it_behaves_like 'logs the exception and fails the import' + shared_examples 'logs the exception and does not fail the import' do + it 'when the failure does not abort the import' do + expect(Gitlab::ErrorTracking) + .to receive(:track_exception) + .with( + exception, + project_id: project.id, + import_type: import_type, + source: 'SomeImporter' + ) + + expect(Gitlab::Import::Logger) + .to receive(:error) + .with( + message: 'importer failed', + 'error.message': 'some error', + project_id: project.id, + import_type: import_type, + source: 'SomeImporter' + ) + + service.execute(**exe_arguments) + + expect(project.import_state.reload.status).to eq('started') + + expect(project.import_failures).not_to be_empty + expect(project.import_failures.last.exception_class).to eq('StandardError') + expect(project.import_failures.last.exception_message).to eq('some error') + end end - context 'when it does not fail the import' do - let(:arguments) do - { - import_state: import_state, - exception: exception, - error_source: 'SomeImporter', - fail_import: false - } + context 'when tracking metrics' do + let(:exe_arguments) { { fail_import: false, metrics: true } } + + it 'tracks the failed import' do + metrics = double(:metrics) + + expect(Gitlab::Import::Metrics).to receive(:new).with("#{project.import_type}_importer", project).and_return(metrics) + expect(metrics).to receive(:track_failed_import) + + service.execute(**exe_arguments) end + end + + context 'when using the project as reference' do + context 'when it fails the import' do + let(:exe_arguments) { { fail_import: true, metrics: false } } - it_behaves_like 'logs the exception and does not fail the import' + it_behaves_like 'logs the exception and fails the import' + end + + context 'when it does not fail the import' do + it_behaves_like 'logs the exception and does not fail the import' + end + end + + context 'when using the import_state as reference' do + let(:arguments) { { import_state: project.import_state } } + + context 'when it fails the import' do + let(:exe_arguments) { { fail_import: true, metrics: false } } + + it_behaves_like 'logs the exception and fails the import' + end + + context 'when it does not fail the import' do + it_behaves_like 'logs the exception and does not fail the import' + end end end end diff --git a/spec/lib/gitlab/import/metrics_spec.rb b/spec/lib/gitlab/import/metrics_spec.rb index 0a912427014..035294a620f 100644 --- a/spec/lib/gitlab/import/metrics_spec.rb +++ b/spec/lib/gitlab/import/metrics_spec.rb @@ -2,20 +2,67 @@ require 'spec_helper' -RSpec.describe Gitlab::Import::Metrics do +RSpec.describe Gitlab::Import::Metrics, :aggregate_failures do let(:importer) { :test_importer } - let(:project) { create(:project) } + let(:project) { build(:project, id: non_existing_record_id, created_at: Time.current) } let(:histogram) { double(:histogram) } let(:counter) { double(:counter) } subject { described_class.new(importer, project) } - describe '#report_import_time' do + before do + allow(Gitlab::Metrics).to receive(:counter) { counter } + allow(counter).to receive(:increment) + allow(histogram).to receive(:observe) + end + + describe '#track_start_import' do + context 'when project is not a github import' do + it 'does not emit importer metrics' do + expect(subject).not_to receive(:track_usage_event) + + subject.track_start_import + end + end + + context 'when project is a github import' do + before do + project.import_type = 'github' + end + + it 'emits importer metrics' do + expect(subject).to receive(:track_usage_event).with(:github_import_project_start, project.id) + + subject.track_start_import + end + end + end + + describe '#track_failed_import' do + context 'when project is not a github import' do + it 'does not emit importer metrics' do + expect(subject).not_to receive(:track_usage_event) + + subject.track_failed_import + end + end + + context 'when project is a github import' do + before do + project.import_type = 'github' + end + + it 'emits importer metrics' do + expect(subject).to receive(:track_usage_event).with(:github_import_project_failure, project.id) + + subject.track_failed_import + end + end + end + + describe '#track_finished_import' do before do - allow(Gitlab::Metrics).to receive(:counter) { counter } allow(Gitlab::Metrics).to receive(:histogram) { histogram } - allow(counter).to receive(:increment) - allow(counter).to receive(:observe) end it 'emits importer metrics' do @@ -32,9 +79,56 @@ RSpec.describe Gitlab::Import::Metrics do ) expect(counter).to receive(:increment) - expect(histogram).to receive(:observe).with({ importer: :test_importer }, anything) subject.track_finished_import + + expect(subject.duration).not_to be_nil + end + + context 'when project is not a github import' do + it 'does not emit importer metrics' do + expect(subject).not_to receive(:track_usage_event) + + subject.track_finished_import + + expect(histogram).to have_received(:observe).with({ importer: :test_importer }, anything) + end + end + + context 'when project is a github import' do + before do + project.import_type = 'github' + end + + it 'emits importer metrics' do + expect(subject).to receive(:track_usage_event).with(:github_import_project_success, project.id) + + subject.track_finished_import + + expect(histogram).to have_received(:observe).with({ project: project.full_path }, anything) + end + end + end + + describe '#issues_counter' do + it 'creates a counter for issues' do + expect(Gitlab::Metrics).to receive(:counter).with( + :test_importer_imported_issues_total, + 'The number of imported issues' + ) + + subject.issues_counter + end + end + + describe '#merge_requests_counter' do + it 'creates a counter for issues' do + expect(Gitlab::Metrics).to receive(:counter).with( + :test_importer_imported_merge_requests_total, + 'The number of imported merge (pull) requests' + ) + + subject.merge_requests_counter end end end diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index 614aa55c3c5..10f0e687077 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -59,6 +59,7 @@ issues: - requirement - incident_management_issuable_escalation_status - pending_escalations +- customer_relations_contacts work_item_type: - issues events: @@ -272,6 +273,8 @@ ci_pipelines: - dast_profiles_pipeline - dast_site_profile - dast_site_profiles_pipeline +- package_build_infos +- package_file_build_infos ci_refs: - project - ci_pipelines @@ -322,7 +325,6 @@ integrations: - jira_tracker_data - zentao_tracker_data - issue_tracker_data -- open_project_tracker_data hooks: - project - web_hook_logs @@ -354,10 +356,7 @@ container_repositories: - name project: - external_status_checks -- taggings - base_tags -- topic_taggings -- topics_acts_as_taggable - project_topics - topics - chat_services @@ -593,6 +592,7 @@ project: - pending_builds - security_scans - ci_feature_usages +- bulk_import_exports award_emoji: - awardable - user diff --git a/spec/lib/gitlab/import_export/attributes_permitter_spec.rb b/spec/lib/gitlab/import_export/attributes_permitter_spec.rb index 36a831a785c..2b974f8985d 100644 --- a/spec/lib/gitlab/import_export/attributes_permitter_spec.rb +++ b/spec/lib/gitlab/import_export/attributes_permitter_spec.rb @@ -83,14 +83,22 @@ RSpec.describe Gitlab::ImportExport::AttributesPermitter do where(:relation_name, :permitted_attributes_defined) do :user | false :author | false - :ci_cd_settings | false - :issuable_sla | false - :push_rule | false + :ci_cd_settings | true :metrics_setting | true :project_badges | true :pipeline_schedules | true :error_tracking_setting | true :auto_devops | true + :boards | true + :custom_attributes | true + :labels | true + :protected_branches | true + :protected_tags | true + :create_access_levels | true + :merge_access_levels | true + :push_access_levels | true + :releases | true + :links | true end with_them do @@ -99,47 +107,11 @@ RSpec.describe Gitlab::ImportExport::AttributesPermitter do end describe 'included_attributes for Project' do - let(:prohibited_attributes) { %i[remote_url my_attributes my_ids token my_id test] } - subject { described_class.new } Gitlab::ImportExport::Config.new.to_h[:included_attributes].each do |relation_sym, permitted_attributes| context "for #{relation_sym}" do - let(:import_export_config) { Gitlab::ImportExport::Config.new.to_h } - let(:project_relation_factory) { Gitlab::ImportExport::Project::RelationFactory } - - let(:relation_hash) { (permitted_attributes + prohibited_attributes).map(&:to_s).zip([]).to_h } - let(:relation_name) { project_relation_factory.overrides[relation_sym]&.to_sym || relation_sym } - let(:relation_class) { project_relation_factory.relation_class(relation_name) } - let(:excluded_keys) { import_export_config.dig(:excluded_keys, relation_sym) || [] } - - let(:cleaned_hash) do - Gitlab::ImportExport::AttributeCleaner.new( - relation_hash: relation_hash, - relation_class: relation_class, - excluded_keys: excluded_keys - ).clean - end - - let(:permitted_hash) { subject.permit(relation_sym, relation_hash) } - - if described_class.new.permitted_attributes_defined?(relation_sym) - it 'contains only attributes that are defined as permitted in the import/export config' do - expect(permitted_hash.keys).to contain_exactly(*permitted_attributes.map(&:to_s)) - end - - it 'does not contain attributes that would be cleaned with AttributeCleaner' do - expect(cleaned_hash.keys).to include(*permitted_hash.keys) - end - - it 'does not contain prohibited attributes that are not related to given relation' do - expect(permitted_hash.keys).not_to include(*prohibited_attributes.map(&:to_s)) - end - else - it 'is disabled' do - expect(subject).not_to be_permitted_attributes_defined(relation_sym) - end - end + it_behaves_like 'a permitted attribute', relation_sym, permitted_attributes end end end diff --git a/spec/lib/gitlab/import_export/command_line_util_spec.rb b/spec/lib/gitlab/import_export/command_line_util_spec.rb index 39a10f87083..59c4e1083ae 100644 --- a/spec/lib/gitlab/import_export/command_line_util_spec.rb +++ b/spec/lib/gitlab/import_export/command_line_util_spec.rb @@ -8,6 +8,7 @@ RSpec.describe Gitlab::ImportExport::CommandLineUtil do let(:path) { "#{Dir.tmpdir}/symlink_test" } let(:archive) { 'spec/fixtures/symlink_export.tar.gz' } let(:shared) { Gitlab::ImportExport::Shared.new(nil) } + let(:tmpdir) { Dir.mktmpdir } subject do Class.new do @@ -26,6 +27,7 @@ RSpec.describe Gitlab::ImportExport::CommandLineUtil do after do FileUtils.rm_rf(path) + FileUtils.remove_entry(tmpdir) end it 'has the right mask for project.json' do @@ -55,7 +57,6 @@ RSpec.describe Gitlab::ImportExport::CommandLineUtil do describe '#gunzip' do it 'decompresses specified file' do - tmpdir = Dir.mktmpdir filename = 'labels.ndjson.gz' gz_filepath = "spec/fixtures/bulk_imports/gz/#{filename}" FileUtils.copy_file(gz_filepath, File.join(tmpdir, filename)) @@ -63,8 +64,6 @@ RSpec.describe Gitlab::ImportExport::CommandLineUtil do subject.gunzip(dir: tmpdir, filename: filename) expect(File.exist?(File.join(tmpdir, 'labels.ndjson'))).to eq(true) - - FileUtils.remove_entry(tmpdir) end context 'when exception occurs' do @@ -73,4 +72,33 @@ RSpec.describe Gitlab::ImportExport::CommandLineUtil do end end end + + describe '#tar_cf' do + let(:archive_dir) { Dir.mktmpdir } + + after do + FileUtils.remove_entry(archive_dir) + end + + it 'archives a folder without compression' do + archive_file = File.join(archive_dir, 'archive.tar') + + result = subject.tar_cf(archive: archive_file, dir: tmpdir) + + expect(result).to eq(true) + expect(File.exist?(archive_file)).to eq(true) + end + + context 'when something goes wrong' do + it 'raises an error' do + expect(Gitlab::Popen).to receive(:popen).and_return(['Error', 1]) + + klass = Class.new do + include Gitlab::ImportExport::CommandLineUtil + end.new + + expect { klass.tar_cf(archive: 'test', dir: 'test') }.to raise_error(Gitlab::ImportExport::Error, 'System call failed') + end + end + end end diff --git a/spec/lib/gitlab/import_export/json/streaming_serializer_spec.rb b/spec/lib/gitlab/import_export/json/streaming_serializer_spec.rb index 9e30564b437..d69d775fffb 100644 --- a/spec/lib/gitlab/import_export/json/streaming_serializer_spec.rb +++ b/spec/lib/gitlab/import_export/json/streaming_serializer_spec.rb @@ -115,7 +115,7 @@ RSpec.describe Gitlab::ImportExport::Json::StreamingSerializer do end it 'orders exported issues by custom column(relative_position)' do - expected_issues = exportable.issues.order_relative_position_desc.order(id: :desc).map(&:to_json) + expected_issues = exportable.issues.reorder(::Gitlab::Database.nulls_first_order('relative_position', 'DESC')).order(id: :desc).map(&:to_json) expect(json_writer).to receive(:write_relation_array).with(exportable_path, :issues, expected_issues) @@ -163,21 +163,10 @@ RSpec.describe Gitlab::ImportExport::Json::StreamingSerializer do stub_feature_flags(load_balancing_for_export_workers: true) end - context 'when enabled', :db_load_balancing do - it 'reads from replica' do - expect(Gitlab::Database::LoadBalancing::Session.current).to receive(:use_replicas_for_read_queries).and_call_original + it 'reads from replica' do + expect(Gitlab::Database::LoadBalancing::Session.current).to receive(:use_replicas_for_read_queries).and_call_original - subject.execute - end - end - - context 'when disabled' do - it 'reads from primary' do - allow(Gitlab::Database::LoadBalancing).to receive(:enable?).and_return(false) - expect(Gitlab::Database::LoadBalancing::Session.current).not_to receive(:use_replicas_for_read_queries) - - subject.execute - end + subject.execute end end diff --git a/spec/lib/gitlab/import_export/merge_request_parser_spec.rb b/spec/lib/gitlab/import_export/merge_request_parser_spec.rb index c558c12f581..550cefea805 100644 --- a/spec/lib/gitlab/import_export/merge_request_parser_spec.rb +++ b/spec/lib/gitlab/import_export/merge_request_parser_spec.rb @@ -13,9 +13,11 @@ RSpec.describe Gitlab::ImportExport::MergeRequestParser do create(:merge_request, source_project: forked_project, target_project: project) end + let(:diff_head_sha) { SecureRandom.hex(20) } + let(:parsed_merge_request) do described_class.new(project, - 'abcd', + diff_head_sha, merge_request, merge_request.as_json).parse! end @@ -34,14 +36,34 @@ RSpec.describe Gitlab::ImportExport::MergeRequestParser do expect(project.repository.branch_exists?(parsed_merge_request.target_branch)).to be true end - it 'parses a MR that has no source branch' do - allow_next_instance_of(described_class) do |instance| - allow(instance).to receive(:branch_exists?).and_call_original - allow(instance).to receive(:branch_exists?).with(merge_request.source_branch).and_return(false) - allow(instance).to receive(:fork_merge_request?).and_return(true) + # Source and target branch are only created when: fork_merge_request + context 'fork merge request' do + before do + allow_next_instance_of(described_class) do |instance| + allow(instance).to receive(:fork_merge_request?).and_return(true) + end + end + + it 'parses a MR that has no source branch' do + allow_next_instance_of(described_class) do |instance| + allow(instance).to receive(:branch_exists?).and_call_original + allow(instance).to receive(:branch_exists?).with(merge_request.source_branch).and_return(false) + end + + expect(parsed_merge_request).to eq(merge_request) end - expect(parsed_merge_request).to eq(merge_request) + it 'parses a MR that is closed' do + merge_request.update!(state: :closed, source_branch: 'new_branch') + + expect(project.repository.branch_exists?(parsed_merge_request.source_branch)).to be false + end + + it 'parses a MR that is merged' do + merge_request.update!(state: :merged, source_branch: 'new_branch') + + expect(project.repository.branch_exists?(parsed_merge_request.source_branch)).to be false + end end context 'when the merge request has diffs' do diff --git a/spec/lib/gitlab/import_export/relation_tree_restorer_spec.rb b/spec/lib/gitlab/import_export/relation_tree_restorer_spec.rb index 9325cdac9ed..5e4075c2b59 100644 --- a/spec/lib/gitlab/import_export/relation_tree_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/relation_tree_restorer_spec.rb @@ -30,18 +30,12 @@ RSpec.describe Gitlab::ImportExport::RelationTreeRestorer do subject { relation_tree_restorer.restore } shared_examples 'import project successfully' do - it 'restores project tree' do - expect(subject).to eq(true) - end - describe 'imported project' do - let(:project) { Project.find_by_path('project') } + it 'has the project attributes and relations', :aggregate_failures do + expect(subject).to eq(true) - before do - subject - end + project = Project.find_by_path('project') - it 'has the project attributes and relations' do expect(project.description).to eq('Nisi et repellendus ut enim quo accusamus vel magnam.') expect(project.labels.count).to eq(3) expect(project.boards.count).to eq(1) @@ -86,7 +80,10 @@ RSpec.describe Gitlab::ImportExport::RelationTreeRestorer do end context 'when restoring a project' do - let(:importable) { create(:project, :builds_enabled, :issues_disabled, name: 'project', path: 'project') } + let_it_be(:importable, reload: true) do + create(:project, :builds_enabled, :issues_disabled, name: 'project', path: 'project') + end + let(:importable_name) { 'project' } let(:importable_path) { 'project' } let(:object_builder) { Gitlab::ImportExport::Project::ObjectBuilder } @@ -108,8 +105,10 @@ RSpec.describe Gitlab::ImportExport::RelationTreeRestorer do it_behaves_like 'import project successfully' context 'logging of relations creation' do - let(:group) { create(:group) } - let(:importable) { create(:project, :builds_enabled, :issues_disabled, name: 'project', path: 'project', group: group) } + let_it_be(:group) { create(:group) } + let_it_be(:importable) do + create(:project, :builds_enabled, :issues_disabled, name: 'project', path: 'project', group: group) + end include_examples 'logging of relations creation' end @@ -120,6 +119,18 @@ RSpec.describe Gitlab::ImportExport::RelationTreeRestorer do let(:relation_reader) { Gitlab::ImportExport::Json::NdjsonReader.new(path) } it_behaves_like 'import project successfully' + + context 'when inside a group' do + let_it_be(:group) do + create(:group, :disabled_and_unoverridable) + end + + before do + importable.update!(shared_runners_enabled: false, group: group) + end + + it_behaves_like 'import project successfully' + end end context 'with invalid relations' do @@ -143,9 +154,10 @@ RSpec.describe Gitlab::ImportExport::RelationTreeRestorer do end context 'when restoring a group' do + let_it_be(:group) { create(:group) } + let_it_be(:importable) { create(:group, parent: group) } + let(:path) { 'spec/fixtures/lib/gitlab/import_export/group_exports/no_children/group.json' } - let(:group) { create(:group) } - let(:importable) { create(:group, parent: group) } let(:importable_name) { nil } let(:importable_path) { nil } let(:object_builder) { Gitlab::ImportExport::Group::ObjectBuilder } diff --git a/spec/lib/gitlab/import_export/snippet_repo_restorer_spec.rb b/spec/lib/gitlab/import_export/snippet_repo_restorer_spec.rb index c1661cf02b6..7d719b6028f 100644 --- a/spec/lib/gitlab/import_export/snippet_repo_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/snippet_repo_restorer_spec.rb @@ -29,6 +29,9 @@ RSpec.describe Gitlab::ImportExport::SnippetRepoRestorer do expect(restorer.restore).to be_truthy end.to change { SnippetRepository.count }.by(1) + snippet.repository.expire_method_caches(%i(exists?)) + expect(snippet.repository_exists?).to be_truthy + blob = snippet.repository.blob_at(snippet.default_branch, snippet.file_name) expect(blob).not_to be_nil expect(blob.data).to eq(snippet.content) diff --git a/spec/lib/gitlab/instrumentation/redis_spec.rb b/spec/lib/gitlab/instrumentation/redis_spec.rb index ebc2e92a0dd..900a079cdd2 100644 --- a/spec/lib/gitlab/instrumentation/redis_spec.rb +++ b/spec/lib/gitlab/instrumentation/redis_spec.rb @@ -76,7 +76,9 @@ RSpec.describe Gitlab::Instrumentation::Redis do details_row.merge(storage: 'Cache'), details_row.merge(storage: 'Queues'), details_row.merge(storage: 'SharedState'), - details_row.merge(storage: 'TraceChunks')) + details_row.merge(storage: 'TraceChunks'), + details_row.merge(storage: 'RateLimiting'), + details_row.merge(storage: 'Sessions')) end end end diff --git a/spec/lib/gitlab/instrumentation_helper_spec.rb b/spec/lib/gitlab/instrumentation_helper_spec.rb index 85daf50717c..52d3623c304 100644 --- a/spec/lib/gitlab/instrumentation_helper_spec.rb +++ b/spec/lib/gitlab/instrumentation_helper_spec.rb @@ -107,75 +107,50 @@ RSpec.describe Gitlab::InstrumentationHelper do end end - context 'when load balancing is enabled' do - before do - allow(Gitlab::Database::LoadBalancing).to receive(:enable?).and_return(true) - end - - it 'includes DB counts' do - subject - - expect(payload).to include(db_replica_count: 0, - db_replica_cached_count: 0, - db_primary_count: 0, - db_primary_cached_count: 0, - db_primary_wal_count: 0, - db_replica_wal_count: 0, - db_primary_wal_cached_count: 0, - db_replica_wal_cached_count: 0) - end - - context 'when replica caught up search was made' do - before do - Gitlab::SafeRequestStore[:caught_up_replica_pick_ok] = 2 - Gitlab::SafeRequestStore[:caught_up_replica_pick_fail] = 1 - end + it 'includes DB counts' do + subject - it 'includes related metrics' do - subject + expect(payload).to include(db_replica_count: 0, + db_replica_cached_count: 0, + db_primary_count: 0, + db_primary_cached_count: 0, + db_primary_wal_count: 0, + db_replica_wal_count: 0, + db_primary_wal_cached_count: 0, + db_replica_wal_cached_count: 0) + end - expect(payload).to include(caught_up_replica_pick_ok: 2) - expect(payload).to include(caught_up_replica_pick_fail: 1) - end + context 'when replica caught up search was made' do + before do + Gitlab::SafeRequestStore[:caught_up_replica_pick_ok] = 2 + Gitlab::SafeRequestStore[:caught_up_replica_pick_fail] = 1 end - context 'when only a single counter was updated' do - before do - Gitlab::SafeRequestStore[:caught_up_replica_pick_ok] = 1 - Gitlab::SafeRequestStore[:caught_up_replica_pick_fail] = nil - end - - it 'includes only that counter into logging' do - subject + it 'includes related metrics' do + subject - expect(payload).to include(caught_up_replica_pick_ok: 1) - expect(payload).not_to include(:caught_up_replica_pick_fail) - end + expect(payload).to include(caught_up_replica_pick_ok: 2) + expect(payload).to include(caught_up_replica_pick_fail: 1) end end - context 'when load balancing is disabled' do + context 'when only a single counter was updated' do before do - allow(Gitlab::Database::LoadBalancing).to receive(:enable?).and_return(false) + Gitlab::SafeRequestStore[:caught_up_replica_pick_ok] = 1 + Gitlab::SafeRequestStore[:caught_up_replica_pick_fail] = nil end - it 'does not include DB counts' do + it 'includes only that counter into logging' do subject - expect(payload).not_to include(db_replica_count: 0, - db_replica_cached_count: 0, - db_primary_count: 0, - db_primary_cached_count: 0, - db_primary_wal_count: 0, - db_replica_wal_count: 0, - db_primary_wal_cached_count: 0, - db_replica_wal_cached_count: 0) + expect(payload).to include(caught_up_replica_pick_ok: 1) + expect(payload).not_to include(:caught_up_replica_pick_fail) end end end - describe '.queue_duration_for_job' do - where(:enqueued_at, :created_at, :time_now, :expected_duration) do + describe 'duration calculations' do + where(:end_time, :start_time, :time_now, :expected_duration) do "2019-06-01T00:00:00.000+0000" | nil | "2019-06-01T02:00:00.000+0000" | 2.hours.to_f "2019-06-01T02:00:00.000+0000" | nil | "2019-06-01T02:00:00.001+0000" | 0.001 "2019-06-01T02:00:00.000+0000" | "2019-05-01T02:00:00.000+0000" | "2019-06-01T02:00:01.000+0000" | 1 @@ -189,15 +164,29 @@ RSpec.describe Gitlab::InstrumentationHelper do 0 | nil | "2019-10-23T12:13:16.000+0200" | nil -1 | nil | "2019-10-23T12:13:16.000+0200" | nil "2019-06-01T02:00:00.000+0000" | nil | "2019-06-01T00:00:00.000+0000" | 0 - Time.at(1571999233) | nil | "2019-10-25T12:29:16.000+0200" | 123 + Time.at(1571999233).utc | nil | "2019-10-25T12:29:16.000+0200" | 123 + end + + describe '.queue_duration_for_job' do + with_them do + let(:job) { { 'enqueued_at' => end_time, 'created_at' => start_time } } + + it "returns the correct duration" do + travel_to(Time.iso8601(time_now)) do + expect(described_class.queue_duration_for_job(job)).to eq(expected_duration) + end + end + end end - with_them do - let(:job) { { 'enqueued_at' => enqueued_at, 'created_at' => created_at } } + describe '.enqueue_latency_for_scheduled_job' do + with_them do + let(:job) { { 'enqueued_at' => end_time, 'scheduled_at' => start_time } } - it "returns the correct duration" do - Timecop.freeze(Time.iso8601(time_now)) do - expect(described_class.queue_duration_for_job(job)).to eq(expected_duration) + it "returns the correct duration" do + travel_to(Time.iso8601(time_now)) do + expect(described_class.enqueue_latency_for_scheduled_job(job)).to eq(expected_duration) + end end end end diff --git a/spec/lib/gitlab/kas_spec.rb b/spec/lib/gitlab/kas_spec.rb index 17d038ed16c..0fbb5f31210 100644 --- a/spec/lib/gitlab/kas_spec.rb +++ b/spec/lib/gitlab/kas_spec.rb @@ -70,30 +70,44 @@ RSpec.describe Gitlab::Kas do stub_config(gitlab_kas: { external_url: external_url }) end + let(:external_url) { 'xyz' } + subject { described_class.tunnel_url } - context 'external_url uses wss://' do - let(:external_url) { 'wss://kas.gitlab.example.com' } + context 'with a gitlab_kas.external_k8s_proxy_url setting' do + let(:external_k8s_proxy_url) { 'abc' } + + before do + stub_config(gitlab_kas: { external_k8s_proxy_url: external_k8s_proxy_url }) + end - it { is_expected.to eq('https://kas.gitlab.example.com/k8s-proxy') } + it { is_expected.to eq(external_k8s_proxy_url) } end - context 'external_url uses ws://' do - let(:external_url) { 'ws://kas.gitlab.example.com' } + context 'without a gitlab_kas.external_k8s_proxy_url setting' do + context 'external_url uses wss://' do + let(:external_url) { 'wss://kas.gitlab.example.com' } - it { is_expected.to eq('http://kas.gitlab.example.com/k8s-proxy') } - end + it { is_expected.to eq('https://kas.gitlab.example.com/k8s-proxy') } + end - context 'external_url uses grpcs://' do - let(:external_url) { 'grpcs://kas.gitlab.example.com' } + context 'external_url uses ws://' do + let(:external_url) { 'ws://kas.gitlab.example.com' } - it { is_expected.to eq('https://kas.gitlab.example.com/k8s-proxy') } - end + it { is_expected.to eq('http://kas.gitlab.example.com/k8s-proxy') } + end + + context 'external_url uses grpcs://' do + let(:external_url) { 'grpcs://kas.gitlab.example.com' } - context 'external_url uses grpc://' do - let(:external_url) { 'grpc://kas.gitlab.example.com' } + it { is_expected.to eq('https://kas.gitlab.example.com/k8s-proxy') } + end + + context 'external_url uses grpc://' do + let(:external_url) { 'grpc://kas.gitlab.example.com' } - it { is_expected.to eq('http://kas.gitlab.example.com/k8s-proxy') } + it { is_expected.to eq('http://kas.gitlab.example.com/k8s-proxy') } + end end end diff --git a/spec/lib/gitlab/mail_room/mail_room_spec.rb b/spec/lib/gitlab/mail_room/mail_room_spec.rb index a42da4ad3e0..0bd1a27c65e 100644 --- a/spec/lib/gitlab/mail_room/mail_room_spec.rb +++ b/spec/lib/gitlab/mail_room/mail_room_spec.rb @@ -93,7 +93,7 @@ RSpec.describe Gitlab::MailRoom do end describe 'setting up redis settings' do - let(:fake_redis_queues) { double(url: "localhost", sentinels: "yes, them", sentinels?: true) } + let(:fake_redis_queues) { double(url: "localhost", db: 99, sentinels: "yes, them", sentinels?: true) } before do allow(Gitlab::Redis::Queues).to receive(:new).and_return(fake_redis_queues) @@ -103,6 +103,7 @@ RSpec.describe Gitlab::MailRoom do config = described_class.enabled_configs.first expect(config[:redis_url]).to eq('localhost') + expect(config[:redis_db]).to eq(99) expect(config[:sentinels]).to eq('yes, them') end end diff --git a/spec/lib/gitlab/merge_requests/mergeability/check_result_spec.rb b/spec/lib/gitlab/merge_requests/mergeability/check_result_spec.rb new file mode 100644 index 00000000000..4f437e57600 --- /dev/null +++ b/spec/lib/gitlab/merge_requests/mergeability/check_result_spec.rb @@ -0,0 +1,140 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::MergeRequests::Mergeability::CheckResult do + subject(:check_result) { described_class } + + let(:time) { Time.current } + + around do |example| + freeze_time do + example.run + end + end + + describe '.default_payload' do + it 'returns the expected defaults' do + expect(check_result.default_payload).to eq({ last_run_at: time }) + end + end + + describe '.success' do + subject(:success) { check_result.success(payload: payload) } + + let(:payload) { {} } + + it 'creates a success result' do + expect(success.status).to eq described_class::SUCCESS_STATUS + end + + it 'uses the default payload' do + expect(success.payload).to eq described_class.default_payload + end + + context 'when given a payload' do + let(:payload) { { last_run_at: time + 1.day, test: 'test' } } + + it 'uses the payload passed' do + expect(success.payload).to eq payload + end + end + end + + describe '.failed' do + subject(:failed) { check_result.failed(payload: payload) } + + let(:payload) { {} } + + it 'creates a failure result' do + expect(failed.status).to eq described_class::FAILED_STATUS + end + + it 'uses the default payload' do + expect(failed.payload).to eq described_class.default_payload + end + + context 'when given a payload' do + let(:payload) { { last_run_at: time + 1.day, test: 'test' } } + + it 'uses the payload passed' do + expect(failed.payload).to eq payload + end + end + end + + describe '.from_hash' do + subject(:from_hash) { described_class.from_hash(hash) } + + let(:status) { described_class::SUCCESS_STATUS } + let(:payload) { { test: 'test' } } + let(:hash) do + { + status: status, + payload: payload + } + end + + it 'returns the expected status and payload' do + expect(from_hash.status).to eq status + expect(from_hash.payload).to eq payload + end + end + + describe '#to_hash' do + subject(:to_hash) { described_class.new(**hash).to_hash } + + let(:status) { described_class::SUCCESS_STATUS } + let(:payload) { { test: 'test' } } + let(:hash) do + { + status: status, + payload: payload + } + end + + it 'returns the expected hash' do + expect(to_hash).to eq hash + end + end + + describe '#failed?' do + subject(:failed) { described_class.new(status: status).failed? } + + context 'when it has failed' do + let(:status) { described_class::FAILED_STATUS } + + it 'returns true' do + expect(failed).to eq true + end + end + + context 'when it has succeeded' do + let(:status) { described_class::SUCCESS_STATUS } + + it 'returns false' do + expect(failed).to eq false + end + end + end + + describe '#success?' do + subject(:success) { described_class.new(status: status).success? } + + context 'when it has failed' do + let(:status) { described_class::FAILED_STATUS } + + it 'returns false' do + expect(success).to eq false + end + end + + context 'when it has succeeded' do + let(:status) { described_class::SUCCESS_STATUS } + + it 'returns true' do + expect(success).to eq true + end + end + end +end diff --git a/spec/lib/gitlab/merge_requests/mergeability/redis_interface_spec.rb b/spec/lib/gitlab/merge_requests/mergeability/redis_interface_spec.rb new file mode 100644 index 00000000000..e5475d04d86 --- /dev/null +++ b/spec/lib/gitlab/merge_requests/mergeability/redis_interface_spec.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::MergeRequests::Mergeability::RedisInterface, :clean_gitlab_redis_shared_state do + subject(:redis_interface) { described_class.new } + + let(:merge_check) { double(cache_key: '13') } + let(:result_hash) { { 'test' => 'test' } } + let(:expected_key) { "#{merge_check.cache_key}:#{described_class::VERSION}" } + + describe '#save_check' do + it 'saves the hash' do + expect(Gitlab::Redis::SharedState.with { |redis| redis.get(expected_key) }).to be_nil + + redis_interface.save_check(merge_check: merge_check, result_hash: result_hash) + + expect(Gitlab::Redis::SharedState.with { |redis| redis.get(expected_key) }).to eq result_hash.to_json + end + end + + describe '#retrieve_check' do + it 'returns the hash' do + Gitlab::Redis::SharedState.with { |redis| redis.set(expected_key, result_hash.to_json) } + + expect(redis_interface.retrieve_check(merge_check: merge_check)).to eq result_hash + end + end +end diff --git a/spec/lib/gitlab/merge_requests/mergeability/results_store_spec.rb b/spec/lib/gitlab/merge_requests/mergeability/results_store_spec.rb new file mode 100644 index 00000000000..d376dcb5b18 --- /dev/null +++ b/spec/lib/gitlab/merge_requests/mergeability/results_store_spec.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::MergeRequests::Mergeability::ResultsStore do + subject(:results_store) { described_class.new(merge_request: merge_request, interface: interface) } + + let(:merge_check) { double } + let(:interface) { double } + let(:merge_request) { double } + + describe '#read' do + it 'calls #retrieve on the interface' do + expect(interface).to receive(:retrieve_check).with(merge_check: merge_check) + + results_store.read(merge_check: merge_check) + end + end + + describe '#write' do + let(:result_hash) { double } + + it 'calls #save_check on the interface' do + expect(interface).to receive(:save_check).with(merge_check: merge_check, result_hash: result_hash) + + results_store.write(merge_check: merge_check, result_hash: result_hash) + end + end +end diff --git a/spec/lib/gitlab/metrics/exporter/web_exporter_spec.rb b/spec/lib/gitlab/metrics/exporter/web_exporter_spec.rb index ce98c807e2e..9deaecbf41b 100644 --- a/spec/lib/gitlab/metrics/exporter/web_exporter_spec.rb +++ b/spec/lib/gitlab/metrics/exporter/web_exporter_spec.rb @@ -30,6 +30,15 @@ RSpec.describe Gitlab::Metrics::Exporter::WebExporter do expect(readiness_probe.json).to include(status: 'ok') expect(readiness_probe.json).to include('web_exporter' => [{ 'status': 'ok' }]) end + + it 'initializes request metrics', :prometheus do + expect(Gitlab::Metrics::RailsSlis).to receive(:initialize_request_slis_if_needed!).and_call_original + + http = Net::HTTP.new(exporter.server.config[:BindAddress], exporter.server.config[:Port]) + response = http.request(Net::HTTP::Get.new('/metrics')) + + expect(response.body).to include('gitlab_sli:rails_request_apdex') + end end describe '#mark_as_not_running!' do diff --git a/spec/lib/gitlab/metrics/instrumentation_spec.rb b/spec/lib/gitlab/metrics/instrumentation_spec.rb deleted file mode 100644 index b15e06a0861..00000000000 --- a/spec/lib/gitlab/metrics/instrumentation_spec.rb +++ /dev/null @@ -1,342 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Metrics::Instrumentation do - let(:env) { {} } - let(:transaction) { Gitlab::Metrics::WebTransaction.new(env) } - - before do - @dummy = Class.new do - def self.foo(text = 'foo') - text - end - - def self.wat(text = 'wat') - text - end - private_class_method :wat - - class << self - def buzz(text = 'buzz') - text - end - private :buzz - - def flaky(text = 'flaky') - text - end - protected :flaky - end - - def bar(text = 'bar') - text - end - - def wadus(text = 'wadus') - text - end - private :wadus - - def chaf(text = 'chaf') - text - end - protected :chaf - end - - allow(@dummy).to receive(:name).and_return('Dummy') - end - - describe '.series' do - it 'returns a String' do - expect(described_class.series).to be_an_instance_of(String) - end - end - - describe '.configure' do - it 'yields self' do - described_class.configure do |c| - expect(c).to eq(described_class) - end - end - end - - describe '.instrument_method' do - describe 'with metrics enabled' do - before do - allow(Gitlab::Metrics).to receive(:enabled?).and_return(true) - - described_class.instrument_method(@dummy, :foo) - end - - it 'instruments the Class' do - target = @dummy.singleton_class - - expect(described_class.instrumented?(target)).to eq(true) - end - - it 'defines a proxy method' do - mod = described_class.proxy_module(@dummy.singleton_class) - - expect(mod.method_defined?(:foo)).to eq(true) - end - - it 'calls the instrumented method with the correct arguments' do - expect(@dummy.foo).to eq('foo') - end - - it 'tracks the call duration upon calling the method' do - allow(Gitlab::Metrics).to receive(:method_call_threshold) - .and_return(0) - - allow(described_class).to receive(:transaction) - .and_return(transaction) - - expect_next_instance_of(Gitlab::Metrics::MethodCall) do |instance| - expect(instance).to receive(:measure) - end - - @dummy.foo - end - - it 'does not track method calls below a given duration threshold' do - allow(Gitlab::Metrics).to receive(:method_call_threshold) - .and_return(100) - - expect(transaction).not_to receive(:add_metric) - - @dummy.foo - end - - it 'generates a method with the correct arity when using methods without arguments' do - dummy = Class.new do - def self.test; end - end - - described_class.instrument_method(dummy, :test) - - expect(dummy.method(:test).arity).to eq(0) - end - - describe 'when a module is instrumented multiple times' do - it 'calls the instrumented method with the correct arguments' do - described_class.instrument_method(@dummy, :foo) - - expect(@dummy.foo).to eq('foo') - end - end - end - - describe 'with metrics disabled' do - before do - allow(Gitlab::Metrics).to receive(:enabled?).and_return(false) - end - - it 'does not instrument the method' do - described_class.instrument_method(@dummy, :foo) - - target = @dummy.singleton_class - - expect(described_class.instrumented?(target)).to eq(false) - end - end - end - - describe '.instrument_instance_method' do - describe 'with metrics enabled' do - before do - allow(Gitlab::Metrics).to receive(:enabled?).and_return(true) - - described_class - .instrument_instance_method(@dummy, :bar) - end - - it 'instruments instances of the Class' do - expect(described_class.instrumented?(@dummy)).to eq(true) - end - - it 'defines a proxy method' do - mod = described_class.proxy_module(@dummy) - - expect(mod.method_defined?(:bar)).to eq(true) - end - - it 'calls the instrumented method with the correct arguments' do - expect(@dummy.new.bar).to eq('bar') - end - - it 'tracks the call duration upon calling the method' do - allow(Gitlab::Metrics).to receive(:method_call_threshold) - .and_return(0) - - allow(described_class).to receive(:transaction) - .and_return(transaction) - - expect_next_instance_of(Gitlab::Metrics::MethodCall) do |instance| - expect(instance).to receive(:measure) - end - - @dummy.new.bar - end - - it 'does not track method calls below a given duration threshold' do - allow(Gitlab::Metrics).to receive(:method_call_threshold) - .and_return(100) - - expect(transaction).not_to receive(:add_metric) - - @dummy.new.bar - end - end - - describe 'with metrics disabled' do - before do - allow(Gitlab::Metrics).to receive(:enabled?).and_return(false) - end - - it 'does not instrument the method' do - described_class - .instrument_instance_method(@dummy, :bar) - - expect(described_class.instrumented?(@dummy)).to eq(false) - end - end - end - - describe '.instrument_class_hierarchy' do - before do - allow(Gitlab::Metrics).to receive(:enabled?).and_return(true) - - @child1 = Class.new(@dummy) do - def self.child1_foo; end - - def child1_bar; end - end - - @child2 = Class.new(@child1) do - def self.child2_foo; end - - def child2_bar; end - end - end - - it 'recursively instruments a class hierarchy' do - described_class.instrument_class_hierarchy(@dummy) - - expect(described_class.instrumented?(@child1.singleton_class)).to eq(true) - expect(described_class.instrumented?(@child2.singleton_class)).to eq(true) - - expect(described_class.instrumented?(@child1)).to eq(true) - expect(described_class.instrumented?(@child2)).to eq(true) - end - - it 'does not instrument the root module' do - described_class.instrument_class_hierarchy(@dummy) - - expect(described_class.instrumented?(@dummy)).to eq(false) - end - end - - describe '.instrument_methods' do - before do - allow(Gitlab::Metrics).to receive(:enabled?).and_return(true) - end - - it 'instruments all public class methods' do - described_class.instrument_methods(@dummy) - - expect(described_class.instrumented?(@dummy.singleton_class)).to eq(true) - expect(@dummy.method(:foo).source_location.first).to match(/instrumentation\.rb/) - expect(@dummy.public_methods).to include(:foo) - end - - it 'instruments all protected class methods' do - described_class.instrument_methods(@dummy) - - expect(described_class.instrumented?(@dummy.singleton_class)).to eq(true) - expect(@dummy.method(:flaky).source_location.first).to match(/instrumentation\.rb/) - expect(@dummy.protected_methods).to include(:flaky) - end - - it 'instruments all private class methods' do - described_class.instrument_methods(@dummy) - - expect(described_class.instrumented?(@dummy.singleton_class)).to eq(true) - expect(@dummy.method(:buzz).source_location.first).to match(/instrumentation\.rb/) - expect(@dummy.private_methods).to include(:buzz) - expect(@dummy.private_methods).to include(:wat) - end - - it 'only instruments methods directly defined in the module' do - mod = Module.new do - def kittens - end - end - - @dummy.extend(mod) - - described_class.instrument_methods(@dummy) - - expect(@dummy).not_to respond_to(:_original_kittens) - end - - it 'can take a block to determine if a method should be instrumented' do - described_class.instrument_methods(@dummy) do - false - end - - expect(@dummy).not_to respond_to(:_original_foo) - end - end - - describe '.instrument_instance_methods' do - before do - allow(Gitlab::Metrics).to receive(:enabled?).and_return(true) - end - - it 'instruments all public instance methods' do - described_class.instrument_instance_methods(@dummy) - - expect(described_class.instrumented?(@dummy)).to eq(true) - expect(@dummy.new.method(:bar).source_location.first).to match(/instrumentation\.rb/) - expect(@dummy.public_instance_methods).to include(:bar) - end - - it 'instruments all protected instance methods' do - described_class.instrument_instance_methods(@dummy) - - expect(described_class.instrumented?(@dummy)).to eq(true) - expect(@dummy.new.method(:chaf).source_location.first).to match(/instrumentation\.rb/) - expect(@dummy.protected_instance_methods).to include(:chaf) - end - - it 'instruments all private instance methods' do - described_class.instrument_instance_methods(@dummy) - - expect(described_class.instrumented?(@dummy)).to eq(true) - expect(@dummy.new.method(:wadus).source_location.first).to match(/instrumentation\.rb/) - expect(@dummy.private_instance_methods).to include(:wadus) - end - - it 'only instruments methods directly defined in the module' do - mod = Module.new do - def kittens - end - end - - @dummy.include(mod) - - described_class.instrument_instance_methods(@dummy) - - expect(@dummy.new.method(:kittens).source_location.first).not_to match(/instrumentation\.rb/) - end - - it 'can take a block to determine if a method should be instrumented' do - described_class.instrument_instance_methods(@dummy) do - false - end - - expect(@dummy.new.method(:bar).source_location.first).not_to match(/instrumentation\.rb/) - end - end -end diff --git a/spec/lib/gitlab/metrics/rails_slis_spec.rb b/spec/lib/gitlab/metrics/rails_slis_spec.rb new file mode 100644 index 00000000000..16fcb9d46a2 --- /dev/null +++ b/spec/lib/gitlab/metrics/rails_slis_spec.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true +require 'spec_helper' + +RSpec.describe Gitlab::Metrics::RailsSlis do + # Limit what routes we'll initialize so we don't have to load the entire thing + before do + api_route = API::API.routes.find do |route| + API::Base.endpoint_id_for_route(route) == "GET /api/:version/version" + end + + allow(Gitlab::RequestEndpoints).to receive(:all_api_endpoints).and_return([api_route]) + allow(Gitlab::RequestEndpoints).to receive(:all_controller_actions).and_return([[ProjectsController, 'show']]) + end + + describe '.initialize_request_slis_if_needed!' do + it "initializes the SLI for all possible endpoints if they weren't" do + possible_labels = [ + { + endpoint_id: "GET /api/:version/version", + feature_category: :not_owned + }, + { + endpoint_id: "ProjectsController#show", + feature_category: :projects + } + ] + + expect(Gitlab::Metrics::Sli).to receive(:initialized?).with(:rails_request_apdex) { false } + expect(Gitlab::Metrics::Sli).to receive(:initialize_sli).with(:rails_request_apdex, array_including(*possible_labels)).and_call_original + + described_class.initialize_request_slis_if_needed! + end + + it 'does not initialize the SLI if they were initialized already' do + expect(Gitlab::Metrics::Sli).to receive(:initialized?).with(:rails_request_apdex) { true } + expect(Gitlab::Metrics::Sli).not_to receive(:initialize_sli) + + described_class.initialize_request_slis_if_needed! + end + + it 'does not initialize anything if the feature flag is disabled' do + stub_feature_flags(request_apdex_counters: false) + + expect(Gitlab::Metrics::Sli).not_to receive(:initialize_sli) + expect(Gitlab::Metrics::Sli).not_to receive(:initialized?) + + described_class.initialize_request_slis_if_needed! + end + end + + describe '.request_apdex' do + it 'returns the initialized request apdex SLI object' do + described_class.initialize_request_slis_if_needed! + + expect(described_class.request_apdex).to be_initialized + end + end +end diff --git a/spec/lib/gitlab/metrics/requests_rack_middleware_spec.rb b/spec/lib/gitlab/metrics/requests_rack_middleware_spec.rb index 9d5c4bdf9e2..5870f9a8f68 100644 --- a/spec/lib/gitlab/metrics/requests_rack_middleware_spec.rb +++ b/spec/lib/gitlab/metrics/requests_rack_middleware_spec.rb @@ -36,6 +36,7 @@ RSpec.describe Gitlab::Metrics::RequestsRackMiddleware, :aggregate_failures do it 'tracks request count and duration' do expect(described_class).to receive_message_chain(:http_requests_total, :increment).with(method: 'get', status: '200', feature_category: 'unknown') expect(described_class).to receive_message_chain(:http_request_duration_seconds, :observe).with({ method: 'get' }, a_positive_execution_time) + expect(Gitlab::Metrics::RailsSlis.request_apdex).to receive(:increment).with(labels: { feature_category: 'unknown', endpoint_id: 'unknown' }, success: true) subject.call(env) end @@ -70,7 +71,7 @@ RSpec.describe Gitlab::Metrics::RequestsRackMiddleware, :aggregate_failures do expect(described_class).not_to receive(:http_health_requests_total) expect(described_class) .to receive_message_chain(:http_request_duration_seconds, :observe) - .with({ method: 'get' }, a_positive_execution_time) + .with({ method: 'get' }, a_positive_execution_time) subject.call(env) end @@ -82,9 +83,10 @@ RSpec.describe Gitlab::Metrics::RequestsRackMiddleware, :aggregate_failures do context '@app.call returns an error code' do let(:status) { '500' } - it 'tracks count but not duration' do + it 'tracks count but not duration or apdex' do expect(described_class).to receive_message_chain(:http_requests_total, :increment).with(method: 'get', status: '500', feature_category: 'unknown') expect(described_class).not_to receive(:http_request_duration_seconds) + expect(Gitlab::Metrics::RailsSlis).not_to receive(:request_apdex) subject.call(env) end @@ -104,20 +106,23 @@ RSpec.describe Gitlab::Metrics::RequestsRackMiddleware, :aggregate_failures do expect(described_class).to receive_message_chain(:rack_uncaught_errors_count, :increment) expect(described_class).to receive_message_chain(:http_requests_total, :increment).with(method: 'get', status: 'undefined', feature_category: 'unknown') expect(described_class.http_request_duration_seconds).not_to receive(:observe) + expect(Gitlab::Metrics::RailsSlis).not_to receive(:request_apdex) expect { subject.call(env) }.to raise_error(StandardError) end end - context 'feature category header' do - context 'when a feature category context is present' do + context 'application context' do + context 'when a context is present' do before do - ::Gitlab::ApplicationContext.push(feature_category: 'issue_tracking') + ::Gitlab::ApplicationContext.push(feature_category: 'issue_tracking', caller_id: 'IssuesController#show') end - it 'adds the feature category to the labels for http_requests_total' do + it 'adds the feature category to the labels for required metrics' do expect(described_class).to receive_message_chain(:http_requests_total, :increment).with(method: 'get', status: '200', feature_category: 'issue_tracking') expect(described_class).not_to receive(:http_health_requests_total) + expect(Gitlab::Metrics::RailsSlis.request_apdex) + .to receive(:increment).with(labels: { feature_category: 'issue_tracking', endpoint_id: 'IssuesController#show' }, success: true) subject.call(env) end @@ -127,6 +132,7 @@ RSpec.describe Gitlab::Metrics::RequestsRackMiddleware, :aggregate_failures do expect(described_class).to receive_message_chain(:http_health_requests_total, :increment).with(method: 'get', status: '200') expect(described_class).not_to receive(:http_requests_total) + expect(Gitlab::Metrics::RailsSlis).not_to receive(:request_apdex) subject.call(env) end @@ -140,19 +146,180 @@ RSpec.describe Gitlab::Metrics::RequestsRackMiddleware, :aggregate_failures do it 'adds the feature category to the labels for http_requests_total' do expect(described_class).to receive_message_chain(:http_requests_total, :increment).with(method: 'get', status: 'undefined', feature_category: 'issue_tracking') + expect(Gitlab::Metrics::RailsSlis).not_to receive(:request_apdex) expect { subject.call(env) }.to raise_error(StandardError) end end - context 'when the feature category context is not available' do - it 'sets the feature category to unknown' do + context 'when the context is not available' do + it 'sets the required labels to unknown' do expect(described_class).to receive_message_chain(:http_requests_total, :increment).with(method: 'get', status: '200', feature_category: 'unknown') expect(described_class).not_to receive(:http_health_requests_total) + expect(Gitlab::Metrics::RailsSlis.request_apdex).to receive(:increment).with(labels: { feature_category: 'unknown', endpoint_id: 'unknown' }, success: true) subject.call(env) end end + + context 'SLI satisfactory' do + where(:request_urgency_name, :duration, :success) do + [ + [:high, 0.1, true], + [:high, 0.25, false], + [:high, 0.3, false], + [:medium, 0.3, true], + [:medium, 0.5, false], + [:medium, 0.6, false], + [:default, 0.6, true], + [:default, 1.0, false], + [:default, 1.2, false], + [:low, 4.5, true], + [:low, 5.0, false], + [:low, 6, false] + ] + end + + with_them do + context 'Grape API handler having expected duration setup' do + let(:api_handler) do + request_urgency = request_urgency_name + Class.new(::API::Base) do + feature_category :hello_world, ['/projects/:id/archive'] + urgency request_urgency, ['/projects/:id/archive'] + end + end + + let(:endpoint) do + route = double(:route, request_method: 'GET', path: '/:version/projects/:id/archive(.:format)') + double(:endpoint, route: route, + options: { for: api_handler, path: [":id/archive"] }, + namespace: "/projects") + end + + let(:env) { { 'api.endpoint' => endpoint, 'REQUEST_METHOD' => 'GET' } } + + before do + ::Gitlab::ApplicationContext.push(feature_category: 'hello_world', caller_id: 'GET /projects/:id/archive') + allow(Gitlab::Metrics::System).to receive(:monotonic_time).and_return(100, 100 + duration) + end + + it "captures SLI metrics" do + expect(Gitlab::Metrics::RailsSlis.request_apdex).to receive(:increment).with( + labels: { feature_category: 'hello_world', endpoint_id: 'GET /projects/:id/archive' }, + success: success + ) + subject.call(env) + end + end + + context 'Rails controller having expected duration setup' do + let(:controller) do + request_urgency = request_urgency_name + Class.new(ApplicationController) do + feature_category :hello_world, [:index, :show] + urgency request_urgency, [:index, :show] + end + end + + let(:env) do + controller_instance = controller.new + controller_instance.action_name = :index + { 'action_controller.instance' => controller_instance, 'REQUEST_METHOD' => 'GET' } + end + + before do + ::Gitlab::ApplicationContext.push(feature_category: 'hello_world', caller_id: 'AnonymousController#index') + allow(Gitlab::Metrics::System).to receive(:monotonic_time).and_return(100, 100 + duration) + end + + it "captures SLI metrics" do + expect(Gitlab::Metrics::RailsSlis.request_apdex).to receive(:increment).with( + labels: { feature_category: 'hello_world', endpoint_id: 'AnonymousController#index' }, + success: success + ) + subject.call(env) + end + end + end + + context 'Grape API without expected duration' do + let(:endpoint) do + route = double(:route, request_method: 'GET', path: '/:version/projects/:id/archive(.:format)') + double(:endpoint, route: route, + options: { for: api_handler, path: [":id/archive"] }, + namespace: "/projects") + end + + let(:env) { { 'api.endpoint' => endpoint, 'REQUEST_METHOD' => 'GET' } } + + let(:api_handler) { Class.new(::API::Base) } + + it "falls back request's expectation to medium (1 second)" do + allow(Gitlab::Metrics::System).to receive(:monotonic_time).and_return(100, 100.9) + expect(Gitlab::Metrics::RailsSlis.request_apdex).to receive(:increment).with( + labels: { feature_category: 'unknown', endpoint_id: 'unknown' }, + success: true + ) + subject.call(env) + + allow(Gitlab::Metrics::System).to receive(:monotonic_time).and_return(100, 101) + expect(Gitlab::Metrics::RailsSlis.request_apdex).to receive(:increment).with( + labels: { feature_category: 'unknown', endpoint_id: 'unknown' }, + success: false + ) + subject.call(env) + end + end + + context 'Rails controller without expected duration' do + let(:controller) { Class.new(ApplicationController) } + + let(:env) do + controller_instance = controller.new + controller_instance.action_name = :index + { 'action_controller.instance' => controller_instance, 'REQUEST_METHOD' => 'GET' } + end + + it "falls back request's expectation to medium (1 second)" do + allow(Gitlab::Metrics::System).to receive(:monotonic_time).and_return(100, 100.9) + expect(Gitlab::Metrics::RailsSlis.request_apdex).to receive(:increment).with( + labels: { feature_category: 'unknown', endpoint_id: 'unknown' }, + success: true + ) + subject.call(env) + + allow(Gitlab::Metrics::System).to receive(:monotonic_time).and_return(100, 101) + expect(Gitlab::Metrics::RailsSlis.request_apdex).to receive(:increment).with( + labels: { feature_category: 'unknown', endpoint_id: 'unknown' }, + success: false + ) + subject.call(env) + end + end + + context 'An unknown request' do + let(:env) do + { 'REQUEST_METHOD' => 'GET' } + end + + it "falls back request's expectation to medium (1 second)" do + allow(Gitlab::Metrics::System).to receive(:monotonic_time).and_return(100, 100.9) + expect(Gitlab::Metrics::RailsSlis.request_apdex).to receive(:increment).with( + labels: { feature_category: 'unknown', endpoint_id: 'unknown' }, + success: true + ) + subject.call(env) + + allow(Gitlab::Metrics::System).to receive(:monotonic_time).and_return(100, 101) + expect(Gitlab::Metrics::RailsSlis.request_apdex).to receive(:increment).with( + labels: { feature_category: 'unknown', endpoint_id: 'unknown' }, + success: false + ) + subject.call(env) + end + end + end end describe '.initialize_metrics', :prometheus do @@ -181,8 +348,8 @@ RSpec.describe Gitlab::Metrics::RequestsRackMiddleware, :aggregate_failures do end it 'has every label in config/feature_categories.yml' do - defaults = [described_class::FEATURE_CATEGORY_DEFAULT, 'not_owned'] - feature_categories = YAML.load_file(Rails.root.join('config', 'feature_categories.yml')).map(&:strip) + defaults + defaults = [::Gitlab::FeatureCategories::FEATURE_CATEGORY_DEFAULT, 'not_owned'] + feature_categories = Gitlab::FeatureCategories.default.categories + defaults expect(described_class::FEATURE_CATEGORIES_TO_INITIALIZE).to all(be_in(feature_categories)) end diff --git a/spec/lib/gitlab/metrics/sli_spec.rb b/spec/lib/gitlab/metrics/sli_spec.rb new file mode 100644 index 00000000000..8ba4bf29568 --- /dev/null +++ b/spec/lib/gitlab/metrics/sli_spec.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +RSpec.describe Gitlab::Metrics::Sli do + let(:prometheus) { double("prometheus") } + + before do + stub_const("Gitlab::Metrics", prometheus) + end + + describe 'Class methods' do + before do + described_class.instance_variable_set(:@known_slis, nil) + end + + describe '.[]' do + it 'warns about an uninitialized SLI but returns and stores a new one' do + sli = described_class[:bar] + + expect(described_class[:bar]).to be(sli) + end + + it 'returns the same object for multiple accesses' do + sli = described_class.initialize_sli(:huzzah, []) + + 2.times do + expect(described_class[:huzzah]).to be(sli) + end + end + end + + describe '.initialized?' do + before do + fake_total_counter(:boom) + fake_success_counter(:boom) + end + + it 'is true when an SLI was initialized with labels' do + expect { described_class.initialize_sli(:boom, [{ hello: :world }]) } + .to change { described_class.initialized?(:boom) }.from(false).to(true) + end + + it 'is false when an SLI was not initialized with labels' do + expect { described_class.initialize_sli(:boom, []) } + .not_to change { described_class.initialized?(:boom) }.from(false) + end + end + end + + describe '#initialize_counters' do + it 'initializes counters for the passed label combinations' do + counters = [fake_total_counter(:hey), fake_success_counter(:hey)] + + described_class.new(:hey).initialize_counters([{ foo: 'bar' }, { foo: 'baz' }]) + + expect(counters).to all(have_received(:get).with({ foo: 'bar' })) + expect(counters).to all(have_received(:get).with({ foo: 'baz' })) + end + end + + describe "#increment" do + let!(:sli) { described_class.new(:heyo) } + let!(:total_counter) { fake_total_counter(:heyo) } + let!(:success_counter) { fake_success_counter(:heyo) } + + it 'increments both counters for labels successes' do + sli.increment(labels: { hello: "world" }, success: true) + + expect(total_counter).to have_received(:increment).with({ hello: 'world' }) + expect(success_counter).to have_received(:increment).with({ hello: 'world' }) + end + + it 'only increments the total counters for labels when not successful' do + sli.increment(labels: { hello: "world" }, success: false) + + expect(total_counter).to have_received(:increment).with({ hello: 'world' }) + expect(success_counter).not_to have_received(:increment).with({ hello: 'world' }) + end + end + + def fake_prometheus_counter(name) + fake_counter = double("prometheus counter: #{name}") + + allow(fake_counter).to receive(:get) + allow(fake_counter).to receive(:increment) + allow(prometheus).to receive(:counter).with(name.to_sym, anything).and_return(fake_counter) + + fake_counter + end + + def fake_total_counter(name) + fake_prometheus_counter("gitlab_sli:#{name}:total") + end + + def fake_success_counter(name) + fake_prometheus_counter("gitlab_sli:#{name}:success_total") + end +end diff --git a/spec/lib/gitlab/metrics/subscribers/active_record_spec.rb b/spec/lib/gitlab/metrics/subscribers/active_record_spec.rb index 3ffbcbea03c..a8e4f039da4 100644 --- a/spec/lib/gitlab/metrics/subscribers/active_record_spec.rb +++ b/spec/lib/gitlab/metrics/subscribers/active_record_spec.rb @@ -7,7 +7,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::ActiveRecord do let(:env) { {} } let(:subscriber) { described_class.new } - let(:connection) { ActiveRecord::Base.connection } + let(:connection) { ActiveRecord::Base.retrieve_connection } let(:db_config_name) { ::Gitlab::Database.db_config_name(connection) } describe '#transaction' do @@ -135,7 +135,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::ActiveRecord do end it_behaves_like 'record ActiveRecord metrics' - it_behaves_like 'store ActiveRecord info in RequestStore' + it_behaves_like 'store ActiveRecord info in RequestStore', :primary end end @@ -195,10 +195,6 @@ RSpec.describe Gitlab::Metrics::Subscribers::ActiveRecord do with_them do let(:payload) { { name: name, sql: sql(sql_query, comments: comments), connection: connection } } - before do - allow(Gitlab::Database::LoadBalancing).to receive(:enable?).and_return(true) - end - context 'query using a connection to a replica' do before do allow(Gitlab::Database::LoadBalancing).to receive(:db_role_for_connection).and_return(:replica) diff --git a/spec/lib/gitlab/metrics/subscribers/load_balancing_spec.rb b/spec/lib/gitlab/metrics/subscribers/load_balancing_spec.rb index 21a6573c6fd..bc6effd0438 100644 --- a/spec/lib/gitlab/metrics/subscribers/load_balancing_spec.rb +++ b/spec/lib/gitlab/metrics/subscribers/load_balancing_spec.rb @@ -5,10 +5,6 @@ require 'spec_helper' RSpec.describe Gitlab::Metrics::Subscribers::LoadBalancing, :request_store do let(:subscriber) { described_class.new } - before do - allow(Gitlab::Database::LoadBalancing).to receive(:enable?).and_return(true) - end - describe '#caught_up_replica_pick' do shared_examples 'having payload result value' do |result, counter_name| subject { subscriber.caught_up_replica_pick(event) } diff --git a/spec/lib/gitlab/metrics/web_transaction_spec.rb b/spec/lib/gitlab/metrics/web_transaction_spec.rb index 5261d04c879..9e22dccb2a2 100644 --- a/spec/lib/gitlab/metrics/web_transaction_spec.rb +++ b/spec/lib/gitlab/metrics/web_transaction_spec.rb @@ -32,7 +32,7 @@ RSpec.describe Gitlab::Metrics::WebTransaction do it 'measures with correct labels and value' do value = 1 - expect(prometheus_metric).to receive(metric_method).with({ controller: 'TestController', action: 'show', feature_category: '' }, value) + expect(prometheus_metric).to receive(metric_method).with({ controller: 'TestController', action: 'show', feature_category: ::Gitlab::FeatureCategories::FEATURE_CATEGORY_DEFAULT }, value) transaction.send(metric_method, :bau, value) end @@ -105,6 +105,9 @@ RSpec.describe Gitlab::Metrics::WebTransaction do namespace: "/projects") env['api.endpoint'] = endpoint + + # This is needed since we're not actually making a request, which would trigger the controller pushing to the context + ::Gitlab::ApplicationContext.push(feature_category: 'projects') end it 'provides labels with the method and path of the route in the grape endpoint' do @@ -129,7 +132,7 @@ RSpec.describe Gitlab::Metrics::WebTransaction do include_context 'ActionController request' it 'tags a transaction with the name and action of a controller' do - expect(transaction.labels).to eq({ controller: 'TestController', action: 'show', feature_category: '' }) + expect(transaction.labels).to eq({ controller: 'TestController', action: 'show', feature_category: ::Gitlab::FeatureCategories::FEATURE_CATEGORY_DEFAULT }) end it 'contains only the labels defined for transactions' do @@ -140,7 +143,7 @@ RSpec.describe Gitlab::Metrics::WebTransaction do let(:request) { double(:request, format: double(:format, ref: :json)) } it 'appends the mime type to the transaction action' do - expect(transaction.labels).to eq({ controller: 'TestController', action: 'show.json', feature_category: '' }) + expect(transaction.labels).to eq({ controller: 'TestController', action: 'show.json', feature_category: ::Gitlab::FeatureCategories::FEATURE_CATEGORY_DEFAULT }) end end @@ -148,13 +151,15 @@ RSpec.describe Gitlab::Metrics::WebTransaction do let(:request) { double(:request, format: double(:format, ref: 'http://example.com')) } it 'does not append the MIME type to the transaction action' do - expect(transaction.labels).to eq({ controller: 'TestController', action: 'show', feature_category: '' }) + expect(transaction.labels).to eq({ controller: 'TestController', action: 'show', feature_category: ::Gitlab::FeatureCategories::FEATURE_CATEGORY_DEFAULT }) end end context 'when the feature category is known' do it 'includes it in the feature category label' do - expect(controller_class).to receive(:feature_category_for_action).with('show').and_return(:source_code_management) + # This is needed since we're not actually making a request, which would trigger the controller pushing to the context + ::Gitlab::ApplicationContext.push(feature_category: 'source_code_management') + expect(transaction.labels).to eq({ controller: 'TestController', action: 'show', feature_category: "source_code_management" }) end end diff --git a/spec/lib/gitlab/middleware/go_spec.rb b/spec/lib/gitlab/middleware/go_spec.rb index 7bac041cd65..0ce95fdb5af 100644 --- a/spec/lib/gitlab/middleware/go_spec.rb +++ b/spec/lib/gitlab/middleware/go_spec.rb @@ -98,7 +98,7 @@ RSpec.describe Gitlab::Middleware::Go do end end - context 'without access to the project' do + context 'without access to the project', :sidekiq_inline do before do project.team.find_member(current_user).destroy end diff --git a/spec/lib/gitlab/middleware/multipart/handler_spec.rb b/spec/lib/gitlab/middleware/multipart/handler_spec.rb index aac3f00defe..53b59b042e2 100644 --- a/spec/lib/gitlab/middleware/multipart/handler_spec.rb +++ b/spec/lib/gitlab/middleware/multipart/handler_spec.rb @@ -16,6 +16,7 @@ RSpec.describe Gitlab::Middleware::Multipart::Handler do ::Gitlab.config.uploads.storage_path, ::JobArtifactUploader.workhorse_upload_path, ::LfsObjectUploader.workhorse_upload_path, + ::DependencyProxy::FileUploader.workhorse_upload_path, File.join(Rails.root, 'public/uploads/tmp') ] end diff --git a/spec/lib/gitlab/middleware/speedscope_spec.rb b/spec/lib/gitlab/middleware/speedscope_spec.rb index bb830a2fbda..c1d452f69f8 100644 --- a/spec/lib/gitlab/middleware/speedscope_spec.rb +++ b/spec/lib/gitlab/middleware/speedscope_spec.rb @@ -46,7 +46,7 @@ RSpec.describe Gitlab::Middleware::Speedscope do allow(env).to receive(:[]).with('warden').and_return(double('Warden', user: create(:admin))) end - it 'runs StackProf and returns a flamegraph' do + it 'returns a flamegraph' do expect(StackProf).to receive(:run).and_call_original status, headers, body = middleware.call(env) @@ -55,6 +55,56 @@ RSpec.describe Gitlab::Middleware::Speedscope do expect(headers).to eq({ 'Content-Type' => 'text/html' }) expect(body.first).to include('speedscope-iframe') end + + context 'when the stackprof_mode parameter is set and valid' do + let(:env) { Rack::MockRequest.env_for('/', params: { 'performance_bar' => 'flamegraph', 'stackprof_mode' => 'cpu' }) } + + it 'runs StackProf in the mode specified in the stackprof_mode parameter' do + expect(StackProf).to receive(:run).with(hash_including(mode: :cpu)) + + middleware.call(env) + end + end + + context 'when the stackprof_mode parameter is not set' do + let(:env) { Rack::MockRequest.env_for('/', params: { 'performance_bar' => 'flamegraph' }) } + + it 'runs StackProf in wall mode' do + expect(StackProf).to receive(:run).with(hash_including(mode: :wall)) + + middleware.call(env) + end + end + + context 'when the stackprof_mode parameter is invalid' do + let(:env) { Rack::MockRequest.env_for('/', params: { 'performance_bar' => 'flamegraph', 'stackprof_mode' => 'invalid' }) } + + it 'runs StackProf in wall mode' do + expect(StackProf).to receive(:run).with(hash_including(mode: :wall)) + + middleware.call(env) + end + end + + context 'when the stackprof_mode parameter is set to object mode' do + let(:env) { Rack::MockRequest.env_for('/', params: { 'performance_bar' => 'flamegraph', 'stackprof_mode' => 'object' }) } + + it 'runs StackProf with an interval of 100' do + expect(StackProf).to receive(:run).with(hash_including(interval: 100)) + + middleware.call(env) + end + end + + context 'when the stackprof_mode parameter is not set to object mode' do + let(:env) { Rack::MockRequest.env_for('/', params: { 'performance_bar' => 'flamegraph', 'stackprof_mode' => 'wall' }) } + + it 'runs StackProf with an interval of 10_100' do + expect(StackProf).to receive(:run).with(hash_including(interval: 10_100)) + + middleware.call(env) + end + end end end end diff --git a/spec/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder_spec.rb b/spec/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder_spec.rb index 4ce51e37685..00beacd4b35 100644 --- a/spec/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder_spec.rb +++ b/spec/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder_spec.rb @@ -41,14 +41,40 @@ RSpec.describe Gitlab::Pagination::Keyset::InOperatorOptimization::QueryBuilder ) end - it 'returns records in correct order' do + let(:all_records) do all_records = [] iterator.each_batch(of: batch_size) do |records| all_records.concat(records) end + all_records + end + it 'returns records in correct order' do expect(all_records).to eq(expected_order) end + + context 'when not passing the finder query' do + before do + in_operator_optimization_options.delete(:finder_query) + end + + it 'returns records in correct order' do + expect(all_records).to eq(expected_order) + end + + it 'loads only the order by column' do + order_by_attribute_names = iterator + .send(:order) + .column_definitions + .map(&:attribute_name) + .map(&:to_s) + + record = all_records.first + loaded_attributes = record.attributes.keys - ['time_estimate'] # time_estimate is always present (has default value) + + expect(loaded_attributes).to eq(order_by_attribute_names) + end + end end context 'when ordering by issues.id DESC' do diff --git a/spec/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/order_values_loader_strategy_spec.rb b/spec/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/order_values_loader_strategy_spec.rb new file mode 100644 index 00000000000..fe95d5406dd --- /dev/null +++ b/spec/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/order_values_loader_strategy_spec.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Pagination::Keyset::InOperatorOptimization::Strategies::OrderValuesLoaderStrategy do + let(:model) { Project } + + let(:keyset_scope) do + scope, _ = Gitlab::Pagination::Keyset::SimpleOrderBuilder.build( + Project.order(:created_at, :id) + ) + + scope + end + + let(:keyset_order) do + Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(keyset_scope) + end + + let(:order_by_columns) do + Gitlab::Pagination::Keyset::InOperatorOptimization::OrderByColumns.new(keyset_order.column_definitions, model.arel_table) + end + + subject(:strategy) { described_class.new(model, order_by_columns) } + + describe '#initializer_columns' do + it 'returns NULLs for each ORDER BY columns' do + expect(strategy.initializer_columns).to eq([ + 'NULL::timestamp without time zone AS created_at', + 'NULL::integer AS id' + ]) + end + end +end diff --git a/spec/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy_spec.rb b/spec/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy_spec.rb new file mode 100644 index 00000000000..5180403b493 --- /dev/null +++ b/spec/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy_spec.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Pagination::Keyset::InOperatorOptimization::Strategies::RecordLoaderStrategy do + let(:finder_query) { -> (created_at_value, id_value) { Project.where(Project.arel_table[:id].eq(id_value)) } } + let(:model) { Project } + + let(:keyset_scope) do + scope, _ = Gitlab::Pagination::Keyset::SimpleOrderBuilder.build( + Project.order(:created_at, :id) + ) + + scope + end + + let(:keyset_order) do + Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(keyset_scope) + end + + let(:order_by_columns) do + Gitlab::Pagination::Keyset::InOperatorOptimization::OrderByColumns.new(keyset_order.column_definitions, model.arel_table) + end + + subject(:strategy) { described_class.new(finder_query, model, order_by_columns) } + + describe '#initializer_columns' do + # Explanation: + # > SELECT NULL::projects AS records + # + # The query returns one row and one column. The column may contain a full project row. + # In this particular case the row is NULL. + it 'returns a NULL table row as the result column' do + expect(strategy.initializer_columns).to eq(["NULL::projects AS records"]) + end + end + + describe '#columns' do + # Explanation: + # > SELECT (SELECT projects FROM projects limit 1) + # + # Selects one row from the database and collapses it into one column. + # + # Side note: Due to the type casts, columns and initializer_columns can be also UNION-ed: + # SELECT * FROM ( + # ( + # SELECT NULL::projects AS records + # UNION + # SELECT (SELECT projects FROM projects limit 1) + # ) + # ) as records + it 'uses the finder query to load the row in the result column' do + expected_loader_query = <<~SQL + (SELECT projects FROM "projects" WHERE "projects"."id" = recursive_keyset_cte.projects_id_array[position] LIMIT 1) + SQL + + expect(strategy.columns).to eq([expected_loader_query.chomp]) + end + end +end diff --git a/spec/lib/gitlab/pagination/keyset/iterator_spec.rb b/spec/lib/gitlab/pagination/keyset/iterator_spec.rb index d8e79287745..09cbca2c1cb 100644 --- a/spec/lib/gitlab/pagination/keyset/iterator_spec.rb +++ b/spec/lib/gitlab/pagination/keyset/iterator_spec.rb @@ -32,8 +32,11 @@ RSpec.describe Gitlab::Pagination::Keyset::Iterator do ]) end + let(:iterator_params) { nil } let(:scope) { project.issues.reorder(custom_reorder) } + subject(:iterator) { described_class.new(**iterator_params) } + shared_examples 'iterator examples' do describe '.each_batch' do it 'yields an ActiveRecord::Relation when a block is given' do @@ -56,6 +59,29 @@ RSpec.describe Gitlab::Pagination::Keyset::Iterator do expect(count).to eq(9) end + it 'continues after the cursor' do + loaded_records = [] + cursor = nil + + # stopping the iterator after the first batch and storing the cursor + iterator.each_batch(of: 2) do |relation| # rubocop: disable Lint/UnreachableLoop + loaded_records.concat(relation.to_a) + record = loaded_records.last + + cursor = custom_reorder.cursor_attributes_for_node(record) + break + end + + expect(loaded_records).to eq(project.issues.order(custom_reorder).take(2)) + + new_iterator = described_class.new(**iterator_params.merge(cursor: cursor)) + new_iterator.each_batch(of: 2) do |relation| + loaded_records.concat(relation.to_a) + end + + expect(loaded_records).to eq(project.issues.order(custom_reorder)) + end + it 'allows updating of the yielded relations' do time = Time.current @@ -73,7 +99,7 @@ RSpec.describe Gitlab::Pagination::Keyset::Iterator do iterator.each_batch(of: 2) { |rel| positions.concat(rel.pluck(:relative_position, :id)) } - expect(positions).to eq(project.issues.order_relative_position_asc.order(id: :asc).pluck(:relative_position, :id)) + expect(positions).to eq(project.issues.reorder(::Gitlab::Database.nulls_last_order('relative_position', 'ASC')).order(id: :asc).pluck(:relative_position, :id)) end end @@ -85,7 +111,7 @@ RSpec.describe Gitlab::Pagination::Keyset::Iterator do iterator.each_batch(of: 2) { |rel| positions.concat(rel.pluck(:relative_position, :id)) } - expect(positions).to eq(project.issues.order_relative_position_desc.order(id: :desc).pluck(:relative_position, :id)) + expect(positions).to eq(project.issues.reorder(::Gitlab::Database.nulls_first_order('relative_position', 'DESC')).order(id: :desc).pluck(:relative_position, :id)) end end @@ -131,13 +157,13 @@ RSpec.describe Gitlab::Pagination::Keyset::Iterator do end context 'when use_union_optimization is used' do - subject(:iterator) { described_class.new(scope: scope, use_union_optimization: true) } + let(:iterator_params) { { scope: scope, use_union_optimization: true } } include_examples 'iterator examples' end context 'when use_union_optimization is not used' do - subject(:iterator) { described_class.new(scope: scope, use_union_optimization: false) } + let(:iterator_params) { { scope: scope, use_union_optimization: false } } include_examples 'iterator examples' end diff --git a/spec/lib/gitlab/path_regex_spec.rb b/spec/lib/gitlab/path_regex_spec.rb index aa13660deb4..2f38ed58727 100644 --- a/spec/lib/gitlab/path_regex_spec.rb +++ b/spec/lib/gitlab/path_regex_spec.rb @@ -561,4 +561,25 @@ RSpec.describe Gitlab::PathRegex do expect(subject.match('sha256:asdf1234%2f')[0]).to eq('sha256:asdf1234') end end + + describe '.dependency_proxy_route_regex' do + subject { described_class.dependency_proxy_route_regex } + + it { is_expected.to match('/v2/group1/dependency_proxy/containers/alpine/manifests/latest') } + it { is_expected.to match('/v2/group1/dependency_proxy/containers/alpine/blobs/sha256:14119a10abf4669e8cdbdff324a9f9605d99697215a0d21c360fe8dfa8471bab') } + + it { is_expected.not_to match('') } + it { is_expected.not_to match('/v3/group1/dependency_proxy/containers/alpine/manifests/latest') } + it { is_expected.not_to match('/v2/group1/dependency_proxy/container/alpine/manifests/latest') } + it { is_expected.not_to match('/v2/group1/dependency_prox/containers/alpine/manifests/latest') } + it { is_expected.not_to match('/v2/group1/dependency_proxy/containers/alpine/manifest/latest') } + it { is_expected.not_to match('/v2/group1/dependency_proxy/containers/alpine/manifest/la%2Ftest') } + it { is_expected.not_to match('/v2/group1/dependency_proxy/containers/alpine/manifest/latest/../one') } + it { is_expected.not_to match('/v3/group1/dependency_proxy/containers/alpine/blobs/sha256:14119a10abf4669e8cdbdff324a9f9605d99697215a0d21c360fe8dfa8471bab') } + it { is_expected.not_to match('/v2/group1/dependency_proxy/container/alpine/blobs/sha256:14119a10abf4669e8cdbdff324a9f9605d99697215a0d21c360fe8dfa8471bab') } + it { is_expected.not_to match('/v2/group1/dependency_prox/containers/alpine/blobs/sha256:14119a10abf4669e8cdbdff324a9f9605d99697215a0d21c360fe8dfa8471bab') } + it { is_expected.not_to match('/v2/group1/dependency_proxy/containers/alpine/blob/sha256:14119a10abf4669e8cdbdff324a9f9605d99697215a0d21c360fe8dfa8471bab') } + it { is_expected.not_to match('/v2/group1/dependency_proxy/containers/alpine/blob/sha256:F14119a10abf4669e8cdbdff324a9f9605d99697215a0d21c360fe8dfa8471bab/../latest') } + it { is_expected.not_to match('/v2/group1/dependency_proxy/containers/alpine/blob/sha256:F14119a10abf4669e8cdbdff324a9f9605d99697215a0d21c360fe8dfa8471bab/latest') } + end end diff --git a/spec/lib/gitlab/performance_bar/stats_spec.rb b/spec/lib/gitlab/performance_bar/stats_spec.rb index ad11eca56d1..b4f90745ee7 100644 --- a/spec/lib/gitlab/performance_bar/stats_spec.rb +++ b/spec/lib/gitlab/performance_bar/stats_spec.rb @@ -23,11 +23,23 @@ RSpec.describe Gitlab::PerformanceBar::Stats do expect(logger).to receive(:info) .with({ duration_ms: 1.096, filename: 'lib/gitlab/pagination/offset_pagination.rb', method_path: 'lib/gitlab/pagination/offset_pagination.rb:add_pagination_headers', - count: 1, request_id: 'foo', type: :sql }) + count: 1, request_id: 'foo', query_type: 'active-record' }) expect(logger).to receive(:info) .with({ duration_ms: 1.634, filename: 'lib/api/helpers.rb', method_path: 'lib/api/helpers.rb:find_project', - count: 2, request_id: 'foo', type: :sql }) + count: 2, request_id: 'foo', query_type: 'active-record' }) + expect(logger).to receive(:info) + .with({ duration_ms: 23.709, filename: 'lib/gitlab/gitaly_client/commit_service.rb', + method_path: 'lib/gitlab/gitaly_client/commit_service.rb:each', + count: 1, request_id: 'foo', query_type: 'gitaly' }) + expect(logger).to receive(:info) + .with({ duration_ms: 6.678, filename: 'lib/gitlab/gitaly_client/commit_service.rb', + method_path: 'lib/gitlab/gitaly_client/commit_service.rb:call_find_commit', + count: 1, request_id: 'foo', query_type: 'gitaly' }) + expect(logger).to receive(:info) + .with({ duration_ms: 0.155, filename: 'lib/feature.rb', + method_path: 'lib/feature.rb:enabled?', + count: 1, request_id: 'foo', query_type: 'redis' }) subject end diff --git a/spec/lib/gitlab/project_authorizations_spec.rb b/spec/lib/gitlab/project_authorizations_spec.rb index d2b41ee31d9..16066934194 100644 --- a/spec/lib/gitlab/project_authorizations_spec.rb +++ b/spec/lib/gitlab/project_authorizations_spec.rb @@ -204,6 +204,43 @@ RSpec.describe Gitlab::ProjectAuthorizations do end end + context 'with shared projects' do + let_it_be(:shared_with_group) { create(:group) } + let_it_be(:user) { create(:user) } + let_it_be(:project) { create(:project, group: create(:group)) } + + let(:mapping) { map_access_levels(authorizations) } + + before do + create(:project_group_link, :developer, project: project, group: shared_with_group) + shared_with_group.add_maintainer(user) + end + + it 'creates proper authorizations' do + expect(mapping[project.id]).to eq(Gitlab::Access::DEVELOPER) + end + + context 'even when the `lock_memberships_to_ldap` setting has been turned ON' do + before do + stub_application_setting(lock_memberships_to_ldap: true) + end + + it 'creates proper authorizations' do + expect(mapping[project.id]).to eq(Gitlab::Access::DEVELOPER) + end + end + + context 'when the group containing the project has forbidden group shares for any of its projects' do + before do + project.namespace.update!(share_with_group_lock: true) + end + + it 'does not create authorizations' do + expect(mapping[project.id]).to be_nil + end + end + end + context 'with shared groups' do let(:parent_group_user) { create(:user) } let(:group_user) { create(:user) } diff --git a/spec/lib/gitlab/rack_attack/request_spec.rb b/spec/lib/gitlab/rack_attack/request_spec.rb index 3be7ec17e45..ecdcc23e588 100644 --- a/spec/lib/gitlab/rack_attack/request_spec.rb +++ b/spec/lib/gitlab/rack_attack/request_spec.rb @@ -3,6 +3,8 @@ require 'spec_helper' RSpec.describe Gitlab::RackAttack::Request do + using RSpec::Parameterized::TableSyntax + describe 'FILES_PATH_REGEX' do subject { described_class::FILES_PATH_REGEX } @@ -13,4 +15,33 @@ RSpec.describe Gitlab::RackAttack::Request do it { is_expected.to match('/api/v4/projects/some%2Fnested%2Frepo/repository/files/README') } it { is_expected.not_to match('/api/v4/projects/some/nested/repo/repository/files/README') } end + + describe '#deprecated_api_request?' do + let(:env) { { 'REQUEST_METHOD' => 'GET', 'rack.input' => StringIO.new, 'PATH_INFO' => path, 'QUERY_STRING' => query } } + let(:request) { ::Rack::Attack::Request.new(env) } + + subject { !!request.__send__(:deprecated_api_request?) } + + where(:path, :query, :expected) do + '/' | '' | false + + '/api/v4/groups/1/' | '' | true + '/api/v4/groups/1' | '' | true + '/api/v4/groups/foo/' | '' | true + '/api/v4/groups/foo' | '' | true + + '/api/v4/groups/1' | 'with_projects=' | true + '/api/v4/groups/1' | 'with_projects=1' | true + '/api/v4/groups/1' | 'with_projects=0' | false + + '/foo/api/v4/groups/1' | '' | false + '/api/v4/groups/1/foo' | '' | false + + '/api/v4/groups/nested%2Fgroup' | '' | true + end + + with_them do + it { is_expected.to eq(expected) } + end + end end diff --git a/spec/lib/gitlab/rate_limit_helpers_spec.rb b/spec/lib/gitlab/rate_limit_helpers_spec.rb index d583c8e58fb..ad0e2de1448 100644 --- a/spec/lib/gitlab/rate_limit_helpers_spec.rb +++ b/spec/lib/gitlab/rate_limit_helpers_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::RateLimitHelpers, :clean_gitlab_redis_cache do +RSpec.describe Gitlab::RateLimitHelpers, :clean_gitlab_redis_rate_limiting do let(:limiter_class) do Class.new do include ::Gitlab::RateLimitHelpers diff --git a/spec/lib/gitlab/redis/queues_spec.rb b/spec/lib/gitlab/redis/queues_spec.rb index 2e396cde3bf..a0f73a654e7 100644 --- a/spec/lib/gitlab/redis/queues_spec.rb +++ b/spec/lib/gitlab/redis/queues_spec.rb @@ -9,10 +9,24 @@ RSpec.describe Gitlab::Redis::Queues do include_examples "redis_shared_examples" describe '#raw_config_hash' do - it 'has a legacy default URL' do - expect(subject).to receive(:fetch_config) { false } + before do + expect(subject).to receive(:fetch_config) { config } + end + + context 'when the config url is blank' do + let(:config) { nil } + + it 'has a legacy default URL' do + expect(subject.send(:raw_config_hash)).to eq(url: 'redis://localhost:6381' ) + end + end + + context 'when the config url is present' do + let(:config) { { url: 'redis://localhost:1111' } } - expect(subject.send(:raw_config_hash)).to eq(url: 'redis://localhost:6381' ) + it 'sets the configured url' do + expect(subject.send(:raw_config_hash)).to eq(url: 'redis://localhost:1111' ) + end end end end diff --git a/spec/lib/gitlab/redis/rate_limiting_spec.rb b/spec/lib/gitlab/redis/rate_limiting_spec.rb new file mode 100644 index 00000000000..e79c070df93 --- /dev/null +++ b/spec/lib/gitlab/redis/rate_limiting_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Redis::RateLimiting do + include_examples "redis_new_instance_shared_examples", 'rate_limiting', Gitlab::Redis::Cache +end diff --git a/spec/lib/gitlab/redis/sessions_spec.rb b/spec/lib/gitlab/redis/sessions_spec.rb new file mode 100644 index 00000000000..7e239c08e9f --- /dev/null +++ b/spec/lib/gitlab/redis/sessions_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Redis::Sessions do + include_examples "redis_new_instance_shared_examples", 'sessions', Gitlab::Redis::SharedState +end diff --git a/spec/lib/gitlab/redis/trace_chunks_spec.rb b/spec/lib/gitlab/redis/trace_chunks_spec.rb index e974dc519d6..bb3c3089430 100644 --- a/spec/lib/gitlab/redis/trace_chunks_spec.rb +++ b/spec/lib/gitlab/redis/trace_chunks_spec.rb @@ -3,53 +3,5 @@ require 'spec_helper' RSpec.describe Gitlab::Redis::TraceChunks do - let(:instance_specific_config_file) { "config/redis.trace_chunks.yml" } - let(:environment_config_file_name) { "GITLAB_REDIS_TRACE_CHUNKS_CONFIG_FILE" } - let(:shared_state_config_file) { nil } - - before do - allow(Gitlab::Redis::SharedState).to receive(:config_file_name).and_return(shared_state_config_file) - end - - include_examples "redis_shared_examples" - - describe '.config_file_name' do - subject { described_class.config_file_name } - - let(:rails_root) { Dir.mktmpdir('redis_shared_examples') } - - before do - # Undo top-level stub of config_file_name because we are testing that method now. - allow(described_class).to receive(:config_file_name).and_call_original - - allow(described_class).to receive(:rails_root).and_return(rails_root) - FileUtils.mkdir_p(File.join(rails_root, 'config')) - end - - after do - FileUtils.rm_rf(rails_root) - end - - context 'when there is only a resque.yml' do - before do - FileUtils.touch(File.join(rails_root, 'config/resque.yml')) - end - - it { expect(subject).to eq("#{rails_root}/config/resque.yml") } - - context 'and there is a global env override' do - before do - stub_env('GITLAB_REDIS_CONFIG_FILE', 'global override') - end - - it { expect(subject).to eq('global override') } - - context 'and SharedState has a different config file' do - let(:shared_state_config_file) { 'shared state config file' } - - it { expect(subject).to eq('shared state config file') } - end - end - end - end + include_examples "redis_new_instance_shared_examples", 'trace_chunks', Gitlab::Redis::SharedState end diff --git a/spec/lib/gitlab/regex_spec.rb b/spec/lib/gitlab/regex_spec.rb index f1b4e50b1eb..9514654204b 100644 --- a/spec/lib/gitlab/regex_spec.rb +++ b/spec/lib/gitlab/regex_spec.rb @@ -12,22 +12,29 @@ RSpec.describe Gitlab::Regex do it { is_expected.to match('Dash – is this') } end - shared_examples_for 'project/group name regex' do + shared_examples_for 'group name regex' do it_behaves_like 'project/group name chars regex' it { is_expected.not_to match('?gitlab') } it { is_expected.not_to match("Users's something") } end + shared_examples_for 'project name regex' do + it_behaves_like 'project/group name chars regex' + it { is_expected.to match("Gitlab++") } + it { is_expected.not_to match('?gitlab') } + it { is_expected.not_to match("Users's something") } + end + describe '.project_name_regex' do subject { described_class.project_name_regex } - it_behaves_like 'project/group name regex' + it_behaves_like 'project name regex' end describe '.group_name_regex' do subject { described_class.group_name_regex } - it_behaves_like 'project/group name regex' + it_behaves_like 'group name regex' it 'allows parenthesis' do is_expected.to match('Group One (Test)') @@ -51,7 +58,7 @@ RSpec.describe Gitlab::Regex do describe '.project_name_regex_message' do subject { described_class.project_name_regex_message } - it { is_expected.to eq("can contain only letters, digits, emojis, '_', '.', dash, space. It must start with letter, digit, emoji or '_'.") } + it { is_expected.to eq("can contain only letters, digits, emojis, '_', '.', '+', dashes, or spaces. It must start with a letter, digit, emoji, or '_'.") } end describe '.group_name_regex_message' do @@ -646,13 +653,24 @@ RSpec.describe Gitlab::Regex do it { is_expected.to match('release') } it { is_expected.to match('my-repo') } - it { is_expected.to match('my-repo42') } + it { is_expected.to match('My-Re_po') } + it { is_expected.to match('my_repo42') } + it { is_expected.to match('1.2.3') } + it { is_expected.to match('v1.2.3-beta-12') } + it { is_expected.to match('renovate_https-github.com-operator-framework-operator-lifecycle-manager.git-0.x') } # Do not allow empty it { is_expected.not_to match('') } # Do not allow Unicode it { is_expected.not_to match('hé') } + + it { is_expected.not_to match('.1.23') } + it { is_expected.not_to match('1..23') } + it { is_expected.not_to match('1.2.3.') } + it { is_expected.not_to match('1..2.3.') } + it { is_expected.not_to match('1/../2.3.') } + it { is_expected.not_to match('1/..%2F2.3.') } end describe '.helm_package_regex' do diff --git a/spec/lib/gitlab/request_endpoints_spec.rb b/spec/lib/gitlab/request_endpoints_spec.rb new file mode 100644 index 00000000000..0c939bfb0ee --- /dev/null +++ b/spec/lib/gitlab/request_endpoints_spec.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true +require 'spec_helper' + +RSpec.describe Gitlab::RequestEndpoints do + describe '.all_api_endpoints' do + it 'selects all feature API classes' do + api_classes = described_class.all_api_endpoints.map { |route| route.app.options[:for] } + + expect(api_classes).to all(include(Gitlab::EndpointAttributes)) + end + end + + describe '.all_controller_actions' do + it 'selects all feature controllers and action names' do + all_controller_actions = described_class.all_controller_actions + controller_classes = all_controller_actions.map(&:first) + all_actions = all_controller_actions.map(&:last) + + expect(controller_classes).to all(include(Gitlab::EndpointAttributes)) + expect(controller_classes).not_to include(ApplicationController, Devise::UnlocksController) + expect(all_actions).to all(be_a(String)) + end + end +end diff --git a/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb b/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb index a98038cd3f8..d801b84775b 100644 --- a/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb +++ b/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb @@ -18,7 +18,7 @@ RSpec.describe Gitlab::SidekiqLogging::StructuredLogger do end it 'logs start and end of job' do - Timecop.freeze(timestamp) do + travel_to(timestamp) do expect(logger).to receive(:info).with(start_payload).ordered expect(logger).to receive(:info).with(end_payload).ordered expect(subject).to receive(:log_job_start).and_call_original @@ -34,7 +34,7 @@ RSpec.describe Gitlab::SidekiqLogging::StructuredLogger do "wrapped" => "TestWorker" ) - Timecop.freeze(timestamp) do + travel_to(timestamp) do expect(logger).to receive(:info).with(start_payload).ordered expect(logger).to receive(:info).with(end_payload).ordered expect(subject).to receive(:log_job_start).and_call_original @@ -45,7 +45,7 @@ RSpec.describe Gitlab::SidekiqLogging::StructuredLogger do end it 'logs an exception in job' do - Timecop.freeze(timestamp) do + travel_to(timestamp) do expect(logger).to receive(:info).with(start_payload) expect(logger).to receive(:warn).with(include(exception_payload)) expect(subject).to receive(:log_job_start).and_call_original @@ -60,7 +60,7 @@ RSpec.describe Gitlab::SidekiqLogging::StructuredLogger do end it 'logs the root cause of an Sidekiq::JobRetry::Skip exception in the job' do - Timecop.freeze(timestamp) do + travel_to(timestamp) do expect(logger).to receive(:info).with(start_payload) expect(logger).to receive(:warn).with(include(exception_payload)) expect(subject).to receive(:log_job_start).and_call_original @@ -77,7 +77,7 @@ RSpec.describe Gitlab::SidekiqLogging::StructuredLogger do end it 'logs the root cause of an Sidekiq::JobRetry::Handled exception in the job' do - Timecop.freeze(timestamp) do + travel_to(timestamp) do expect(logger).to receive(:info).with(start_payload) expect(logger).to receive(:warn).with(include(exception_payload)) expect(subject).to receive(:log_job_start).and_call_original @@ -94,7 +94,7 @@ RSpec.describe Gitlab::SidekiqLogging::StructuredLogger do end it 'keeps Sidekiq::JobRetry::Handled exception if the cause does not exist' do - Timecop.freeze(timestamp) do + travel_to(timestamp) do expect(logger).to receive(:info).with(start_payload) expect(logger).to receive(:warn).with( include( @@ -116,7 +116,7 @@ RSpec.describe Gitlab::SidekiqLogging::StructuredLogger do end it 'does not modify the job' do - Timecop.freeze(timestamp) do + travel_to(timestamp) do job_copy = job.deep_dup allow(logger).to receive(:info) @@ -130,7 +130,7 @@ RSpec.describe Gitlab::SidekiqLogging::StructuredLogger do end it 'does not modify the wrapped job' do - Timecop.freeze(timestamp) do + travel_to(timestamp) do wrapped_job = job.merge( "class" => "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper", "wrapped" => "TestWorker" @@ -154,7 +154,7 @@ RSpec.describe Gitlab::SidekiqLogging::StructuredLogger do end it 'logs start and end of job without args' do - Timecop.freeze(timestamp) do + travel_to(timestamp) do expect(logger).to receive(:info).with(start_payload.except('args')).ordered expect(logger).to receive(:info).with(end_payload.except('args')).ordered expect(subject).to receive(:log_job_start).and_call_original @@ -165,7 +165,7 @@ RSpec.describe Gitlab::SidekiqLogging::StructuredLogger do end it 'logs without created_at and enqueued_at fields' do - Timecop.freeze(timestamp) do + travel_to(timestamp) do excluded_fields = %w(created_at enqueued_at args scheduling_latency_s) expect(logger).to receive(:info).with(start_payload.except(*excluded_fields)).ordered @@ -183,7 +183,7 @@ RSpec.describe Gitlab::SidekiqLogging::StructuredLogger do let(:scheduling_latency_s) { 7200.0 } it 'logs with scheduling latency' do - Timecop.freeze(timestamp) do + travel_to(timestamp) do expect(logger).to receive(:info).with(start_payload).ordered expect(logger).to receive(:info).with(end_payload).ordered expect(subject).to receive(:log_job_start).and_call_original @@ -194,6 +194,35 @@ RSpec.describe Gitlab::SidekiqLogging::StructuredLogger do end end + context 'with enqueue latency' do + let(:expected_start_payload) do + start_payload.merge( + 'scheduled_at' => job['scheduled_at'], + 'enqueue_latency_s' => 1.hour.to_f + ) + end + + let(:expected_end_payload) do + end_payload.merge('enqueue_latency_s' => 1.hour.to_f) + end + + before do + # enqueued_at is set to created_at + job['scheduled_at'] = created_at - 1.hour + end + + it 'logs with scheduling latency' do + travel_to(timestamp) do + expect(logger).to receive(:info).with(expected_start_payload).ordered + expect(logger).to receive(:info).with(expected_end_payload).ordered + expect(subject).to receive(:log_job_start).and_call_original + expect(subject).to receive(:log_job_done).and_call_original + + call_subject(job, 'test_queue') { } + end + end + end + context 'with Gitaly, Rugged, and Redis calls' do let(:timing_data) do { @@ -218,7 +247,7 @@ RSpec.describe Gitlab::SidekiqLogging::StructuredLogger do end it 'logs with Gitaly and Rugged timing data', :aggregate_failures do - Timecop.freeze(timestamp) do + travel_to(timestamp) do expect(logger).to receive(:info).with(start_payload).ordered expect(logger).to receive(:info).with(expected_end_payload).ordered @@ -243,8 +272,22 @@ RSpec.describe Gitlab::SidekiqLogging::StructuredLogger do expected_end_payload.merge( 'db_duration_s' => a_value >= 0.1, 'db_count' => a_value >= 1, - 'db_cached_count' => 0, - 'db_write_count' => 0 + "db_replica_#{db_config_name}_count" => 0, + 'db_replica_duration_s' => a_value >= 0, + 'db_primary_count' => a_value >= 1, + "db_primary_#{db_config_name}_count" => a_value >= 1, + 'db_primary_duration_s' => a_value > 0, + "db_primary_#{db_config_name}_duration_s" => a_value > 0 + ) + end + + let(:end_payload) do + start_payload.merge(db_payload_defaults).merge( + 'message' => 'TestWorker JID-da883554ee4fe414012f5f42: done: 0.0 sec', + 'job_status' => 'done', + 'duration_s' => 0.0, + 'completed_at' => timestamp.to_f, + 'cpu_s' => 1.111112 ) end @@ -274,59 +317,9 @@ RSpec.describe Gitlab::SidekiqLogging::StructuredLogger do end end - context 'when load balancing is disabled' do - before do - allow(Gitlab::Database::LoadBalancing).to receive(:enable?).and_return(false) - end - - let(:expected_end_payload_with_db) do - expected_end_payload.merge( - 'db_duration_s' => a_value >= 0.1, - 'db_count' => a_value >= 1, - 'db_cached_count' => 0, - 'db_write_count' => 0 - ) - end - - include_examples 'performs database queries' - end - - context 'when load balancing is enabled', :db_load_balancing do - let(:db_config_name) { ::Gitlab::Database.db_config_name(ApplicationRecord.connection) } - - let(:expected_db_payload_defaults) do - metrics = - ::Gitlab::Metrics::Subscribers::ActiveRecord.load_balancing_metric_counter_keys + - ::Gitlab::Metrics::Subscribers::ActiveRecord.load_balancing_metric_duration_keys + - ::Gitlab::Metrics::Subscribers::ActiveRecord.db_counter_keys + - [:db_duration_s] - - metrics.each_with_object({}) do |key, result| - result[key.to_s] = 0 - end - end - - let(:expected_end_payload_with_db) do - expected_end_payload.merge(expected_db_payload_defaults).merge( - 'db_duration_s' => a_value >= 0.1, - 'db_count' => a_value >= 1, - "db_replica_#{db_config_name}_count" => 0, - 'db_replica_duration_s' => a_value >= 0, - 'db_primary_count' => a_value >= 1, - "db_primary_#{db_config_name}_count" => a_value >= 1, - 'db_primary_duration_s' => a_value > 0, - "db_primary_#{db_config_name}_duration_s" => a_value > 0 - ) - end - - let(:end_payload) do - start_payload.merge(expected_db_payload_defaults).merge( - 'message' => 'TestWorker JID-da883554ee4fe414012f5f42: done: 0.0 sec', - 'job_status' => 'done', - 'duration_s' => 0.0, - 'completed_at' => timestamp.to_f, - 'cpu_s' => 1.111112 - ) + context 'when load balancing is enabled' do + let(:db_config_name) do + ::Gitlab::Database.db_config_name(ApplicationRecord.retrieve_connection) end include_examples 'performs database queries' @@ -359,7 +352,7 @@ RSpec.describe Gitlab::SidekiqLogging::StructuredLogger do end it 'logs it in the done log' do - Timecop.freeze(timestamp) do + travel_to(timestamp) do expect(logger).to receive(:info).with(expected_start_payload).ordered expect(logger).to receive(:info).with(expected_end_payload).ordered @@ -401,7 +394,7 @@ RSpec.describe Gitlab::SidekiqLogging::StructuredLogger do end it 'logs it in the done log' do - Timecop.freeze(timestamp) do + travel_to(timestamp) do expect(logger).to receive(:info).with(expected_start_payload).ordered expect(logger).to receive(:info).with(expected_end_payload).ordered @@ -426,13 +419,13 @@ RSpec.describe Gitlab::SidekiqLogging::StructuredLogger do 'message' => 'my-message', 'job_status' => 'my-job-status', 'duration_s' => 0.123123, - 'completed_at' => current_utc_time.to_f } + 'completed_at' => current_utc_time.to_i } end subject { described_class.new } it 'update payload correctly' do - Timecop.freeze(current_utc_time) do + travel_to(current_utc_time) do subject.send(:add_time_keys!, time, payload) expect(payload).to eq(payload_with_time_keys) diff --git a/spec/lib/gitlab/sidekiq_middleware/client_metrics_spec.rb b/spec/lib/gitlab/sidekiq_middleware/client_metrics_spec.rb index 698758a13fd..dca00c85e30 100644 --- a/spec/lib/gitlab/sidekiq_middleware/client_metrics_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware/client_metrics_spec.rb @@ -62,6 +62,27 @@ RSpec.describe Gitlab::SidekiqMiddleware::ClientMetrics do Sidekiq::Testing.inline! { TestWorker.perform_in(1.second) } end + + it 'sets the scheduled_at field' do + job = { 'at' => Time.current } + + subject.call('TestWorker', job, 'queue', nil) do + expect(job[:scheduled_at]).to eq(job['at']) + end + end + end + + context 'when the worker class cannot be found' do + it 'increments enqueued jobs metric with the worker labels set to NilClass' do + test_anonymous_worker = Class.new(TestWorker) + + expect(enqueued_jobs_metric).to receive(:increment).with(a_hash_including(worker: 'NilClass'), 1) + + # Sidekiq won't be able to create an instance of this class + expect do + Sidekiq::Testing.inline! { test_anonymous_worker.perform_async } + end.to raise_error(NameError) + end end end end diff --git a/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job_spec.rb b/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job_spec.rb index cc69a11f7f8..5083ac514db 100644 --- a/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job_spec.rb @@ -472,6 +472,26 @@ RSpec.describe Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob, :clean_gi expect(duplicate_job).to be_idempotent end end + + context 'when worker class is utilizing load balancing capabilities' do + before do + allow(AuthorizedProjectsWorker).to receive(:utilizes_load_balancing_capabilities?).and_return(true) + end + + it 'returns true' do + expect(duplicate_job).to be_idempotent + end + + context 'when preserve_latest_wal_locations_for_idempotent_jobs feature flag is disabled' do + before do + stub_feature_flags(preserve_latest_wal_locations_for_idempotent_jobs: false) + end + + it 'returns false' do + expect(duplicate_job).not_to be_idempotent + end + end + end end def existing_wal_location_key(idempotency_key, config_name) diff --git a/spec/lib/gitlab/sidekiq_middleware/memory_killer_spec.rb b/spec/lib/gitlab/sidekiq_middleware/memory_killer_spec.rb index 0d1616c4aed..1667622ad8e 100644 --- a/spec/lib/gitlab/sidekiq_middleware/memory_killer_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware/memory_killer_spec.rb @@ -45,7 +45,7 @@ RSpec.describe Gitlab::SidekiqMiddleware::MemoryKiller do expect(subject).to receive(:sleep).with(30).ordered expect(Process).to receive(:kill).with('SIGTERM', pid).ordered - expect(subject).to receive(:sleep).with(10).ordered + expect(subject).to receive(:sleep).with(Sidekiq.options[:timeout] + 2).ordered expect(Process).to receive(:kill).with('SIGKILL', pid).ordered expect(Sidekiq.logger) diff --git a/spec/lib/gitlab/sidekiq_middleware/server_metrics_spec.rb b/spec/lib/gitlab/sidekiq_middleware/server_metrics_spec.rb index cae0bb6b167..914f5a30c3a 100644 --- a/spec/lib/gitlab/sidekiq_middleware/server_metrics_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware/server_metrics_spec.rb @@ -211,6 +211,9 @@ RSpec.describe Gitlab::SidekiqMiddleware::ServerMetrics do end end + include_context 'server metrics with mocked prometheus' + include_context 'server metrics call' + before do stub_const('TestWorker', Class.new) TestWorker.class_eval do @@ -234,9 +237,6 @@ RSpec.describe Gitlab::SidekiqMiddleware::ServerMetrics do end end - include_context 'server metrics with mocked prometheus' - include_context 'server metrics call' - shared_context 'worker declaring data consistency' do let(:worker_class) { LBTestWorker } @@ -250,61 +250,93 @@ RSpec.describe Gitlab::SidekiqMiddleware::ServerMetrics do end end - context 'when load_balancing is enabled' do - before do - allow(::Gitlab::Database::LoadBalancing).to receive(:enable?).and_return(true) - end + describe '#call' do + context 'when worker declares data consistency' do + include_context 'worker declaring data consistency' - describe '#call' do - context 'when worker declares data consistency' do - include_context 'worker declaring data consistency' + it 'increments load balancing counter with defined data consistency' do + process_job + + expect(load_balancing_metric).to have_received(:increment).with( + a_hash_including( + data_consistency: :delayed, + load_balancing_strategy: 'replica' + ), 1) + end + end - it 'increments load balancing counter with defined data consistency' do - process_job + context 'when worker does not declare data consistency' do + it 'increments load balancing counter with default data consistency' do + process_job - expect(load_balancing_metric).to have_received(:increment).with( - a_hash_including( - data_consistency: :delayed, - load_balancing_strategy: 'replica' - ), 1) - end + expect(load_balancing_metric).to have_received(:increment).with( + a_hash_including( + data_consistency: :always, + load_balancing_strategy: 'primary' + ), 1) end + end + end + end - context 'when worker does not declare data consistency' do - it 'increments load balancing counter with default data consistency' do - process_job + context 'feature attribution' do + let(:test_worker) do + category = worker_category - expect(load_balancing_metric).to have_received(:increment).with( - a_hash_including( - data_consistency: :always, - load_balancing_strategy: 'primary' - ), 1) - end + Class.new do + include Sidekiq::Worker + include WorkerAttributes + + if category + feature_category category + else + feature_category_not_owned! + end + + def perform end end end - context 'when load_balancing is disabled' do - include_context 'worker declaring data consistency' + let(:context_category) { 'continuous_integration' } + let(:job) { { 'meta.feature_category' => 'continuous_integration' } } - before do - allow(::Gitlab::Database::LoadBalancing).to receive(:enable?).and_return(false) + before do + stub_const('TestWorker', test_worker) + end + + around do |example| + with_sidekiq_server_middleware do |chain| + Gitlab::SidekiqMiddleware.server_configurator( + metrics: true, + arguments_logger: false, + memory_killer: false + ).call(chain) + + Sidekiq::Testing.inline! { example.run } end + end - describe '#initialize' do - it 'does not set load_balancing metrics' do - expect(Gitlab::Metrics).not_to receive(:counter).with(:sidekiq_load_balancing_count, anything) + include_context 'server metrics with mocked prometheus' + include_context 'server metrics call' - subject - end + context 'when a worker has a feature category' do + let(:worker_category) { 'authentication_and_authorization' } + + it 'uses that category for metrics' do + expect(completion_seconds_metric).to receive(:observe).with(a_hash_including(feature_category: worker_category), anything) + + TestWorker.process_job(job) end + end - describe '#call' do - it 'does not increment load balancing counter' do - process_job + context 'when a worker does not have a feature category' do + let(:worker_category) { nil } - expect(load_balancing_metric).not_to have_received(:increment) - end + it 'uses the category from the context for metrics' do + expect(completion_seconds_metric).to receive(:observe).with(a_hash_including(feature_category: context_category), anything) + + TestWorker.process_job(job) end end end diff --git a/spec/lib/gitlab/sidekiq_middleware/worker_context/client_spec.rb b/spec/lib/gitlab/sidekiq_middleware/worker_context/client_spec.rb index d6cc787f53d..92a11c83a4a 100644 --- a/spec/lib/gitlab/sidekiq_middleware/worker_context/client_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware/worker_context/client_spec.rb @@ -22,8 +22,31 @@ RSpec.describe Gitlab::SidekiqMiddleware::WorkerContext::Client do end end + let(:not_owned_worker_class) do + Class.new(worker_class) do + def self.name + 'TestNotOwnedWithContextWorker' + end + + feature_category_not_owned! + end + end + + let(:mailer_class) do + Class.new(ApplicationMailer) do + def self.name + 'TestMailer' + end + + def test_mail + end + end + end + before do - stub_const('TestWithContextWorker', worker_class) + stub_const(worker_class.name, worker_class) + stub_const(not_owned_worker_class.name, not_owned_worker_class) + stub_const(mailer_class.name, mailer_class) end describe "#call" do @@ -58,6 +81,26 @@ RSpec.describe Gitlab::SidekiqMiddleware::WorkerContext::Client do expect(job1['meta.feature_category']).to eq('issue_tracking') expect(job2['meta.feature_category']).to eq('issue_tracking') end + + it 'takes the feature category from the caller if the worker is not owned' do + TestNotOwnedWithContextWorker.bulk_perform_async_with_contexts( + %w(job1 job2), + arguments_proc: -> (name) { [name, 1, 2, 3] }, + context_proc: -> (_) { { feature_category: 'code_review' } } + ) + + job1 = TestNotOwnedWithContextWorker.job_for_args(['job1', 1, 2, 3]) + job2 = TestNotOwnedWithContextWorker.job_for_args(['job2', 1, 2, 3]) + + expect(job1['meta.feature_category']).to eq('code_review') + expect(job2['meta.feature_category']).to eq('code_review') + end + + it 'does not set any explicit feature category for mailers', :sidekiq_mailers do + expect(Gitlab::ApplicationContext).not_to receive(:with_context) + + TestMailer.test_mail.deliver_later + end end context 'when the feature category is already set in the surrounding block' do @@ -76,6 +119,22 @@ RSpec.describe Gitlab::SidekiqMiddleware::WorkerContext::Client do expect(job1['meta.feature_category']).to eq('issue_tracking') expect(job2['meta.feature_category']).to eq('issue_tracking') end + + it 'takes the feature category from the caller if the worker is not owned' do + Gitlab::ApplicationContext.with_context(feature_category: 'authentication_and_authorization') do + TestNotOwnedWithContextWorker.bulk_perform_async_with_contexts( + %w(job1 job2), + arguments_proc: -> (name) { [name, 1, 2, 3] }, + context_proc: -> (_) { {} } + ) + end + + job1 = TestNotOwnedWithContextWorker.job_for_args(['job1', 1, 2, 3]) + job2 = TestNotOwnedWithContextWorker.job_for_args(['job2', 1, 2, 3]) + + expect(job1['meta.feature_category']).to eq('authentication_and_authorization') + expect(job2['meta.feature_category']).to eq('authentication_and_authorization') + end end end end diff --git a/spec/lib/gitlab/sidekiq_middleware/worker_context/server_spec.rb b/spec/lib/gitlab/sidekiq_middleware/worker_context/server_spec.rb index f736a7db774..377ff6fd166 100644 --- a/spec/lib/gitlab/sidekiq_middleware/worker_context/server_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware/worker_context/server_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::SidekiqMiddleware::WorkerContext::Server do - let(:worker_class) do + let(:test_worker) do Class.new do def self.name "TestWorker" @@ -23,6 +23,16 @@ RSpec.describe Gitlab::SidekiqMiddleware::WorkerContext::Server do end end + let(:not_owned_worker) do + Class.new(test_worker) do + def self.name + "NotOwnedWorker" + end + + feature_category_not_owned! + end + end + let(:other_worker) do Class.new do def self.name @@ -37,7 +47,8 @@ RSpec.describe Gitlab::SidekiqMiddleware::WorkerContext::Server do end before do - stub_const("TestWorker", worker_class) + stub_const("TestWorker", test_worker) + stub_const("NotOwnedWorker", not_owned_worker) stub_const("OtherWorker", other_worker) end @@ -57,10 +68,24 @@ RSpec.describe Gitlab::SidekiqMiddleware::WorkerContext::Server do expect(TestWorker.contexts['identifier'].keys).not_to include('meta.user') end - it 'takes the feature category from the worker' do - TestWorker.perform_async('identifier', 1) + context 'feature category' do + it 'takes the feature category from the worker' do + Gitlab::ApplicationContext.with_context(feature_category: 'authentication_and_authorization') do + TestWorker.perform_async('identifier', 1) + end + + expect(TestWorker.contexts['identifier']).to include('meta.feature_category' => 'foo') + end - expect(TestWorker.contexts['identifier']).to include('meta.feature_category' => 'foo') + context 'when the worker is not owned' do + it 'takes the feature category from the surrounding context' do + Gitlab::ApplicationContext.with_context(feature_category: 'authentication_and_authorization') do + NotOwnedWorker.perform_async('identifier', 1) + end + + expect(NotOwnedWorker.contexts['identifier']).to include('meta.feature_category' => 'authentication_and_authorization') + end + end end it "doesn't fail for unknown workers" do diff --git a/spec/lib/gitlab/sidekiq_middleware_spec.rb b/spec/lib/gitlab/sidekiq_middleware_spec.rb index 8285cf960d2..e687c8e8cf7 100644 --- a/spec/lib/gitlab/sidekiq_middleware_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware_spec.rb @@ -28,9 +28,8 @@ RSpec.describe Gitlab::SidekiqMiddleware do stub_const('TestWorker', worker_class) end - shared_examples "a middleware chain" do |load_balancing_enabled| + shared_examples "a middleware chain" do before do - allow(::Gitlab::Database::LoadBalancing).to receive(:enable?).and_return(load_balancing_enabled) configurator.call(chain) end @@ -45,10 +44,10 @@ RSpec.describe Gitlab::SidekiqMiddleware do end end - shared_examples "a middleware chain for mailer" do |load_balancing_enabled| + shared_examples "a middleware chain for mailer" do let(:worker_class) { ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper } - it_behaves_like "a middleware chain", load_balancing_enabled + it_behaves_like "a middleware chain" end describe '.server_configurator' do @@ -58,13 +57,13 @@ RSpec.describe Gitlab::SidekiqMiddleware do let(:all_sidekiq_middlewares) do [ ::Gitlab::SidekiqMiddleware::Monitor, + ::Labkit::Middleware::Sidekiq::Server, ::Gitlab::SidekiqMiddleware::ServerMetrics, ::Gitlab::SidekiqMiddleware::ArgumentsLogger, ::Gitlab::SidekiqMiddleware::MemoryKiller, ::Gitlab::SidekiqMiddleware::RequestStoreMiddleware, ::Gitlab::SidekiqMiddleware::ExtraDoneLogMetadata, ::Gitlab::SidekiqMiddleware::BatchLoader, - ::Labkit::Middleware::Sidekiq::Server, ::Gitlab::SidekiqMiddleware::InstrumentationLogger, ::Gitlab::SidekiqMiddleware::AdminMode::Server, ::Gitlab::SidekiqVersioning::Middleware, @@ -105,25 +104,8 @@ RSpec.describe Gitlab::SidekiqMiddleware do end context "all optional middlewares on" do - context "when load balancing is enabled" do - before do - allow(::Gitlab::Database::LoadBalancing).to receive_message_chain(:proxy, :load_balancer, :release_host) - end - - it_behaves_like "a middleware chain", true - it_behaves_like "a middleware chain for mailer", true - end - - context "when load balancing is disabled" do - let(:disabled_sidekiq_middlewares) do - [ - Gitlab::Database::LoadBalancing::SidekiqServerMiddleware - ] - end - - it_behaves_like "a middleware chain", false - it_behaves_like "a middleware chain for mailer", false - end + it_behaves_like "a middleware chain" + it_behaves_like "a middleware chain for mailer" end context "all optional middlewares off" do @@ -135,36 +117,16 @@ RSpec.describe Gitlab::SidekiqMiddleware do ) end - context "when load balancing is enabled" do - let(:disabled_sidekiq_middlewares) do - [ - Gitlab::SidekiqMiddleware::ServerMetrics, - Gitlab::SidekiqMiddleware::ArgumentsLogger, - Gitlab::SidekiqMiddleware::MemoryKiller - ] - end - - before do - allow(::Gitlab::Database::LoadBalancing).to receive_message_chain(:proxy, :load_balancer, :release_host) - end - - it_behaves_like "a middleware chain", true - it_behaves_like "a middleware chain for mailer", true + let(:disabled_sidekiq_middlewares) do + [ + Gitlab::SidekiqMiddleware::ServerMetrics, + Gitlab::SidekiqMiddleware::ArgumentsLogger, + Gitlab::SidekiqMiddleware::MemoryKiller + ] end - context "when load balancing is disabled" do - let(:disabled_sidekiq_middlewares) do - [ - Gitlab::SidekiqMiddleware::ServerMetrics, - Gitlab::SidekiqMiddleware::ArgumentsLogger, - Gitlab::SidekiqMiddleware::MemoryKiller, - Gitlab::Database::LoadBalancing::SidekiqServerMiddleware - ] - end - - it_behaves_like "a middleware chain", false - it_behaves_like "a middleware chain for mailer", false - end + it_behaves_like "a middleware chain" + it_behaves_like "a middleware chain for mailer" end end @@ -186,30 +148,7 @@ RSpec.describe Gitlab::SidekiqMiddleware do ] end - context "when load balancing is disabled" do - let(:disabled_sidekiq_middlewares) do - [ - Gitlab::Database::LoadBalancing::SidekiqClientMiddleware - ] - end - - it_behaves_like "a middleware chain", false - it_behaves_like "a middleware chain for mailer", false - - # Sidekiq documentation states that the worker class could be a string - # or a class reference. We should test for both - context "worker_class as string value" do - let(:worker_args) { [worker_class.to_s, { 'args' => job_args }, queue, redis_pool] } - let(:middleware_expected_args) { [worker_class.to_s, hash_including({ 'args' => job_args }), queue, redis_pool] } - - it_behaves_like "a middleware chain", false - it_behaves_like "a middleware chain for mailer", false - end - end - - context "when load balancing is enabled" do - it_behaves_like "a middleware chain", true - it_behaves_like "a middleware chain for mailer", true - end + it_behaves_like "a middleware chain" + it_behaves_like "a middleware chain for mailer" end end diff --git a/spec/lib/gitlab/sidekiq_migrate_jobs_spec.rb b/spec/lib/gitlab/sidekiq_migrate_jobs_spec.rb index b30143ed196..d4391d3023a 100644 --- a/spec/lib/gitlab/sidekiq_migrate_jobs_spec.rb +++ b/spec/lib/gitlab/sidekiq_migrate_jobs_spec.rb @@ -65,7 +65,7 @@ RSpec.describe Gitlab::SidekiqMigrateJobs, :clean_gitlab_redis_queues do expect(item).to include('queue' => 'post_receive', 'args' => [i]) end - expect(score).to eq(i.succ.hours.from_now.to_i) + expect(score).to be_within(schedule_jitter).of(i.succ.hours.from_now.to_i) end end end @@ -84,7 +84,7 @@ RSpec.describe Gitlab::SidekiqMigrateJobs, :clean_gitlab_redis_queues do expect(item).to include('queue' => 'another_queue', 'args' => [i]) end - expect(score).to eq(i.succ.hours.from_now.to_i) + expect(score).to be_within(schedule_jitter).of(i.succ.hours.from_now.to_i) end end end @@ -98,7 +98,7 @@ RSpec.describe Gitlab::SidekiqMigrateJobs, :clean_gitlab_redis_queues do set_after.each.with_index do |(item, score), i| expect(item).to include('queue' => 'new_queue', 'args' => [i]) - expect(score).to eq(i.succ.hours.from_now.to_i) + expect(score).to be_within(schedule_jitter).of(i.succ.hours.from_now.to_i) end end end @@ -173,6 +173,7 @@ RSpec.describe Gitlab::SidekiqMigrateJobs, :clean_gitlab_redis_queues do context 'scheduled jobs' do let(:set_name) { 'schedule' } + let(:schedule_jitter) { 0 } def create_jobs(include_post_receive: true) AuthorizedProjectsWorker.perform_in(1.hour, 0) @@ -186,12 +187,14 @@ RSpec.describe Gitlab::SidekiqMigrateJobs, :clean_gitlab_redis_queues do context 'retried jobs' do let(:set_name) { 'retry' } + # Account for Sidekiq retry jitter + # https://github.com/mperham/sidekiq/blob/3575ccb44c688dd08bfbfd937696260b12c622fb/lib/sidekiq/job_retry.rb#L217 + let(:schedule_jitter) { 10 } # Try to mimic as closely as possible what Sidekiq will actually # do to retry a job. def retry_in(klass, time, args) - # In Sidekiq 6, this argument will become a JSON string - message = { 'class' => klass, 'args' => [args], 'retry' => true } + message = { 'class' => klass.name, 'args' => [args], 'retry' => true }.to_json allow(klass).to receive(:sidekiq_retry_in_block).and_return(proc { time }) diff --git a/spec/lib/gitlab/sidekiq_versioning/manager_spec.rb b/spec/lib/gitlab/sidekiq_versioning/manager_spec.rb deleted file mode 100644 index 84161d9236e..00000000000 --- a/spec/lib/gitlab/sidekiq_versioning/manager_spec.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::SidekiqVersioning::Manager do - before do - Sidekiq::Manager.prepend described_class - end - - describe '#initialize' do - it 'listens on all expanded queues' do - manager = Sidekiq::Manager.new(queues: %w[post_receive repository_fork cronjob unknown]) - - queues = manager.options[:queues] - - expect(queues).to include('post_receive') - expect(queues).to include('repository_fork') - expect(queues).to include('cronjob') - expect(queues).to include('cronjob:import_stuck_project_import_jobs') - expect(queues).to include('cronjob:jira_import_stuck_jira_import_jobs') - expect(queues).to include('cronjob:stuck_merge_jobs') - expect(queues).to include('unknown') - end - end -end diff --git a/spec/lib/gitlab/sidekiq_versioning_spec.rb b/spec/lib/gitlab/sidekiq_versioning_spec.rb index ed9650fc166..afafd04d87d 100644 --- a/spec/lib/gitlab/sidekiq_versioning_spec.rb +++ b/spec/lib/gitlab/sidekiq_versioning_spec.rb @@ -29,12 +29,6 @@ RSpec.describe Gitlab::SidekiqVersioning, :redis do end describe '.install!' do - it 'prepends SidekiqVersioning::Manager into Sidekiq::Manager' do - described_class.install! - - expect(Sidekiq::Manager).to include(Gitlab::SidekiqVersioning::Manager) - end - it 'registers all versionless and versioned queues with Redis' do described_class.install! diff --git a/spec/lib/gitlab/slash_commands/issue_move_spec.rb b/spec/lib/gitlab/slash_commands/issue_move_spec.rb index 5fffbb2d4cc..aa1341b4148 100644 --- a/spec/lib/gitlab/slash_commands/issue_move_spec.rb +++ b/spec/lib/gitlab/slash_commands/issue_move_spec.rb @@ -95,7 +95,7 @@ RSpec.describe Gitlab::SlashCommands::IssueMove, service: true do end end - context 'when the user cannot see the target project' do + context 'when the user cannot see the target project', :sidekiq_inline do it 'returns not found' do message = "issue move #{issue.iid} #{other_project.full_path}" other_project.team.truncate diff --git a/spec/lib/gitlab/subscription_portal_spec.rb b/spec/lib/gitlab/subscription_portal_spec.rb index 628eb380396..a3808b0f0e2 100644 --- a/spec/lib/gitlab/subscription_portal_spec.rb +++ b/spec/lib/gitlab/subscription_portal_spec.rb @@ -5,23 +5,96 @@ require 'spec_helper' RSpec.describe ::Gitlab::SubscriptionPortal do using RSpec::Parameterized::TableSyntax - where(:method_name, :test, :development, :result) do - :default_subscriptions_url | false | false | 'https://customers.gitlab.com' - :default_subscriptions_url | false | true | 'https://customers.stg.gitlab.com' - :default_subscriptions_url | true | false | 'https://customers.stg.gitlab.com' - :payment_form_url | false | false | 'https://customers.gitlab.com/payment_forms/cc_validation' - :payment_form_url | false | true | 'https://customers.stg.gitlab.com/payment_forms/cc_validation' - :payment_form_url | true | false | 'https://customers.stg.gitlab.com/payment_forms/cc_validation' + let(:env_value) { nil } + + before do + stub_env('CUSTOMER_PORTAL_URL', env_value) + stub_feature_flags(new_customersdot_staging_url: false) end - with_them do - subject { described_class.method(method_name).call } + describe '.default_subscriptions_url' do + where(:test, :development, :result) do + false | false | 'https://customers.gitlab.com' + false | true | 'https://customers.stg.gitlab.com' + true | false | 'https://customers.stg.gitlab.com' + end before do allow(Rails).to receive_message_chain(:env, :test?).and_return(test) allow(Rails).to receive_message_chain(:env, :development?).and_return(development) end - it { is_expected.to eq(result) } + with_them do + subject { described_class.default_subscriptions_url } + + it { is_expected.to eq(result) } + end + end + + describe '.subscriptions_url' do + subject { described_class.subscriptions_url } + + context 'when CUSTOMER_PORTAL_URL ENV is unset' do + it { is_expected.to eq('https://customers.stg.gitlab.com') } + end + + context 'when CUSTOMER_PORTAL_URL ENV is set' do + let(:env_value) { 'https://customers.example.com' } + + it { is_expected.to eq(env_value) } + end + end + + describe '.subscriptions_comparison_url' do + subject { described_class.subscriptions_comparison_url } + + link_match = %r{\Ahttps://about\.gitlab\.((cn/pricing/saas)|(com/pricing/gitlab-com))/feature-comparison\z} + + it { is_expected.to match(link_match) } + end + + context 'url methods' do + where(:method_name, :result) do + :default_subscriptions_url | 'https://customers.stg.gitlab.com' + :payment_form_url | 'https://customers.stg.gitlab.com/payment_forms/cc_validation' + :subscriptions_graphql_url | 'https://customers.stg.gitlab.com/graphql' + :subscriptions_more_minutes_url | 'https://customers.stg.gitlab.com/buy_pipeline_minutes' + :subscriptions_more_storage_url | 'https://customers.stg.gitlab.com/buy_storage' + :subscriptions_manage_url | 'https://customers.stg.gitlab.com/subscriptions' + :subscriptions_plans_url | 'https://customers.stg.gitlab.com/plans' + :subscriptions_instance_review_url | 'https://customers.stg.gitlab.com/instance_review' + :subscriptions_gitlab_plans_url | 'https://customers.stg.gitlab.com/gitlab_plans' + end + + with_them do + subject { described_class.send(method_name) } + + it { is_expected.to eq(result) } + end + end + + describe '.add_extra_seats_url' do + subject { described_class.add_extra_seats_url(group_id) } + + let(:group_id) { 153 } + + it { is_expected.to eq("https://customers.stg.gitlab.com/gitlab/namespaces/#{group_id}/extra_seats") } + end + + describe '.upgrade_subscription_url' do + subject { described_class.upgrade_subscription_url(group_id, plan_id) } + + let(:group_id) { 153 } + let(:plan_id) { 5 } + + it { is_expected.to eq("https://customers.stg.gitlab.com/gitlab/namespaces/#{group_id}/upgrade/#{plan_id}") } + end + + describe '.renew_subscription_url' do + subject { described_class.renew_subscription_url(group_id) } + + let(:group_id) { 153 } + + it { is_expected.to eq("https://customers.stg.gitlab.com/gitlab/namespaces/#{group_id}/renew") } end end diff --git a/spec/lib/gitlab/tracking/docs/helper_spec.rb b/spec/lib/gitlab/tracking/docs/helper_spec.rb deleted file mode 100644 index 5f7965502f1..00000000000 --- a/spec/lib/gitlab/tracking/docs/helper_spec.rb +++ /dev/null @@ -1,91 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Tracking::Docs::Helper do - let_it_be(:klass) do - Class.new do - include Gitlab::Tracking::Docs::Helper - end - end - - describe '#auto_generated_comment' do - it 'renders information about missing description' do - expect(klass.new.auto_generated_comment).to match /This documentation is auto generated by a script/ - end - end - - describe '#render_description' do - context 'description is empty' do - it 'renders information about missing description' do - object = double(description: '') - - expect(klass.new.render_description(object)).to eq('Missing description') - end - end - - context 'description is present' do - it 'render description' do - object = double(description: 'some description') - - expect(klass.new.render_description(object)).to eq('some description') - end - end - end - - describe '#render_event_taxonomy' do - it 'render table with event taxonomy' do - attributes = { - category: 'epics', - action: 'promote', - label: nil, - property_description: 'String with issue id', - value_description: 'Integer issue id' - } - object = double(attributes: attributes) - event_taxonomy = <<~MD.chomp - | category | action | label | property | value | - |---|---|---|---|---| - | `epics` | `promote` | `` | `String with issue id` | `Integer issue id` | - MD - - expect(klass.new.render_event_taxonomy(object)).to eq(event_taxonomy) - end - end - - describe '#md_link_to' do - it 'render link in md format' do - expect(klass.new.md_link_to('zelda', 'link')).to eq('[zelda](link)') - end - end - - describe '#render_owner' do - it 'render information about group owning event' do - object = double(product_group: "group::product intelligence") - - expect(klass.new.render_owner(object)).to eq("Owner: `group::product intelligence`") - end - end - - describe '#render_tiers' do - it 'render information about tiers' do - object = double(tiers: %w[bronze silver gold]) - - expect(klass.new.render_tiers(object)).to eq("Tiers: `bronze`, `silver`, `gold`") - end - end - - describe '#render_yaml_definition_path' do - it 'render relative location of yaml definition' do - object = double(yaml_path: 'config/events/button_click.yaml') - - expect(klass.new.render_yaml_definition_path(object)).to eq("YAML definition: `config/events/button_click.yaml`") - end - end - - describe '#backtick' do - it 'wraps string in backticks chars' do - expect(klass.new.backtick('test')).to eql("`test`") - end - end -end diff --git a/spec/lib/gitlab/tracking/docs/renderer_spec.rb b/spec/lib/gitlab/tracking/docs/renderer_spec.rb deleted file mode 100644 index 386aea6c23a..00000000000 --- a/spec/lib/gitlab/tracking/docs/renderer_spec.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Tracking::Docs::Renderer do - describe 'contents' do - let(:dictionary_path) { described_class::DICTIONARY_PATH } - let(:items) { Gitlab::Tracking::EventDefinition.definitions.first(10).to_h } - - it 'generates dictionary for given items' do - generated_dictionary = described_class.new(items).contents - table_of_contents_items = items.values.map { |item| "#{item.category} #{item.action}"} - - generated_dictionary_keys = RDoc::Markdown - .parse(generated_dictionary) - .table_of_contents - .select { |metric_doc| metric_doc.level == 3 } - .map { |item| item.text.match(%r{<code>(.*)</code>})&.captures&.first } - - expect(generated_dictionary_keys).to match_array(table_of_contents_items) - end - end -end diff --git a/spec/lib/gitlab/tracking/standard_context_spec.rb b/spec/lib/gitlab/tracking/standard_context_spec.rb index ca7a6b6b1c3..8ded80dd191 100644 --- a/spec/lib/gitlab/tracking/standard_context_spec.rb +++ b/spec/lib/gitlab/tracking/standard_context_spec.rb @@ -5,6 +5,7 @@ require 'spec_helper' RSpec.describe Gitlab::Tracking::StandardContext do let_it_be(:project) { create(:project) } let_it_be(:namespace) { create(:namespace) } + let_it_be(:user) { create(:user) } let(:snowplow_context) { subject.to_context } @@ -87,8 +88,8 @@ RSpec.describe Gitlab::Tracking::StandardContext do end end - it 'does not contain user id' do - expect(snowplow_context.to_json[:data].keys).not_to include(:user_id) + it 'contains user id' do + expect(snowplow_context.to_json[:data].keys).to include(:user_id) end it 'contains namespace and project ids' do @@ -104,8 +105,18 @@ RSpec.describe Gitlab::Tracking::StandardContext do stub_feature_flags(add_namespace_and_project_to_snowplow_tracking: false) end - it 'does not contain any ids' do - expect(snowplow_context.to_json[:data].keys).not_to include(:user_id, :project_id, :namespace_id) + it 'does not contain project or namespace ids' do + expect(snowplow_context.to_json[:data].keys).not_to include(:project_id, :namespace_id) + end + end + + context 'without add_actor_based_user_to_snowplow_tracking feature' do + before do + stub_feature_flags(add_actor_based_user_to_snowplow_tracking: false) + end + + it 'does not contain user_id' do + expect(snowplow_context.to_json[:data].keys).not_to include(:user_id) end end end diff --git a/spec/lib/gitlab/tracking_spec.rb b/spec/lib/gitlab/tracking_spec.rb index 02e66458f46..dacaae55676 100644 --- a/spec/lib/gitlab/tracking_spec.rb +++ b/spec/lib/gitlab/tracking_spec.rb @@ -48,7 +48,7 @@ RSpec.describe Gitlab::Tracking do other_context = double(:context) project = build_stubbed(:project) - user = double(:user) + user = build_stubbed(:user) expect(Gitlab::Tracking::StandardContext) .to receive(:new) diff --git a/spec/lib/gitlab/usage/metric_definition_spec.rb b/spec/lib/gitlab/usage/metric_definition_spec.rb index 6406c0b5458..522f69062fb 100644 --- a/spec/lib/gitlab/usage/metric_definition_spec.rb +++ b/spec/lib/gitlab/usage/metric_definition_spec.rb @@ -49,6 +49,37 @@ RSpec.describe Gitlab::Usage::MetricDefinition do expect { described_class.definitions }.not_to raise_error end + describe '#with_instrumentation_class' do + let(:metric_status) { 'active' } + let(:all_definitions) do + metrics_definitions = [ + { key_path: 'metric1', instrumentation_class: 'RedisHLLMetric', status: 'data_available' }, + { key_path: 'metric2', instrumentation_class: 'RedisHLLMetric', status: 'implemented' }, + { key_path: 'metric3', instrumentation_class: 'RedisHLLMetric', status: 'deprecated' }, + { key_path: 'metric4', instrumentation_class: 'RedisHLLMetric', status: metric_status }, + { key_path: 'metric5', status: 'active' }, + { key_path: 'metric_missing_status' } + ] + metrics_definitions.map { |definition| described_class.new(definition[:key_path], definition.symbolize_keys) } + end + + before do + allow(described_class).to receive(:all).and_return(all_definitions) + end + + it 'includes definitions with instrumentation_class' do + expect(described_class.with_instrumentation_class.count).to eq(4) + end + + context 'with removed metric' do + let(:metric_status) { 'removed' } + + it 'excludes removed definitions' do + expect(described_class.with_instrumentation_class.count).to eq(3) + end + end + end + describe '#key' do subject { definition.key } diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/active_user_count_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/active_user_count_metric_spec.rb new file mode 100644 index 00000000000..f0ee6c38f2e --- /dev/null +++ b/spec/lib/gitlab/usage/metrics/instrumentations/active_user_count_metric_spec.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Usage::Metrics::Instrumentations::ActiveUserCountMetric do + before do + create(:user) + end + + it_behaves_like 'a correct instrumented metric value', { time_frame: 'none', data_source: 'ruby' } do + let(:expected_value) { ::User.active.count } + end +end diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/count_users_associating_milestones_to_releases_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/count_users_associating_milestones_to_releases_metric_spec.rb new file mode 100644 index 00000000000..e2bb99c832a --- /dev/null +++ b/spec/lib/gitlab/usage/metrics/instrumentations/count_users_associating_milestones_to_releases_metric_spec.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Usage::Metrics::Instrumentations::CountUsersAssociatingMilestonesToReleasesMetric do + let_it_be(:release) { create(:release, created_at: 3.days.ago) } + let_it_be(:release_with_milestone) { create(:release, :with_milestones, created_at: 3.days.ago) } + + it_behaves_like 'a correct instrumented metric value', { time_frame: '28d', data_source: 'database' } do + let(:expected_value) { 1 } + end +end diff --git a/spec/lib/gitlab/usage_data_counters/ci_template_unique_counter_spec.rb b/spec/lib/gitlab/usage_data_counters/ci_template_unique_counter_spec.rb index 4996b0a0089..222198a58ac 100644 --- a/spec/lib/gitlab/usage_data_counters/ci_template_unique_counter_spec.rb +++ b/spec/lib/gitlab/usage_data_counters/ci_template_unique_counter_spec.rb @@ -6,97 +6,62 @@ RSpec.describe Gitlab::UsageDataCounters::CiTemplateUniqueCounter do describe '.track_unique_project_event' do using RSpec::Parameterized::TableSyntax - where(:template, :config_source, :expected_event) do - # Implicit Auto DevOps usage - 'Auto-DevOps.gitlab-ci.yml' | :auto_devops_source | 'p_ci_templates_implicit_auto_devops' - 'Jobs/Build.gitlab-ci.yml' | :auto_devops_source | 'p_ci_templates_implicit_auto_devops_build' - 'Jobs/Deploy.gitlab-ci.yml' | :auto_devops_source | 'p_ci_templates_implicit_auto_devops_deploy' - 'Security/SAST.gitlab-ci.yml' | :auto_devops_source | 'p_ci_templates_implicit_security_sast' - 'Security/Secret-Detection.gitlab-ci.yml' | :auto_devops_source | 'p_ci_templates_implicit_security_secret_detection' - # Explicit include:template usage - '5-Minute-Production-App.gitlab-ci.yml' | :repository_source | 'p_ci_templates_5_min_production_app' - 'Auto-DevOps.gitlab-ci.yml' | :repository_source | 'p_ci_templates_auto_devops' - 'AWS/CF-Provision-and-Deploy-EC2.gitlab-ci.yml' | :repository_source | 'p_ci_templates_aws_cf_deploy_ec2' - 'AWS/Deploy-ECS.gitlab-ci.yml' | :repository_source | 'p_ci_templates_aws_deploy_ecs' - 'Jobs/Build.gitlab-ci.yml' | :repository_source | 'p_ci_templates_auto_devops_build' - 'Jobs/Deploy.gitlab-ci.yml' | :repository_source | 'p_ci_templates_auto_devops_deploy' - 'Jobs/Deploy.latest.gitlab-ci.yml' | :repository_source | 'p_ci_templates_auto_devops_deploy_latest' - 'Security/SAST.gitlab-ci.yml' | :repository_source | 'p_ci_templates_security_sast' - 'Security/Secret-Detection.gitlab-ci.yml' | :repository_source | 'p_ci_templates_security_secret_detection' - 'Terraform/Base.latest.gitlab-ci.yml' | :repository_source | 'p_ci_templates_terraform_base_latest' - end - - with_them do - it_behaves_like 'tracking unique hll events' do - subject(:request) { described_class.track_unique_project_event(project_id: project_id, template: template, config_source: config_source) } + let(:project_id) { 1 } - let(:project_id) { 1 } - let(:target_id) { expected_event } - let(:expected_type) { instance_of(Integer) } + shared_examples 'tracks template' do + it "has an event defined for template" do + expect do + described_class.track_unique_project_event( + project_id: project_id, + template: template_path, + config_source: config_source + ) + end.not_to raise_error end - end - context 'known_events coverage tests' do - let(:project_id) { 1 } - let(:config_source) { :repository_source } + it "tracks template" do + expanded_template_name = described_class.expand_template_name(template_path) + expected_template_event_name = described_class.ci_template_event_name(expanded_template_name, config_source) + expect(Gitlab::UsageDataCounters::HLLRedisCounter).to(receive(:track_event)).with(expected_template_event_name, values: project_id) - # These tests help guard against missing "explicit" events in known_events/ci_templates.yml - context 'explicit include:template events' do - described_class::TEMPLATE_TO_EVENT.keys.each do |template| - it "does not raise error for #{template}" do - expect do - described_class.track_unique_project_event(project_id: project_id, template: template, config_source: config_source) - end.not_to raise_error - end - end + described_class.track_unique_project_event(project_id: project_id, template: template_path, config_source: config_source) end + end - # This test is to help guard against missing "implicit" events in known_events/ci_templates.yml - it 'does not raise error for any template in an implicit Auto DevOps pipeline' do - project = create(:project, :auto_devops) - pipeline = double(project: project) - command = double - result = Gitlab::Ci::YamlProcessor.new( - Gitlab::Ci::Pipeline::Chain::Config::Content::AutoDevops.new(pipeline, command).content, - project: project, - user: double, - sha: 'd310cc759caaa20cd05a9e0983d6017896d9c34c' - ).execute + context 'with explicit includes' do + let(:config_source) { :repository_source } - config_source = :auto_devops_source + (described_class.ci_templates - ['Verify/Browser-Performance.latest.gitlab-ci.yml', 'Verify/Browser-Performance.gitlab-ci.yml']).each do |template| + context "for #{template}" do + let(:template_path) { template } - result.included_templates.each do |template| - expect do - described_class.track_unique_project_event(project_id: project.id, template: template, config_source: config_source) - end.not_to raise_error + include_examples 'tracks template' end end end - context 'templates outside of TEMPLATE_TO_EVENT' do - let(:project_id) { 1 } - let(:config_source) { :repository_source } - - described_class.ci_templates.each do |template| - next if described_class::TEMPLATE_TO_EVENT.key?(template) - - it "has an event defined for #{template}" do - expect do - described_class.track_unique_project_event( - project_id: project_id, - template: template, - config_source: config_source - ) - end.not_to raise_error - end + context 'with implicit includes' do + let(:config_source) { :auto_devops_source } - it "tracks #{template}" do - expected_template_event_name = described_class.ci_template_event_name(template, :repository_source) - expect(Gitlab::UsageDataCounters::HLLRedisCounter).to(receive(:track_event)).with(expected_template_event_name, values: project_id) + [ + ['', ['Auto-DevOps.gitlab-ci.yml']], + ['Jobs', described_class.ci_templates('lib/gitlab/ci/templates/Jobs')], + ['Security', described_class.ci_templates('lib/gitlab/ci/templates/Security')] + ].each do |directory, templates| + templates.each do |template| + context "for #{template}" do + let(:template_path) { File.join(directory, template) } - described_class.track_unique_project_event(project_id: project_id, template: template, config_source: config_source) + include_examples 'tracks template' + end end end end + + it 'expands short template names' do + expect do + described_class.track_unique_project_event(project_id: 1, template: 'Dependency-Scanning.gitlab-ci.yml', config_source: :repository_source) + end.not_to raise_error + end end end diff --git a/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb b/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb index 427dd4a205e..0ec805714e3 100644 --- a/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb +++ b/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb @@ -47,6 +47,7 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s 'epics_usage', 'epic_boards_usage', 'secure', + 'importer', 'network_policies' ) end diff --git a/spec/lib/gitlab/usage_data_metrics_spec.rb b/spec/lib/gitlab/usage_data_metrics_spec.rb index e0063194f9b..ee0cfb1407e 100644 --- a/spec/lib/gitlab/usage_data_metrics_spec.rb +++ b/spec/lib/gitlab/usage_data_metrics_spec.rb @@ -46,7 +46,7 @@ RSpec.describe Gitlab::UsageDataMetrics do let(:metric_files_key_paths) do Gitlab::Usage::MetricDefinition .definitions - .select { |k, v| v.attributes[:data_source] == 'redis_hll' && v.key_path.starts_with?('redis_hll_counters') } + .select { |k, v| v.attributes[:data_source] == 'redis_hll' && v.key_path.starts_with?('redis_hll_counters') && v.available? } .keys .sort end diff --git a/spec/lib/gitlab/usage_data_spec.rb b/spec/lib/gitlab/usage_data_spec.rb index a70b68a181f..833bf260019 100644 --- a/spec/lib/gitlab/usage_data_spec.rb +++ b/spec/lib/gitlab/usage_data_spec.rb @@ -101,11 +101,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do it 'includes accurate usage_activity_by_stage data' do for_defined_days_back do user = create(:user) - cluster = create(:cluster, user: user) - create(:clusters_applications_cert_manager, :installed, cluster: cluster) - create(:clusters_applications_helm, :installed, cluster: cluster) - create(:clusters_applications_ingress, :installed, cluster: cluster) - create(:clusters_applications_knative, :installed, cluster: cluster) + create(:cluster, user: user) create(:cluster, :disabled, user: user) create(:cluster_provider_gcp, :created) create(:cluster_provider_aws, :created) @@ -118,10 +114,6 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do end expect(described_class.usage_activity_by_stage_configure({})).to include( - clusters_applications_cert_managers: 2, - clusters_applications_helm: 2, - clusters_applications_ingress: 2, - clusters_applications_knative: 2, clusters_management_project: 2, clusters_disabled: 4, clusters_enabled: 12, @@ -136,10 +128,6 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do project_clusters_enabled: 10 ) expect(described_class.usage_activity_by_stage_configure(described_class.monthly_time_range_db_params)).to include( - clusters_applications_cert_managers: 1, - clusters_applications_helm: 1, - clusters_applications_ingress: 1, - clusters_applications_knative: 1, clusters_management_project: 1, clusters_disabled: 2, clusters_enabled: 6, @@ -392,7 +380,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do user = create(:user, dashboard: 'operations') cluster = create(:cluster, user: user) project = create(:project, creator: user) - create(:clusters_applications_prometheus, :installed, cluster: cluster) + create(:clusters_integrations_prometheus, cluster: cluster) create(:project_tracing_setting) create(:project_error_tracking_setting) create(:incident) @@ -402,7 +390,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do expect(described_class.usage_activity_by_stage_monitor({})).to include( clusters: 2, - clusters_applications_prometheus: 2, + clusters_integrations_prometheus: 2, operations_dashboard_default_dashboard: 2, projects_with_tracing_enabled: 2, projects_with_error_tracking_enabled: 2, @@ -414,7 +402,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do data_28_days = described_class.usage_activity_by_stage_monitor(described_class.monthly_time_range_db_params) expect(data_28_days).to include( clusters: 1, - clusters_applications_prometheus: 1, + clusters_integrations_prometheus: 1, operations_dashboard_default_dashboard: 1, projects_with_tracing_enabled: 1, projects_with_error_tracking_enabled: 1, @@ -469,7 +457,8 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do for_defined_days_back do user = create(:user) create(:deployment, :failed, user: user) - create(:release, author: user) + release = create(:release, author: user) + create(:milestone, project: release.project, releases: [release]) create(:deployment, :success, user: user) end @@ -477,13 +466,15 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do deployments: 2, failed_deployments: 2, releases: 2, - successful_deployments: 2 + successful_deployments: 2, + releases_with_milestones: 2 ) expect(described_class.usage_activity_by_stage_release(described_class.monthly_time_range_db_params)).to include( deployments: 1, failed_deployments: 1, releases: 1, - successful_deployments: 1 + successful_deployments: 1, + releases_with_milestones: 1 ) end end @@ -499,7 +490,6 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do create(:ci_pipeline, :repository_source, user: user) create(:ci_pipeline_schedule, owner: user) create(:ci_trigger, owner: user) - create(:clusters_applications_runner, :installed) end expect(described_class.usage_activity_by_stage_verify({})).to include( @@ -510,8 +500,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do ci_pipeline_config_repository: 2, ci_pipeline_schedules: 2, ci_pipelines: 2, - ci_triggers: 2, - clusters_applications_runner: 2 + ci_triggers: 2 ) expect(described_class.usage_activity_by_stage_verify(described_class.monthly_time_range_db_params)).to include( ci_builds: 1, @@ -521,8 +510,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do ci_pipeline_config_repository: 1, ci_pipeline_schedules: 1, ci_pipelines: 1, - ci_triggers: 1, - clusters_applications_runner: 1 + ci_triggers: 1 ) end end @@ -604,17 +592,9 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do expect(count_data[:clusters_platforms_eks]).to eq(1) expect(count_data[:clusters_platforms_gke]).to eq(1) expect(count_data[:clusters_platforms_user]).to eq(1) - expect(count_data[:clusters_applications_helm]).to eq(1) - expect(count_data[:clusters_applications_ingress]).to eq(1) - expect(count_data[:clusters_applications_cert_managers]).to eq(1) - expect(count_data[:clusters_applications_crossplane]).to eq(1) - expect(count_data[:clusters_applications_prometheus]).to eq(1) - expect(count_data[:clusters_applications_runner]).to eq(1) - expect(count_data[:clusters_applications_knative]).to eq(1) - expect(count_data[:clusters_applications_elastic_stack]).to eq(1) + expect(count_data[:clusters_integrations_elastic_stack]).to eq(1) + expect(count_data[:clusters_integrations_prometheus]).to eq(1) expect(count_data[:grafana_integrated_projects]).to eq(2) - expect(count_data[:clusters_applications_jupyter]).to eq(1) - expect(count_data[:clusters_applications_cilium]).to eq(1) expect(count_data[:clusters_management_project]).to eq(1) expect(count_data[:kubernetes_agents]).to eq(2) expect(count_data[:kubernetes_agents_with_token]).to eq(1) @@ -662,13 +642,13 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do subject { described_class.data[:counts] } it 'gathers usage data' do - expect(subject[:projects_with_expiration_policy_enabled]).to eq 18 + expect(subject[:projects_with_expiration_policy_enabled]).to eq 19 expect(subject[:projects_with_expiration_policy_disabled]).to eq 5 expect(subject[:projects_with_expiration_policy_enabled_with_keep_n_unset]).to eq 1 expect(subject[:projects_with_expiration_policy_enabled_with_keep_n_set_to_1]).to eq 1 expect(subject[:projects_with_expiration_policy_enabled_with_keep_n_set_to_5]).to eq 1 - expect(subject[:projects_with_expiration_policy_enabled_with_keep_n_set_to_10]).to eq 12 + expect(subject[:projects_with_expiration_policy_enabled_with_keep_n_set_to_10]).to eq 13 expect(subject[:projects_with_expiration_policy_enabled_with_keep_n_set_to_25]).to eq 1 expect(subject[:projects_with_expiration_policy_enabled_with_keep_n_set_to_50]).to eq 1 @@ -676,9 +656,10 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do expect(subject[:projects_with_expiration_policy_enabled_with_older_than_set_to_7d]).to eq 1 expect(subject[:projects_with_expiration_policy_enabled_with_older_than_set_to_14d]).to eq 1 expect(subject[:projects_with_expiration_policy_enabled_with_older_than_set_to_30d]).to eq 1 + expect(subject[:projects_with_expiration_policy_enabled_with_older_than_set_to_60d]).to eq 1 expect(subject[:projects_with_expiration_policy_enabled_with_older_than_set_to_90d]).to eq 14 - expect(subject[:projects_with_expiration_policy_enabled_with_cadence_set_to_1d]).to eq 14 + expect(subject[:projects_with_expiration_policy_enabled_with_cadence_set_to_1d]).to eq 15 expect(subject[:projects_with_expiration_policy_enabled_with_cadence_set_to_7d]).to eq 1 expect(subject[:projects_with_expiration_policy_enabled_with_cadence_set_to_14d]).to eq 1 expect(subject[:projects_with_expiration_policy_enabled_with_cadence_set_to_1month]).to eq 1 @@ -743,7 +724,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do expect(counts_monthly[:projects_with_alerts_created]).to eq(1) expect(counts_monthly[:projects]).to eq(1) expect(counts_monthly[:packages]).to eq(1) - expect(counts_monthly[:promoted_issues]).to eq(1) + expect(counts_monthly[:promoted_issues]).to eq(Gitlab::UsageData::DEPRECATED_VALUE) end end @@ -1093,6 +1074,10 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do it 'gathers service_ping_features_enabled' do expect(subject[:settings][:service_ping_features_enabled]).to eq(Gitlab::CurrentSettings.usage_ping_features_enabled) end + + it 'gathers user_cap_feature_enabled' do + expect(subject[:settings][:user_cap_feature_enabled]).to eq(Gitlab::CurrentSettings.new_user_signups_cap) + end end end @@ -1438,48 +1423,4 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do end end end - - describe '.snowplow_event_counts' do - let_it_be(:time_period) { { collector_tstamp: 8.days.ago..1.day.ago } } - - context 'when self-monitoring project exists' do - let_it_be(:project) { create(:project) } - - before do - stub_application_setting(self_monitoring_project: project) - end - - context 'and product_analytics FF is enabled for it' do - before do - stub_feature_flags(product_analytics_tracking: true) - - create(:product_analytics_event, project: project, se_category: 'epics', se_action: 'promote') - create(:product_analytics_event, project: project, se_category: 'epics', se_action: 'promote', collector_tstamp: 2.days.ago) - create(:product_analytics_event, project: project, se_category: 'epics', se_action: 'promote', collector_tstamp: 9.days.ago) - - create(:product_analytics_event, project: project, se_category: 'foo', se_action: 'bar', collector_tstamp: 2.days.ago) - end - - it 'returns promoted_issues for the time period' do - expect(described_class.snowplow_event_counts(time_period)[:promoted_issues]).to eq(1) - end - end - - context 'and product_analytics FF is disabled' do - before do - stub_feature_flags(product_analytics_tracking: false) - end - - it 'returns an empty hash' do - expect(described_class.snowplow_event_counts(time_period)).to eq({}) - end - end - end - - context 'when self-monitoring project does not exist' do - it 'returns an empty hash' do - expect(described_class.snowplow_event_counts(time_period)).to eq({}) - end - end - end end diff --git a/spec/lib/gitlab/utils/delegator_override/error_spec.rb b/spec/lib/gitlab/utils/delegator_override/error_spec.rb new file mode 100644 index 00000000000..59b67676eff --- /dev/null +++ b/spec/lib/gitlab/utils/delegator_override/error_spec.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +RSpec.describe Gitlab::Utils::DelegatorOverride::Error do + let(:error) { described_class.new('foo', 'Target', '/path/to/target', 'Delegator', '/path/to/delegator') } + + describe '#to_s' do + subject { error.to_s } + + it { is_expected.to eq("Delegator#foo is overriding Target#foo. delegator_location: /path/to/delegator target_location: /path/to/target") } + end +end diff --git a/spec/lib/gitlab/utils/delegator_override/validator_spec.rb b/spec/lib/gitlab/utils/delegator_override/validator_spec.rb new file mode 100644 index 00000000000..4cd1b18de82 --- /dev/null +++ b/spec/lib/gitlab/utils/delegator_override/validator_spec.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +RSpec.describe Gitlab::Utils::DelegatorOverride::Validator do + let(:delegator_class) do + Class.new(::SimpleDelegator) do + extend(::Gitlab::Utils::DelegatorOverride) + + def foo + end + end.prepend(ee_delegator_extension) + end + + let(:ee_delegator_extension) do + Module.new do + extend(::Gitlab::Utils::DelegatorOverride) + + def bar + end + end + end + + let(:target_class) do + Class.new do + def foo + end + + def bar + end + end + end + + let(:validator) { described_class.new(delegator_class) } + + describe '#add_allowlist' do + it 'adds a method name to the allowlist' do + validator.add_allowlist([:foo]) + + expect(validator.allowed_method_names).to contain_exactly(:foo) + end + end + + describe '#add_target' do + it 'adds the target class' do + validator.add_target(target_class) + + expect(validator.target_classes).to contain_exactly(target_class) + end + end + + describe '#expand_on_ancestors' do + it 'adds the allowlist in the ancestors' do + ancestor_validator = described_class.new(ee_delegator_extension) + ancestor_validator.add_allowlist([:bar]) + validator.expand_on_ancestors( { ee_delegator_extension => ancestor_validator }) + + expect(validator.allowed_method_names).to contain_exactly(:bar) + end + end + + describe '#validate_overrides!' do + before do + validator.add_target(target_class) + end + + it 'does not raise an error when the overrides are allowed' do + validator.add_allowlist([:foo]) + ancestor_validator = described_class.new(ee_delegator_extension) + ancestor_validator.add_allowlist([:bar]) + validator.expand_on_ancestors( { ee_delegator_extension => ancestor_validator }) + + expect { validator.validate_overrides! }.not_to raise_error + end + + it 'raises an error when there is an override' do + expect { validator.validate_overrides! } + .to raise_error(described_class::UnexpectedDelegatorOverrideError) + end + end +end diff --git a/spec/lib/gitlab/utils/delegator_override_spec.rb b/spec/lib/gitlab/utils/delegator_override_spec.rb new file mode 100644 index 00000000000..af4c7fa5d8e --- /dev/null +++ b/spec/lib/gitlab/utils/delegator_override_spec.rb @@ -0,0 +1,97 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +RSpec.describe Gitlab::Utils::DelegatorOverride do + let(:delegator_class) do + Class.new(::SimpleDelegator) do + extend(::Gitlab::Utils::DelegatorOverride) + + def foo + end + end + end + + let(:target_class) do + Class.new do + def foo + end + + def bar + end + end + end + + let(:dummy_module) do + Module.new do + def foobar + end + end + end + + before do + stub_env('STATIC_VERIFICATION', 'true') + end + + describe '.delegator_target' do + subject { delegator_class.delegator_target(target_class) } + + it 'sets the delegator target to the validator' do + expect(described_class.validator(delegator_class)) + .to receive(:add_target).with(target_class) + + subject + end + + context 'when the class does not inherit SimpleDelegator' do + let(:delegator_class) do + Class.new do + extend(::Gitlab::Utils::DelegatorOverride) + end + end + + it 'raises an error' do + expect { subject }.to raise_error(ArgumentError, /not a subclass of 'SimpleDelegator' class/) + end + end + end + + describe '.delegator_override' do + subject { delegator_class.delegator_override(:foo) } + + it 'adds the method name to the allowlist' do + expect(described_class.validator(delegator_class)) + .to receive(:add_allowlist).with([:foo]) + + subject + end + end + + describe '.delegator_override_with' do + subject { delegator_class.delegator_override_with(dummy_module) } + + it 'adds the method names of the module to the allowlist' do + expect(described_class.validator(delegator_class)) + .to receive(:add_allowlist).with([:foobar]) + + subject + end + end + + describe '.verify!' do + subject { described_class.verify! } + + it 'does not raise an error when an override is in allowlist' do + delegator_class.delegator_target(target_class) + delegator_class.delegator_override(:foo) + + expect { subject }.not_to raise_error + end + + it 'raises an error when there is an override' do + delegator_class.delegator_target(target_class) + + expect { subject }.to raise_error(Gitlab::Utils::DelegatorOverride::Validator::UnexpectedDelegatorOverrideError) + end + end +end diff --git a/spec/lib/gitlab/view/presenter/base_spec.rb b/spec/lib/gitlab/view/presenter/base_spec.rb index 97d5e2b280d..a7083bd2722 100644 --- a/spec/lib/gitlab/view/presenter/base_spec.rb +++ b/spec/lib/gitlab/view/presenter/base_spec.rb @@ -18,11 +18,43 @@ RSpec.describe Gitlab::View::Presenter::Base do describe '.presents' do it 'exposes #subject with the given keyword' do - presenter_class.presents(:foo) + presenter_class.presents(Object, as: :foo) presenter = presenter_class.new(project) expect(presenter.foo).to eq(project) end + + it 'raises an error when symbol is passed' do + expect { presenter_class.presents(:foo) }.to raise_error(ArgumentError) + end + + context 'when the presenter class inherits Presenter::Delegated' do + let(:presenter_class) do + Class.new(::Gitlab::View::Presenter::Delegated) do + include(::Gitlab::View::Presenter::Base) + end + end + + it 'sets the delegator target' do + expect(presenter_class).to receive(:delegator_target).with(Object) + + presenter_class.presents(Object, as: :foo) + end + end + + context 'when the presenter class inherits Presenter::Simple' do + let(:presenter_class) do + Class.new(::Gitlab::View::Presenter::Simple) do + include(::Gitlab::View::Presenter::Base) + end + end + + it 'does not set the delegator target' do + expect(presenter_class).not_to receive(:delegator_target).with(Object) + + presenter_class.presents(Object, as: :foo) + end + end end describe '#can?' do diff --git a/spec/lib/gitlab/with_feature_category_spec.rb b/spec/lib/gitlab/with_feature_category_spec.rb deleted file mode 100644 index b6fe1c84b26..00000000000 --- a/spec/lib/gitlab/with_feature_category_spec.rb +++ /dev/null @@ -1,69 +0,0 @@ -# frozen_string_literal: true - -require 'fast_spec_helper' -require_relative "../../../lib/gitlab/with_feature_category" - -RSpec.describe Gitlab::WithFeatureCategory do - describe ".feature_category_for_action" do - let(:base_controller) do - Class.new do - include ::Gitlab::WithFeatureCategory - end - end - - let(:controller) do - Class.new(base_controller) do - feature_category :foo, %w(update edit) - feature_category :bar, %w(index show) - feature_category :quux, %w(destroy) - end - end - - let(:subclass) do - Class.new(controller) do - feature_category :baz, %w(subclass_index) - end - end - - it "is nil when nothing was defined" do - expect(base_controller.feature_category_for_action("hello")).to be_nil - end - - it "returns the expected category", :aggregate_failures do - expect(controller.feature_category_for_action("update")).to eq(:foo) - expect(controller.feature_category_for_action("index")).to eq(:bar) - expect(controller.feature_category_for_action("destroy")).to eq(:quux) - end - - it "returns the expected category for categories defined in subclasses" do - expect(subclass.feature_category_for_action("subclass_index")).to eq(:baz) - end - - it "raises an error when defining for the controller and for individual actions" do - expect do - Class.new(base_controller) do - feature_category :hello - feature_category :goodbye, [:world] - end - end.to raise_error(ArgumentError, "hello is defined for all actions, but other categories are set") - end - - it "raises an error when multiple calls define the same action" do - expect do - Class.new(base_controller) do - feature_category :hello, [:world] - feature_category :goodbye, ["world"] - end - end.to raise_error(ArgumentError, "Actions have multiple feature categories: world") - end - - it "does not raise an error when multiple calls define the same action and feature category" do - expect do - Class.new(base_controller) do - feature_category :hello, [:world] - feature_category :hello, ["world"] - end - end.not_to raise_error - end - end -end diff --git a/spec/lib/gitlab/workhorse_spec.rb b/spec/lib/gitlab/workhorse_spec.rb index 09f90a3e5b6..8ba56af561d 100644 --- a/spec/lib/gitlab/workhorse_spec.rb +++ b/spec/lib/gitlab/workhorse_spec.rb @@ -244,13 +244,15 @@ RSpec.describe Gitlab::Workhorse do GitalyServer: { features: { 'gitaly-feature-enforce-requests-limits' => 'true' }, address: Gitlab::GitalyClient.address('default'), - token: Gitlab::GitalyClient.token('default') + token: Gitlab::GitalyClient.token('default'), + sidechannel: false } } end before do allow(Gitlab.config.gitaly).to receive(:enabled).and_return(true) + stub_feature_flags(workhorse_use_sidechannel: false) end it 'includes a Repository param' do @@ -332,6 +334,46 @@ RSpec.describe Gitlab::Workhorse do it { expect { subject }.to raise_exception('Unsupported action: download') } end + + context 'when workhorse_use_sidechannel flag is set' do + context 'when a feature flag is set globally' do + before do + stub_feature_flags(workhorse_use_sidechannel: true) + end + + it 'sets the flag to true' do + response = described_class.git_http_ok(repository, Gitlab::GlRepository::PROJECT, user, action) + + expect(response.dig(:GitalyServer, :sidechannel)).to eq(true) + end + end + + context 'when a feature flag is set for a single project' do + before do + stub_feature_flags(workhorse_use_sidechannel: project) + end + + it 'sets the flag to true for that project' do + response = described_class.git_http_ok(repository, Gitlab::GlRepository::PROJECT, user, action) + + expect(response.dig(:GitalyServer, :sidechannel)).to eq(true) + end + + it 'sets the flag to false for other projects' do + other_project = create(:project, :public, :repository) + response = described_class.git_http_ok(other_project.repository, Gitlab::GlRepository::PROJECT, user, action) + + expect(response.dig(:GitalyServer, :sidechannel)).to eq(false) + end + + it 'sets the flag to false when there is no project' do + snippet = create(:personal_snippet, :repository) + response = described_class.git_http_ok(snippet.repository, Gitlab::GlRepository::SNIPPET, user, action) + + expect(response.dig(:GitalyServer, :sidechannel)).to eq(false) + end + end + end end context 'when receive_max_input_size has been updated' do diff --git a/spec/lib/gitlab/email/smime/certificate_spec.rb b/spec/lib/gitlab/x509/certificate_spec.rb index f7bb933e348..a5b192dd051 100644 --- a/spec/lib/gitlab/email/smime/certificate_spec.rb +++ b/spec/lib/gitlab/x509/certificate_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Email::Smime::Certificate do +RSpec.describe Gitlab::X509::Certificate do include SmimeHelper # cert generation is an expensive operation and they are used read-only, |