diff options
Diffstat (limited to 'spec/lib/gitlab')
193 files changed, 4719 insertions, 3517 deletions
diff --git a/spec/lib/gitlab/analytics/cycle_analytics/aggregated/records_fetcher_spec.rb b/spec/lib/gitlab/analytics/cycle_analytics/aggregated/records_fetcher_spec.rb index 045cdb129cb..55ba6e56237 100644 --- a/spec/lib/gitlab/analytics/cycle_analytics/aggregated/records_fetcher_spec.rb +++ b/spec/lib/gitlab/analytics/cycle_analytics/aggregated/records_fetcher_spec.rb @@ -41,6 +41,19 @@ RSpec.describe Gitlab::Analytics::CycleAnalytics::Aggregated::RecordsFetcher do it_behaves_like 'match returned records' end + context 'when intervalstyle setting is configured to "postgres"' do + it 'avoids nil durations' do + # ActiveRecord cannot parse the 'postgres' intervalstyle, it returns nil + # The setting is rolled back after the test case. + Analytics::CycleAnalytics::IssueStageEvent.connection.execute("SET LOCAL intervalstyle='postgres'") + + records_fetcher.serialized_records do |relation| + durations = relation.map(&:total_time) + expect(durations).to all(be > 0) + end + end + end + context 'when sorting by end event ASC' do let(:expected_issue_ids) { [issue_2.iid, issue_1.iid, issue_3.iid] } diff --git a/spec/lib/gitlab/anonymous_session_spec.rb b/spec/lib/gitlab/anonymous_session_spec.rb index 245ca02e91a..64186e9003a 100644 --- a/spec/lib/gitlab/anonymous_session_spec.rb +++ b/spec/lib/gitlab/anonymous_session_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::AnonymousSession, :clean_gitlab_redis_shared_state do +RSpec.describe Gitlab::AnonymousSession, :clean_gitlab_redis_sessions do let(:default_session_id) { '6919a6f1bb119dd7396fadc38fd18d0d' } let(:additional_session_id) { '7919a6f1bb119dd7396fadc38fd18d0d' } @@ -16,7 +16,7 @@ RSpec.describe Gitlab::AnonymousSession, :clean_gitlab_redis_shared_state do it 'adds session id to proper key' do subject.count_session_ip - Gitlab::Redis::SharedState.with do |redis| + Gitlab::Redis::Sessions.with do |redis| expect(redis.get("session:lookup:ip:gitlab2:127.0.0.1").to_i).to eq 1 end end @@ -25,7 +25,7 @@ RSpec.describe Gitlab::AnonymousSession, :clean_gitlab_redis_shared_state do freeze_time do subject.count_session_ip - Gitlab::Redis::SharedState.with do |redis| + Gitlab::Redis::Sessions.with do |redis| expect(redis.ttl("session:lookup:ip:gitlab2:127.0.0.1")).to eq(24.hours.to_i) end end @@ -36,7 +36,7 @@ RSpec.describe Gitlab::AnonymousSession, :clean_gitlab_redis_shared_state do subject.count_session_ip new_anonymous_session.count_session_ip - Gitlab::Redis::SharedState.with do |redis| + Gitlab::Redis::Sessions.with do |redis| expect(redis.get("session:lookup:ip:gitlab2:127.0.0.1").to_i).to eq(2) end end @@ -45,7 +45,7 @@ RSpec.describe Gitlab::AnonymousSession, :clean_gitlab_redis_shared_state do describe '#stored_sessions' do it 'returns all anonymous sessions per ip' do - Gitlab::Redis::SharedState.with do |redis| + Gitlab::Redis::Sessions.with do |redis| redis.set("session:lookup:ip:gitlab2:127.0.0.1", 2) end @@ -54,13 +54,13 @@ RSpec.describe Gitlab::AnonymousSession, :clean_gitlab_redis_shared_state do end it 'removes obsolete lookup through ip entries' do - Gitlab::Redis::SharedState.with do |redis| + Gitlab::Redis::Sessions.with do |redis| redis.set("session:lookup:ip:gitlab2:127.0.0.1", 2) end subject.cleanup_session_per_ip_count - Gitlab::Redis::SharedState.with do |redis| + Gitlab::Redis::Sessions.with do |redis| expect(redis.exists("session:lookup:ip:gitlab2:127.0.0.1")).to eq(false) end end diff --git a/spec/lib/gitlab/application_context_spec.rb b/spec/lib/gitlab/application_context_spec.rb index ecd68caba79..5ecec978017 100644 --- a/spec/lib/gitlab/application_context_spec.rb +++ b/spec/lib/gitlab/application_context_spec.rb @@ -152,6 +152,38 @@ RSpec.describe Gitlab::ApplicationContext do end end end + + context 'when using a runner project' do + let_it_be_with_reload(:runner) { create(:ci_runner, :project) } + + it 'sets project path from runner project' do + context = described_class.new(runner: runner) + + expect(result(context)).to include(project: runner.runner_projects.first.project.full_path) + end + + context 'when the runner serves multiple projects' do + before do + create(:ci_runner_project, runner: runner, project: create(:project)) + end + + it 'does not set project path' do + context = described_class.new(runner: runner) + + expect(result(context)).to include(project: nil) + end + end + end + + context 'when using an instance runner' do + let_it_be(:runner) { create(:ci_runner, :instance) } + + it 'does not sets project path' do + context = described_class.new(runner: runner) + + expect(result(context)).to include(project: nil) + end + end end describe '#use' do diff --git a/spec/lib/gitlab/application_rate_limiter_spec.rb b/spec/lib/gitlab/application_rate_limiter_spec.rb index c74bcf8d678..20c89eab5f5 100644 --- a/spec/lib/gitlab/application_rate_limiter_spec.rb +++ b/spec/lib/gitlab/application_rate_limiter_spec.rb @@ -2,37 +2,37 @@ require 'spec_helper' -RSpec.describe Gitlab::ApplicationRateLimiter do +RSpec.describe Gitlab::ApplicationRateLimiter, :clean_gitlab_redis_rate_limiting do let_it_be(:user) { create(:user) } let_it_be(:project) { create(:project) } - subject { described_class } - - describe '.throttled?', :clean_gitlab_redis_rate_limiting do - let(:rate_limits) do - { - test_action: { - threshold: 1, - interval: 2.minutes - }, - another_action: { - threshold: 2, - interval: 3.minutes - } + let(:rate_limits) do + { + test_action: { + threshold: 1, + interval: 2.minutes + }, + another_action: { + threshold: 2, + interval: 3.minutes } - end + } + end - before do - allow(described_class).to receive(:rate_limits).and_return(rate_limits) - end + subject { described_class } + + before do + allow(described_class).to receive(:rate_limits).and_return(rate_limits) + end + describe '.throttled?' do context 'when the key is invalid' do context 'is provided as a Symbol' do context 'but is not defined in the rate_limits Hash' do it 'raises an InvalidKeyError exception' do key = :key_not_in_rate_limits_hash - expect { subject.throttled?(key) }.to raise_error(Gitlab::ApplicationRateLimiter::InvalidKeyError) + expect { subject.throttled?(key, scope: [user]) }.to raise_error(Gitlab::ApplicationRateLimiter::InvalidKeyError) end end end @@ -42,7 +42,7 @@ RSpec.describe Gitlab::ApplicationRateLimiter do it 'raises an InvalidKeyError exception' do key = rate_limits.keys[0].to_s - expect { subject.throttled?(key) }.to raise_error(Gitlab::ApplicationRateLimiter::InvalidKeyError) + expect { subject.throttled?(key, scope: [user]) }.to raise_error(Gitlab::ApplicationRateLimiter::InvalidKeyError) end end @@ -50,7 +50,7 @@ RSpec.describe Gitlab::ApplicationRateLimiter do it 'raises an InvalidKeyError exception' do key = 'key_not_in_rate_limits_hash' - expect { subject.throttled?(key) }.to raise_error(Gitlab::ApplicationRateLimiter::InvalidKeyError) + expect { subject.throttled?(key, scope: [user]) }.to raise_error(Gitlab::ApplicationRateLimiter::InvalidKeyError) end end end @@ -89,6 +89,17 @@ RSpec.describe Gitlab::ApplicationRateLimiter do expect(subject.throttled?(:another_action, scope: scope)).to eq(true) end end + + it 'allows peeking at the current state without changing its value' do + travel_to(start_time) do + expect(subject.throttled?(:test_action, scope: scope)).to eq(false) + 2.times do + expect(subject.throttled?(:test_action, scope: scope, peek: true)).to eq(false) + end + expect(subject.throttled?(:test_action, scope: scope)).to eq(true) + expect(subject.throttled?(:test_action, scope: scope, peek: true)).to eq(true) + end + end end context 'when using ActiveRecord models as scope' do @@ -104,6 +115,20 @@ RSpec.describe Gitlab::ApplicationRateLimiter do end end + describe '.peek' do + it 'peeks at the current state without changing its value' do + freeze_time do + expect(subject.peek(:test_action, scope: [user])).to eq(false) + expect(subject.throttled?(:test_action, scope: [user])).to eq(false) + 2.times do + expect(subject.peek(:test_action, scope: [user])).to eq(false) + end + expect(subject.throttled?(:test_action, scope: [user])).to eq(true) + expect(subject.peek(:test_action, scope: [user])).to eq(true) + end + end + end + describe '.log_request' do let(:file_path) { 'master/README.md' } let(:type) { :raw_blob_request_limit } diff --git a/spec/lib/gitlab/asciidoc_spec.rb b/spec/lib/gitlab/asciidoc_spec.rb index ac29bb22865..7200ff3c4db 100644 --- a/spec/lib/gitlab/asciidoc_spec.rb +++ b/spec/lib/gitlab/asciidoc_spec.rb @@ -96,10 +96,10 @@ module Gitlab it "does not convert dangerous fenced code with inline script into HTML" do input = '```mypre"><script>alert(3)</script>' output = - if Feature.enabled?(:use_cmark_renderer) - "<div>\n<div>\n<pre class=\"code highlight js-syntax-highlight language-plaintext\" lang=\"plaintext\" v-pre=\"true\"><code></code></pre>\n</div>\n</div>" + if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) + "<div>\n<div>\n<div class=\"gl-relative markdown-code-block js-markdown-code\">\n<pre class=\"code highlight js-syntax-highlight language-plaintext\" lang=\"plaintext\" v-pre=\"true\"><code></code></pre>\n<copy-code></copy-code>\n</div>\n</div>\n</div>" else - "<div>\n<div>\n<pre class=\"code highlight js-syntax-highlight language-plaintext\" lang=\"plaintext\" v-pre=\"true\"><code><span id=\"LC1\" class=\"line\" lang=\"plaintext\">\"></span></code></pre>\n</div>\n</div>" + "<div>\n<div>\n<div class=\"gl-relative markdown-code-block js-markdown-code\">\n<pre class=\"code highlight js-syntax-highlight language-plaintext\" lang=\"plaintext\" v-pre=\"true\"><code><span id=\"LC1\" class=\"line\" lang=\"plaintext\">\"></span></code></pre>\n<copy-code></copy-code>\n</div>\n</div>\n</div>" end expect(render(input, context)).to include(output) @@ -365,7 +365,10 @@ module Gitlab output = <<~HTML <div> <div> + <div class="gl-relative markdown-code-block js-markdown-code"> <pre class="code highlight js-syntax-highlight language-javascript" lang="javascript" v-pre="true"><code><span id="LC1" class="line" lang="javascript"><span class="nx">console</span><span class="p">.</span><span class="nx">log</span><span class="p">(</span><span class="dl">'</span><span class="s1">hello world</span><span class="dl">'</span><span class="p">)</span></span></code></pre> + <copy-code></copy-code> + </div> </div> </div> HTML @@ -392,11 +395,14 @@ module Gitlab <div> <div>class.cpp</div> <div> + <div class="gl-relative markdown-code-block js-markdown-code"> <pre class="code highlight js-syntax-highlight language-cpp" lang="cpp" v-pre="true"><code><span id="LC1" class="line" lang="cpp"><span class="cp">#include <stdio.h></span></span> <span id="LC2" class="line" lang="cpp"></span> <span id="LC3" class="line" lang="cpp"><span class="k">for</span> <span class="p">(</span><span class="kt">int</span> <span class="n">i</span> <span class="o">=</span> <span class="mi">0</span><span class="p">;</span> <span class="n">i</span> <span class="o"><</span> <span class="mi">5</span><span class="p">;</span> <span class="n">i</span><span class="o">++</span><span class="p">)</span> <span class="p">{</span></span> <span id="LC4" class="line" lang="cpp"> <span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="o"><<</span><span class="s">"*"</span><span class="o"><<</span><span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span></span> <span id="LC5" class="line" lang="cpp"><span class="p">}</span></span></code></pre> + <copy-code></copy-code> + </div> </div> </div> HTML diff --git a/spec/lib/gitlab/auth/user_access_denied_reason_spec.rb b/spec/lib/gitlab/auth/user_access_denied_reason_spec.rb index 102d6fba97f..e5bc51edc2d 100644 --- a/spec/lib/gitlab/auth/user_access_denied_reason_spec.rb +++ b/spec/lib/gitlab/auth/user_access_denied_reason_spec.rb @@ -26,6 +26,7 @@ RSpec.describe Gitlab::Auth::UserAccessDeniedReason do it { is_expected.to match /must accept the Terms of Service/ } it { is_expected.to include(user.username) } + it { is_expected.to include(Gitlab.config.gitlab.url) } end context 'when the user is internal' do diff --git a/spec/lib/gitlab/auth_spec.rb b/spec/lib/gitlab/auth_spec.rb index 5ec6e23774a..32e647688ff 100644 --- a/spec/lib/gitlab/auth_spec.rb +++ b/spec/lib/gitlab/auth_spec.rb @@ -259,30 +259,48 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching do context 'while using OAuth tokens as passwords' do let(:user) { create(:user) } - let(:token_w_api_scope) { Doorkeeper::AccessToken.create!(application_id: application.id, resource_owner_id: user.id, scopes: 'api') } let(:application) { Doorkeeper::Application.create!(name: 'MyApp', redirect_uri: 'https://app.com', owner: user) } shared_examples 'an oauth failure' do it 'fails' do - expect(gl_auth.find_for_git_client("oauth2", token_w_api_scope.token, project: nil, ip: 'ip')) + access_token = Doorkeeper::AccessToken.create!(application_id: application.id, resource_owner_id: user.id, scopes: 'api') + + expect(gl_auth.find_for_git_client("oauth2", access_token.token, project: nil, ip: 'ip')) .to have_attributes(auth_failure) end end - it 'succeeds for OAuth tokens with the `api` scope' do - expect(gl_auth.find_for_git_client("oauth2", token_w_api_scope.token, project: nil, ip: 'ip')).to have_attributes(actor: user, project: nil, type: :oauth, authentication_abilities: described_class.full_authentication_abilities) - end + context 'with specified scopes' do + using RSpec::Parameterized::TableSyntax + + where(:scopes, :abilities) do + 'api' | described_class.full_authentication_abilities + 'read_api' | described_class.read_only_authentication_abilities + 'read_repository' | [:download_code] + 'write_repository' | [:download_code, :push_code] + 'read_user' | [] + 'sudo' | [] + 'openid' | [] + 'profile' | [] + 'email' | [] + end - it 'fails for OAuth tokens with other scopes' do - token = Doorkeeper::AccessToken.create!(application_id: application.id, resource_owner_id: user.id, scopes: 'read_user') + with_them do + it 'authenticates with correct abilities' do + access_token = Doorkeeper::AccessToken.create!(application_id: application.id, resource_owner_id: user.id, scopes: scopes) - expect(gl_auth.find_for_git_client("oauth2", token.token, project: nil, ip: 'ip')).to have_attributes(auth_failure) + expect(gl_auth.find_for_git_client("oauth2", access_token.token, project: nil, ip: 'ip')) + .to have_attributes(actor: user, project: nil, type: :oauth, authentication_abilities: abilities) + end + end end it 'does not try password auth before oauth' do + access_token = Doorkeeper::AccessToken.create!(application_id: application.id, resource_owner_id: user.id, scopes: 'api') + expect(gl_auth).not_to receive(:find_with_user_password) - gl_auth.find_for_git_client("oauth2", token_w_api_scope.token, project: nil, ip: 'ip') + gl_auth.find_for_git_client("oauth2", access_token.token, project: nil, ip: 'ip') end context 'blocked user' do diff --git a/spec/lib/gitlab/background_migration/add_merge_request_diff_commits_count_spec.rb b/spec/lib/gitlab/background_migration/add_merge_request_diff_commits_count_spec.rb deleted file mode 100644 index 1e72b249c19..00000000000 --- a/spec/lib/gitlab/background_migration/add_merge_request_diff_commits_count_spec.rb +++ /dev/null @@ -1,66 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::AddMergeRequestDiffCommitsCount do - let(:namespaces_table) { table(:namespaces) } - let(:projects_table) { table(:projects) } - let(:merge_requests_table) { table(:merge_requests) } - let(:merge_request_diffs_table) { table(:merge_request_diffs) } - let(:merge_request_diff_commits_table) { table(:merge_request_diff_commits) } - - let(:namespace) { namespaces_table.create!(name: 'gitlab-org', path: 'gitlab-org') } - let(:project) { projects_table.create!(name: 'gitlab', path: 'gitlab-org/gitlab-ce', namespace_id: namespace.id) } - let(:merge_request) do - merge_requests_table.create!(target_project_id: project.id, - target_branch: 'master', - source_project_id: project.id, - source_branch: 'mr name', - title: 'mr name') - end - - def create_diff!(name, commits: 0) - mr_diff = merge_request_diffs_table.create!( - merge_request_id: merge_request.id) - - commits.times do |i| - merge_request_diff_commits_table.create!( - merge_request_diff_id: mr_diff.id, - relative_order: i, sha: i) - end - - mr_diff - end - - describe '#perform' do - it 'migrates diffs that have no commits' do - diff = create_diff!('with_multiple_commits', commits: 0) - - subject.perform(diff.id, diff.id) - - expect(diff.reload.commits_count).to eq(0) - end - - it 'skips diffs that have commits_count already set' do - timestamp = 2.days.ago - diff = merge_request_diffs_table.create!( - merge_request_id: merge_request.id, - commits_count: 0, - updated_at: timestamp) - - subject.perform(diff.id, diff.id) - - expect(diff.reload.updated_at).to be_within(1.second).of(timestamp) - end - - it 'migrates multiple diffs to the correct values' do - diffs = Array.new(3).map.with_index { |_, i| create_diff!(i, commits: 3) } - - subject.perform(diffs.first.id, diffs.last.id) - - diffs.each do |diff| - expect(diff.reload.commits_count).to eq(3) - end - end - end -end diff --git a/spec/lib/gitlab/background_migration/add_modified_to_approval_merge_request_rule_spec.rb b/spec/lib/gitlab/background_migration/add_modified_to_approval_merge_request_rule_spec.rb deleted file mode 100644 index 0b29163671c..00000000000 --- a/spec/lib/gitlab/background_migration/add_modified_to_approval_merge_request_rule_spec.rb +++ /dev/null @@ -1,61 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::AddModifiedToApprovalMergeRequestRule, schema: 20181228175414 do - let(:determine_if_rules_are_modified) { described_class.new } - - let(:namespace) { table(:namespaces).create!(name: 'gitlab', path: 'gitlab') } - let(:projects) { table(:projects) } - let(:normal_project) { projects.create!(namespace_id: namespace.id) } - let(:overridden_project) { projects.create!(namespace_id: namespace.id) } - let(:rules) { table(:approval_merge_request_rules) } - let(:project_rules) { table(:approval_project_rules) } - let(:sources) { table(:approval_merge_request_rule_sources) } - let(:merge_requests) { table(:merge_requests) } - let(:groups) { table(:namespaces) } - let(:mr_groups) { table(:approval_merge_request_rules_groups) } - let(:project_groups) { table(:approval_project_rules_groups) } - - before do - project_rule = project_rules.create!(project_id: normal_project.id, approvals_required: 3, name: 'test rule') - overridden_project_rule = project_rules.create!(project_id: overridden_project.id, approvals_required: 5, name: 'other test rule') - overridden_project_rule_two = project_rules.create!(project_id: overridden_project.id, approvals_required: 7, name: 'super cool rule') - - merge_request = merge_requests.create!(target_branch: 'feature', source_branch: 'default', source_project_id: normal_project.id, target_project_id: normal_project.id) - overridden_merge_request = merge_requests.create!(target_branch: 'feature-2', source_branch: 'default', source_project_id: overridden_project.id, target_project_id: overridden_project.id) - - merge_rule = rules.create!(merge_request_id: merge_request.id, approvals_required: 3, name: 'test rule') - overridden_merge_rule = rules.create!(merge_request_id: overridden_merge_request.id, approvals_required: 6, name: 'other test rule') - overridden_merge_rule_two = rules.create!(merge_request_id: overridden_merge_request.id, approvals_required: 7, name: 'super cool rule') - - sources.create!(approval_project_rule_id: project_rule.id, approval_merge_request_rule_id: merge_rule.id) - sources.create!(approval_project_rule_id: overridden_project_rule.id, approval_merge_request_rule_id: overridden_merge_rule.id) - sources.create!(approval_project_rule_id: overridden_project_rule_two.id, approval_merge_request_rule_id: overridden_merge_rule_two.id) - - group1 = groups.create!(name: "group1", path: "test_group1", type: 'Group') - group2 = groups.create!(name: "group2", path: "test_group2", type: 'Group') - group3 = groups.create!(name: "group3", path: "test_group3", type: 'Group') - - project_groups.create!(approval_project_rule_id: overridden_project_rule_two.id, group_id: group1.id) - project_groups.create!(approval_project_rule_id: overridden_project_rule_two.id, group_id: group2.id) - project_groups.create!(approval_project_rule_id: overridden_project_rule_two.id, group_id: group3.id) - - mr_groups.create!(approval_merge_request_rule_id: overridden_merge_rule.id, group_id: group1.id) - mr_groups.create!(approval_merge_request_rule_id: overridden_merge_rule_two.id, group_id: group2.id) - end - - describe '#perform' do - it 'changes the correct rules' do - original_count = rules.all.count - - determine_if_rules_are_modified.perform(rules.minimum(:id), rules.maximum(:id)) - - results = rules.where(modified_from_project_rule: true) - - expect(results.count).to eq 2 - expect(results.collect(&:name)).to eq(['other test rule', 'super cool rule']) - expect(rules.count).to eq original_count - end - end -end diff --git a/spec/lib/gitlab/background_migration/backfill_deployment_clusters_from_deployments_spec.rb b/spec/lib/gitlab/background_migration/backfill_deployment_clusters_from_deployments_spec.rb deleted file mode 100644 index 1404ada3647..00000000000 --- a/spec/lib/gitlab/background_migration/backfill_deployment_clusters_from_deployments_spec.rb +++ /dev/null @@ -1,44 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::BackfillDeploymentClustersFromDeployments, :migration, schema: 20181228175414 do - subject { described_class.new } - - describe '#perform' do - it 'backfills deployment_cluster for all deployments in the given range with a non-null cluster_id' do - deployment_clusters = table(:deployment_clusters) - - namespace = table(:namespaces).create!(name: 'the-namespace', path: 'the-path') - project = table(:projects).create!(name: 'the-project', namespace_id: namespace.id) - environment = table(:environments).create!(name: 'the-environment', project_id: project.id, slug: 'slug') - cluster = table(:clusters).create!(name: 'the-cluster') - - deployment_data = { cluster_id: cluster.id, project_id: project.id, environment_id: environment.id, ref: 'abc', tag: false, sha: 'sha', status: 1 } - expected_deployment_1 = create_deployment(**deployment_data) - create_deployment(**deployment_data, cluster_id: nil) # no cluster_id - expected_deployment_2 = create_deployment(**deployment_data) - out_of_range_deployment = create_deployment(**deployment_data, cluster_id: cluster.id) # expected to be out of range - - # to test "ON CONFLICT DO NOTHING" - existing_record_for_deployment_2 = deployment_clusters.create!( - deployment_id: expected_deployment_2.id, - cluster_id: expected_deployment_2.cluster_id, - kubernetes_namespace: 'production' - ) - - subject.perform(expected_deployment_1.id, out_of_range_deployment.id - 1) - - expect(deployment_clusters.all.pluck(:deployment_id, :cluster_id, :kubernetes_namespace)).to contain_exactly( - [expected_deployment_1.id, cluster.id, nil], - [expected_deployment_2.id, cluster.id, existing_record_for_deployment_2.kubernetes_namespace] - ) - end - - def create_deployment(**data) - @iid ||= 0 - @iid += 1 - table(:deployments).create!(iid: @iid, **data) - end - end -end diff --git a/spec/lib/gitlab/background_migration/backfill_environment_id_deployment_merge_requests_spec.rb b/spec/lib/gitlab/background_migration/backfill_environment_id_deployment_merge_requests_spec.rb deleted file mode 100644 index 9194525e713..00000000000 --- a/spec/lib/gitlab/background_migration/backfill_environment_id_deployment_merge_requests_spec.rb +++ /dev/null @@ -1,46 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::BackfillEnvironmentIdDeploymentMergeRequests, schema: 20181228175414 do - let(:environments) { table(:environments) } - let(:merge_requests) { table(:merge_requests) } - let(:deployments) { table(:deployments) } - let(:deployment_merge_requests) { table(:deployment_merge_requests) } - let(:namespaces) { table(:namespaces) } - let(:projects) { table(:projects) } - - subject(:migration) { described_class.new } - - it 'correctly backfills environment_id column' do - namespace = namespaces.create!(name: 'foo', path: 'foo') - project = projects.create!(namespace_id: namespace.id) - - production = environments.create!(project_id: project.id, name: 'production', slug: 'production') - staging = environments.create!(project_id: project.id, name: 'staging', slug: 'staging') - - mr = merge_requests.create!(source_branch: 'x', target_branch: 'master', target_project_id: project.id) - - deployment1 = deployments.create!(environment_id: staging.id, iid: 1, project_id: project.id, ref: 'master', tag: false, sha: '123abcdef', status: 1) - deployment2 = deployments.create!(environment_id: production.id, iid: 2, project_id: project.id, ref: 'master', tag: false, sha: '123abcdef', status: 1) - deployment3 = deployments.create!(environment_id: production.id, iid: 3, project_id: project.id, ref: 'master', tag: false, sha: '123abcdef', status: 1) - - # mr is tracked twice in production through deployment2 and deployment3 - deployment_merge_requests.create!(deployment_id: deployment1.id, merge_request_id: mr.id) - deployment_merge_requests.create!(deployment_id: deployment2.id, merge_request_id: mr.id) - deployment_merge_requests.create!(deployment_id: deployment3.id, merge_request_id: mr.id) - - expect(deployment_merge_requests.where(environment_id: nil).count).to eq(3) - - migration.backfill_range(1, mr.id) - - expect(deployment_merge_requests.where(environment_id: nil).count).to be_zero - expect(deployment_merge_requests.count).to eq(2) - - production_deployments = deployment_merge_requests.where(environment_id: production.id) - expect(production_deployments.count).to eq(1) - expect(production_deployments.first.deployment_id).to eq(deployment2.id) - - expect(deployment_merge_requests.where(environment_id: staging.id).count).to eq(1) - end -end diff --git a/spec/lib/gitlab/background_migration/backfill_merge_request_cleanup_schedules_spec.rb b/spec/lib/gitlab/background_migration/backfill_merge_request_cleanup_schedules_spec.rb deleted file mode 100644 index d33f52514da..00000000000 --- a/spec/lib/gitlab/background_migration/backfill_merge_request_cleanup_schedules_spec.rb +++ /dev/null @@ -1,53 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::BackfillMergeRequestCleanupSchedules, schema: 20181228175414 do - let(:merge_requests) { table(:merge_requests) } - let(:cleanup_schedules) { table(:merge_request_cleanup_schedules) } - let(:metrics) { table(:merge_request_metrics) } - - let(:namespace) { table(:namespaces).create!(name: 'name', path: 'path') } - let(:project) { table(:projects).create!(namespace_id: namespace.id) } - - subject { described_class.new } - - describe '#perform' do - let!(:open_mr) { merge_requests.create!(target_project_id: project.id, source_branch: 'master', target_branch: 'master') } - - let!(:closed_mr_1) { merge_requests.create!(target_project_id: project.id, source_branch: 'master', target_branch: 'master', state_id: 2) } - let!(:closed_mr_2) { merge_requests.create!(target_project_id: project.id, source_branch: 'master', target_branch: 'master', state_id: 2) } - let!(:closed_mr_1_metrics) { metrics.create!(merge_request_id: closed_mr_1.id, target_project_id: project.id, latest_closed_at: Time.current, created_at: Time.current, updated_at: Time.current) } - let!(:closed_mr_2_metrics) { metrics.create!(merge_request_id: closed_mr_2.id, target_project_id: project.id, latest_closed_at: Time.current, created_at: Time.current, updated_at: Time.current) } - let!(:closed_mr_2_cleanup_schedule) { cleanup_schedules.create!(merge_request_id: closed_mr_2.id, scheduled_at: Time.current) } - - let!(:merged_mr_1) { merge_requests.create!(target_project_id: project.id, source_branch: 'master', target_branch: 'master', state_id: 3) } - let!(:merged_mr_2) { merge_requests.create!(target_project_id: project.id, source_branch: 'master', target_branch: 'master', state_id: 3, updated_at: Time.current) } - let!(:merged_mr_1_metrics) { metrics.create!(merge_request_id: merged_mr_1.id, target_project_id: project.id, merged_at: Time.current, created_at: Time.current, updated_at: Time.current) } - - let!(:closed_mr_3) { merge_requests.create!(target_project_id: project.id, source_branch: 'master', target_branch: 'master', state_id: 2) } - let!(:closed_mr_3_metrics) { metrics.create!(merge_request_id: closed_mr_3.id, target_project_id: project.id, latest_closed_at: Time.current, created_at: Time.current, updated_at: Time.current) } - - it 'creates records for all closed and merged merge requests in range' do - expect(Gitlab::BackgroundMigration::Logger).to receive(:info).with( - message: 'Backfilled merge_request_cleanup_schedules records', - count: 3 - ) - - subject.perform(open_mr.id, merged_mr_2.id) - - aggregate_failures do - expect(cleanup_schedules.all.pluck(:merge_request_id)) - .to contain_exactly(closed_mr_1.id, closed_mr_2.id, merged_mr_1.id, merged_mr_2.id) - expect(cleanup_schedules.find_by(merge_request_id: closed_mr_1.id).scheduled_at.to_s) - .to eq((closed_mr_1_metrics.latest_closed_at + 14.days).to_s) - expect(cleanup_schedules.find_by(merge_request_id: closed_mr_2.id).scheduled_at.to_s) - .to eq(closed_mr_2_cleanup_schedule.scheduled_at.to_s) - expect(cleanup_schedules.find_by(merge_request_id: merged_mr_1.id).scheduled_at.to_s) - .to eq((merged_mr_1_metrics.merged_at + 14.days).to_s) - expect(cleanup_schedules.find_by(merge_request_id: merged_mr_2.id).scheduled_at.to_s) - .to eq((merged_mr_2.updated_at + 14.days).to_s) - end - end - end -end diff --git a/spec/lib/gitlab/background_migration/backfill_namespace_settings_spec.rb b/spec/lib/gitlab/background_migration/backfill_namespace_settings_spec.rb deleted file mode 100644 index 0f8adca2ca4..00000000000 --- a/spec/lib/gitlab/background_migration/backfill_namespace_settings_spec.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::BackfillNamespaceSettings, schema: 20181228175414 do - let(:namespaces) { table(:namespaces) } - let(:namespace_settings) { table(:namespace_settings) } - let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') } - - subject { described_class.new } - - describe '#perform' do - it 'creates settings for all projects in range' do - namespaces.create!(id: 5, name: 'test1', path: 'test1') - namespaces.create!(id: 7, name: 'test2', path: 'test2') - namespaces.create!(id: 8, name: 'test3', path: 'test3') - - subject.perform(5, 7) - - expect(namespace_settings.all.pluck(:namespace_id)).to contain_exactly(5, 7) - end - end -end diff --git a/spec/lib/gitlab/background_migration/backfill_project_settings_spec.rb b/spec/lib/gitlab/background_migration/backfill_project_settings_spec.rb deleted file mode 100644 index e6b0db2ab73..00000000000 --- a/spec/lib/gitlab/background_migration/backfill_project_settings_spec.rb +++ /dev/null @@ -1,24 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::BackfillProjectSettings, schema: 20181228175414 do - let(:projects) { table(:projects) } - let(:project_settings) { table(:project_settings) } - let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') } - let(:project) { projects.create!(namespace_id: namespace.id) } - - subject { described_class.new } - - describe '#perform' do - it 'creates settings for all projects in range' do - projects.create!(id: 5, namespace_id: namespace.id) - projects.create!(id: 7, namespace_id: namespace.id) - projects.create!(id: 8, namespace_id: namespace.id) - - subject.perform(5, 7) - - expect(project_settings.all.pluck(:project_id)).to contain_exactly(5, 7) - end - end -end diff --git a/spec/lib/gitlab/background_migration/backfill_push_rules_id_in_projects_spec.rb b/spec/lib/gitlab/background_migration/backfill_push_rules_id_in_projects_spec.rb deleted file mode 100644 index 3468df3dccd..00000000000 --- a/spec/lib/gitlab/background_migration/backfill_push_rules_id_in_projects_spec.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::BackfillPushRulesIdInProjects, :migration, schema: 20181228175414 do - let(:push_rules) { table(:push_rules) } - let(:projects) { table(:projects) } - let(:project_settings) { table(:project_settings) } - let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') } - - subject { described_class.new } - - describe '#perform' do - it 'creates new project push_rules for all push rules in the range' do - project_1 = projects.create!(id: 1, namespace_id: namespace.id) - project_2 = projects.create!(id: 2, namespace_id: namespace.id) - project_3 = projects.create!(id: 3, namespace_id: namespace.id) - project_settings_1 = project_settings.create!(project_id: project_1.id) - project_settings_2 = project_settings.create!(project_id: project_2.id) - project_settings_3 = project_settings.create!(project_id: project_3.id) - push_rule_1 = push_rules.create!(id: 5, is_sample: false, project_id: project_1.id) - push_rule_2 = push_rules.create!(id: 6, is_sample: false, project_id: project_2.id) - push_rules.create!(id: 8, is_sample: false, project_id: 3) - - subject.perform(5, 7) - - expect(project_settings_1.reload.push_rule_id).to eq(push_rule_1.id) - expect(project_settings_2.reload.push_rule_id).to eq(push_rule_2.id) - expect(project_settings_3.reload.push_rule_id).to be_nil - end - end -end diff --git a/spec/lib/gitlab/background_migration/drop_invalid_security_findings_spec.rb b/spec/lib/gitlab/background_migration/drop_invalid_security_findings_spec.rb new file mode 100644 index 00000000000..7cc64889fc8 --- /dev/null +++ b/spec/lib/gitlab/background_migration/drop_invalid_security_findings_spec.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::DropInvalidSecurityFindings, schema: 20211108211434 do + let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user', type: Namespaces::UserNamespace.sti_name) } + let(:project) { table(:projects).create!(namespace_id: namespace.id) } + + let(:pipelines) { table(:ci_pipelines) } + let!(:pipeline) { pipelines.create!(project_id: project.id) } + + let(:ci_builds) { table(:ci_builds) } + let!(:ci_build) { ci_builds.create! } + + let(:security_scans) { table(:security_scans) } + let!(:security_scan) do + security_scans.create!( + scan_type: 1, + status: 1, + build_id: ci_build.id, + project_id: project.id, + pipeline_id: pipeline.id + ) + end + + let(:vulnerability_scanners) { table(:vulnerability_scanners) } + let!(:vulnerability_scanner) { vulnerability_scanners.create!(project_id: project.id, external_id: 'test 1', name: 'test scanner 1') } + + let(:security_findings) { table(:security_findings) } + let!(:security_finding_without_uuid) do + security_findings.create!( + severity: 1, + confidence: 1, + scan_id: security_scan.id, + scanner_id: vulnerability_scanner.id, + uuid: nil + ) + end + + let!(:security_finding_with_uuid) do + security_findings.create!( + severity: 1, + confidence: 1, + scan_id: security_scan.id, + scanner_id: vulnerability_scanner.id, + uuid: 'bd95c085-71aa-51d7-9bb6-08ae669c262e' + ) + end + + let(:sub_batch_size) { 10_000 } + + subject { described_class.new.perform(security_finding_without_uuid.id, security_finding_with_uuid.id, sub_batch_size) } + + it 'drops Security::Finding objects with no UUID' do + expect { subject }.to change(security_findings, :count).from(2).to(1) + end +end diff --git a/spec/lib/gitlab/background_migration/fix_promoted_epics_discussion_ids_spec.rb b/spec/lib/gitlab/background_migration/fix_promoted_epics_discussion_ids_spec.rb deleted file mode 100644 index 35ec8be691a..00000000000 --- a/spec/lib/gitlab/background_migration/fix_promoted_epics_discussion_ids_spec.rb +++ /dev/null @@ -1,49 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::FixPromotedEpicsDiscussionIds, schema: 20181228175414 do - let(:namespaces) { table(:namespaces) } - let(:users) { table(:users) } - let(:epics) { table(:epics) } - let(:notes) { table(:notes) } - - let(:user) { users.create!(email: 'test@example.com', projects_limit: 100, username: 'test') } - let(:namespace) { namespaces.create!(name: 'gitlab', path: 'gitlab-org') } - let(:epic1) { epics.create!(id: 1, author_id: user.id, iid: 1, group_id: namespace.id, title: 'Epic with discussion', title_html: 'Epic with discussion') } - - def create_note(discussion_id) - notes.create!(note: 'note comment', - noteable_id: epic1.id, - noteable_type: 'Epic', - discussion_id: discussion_id) - end - - def expect_valid_discussion_id(id) - expect(id).to match(/\A\h{40}\z/) - end - - describe '#perform with batch of discussion ids' do - it 'updates discussion ids' do - note1 = create_note('00000000') - note2 = create_note('00000000') - note3 = create_note('10000000') - - subject.perform(%w(00000000 10000000)) - - expect_valid_discussion_id(note1.reload.discussion_id) - expect_valid_discussion_id(note2.reload.discussion_id) - expect_valid_discussion_id(note3.reload.discussion_id) - expect(note1.discussion_id).to eq(note2.discussion_id) - expect(note1.discussion_id).not_to eq(note3.discussion_id) - end - - it 'skips notes with discussion id not in range' do - note4 = create_note('20000000') - - subject.perform(%w(00000000 10000000)) - - expect(note4.reload.discussion_id).to eq('20000000') - end - end -end diff --git a/spec/lib/gitlab/background_migration/fix_user_namespace_names_spec.rb b/spec/lib/gitlab/background_migration/fix_user_namespace_names_spec.rb deleted file mode 100644 index 95509f9b897..00000000000 --- a/spec/lib/gitlab/background_migration/fix_user_namespace_names_spec.rb +++ /dev/null @@ -1,104 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::FixUserNamespaceNames, schema: 20181228175414 do - let(:namespaces) { table(:namespaces) } - let(:users) { table(:users) } - let(:user) { users.create!(name: "The user's full name", projects_limit: 10, username: 'not-null', email: '1') } - - context 'updating the namespace names' do - it 'updates a user namespace within range' do - user2 = users.create!(name: "Other user's full name", projects_limit: 10, username: 'also-not-null', email: '2') - user_namespace1 = namespaces.create!( - id: 2, - owner_id: user.id, - name: "Should be the user's name", - path: user.username - ) - user_namespace2 = namespaces.create!( - id: 3, - owner_id: user2.id, - name: "Should also be the user's name", - path: user.username - ) - - described_class.new.perform(1, 5) - - expect(user_namespace1.reload.name).to eq("The user's full name") - expect(user_namespace2.reload.name).to eq("Other user's full name") - end - - it 'does not update namespaces out of range' do - user_namespace = namespaces.create!( - id: 6, - owner_id: user.id, - name: "Should be the user's name", - path: user.username - ) - - expect { described_class.new.perform(1, 5) } - .not_to change { user_namespace.reload.name } - end - - it 'does not update groups owned by the users' do - user_group = namespaces.create!( - id: 2, - owner_id: user.id, - name: 'A group name', - path: 'the-path', - type: 'Group' - ) - - expect { described_class.new.perform(1, 5) } - .not_to change { user_group.reload.name } - end - end - - context 'namespace route names' do - let(:routes) { table(:routes) } - let(:namespace) do - namespaces.create!( - id: 2, - owner_id: user.id, - name: "Will be updated to the user's name", - path: user.username - ) - end - - it "updates the route name if it didn't match the namespace" do - route = routes.create!(path: namespace.path, name: 'Incorrect name', source_type: 'Namespace', source_id: namespace.id) - - described_class.new.perform(1, 5) - - expect(route.reload.name).to eq("The user's full name") - end - - it 'updates the route name if it was nil match the namespace' do - route = routes.create!(path: namespace.path, name: nil, source_type: 'Namespace', source_id: namespace.id) - - described_class.new.perform(1, 5) - - expect(route.reload.name).to eq("The user's full name") - end - - it "doesn't update group routes" do - route = routes.create!(path: 'group-path', name: 'Group name', source_type: 'Group', source_id: namespace.id) - - expect { described_class.new.perform(1, 5) } - .not_to change { route.reload.name } - end - - it "doesn't touch routes for namespaces out of range" do - user_namespace = namespaces.create!( - id: 6, - owner_id: user.id, - name: "Should be the user's name", - path: user.username - ) - - expect { described_class.new.perform(1, 5) } - .not_to change { user_namespace.reload.name } - end - end -end diff --git a/spec/lib/gitlab/background_migration/fix_user_project_route_names_spec.rb b/spec/lib/gitlab/background_migration/fix_user_project_route_names_spec.rb deleted file mode 100644 index b4444df674e..00000000000 --- a/spec/lib/gitlab/background_migration/fix_user_project_route_names_spec.rb +++ /dev/null @@ -1,98 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::FixUserProjectRouteNames, schema: 20181228175414 do - let(:namespaces) { table(:namespaces) } - let(:users) { table(:users) } - let(:routes) { table(:routes) } - let(:projects) { table(:projects) } - - let(:user) { users.create!(name: "The user's full name", projects_limit: 10, username: 'not-null', email: '1') } - - let(:namespace) do - namespaces.create!( - owner_id: user.id, - name: "Should eventually be the user's name", - path: user.username - ) - end - - let(:project) do - projects.create!(namespace_id: namespace.id, name: 'Project Name') - end - - it "updates the route for a project if it did not match the user's name" do - route = routes.create!( - id: 1, - path: "#{user.username}/#{project.path}", - source_id: project.id, - source_type: 'Project', - name: 'Completely wrong' - ) - - described_class.new.perform(1, 5) - - expect(route.reload.name).to eq("The user's full name / Project Name") - end - - it 'updates the route for a project if the name was nil' do - route = routes.create!( - id: 1, - path: "#{user.username}/#{project.path}", - source_id: project.id, - source_type: 'Project', - name: nil - ) - - described_class.new.perform(1, 5) - - expect(route.reload.name).to eq("The user's full name / Project Name") - end - - it 'does not update routes that were are out of the range' do - route = routes.create!( - id: 6, - path: "#{user.username}/#{project.path}", - source_id: project.id, - source_type: 'Project', - name: 'Completely wrong' - ) - - expect { described_class.new.perform(1, 5) } - .not_to change { route.reload.name } - end - - it 'does not update routes for projects in groups owned by the user' do - group = namespaces.create!( - owner_id: user.id, - name: 'A group', - path: 'a-path', - type: '' - ) - project = projects.create!(namespace_id: group.id, name: 'Project Name') - route = routes.create!( - id: 1, - path: "#{group.path}/#{project.path}", - source_id: project.id, - source_type: 'Project', - name: 'Completely wrong' - ) - - expect { described_class.new.perform(1, 5) } - .not_to change { route.reload.name } - end - - it 'does not update routes for namespaces' do - route = routes.create!( - id: 1, - path: namespace.path, - source_id: namespace.id, - source_type: 'Namespace', - name: 'Completely wrong' - ) - - expect { described_class.new.perform(1, 5) } - .not_to change { route.reload.name } - end -end diff --git a/spec/lib/gitlab/background_migration/job_coordinator_spec.rb b/spec/lib/gitlab/background_migration/job_coordinator_spec.rb index a0543ca9958..7a524d1489a 100644 --- a/spec/lib/gitlab/background_migration/job_coordinator_spec.rb +++ b/spec/lib/gitlab/background_migration/job_coordinator_spec.rb @@ -3,32 +3,22 @@ require 'spec_helper' RSpec.describe Gitlab::BackgroundMigration::JobCoordinator do - let(:database) { :main } let(:worker_class) { BackgroundMigrationWorker } - let(:coordinator) { described_class.new(database, worker_class) } + let(:tracking_database) { worker_class.tracking_database } + let(:coordinator) { described_class.new(worker_class) } - describe '.for_database' do + describe '.for_tracking_database' do it 'returns an executor with the correct worker class and database' do - coordinator = described_class.for_database(database) + coordinator = described_class.for_tracking_database(tracking_database) - expect(coordinator.database).to eq(database) expect(coordinator.worker_class).to eq(worker_class) end - context 'when passed in as a string' do - it 'retruns an executor with the correct worker class and database' do - coordinator = described_class.for_database(database.to_s) - - expect(coordinator.database).to eq(database) - expect(coordinator.worker_class).to eq(worker_class) - end - end - context 'when an invalid value is given' do it 'raises an error' do expect do - described_class.for_database('notvalid') - end.to raise_error(ArgumentError, "database must be one of [main], got 'notvalid'") + described_class.for_tracking_database('notvalid') + end.to raise_error(ArgumentError, /tracking_database must be one of/) end end end diff --git a/spec/lib/gitlab/background_migration/link_lfs_objects_projects_spec.rb b/spec/lib/gitlab/background_migration/link_lfs_objects_projects_spec.rb deleted file mode 100644 index 64e8afedf52..00000000000 --- a/spec/lib/gitlab/background_migration/link_lfs_objects_projects_spec.rb +++ /dev/null @@ -1,113 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::LinkLfsObjectsProjects, :migration, schema: 20181228175414 do - let(:namespaces) { table(:namespaces) } - let(:projects) { table(:projects) } - let(:fork_networks) { table(:fork_networks) } - let(:fork_network_members) { table(:fork_network_members) } - let(:lfs_objects) { table(:lfs_objects) } - let(:lfs_objects_projects) { table(:lfs_objects_projects) } - - let(:namespace) { namespaces.create!(name: 'GitLab', path: 'gitlab') } - - let(:fork_network) { fork_networks.create!(root_project_id: source_project.id) } - let(:another_fork_network) { fork_networks.create!(root_project_id: another_source_project.id) } - - let(:source_project) { projects.create!(namespace_id: namespace.id) } - let(:another_source_project) { projects.create!(namespace_id: namespace.id) } - let(:project) { projects.create!(namespace_id: namespace.id) } - let(:another_project) { projects.create!(namespace_id: namespace.id) } - let(:partially_linked_project) { projects.create!(namespace_id: namespace.id) } - let(:fully_linked_project) { projects.create!(namespace_id: namespace.id) } - - let(:lfs_object) { lfs_objects.create!(oid: 'abc123', size: 100) } - let(:another_lfs_object) { lfs_objects.create!(oid: 'def456', size: 200) } - - let!(:source_project_lop_1) do - lfs_objects_projects.create!( - lfs_object_id: lfs_object.id, - project_id: source_project.id - ) - end - - let!(:source_project_lop_2) do - lfs_objects_projects.create!( - lfs_object_id: another_lfs_object.id, - project_id: source_project.id - ) - end - - let!(:another_source_project_lop_1) do - lfs_objects_projects.create!( - lfs_object_id: lfs_object.id, - project_id: another_source_project.id - ) - end - - let!(:another_source_project_lop_2) do - lfs_objects_projects.create!( - lfs_object_id: another_lfs_object.id, - project_id: another_source_project.id - ) - end - - before do - stub_const("#{described_class}::BATCH_SIZE", 2) - - # Create links between projects - fork_network_members.create!(fork_network_id: fork_network.id, project_id: source_project.id, forked_from_project_id: nil) - - [project, partially_linked_project, fully_linked_project].each do |p| - fork_network_members.create!( - fork_network_id: fork_network.id, - project_id: p.id, - forked_from_project_id: fork_network.root_project_id - ) - end - - fork_network_members.create!(fork_network_id: another_fork_network.id, project_id: another_source_project.id, forked_from_project_id: nil) - fork_network_members.create!(fork_network_id: another_fork_network.id, project_id: another_project.id, forked_from_project_id: another_fork_network.root_project_id) - - # Links LFS objects to some projects - lfs_objects_projects.create!(lfs_object_id: lfs_object.id, project_id: fully_linked_project.id) - lfs_objects_projects.create!(lfs_object_id: another_lfs_object.id, project_id: fully_linked_project.id) - lfs_objects_projects.create!(lfs_object_id: lfs_object.id, project_id: partially_linked_project.id) - end - - context 'when there are LFS objects to be linked' do - it 'creates LfsObjectsProject records for forks based on the specified range of LfsObjectProject id' do - expect_next_instance_of(Gitlab::BackgroundMigration::Logger) do |logger| - expect(logger).to receive(:info).exactly(4).times - end - - expect { subject.perform(source_project_lop_1.id, another_source_project_lop_2.id) }.to change { lfs_objects_projects.count }.by(5) - - expect(lfs_object_ids_for(project)).to match_array(lfs_object_ids_for(source_project)) - expect(lfs_object_ids_for(another_project)).to match_array(lfs_object_ids_for(another_source_project)) - expect(lfs_object_ids_for(partially_linked_project)).to match_array(lfs_object_ids_for(source_project)) - - expect { subject.perform(source_project_lop_1.id, another_source_project_lop_2.id) }.not_to change { lfs_objects_projects.count } - end - end - - context 'when there are no LFS objects to be linked' do - before do - # Links LFS objects to all projects - projects.all.each do |p| - lfs_objects_projects.create!(lfs_object_id: lfs_object.id, project_id: p.id) - lfs_objects_projects.create!(lfs_object_id: another_lfs_object.id, project_id: p.id) - end - end - - it 'does not create LfsObjectProject records' do - expect { subject.perform(source_project_lop_1.id, another_source_project_lop_2.id) } - .not_to change { lfs_objects_projects.count } - end - end - - def lfs_object_ids_for(project) - lfs_objects_projects.where(project_id: project.id).pluck(:lfs_object_id) - end -end diff --git a/spec/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys_spec.rb b/spec/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys_spec.rb deleted file mode 100644 index 4287d6723cf..00000000000 --- a/spec/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys_spec.rb +++ /dev/null @@ -1,93 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::MigrateFingerprintSha256WithinKeys, schema: 20181228175414 do - subject(:fingerprint_migrator) { described_class.new } - - let(:key_table) { table(:keys) } - - before do - generate_fingerprints! - end - - it 'correctly creates a sha256 fingerprint for a key' do - key_1 = Key.find(1017) - key_2 = Key.find(1027) - - expect(key_1.fingerprint_md5).to eq('ba:81:59:68:d7:6c:cd:02:02:bf:6a:9b:55:4e:af:d1') - expect(key_1.fingerprint_sha256).to eq(nil) - - expect(key_2.fingerprint_md5).to eq('39:e3:64:a6:24:ea:45:a2:8c:55:2a:e9:4d:4f:1f:b4') - expect(key_2.fingerprint_sha256).to eq(nil) - - query_count = ActiveRecord::QueryRecorder.new do - fingerprint_migrator.perform(1, 10000) - end.count - - expect(query_count).to eq(8) - - key_1.reload - key_2.reload - - expect(key_1.fingerprint_md5).to eq('ba:81:59:68:d7:6c:cd:02:02:bf:6a:9b:55:4e:af:d1') - expect(key_1.fingerprint_sha256).to eq('nUhzNyftwADy8AH3wFY31tAKs7HufskYTte2aXo/lCg') - - expect(key_2.fingerprint_md5).to eq('39:e3:64:a6:24:ea:45:a2:8c:55:2a:e9:4d:4f:1f:b4') - expect(key_2.fingerprint_sha256).to eq('zMNbLekgdjtcgDv8VSC0z5lpdACMG3Q4PUoIz5+H2jM') - end - - context 'with invalid keys' do - before do - key = Key.find(1017) - # double space after "ssh-rsa" leads to a - # OpenSSL::PKey::PKeyError in Net::SSH::KeyFactory.load_data_public_key - key.update_column(:key, key.key.gsub('ssh-rsa ', 'ssh-rsa ')) - end - - it 'ignores errors and does not set the fingerprint' do - fingerprint_migrator.perform(1, 10000) - - key_1 = Key.find(1017) - key_2 = Key.find(1027) - - expect(key_1.fingerprint_sha256).to be_nil - expect(key_2.fingerprint_sha256).not_to be_nil - end - end - - it 'migrates all keys' do - expect(Key.where(fingerprint_sha256: nil).count).to eq(Key.all.count) - - fingerprint_migrator.perform(1, 10000) - - expect(Key.where(fingerprint_sha256: nil).count).to eq(0) - end - - def generate_fingerprints! - values = "" - (1000..2000).to_a.each do |record| - key = base_key_for(record) - fingerprint = fingerprint_for(key) - - values += "(#{record}, #{record}, 'test-#{record}', '#{key}', '#{fingerprint}')," - end - - update_query = <<~SQL - INSERT INTO keys ( id, user_id, title, key, fingerprint ) - VALUES - #{values.chomp(",")}; - SQL - - ActiveRecord::Base.connection.execute(update_query) - end - - def base_key_for(record) - 'ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAIEAiPWx6WM4lhHNedGfBpPJNPpZ7yKu+dnn1SJejgt0000k6YjzGGphH2TUxwKzxcKDKKezwkpfnxPkSMkuEspGRt/aZZ9wa++Oi7Qkr8prgHc4soW6NUlfDzpvZK2H5E7eQaSeP3SAwGmQKUFHCddNaP0L+hM7zhFNzjFvpaMgJw0=' - .gsub("0000", "%04d" % (record - 1)) # generate arbitrary keys with placeholder 0000 within the key above - end - - def fingerprint_for(key) - Gitlab::SSHPublicKey.new(key).fingerprint("md5") - end -end diff --git a/spec/lib/gitlab/background_migration/migrate_pages_metadata_spec.rb b/spec/lib/gitlab/background_migration/migrate_pages_metadata_spec.rb deleted file mode 100644 index b6d93b9ff54..00000000000 --- a/spec/lib/gitlab/background_migration/migrate_pages_metadata_spec.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::MigratePagesMetadata, schema: 20181228175414 do - let(:projects) { table(:projects) } - - subject(:migrate_pages_metadata) { described_class.new } - - describe '#perform' do - let(:namespaces) { table(:namespaces) } - let(:builds) { table(:ci_builds) } - let(:pages_metadata) { table(:project_pages_metadata) } - - it 'marks specified projects with successful pages deployment' do - namespace = namespaces.create!(name: 'gitlab', path: 'gitlab-org') - not_migrated_with_pages = projects.create!(namespace_id: namespace.id, name: 'Not Migrated With Pages') - builds.create!(project_id: not_migrated_with_pages.id, type: 'GenericCommitStatus', status: 'success', stage: 'deploy', name: 'pages:deploy') - - migrated = projects.create!(namespace_id: namespace.id, name: 'Migrated') - pages_metadata.create!(project_id: migrated.id, deployed: true) - - not_migrated_no_pages = projects.create!(namespace_id: namespace.id, name: 'Not Migrated No Pages') - project_not_in_relation_scope = projects.create!(namespace_id: namespace.id, name: 'Other') - - ids = [not_migrated_no_pages.id, not_migrated_with_pages.id, migrated.id] - - migrate_pages_metadata.perform(ids.min, ids.max) - - expect(pages_metadata.find_by_project_id(not_migrated_with_pages.id).deployed).to eq(true) - expect(pages_metadata.find_by_project_id(not_migrated_no_pages.id).deployed).to eq(false) - expect(pages_metadata.find_by_project_id(migrated.id).deployed).to eq(true) - expect(pages_metadata.find_by_project_id(project_not_in_relation_scope.id)).to be_nil - end - end -end diff --git a/spec/lib/gitlab/background_migration/migrate_to_hashed_storage_spec.rb b/spec/lib/gitlab/background_migration/migrate_to_hashed_storage_spec.rb deleted file mode 100644 index 0f7bb06e830..00000000000 --- a/spec/lib/gitlab/background_migration/migrate_to_hashed_storage_spec.rb +++ /dev/null @@ -1,43 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -# rubocop:disable RSpec/FactoriesInMigrationSpecs -RSpec.describe Gitlab::BackgroundMigration::MigrateToHashedStorage, :sidekiq, :redis do - let(:migrator) { Gitlab::HashedStorage::Migrator.new } - - subject(:background_migration) { described_class.new } - - describe '#perform' do - let!(:project) { create(:project, :empty_repo, :legacy_storage) } - - context 'with pending rollback' do - it 'aborts rollback operation' do - Sidekiq::Testing.disable! do - Sidekiq::Client.push( - 'queue' => ::HashedStorage::ProjectRollbackWorker.queue, - 'class' => ::HashedStorage::ProjectRollbackWorker, - 'args' => [project.id] - ) - - expect { background_migration.perform }.to change { migrator.rollback_pending? }.from(true).to(false) - end - end - end - - it 'enqueues legacy projects to be migrated' do - Sidekiq::Testing.fake! do - expect { background_migration.perform }.to change { Sidekiq::Queues[::HashedStorage::MigratorWorker.queue].size }.by(1) - end - end - - context 'when executing all jobs' do - it 'migrates legacy projects' do - Sidekiq::Testing.inline! do - expect { background_migration.perform }.to change { project.reload.legacy_storage? }.from(true).to(false) - end - end - end - end -end -# rubocop:enable RSpec/FactoriesInMigrationSpecs diff --git a/spec/lib/gitlab/background_migration/populate_canonical_emails_spec.rb b/spec/lib/gitlab/background_migration/populate_canonical_emails_spec.rb deleted file mode 100644 index 944ee98ed4a..00000000000 --- a/spec/lib/gitlab/background_migration/populate_canonical_emails_spec.rb +++ /dev/null @@ -1,94 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::PopulateCanonicalEmails, :migration, schema: 20181228175414 do - let(:migration) { described_class.new } - - let_it_be(:users_table) { table(:users) } - let_it_be(:user_canonical_emails_table) { table(:user_canonical_emails) } - - let_it_be(:users) { users_table.all } - let_it_be(:user_canonical_emails) { user_canonical_emails_table.all } - - subject { migration.perform(1, 1) } - - describe 'gmail users' do - using RSpec::Parameterized::TableSyntax - - where(:original_email, :expected_result) do - 'legitimateuser@gmail.com' | 'legitimateuser@gmail.com' - 'userwithplus+somestuff@gmail.com' | 'userwithplus@gmail.com' - 'user.with.periods@gmail.com' | 'userwithperiods@gmail.com' - 'user.with.periods.and.plus+someotherstuff@gmail.com' | 'userwithperiodsandplus@gmail.com' - end - - with_them do - it 'generates the correct canonical email' do - create_user(email: original_email, id: 1) - - subject - - result = canonical_emails - expect(result.count).to eq 1 - expect(result.first).to match({ - 'user_id' => 1, - 'canonical_email' => expected_result - }) - end - end - end - - describe 'non gmail.com domain users' do - %w[ - legitimateuser@somedomain.com - userwithplus+somestuff@other.com - user.with.periods@gmail.org - user.with.periods.and.plus+someotherstuff@orangmail.com - ].each do |non_gmail_address| - it 'does not generate a canonical email' do - create_user(email: non_gmail_address, id: 1) - - subject - - expect(canonical_emails(user_id: 1).count).to eq 0 - end - end - end - - describe 'gracefully handles missing records' do - specify { expect { subject }.not_to raise_error } - end - - describe 'gracefully handles existing records, some of which may have an already-existing identical canonical_email field' do - let_it_be(:user_one) { create_user(email: "example.user@gmail.com", id: 1) } - let_it_be(:user_two) { create_user(email: "exampleuser@gmail.com", id: 2) } - let_it_be(:user_email_one) { user_canonical_emails.create!(canonical_email: "exampleuser@gmail.com", user_id: user_one.id) } - - subject { migration.perform(1, 2) } - - it 'only creates one record' do - subject - - expect(canonical_emails.count).not_to be_nil - end - end - - def create_user(attributes) - default_attributes = { - projects_limit: 0 - } - - users.create!(default_attributes.merge!(attributes)) - end - - def canonical_emails(user_id: nil) - filter_by_id = user_id ? "WHERE user_id = #{user_id}" : "" - - ApplicationRecord.connection.execute <<~SQL - SELECT canonical_email, user_id - FROM user_canonical_emails - #{filter_by_id}; - SQL - end -end diff --git a/spec/lib/gitlab/background_migration/populate_dismissed_state_for_vulnerabilities_spec.rb b/spec/lib/gitlab/background_migration/populate_dismissed_state_for_vulnerabilities_spec.rb deleted file mode 100644 index dc8c8c75b83..00000000000 --- a/spec/lib/gitlab/background_migration/populate_dismissed_state_for_vulnerabilities_spec.rb +++ /dev/null @@ -1,44 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe ::Gitlab::BackgroundMigration::PopulateDismissedStateForVulnerabilities, schema: 20181228175414 do - let(:users) { table(:users) } - let(:namespaces) { table(:namespaces) } - let(:projects) { table(:projects) } - let(:vulnerabilities) { table(:vulnerabilities) } - - let!(:namespace) { namespaces.create!(name: "foo", path: "bar") } - let!(:user) { users.create!(name: 'John Doe', email: 'test@example.com', projects_limit: 5) } - let!(:project) { projects.create!(namespace_id: namespace.id) } - let!(:vulnerability_params) do - { - project_id: project.id, - author_id: user.id, - title: 'Vulnerability', - severity: 5, - confidence: 5, - report_type: 5 - } - end - - let!(:vulnerability_1) { vulnerabilities.create!(vulnerability_params.merge(state: 1)) } - let!(:vulnerability_2) { vulnerabilities.create!(vulnerability_params.merge(state: 3)) } - - describe '#perform' do - it 'changes state of vulnerability to dismissed' do - subject.perform(vulnerability_1.id, vulnerability_2.id) - - expect(vulnerability_1.reload.state).to eq(2) - expect(vulnerability_2.reload.state).to eq(2) - end - - it 'populates missing dismissal information' do - expect_next_instance_of(::Gitlab::BackgroundMigration::PopulateMissingVulnerabilityDismissalInformation) do |migration| - expect(migration).to receive(:perform).with(vulnerability_1.id, vulnerability_2.id) - end - - subject.perform(vulnerability_1.id, vulnerability_2.id) - end - end -end diff --git a/spec/lib/gitlab/background_migration/populate_has_vulnerabilities_spec.rb b/spec/lib/gitlab/background_migration/populate_has_vulnerabilities_spec.rb deleted file mode 100644 index 6722321d5f7..00000000000 --- a/spec/lib/gitlab/background_migration/populate_has_vulnerabilities_spec.rb +++ /dev/null @@ -1,63 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::PopulateHasVulnerabilities, schema: 20181228175414 do - let(:users) { table(:users) } - let(:namespaces) { table(:namespaces) } - let(:projects) { table(:projects) } - let(:project_settings) { table(:project_settings) } - let(:vulnerabilities) { table(:vulnerabilities) } - - let(:user) { users.create!(name: 'test', email: 'test@example.com', projects_limit: 5) } - let(:namespace) { namespaces.create!(name: 'gitlab', path: 'gitlab-org') } - let(:vulnerability_base_params) { { title: 'title', state: 2, severity: 0, confidence: 5, report_type: 2, author_id: user.id } } - - let!(:project_1) { projects.create!(namespace_id: namespace.id, name: 'foo_1') } - let!(:project_2) { projects.create!(namespace_id: namespace.id, name: 'foo_2') } - let!(:project_3) { projects.create!(namespace_id: namespace.id, name: 'foo_3') } - - before do - project_settings.create!(project_id: project_1.id) - vulnerabilities.create!(vulnerability_base_params.merge(project_id: project_1.id)) - vulnerabilities.create!(vulnerability_base_params.merge(project_id: project_3.id)) - - allow(::Gitlab::BackgroundMigration::Logger).to receive_messages(info: true, error: true) - end - - describe '#perform' do - it 'sets `has_vulnerabilities` attribute of project_settings' do - expect { subject.perform(project_1.id, project_3.id) }.to change { project_settings.count }.from(1).to(2) - .and change { project_settings.where(has_vulnerabilities: true).count }.from(0).to(2) - end - - it 'writes info log message' do - subject.perform(project_1.id, project_3.id) - - expect(::Gitlab::BackgroundMigration::Logger).to have_received(:info).with(migrator: described_class.name, - message: 'Projects has been processed to populate `has_vulnerabilities` information', - count: 2) - end - - context 'when non-existing project_id is given' do - it 'populates only for the existing projects' do - expect { subject.perform(project_1.id, 0, project_3.id) }.to change { project_settings.count }.from(1).to(2) - .and change { project_settings.where(has_vulnerabilities: true).count }.from(0).to(2) - end - end - - context 'when an error happens' do - before do - allow(described_class::ProjectSetting).to receive(:upsert_for).and_raise('foo') - end - - it 'writes error log message' do - subject.perform(project_1.id, project_3.id) - - expect(::Gitlab::BackgroundMigration::Logger).to have_received(:error).with(migrator: described_class.name, - message: 'foo', - project_ids: [project_1.id, project_3.id]) - end - end - end -end diff --git a/spec/lib/gitlab/background_migration/populate_merge_request_assignees_table_spec.rb b/spec/lib/gitlab/background_migration/populate_merge_request_assignees_table_spec.rb deleted file mode 100644 index 1d8eed53553..00000000000 --- a/spec/lib/gitlab/background_migration/populate_merge_request_assignees_table_spec.rb +++ /dev/null @@ -1,70 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::PopulateMergeRequestAssigneesTable, schema: 20181228175414 do - let(:namespaces) { table(:namespaces) } - let(:projects) { table(:projects) } - let(:users) { table(:users) } - - let(:user) { users.create!(email: 'test@example.com', projects_limit: 100, username: 'test') } - let(:user_2) { users.create!(email: 'test2@example.com', projects_limit: 100, username: 'test') } - let(:user_3) { users.create!(email: 'test3@example.com', projects_limit: 100, username: 'test') } - - let(:namespace) { namespaces.create!(name: 'gitlab', path: 'gitlab-org') } - let(:project) { projects.create!(namespace_id: namespace.id, name: 'foo') } - let(:merge_requests) { table(:merge_requests) } - let(:merge_request_assignees) { table(:merge_request_assignees) } - - def create_merge_request(id, params = {}) - params.merge!(id: id, - target_project_id: project.id, - target_branch: 'master', - source_project_id: project.id, - source_branch: 'mr name', - title: "mr name#{id}") - - merge_requests.create!(params) - end - - before do - create_merge_request(2, assignee_id: user.id) - create_merge_request(3, assignee_id: user_2.id) - create_merge_request(4, assignee_id: user_3.id) - - # Test filtering MRs without assignees - create_merge_request(5, assignee_id: nil) - # Test filtering already migrated row - merge_request_assignees.create!(merge_request_id: 2, user_id: user_3.id) - end - - describe '#perform' do - it 'creates merge_request_assignees rows according to merge_requests' do - subject.perform(1, 4) - - rows = merge_request_assignees.order(:id).map { |row| row.attributes.slice('merge_request_id', 'user_id') } - existing_rows = [ - { 'merge_request_id' => 2, 'user_id' => user_3.id } - ] - created_rows = [ - { 'merge_request_id' => 3, 'user_id' => user_2.id }, - { 'merge_request_id' => 4, 'user_id' => user_3.id } - ] - expected_rows = existing_rows + created_rows - - expect(rows.size).to eq(expected_rows.size) - expected_rows.each do |expected_row| - expect(rows).to include(expected_row) - end - end - end - - describe '#perform_all_sync' do - it 'executes peform for all merge requests in batches' do - expect(subject).to receive(:perform).with(2, 4).ordered - expect(subject).to receive(:perform).with(5, 5).ordered - - subject.perform_all_sync(batch_size: 3) - end - end -end diff --git a/spec/lib/gitlab/background_migration/populate_missing_vulnerability_dismissal_information_spec.rb b/spec/lib/gitlab/background_migration/populate_missing_vulnerability_dismissal_information_spec.rb deleted file mode 100644 index 1c987d3876f..00000000000 --- a/spec/lib/gitlab/background_migration/populate_missing_vulnerability_dismissal_information_spec.rb +++ /dev/null @@ -1,65 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::PopulateMissingVulnerabilityDismissalInformation, schema: 20181228175414 do - let(:users) { table(:users) } - let(:namespaces) { table(:namespaces) } - let(:projects) { table(:projects) } - let(:vulnerabilities) { table(:vulnerabilities) } - let(:findings) { table(:vulnerability_occurrences) } - let(:scanners) { table(:vulnerability_scanners) } - let(:identifiers) { table(:vulnerability_identifiers) } - let(:feedback) { table(:vulnerability_feedback) } - - let(:user) { users.create!(name: 'test', email: 'test@example.com', projects_limit: 5) } - let(:namespace) { namespaces.create!(name: 'gitlab', path: 'gitlab-org') } - let(:project) { projects.create!(namespace_id: namespace.id, name: 'foo') } - let(:vulnerability_1) { vulnerabilities.create!(title: 'title', state: 2, severity: 0, confidence: 5, report_type: 2, project_id: project.id, author_id: user.id) } - let(:vulnerability_2) { vulnerabilities.create!(title: 'title', state: 2, severity: 0, confidence: 5, report_type: 2, project_id: project.id, author_id: user.id) } - let(:scanner) { scanners.create!(project_id: project.id, external_id: 'foo', name: 'bar') } - let(:identifier) { identifiers.create!(project_id: project.id, fingerprint: 'foo', external_type: 'bar', external_id: 'zoo', name: 'identifier') } - - before do - feedback.create!(feedback_type: 0, - category: 'sast', - project_fingerprint: '418291a26024a1445b23fe64de9380cdcdfd1fa8', - project_id: project.id, - author_id: user.id, - created_at: Time.current) - - findings.create!(name: 'Finding', - report_type: 'sast', - project_fingerprint: Gitlab::Database::ShaAttribute.new.serialize('418291a26024a1445b23fe64de9380cdcdfd1fa8'), - location_fingerprint: 'bar', - severity: 1, - confidence: 1, - metadata_version: 1, - raw_metadata: '', - uuid: SecureRandom.uuid, - project_id: project.id, - vulnerability_id: vulnerability_1.id, - scanner_id: scanner.id, - primary_identifier_id: identifier.id) - - allow(::Gitlab::BackgroundMigration::Logger).to receive_messages(info: true, warn: true, error: true) - end - - describe '#perform' do - it 'updates the missing dismissal information of the vulnerability' do - expect { subject.perform(vulnerability_1.id, vulnerability_2.id) }.to change { vulnerability_1.reload.dismissed_at }.from(nil) - .and change { vulnerability_1.reload.dismissed_by_id }.from(nil).to(user.id) - end - - it 'writes log messages' do - subject.perform(vulnerability_1.id, vulnerability_2.id) - - expect(::Gitlab::BackgroundMigration::Logger).to have_received(:info).with(migrator: described_class.name, - message: 'Dismissal information has been copied', - count: 2) - expect(::Gitlab::BackgroundMigration::Logger).to have_received(:warn).with(migrator: described_class.name, - message: 'Could not update vulnerability!', - vulnerability_id: vulnerability_2.id) - end - end -end diff --git a/spec/lib/gitlab/background_migration/populate_personal_snippet_statistics_spec.rb b/spec/lib/gitlab/background_migration/populate_personal_snippet_statistics_spec.rb deleted file mode 100644 index f9628849dbf..00000000000 --- a/spec/lib/gitlab/background_migration/populate_personal_snippet_statistics_spec.rb +++ /dev/null @@ -1,141 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::PopulatePersonalSnippetStatistics do - let(:file_name) { 'file_name.rb' } - let(:content) { 'content' } - let(:snippets) { table(:snippets) } - let(:snippet_repositories) { table(:snippet_repositories) } - let(:users) { table(:users) } - let(:namespaces) { table(:namespaces) } - let(:snippet_statistics) { table(:snippet_statistics) } - let(:namespace_statistics) { table(:namespace_root_storage_statistics) } - let(:routes) { table(:routes) } - let(:repo_size) { 123456 } - let(:expected_repo_size) { repo_size.megabytes } - - let(:user1) { users.create!(id: 1, email: 'test@example.com', projects_limit: 100, username: 'test1') } - let(:user2) { users.create!(id: 2, email: 'test2@example.com', projects_limit: 100, username: 'test2') } - let!(:user1_namespace) { namespaces.create!(id: 1, name: 'user1', path: 'user1', owner_id: user1.id) } - let!(:user2_namespace) { namespaces.create!(id: 2, name: 'user2', path: 'user2', owner_id: user2.id) } - let(:user1_namespace_statistics) { namespace_statistics.find_by(namespace_id: user1_namespace.id) } - let(:user2_namespace_statistics) { namespace_statistics.find_by(namespace_id: user2_namespace.id) } - - let(:ids) { snippets.pluck(:id) } - let(:migration) { described_class.new } - - subject do - migration.perform(ids) - end - - before do - allow_any_instance_of(Repository).to receive(:size).and_return(repo_size) - end - - after do - snippets.all.each { |s| raw_repository(s).remove } - end - - context 'with existing personal snippets' do - let!(:snippet1) { create_snippet(1, user1) } - let!(:snippet2) { create_snippet(2, user1) } - let!(:snippet3) { create_snippet(3, user2) } - let!(:snippet4) { create_snippet(4, user2) } - - before do - create_snippet_statistics(2, 0) - create_snippet_statistics(4, 123) - end - - it 'creates/updates all snippet_statistics' do - expect { subject }.to change { snippet_statistics.count }.from(2).to(4) - - expect(snippet_statistics.pluck(:repository_size)).to be_all(expected_repo_size) - end - - it 'creates/updates the associated namespace statistics' do - expect(migration).to receive(:update_namespace_statistics).twice.and_call_original - - subject - - stats = snippet_statistics.where(snippet_id: [snippet1, snippet2]).sum(:repository_size) - expect(user1_namespace_statistics.snippets_size).to eq stats - - stats = snippet_statistics.where(snippet_id: [snippet3, snippet4]).sum(:repository_size) - expect(user2_namespace_statistics.snippets_size).to eq stats - end - - context 'when an error is raised when updating a namespace statistics' do - it 'logs the error and continue execution' do - expect_next_instance_of(Namespaces::StatisticsRefresherService) do |instance| - expect(instance).to receive(:execute).with(Namespace.find(user1_namespace.id)).and_raise('Error') - end - - expect_next_instance_of(Namespaces::StatisticsRefresherService) do |instance| - expect(instance).to receive(:execute).and_call_original - end - - expect_next_instance_of(Gitlab::BackgroundMigration::Logger) do |instance| - expect(instance).to receive(:error).with(message: /Error updating statistics for namespace/).once - end - - subject - - expect(user1_namespace_statistics).to be_nil - - stats = snippet_statistics.where(snippet_id: [snippet3, snippet4]).sum(:repository_size) - expect(user2_namespace_statistics.snippets_size).to eq stats - end - end - end - - context 'when a snippet repository is empty' do - let!(:snippet1) { create_snippet(1, user1, with_repo: false) } - let!(:snippet2) { create_snippet(2, user1) } - - it 'logs error and continues execution' do - expect_next_instance_of(Gitlab::BackgroundMigration::Logger) do |instance| - expect(instance).to receive(:error).with(message: /Invalid snippet repository/).once - end - - subject - - expect(snippet_statistics.find_by(snippet_id: snippet1.id)).to be_nil - expect(user1_namespace_statistics.snippets_size).to eq expected_repo_size - end - end - - def create_snippet(id, author, with_repo: true) - snippets.create!(id: id, type: 'PersonalSnippet', author_id: author.id, file_name: file_name, content: content).tap do |snippet| - if with_repo - allow(snippet).to receive(:disk_path).and_return(disk_path(snippet)) - - raw_repository(snippet).create_repository - - TestEnv.copy_repo(snippet, - bare_repo: TestEnv.factory_repo_path_bare, - refs: TestEnv::BRANCH_SHA) - end - end - end - - def create_snippet_statistics(snippet_id, repository_size = 0) - snippet_statistics.create!(snippet_id: snippet_id, repository_size: repository_size) - end - - def raw_repository(snippet) - Gitlab::Git::Repository.new('default', - "#{disk_path(snippet)}.git", - Gitlab::GlRepository::SNIPPET.identifier_for_container(snippet), - "@snippets/#{snippet.id}") - end - - def hashed_repository(snippet) - Storage::Hashed.new(snippet, prefix: '@snippets') - end - - def disk_path(snippet) - hashed_repository(snippet).disk_path - end -end diff --git a/spec/lib/gitlab/background_migration/populate_project_snippet_statistics_spec.rb b/spec/lib/gitlab/background_migration/populate_project_snippet_statistics_spec.rb deleted file mode 100644 index 7884e0d97c0..00000000000 --- a/spec/lib/gitlab/background_migration/populate_project_snippet_statistics_spec.rb +++ /dev/null @@ -1,224 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::PopulateProjectSnippetStatistics do - let(:file_name) { 'file_name.rb' } - let(:content) { 'content' } - let(:snippets) { table(:snippets) } - let(:snippet_repositories) { table(:snippet_repositories) } - let(:users) { table(:users) } - let(:namespaces) { table(:namespaces) } - let(:snippet_statistics) { table(:snippet_statistics) } - let(:project_statistics) { table(:project_statistics) } - let(:projects) { table(:projects) } - let(:namespace_statistics) { table(:namespace_root_storage_statistics) } - let(:routes) { table(:routes) } - let(:repo_size) { 123456 } - let(:expected_repo_size) { repo_size.megabytes } - - let(:user) { users.create!(id: 1, email: 'test@example.com', projects_limit: 100, username: 'test') } - let(:group) { namespaces.create!(id: 10, type: 'Group', name: 'group1', path: 'group1') } - let(:user_namespace) { namespaces.create!(id: 20, name: 'user', path: 'user', owner_id: user.id) } - - let(:project1) { create_project(1, 'test', group) } - let(:project2) { create_project(2, 'test1', user_namespace) } - let(:project3) { create_project(3, 'test2', group) } - - let!(:project_stats1) { create_project_statistics(project1) } - let!(:project_stats2) { create_project_statistics(project2) } - let!(:project_stats3) { create_project_statistics(project3) } - - let(:ids) { snippets.pluck(:id) } - let(:migration) { described_class.new } - - subject do - migration.perform(ids) - - project_stats1.reload if project_stats1.persisted? - project_stats2.reload if project_stats2.persisted? - project_stats3.reload if project_stats3.persisted? - end - - before do - allow_any_instance_of(Repository).to receive(:size).and_return(repo_size) - end - - after do - snippets.all.each { |s| raw_repository(s).remove } - end - - context 'with existing user and group snippets' do - let!(:snippet1) { create_snippet(1, project1) } - let!(:snippet2) { create_snippet(2, project1) } - let!(:snippet3) { create_snippet(3, project2) } - let!(:snippet4) { create_snippet(4, project2) } - let!(:snippet5) { create_snippet(5, project3) } - - before do - create_snippet_statistics(2, 0) - create_snippet_statistics(4, 123) - end - - it 'creates/updates all snippet_statistics' do - expect(snippet_statistics.count).to eq 2 - - subject - - expect(snippet_statistics.count).to eq 5 - - snippet_statistics.all.each do |stat| - expect(stat.repository_size).to eq expected_repo_size - end - end - - it 'updates associated snippet project statistics' do - expect(project_stats1.snippets_size).to be_nil - expect(project_stats2.snippets_size).to be_nil - - subject - - snippets_size = snippet_statistics.where(snippet_id: [snippet1.id, snippet2.id]).sum(:repository_size) - expect(project_stats1.snippets_size).to eq snippets_size - - snippets_size = snippet_statistics.where(snippet_id: [snippet3.id, snippet4.id]).sum(:repository_size) - expect(project_stats2.snippets_size).to eq snippets_size - - snippets_size = snippet_statistics.where(snippet_id: snippet5.id).sum(:repository_size) - expect(project_stats3.snippets_size).to eq snippets_size - end - - it 'forces the project statistics refresh' do - expect(migration).to receive(:update_project_statistics).exactly(3).times - - subject - end - - it 'creates/updates the associated namespace statistics' do - expect(migration).to receive(:update_namespace_statistics).twice.and_call_original - - subject - - expect(namespace_statistics.find_by(namespace_id: group.id).snippets_size).to eq project_stats1.snippets_size + project_stats3.snippets_size - expect(namespace_statistics.find_by(namespace_id: user_namespace.id).snippets_size).to eq project_stats2.snippets_size - end - - context 'when the project statistics does not exists' do - it 'does not raise any error' do - project_stats3.delete - - subject - - expect(namespace_statistics.find_by(namespace_id: group.id).snippets_size).to eq project_stats1.snippets_size - expect(namespace_statistics.find_by(namespace_id: user_namespace.id).snippets_size).to eq project_stats2.snippets_size - end - end - - context 'when an error is raised when updating a project statistics' do - it 'logs the error and continue execution' do - expect(migration).to receive(:update_project_statistics).with(Project.find(project1.id)).and_raise('Error') - expect(migration).to receive(:update_project_statistics).with(Project.find(project2.id)).and_call_original - expect(migration).to receive(:update_project_statistics).with(Project.find(project3.id)).and_call_original - - expect_next_instance_of(Gitlab::BackgroundMigration::Logger) do |instance| - expect(instance).to receive(:error).with(message: /Error updating statistics for project #{project1.id}/).once - end - - subject - - expect(project_stats2.snippets_size).not_to be_nil - expect(project_stats3.snippets_size).not_to be_nil - end - end - - context 'when an error is raised when updating a namespace statistics' do - it 'logs the error and continue execution' do - expect(migration).to receive(:update_namespace_statistics).with(Group.find(group.id)).and_raise('Error') - expect(migration).to receive(:update_namespace_statistics).with(Namespace.find(user_namespace.id)).and_call_original - - expect_next_instance_of(Gitlab::BackgroundMigration::Logger) do |instance| - expect(instance).to receive(:error).with(message: /Error updating statistics for namespace/).once - end - - subject - - expect(namespace_statistics.find_by(namespace_id: user_namespace.id).snippets_size).to eq project_stats2.snippets_size - end - end - end - - context 'when project snippet is in a subgroup' do - let(:subgroup) { namespaces.create!(id: 30, type: 'Group', name: 'subgroup', path: 'subgroup', parent_id: group.id) } - let(:project1) { create_project(1, 'test', subgroup, "#{group.path}/#{subgroup.path}/test") } - let!(:snippet1) { create_snippet(1, project1) } - - it 'updates the root namespace statistics' do - subject - - expect(snippet_statistics.count).to eq 1 - expect(project_stats1.snippets_size).to eq snippet_statistics.first.repository_size - expect(namespace_statistics.find_by(namespace_id: subgroup.id)).to be_nil - expect(namespace_statistics.find_by(namespace_id: group.id).snippets_size).to eq project_stats1.snippets_size - end - end - - context 'when a snippet repository is empty' do - let!(:snippet1) { create_snippet(1, project1, with_repo: false) } - let!(:snippet2) { create_snippet(2, project1) } - - it 'logs error and continues execution' do - expect_next_instance_of(Gitlab::BackgroundMigration::Logger) do |instance| - expect(instance).to receive(:error).with(message: /Invalid snippet repository/).once - end - - subject - - expect(snippet_statistics.find_by(snippet_id: snippet1.id)).to be_nil - expect(project_stats1.snippets_size).to eq snippet_statistics.find(snippet2.id).repository_size - end - end - - def create_snippet(id, project, with_repo: true) - snippets.create!(id: id, type: 'ProjectSnippet', project_id: project.id, author_id: user.id, file_name: file_name, content: content).tap do |snippet| - if with_repo - allow(snippet).to receive(:disk_path).and_return(disk_path(snippet)) - - raw_repository(snippet).create_repository - - TestEnv.copy_repo(snippet, - bare_repo: TestEnv.factory_repo_path_bare, - refs: TestEnv::BRANCH_SHA) - end - end - end - - def create_project(id, name, namespace, path = nil) - projects.create!(id: id, name: name, path: name.downcase.gsub(/\s/, '_'), namespace_id: namespace.id).tap do |project| - path ||= "#{namespace.path}/#{project.path}" - routes.create!(id: id, source_type: 'Project', source_id: project.id, path: path) - end - end - - def create_snippet_statistics(snippet_id, repository_size = 0) - snippet_statistics.create!(snippet_id: snippet_id, repository_size: repository_size) - end - - def create_project_statistics(project, snippets_size = nil) - project_statistics.create!(id: project.id, project_id: project.id, namespace_id: project.namespace_id, snippets_size: snippets_size) - end - - def raw_repository(snippet) - Gitlab::Git::Repository.new('default', - "#{disk_path(snippet)}.git", - Gitlab::GlRepository::SNIPPET.identifier_for_container(snippet), - "@snippets/#{snippet.id}") - end - - def hashed_repository(snippet) - Storage::Hashed.new(snippet, prefix: '@snippets') - end - - def disk_path(snippet) - hashed_repository(snippet).disk_path - end -end diff --git a/spec/lib/gitlab/background_migration/recalculate_project_authorizations_with_min_max_user_id_spec.rb b/spec/lib/gitlab/background_migration/recalculate_project_authorizations_with_min_max_user_id_spec.rb deleted file mode 100644 index 1830a7fc099..00000000000 --- a/spec/lib/gitlab/background_migration/recalculate_project_authorizations_with_min_max_user_id_spec.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::RecalculateProjectAuthorizationsWithMinMaxUserId, schema: 20181228175414 do - let(:users_table) { table(:users) } - let(:min) { 1 } - let(:max) { 5 } - - before do - min.upto(max) do |i| - users_table.create!(id: i, email: "user#{i}@example.com", projects_limit: 10) - end - end - - describe '#perform' do - it 'initializes Users::RefreshAuthorizedProjectsService with correct users' do - min.upto(max) do |i| - user = User.find(i) - expect(Users::RefreshAuthorizedProjectsService).to( - receive(:new).with(user, any_args).and_call_original) - end - - described_class.new.perform(min, max) - end - - it 'executes Users::RefreshAuthorizedProjectsService' do - expected_call_counts = max - min + 1 - - service = instance_double(Users::RefreshAuthorizedProjectsService) - expect(Users::RefreshAuthorizedProjectsService).to( - receive(:new).exactly(expected_call_counts).times.and_return(service)) - expect(service).to receive(:execute).exactly(expected_call_counts).times - - described_class.new.perform(min, max) - end - end -end diff --git a/spec/lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings_spec.rb b/spec/lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings_spec.rb new file mode 100644 index 00000000000..28aa9efde4f --- /dev/null +++ b/spec/lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings_spec.rb @@ -0,0 +1,175 @@ +# frozen_string_literal: true +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::RemoveOccurrencePipelinesAndDuplicateVulnerabilitiesFindings do + let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') } + let(:users) { table(:users) } + let(:user) { create_user! } + let(:project) { table(:projects).create!(id: 14219619, namespace_id: namespace.id) } + let(:scanners) { table(:vulnerability_scanners) } + let!(:scanner1) { scanners.create!(project_id: project.id, external_id: 'test 1', name: 'test scanner 1') } + let!(:scanner2) { scanners.create!(project_id: project.id, external_id: 'test 2', name: 'test scanner 2') } + let!(:scanner3) { scanners.create!(project_id: project.id, external_id: 'test 3', name: 'test scanner 3') } + let!(:unrelated_scanner) { scanners.create!(project_id: project.id, external_id: 'unreleated_scanner', name: 'unrelated scanner') } + let(:vulnerabilities) { table(:vulnerabilities) } + let(:vulnerability_findings) { table(:vulnerability_occurrences) } + let(:vulnerability_finding_pipelines) { table(:vulnerability_occurrence_pipelines) } + let(:vulnerability_identifiers) { table(:vulnerability_identifiers) } + let(:vulnerability_identifier) do + vulnerability_identifiers.create!( + id: 1244459, + project_id: project.id, + external_type: 'vulnerability-identifier', + external_id: 'vulnerability-identifier', + fingerprint: '0a203e8cd5260a1948edbedc76c7cb91ad6a2e45', + name: 'vulnerability identifier') + end + + let!(:vulnerability_for_first_duplicate) do + create_vulnerability!( + project_id: project.id, + author_id: user.id + ) + end + + let!(:first_finding_duplicate) do + create_finding!( + id: 5606961, + uuid: "bd95c085-71aa-51d7-9bb6-08ae669c262e", + vulnerability_id: vulnerability_for_first_duplicate.id, + report_type: 0, + location_fingerprint: '00049d5119c2cb3bfb3d1ee1f6e031fe925aed75', + primary_identifier_id: vulnerability_identifier.id, + scanner_id: scanner1.id, + project_id: project.id + ) + end + + let!(:vulnerability_for_second_duplicate) do + create_vulnerability!( + project_id: project.id, + author_id: user.id + ) + end + + let!(:second_finding_duplicate) do + create_finding!( + id: 8765432, + uuid: "5b714f58-1176-5b26-8fd5-e11dfcb031b5", + vulnerability_id: vulnerability_for_second_duplicate.id, + report_type: 0, + location_fingerprint: '00049d5119c2cb3bfb3d1ee1f6e031fe925aed75', + primary_identifier_id: vulnerability_identifier.id, + scanner_id: scanner2.id, + project_id: project.id + ) + end + + let!(:vulnerability_for_third_duplicate) do + create_vulnerability!( + project_id: project.id, + author_id: user.id + ) + end + + let!(:third_finding_duplicate) do + create_finding!( + id: 8832995, + uuid: "cfe435fa-b25b-5199-a56d-7b007cc9e2d4", + vulnerability_id: vulnerability_for_third_duplicate.id, + report_type: 0, + location_fingerprint: '00049d5119c2cb3bfb3d1ee1f6e031fe925aed75', + primary_identifier_id: vulnerability_identifier.id, + scanner_id: scanner3.id, + project_id: project.id + ) + end + + let!(:unrelated_finding) do + create_finding!( + id: 9999999, + uuid: "unreleated_finding", + vulnerability_id: nil, + report_type: 1, + location_fingerprint: 'random_location_fingerprint', + primary_identifier_id: vulnerability_identifier.id, + scanner_id: unrelated_scanner.id, + project_id: project.id + ) + end + + subject { described_class.new.perform(first_finding_duplicate.id, unrelated_finding.id) } + + before do + 4.times do + create_finding_pipeline!(project_id: project.id, finding_id: first_finding_duplicate.id) + create_finding_pipeline!(project_id: project.id, finding_id: second_finding_duplicate.id) + create_finding_pipeline!(project_id: project.id, finding_id: third_finding_duplicate.id) + create_finding_pipeline!(project_id: project.id, finding_id: unrelated_finding.id) + end + end + + it 'removes Vulnerabilities::OccurrencePipelines for matching Vulnerabilities::Finding' do + expect(vulnerability_findings.count).to eq(4) + expect(vulnerability_finding_pipelines.count).to eq(16) + + expect { subject }.to change(vulnerability_finding_pipelines, :count).from(16).to(8) + .and change(vulnerability_findings, :count).from(4).to(2) + end + + private + + def create_vulnerability!(project_id:, author_id:, title: 'test', severity: 7, confidence: 7, report_type: 0) + vulnerabilities.create!( + project_id: project_id, + author_id: author_id, + title: title, + severity: severity, + confidence: confidence, + report_type: report_type + ) + end + + # rubocop:disable Metrics/ParameterLists + def create_finding!( + id: nil, + vulnerability_id:, project_id:, scanner_id:, primary_identifier_id:, + name: "test", severity: 7, confidence: 7, report_type: 0, + project_fingerprint: '123qweasdzxc', location_fingerprint: 'test', + metadata_version: 'test', raw_metadata: 'test', uuid: 'test') + params = { + vulnerability_id: vulnerability_id, + project_id: project_id, + name: name, + severity: severity, + confidence: confidence, + report_type: report_type, + project_fingerprint: project_fingerprint, + scanner_id: scanner_id, + primary_identifier_id: vulnerability_identifier.id, + location_fingerprint: location_fingerprint, + metadata_version: metadata_version, + raw_metadata: raw_metadata, + uuid: uuid + } + params[:id] = id unless id.nil? + vulnerability_findings.create!(params) + end + # rubocop:enable Metrics/ParameterLists + + def create_user!(name: "Example User", email: "user@example.com", user_type: nil, created_at: Time.zone.now, confirmed_at: Time.zone.now) + table(:users).create!( + name: name, + email: email, + username: name, + projects_limit: 0, + user_type: user_type, + confirmed_at: confirmed_at + ) + end + + def create_finding_pipeline!(project_id:, finding_id:) + pipeline = table(:ci_pipelines).create!(project_id: project_id) + vulnerability_finding_pipelines.create!(pipeline_id: pipeline.id, occurrence_id: finding_id) + end +end diff --git a/spec/lib/gitlab/background_migration/remove_vulnerability_finding_links_spec.rb b/spec/lib/gitlab/background_migration/remove_vulnerability_finding_links_spec.rb new file mode 100644 index 00000000000..fadee64886f --- /dev/null +++ b/spec/lib/gitlab/background_migration/remove_vulnerability_finding_links_spec.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::RemoveVulnerabilityFindingLinks, :migration, schema: 20211104165220 do + let(:vulnerability_findings) { table(:vulnerability_occurrences) } + let(:finding_links) { table(:vulnerability_finding_links) } + + let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user', type: Namespaces::UserNamespace.sti_name) } + let(:project) { table(:projects).create!(namespace_id: namespace.id) } + let(:scanner) { table(:vulnerability_scanners).create!(project_id: project.id, external_id: 'scanner', name: 'scanner') } + let(:vulnerability_identifier) do + table(:vulnerability_identifiers).create!( + project_id: project.id, + external_type: 'vulnerability-identifier', + external_id: 'vulnerability-identifier', + fingerprint: '7e394d1b1eb461a7406d7b1e08f057a1cf11287a', + name: 'vulnerability identifier') + end + + # vulnerability findings + let!(:findings) do + Array.new(2) do |id| + vulnerability_findings.create!( + project_id: project.id, + name: 'Vulnerability Name', + severity: 7, + confidence: 7, + report_type: 0, + project_fingerprint: '123qweasdzxc', + scanner_id: scanner.id, + primary_identifier_id: vulnerability_identifier.id, + location_fingerprint: "location_fingerprint_#{id}", + metadata_version: 'metadata_version', + raw_metadata: 'raw_metadata', + uuid: "uuid_#{id}" + ) + end + end + + # vulnerability finding links + let!(:links) do + { + findings.first => Array.new(5) { |id| finding_links.create!(vulnerability_occurrence_id: findings.first.id, name: "Link Name 1", url: "link_url1.example") }, + findings.second => Array.new(5) { |id| finding_links.create!(vulnerability_occurrence_id: findings.second.id, name: "Link Name 2", url: "link_url2.example") } + } + end + + it 'removes vulnerability links' do + expect do + subject.perform(links[findings.first].first.id, links[findings.second].last.id) + end.to change { finding_links.count }.from(10).to(0) + + expect(finding_links.all).to be_empty + end + + it 'only deletes vulnerability links for the current batch' do + expected_links = [finding_links.where(vulnerability_occurrence_id: findings.second.id)].flatten + + expect do + subject.perform(links[findings.first].first.id, links[findings.first].last.id) + end.to change { finding_links.count }.from(10).to(5) + + expect(finding_links.all).to match_array(expected_links) + end +end diff --git a/spec/lib/gitlab/background_migration/replace_blocked_by_links_spec.rb b/spec/lib/gitlab/background_migration/replace_blocked_by_links_spec.rb deleted file mode 100644 index 6cfdbb5a14e..00000000000 --- a/spec/lib/gitlab/background_migration/replace_blocked_by_links_spec.rb +++ /dev/null @@ -1,42 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::ReplaceBlockedByLinks, schema: 20181228175414 do - let(:namespace) { table(:namespaces).create!(name: 'gitlab', path: 'gitlab-org') } - let(:project) { table(:projects).create!(namespace_id: namespace.id, name: 'gitlab') } - let(:issue1) { table(:issues).create!(project_id: project.id, title: 'a') } - let(:issue2) { table(:issues).create!(project_id: project.id, title: 'b') } - let(:issue3) { table(:issues).create!(project_id: project.id, title: 'c') } - let(:issue_links) { table(:issue_links) } - let!(:blocked_link1) { issue_links.create!(source_id: issue2.id, target_id: issue1.id, link_type: 2) } - let!(:opposite_link1) { issue_links.create!(source_id: issue1.id, target_id: issue2.id, link_type: 1) } - let!(:blocked_link2) { issue_links.create!(source_id: issue1.id, target_id: issue3.id, link_type: 2) } - let!(:opposite_link2) { issue_links.create!(source_id: issue3.id, target_id: issue1.id, link_type: 0) } - let!(:nochange_link) { issue_links.create!(source_id: issue2.id, target_id: issue3.id, link_type: 1) } - - subject { described_class.new.perform(issue_links.minimum(:id), issue_links.maximum(:id)) } - - it 'deletes any opposite relations' do - subject - - expect(issue_links.ids).to match_array([nochange_link.id, blocked_link1.id, blocked_link2.id]) - end - - it 'ignores issue links other than blocked_by' do - subject - - expect(nochange_link.reload.link_type).to eq(1) - end - - it 'updates blocked_by issue links' do - subject - - expect(blocked_link1.reload.link_type).to eq(1) - expect(blocked_link1.source_id).to eq(issue1.id) - expect(blocked_link1.target_id).to eq(issue2.id) - expect(blocked_link2.reload.link_type).to eq(1) - expect(blocked_link2.source_id).to eq(issue3.id) - expect(blocked_link2.target_id).to eq(issue1.id) - end -end diff --git a/spec/lib/gitlab/background_migration/reset_merge_status_spec.rb b/spec/lib/gitlab/background_migration/reset_merge_status_spec.rb deleted file mode 100644 index 2f5074649c4..00000000000 --- a/spec/lib/gitlab/background_migration/reset_merge_status_spec.rb +++ /dev/null @@ -1,48 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::ResetMergeStatus do - let(:namespaces) { table(:namespaces) } - let(:projects) { table(:projects) } - let(:namespace) { namespaces.create!(name: 'gitlab', path: 'gitlab-org') } - let(:project) { projects.create!(namespace_id: namespace.id, name: 'foo') } - let(:merge_requests) { table(:merge_requests) } - - def create_merge_request(id, extra_params = {}) - params = { - id: id, - target_project_id: project.id, - target_branch: 'master', - source_project_id: project.id, - source_branch: 'mr name', - title: "mr name#{id}" - }.merge(extra_params) - - merge_requests.create!(params) - end - - it 'correctly updates opened mergeable MRs to unchecked' do - create_merge_request(1, state_id: MergeRequest.available_states[:opened], merge_status: 'can_be_merged') - create_merge_request(2, state_id: MergeRequest.available_states[:opened], merge_status: 'can_be_merged') - create_merge_request(3, state_id: MergeRequest.available_states[:opened], merge_status: 'can_be_merged') - create_merge_request(4, state_id: MergeRequest.available_states[:merged], merge_status: 'can_be_merged') - create_merge_request(5, state_id: MergeRequest.available_states[:opened], merge_status: 'cannot_be_merged') - - subject.perform(1, 5) - - expected_rows = [ - { id: 1, state_id: MergeRequest.available_states[:opened], merge_status: 'unchecked' }, - { id: 2, state_id: MergeRequest.available_states[:opened], merge_status: 'unchecked' }, - { id: 3, state_id: MergeRequest.available_states[:opened], merge_status: 'unchecked' }, - { id: 4, state_id: MergeRequest.available_states[:merged], merge_status: 'can_be_merged' }, - { id: 5, state_id: MergeRequest.available_states[:opened], merge_status: 'cannot_be_merged' } - ] - - rows = merge_requests.order(:id).map do |row| - row.attributes.slice('id', 'state_id', 'merge_status').symbolize_keys - end - - expect(rows).to eq(expected_rows) - end -end diff --git a/spec/lib/gitlab/background_migration/reset_shared_runners_for_transferred_projects_spec.rb b/spec/lib/gitlab/background_migration/reset_shared_runners_for_transferred_projects_spec.rb deleted file mode 100644 index ef90b5674f0..00000000000 --- a/spec/lib/gitlab/background_migration/reset_shared_runners_for_transferred_projects_spec.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::ResetSharedRunnersForTransferredProjects, schema: 20181228175414 do - let(:namespaces) { table(:namespaces) } - let(:projects) { table(:projects) } - - let(:namespace_1) { namespaces.create!(name: 'foo', path: 'foo', shared_runners_enabled: true, allow_descendants_override_disabled_shared_runners: false ) } - let(:namespace_2) { namespaces.create!(name: 'foo', path: 'foo', shared_runners_enabled: false, allow_descendants_override_disabled_shared_runners: false ) } - let(:namespace_3) { namespaces.create!(name: 'bar', path: 'bar', shared_runners_enabled: false, allow_descendants_override_disabled_shared_runners: true ) } - let(:project_1_1) { projects.create!(namespace_id: namespace_1.id, shared_runners_enabled: true) } - let(:project_1_2) { projects.create!(namespace_id: namespace_1.id, shared_runners_enabled: false) } - let(:project_2_1) { projects.create!(namespace_id: namespace_2.id, shared_runners_enabled: true) } - let(:project_2_2) { projects.create!(namespace_id: namespace_2.id, shared_runners_enabled: false) } - let(:project_3_1) { projects.create!(namespace_id: namespace_3.id, shared_runners_enabled: true) } - let(:project_3_2) { projects.create!(namespace_id: namespace_3.id, shared_runners_enabled: false) } - - it 'corrects each project shared_runners_enabled column' do - expect do - described_class.new.perform(namespace_1.id, namespace_3.id) - project_1_1.reload - project_1_2.reload - project_2_1.reload - project_2_2.reload - project_3_1.reload - project_3_2.reload - end.to not_change(project_1_1, :shared_runners_enabled).from(true) - .and not_change(project_1_2, :shared_runners_enabled).from(false) - .and change(project_2_1, :shared_runners_enabled).from(true).to(false) - .and not_change(project_2_2, :shared_runners_enabled).from(false) - .and not_change(project_3_1, :shared_runners_enabled).from(true) - .and not_change(project_3_2, :shared_runners_enabled).from(false) - end -end diff --git a/spec/lib/gitlab/background_migration/set_merge_request_diff_files_count_spec.rb b/spec/lib/gitlab/background_migration/set_merge_request_diff_files_count_spec.rb deleted file mode 100644 index 1fdbdf25706..00000000000 --- a/spec/lib/gitlab/background_migration/set_merge_request_diff_files_count_spec.rb +++ /dev/null @@ -1,51 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::SetMergeRequestDiffFilesCount, schema: 20181228175414 do - let(:merge_request_diff_files) { table(:merge_request_diff_files) } - let(:merge_request_diffs) { table(:merge_request_diffs) } - let(:merge_requests) { table(:merge_requests) } - let(:namespaces) { table(:namespaces) } - let(:projects) { table(:projects) } - - let(:namespace) { namespaces.create!(name: 'foo', path: 'foo') } - let(:project) { projects.create!(namespace_id: namespace.id) } - let(:merge_request) { merge_requests.create!(source_branch: 'x', target_branch: 'master', target_project_id: project.id) } - - let!(:empty_diff) { merge_request_diffs.create!(merge_request_id: merge_request.id) } - let!(:filled_diff) { merge_request_diffs.create!(merge_request_id: merge_request.id) } - - let!(:filled_diff_files) do - 1.upto(3).map do |n| - merge_request_diff_files.create!( - merge_request_diff_id: filled_diff.id, - relative_order: n, - new_file: false, - renamed_file: false, - deleted_file: false, - too_large: false, - a_mode: '', - b_mode: '', - old_path: '', - new_path: '' - ) - end - end - - it 'fills the files_count column' do - described_class.new.perform(empty_diff.id, filled_diff.id) - - expect(empty_diff.reload.files_count).to eq(0) - expect(filled_diff.reload.files_count).to eq(3) - end - - it 'uses the sentinel value if the actual count is too high' do - stub_const("#{described_class}::FILES_COUNT_SENTINEL", filled_diff_files.size - 1) - - described_class.new.perform(empty_diff.id, filled_diff.id) - - expect(empty_diff.reload.files_count).to eq(0) - expect(filled_diff.reload.files_count).to eq(described_class::FILES_COUNT_SENTINEL) - end -end diff --git a/spec/lib/gitlab/background_migration/update_existing_subgroup_to_match_visibility_level_of_parent_spec.rb b/spec/lib/gitlab/background_migration/update_existing_subgroup_to_match_visibility_level_of_parent_spec.rb deleted file mode 100644 index de9799c3642..00000000000 --- a/spec/lib/gitlab/background_migration/update_existing_subgroup_to_match_visibility_level_of_parent_spec.rb +++ /dev/null @@ -1,46 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::UpdateExistingSubgroupToMatchVisibilityLevelOfParent, schema: 20181228175414 do - include MigrationHelpers::NamespacesHelpers - - context 'private visibility level' do - it 'updates the project visibility' do - parent = create_namespace('parent', Gitlab::VisibilityLevel::PRIVATE) - child = create_namespace('child', Gitlab::VisibilityLevel::PUBLIC, parent_id: parent.id) - - expect { subject.perform([parent.id], Gitlab::VisibilityLevel::PRIVATE) }.to change { child.reload.visibility_level }.to(Gitlab::VisibilityLevel::PRIVATE) - end - - it 'updates sub-sub groups' do - parent = create_namespace('parent', Gitlab::VisibilityLevel::PRIVATE) - middle_group = create_namespace('middle', Gitlab::VisibilityLevel::PRIVATE, parent_id: parent.id) - child = create_namespace('child', Gitlab::VisibilityLevel::PUBLIC, parent_id: middle_group.id) - - subject.perform([parent.id, middle_group.id], Gitlab::VisibilityLevel::PRIVATE) - - expect(child.reload.visibility_level).to eq(Gitlab::VisibilityLevel::PRIVATE) - end - - it 'updates all sub groups' do - parent = create_namespace('parent', Gitlab::VisibilityLevel::PRIVATE) - middle_group = create_namespace('middle', Gitlab::VisibilityLevel::PUBLIC, parent_id: parent.id) - child = create_namespace('child', Gitlab::VisibilityLevel::PUBLIC, parent_id: middle_group.id) - - subject.perform([parent.id], Gitlab::VisibilityLevel::PRIVATE) - - expect(child.reload.visibility_level).to eq(Gitlab::VisibilityLevel::PRIVATE) - expect(middle_group.reload.visibility_level).to eq(Gitlab::VisibilityLevel::PRIVATE) - end - end - - context 'internal visibility level' do - it 'updates the project visibility' do - parent = create_namespace('parent', Gitlab::VisibilityLevel::INTERNAL) - child = create_namespace('child', Gitlab::VisibilityLevel::PUBLIC, parent_id: parent.id) - - expect { subject.perform([parent.id], Gitlab::VisibilityLevel::INTERNAL) }.to change { child.reload.visibility_level }.to(Gitlab::VisibilityLevel::INTERNAL) - end - end -end diff --git a/spec/lib/gitlab/background_migration/update_existing_users_that_require_two_factor_auth_spec.rb b/spec/lib/gitlab/background_migration/update_existing_users_that_require_two_factor_auth_spec.rb deleted file mode 100644 index 33f5e38100e..00000000000 --- a/spec/lib/gitlab/background_migration/update_existing_users_that_require_two_factor_auth_spec.rb +++ /dev/null @@ -1,74 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::UpdateExistingUsersThatRequireTwoFactorAuth, schema: 20181228175414 do - include MigrationHelpers::NamespacesHelpers - - let(:group_with_2fa_parent) { create_namespace('parent', Gitlab::VisibilityLevel::PRIVATE) } - let(:group_with_2fa_child) { create_namespace('child', Gitlab::VisibilityLevel::PRIVATE, parent_id: group_with_2fa_parent.id) } - let(:members_table) { table(:members) } - let(:users_table) { table(:users) } - - subject { described_class.new } - - describe '#perform' do - context 'with group members' do - let(:user_1) { create_user('user@example.com') } - let!(:member) { create_group_member(user_1, group_with_2fa_parent) } - let!(:user_without_group) { create_user('user_without@example.com') } - let(:user_other) { create_user('user_other@example.com') } - let!(:member_other) { create_group_member(user_other, group_with_2fa_parent) } - - it 'updates user when user should not be required to establish two factor authentication' do - subject.perform(user_1.id, user_without_group.id) - - expect(user_1.reload.require_two_factor_authentication_from_group).to eq(false) - end - - it 'does not update user when user is member of group that requires two factor authentication' do - group = create_namespace('other', Gitlab::VisibilityLevel::PRIVATE, require_two_factor_authentication: true) - create_group_member(user_1, group) - - subject.perform(user_1.id, user_without_group.id) - - expect(user_1.reload.require_two_factor_authentication_from_group).to eq(true) - end - - it 'does not update user who is not in current batch' do - subject.perform(user_1.id, user_without_group.id) - - expect(user_other.reload.require_two_factor_authentication_from_group).to eq(true) - end - - it 'updates all users in current batch' do - subject.perform(user_1.id, user_other.id) - - expect(user_other.reload.require_two_factor_authentication_from_group).to eq(false) - end - - it 'does not update user when user is member of group which parent group requires two factor authentication' do - group_with_2fa_parent.update!(require_two_factor_authentication: true) - subject.perform(user_1.id, user_other.id) - - expect(user_1.reload.require_two_factor_authentication_from_group).to eq(true) - end - - it 'does not update user when user is member of group which has subgroup that requires two factor authentication' do - create_namespace('subgroup', Gitlab::VisibilityLevel::PRIVATE, require_two_factor_authentication: true, parent_id: group_with_2fa_child.id) - - subject.perform(user_1.id, user_other.id) - - expect(user_1.reload.require_two_factor_authentication_from_group).to eq(true) - end - end - end - - def create_user(email, require_2fa: true) - users_table.create!(email: email, projects_limit: 10, require_two_factor_authentication_from_group: require_2fa) - end - - def create_group_member(user, group) - members_table.create!(user_id: user.id, source_id: group.id, access_level: GroupMember::MAINTAINER, source_type: "Namespace", type: "GroupMember", notification_level: 3) - end -end diff --git a/spec/lib/gitlab/background_migration/update_timelogs_null_spent_at_spec.rb b/spec/lib/gitlab/background_migration/update_timelogs_null_spent_at_spec.rb new file mode 100644 index 00000000000..982e3319063 --- /dev/null +++ b/spec/lib/gitlab/background_migration/update_timelogs_null_spent_at_spec.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::UpdateTimelogsNullSpentAt, schema: 20211215090620 do + let_it_be(:previous_time) { 10.days.ago } + let_it_be(:namespace) { table(:namespaces).create!(name: 'namespace', path: 'namespace') } + let_it_be(:project) { table(:projects).create!(namespace_id: namespace.id) } + let_it_be(:issue) { table(:issues).create!(project_id: project.id) } + let_it_be(:merge_request) { table(:merge_requests).create!(target_project_id: project.id, source_branch: 'master', target_branch: 'feature') } + let_it_be(:timelog1) { create_timelog!(issue_id: issue.id) } + let_it_be(:timelog2) { create_timelog!(merge_request_id: merge_request.id) } + let_it_be(:timelog3) { create_timelog!(issue_id: issue.id, spent_at: previous_time) } + let_it_be(:timelog4) { create_timelog!(merge_request_id: merge_request.id, spent_at: previous_time) } + + subject(:background_migration) { described_class.new } + + before_all do + table(:timelogs).where.not(id: [timelog3.id, timelog4.id]).update_all(spent_at: nil) + end + + describe '#perform' do + it 'sets correct spent_at' do + background_migration.perform(timelog1.id, timelog4.id) + + expect(timelog1.reload.spent_at).to be_like_time(timelog1.created_at) + expect(timelog2.reload.spent_at).to be_like_time(timelog2.created_at) + expect(timelog3.reload.spent_at).to be_like_time(previous_time) + expect(timelog4.reload.spent_at).to be_like_time(previous_time) + expect(timelog3.reload.spent_at).not_to be_like_time(timelog3.created_at) + expect(timelog4.reload.spent_at).not_to be_like_time(timelog4.created_at) + end + end + + private + + def create_timelog!(**args) + table(:timelogs).create!(**args, time_spent: 1) + end +end diff --git a/spec/lib/gitlab/background_migration/user_mentions/create_resource_user_mention_spec.rb b/spec/lib/gitlab/background_migration/user_mentions/create_resource_user_mention_spec.rb deleted file mode 100644 index 7af11ffa1e0..00000000000 --- a/spec/lib/gitlab/background_migration/user_mentions/create_resource_user_mention_spec.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::UserMentions::CreateResourceUserMention, schema: 20181228175414 do - context 'checks no_quote_columns' do - it 'has correct no_quote_columns' do - expect(Gitlab::BackgroundMigration::UserMentions::Models::MergeRequest.no_quote_columns).to match([:note_id, :merge_request_id]) - end - - it 'commit has correct no_quote_columns' do - expect(Gitlab::BackgroundMigration::UserMentions::Models::Commit.no_quote_columns).to match([:note_id]) - end - end -end diff --git a/spec/lib/gitlab/background_migration_spec.rb b/spec/lib/gitlab/background_migration_spec.rb index 777dc8112a7..8dd7f6892a6 100644 --- a/spec/lib/gitlab/background_migration_spec.rb +++ b/spec/lib/gitlab/background_migration_spec.rb @@ -3,11 +3,12 @@ require 'spec_helper' RSpec.describe Gitlab::BackgroundMigration do - let(:coordinator) { described_class::JobCoordinator.for_database(:main) } + let(:default_tracking_database) { described_class::DEFAULT_TRACKING_DATABASE } + let(:coordinator) { described_class::JobCoordinator.for_tracking_database(default_tracking_database) } before do allow(described_class).to receive(:coordinator_for_database) - .with(:main) + .with(default_tracking_database) .and_return(coordinator) end diff --git a/spec/lib/gitlab/bitbucket_server_import/importer_spec.rb b/spec/lib/gitlab/bitbucket_server_import/importer_spec.rb index f9313f0ff28..0380ddd9a2e 100644 --- a/spec/lib/gitlab/bitbucket_server_import/importer_spec.rb +++ b/spec/lib/gitlab/bitbucket_server_import/importer_spec.rb @@ -27,20 +27,26 @@ RSpec.describe Gitlab::BitbucketServerImport::Importer do end describe '#import_repository' do + let(:repo_url) { 'http://bitbucket:test@my-bitbucket' } + + before do + expect(project.repository).to receive(:import_repository).with(repo_url) + end + it 'adds a remote' do expect(subject).to receive(:import_pull_requests) expect(subject).to receive(:delete_temp_branches) expect(project.repository).to receive(:fetch_as_mirror) - .with('http://bitbucket:test@my-bitbucket', - refmap: [:heads, :tags, '+refs/pull-requests/*/to:refs/merge-requests/*/head']) + .with(repo_url, + refmap: ['+refs/pull-requests/*/to:refs/merge-requests/*/head']) subject.execute end - it 'raises a Gitlab::Shell exception in the fetch' do - expect(project.repository).to receive(:fetch_as_mirror).and_raise(Gitlab::Shell::Error) + it 'raises a Gitlab::Git::CommandError in the fetch' do + expect(project.repository).to receive(:fetch_as_mirror).and_raise(::Gitlab::Git::CommandError) - expect { subject.execute }.to raise_error(Gitlab::Shell::Error) + expect { subject.execute }.to raise_error(::Gitlab::Git::CommandError) end it 'raises an unhandled exception in the fetch' do diff --git a/spec/lib/gitlab/ci/build/context/build_spec.rb b/spec/lib/gitlab/ci/build/context/build_spec.rb index 46447231424..7f862a3b80a 100644 --- a/spec/lib/gitlab/ci/build/context/build_spec.rb +++ b/spec/lib/gitlab/ci/build/context/build_spec.rb @@ -8,11 +8,7 @@ RSpec.describe Gitlab::Ci::Build::Context::Build do let(:context) { described_class.new(pipeline, seed_attributes) } - describe '#variables' do - subject { context.variables.to_hash } - - it { expect(context.variables).to be_instance_of(Gitlab::Ci::Variables::Collection) } - + shared_examples 'variables collection' do it { is_expected.to include('CI_COMMIT_REF_NAME' => 'master') } it { is_expected.to include('CI_PIPELINE_IID' => pipeline.iid.to_s) } it { is_expected.to include('CI_PROJECT_PATH' => pipeline.project.full_path) } @@ -27,4 +23,20 @@ RSpec.describe Gitlab::Ci::Build::Context::Build do it { is_expected.to include('CI_PROJECT_PATH' => pipeline.project.full_path) } end end + + describe '#variables' do + subject { context.variables.to_hash } + + it { expect(context.variables).to be_instance_of(Gitlab::Ci::Variables::Collection) } + + it_behaves_like 'variables collection' + end + + describe '#variables_hash' do + subject { context.variables_hash } + + it { expect(context.variables_hash).to be_instance_of(ActiveSupport::HashWithIndifferentAccess) } + + it_behaves_like 'variables collection' + end end diff --git a/spec/lib/gitlab/ci/build/context/global_spec.rb b/spec/lib/gitlab/ci/build/context/global_spec.rb index 61f2b90426d..d4141eb8389 100644 --- a/spec/lib/gitlab/ci/build/context/global_spec.rb +++ b/spec/lib/gitlab/ci/build/context/global_spec.rb @@ -8,11 +8,7 @@ RSpec.describe Gitlab::Ci::Build::Context::Global do let(:context) { described_class.new(pipeline, yaml_variables: yaml_variables) } - describe '#variables' do - subject { context.variables.to_hash } - - it { expect(context.variables).to be_instance_of(Gitlab::Ci::Variables::Collection) } - + shared_examples 'variables collection' do it { is_expected.to include('CI_COMMIT_REF_NAME' => 'master') } it { is_expected.to include('CI_PIPELINE_IID' => pipeline.iid.to_s) } it { is_expected.to include('CI_PROJECT_PATH' => pipeline.project.full_path) } @@ -26,4 +22,20 @@ RSpec.describe Gitlab::Ci::Build::Context::Global do it { is_expected.to include('SUPPORTED' => 'parsed') } end end + + describe '#variables' do + subject { context.variables.to_hash } + + it { expect(context.variables).to be_instance_of(Gitlab::Ci::Variables::Collection) } + + it_behaves_like 'variables collection' + end + + describe '#variables_hash' do + subject { context.variables_hash } + + it { is_expected.to be_instance_of(ActiveSupport::HashWithIndifferentAccess) } + + it_behaves_like 'variables collection' + end end diff --git a/spec/lib/gitlab/ci/build/policy/variables_spec.rb b/spec/lib/gitlab/ci/build/policy/variables_spec.rb index 6c8c968dc0c..436ad59bdf7 100644 --- a/spec/lib/gitlab/ci/build/policy/variables_spec.rb +++ b/spec/lib/gitlab/ci/build/policy/variables_spec.rb @@ -16,7 +16,7 @@ RSpec.describe Gitlab::Ci::Build::Policy::Variables do let(:seed) do double('build seed', to_resource: ci_build, - variables: ci_build.scoped_variables + variables_hash: ci_build.scoped_variables.to_hash ) end @@ -91,7 +91,7 @@ RSpec.describe Gitlab::Ci::Build::Policy::Variables do let(:seed) do double('bridge seed', to_resource: bridge, - variables: ci_build.scoped_variables + variables_hash: ci_build.scoped_variables.to_hash ) end diff --git a/spec/lib/gitlab/ci/build/rules/rule/clause/changes_spec.rb b/spec/lib/gitlab/ci/build/rules/rule/clause/changes_spec.rb index d20ea6c9202..532c83f6768 100644 --- a/spec/lib/gitlab/ci/build/rules/rule/clause/changes_spec.rb +++ b/spec/lib/gitlab/ci/build/rules/rule/clause/changes_spec.rb @@ -33,12 +33,12 @@ RSpec.describe Gitlab::Ci::Build::Rules::Rule::Clause::Changes do end context 'when context has the specified variables' do - let(:variables) do - [{ key: "HELM_DIR", value: "helm", public: true }] + let(:variables_hash) do + { 'HELM_DIR' => 'helm' } end before do - allow(context).to receive(:variables).and_return(variables) + allow(context).to receive(:variables_hash).and_return(variables_hash) end it { is_expected.to be_truthy } @@ -49,7 +49,7 @@ RSpec.describe Gitlab::Ci::Build::Rules::Rule::Clause::Changes do let(:modified_paths) { ['path/with/$in/it/file.txt'] } before do - allow(context).to receive(:variables).and_return([]) + allow(context).to receive(:variables_hash).and_return({}) end it { is_expected.to be_truthy } diff --git a/spec/lib/gitlab/ci/build/rules/rule_spec.rb b/spec/lib/gitlab/ci/build/rules/rule_spec.rb index 6f3c9278677..f905e229415 100644 --- a/spec/lib/gitlab/ci/build/rules/rule_spec.rb +++ b/spec/lib/gitlab/ci/build/rules/rule_spec.rb @@ -6,7 +6,7 @@ RSpec.describe Gitlab::Ci::Build::Rules::Rule do let(:seed) do double('build seed', to_resource: ci_build, - variables: ci_build.scoped_variables + variables_hash: ci_build.scoped_variables.to_hash ) end diff --git a/spec/lib/gitlab/ci/build/rules_spec.rb b/spec/lib/gitlab/ci/build/rules_spec.rb index 1d5bdf30278..37bfdca4d1d 100644 --- a/spec/lib/gitlab/ci/build/rules_spec.rb +++ b/spec/lib/gitlab/ci/build/rules_spec.rb @@ -3,13 +3,13 @@ require 'spec_helper' RSpec.describe Gitlab::Ci::Build::Rules do - let(:pipeline) { create(:ci_pipeline) } - let(:ci_build) { build(:ci_build, pipeline: pipeline) } + let_it_be(:pipeline) { create(:ci_pipeline) } + let_it_be(:ci_build) { build(:ci_build, pipeline: pipeline) } let(:seed) do double('build seed', to_resource: ci_build, - variables: ci_build.scoped_variables + variables_hash: ci_build.scoped_variables.to_hash ) end diff --git a/spec/lib/gitlab/ci/config/entry/bridge_spec.rb b/spec/lib/gitlab/ci/config/entry/bridge_spec.rb index 6c9c8fa5df5..62feed3dda0 100644 --- a/spec/lib/gitlab/ci/config/entry/bridge_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/bridge_spec.rb @@ -163,7 +163,7 @@ RSpec.describe Gitlab::Ci::Config::Entry::Bridge do }) end - it { is_expected.not_to be_valid } + it { is_expected.to be_valid } end context 'when bridge configuration uses rules with only' do diff --git a/spec/lib/gitlab/ci/config/entry/job_spec.rb b/spec/lib/gitlab/ci/config/entry/job_spec.rb index 0bb26babfc0..885f3eaff79 100644 --- a/spec/lib/gitlab/ci/config/entry/job_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/job_spec.rb @@ -118,6 +118,20 @@ RSpec.describe Gitlab::Ci::Config::Entry::Job do end end + context 'when config uses both "when:" and "rules:"' do + let(:config) do + { + script: 'echo', + when: 'on_failure', + rules: [{ if: '$VARIABLE', when: 'on_success' }] + } + end + + it 'is valid' do + expect(entry).to be_valid + end + end + context 'when delayed job' do context 'when start_in is specified' do let(:config) { { script: 'echo', when: 'delayed', start_in: '1 week' } } @@ -268,21 +282,6 @@ RSpec.describe Gitlab::Ci::Config::Entry::Job do end end - context 'when it uses both "when:" and "rules:"' do - let(:config) do - { - script: 'echo', - when: 'on_failure', - rules: [{ if: '$VARIABLE', when: 'on_success' }] - } - end - - it 'returns an error about when: being combined with rules' do - expect(entry).not_to be_valid - expect(entry.errors).to include 'job config key may not be used with `rules`: when' - end - end - context 'when delayed job' do context 'when start_in is specified' do let(:config) { { script: 'echo', when: 'delayed', start_in: '1 week' } } diff --git a/spec/lib/gitlab/ci/config/entry/processable_spec.rb b/spec/lib/gitlab/ci/config/entry/processable_spec.rb index c9c28e2eb8b..5b9337ede34 100644 --- a/spec/lib/gitlab/ci/config/entry/processable_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/processable_spec.rb @@ -33,6 +33,20 @@ RSpec.describe Gitlab::Ci::Config::Entry::Processable do end end + context 'when config uses both "when:" and "rules:"' do + let(:config) do + { + script: 'echo', + when: 'on_failure', + rules: [{ if: '$VARIABLE', when: 'on_success' }] + } + end + + it 'is valid' do + expect(entry).to be_valid + end + end + context 'when job name is more than 255' do let(:entry) { node_class.new(config, name: ('a' * 256).to_sym) } @@ -90,21 +104,6 @@ RSpec.describe Gitlab::Ci::Config::Entry::Processable do end end - context 'when it uses both "when:" and "rules:"' do - let(:config) do - { - script: 'echo', - when: 'on_failure', - rules: [{ if: '$VARIABLE', when: 'on_success' }] - } - end - - it 'returns an error about when: being combined with rules' do - expect(entry).not_to be_valid - expect(entry.errors).to include 'job config key may not be used with `rules`: when' - end - end - context 'when only: is used with rules:' do let(:config) { { only: ['merge_requests'], rules: [{ if: '$THIS' }] } } diff --git a/spec/lib/gitlab/ci/config/entry/tags_spec.rb b/spec/lib/gitlab/ci/config/entry/tags_spec.rb index 79317de373b..e05d4ae52b2 100644 --- a/spec/lib/gitlab/ci/config/entry/tags_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/tags_spec.rb @@ -36,25 +36,9 @@ RSpec.describe Gitlab::Ci::Config::Entry::Tags do context 'when tags limit is reached' do let(:config) { Array.new(50) {|i| "tag-#{i}" } } - context 'when ci_build_tags_limit is enabled' do - before do - stub_feature_flags(ci_build_tags_limit: true) - end - - it 'reports error' do - expect(entry.errors) - .to include "tags config must be less than the limit of #{described_class::TAGS_LIMIT} tags" - end - end - - context 'when ci_build_tags_limit is disabled' do - before do - stub_feature_flags(ci_build_tags_limit: false) - end - - it 'does not report an error' do - expect(entry.errors).to be_empty - end + it 'reports error' do + expect(entry.errors) + .to include "tags config must be less than the limit of #{described_class::TAGS_LIMIT} tags" end end end diff --git a/spec/lib/gitlab/ci/config/external/context_spec.rb b/spec/lib/gitlab/ci/config/external/context_spec.rb index 4b9adf7e87b..800c563cd0b 100644 --- a/spec/lib/gitlab/ci/config/external/context_spec.rb +++ b/spec/lib/gitlab/ci/config/external/context_spec.rb @@ -6,7 +6,8 @@ RSpec.describe Gitlab::Ci::Config::External::Context do let(:project) { double('Project') } let(:user) { double('User') } let(:sha) { '12345' } - let(:attributes) { { project: project, user: user, sha: sha } } + let(:variables) { Gitlab::Ci::Variables::Collection.new([{ 'key' => 'a', 'value' => 'b' }]) } + let(:attributes) { { project: project, user: user, sha: sha, variables: variables } } subject(:subject) { described_class.new(**attributes) } @@ -15,6 +16,9 @@ RSpec.describe Gitlab::Ci::Config::External::Context do it { is_expected.to have_attributes(**attributes) } it { expect(subject.expandset).to eq(Set.new) } it { expect(subject.execution_deadline).to eq(0) } + it { expect(subject.variables).to be_instance_of(Gitlab::Ci::Variables::Collection) } + it { expect(subject.variables_hash).to be_instance_of(ActiveSupport::HashWithIndifferentAccess) } + it { expect(subject.variables_hash).to include('a' => 'b') } end context 'without values' do @@ -23,6 +27,8 @@ RSpec.describe Gitlab::Ci::Config::External::Context do it { is_expected.to have_attributes(**attributes) } it { expect(subject.expandset).to eq(Set.new) } it { expect(subject.execution_deadline).to eq(0) } + it { expect(subject.variables).to be_instance_of(Gitlab::Ci::Variables::Collection) } + it { expect(subject.variables_hash).to be_instance_of(ActiveSupport::HashWithIndifferentAccess) } end end @@ -94,6 +100,15 @@ RSpec.describe Gitlab::Ci::Config::External::Context do end describe '#mutate' do + let(:attributes) do + { + project: project, + user: user, + sha: sha, + logger: double('logger') + } + end + shared_examples 'a mutated context' do let(:mutated) { subject.mutate(new_attributes) } @@ -107,6 +122,7 @@ RSpec.describe Gitlab::Ci::Config::External::Context do it { expect(mutated).to have_attributes(new_attributes) } it { expect(mutated.expandset).to eq(subject.expandset) } it { expect(mutated.execution_deadline).to eq(mutated.execution_deadline) } + it { expect(mutated.logger).to eq(mutated.logger) } end context 'with attributes' do diff --git a/spec/lib/gitlab/ci/config/external/processor_spec.rb b/spec/lib/gitlab/ci/config/external/processor_spec.rb index 2e9e6f95071..97bd74721f2 100644 --- a/spec/lib/gitlab/ci/config/external/processor_spec.rb +++ b/spec/lib/gitlab/ci/config/external/processor_spec.rb @@ -6,7 +6,7 @@ RSpec.describe Gitlab::Ci::Config::External::Processor do include StubRequests let_it_be(:project) { create(:project, :repository) } - let_it_be(:another_project) { create(:project, :repository) } + let_it_be_with_reload(:another_project) { create(:project, :repository) } let_it_be(:user) { create(:user) } let(:sha) { '12345' } @@ -251,6 +251,17 @@ RSpec.describe Gitlab::Ci::Config::External::Processor do it 'properly expands all includes' do is_expected.to include(:my_build, :remote_build, :rspec) end + + it 'propagates the pipeline logger' do + processor.perform + + process_obs_count = processor + .logger + .observations_hash + .dig('config_mapper_process_duration_s', 'count') + + expect(process_obs_count).to eq(3) + end end context 'when user is reporter of another project' do diff --git a/spec/lib/gitlab/ci/config/external/rules_spec.rb b/spec/lib/gitlab/ci/config/external/rules_spec.rb index 1e42cb30ae7..091bd3b07e6 100644 --- a/spec/lib/gitlab/ci/config/external/rules_spec.rb +++ b/spec/lib/gitlab/ci/config/external/rules_spec.rb @@ -8,7 +8,7 @@ RSpec.describe Gitlab::Ci::Config::External::Rules do subject(:rules) { described_class.new(rule_hashes) } describe '#evaluate' do - let(:context) { double(variables: {}) } + let(:context) { double(variables_hash: {}) } subject(:result) { rules.evaluate(context).pass? } @@ -20,13 +20,13 @@ RSpec.describe Gitlab::Ci::Config::External::Rules do let(:rule_hashes) { [{ if: '$MY_VAR == "hello"' }] } context 'when the rule matches' do - let(:context) { double(variables: { MY_VAR: 'hello' }) } + let(:context) { double(variables_hash: { 'MY_VAR' => 'hello' }) } it { is_expected.to eq(true) } end context 'when the rule does not match' do - let(:context) { double(variables: { MY_VAR: 'invalid' }) } + let(:context) { double(variables_hash: { 'MY_VAR' => 'invalid' }) } it { is_expected.to eq(false) } end diff --git a/spec/lib/gitlab/ci/parsers/terraform/tfplan_spec.rb b/spec/lib/gitlab/ci/parsers/terraform/tfplan_spec.rb index f487fccdab7..60b4e01f382 100644 --- a/spec/lib/gitlab/ci/parsers/terraform/tfplan_spec.rb +++ b/spec/lib/gitlab/ci/parsers/terraform/tfplan_spec.rb @@ -103,7 +103,7 @@ RSpec.describe Gitlab::Ci::Parsers::Terraform::Tfplan do 'create' => 0, 'update' => 1, 'delete' => 0, - 'job_name' => artifact.job.options.dig(:artifacts, :name).to_s + 'job_name' => artifact.job.name ) ) ) @@ -124,7 +124,7 @@ RSpec.describe Gitlab::Ci::Parsers::Terraform::Tfplan do 'create' => 0, 'update' => 1, 'delete' => 0, - 'job_name' => artifact.job.options.dig(:artifacts, :name).to_s + 'job_name' => artifact.job.name ) ) ) diff --git a/spec/lib/gitlab/ci/pipeline/chain/create_deployments_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/create_deployments_spec.rb new file mode 100644 index 00000000000..28bc685286f --- /dev/null +++ b/spec/lib/gitlab/ci/pipeline/chain/create_deployments_spec.rb @@ -0,0 +1,97 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Pipeline::Chain::CreateDeployments do + let_it_be(:project) { create(:project, :repository) } + let_it_be(:user) { create(:user) } + + let(:stage) { build(:ci_stage_entity, project: project, statuses: [job]) } + let(:pipeline) { create(:ci_pipeline, project: project, stages: [stage]) } + + let(:command) do + Gitlab::Ci::Pipeline::Chain::Command.new(project: project, current_user: user) + end + + let(:step) { described_class.new(pipeline, command) } + + describe '#perform!' do + subject { step.perform! } + + before do + job.pipeline = pipeline + end + + context 'when a pipeline contains a deployment job' do + let!(:job) { build(:ci_build, :start_review_app, project: project) } + let!(:environment) { create(:environment, project: project, name: job.expanded_environment_name) } + + it 'creates a deployment record' do + expect { subject }.to change { Deployment.count }.by(1) + + job.reset + expect(job.deployment.project).to eq(job.project) + expect(job.deployment.ref).to eq(job.ref) + expect(job.deployment.sha).to eq(job.sha) + expect(job.deployment.deployable).to eq(job) + expect(job.deployment.deployable_type).to eq('CommitStatus') + expect(job.deployment.environment).to eq(job.persisted_environment) + end + + context 'when creation failure occures' do + before do + allow_next_instance_of(Deployment) do |deployment| + allow(deployment).to receive(:save!) { raise ActiveRecord::RecordInvalid } + end + end + + it 'trackes the exception' do + expect { subject }.to raise_error(described_class::DeploymentCreationError) + + expect(Deployment.count).to eq(0) + end + end + + context 'when the corresponding environment does not exist' do + let!(:environment) { } + + it 'does not create a deployment record' do + expect { subject }.not_to change { Deployment.count } + + expect(job.deployment).to be_nil + end + end + + context 'when create_deployment_in_separate_transaction feature flag is disabled' do + before do + stub_feature_flags(create_deployment_in_separate_transaction: false) + end + + it 'does not create a deployment record' do + expect { subject }.not_to change { Deployment.count } + + expect(job.deployment).to be_nil + end + end + end + + context 'when a pipeline contains a teardown job' do + let!(:job) { build(:ci_build, :stop_review_app, project: project) } + let!(:environment) { create(:environment, name: job.expanded_environment_name) } + + it 'does not create a deployment record' do + expect { subject }.not_to change { Deployment.count } + + expect(job.deployment).to be_nil + end + end + + context 'when a pipeline does not contain a deployment job' do + let!(:job) { build(:ci_build, project: project) } + + it 'does not create any deployments' do + expect { subject }.not_to change { Deployment.count } + end + end + end +end diff --git a/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb index d60ecc80a6e..4206483b228 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb @@ -56,4 +56,74 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Create do .to include /Failed to persist the pipeline/ end end + + context 'tags persistence' do + let(:stage) do + build(:ci_stage_entity, pipeline: pipeline) + end + + let(:job) do + build(:ci_build, stage: stage, pipeline: pipeline, project: project) + end + + let(:bridge) do + build(:ci_bridge, stage: stage, pipeline: pipeline, project: project) + end + + before do + pipeline.stages = [stage] + stage.statuses = [job, bridge] + end + + context 'without tags' do + it 'extracts an empty tag list' do + expect(CommitStatus) + .to receive(:bulk_insert_tags!) + .with(stage.statuses, {}) + .and_call_original + + step.perform! + + expect(job.instance_variable_defined?(:@tag_list)).to be_falsey + expect(job).to be_persisted + expect(job.tag_list).to eq([]) + end + end + + context 'with tags' do + before do + job.tag_list = %w[tag1 tag2] + end + + it 'bulk inserts tags' do + expect(CommitStatus) + .to receive(:bulk_insert_tags!) + .with(stage.statuses, { job.name => %w[tag1 tag2] }) + .and_call_original + + step.perform! + + expect(job.instance_variable_defined?(:@tag_list)).to be_falsey + expect(job).to be_persisted + expect(job.tag_list).to match_array(%w[tag1 tag2]) + end + end + + context 'when the feature flag is disabled' do + before do + job.tag_list = %w[tag1 tag2] + stub_feature_flags(ci_bulk_insert_tags: false) + end + + it 'follows the old code path' do + expect(CommitStatus).not_to receive(:bulk_insert_tags!) + + step.perform! + + expect(job.instance_variable_defined?(:@tag_list)).to be_truthy + expect(job).to be_persisted + expect(job.reload.tag_list).to match_array(%w[tag1 tag2]) + end + end + end end diff --git a/spec/lib/gitlab/ci/pipeline/chain/ensure_environments_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/ensure_environments_spec.rb new file mode 100644 index 00000000000..253928e1a19 --- /dev/null +++ b/spec/lib/gitlab/ci/pipeline/chain/ensure_environments_spec.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Pipeline::Chain::EnsureEnvironments do + let(:project) { create(:project) } + let(:user) { create(:user) } + let(:stage) { build(:ci_stage_entity, project: project, statuses: [job]) } + let(:pipeline) { build(:ci_pipeline, project: project, stages: [stage]) } + + let(:command) do + Gitlab::Ci::Pipeline::Chain::Command.new(project: project, current_user: user) + end + + let(:step) { described_class.new(pipeline, command) } + + describe '#perform!' do + subject { step.perform! } + + before do + job.pipeline = pipeline + end + + context 'when a pipeline contains a deployment job' do + let!(:job) { build(:ci_build, :start_review_app, project: project) } + + it 'ensures environment existence for the job' do + expect { subject }.to change { Environment.count }.by(1) + + expect(project.environments.find_by_name('review/master')).to be_present + expect(job.persisted_environment.name).to eq('review/master') + expect(job.metadata.expanded_environment_name).to eq('review/master') + end + + context 'when an environment has already been existed' do + before do + create(:environment, project: project, name: 'review/master') + end + + it 'ensures environment existence for the job' do + expect { subject }.not_to change { Environment.count } + + expect(project.environments.find_by_name('review/master')).to be_present + expect(job.persisted_environment.name).to eq('review/master') + expect(job.metadata.expanded_environment_name).to eq('review/master') + end + end + + context 'when an environment name contains an invalid character' do + let(:pipeline) { build(:ci_pipeline, ref: '!!!', project: project, stages: [stage]) } + + it 'sets the failure status' do + expect { subject }.not_to change { Environment.count } + + expect(job).to be_failed + expect(job).to be_environment_creation_failure + expect(job.persisted_environment).to be_nil + end + end + + context 'when create_deployment_in_separate_transaction feature flag is disabled' do + before do + stub_feature_flags(create_deployment_in_separate_transaction: false) + end + + it 'does not create any environments' do + expect { subject }.not_to change { Environment.count } + + expect(job.persisted_environment).to be_nil + end + end + end + + context 'when a pipeline contains a teardown job' do + let!(:job) { build(:ci_build, :stop_review_app, project: project) } + + it 'ensures environment existence for the job' do + expect { subject }.to change { Environment.count }.by(1) + + expect(project.environments.find_by_name('review/master')).to be_present + expect(job.persisted_environment.name).to eq('review/master') + expect(job.metadata.expanded_environment_name).to eq('review/master') + end + end + + context 'when a pipeline does not contain a deployment job' do + let!(:job) { build(:ci_build, project: project) } + + it 'does not create any environments' do + expect { subject }.not_to change { Environment.count } + end + end + end +end diff --git a/spec/lib/gitlab/ci/pipeline/chain/ensure_resource_groups_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/ensure_resource_groups_spec.rb new file mode 100644 index 00000000000..87df5a3e21b --- /dev/null +++ b/spec/lib/gitlab/ci/pipeline/chain/ensure_resource_groups_spec.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Pipeline::Chain::EnsureResourceGroups do + let(:project) { create(:project) } + let(:user) { create(:user) } + let(:stage) { build(:ci_stage_entity, project: project, statuses: [job]) } + let(:pipeline) { build(:ci_pipeline, project: project, stages: [stage]) } + let!(:environment) { create(:environment, name: 'production', project: project) } + + let(:command) do + Gitlab::Ci::Pipeline::Chain::Command.new(project: project, current_user: user) + end + + let(:step) { described_class.new(pipeline, command) } + + describe '#perform!' do + subject { step.perform! } + + before do + job.pipeline = pipeline + end + + context 'when a pipeline contains a job that requires a resource group' do + let!(:job) do + build(:ci_build, project: project, environment: 'production', options: { resource_group_key: '$CI_ENVIRONMENT_NAME' }) + end + + it 'ensures the resource group existence' do + expect { subject }.to change { Ci::ResourceGroup.count }.by(1) + + expect(project.resource_groups.find_by_key('production')).to be_present + expect(job.resource_group.key).to eq('production') + expect(job.options[:resource_group_key]).to be_nil + end + + context 'when a resource group has already been existed' do + before do + create(:ci_resource_group, project: project, key: 'production') + end + + it 'ensures the resource group existence' do + expect { subject }.not_to change { Ci::ResourceGroup.count } + + expect(project.resource_groups.find_by_key('production')).to be_present + expect(job.resource_group.key).to eq('production') + expect(job.options[:resource_group_key]).to be_nil + end + end + + context 'when a resource group key contains an invalid character' do + let!(:job) do + build(:ci_build, project: project, environment: '!!!', options: { resource_group_key: '$CI_ENVIRONMENT_NAME' }) + end + + it 'does not create any resource groups' do + expect { subject }.not_to change { Ci::ResourceGroup.count } + + expect(job.resource_group).to be_nil + end + end + + context 'when create_deployment_in_separate_transaction feature flag is disabled' do + before do + stub_feature_flags(create_deployment_in_separate_transaction: false) + end + + it 'does not create any resource groups' do + expect { subject }.not_to change { Ci::ResourceGroup.count } + + expect(job.resource_group).to be_nil + end + end + end + + context 'when a pipeline does not contain a job that requires a resource group' do + let!(:job) { build(:ci_build, project: project) } + + it 'does not create any resource groups' do + expect { subject }.not_to change { Ci::ResourceGroup.count } + end + end + end +end diff --git a/spec/lib/gitlab/ci/pipeline/chain/validate/external_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/validate/external_spec.rb index cf21c98dbd5..cebc4c02d11 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/validate/external_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/validate/external_spec.rb @@ -24,6 +24,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Validate::External do second_stage_job_name: stage: second_stage services: + - - postgres before_script: - echo 'first hello' @@ -142,6 +143,23 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Validate::External do perform! end + + it 'returns expected payload' do + expect(::Gitlab::HTTP).to receive(:post) do |_url, params| + payload = Gitlab::Json.parse(params[:body]) + + builds = payload['builds'] + expect(builds.count).to eq(2) + expect(builds[0]['services']).to be_nil + expect(builds[0]['stage']).to eq('first_stage') + expect(builds[0]['image']).to eq('hello_world') + expect(builds[1]['services']).to eq(['postgres']) + expect(builds[1]['stage']).to eq('second_stage') + expect(builds[1]['image']).to be_nil + end + + perform! + end end context 'when EXTERNAL_VALIDATION_SERVICE_TOKEN is set' do diff --git a/spec/lib/gitlab/ci/pipeline/expression/lexeme/variable_spec.rb b/spec/lib/gitlab/ci/pipeline/expression/lexeme/variable_spec.rb index 115674edc48..3e10ca686ba 100644 --- a/spec/lib/gitlab/ci/pipeline/expression/lexeme/variable_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/expression/lexeme/variable_spec.rb @@ -17,30 +17,33 @@ RSpec.describe Gitlab::Ci::Pipeline::Expression::Lexeme::Variable do end describe '#evaluate' do - it 'returns variable value if it is defined' do - variable = described_class.new('VARIABLE') + let(:lexeme) { described_class.new('VARIABLE') } - expect(variable.evaluate(VARIABLE: 'my variable')) + it 'returns variable value if it is defined' do + expect(lexeme.evaluate(VARIABLE: 'my variable')) .to eq 'my variable' end it 'allows to use a string as a variable key too' do - variable = described_class.new('VARIABLE') - - expect(variable.evaluate('VARIABLE' => 'my variable')) + expect(lexeme.evaluate('VARIABLE' => 'my variable')) .to eq 'my variable' end it 'returns nil if it is not defined' do - variable = described_class.new('VARIABLE') - - expect(variable.evaluate(OTHER: 'variable')).to be_nil + expect(lexeme.evaluate('OTHER' => 'variable')).to be_nil + expect(lexeme.evaluate(OTHER: 'variable')).to be_nil end it 'returns an empty string if it is empty' do - variable = described_class.new('VARIABLE') + expect(lexeme.evaluate('VARIABLE' => '')).to eq '' + expect(lexeme.evaluate(VARIABLE: '')).to eq '' + end + + it 'does not call with_indifferent_access unnecessarily' do + variables_hash = { VARIABLE: 'my variable' }.with_indifferent_access - expect(variable.evaluate(VARIABLE: '')).to eq '' + expect(variables_hash).not_to receive(:with_indifferent_access) + expect(lexeme.evaluate(variables_hash)).to eq 'my variable' end end end diff --git a/spec/lib/gitlab/ci/pipeline/expression/statement_spec.rb b/spec/lib/gitlab/ci/pipeline/expression/statement_spec.rb index ec7eebdc056..84713e2a798 100644 --- a/spec/lib/gitlab/ci/pipeline/expression/statement_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/expression/statement_spec.rb @@ -9,6 +9,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Expression::Statement do .append(key: 'PATH_VARIABLE', value: 'a/path/variable/value') .append(key: 'FULL_PATH_VARIABLE', value: '/a/full/path/variable/value') .append(key: 'EMPTY_VARIABLE', value: '') + .to_hash end subject do diff --git a/spec/lib/gitlab/ci/pipeline/logger_spec.rb b/spec/lib/gitlab/ci/pipeline/logger_spec.rb new file mode 100644 index 00000000000..0b44e35dec1 --- /dev/null +++ b/spec/lib/gitlab/ci/pipeline/logger_spec.rb @@ -0,0 +1,132 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe ::Gitlab::Ci::Pipeline::Logger do + let_it_be(:project) { build_stubbed(:project) } + let_it_be(:pipeline) { build_stubbed(:ci_pipeline, project: project) } + + subject(:logger) { described_class.new(project: project) } + + describe '#log_when' do + it 'stores blocks for later evaluation' do + logger.log_when { |obs| true } + + expect(logger.send(:log_conditions).first).to be_a(Proc) + end + end + + describe '#instrument' do + it "returns the block's value" do + expect(logger.instrument(:expensive_operation) { 123 }).to eq(123) + end + + it 'records durations of instrumented operations' do + loggable_data = { + 'expensive_operation_duration_s' => { + 'count' => 1, + 'avg' => a_kind_of(Numeric), + 'max' => a_kind_of(Numeric), + 'min' => a_kind_of(Numeric) + } + } + + logger.instrument(:expensive_operation) { 123 } + expect(logger.observations_hash).to match(a_hash_including(loggable_data)) + end + + it 'raises an error when block is not provided' do + expect { logger.instrument(:expensive_operation) } + .to raise_error(ArgumentError, 'block not given') + end + end + + describe '#observe' do + it 'records durations of observed operations' do + loggable_data = { + 'pipeline_creation_duration_s' => { + 'avg' => 30, 'count' => 1, 'max' => 30, 'min' => 30 + } + } + + expect(logger.observe(:pipeline_creation_duration_s, 30)).to be_truthy + expect(logger.observations_hash).to match(a_hash_including(loggable_data)) + end + end + + describe '#commit' do + subject(:commit) { logger.commit(pipeline: pipeline, caller: 'source') } + + before do + stub_feature_flags(ci_pipeline_creation_logger: flag) + allow(logger).to receive(:current_monotonic_time) { Time.current.to_i } + + logger.instrument(:pipeline_save) { travel(60.seconds) } + logger.observe(:pipeline_creation_duration_s, 30) + logger.observe(:pipeline_creation_duration_s, 10) + end + + context 'when the feature flag is enabled' do + let(:flag) { true } + + let(:loggable_data) do + { + 'class' => described_class.name.to_s, + 'pipeline_id' => pipeline.id, + 'pipeline_persisted' => true, + 'project_id' => project.id, + 'pipeline_creation_service_duration_s' => a_kind_of(Numeric), + 'pipeline_creation_caller' => 'source', + 'pipeline_source' => pipeline.source, + 'pipeline_save_duration_s' => { + 'avg' => 60, 'count' => 1, 'max' => 60, 'min' => 60 + }, + 'pipeline_creation_duration_s' => { + 'avg' => 20, 'count' => 2, 'max' => 30, 'min' => 10 + } + } + end + + it 'logs to application.json' do + expect(Gitlab::AppJsonLogger) + .to receive(:info) + .with(a_hash_including(loggable_data)) + .and_call_original + + expect(commit).to be_truthy + end + + context 'with log conditions' do + it 'does not log when the conditions are false' do + logger.log_when { |_obs| false } + + expect(Gitlab::AppJsonLogger).not_to receive(:info) + + expect(commit).to be_falsey + end + + it 'logs when a condition is true' do + logger.log_when { |_obs| true } + logger.log_when { |_obs| false } + + expect(Gitlab::AppJsonLogger) + .to receive(:info) + .with(a_hash_including(loggable_data)) + .and_call_original + + expect(commit).to be_truthy + end + end + end + + context 'when the feature flag is disabled' do + let(:flag) { false } + + it 'does not log' do + expect(Gitlab::AppJsonLogger).not_to receive(:info) + + expect(commit).to be_falsey + end + end + end +end diff --git a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb index e2b64e65938..68806fbf287 100644 --- a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb @@ -9,7 +9,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build do let(:pipeline) { build(:ci_empty_pipeline, project: project, sha: head_sha) } let(:root_variables) { [] } let(:seed_context) { double(pipeline: pipeline, root_variables: root_variables) } - let(:attributes) { { name: 'rspec', ref: 'master', scheduling_type: :stage } } + let(:attributes) { { name: 'rspec', ref: 'master', scheduling_type: :stage, when: 'on_success' } } let(:previous_stages) { [] } let(:current_stage) { double(seeds_names: [attributes[:name]]) } @@ -61,17 +61,35 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build do end end - context 'with job:rules but no explicit when:' do - context 'is matched' do - let(:attributes) { { name: 'rspec', ref: 'master', rules: [{ if: '$VAR == null' }] } } + context 'with job: rules but no explicit when:' do + let(:base_attributes) { { name: 'rspec', ref: 'master' } } + + context 'with a manual job' do + context 'with a matched rule' do + let(:attributes) { base_attributes.merge(when: 'manual', rules: [{ if: '$VAR == null' }]) } + + it { is_expected.to include(when: 'manual') } + end - it { is_expected.to include(when: 'on_success') } + context 'is not matched' do + let(:attributes) { base_attributes.merge(when: 'manual', rules: [{ if: '$VAR != null' }]) } + + it { is_expected.to include(when: 'never') } + end end - context 'is not matched' do - let(:attributes) { { name: 'rspec', ref: 'master', rules: [{ if: '$VAR != null' }] } } + context 'with an automatic job' do + context 'is matched' do + let(:attributes) { base_attributes.merge(when: 'on_success', rules: [{ if: '$VAR == null' }]) } - it { is_expected.to include(when: 'never') } + it { is_expected.to include(when: 'on_success') } + end + + context 'is not matched' do + let(:attributes) { base_attributes.merge(when: 'on_success', rules: [{ if: '$VAR != null' }]) } + + it { is_expected.to include(when: 'never') } + end end end @@ -393,6 +411,10 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build do describe '#to_resource' do subject { seed_build.to_resource } + before do + stub_feature_flags(create_deployment_in_separate_transaction: false) + end + context 'when job is Ci::Build' do it { is_expected.to be_a(::Ci::Build) } it { is_expected.to be_valid } @@ -443,6 +465,18 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build do it_behaves_like 'deployment job' it_behaves_like 'ensures environment existence' + context 'when create_deployment_in_separate_transaction feature flag is enabled' do + before do + stub_feature_flags(create_deployment_in_separate_transaction: true) + end + + it 'does not create any deployments nor environments' do + expect(subject.deployment).to be_nil + expect(Environment.count).to eq(0) + expect(Deployment.count).to eq(0) + end + end + context 'when the environment name is invalid' do let(:attributes) { { name: 'deploy', ref: 'master', environment: '!!!' } } @@ -452,25 +486,6 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build do expect(subject.metadata.expanded_environment_name).to be_nil expect(Environment.exists?(name: expected_environment_name)).to eq(false) end - - context 'when surface_environment_creation_failure feature flag is disabled' do - before do - stub_feature_flags(surface_environment_creation_failure: false) - end - - it_behaves_like 'non-deployment job' - it_behaves_like 'ensures environment inexistence' - - it 'tracks an exception' do - expect(Gitlab::ErrorTracking).to receive(:track_exception) - .with(an_instance_of(described_class::EnvironmentCreationFailure), - project_id: project.id, - reason: %q{Name can contain only letters, digits, '-', '_', '/', '$', '{', '}', '.', and spaces, but it cannot start or end with '/'}) - .once - - subject - end - end end end @@ -515,6 +530,18 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build do it 'returns a job with resource group' do expect(subject.resource_group).not_to be_nil expect(subject.resource_group.key).to eq('iOS') + expect(Ci::ResourceGroup.count).to eq(1) + end + + context 'when create_deployment_in_separate_transaction feature flag is enabled' do + before do + stub_feature_flags(create_deployment_in_separate_transaction: true) + end + + it 'does not create any resource groups' do + expect(subject.resource_group).to be_nil + expect(Ci::ResourceGroup.count).to eq(0) + end end context 'when resource group has $CI_ENVIRONMENT_NAME in it' do @@ -892,7 +919,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build do context 'using rules:' do using RSpec::Parameterized - let(:attributes) { { name: 'rspec', rules: rule_set } } + let(:attributes) { { name: 'rspec', rules: rule_set, when: 'on_success' } } context 'with a matching if: rule' do context 'with an explicit `when: never`' do diff --git a/spec/lib/gitlab/ci/status/bridge/common_spec.rb b/spec/lib/gitlab/ci/status/bridge/common_spec.rb index 37524afc83d..30e6ad234a0 100644 --- a/spec/lib/gitlab/ci/status/bridge/common_spec.rb +++ b/spec/lib/gitlab/ci/status/bridge/common_spec.rb @@ -29,7 +29,15 @@ RSpec.describe Gitlab::Ci::Status::Bridge::Common do end it { expect(subject).to have_details } - it { expect(subject.details_path).to include "pipelines/#{downstream_pipeline.id}" } + it { expect(subject.details_path).to include "jobs/#{bridge.id}" } + + context 'with ci_retry_downstream_pipeline ff disabled' do + before do + stub_feature_flags(ci_retry_downstream_pipeline: false) + end + + it { expect(subject.details_path).to include "pipelines/#{downstream_pipeline.id}" } + end end context 'when user does not have access to read downstream pipeline' do diff --git a/spec/lib/gitlab/ci/tags/bulk_insert_spec.rb b/spec/lib/gitlab/ci/tags/bulk_insert_spec.rb new file mode 100644 index 00000000000..6c1f56de840 --- /dev/null +++ b/spec/lib/gitlab/ci/tags/bulk_insert_spec.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Tags::BulkInsert do + let_it_be(:project) { create(:project, :repository) } + let_it_be(:pipeline) { create(:ci_pipeline, project: project) } + let_it_be_with_refind(:job) { create(:ci_build, :unique_name, pipeline: pipeline, project: project) } + let_it_be_with_refind(:other_job) { create(:ci_build, :unique_name, pipeline: pipeline, project: project) } + let_it_be_with_refind(:bridge) { create(:ci_bridge, pipeline: pipeline, project: project) } + + let(:statuses) { [job, bridge, other_job] } + + subject(:service) { described_class.new(statuses, tags_list) } + + describe '#insert!' do + context 'without tags' do + let(:tags_list) { {} } + + it { expect(service.insert!).to be_falsey } + end + + context 'with tags' do + let(:tags_list) do + { + job.name => %w[tag1 tag2], + other_job.name => %w[tag2 tag3 tag4] + } + end + + it 'persists tags' do + expect(service.insert!).to be_truthy + + expect(job.reload.tag_list).to match_array(%w[tag1 tag2]) + expect(other_job.reload.tag_list).to match_array(%w[tag2 tag3 tag4]) + end + end + end +end diff --git a/spec/lib/gitlab/ci/variables/builder_spec.rb b/spec/lib/gitlab/ci/variables/builder_spec.rb index 10275f33484..5ff34592b2f 100644 --- a/spec/lib/gitlab/ci/variables/builder_spec.rb +++ b/spec/lib/gitlab/ci/variables/builder_spec.rb @@ -24,15 +24,5 @@ RSpec.describe Gitlab::Ci::Variables::Builder do expect(names).to include(*keys) end end - - context 'feature flag disabled' do - before do - stub_feature_flags(ci_predefined_vars_in_builder: false) - end - - it 'returns no variables' do - expect(subject.map { |env| env[:key] }).to be_empty - end - end end end diff --git a/spec/lib/gitlab/ci/yaml_processor_spec.rb b/spec/lib/gitlab/ci/yaml_processor_spec.rb index f00a801286d..e8b38b21ef8 100644 --- a/spec/lib/gitlab/ci/yaml_processor_spec.rb +++ b/spec/lib/gitlab/ci/yaml_processor_spec.rb @@ -2139,7 +2139,7 @@ module Gitlab end end - context 'with when/rules conflict' do + context 'with when/rules' do subject { Gitlab::Ci::YamlProcessor.new(YAML.dump(config)).execute } let(:config) do @@ -2174,7 +2174,7 @@ module Gitlab } end - it_behaves_like 'returns errors', /may not be used with `rules`: when/ + it { is_expected.to be_valid } end context 'used with job-level when:delayed' do @@ -2190,7 +2190,7 @@ module Gitlab } end - it_behaves_like 'returns errors', /may not be used with `rules`: when, start_in/ + it_behaves_like 'returns errors', /may not be used with `rules`: start_in/ end end diff --git a/spec/lib/gitlab/cleanup/orphan_lfs_file_references_spec.rb b/spec/lib/gitlab/cleanup/orphan_lfs_file_references_spec.rb index b0f7703462a..f5f02046d4e 100644 --- a/spec/lib/gitlab/cleanup/orphan_lfs_file_references_spec.rb +++ b/spec/lib/gitlab/cleanup/orphan_lfs_file_references_spec.rb @@ -97,6 +97,6 @@ RSpec.describe Gitlab::Cleanup::OrphanLfsFileReferences do def stub_lfs_pointers(repo, *oids) expect(repo.gitaly_blob_client) .to receive(:get_all_lfs_pointers) - .and_return(oids.map { |oid| OpenStruct.new(lfs_oid: oid) }) + .and_return(oids.map { |oid| double('pointers', lfs_oid: oid) }) end end diff --git a/spec/lib/gitlab/config/entry/undefined_spec.rb b/spec/lib/gitlab/config/entry/undefined_spec.rb index 36faabd8e31..31e0f9487aa 100644 --- a/spec/lib/gitlab/config/entry/undefined_spec.rb +++ b/spec/lib/gitlab/config/entry/undefined_spec.rb @@ -40,4 +40,10 @@ RSpec.describe Gitlab::Config::Entry::Undefined do expect(entry.specified?).to eq false end end + + describe '#type' do + it 'returns nil' do + expect(entry.type).to eq nil + end + end end diff --git a/spec/lib/gitlab/content_security_policy/config_loader_spec.rb b/spec/lib/gitlab/content_security_policy/config_loader_spec.rb index c0476d38380..56e3fc269e6 100644 --- a/spec/lib/gitlab/content_security_policy/config_loader_spec.rb +++ b/spec/lib/gitlab/content_security_policy/config_loader_spec.rb @@ -85,7 +85,7 @@ RSpec.describe Gitlab::ContentSecurityPolicy::ConfigLoader do expect(directives['style_src']).to eq("'self' 'unsafe-inline' https://cdn.example.com") expect(directives['font_src']).to eq("'self' https://cdn.example.com") expect(directives['worker_src']).to eq('http://localhost/assets/ blob: data: https://cdn.example.com') - expect(directives['frame_src']).to eq(::Gitlab::ContentSecurityPolicy::Directives.frame_src + " https://cdn.example.com http://localhost/admin/sidekiq http://localhost/admin/sidekiq/ http://localhost/-/speedscope/index.html") + expect(directives['frame_src']).to eq(::Gitlab::ContentSecurityPolicy::Directives.frame_src + " https://cdn.example.com http://localhost/admin/ http://localhost/assets/ http://localhost/-/speedscope/index.html") end end @@ -113,7 +113,7 @@ RSpec.describe Gitlab::ContentSecurityPolicy::ConfigLoader do end it 'does not add CUSTOMER_PORTAL_URL to CSP' do - expect(directives['frame_src']).to eq(::Gitlab::ContentSecurityPolicy::Directives.frame_src + " http://localhost/admin/sidekiq http://localhost/admin/sidekiq/ http://localhost/-/speedscope/index.html") + expect(directives['frame_src']).to eq(::Gitlab::ContentSecurityPolicy::Directives.frame_src + " http://localhost/admin/ http://localhost/assets/ http://localhost/-/speedscope/index.html") end end @@ -123,12 +123,12 @@ RSpec.describe Gitlab::ContentSecurityPolicy::ConfigLoader do end it 'adds CUSTOMER_PORTAL_URL to CSP' do - expect(directives['frame_src']).to eq(::Gitlab::ContentSecurityPolicy::Directives.frame_src + " http://localhost/rails/letter_opener/ https://customers.example.com http://localhost/admin/sidekiq http://localhost/admin/sidekiq/ http://localhost/-/speedscope/index.html") + expect(directives['frame_src']).to eq(::Gitlab::ContentSecurityPolicy::Directives.frame_src + " http://localhost/rails/letter_opener/ https://customers.example.com http://localhost/admin/ http://localhost/assets/ http://localhost/-/speedscope/index.html") end end end - context 'letter_opener applicaiton URL' do + context 'letter_opener application URL' do let(:gitlab_url) { 'http://gitlab.example.com' } let(:letter_opener_url) { "#{gitlab_url}/rails/letter_opener/" } @@ -156,6 +156,46 @@ RSpec.describe Gitlab::ContentSecurityPolicy::ConfigLoader do end end end + + context 'Snowplow Micro event collector' do + let(:snowplow_micro_hostname) { 'localhost:9090' } + let(:snowplow_micro_url) { "http://#{snowplow_micro_hostname}/" } + + before do + stub_env('SNOWPLOW_MICRO_ENABLE', 1) + allow(Gitlab::Tracking).to receive(:collector_hostname).and_return(snowplow_micro_hostname) + end + + context 'when in production' do + before do + stub_rails_env('production') + end + + it 'does not add Snowplow Micro URL to connect-src' do + expect(directives['connect_src']).not_to include(snowplow_micro_url) + end + end + + context 'when in development' do + before do + stub_rails_env('development') + end + + it 'adds Snowplow Micro URL with trailing slash to connect-src' do + expect(directives['connect_src']).to match(Regexp.new(snowplow_micro_url)) + end + + context 'when not enabled using ENV[SNOWPLOW_MICRO_ENABLE]' do + before do + stub_env('SNOWPLOW_MICRO_ENABLE', nil) + end + + it 'does not add Snowplow Micro URL to connect-src' do + expect(directives['connect_src']).not_to include(snowplow_micro_url) + end + end + end + end end describe '#load' do diff --git a/spec/lib/gitlab/contributions_calendar_spec.rb b/spec/lib/gitlab/contributions_calendar_spec.rb index 384609c6664..8a9ab736d46 100644 --- a/spec/lib/gitlab/contributions_calendar_spec.rb +++ b/spec/lib/gitlab/contributions_calendar_spec.rb @@ -50,7 +50,8 @@ RSpec.describe Gitlab::ContributionsCalendar do Event.create!( project: project, action: action, - target: @targets[project], + target_type: @targets[project].class.name, + target_id: @targets[project].id, author: contributor, created_at: DateTime.new(day.year, day.month, day.day, hour) ) @@ -66,14 +67,34 @@ RSpec.describe Gitlab::ContributionsCalendar do end context "when the user has opted-in for private contributions" do + before do + contributor.update_column(:include_private_contributions, true) + end + it "shows private and public events to all users" do - user.update_column(:include_private_contributions, true) create_event(private_project, today) create_event(public_project, today) + expect(calendar.activity_dates[today]).to eq(2) + expect(calendar(user).activity_dates[today]).to eq(2) + expect(calendar(contributor).activity_dates[today]).to eq(2) + end + + # tests for bug https://gitlab.com/gitlab-org/gitlab/-/merge_requests/74826 + it "still counts correct with feature access levels set to private" do + create_event(private_project, today) + + private_project.project_feature.update_attribute(:issues_access_level, ProjectFeature::PRIVATE) + private_project.project_feature.update_attribute(:repository_access_level, ProjectFeature::PRIVATE) + private_project.project_feature.update_attribute(:merge_requests_access_level, ProjectFeature::PRIVATE) + expect(calendar.activity_dates[today]).to eq(1) expect(calendar(user).activity_dates[today]).to eq(1) - expect(calendar(contributor).activity_dates[today]).to eq(2) + expect(calendar(contributor).activity_dates[today]).to eq(1) + end + + it "does not fail if there are no contributed projects" do + expect(calendar.activity_dates[today]).to eq(nil) end end @@ -125,6 +146,7 @@ RSpec.describe Gitlab::ContributionsCalendar do create_event(public_project, today, 10) create_event(public_project, today, 16) create_event(public_project, today, 23) + create_event(public_project, tomorrow, 1) end it "renders correct event counts within the UTC timezone" do @@ -137,14 +159,14 @@ RSpec.describe Gitlab::ContributionsCalendar do it "renders correct event counts within the Sydney timezone" do Time.use_zone('UTC') do contributor.timezone = 'Sydney' - expect(calendar.activity_dates).to eq(today => 3, tomorrow => 2) + expect(calendar.activity_dates).to eq(today => 3, tomorrow => 3) end end it "renders correct event counts within the US Central timezone" do Time.use_zone('UTC') do contributor.timezone = 'Central Time (US & Canada)' - expect(calendar.activity_dates).to eq(yesterday => 2, today => 3) + expect(calendar.activity_dates).to eq(yesterday => 2, today => 4) end end end @@ -169,6 +191,12 @@ RSpec.describe Gitlab::ContributionsCalendar do expect(calendar(contributor).events_by_date(today)).to contain_exactly(e1, e2, e3) end + it "includes diff notes on merge request" do + e1 = create_event(public_project, today, 0, :commented, :diff_note_on_merge_request) + + expect(calendar.events_by_date(today)).to contain_exactly(e1) + end + context 'when the user cannot read cross project' do before do allow(Ability).to receive(:allowed?).and_call_original diff --git a/spec/lib/gitlab/daemon_spec.rb b/spec/lib/gitlab/daemon_spec.rb index 075a1e414c7..4d11b0bdc6c 100644 --- a/spec/lib/gitlab/daemon_spec.rb +++ b/spec/lib/gitlab/daemon_spec.rb @@ -46,6 +46,30 @@ RSpec.describe Gitlab::Daemon do expect(subject).to have_received(:run_thread) end + + context '@synchronous' do + context 'when @synchronous is set to true' do + subject { described_class.instance(synchronous: true) } + + it 'calls join on the thread' do + # Thread has to be run in a block, expect_next_instance_of does not support this. + expect_any_instance_of(Thread).to receive(:join) # rubocop:disable RSpec/AnyInstanceOf + + subject.start + end + end + + context 'when @synchronous is not set to a truthy value' do + subject { described_class.instance } + + it 'does not call join on the thread' do + # Thread has to be run in a block, expect_next_instance_of does not support this. + expect_any_instance_of(Thread).not_to receive(:join) # rubocop:disable RSpec/AnyInstanceOf + + subject.start + end + end + end end describe '#stop' do diff --git a/spec/lib/gitlab/database/async_indexes/index_creator_spec.rb b/spec/lib/gitlab/database/async_indexes/index_creator_spec.rb index b4010d0fe8d..7ad3eb395a9 100644 --- a/spec/lib/gitlab/database/async_indexes/index_creator_spec.rb +++ b/spec/lib/gitlab/database/async_indexes/index_creator_spec.rb @@ -3,6 +3,8 @@ require 'spec_helper' RSpec.describe Gitlab::Database::AsyncIndexes::IndexCreator do + include ExclusiveLeaseHelpers + describe '#perform' do subject { described_class.new(async_index) } @@ -10,7 +12,18 @@ RSpec.describe Gitlab::Database::AsyncIndexes::IndexCreator do let(:index_model) { Gitlab::Database::AsyncIndexes::PostgresAsyncIndex } - let(:connection) { ApplicationRecord.connection } + let(:model) { Gitlab::Database.database_base_models[Gitlab::Database::PRIMARY_DATABASE_NAME] } + let(:connection) { model.connection } + + let!(:lease) { stub_exclusive_lease(lease_key, :uuid, timeout: lease_timeout) } + let(:lease_key) { "gitlab/database/async_indexes/index_creator/#{Gitlab::Database::PRIMARY_DATABASE_NAME}" } + let(:lease_timeout) { described_class::TIMEOUT_PER_ACTION } + + around do |example| + Gitlab::Database::SharedModel.using_connection(connection) do + example.run + end + end context 'when the index already exists' do before do @@ -40,7 +53,7 @@ RSpec.describe Gitlab::Database::AsyncIndexes::IndexCreator do end it 'skips logic if not able to acquire exclusive lease' do - expect(subject).to receive(:try_obtain_lease).and_return(false) + expect(lease).to receive(:try_obtain).ordered.and_return(false) expect(connection).not_to receive(:execute).with(/CREATE INDEX/) expect(async_index).not_to receive(:destroy) diff --git a/spec/lib/gitlab/database/background_migration/batched_job_spec.rb b/spec/lib/gitlab/database/background_migration/batched_job_spec.rb index 0182e0f7651..c4364826ee2 100644 --- a/spec/lib/gitlab/database/background_migration/batched_job_spec.rb +++ b/spec/lib/gitlab/database/background_migration/batched_job_spec.rb @@ -17,15 +17,19 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedJob, type: :model d let_it_be(:stuck_job) { create(:batched_background_migration_job, status: :pending, updated_at: fixed_time - described_class::STUCK_JOBS_TIMEOUT) } let_it_be(:failed_job) { create(:batched_background_migration_job, status: :failed, attempts: 1) } - before_all do - create(:batched_background_migration_job, status: :failed, attempts: described_class::MAX_ATTEMPTS) - create(:batched_background_migration_job, status: :succeeded) - end + let!(:max_attempts_failed_job) { create(:batched_background_migration_job, status: :failed, attempts: described_class::MAX_ATTEMPTS) } + let!(:succeeded_job) { create(:batched_background_migration_job, status: :succeeded) } before do travel_to fixed_time end + describe '.except_succeeded' do + it 'returns not succeeded jobs' do + expect(described_class.except_succeeded).to contain_exactly(pending_job, running_job, stuck_job, failed_job, max_attempts_failed_job) + end + end + describe '.active' do it 'returns active jobs' do expect(described_class.active).to contain_exactly(pending_job, running_job, stuck_job) diff --git a/spec/lib/gitlab/database/background_migration/batched_migration_spec.rb b/spec/lib/gitlab/database/background_migration/batched_migration_spec.rb index a1c2634f59c..49714cfc4dd 100644 --- a/spec/lib/gitlab/database/background_migration/batched_migration_spec.rb +++ b/spec/lib/gitlab/database/background_migration/batched_migration_spec.rb @@ -23,6 +23,28 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigration, type: :m subject { build(:batched_background_migration) } it { is_expected.to validate_uniqueness_of(:job_arguments).scoped_to(:job_class_name, :table_name, :column_name) } + + context 'when there are failed jobs' do + let(:batched_migration) { create(:batched_background_migration, status: :active, total_tuple_count: 100) } + let!(:batched_job) { create(:batched_background_migration_job, batched_migration: batched_migration, status: :failed) } + + it 'raises an exception' do + expect { batched_migration.finished! }.to raise_error(ActiveRecord::RecordInvalid) + + expect(batched_migration.reload.status).to eql 'active' + end + end + + context 'when the jobs are completed' do + let(:batched_migration) { create(:batched_background_migration, status: :active, total_tuple_count: 100) } + let!(:batched_job) { create(:batched_background_migration_job, batched_migration: batched_migration, status: :succeeded) } + + it 'finishes the migration' do + batched_migration.finished! + + expect(batched_migration.status).to eql 'finished' + end + end end describe '.queue_order' do @@ -214,14 +236,20 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigration, type: :m end end - shared_examples_for 'an attr_writer that demodulizes assigned class names' do |attribute_name| + shared_examples_for 'an attr_writer that assigns class names' do |attribute_name| let(:batched_migration) { build(:batched_background_migration) } context 'when a module name exists' do - it 'removes the module name' do + it 'keeps the class with module name' do + batched_migration.public_send(:"#{attribute_name}=", 'Foo::Bar') + + expect(batched_migration[attribute_name]).to eq('Foo::Bar') + end + + it 'removes leading namespace resolution operator' do batched_migration.public_send(:"#{attribute_name}=", '::Foo::Bar') - expect(batched_migration[attribute_name]).to eq('Bar') + expect(batched_migration[attribute_name]).to eq('Foo::Bar') end end @@ -271,11 +299,11 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigration, type: :m end describe '#job_class_name=' do - it_behaves_like 'an attr_writer that demodulizes assigned class names', :job_class_name + it_behaves_like 'an attr_writer that assigns class names', :job_class_name end describe '#batch_class_name=' do - it_behaves_like 'an attr_writer that demodulizes assigned class names', :batch_class_name + it_behaves_like 'an attr_writer that assigns class names', :batch_class_name end describe '#migrated_tuple_count' do diff --git a/spec/lib/gitlab/database/count/reltuples_count_strategy_spec.rb b/spec/lib/gitlab/database/count/reltuples_count_strategy_spec.rb index 9d49db1f018..e7b9c5fcd02 100644 --- a/spec/lib/gitlab/database/count/reltuples_count_strategy_spec.rb +++ b/spec/lib/gitlab/database/count/reltuples_count_strategy_spec.rb @@ -5,24 +5,24 @@ require 'spec_helper' RSpec.describe Gitlab::Database::Count::ReltuplesCountStrategy do before do create_list(:project, 3) - create(:identity) + create_list(:ci_instance_variable, 2) end subject { described_class.new(models).count } describe '#count' do - let(:models) { [Project, Identity] } + let(:models) { [Project, Ci::InstanceVariable] } context 'when reltuples is up to date' do before do - ActiveRecord::Base.connection.execute('ANALYZE projects') - ActiveRecord::Base.connection.execute('ANALYZE identities') + Project.connection.execute('ANALYZE projects') + Ci::InstanceVariable.connection.execute('ANALYZE ci_instance_variables') end it 'uses statistics to do the count' do models.each { |model| expect(model).not_to receive(:count) } - expect(subject).to eq({ Project => 3, Identity => 1 }) + expect(subject).to eq({ Project => 3, Ci::InstanceVariable => 2 }) end end @@ -31,7 +31,7 @@ RSpec.describe Gitlab::Database::Count::ReltuplesCountStrategy do before do models.each do |model| - ActiveRecord::Base.connection.execute("ANALYZE #{model.table_name}") + model.connection.execute("ANALYZE #{model.table_name}") end end @@ -45,7 +45,9 @@ RSpec.describe Gitlab::Database::Count::ReltuplesCountStrategy do context 'insufficient permissions' do it 'returns an empty hash' do - allow(ActiveRecord::Base).to receive(:transaction).and_raise(PG::InsufficientPrivilege) + Gitlab::Database.database_base_models.each_value do |base_model| + allow(base_model).to receive(:transaction).and_raise(PG::InsufficientPrivilege) + end expect(subject).to eq({}) end diff --git a/spec/lib/gitlab/database/count/tablesample_count_strategy_spec.rb b/spec/lib/gitlab/database/count/tablesample_count_strategy_spec.rb index 2f261aebf02..37d3e13a7ab 100644 --- a/spec/lib/gitlab/database/count/tablesample_count_strategy_spec.rb +++ b/spec/lib/gitlab/database/count/tablesample_count_strategy_spec.rb @@ -5,11 +5,12 @@ require 'spec_helper' RSpec.describe Gitlab::Database::Count::TablesampleCountStrategy do before do create_list(:project, 3) + create_list(:ci_instance_variable, 2) create(:identity) create(:group) end - let(:models) { [Project, Identity, Group, Namespace] } + let(:models) { [Project, Ci::InstanceVariable, Identity, Group, Namespace] } let(:strategy) { described_class.new(models) } subject { strategy.count } @@ -20,7 +21,8 @@ RSpec.describe Gitlab::Database::Count::TablesampleCountStrategy do Project => threshold + 1, Identity => threshold - 1, Group => threshold + 1, - Namespace => threshold + 1 + Namespace => threshold + 1, + Ci::InstanceVariable => threshold + 1 } end @@ -43,12 +45,14 @@ RSpec.describe Gitlab::Database::Count::TablesampleCountStrategy do expect(Project).not_to receive(:count) expect(Group).not_to receive(:count) expect(Namespace).not_to receive(:count) + expect(Ci::InstanceVariable).not_to receive(:count) result = subject expect(result[Project]).to eq(3) expect(result[Group]).to eq(1) # 1-Group, 3 namespaces for each project and 3 project namespaces for each project expect(result[Namespace]).to eq(7) + expect(result[Ci::InstanceVariable]).to eq(2) end end diff --git a/spec/lib/gitlab/database/load_balancing/configuration_spec.rb b/spec/lib/gitlab/database/load_balancing/configuration_spec.rb index eef248afdf2..796c14c1038 100644 --- a/spec/lib/gitlab/database/load_balancing/configuration_spec.rb +++ b/spec/lib/gitlab/database/load_balancing/configuration_spec.rb @@ -140,6 +140,15 @@ RSpec.describe Gitlab::Database::LoadBalancing::Configuration do end describe '#service_discovery_enabled?' do + it 'returns false when running inside a Rake task' do + allow(Gitlab::Runtime).to receive(:rake?).and_return(true) + + config = described_class.new(ActiveRecord::Base) + config.service_discovery[:record] = 'foo' + + expect(config.service_discovery_enabled?).to eq(false) + end + it 'returns true when a record is configured' do config = described_class.new(ActiveRecord::Base) config.service_discovery[:record] = 'foo' diff --git a/spec/lib/gitlab/database/load_balancing/load_balancer_spec.rb b/spec/lib/gitlab/database/load_balancing/load_balancer_spec.rb index 37b83729125..3c7819c04b6 100644 --- a/spec/lib/gitlab/database/load_balancing/load_balancer_spec.rb +++ b/spec/lib/gitlab/database/load_balancing/load_balancer_spec.rb @@ -487,25 +487,9 @@ RSpec.describe Gitlab::Database::LoadBalancing::LoadBalancer, :request_store do end end - describe 'primary connection re-use', :reestablished_active_record_base do + describe 'primary connection re-use', :reestablished_active_record_base, :add_ci_connection do let(:model) { Ci::ApplicationRecord } - around do |example| - if Gitlab::Database.has_config?(:ci) - example.run - else - # fake additional Database - model.establish_connection( - ActiveRecord::DatabaseConfigurations::HashConfig.new(Rails.env, 'ci', ActiveRecord::Base.connection_db_config.configuration_hash) - ) - - example.run - - # Cleanup connection_specification_name for Ci::ApplicationRecord - model.remove_connection - end - end - describe '#read' do it 'returns ci replica connection' do expect { |b| lb.read(&b) }.to yield_with_args do |args| diff --git a/spec/lib/gitlab/database/load_balancing/service_discovery_spec.rb b/spec/lib/gitlab/database/load_balancing/service_discovery_spec.rb index e9bc465b1c7..f05910e5123 100644 --- a/spec/lib/gitlab/database/load_balancing/service_discovery_spec.rb +++ b/spec/lib/gitlab/database/load_balancing/service_discovery_spec.rb @@ -4,9 +4,10 @@ require 'spec_helper' RSpec.describe Gitlab::Database::LoadBalancing::ServiceDiscovery do let(:load_balancer) do - Gitlab::Database::LoadBalancing::LoadBalancer.new( - Gitlab::Database::LoadBalancing::Configuration.new(ActiveRecord::Base) - ) + configuration = Gitlab::Database::LoadBalancing::Configuration.new(ActiveRecord::Base) + configuration.service_discovery[:record] = 'localhost' + + Gitlab::Database::LoadBalancing::LoadBalancer.new(configuration) end let(:service) do @@ -86,6 +87,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::ServiceDiscovery do service.perform_service_discovery end end + context 'with failures' do before do allow(Gitlab::ErrorTracking).to receive(:track_exception) diff --git a/spec/lib/gitlab/database/load_balancing/sidekiq_server_middleware_spec.rb b/spec/lib/gitlab/database/load_balancing/sidekiq_server_middleware_spec.rb index de2ad662d16..31be3963565 100644 --- a/spec/lib/gitlab/database/load_balancing/sidekiq_server_middleware_spec.rb +++ b/spec/lib/gitlab/database/load_balancing/sidekiq_server_middleware_spec.rb @@ -5,7 +5,9 @@ require 'spec_helper' RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware, :clean_gitlab_redis_queues do let(:middleware) { described_class.new } let(:worker) { worker_class.new } - let(:job) { { "retry" => 3, "job_id" => "a180b47c-3fd6-41b8-81e9-34da61c3400e", 'database_replica_location' => '0/D525E3A8' } } + let(:location) {'0/D525E3A8' } + let(:wal_locations) { { Gitlab::Database::MAIN_DATABASE_NAME.to_sym => location } } + let(:job) { { "retry" => 3, "job_id" => "a180b47c-3fd6-41b8-81e9-34da61c3400e", 'wal_locations' => wal_locations } } before do skip_feature_flags_yaml_validation @@ -60,9 +62,6 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware, :clean_ end shared_examples_for 'replica is up to date' do |expected_strategy| - let(:location) {'0/D525E3A8' } - let(:wal_locations) { { Gitlab::Database::MAIN_DATABASE_NAME.to_sym => location } } - it 'does not stick to the primary', :aggregate_failures do expect(ActiveRecord::Base.load_balancer) .to receive(:select_up_to_date_host) @@ -77,9 +76,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware, :clean_ include_examples 'load balancing strategy', expected_strategy end - shared_examples_for 'sticks based on data consistency' do |data_consistency| - include_context 'data consistency worker class', data_consistency, :load_balancing_for_test_data_consistency_worker - + shared_examples_for 'sticks based on data consistency' do context 'when load_balancing_for_test_data_consistency_worker is disabled' do before do stub_feature_flags(load_balancing_for_test_data_consistency_worker: false) @@ -116,23 +113,78 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware, :clean_ it_behaves_like 'replica is up to date', 'replica' end - context 'when legacy wal location is set' do - let(:job) { { 'job_id' => 'a180b47c-3fd6-41b8-81e9-34da61c3400e', 'database_write_location' => '0/D525E3A8' } } + context 'when database location is not set' do + let(:job) { { 'job_id' => 'a180b47c-3fd6-41b8-81e9-34da61c3400e' } } - before do - allow(ActiveRecord::Base.load_balancer) - .to receive(:select_up_to_date_host) - .with('0/D525E3A8') - .and_return(true) - end + include_examples 'stick to the primary', 'primary_no_wal' + end + end - it_behaves_like 'replica is up to date', 'replica' + shared_examples_for 'sleeps when necessary' do + context 'when WAL locations are blank', :freeze_time do + let(:job) { { "retry" => 3, "job_id" => "a180b47c-3fd6-41b8-81e9-34da61c3400e", "wal_locations" => {}, "created_at" => Time.current.to_f - (described_class::MINIMUM_DELAY_INTERVAL_SECONDS - 0.3) } } + + it 'does not sleep' do + expect(middleware).not_to receive(:sleep) + + run_middleware + end end - context 'when database location is not set' do - let(:job) { { 'job_id' => 'a180b47c-3fd6-41b8-81e9-34da61c3400e' } } + context 'when WAL locations are present', :freeze_time do + let(:job) { { "retry" => 3, "job_id" => "a180b47c-3fd6-41b8-81e9-34da61c3400e", 'wal_locations' => wal_locations, "created_at" => Time.current.to_f - elapsed_time } } - include_examples 'stick to the primary', 'primary_no_wal' + context 'when delay interval has not elapsed' do + let(:elapsed_time) { described_class::MINIMUM_DELAY_INTERVAL_SECONDS - 0.3 } + + context 'when replica is up to date' do + before do + Gitlab::Database::LoadBalancing.each_load_balancer do |lb| + allow(lb).to receive(:select_up_to_date_host).and_return(true) + end + end + + it 'does not sleep' do + expect(middleware).not_to receive(:sleep) + + run_middleware + end + end + + context 'when replica is not up to date' do + before do + Gitlab::Database::LoadBalancing.each_load_balancer do |lb| + allow(lb).to receive(:select_up_to_date_host).and_return(false, true) + end + end + + it 'sleeps until the minimum delay is reached' do + expect(middleware).to receive(:sleep).with(be_within(0.01).of(described_class::MINIMUM_DELAY_INTERVAL_SECONDS - elapsed_time)) + + run_middleware + end + end + end + + context 'when delay interval has elapsed' do + let(:elapsed_time) { described_class::MINIMUM_DELAY_INTERVAL_SECONDS + 0.3 } + + it 'does not sleep' do + expect(middleware).not_to receive(:sleep) + + run_middleware + end + end + + context 'when created_at is in the future' do + let(:elapsed_time) { -5 } + + it 'does not sleep' do + expect(middleware).not_to receive(:sleep) + + run_middleware + end + end end end @@ -146,10 +198,24 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware, :clean_ include_context 'data consistency worker class', :always, :load_balancing_for_test_data_consistency_worker include_examples 'stick to the primary', 'primary' + + context 'when delay interval has not elapsed', :freeze_time do + let(:job) { { "retry" => 3, "job_id" => "a180b47c-3fd6-41b8-81e9-34da61c3400e", 'wal_locations' => wal_locations, "created_at" => Time.current.to_f - elapsed_time } } + let(:elapsed_time) { described_class::MINIMUM_DELAY_INTERVAL_SECONDS - 0.3 } + + it 'does not sleep' do + expect(middleware).not_to receive(:sleep) + + run_middleware + end + end end context 'when worker data consistency is :delayed' do - include_examples 'sticks based on data consistency', :delayed + include_context 'data consistency worker class', :delayed, :load_balancing_for_test_data_consistency_worker + + include_examples 'sticks based on data consistency' + include_examples 'sleeps when necessary' context 'when replica is not up to date' do before do @@ -177,7 +243,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware, :clean_ end context 'when job is retried' do - let(:job) { { "retry" => 3, "job_id" => "a180b47c-3fd6-41b8-81e9-34da61c3400e", 'database_replica_location' => '0/D525E3A8', 'retry_count' => 0 } } + let(:job) { { "retry" => 3, "job_id" => "a180b47c-3fd6-41b8-81e9-34da61c3400e", 'wal_locations' => wal_locations, 'retry_count' => 0 } } context 'and replica still lagging behind' do include_examples 'stick to the primary', 'primary' @@ -195,7 +261,10 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware, :clean_ end context 'when worker data consistency is :sticky' do - include_examples 'sticks based on data consistency', :sticky + include_context 'data consistency worker class', :sticky, :load_balancing_for_test_data_consistency_worker + + include_examples 'sticks based on data consistency' + include_examples 'sleeps when necessary' context 'when replica is not up to date' do before do @@ -255,7 +324,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware, :clean_ end def run_middleware - middleware.call(worker, job, double(:queue)) { yield } + middleware.call(worker, job, double(:queue)) { yield if block_given? } rescue described_class::JobReplicaNotUpToDate # we silence errors here that cause the job to retry end diff --git a/spec/lib/gitlab/database/load_balancing/sticking_spec.rb b/spec/lib/gitlab/database/load_balancing/sticking_spec.rb index d88554614cf..f3139bb1b4f 100644 --- a/spec/lib/gitlab/database/load_balancing/sticking_spec.rb +++ b/spec/lib/gitlab/database/load_balancing/sticking_spec.rb @@ -256,15 +256,6 @@ RSpec.describe Gitlab::Database::LoadBalancing::Sticking, :redis do expect(sticking.last_write_location_for(:user, 4)).to be_nil end - - it 'removes the old key' do - Gitlab::Redis::SharedState.with do |redis| - redis.set(sticking.send(:old_redis_key_for, :user, 4), 'foo', ex: 30) - end - - sticking.unstick(:user, 4) - expect(sticking.last_write_location_for(:user, 4)).to be_nil - end end describe '#last_write_location_for' do @@ -273,14 +264,6 @@ RSpec.describe Gitlab::Database::LoadBalancing::Sticking, :redis do expect(sticking.last_write_location_for(:user, 4)).to eq('foo') end - - it 'falls back to reading the old key' do - Gitlab::Redis::SharedState.with do |redis| - redis.set(sticking.send(:old_redis_key_for, :user, 4), 'foo', ex: 30) - end - - expect(sticking.last_write_location_for(:user, 4)).to eq('foo') - end end describe '#redis_key_for' do diff --git a/spec/lib/gitlab/database/load_balancing_spec.rb b/spec/lib/gitlab/database/load_balancing_spec.rb index 65ffe539910..45878b2e266 100644 --- a/spec/lib/gitlab/database/load_balancing_spec.rb +++ b/spec/lib/gitlab/database/load_balancing_spec.rb @@ -38,6 +38,24 @@ RSpec.describe Gitlab::Database::LoadBalancing do end end + describe '.primary_only?' do + it 'returns true if all load balancers have no replicas' do + described_class.each_load_balancer do |lb| + allow(lb).to receive(:primary_only?).and_return(true) + end + + expect(described_class.primary_only?).to eq(true) + end + + it 'returns false if at least one has replicas' do + described_class.each_load_balancer.with_index do |lb, index| + allow(lb).to receive(:primary_only?).and_return(index != 0) + end + + expect(described_class.primary_only?).to eq(false) + end + end + describe '.release_hosts' do it 'releases the host of every load balancer' do described_class.each_load_balancer do |lb| diff --git a/spec/lib/gitlab/database/loose_foreign_keys_spec.rb b/spec/lib/gitlab/database/loose_foreign_keys_spec.rb new file mode 100644 index 00000000000..13f2d31bc32 --- /dev/null +++ b/spec/lib/gitlab/database/loose_foreign_keys_spec.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::LooseForeignKeys do + describe 'verify all definitions' do + subject(:definitions) { described_class.definitions } + + it 'all definitions have assigned a known gitlab_schema and on_delete' do + is_expected.to all(have_attributes( + options: a_hash_including( + column: be_a(String), + gitlab_schema: be_in(Gitlab::Database.schemas_to_base_models.symbolize_keys.keys), + on_delete: be_in([:async_delete, :async_nullify]) + ), + from_table: be_a(String), + to_table: be_a(String) + )) + end + + describe 'ensuring database integrity' do + def base_models_for(table) + parent_table_schema = Gitlab::Database::GitlabSchema.table_schema(table) + Gitlab::Database.schemas_to_base_models.fetch(parent_table_schema) + end + + it 'all `to_table` tables are present' do + definitions.each do |definition| + base_models_for(definition.to_table).each do |model| + expect(model.connection).to be_table_exist(definition.to_table) + end + end + end + + it 'all `from_table` tables are present' do + definitions.each do |definition| + base_models_for(definition.from_table).each do |model| + expect(model.connection).to be_table_exist(definition.from_table) + expect(model.connection).to be_column_exist(definition.from_table, definition.column) + end + end + end + end + end +end diff --git a/spec/lib/gitlab/database/migration_helpers/loose_foreign_key_helpers_spec.rb b/spec/lib/gitlab/database/migration_helpers/loose_foreign_key_helpers_spec.rb index f1dbfbbff18..25fc676d09e 100644 --- a/spec/lib/gitlab/database/migration_helpers/loose_foreign_key_helpers_spec.rb +++ b/spec/lib/gitlab/database/migration_helpers/loose_foreign_key_helpers_spec.rb @@ -47,11 +47,16 @@ RSpec.describe Gitlab::Database::MigrationHelpers::LooseForeignKeyHelpers do record_to_be_deleted.delete expect(LooseForeignKeys::DeletedRecord.count).to eq(1) - deleted_record = LooseForeignKeys::DeletedRecord.all.first + + arel_table = LooseForeignKeys::DeletedRecord.arel_table + deleted_record = LooseForeignKeys::DeletedRecord + .select(arel_table[Arel.star], arel_table[:partition].as('partition_number')) # aliasing the ignored partition column to partition_number + .all + .first expect(deleted_record.primary_key_value).to eq(record_to_be_deleted.id) expect(deleted_record.fully_qualified_table_name).to eq('public._test_loose_fk_test_table') - expect(deleted_record.partition).to eq(1) + expect(deleted_record.partition_number).to eq(1) end it 'stores multiple record deletions' do diff --git a/spec/lib/gitlab/database/migration_helpers_spec.rb b/spec/lib/gitlab/database/migration_helpers_spec.rb index ea755f5a368..7f80bed04a4 100644 --- a/spec/lib/gitlab/database/migration_helpers_spec.rb +++ b/spec/lib/gitlab/database/migration_helpers_spec.rb @@ -2431,7 +2431,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers do let(:issues) { table(:issues) } def setup - namespace = namespaces.create!(name: 'foo', path: 'foo') + namespace = namespaces.create!(name: 'foo', path: 'foo', type: Namespaces::UserNamespace.sti_name) projects.create!(namespace_id: namespace.id) end diff --git a/spec/lib/gitlab/database/migrations/background_migration_helpers_spec.rb b/spec/lib/gitlab/database/migrations/background_migration_helpers_spec.rb index e42a6c970ea..99c7d70724c 100644 --- a/spec/lib/gitlab/database/migrations/background_migration_helpers_spec.rb +++ b/spec/lib/gitlab/database/migrations/background_migration_helpers_spec.rb @@ -7,78 +7,6 @@ RSpec.describe Gitlab::Database::Migrations::BackgroundMigrationHelpers do ActiveRecord::Migration.new.extend(described_class) end - describe '#bulk_queue_background_migration_jobs_by_range' do - context 'when the model has an ID column' do - let!(:id1) { create(:user).id } - let!(:id2) { create(:user).id } - let!(:id3) { create(:user).id } - - before do - User.class_eval do - include EachBatch - end - end - - context 'with enough rows to bulk queue jobs more than once' do - before do - stub_const('Gitlab::Database::Migrations::BackgroundMigrationHelpers::JOB_BUFFER_SIZE', 1) - end - - it 'queues jobs correctly' do - Sidekiq::Testing.fake! do - model.bulk_queue_background_migration_jobs_by_range(User, 'FooJob', batch_size: 2) - - expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['FooJob', [id1, id2]]) - expect(BackgroundMigrationWorker.jobs[1]['args']).to eq(['FooJob', [id3, id3]]) - end - end - - it 'queues jobs in groups of buffer size 1' do - expect(BackgroundMigrationWorker).to receive(:bulk_perform_async).with([['FooJob', [id1, id2]]]) - expect(BackgroundMigrationWorker).to receive(:bulk_perform_async).with([['FooJob', [id3, id3]]]) - - model.bulk_queue_background_migration_jobs_by_range(User, 'FooJob', batch_size: 2) - end - end - - context 'with not enough rows to bulk queue jobs more than once' do - it 'queues jobs correctly' do - Sidekiq::Testing.fake! do - model.bulk_queue_background_migration_jobs_by_range(User, 'FooJob', batch_size: 2) - - expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['FooJob', [id1, id2]]) - expect(BackgroundMigrationWorker.jobs[1]['args']).to eq(['FooJob', [id3, id3]]) - end - end - - it 'queues jobs in bulk all at once (big buffer size)' do - expect(BackgroundMigrationWorker).to receive(:bulk_perform_async).with([['FooJob', [id1, id2]], - ['FooJob', [id3, id3]]]) - - model.bulk_queue_background_migration_jobs_by_range(User, 'FooJob', batch_size: 2) - end - end - - context 'without specifying batch_size' do - it 'queues jobs correctly' do - Sidekiq::Testing.fake! do - model.bulk_queue_background_migration_jobs_by_range(User, 'FooJob') - - expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['FooJob', [id1, id3]]) - end - end - end - end - - context "when the model doesn't have an ID column" do - it 'raises error (for now)' do - expect do - model.bulk_queue_background_migration_jobs_by_range(ProjectAuthorization, 'FooJob') - end.to raise_error(StandardError, /does not have an ID/) - end - end - end - describe '#queue_background_migration_jobs_by_range_at_intervals' do context 'when the model has an ID column' do let!(:id1) { create(:user).id } @@ -354,161 +282,6 @@ RSpec.describe Gitlab::Database::Migrations::BackgroundMigrationHelpers do end end - describe '#queue_batched_background_migration' do - let(:pgclass_info) { instance_double('Gitlab::Database::PgClass', cardinality_estimate: 42) } - - before do - allow(Gitlab::Database::PgClass).to receive(:for_table).and_call_original - end - - context 'when such migration already exists' do - it 'does not create duplicate migration' do - create( - :batched_background_migration, - job_class_name: 'MyJobClass', - table_name: :projects, - column_name: :id, - interval: 10.minutes, - min_value: 5, - max_value: 1005, - batch_class_name: 'MyBatchClass', - batch_size: 200, - sub_batch_size: 20, - job_arguments: [[:id], [:id_convert_to_bigint]] - ) - - expect do - model.queue_batched_background_migration( - 'MyJobClass', - :projects, - :id, - [:id], [:id_convert_to_bigint], - job_interval: 5.minutes, - batch_min_value: 5, - batch_max_value: 1000, - batch_class_name: 'MyBatchClass', - batch_size: 100, - sub_batch_size: 10) - end.not_to change { Gitlab::Database::BackgroundMigration::BatchedMigration.count } - end - end - - it 'creates the database record for the migration' do - expect(Gitlab::Database::PgClass).to receive(:for_table).with(:projects).and_return(pgclass_info) - - expect do - model.queue_batched_background_migration( - 'MyJobClass', - :projects, - :id, - job_interval: 5.minutes, - batch_min_value: 5, - batch_max_value: 1000, - batch_class_name: 'MyBatchClass', - batch_size: 100, - sub_batch_size: 10) - end.to change { Gitlab::Database::BackgroundMigration::BatchedMigration.count }.by(1) - - expect(Gitlab::Database::BackgroundMigration::BatchedMigration.last).to have_attributes( - job_class_name: 'MyJobClass', - table_name: 'projects', - column_name: 'id', - interval: 300, - min_value: 5, - max_value: 1000, - batch_class_name: 'MyBatchClass', - batch_size: 100, - sub_batch_size: 10, - job_arguments: %w[], - status: 'active', - total_tuple_count: pgclass_info.cardinality_estimate) - end - - context 'when the job interval is lower than the minimum' do - let(:minimum_delay) { described_class::BATCH_MIN_DELAY } - - it 'sets the job interval to the minimum value' do - expect do - model.queue_batched_background_migration('MyJobClass', :events, :id, job_interval: minimum_delay - 1.minute) - end.to change { Gitlab::Database::BackgroundMigration::BatchedMigration.count }.by(1) - - created_migration = Gitlab::Database::BackgroundMigration::BatchedMigration.last - - expect(created_migration.interval).to eq(minimum_delay) - end - end - - context 'when additional arguments are passed to the method' do - it 'saves the arguments on the database record' do - expect do - model.queue_batched_background_migration( - 'MyJobClass', - :projects, - :id, - 'my', - 'arguments', - job_interval: 5.minutes, - batch_max_value: 1000) - end.to change { Gitlab::Database::BackgroundMigration::BatchedMigration.count }.by(1) - - expect(Gitlab::Database::BackgroundMigration::BatchedMigration.last).to have_attributes( - job_class_name: 'MyJobClass', - table_name: 'projects', - column_name: 'id', - interval: 300, - min_value: 1, - max_value: 1000, - job_arguments: %w[my arguments]) - end - end - - context 'when the max_value is not given' do - context 'when records exist in the database' do - let!(:event1) { create(:event) } - let!(:event2) { create(:event) } - let!(:event3) { create(:event) } - - it 'creates the record with the current max value' do - expect do - model.queue_batched_background_migration('MyJobClass', :events, :id, job_interval: 5.minutes) - end.to change { Gitlab::Database::BackgroundMigration::BatchedMigration.count }.by(1) - - created_migration = Gitlab::Database::BackgroundMigration::BatchedMigration.last - - expect(created_migration.max_value).to eq(event3.id) - end - - it 'creates the record with an active status' do - expect do - model.queue_batched_background_migration('MyJobClass', :events, :id, job_interval: 5.minutes) - end.to change { Gitlab::Database::BackgroundMigration::BatchedMigration.count }.by(1) - - expect(Gitlab::Database::BackgroundMigration::BatchedMigration.last).to be_active - end - end - - context 'when the database is empty' do - it 'sets the max value to the min value' do - expect do - model.queue_batched_background_migration('MyJobClass', :events, :id, job_interval: 5.minutes) - end.to change { Gitlab::Database::BackgroundMigration::BatchedMigration.count }.by(1) - - created_migration = Gitlab::Database::BackgroundMigration::BatchedMigration.last - - expect(created_migration.max_value).to eq(created_migration.min_value) - end - - it 'creates the record with a finished status' do - expect do - model.queue_batched_background_migration('MyJobClass', :projects, :id, job_interval: 5.minutes) - end.to change { Gitlab::Database::BackgroundMigration::BatchedMigration.count }.by(1) - - expect(Gitlab::Database::BackgroundMigration::BatchedMigration.last).to be_finished - end - end - end - end - describe '#migrate_async' do it 'calls BackgroundMigrationWorker.perform_async' do expect(BackgroundMigrationWorker).to receive(:perform_async).with("Class", "hello", "world") @@ -583,7 +356,7 @@ RSpec.describe Gitlab::Database::Migrations::BackgroundMigrationHelpers do end describe '#finalized_background_migration' do - let(:job_coordinator) { Gitlab::BackgroundMigration::JobCoordinator.new(:main, BackgroundMigrationWorker) } + let(:job_coordinator) { Gitlab::BackgroundMigration::JobCoordinator.new(BackgroundMigrationWorker) } let!(:job_class_name) { 'TestJob' } let!(:job_class) { Class.new } @@ -605,7 +378,7 @@ RSpec.describe Gitlab::Database::Migrations::BackgroundMigrationHelpers do job_class.define_method(:perform, job_perform_method) allow(Gitlab::BackgroundMigration).to receive(:coordinator_for_database) - .with(:main).and_return(job_coordinator) + .with('main').and_return(job_coordinator) expect(job_coordinator).to receive(:migration_class_for) .with(job_class_name).at_least(:once) { job_class } diff --git a/spec/lib/gitlab/database/migrations/batched_background_migration_helpers_spec.rb b/spec/lib/gitlab/database/migrations/batched_background_migration_helpers_spec.rb new file mode 100644 index 00000000000..c45149d67bf --- /dev/null +++ b/spec/lib/gitlab/database/migrations/batched_background_migration_helpers_spec.rb @@ -0,0 +1,164 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::Migrations::BatchedBackgroundMigrationHelpers do + let(:migration) do + ActiveRecord::Migration.new.extend(described_class) + end + + describe '#queue_batched_background_migration' do + let(:pgclass_info) { instance_double('Gitlab::Database::PgClass', cardinality_estimate: 42) } + + before do + allow(Gitlab::Database::PgClass).to receive(:for_table).and_call_original + end + + context 'when such migration already exists' do + it 'does not create duplicate migration' do + create( + :batched_background_migration, + job_class_name: 'MyJobClass', + table_name: :projects, + column_name: :id, + interval: 10.minutes, + min_value: 5, + max_value: 1005, + batch_class_name: 'MyBatchClass', + batch_size: 200, + sub_batch_size: 20, + job_arguments: [[:id], [:id_convert_to_bigint]] + ) + + expect do + migration.queue_batched_background_migration( + 'MyJobClass', + :projects, + :id, + [:id], [:id_convert_to_bigint], + job_interval: 5.minutes, + batch_min_value: 5, + batch_max_value: 1000, + batch_class_name: 'MyBatchClass', + batch_size: 100, + sub_batch_size: 10) + end.not_to change { Gitlab::Database::BackgroundMigration::BatchedMigration.count } + end + end + + it 'creates the database record for the migration' do + expect(Gitlab::Database::PgClass).to receive(:for_table).with(:projects).and_return(pgclass_info) + + expect do + migration.queue_batched_background_migration( + 'MyJobClass', + :projects, + :id, + job_interval: 5.minutes, + batch_min_value: 5, + batch_max_value: 1000, + batch_class_name: 'MyBatchClass', + batch_size: 100, + sub_batch_size: 10) + end.to change { Gitlab::Database::BackgroundMigration::BatchedMigration.count }.by(1) + + expect(Gitlab::Database::BackgroundMigration::BatchedMigration.last).to have_attributes( + job_class_name: 'MyJobClass', + table_name: 'projects', + column_name: 'id', + interval: 300, + min_value: 5, + max_value: 1000, + batch_class_name: 'MyBatchClass', + batch_size: 100, + sub_batch_size: 10, + job_arguments: %w[], + status: 'active', + total_tuple_count: pgclass_info.cardinality_estimate) + end + + context 'when the job interval is lower than the minimum' do + let(:minimum_delay) { described_class::BATCH_MIN_DELAY } + + it 'sets the job interval to the minimum value' do + expect do + migration.queue_batched_background_migration('MyJobClass', :events, :id, job_interval: minimum_delay - 1.minute) + end.to change { Gitlab::Database::BackgroundMigration::BatchedMigration.count }.by(1) + + created_migration = Gitlab::Database::BackgroundMigration::BatchedMigration.last + + expect(created_migration.interval).to eq(minimum_delay) + end + end + + context 'when additional arguments are passed to the method' do + it 'saves the arguments on the database record' do + expect do + migration.queue_batched_background_migration( + 'MyJobClass', + :projects, + :id, + 'my', + 'arguments', + job_interval: 5.minutes, + batch_max_value: 1000) + end.to change { Gitlab::Database::BackgroundMigration::BatchedMigration.count }.by(1) + + expect(Gitlab::Database::BackgroundMigration::BatchedMigration.last).to have_attributes( + job_class_name: 'MyJobClass', + table_name: 'projects', + column_name: 'id', + interval: 300, + min_value: 1, + max_value: 1000, + job_arguments: %w[my arguments]) + end + end + + context 'when the max_value is not given' do + context 'when records exist in the database' do + let!(:event1) { create(:event) } + let!(:event2) { create(:event) } + let!(:event3) { create(:event) } + + it 'creates the record with the current max value' do + expect do + migration.queue_batched_background_migration('MyJobClass', :events, :id, job_interval: 5.minutes) + end.to change { Gitlab::Database::BackgroundMigration::BatchedMigration.count }.by(1) + + created_migration = Gitlab::Database::BackgroundMigration::BatchedMigration.last + + expect(created_migration.max_value).to eq(event3.id) + end + + it 'creates the record with an active status' do + expect do + migration.queue_batched_background_migration('MyJobClass', :events, :id, job_interval: 5.minutes) + end.to change { Gitlab::Database::BackgroundMigration::BatchedMigration.count }.by(1) + + expect(Gitlab::Database::BackgroundMigration::BatchedMigration.last).to be_active + end + end + + context 'when the database is empty' do + it 'sets the max value to the min value' do + expect do + migration.queue_batched_background_migration('MyJobClass', :events, :id, job_interval: 5.minutes) + end.to change { Gitlab::Database::BackgroundMigration::BatchedMigration.count }.by(1) + + created_migration = Gitlab::Database::BackgroundMigration::BatchedMigration.last + + expect(created_migration.max_value).to eq(created_migration.min_value) + end + + it 'creates the record with a finished status' do + expect do + migration.queue_batched_background_migration('MyJobClass', :projects, :id, job_interval: 5.minutes) + end.to change { Gitlab::Database::BackgroundMigration::BatchedMigration.count }.by(1) + + expect(Gitlab::Database::BackgroundMigration::BatchedMigration.last).to be_finished + end + end + end + end +end diff --git a/spec/lib/gitlab/database/migrations/instrumentation_spec.rb b/spec/lib/gitlab/database/migrations/instrumentation_spec.rb index 841d2a98a16..902d8e13a63 100644 --- a/spec/lib/gitlab/database/migrations/instrumentation_spec.rb +++ b/spec/lib/gitlab/database/migrations/instrumentation_spec.rb @@ -3,6 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::Database::Migrations::Instrumentation do let(:result_dir) { Dir.mktmpdir } + let(:connection) { ActiveRecord::Migration.connection } after do FileUtils.rm_rf(result_dir) @@ -14,11 +15,11 @@ RSpec.describe Gitlab::Database::Migrations::Instrumentation do let(:migration_version) { '12345' } it 'executes the given block' do - expect { |b| subject.observe(version: migration_version, name: migration_name, &b) }.to yield_control + expect { |b| subject.observe(version: migration_version, name: migration_name, connection: connection, &b) }.to yield_control end context 'behavior with observers' do - subject { described_class.new(observer_classes: [Gitlab::Database::Migrations::Observers::MigrationObserver], result_dir: result_dir).observe(version: migration_version, name: migration_name) {} } + subject { described_class.new(observer_classes: [Gitlab::Database::Migrations::Observers::MigrationObserver], result_dir: result_dir).observe(version: migration_version, name: migration_name, connection: connection) {} } let(:observer) { instance_double('Gitlab::Database::Migrations::Observers::MigrationObserver', before: nil, after: nil, record: nil) } @@ -29,7 +30,7 @@ RSpec.describe Gitlab::Database::Migrations::Instrumentation do it 'instantiates observer with observation' do expect(Gitlab::Database::Migrations::Observers::MigrationObserver) .to receive(:new) - .with(instance_of(Gitlab::Database::Migrations::Observation), anything) { |observation| expect(observation.version).to eq(migration_version) } + .with(instance_of(Gitlab::Database::Migrations::Observation), anything, connection) { |observation| expect(observation.version).to eq(migration_version) } .and_return(observer) subject @@ -63,7 +64,7 @@ RSpec.describe Gitlab::Database::Migrations::Instrumentation do end context 'on successful execution' do - subject { described_class.new(result_dir: result_dir).observe(version: migration_version, name: migration_name) {} } + subject { described_class.new(result_dir: result_dir).observe(version: migration_version, name: migration_name, connection: connection) {} } it 'records walltime' do expect(subject.walltime).not_to be_nil @@ -83,7 +84,7 @@ RSpec.describe Gitlab::Database::Migrations::Instrumentation do end context 'upon failure' do - subject { described_class.new(result_dir: result_dir).observe(version: migration_version, name: migration_name) { raise 'something went wrong' } } + subject { described_class.new(result_dir: result_dir).observe(version: migration_version, name: migration_name, connection: connection) { raise 'something went wrong' } } it 'raises the exception' do expect { subject }.to raise_error(/something went wrong/) @@ -93,7 +94,7 @@ RSpec.describe Gitlab::Database::Migrations::Instrumentation do subject { instance.observations.first } before do - instance.observe(version: migration_version, name: migration_name) { raise 'something went wrong' } + instance.observe(version: migration_version, name: migration_name, connection: connection) { raise 'something went wrong' } rescue StandardError # ignore end @@ -125,8 +126,8 @@ RSpec.describe Gitlab::Database::Migrations::Instrumentation do let(:migration2) { double('migration2', call: nil) } it 'records observations for all migrations' do - subject.observe(version: migration_version, name: migration_name) {} - subject.observe(version: migration_version, name: migration_name) { raise 'something went wrong' } rescue nil + subject.observe(version: migration_version, name: migration_name, connection: connection) {} + subject.observe(version: migration_version, name: migration_name, connection: connection) { raise 'something went wrong' } rescue nil expect(subject.observations.size).to eq(2) end diff --git a/spec/lib/gitlab/database/migrations/observers/query_details_spec.rb b/spec/lib/gitlab/database/migrations/observers/query_details_spec.rb index 191ac29e3b3..5a19ae6581d 100644 --- a/spec/lib/gitlab/database/migrations/observers/query_details_spec.rb +++ b/spec/lib/gitlab/database/migrations/observers/query_details_spec.rb @@ -2,10 +2,10 @@ require 'spec_helper' RSpec.describe Gitlab::Database::Migrations::Observers::QueryDetails do - subject { described_class.new(observation, directory_path) } + subject { described_class.new(observation, directory_path, connection) } + let(:connection) { ActiveRecord::Migration.connection } let(:observation) { Gitlab::Database::Migrations::Observation.new(migration_version, migration_name) } - let(:connection) { ActiveRecord::Base.connection } let(:query) { "select date_trunc('day', $1::timestamptz) + $2 * (interval '1 hour')" } let(:query_binds) { [Time.current, 3] } let(:directory_path) { Dir.mktmpdir } diff --git a/spec/lib/gitlab/database/migrations/observers/query_log_spec.rb b/spec/lib/gitlab/database/migrations/observers/query_log_spec.rb index 2e70a85fd5b..7b01e39f5f1 100644 --- a/spec/lib/gitlab/database/migrations/observers/query_log_spec.rb +++ b/spec/lib/gitlab/database/migrations/observers/query_log_spec.rb @@ -2,10 +2,10 @@ require 'spec_helper' RSpec.describe Gitlab::Database::Migrations::Observers::QueryLog do - subject { described_class.new(observation, directory_path) } + subject { described_class.new(observation, directory_path, connection) } let(:observation) { Gitlab::Database::Migrations::Observation.new(migration_version, migration_name) } - let(:connection) { ActiveRecord::Base.connection } + let(:connection) { ActiveRecord::Migration.connection } let(:query) { 'select 1' } let(:directory_path) { Dir.mktmpdir } let(:migration_version) { 20210422152437 } diff --git a/spec/lib/gitlab/database/migrations/observers/query_statistics_spec.rb b/spec/lib/gitlab/database/migrations/observers/query_statistics_spec.rb index 9727a215d71..2515f0d4a06 100644 --- a/spec/lib/gitlab/database/migrations/observers/query_statistics_spec.rb +++ b/spec/lib/gitlab/database/migrations/observers/query_statistics_spec.rb @@ -2,10 +2,10 @@ require 'spec_helper' RSpec.describe Gitlab::Database::Migrations::Observers::QueryStatistics do - subject { described_class.new(observation, double("unused path")) } + subject { described_class.new(observation, double("unused path"), connection) } let(:observation) { Gitlab::Database::Migrations::Observation.new } - let(:connection) { ActiveRecord::Base.connection } + let(:connection) { ActiveRecord::Migration.connection } def mock_pgss(enabled: true) if enabled diff --git a/spec/lib/gitlab/database/migrations/observers/total_database_size_change_spec.rb b/spec/lib/gitlab/database/migrations/observers/total_database_size_change_spec.rb index e689759c574..4b08838d6bb 100644 --- a/spec/lib/gitlab/database/migrations/observers/total_database_size_change_spec.rb +++ b/spec/lib/gitlab/database/migrations/observers/total_database_size_change_spec.rb @@ -2,10 +2,10 @@ require 'spec_helper' RSpec.describe Gitlab::Database::Migrations::Observers::TotalDatabaseSizeChange do - subject { described_class.new(observation, double('unused path')) } + subject { described_class.new(observation, double('unused path'), connection) } let(:observation) { Gitlab::Database::Migrations::Observation.new } - let(:connection) { ActiveRecord::Base.connection } + let(:connection) { ActiveRecord::Migration.connection } let(:query) { 'select pg_database_size(current_database())' } it 'records the size change' do diff --git a/spec/lib/gitlab/database/migrations/observers/transaction_duration_spec.rb b/spec/lib/gitlab/database/migrations/observers/transaction_duration_spec.rb index e65f89747c4..b26bb8fbe41 100644 --- a/spec/lib/gitlab/database/migrations/observers/transaction_duration_spec.rb +++ b/spec/lib/gitlab/database/migrations/observers/transaction_duration_spec.rb @@ -2,8 +2,9 @@ require 'spec_helper' RSpec.describe Gitlab::Database::Migrations::Observers::TransactionDuration do - subject(:transaction_duration_observer) { described_class.new(observation, directory_path) } + subject(:transaction_duration_observer) { described_class.new(observation, directory_path, connection) } + let(:connection) { ActiveRecord::Migration.connection } let(:observation) { Gitlab::Database::Migrations::Observation.new(migration_version, migration_name) } let(:directory_path) { Dir.mktmpdir } let(:log_file) { "#{directory_path}/#{migration_version}_#{migration_name}-transaction-duration.json" } @@ -78,17 +79,17 @@ RSpec.describe Gitlab::Database::Migrations::Observers::TransactionDuration do end def run_real_transactions - ActiveRecord::Base.transaction do + ApplicationRecord.transaction do end end def run_sub_transactions - ActiveRecord::Base.transaction(requires_new: true) do + ApplicationRecord.transaction(requires_new: true) do end end def run_transaction - ActiveRecord::Base.connection_pool.with_connection do |connection| + ApplicationRecord.connection_pool.with_connection do |connection| Gitlab::Database::SharedModel.using_connection(connection) do Gitlab::Database::SharedModel.transaction do Gitlab::Database::SharedModel.transaction(requires_new: true) do diff --git a/spec/lib/gitlab/database/migrations/runner_spec.rb b/spec/lib/gitlab/database/migrations/runner_spec.rb index 52fb5ec2ba8..4616bd6941e 100644 --- a/spec/lib/gitlab/database/migrations/runner_spec.rb +++ b/spec/lib/gitlab/database/migrations/runner_spec.rb @@ -76,7 +76,7 @@ RSpec.describe Gitlab::Database::Migrations::Runner do it 'runs the unapplied migrations in version order', :aggregate_failures do up.run - expect(migration_runs.map(&:dir)).to eq([:up, :up]) + expect(migration_runs.map(&:dir)).to match_array([:up, :up]) expect(migration_runs.map(&:version_to_migrate)).to eq(pending_migrations.map(&:version)) end end @@ -101,7 +101,7 @@ RSpec.describe Gitlab::Database::Migrations::Runner do it 'runs the applied migrations for the current branch in reverse order', :aggregate_failures do down.run - expect(migration_runs.map(&:dir)).to eq([:down, :down]) + expect(migration_runs.map(&:dir)).to match_array([:down, :down]) expect(migration_runs.map(&:version_to_migrate)).to eq(applied_migrations_this_branch.reverse.map(&:version)) end end diff --git a/spec/lib/gitlab/database/partitioning/detached_partition_dropper_spec.rb b/spec/lib/gitlab/database/partitioning/detached_partition_dropper_spec.rb index b2c4e4b54a4..2ef873e8adb 100644 --- a/spec/lib/gitlab/database/partitioning/detached_partition_dropper_spec.rb +++ b/spec/lib/gitlab/database/partitioning/detached_partition_dropper_spec.rb @@ -90,18 +90,6 @@ RSpec.describe Gitlab::Database::Partitioning::DetachedPartitionDropper do expect(table_oid('test_partition')).to be_nil end - context 'when the drop_detached_partitions feature flag is disabled' do - before do - stub_feature_flags(drop_detached_partitions: false) - end - - it 'does not drop the partition' do - dropper.perform - - expect(table_oid('test_partition')).not_to be_nil - end - end - context 'removing foreign keys' do it 'removes foreign keys from the table before dropping it' do expect(dropper).to receive(:drop_detached_partition).and_wrap_original do |drop_method, partition_name| diff --git a/spec/lib/gitlab/database/partitioning/partition_manager_spec.rb b/spec/lib/gitlab/database/partitioning/partition_manager_spec.rb index 1c6f5c5c694..5e107109fc9 100644 --- a/spec/lib/gitlab/database/partitioning/partition_manager_spec.rb +++ b/spec/lib/gitlab/database/partitioning/partition_manager_spec.rb @@ -16,7 +16,7 @@ RSpec.describe Gitlab::Database::Partitioning::PartitionManager do subject(:sync_partitions) { described_class.new(model).sync_partitions } let(:model) { double(partitioning_strategy: partitioning_strategy, table_name: table, connection: connection) } - let(:partitioning_strategy) { double(missing_partitions: partitions, extra_partitions: []) } + let(:partitioning_strategy) { double(missing_partitions: partitions, extra_partitions: [], after_adding_partitions: nil) } let(:connection) { ActiveRecord::Base.connection } let(:table) { "some_table" } @@ -83,7 +83,7 @@ RSpec.describe Gitlab::Database::Partitioning::PartitionManager do let(:manager) { described_class.new(model) } let(:model) { double(partitioning_strategy: partitioning_strategy, table_name: table, connection: connection) } - let(:partitioning_strategy) { double(extra_partitions: extra_partitions, missing_partitions: []) } + let(:partitioning_strategy) { double(extra_partitions: extra_partitions, missing_partitions: [], after_adding_partitions: nil) } let(:connection) { ActiveRecord::Base.connection } let(:table) { "foo" } @@ -101,28 +101,10 @@ RSpec.describe Gitlab::Database::Partitioning::PartitionManager do ] end - context 'with the partition_pruning feature flag enabled' do - before do - stub_feature_flags(partition_pruning: true) - end - - it 'detaches each extra partition' do - extra_partitions.each { |p| expect(manager).to receive(:detach_one_partition).with(p) } - - sync_partitions - end - end + it 'detaches each extra partition' do + extra_partitions.each { |p| expect(manager).to receive(:detach_one_partition).with(p) } - context 'with the partition_pruning feature flag disabled' do - before do - stub_feature_flags(partition_pruning: false) - end - - it 'returns immediately' do - expect(manager).not_to receive(:detach) - - sync_partitions - end + sync_partitions end end diff --git a/spec/lib/gitlab/database/partitioning/single_numeric_list_partition_spec.rb b/spec/lib/gitlab/database/partitioning/single_numeric_list_partition_spec.rb new file mode 100644 index 00000000000..9941241e846 --- /dev/null +++ b/spec/lib/gitlab/database/partitioning/single_numeric_list_partition_spec.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::Partitioning::SingleNumericListPartition do + describe '.from_sql' do + subject(:parsed_partition) { described_class.from_sql(table, partition_name, definition) } + + let(:table) { 'partitioned_table' } + let(:partition_value) { 0 } + let(:partition_name) { "partitioned_table_#{partition_value}" } + let(:definition) { "FOR VALUES IN ('#{partition_value}')" } + + it 'uses specified table name' do + expect(parsed_partition.table).to eq(table) + end + + it 'uses specified partition name' do + expect(parsed_partition.partition_name).to eq(partition_name) + end + + it 'parses the definition' do + expect(parsed_partition.value).to eq(partition_value) + end + end + + describe '#partition_name' do + it 'is the explicit name if provided' do + expect(described_class.new('table', 1, partition_name: 'some_other_name').partition_name).to eq('some_other_name') + end + + it 'defaults to the table name followed by the partition value' do + expect(described_class.new('table', 1).partition_name).to eq('table_1') + end + end + + context 'sorting' do + it 'is incomparable if the tables do not match' do + expect(described_class.new('table1', 1) <=> described_class.new('table2', 2)).to be_nil + end + + it 'sorts by the value when the tables match' do + expect(described_class.new('table1', 1) <=> described_class.new('table1', 2)).to eq(1 <=> 2) + end + + it 'sorts by numeric value rather than text value' do + expect(described_class.new('table', 10)).to be > described_class.new('table', 9) + end + end +end diff --git a/spec/lib/gitlab/database/partitioning/sliding_list_strategy_spec.rb b/spec/lib/gitlab/database/partitioning/sliding_list_strategy_spec.rb new file mode 100644 index 00000000000..636a09e5710 --- /dev/null +++ b/spec/lib/gitlab/database/partitioning/sliding_list_strategy_spec.rb @@ -0,0 +1,214 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::Partitioning::SlidingListStrategy do + let(:connection) { ActiveRecord::Base.connection } + let(:table_name) { :_test_partitioned_test } + let(:model) { double('model', table_name: table_name, ignored_columns: %w[partition]) } + let(:next_partition_if) { double('next_partition_if') } + let(:detach_partition_if) { double('detach_partition_if') } + + subject(:strategy) do + described_class.new(model, :partition, + next_partition_if: next_partition_if, + detach_partition_if: detach_partition_if) + end + + before do + connection.execute(<<~SQL) + create table #{table_name} + ( + id serial not null, + partition bigint not null default 2, + created_at timestamptz not null, + primary key (id, partition) + ) + partition by list(partition); + + create table #{table_name}_1 + partition of #{table_name} for values in (1); + + create table #{table_name}_2 + partition of #{table_name} for values in (2); + SQL + end + + describe '#current_partitions' do + it 'detects both partitions' do + expect(strategy.current_partitions).to eq([ + Gitlab::Database::Partitioning::SingleNumericListPartition.new(table_name, 1, partition_name: '_test_partitioned_test_1'), + Gitlab::Database::Partitioning::SingleNumericListPartition.new(table_name, 2, partition_name: '_test_partitioned_test_2') + ]) + end + end + + describe '#active_partition' do + it 'is the partition with the largest value' do + expect(strategy.active_partition.value).to eq(2) + end + end + + describe '#missing_partitions' do + context 'when next_partition_if returns true' do + let(:next_partition_if) { proc { true } } + + it 'is a partition definition for the next partition in the series' do + extra = strategy.missing_partitions + + expect(extra.length).to eq(1) + expect(extra.first.value).to eq(3) + end + end + + context 'when next_partition_if returns false' do + let(:next_partition_if) { proc { false } } + + it 'is empty' do + expect(strategy.missing_partitions).to be_empty + end + end + + context 'when there are no partitions for the table' do + it 'returns a partition for value 1' do + connection.execute("drop table #{table_name}_1; drop table #{table_name}_2;") + + missing_partitions = strategy.missing_partitions + + expect(missing_partitions.size).to eq(1) + missing_partition = missing_partitions.first + + expect(missing_partition.value).to eq(1) + end + end + end + + describe '#extra_partitions' do + before do + (3..10).each do |i| + connection.execute("CREATE TABLE #{table_name}_#{i} PARTITION OF #{table_name} FOR VALUES IN (#{i})") + end + end + + context 'when some partitions are true for detach_partition_if' do + let(:detach_partition_if) { ->(p) { p != 5 } } + + it 'is the leading set of partitions before that value' do + expect(strategy.extra_partitions.map(&:value)).to contain_exactly(1, 2, 3, 4) + end + end + + context 'when all partitions are true for detach_partition_if' do + let(:detach_partition_if) { proc { true } } + + it 'is all but the most recent partition', :aggregate_failures do + expect(strategy.extra_partitions.map(&:value)).to contain_exactly(1, 2, 3, 4, 5, 6, 7, 8, 9) + + expect(strategy.current_partitions.map(&:value).max).to eq(10) + end + end + end + + describe '#initial_partition' do + it 'starts with the value 1', :aggregate_failures do + initial_partition = strategy.initial_partition + expect(initial_partition.value).to eq(1) + expect(initial_partition.table).to eq(strategy.table_name) + expect(initial_partition.partition_name).to eq("#{strategy.table_name}_1") + end + end + + describe '#next_partition' do + it 'is one after the active partition', :aggregate_failures do + expect(strategy).to receive(:active_partition).and_return(double(value: 5)) + next_partition = strategy.next_partition + + expect(next_partition.value).to eq(6) + expect(next_partition.table).to eq(strategy.table_name) + expect(next_partition.partition_name).to eq("#{strategy.table_name}_6") + end + end + + describe '#ensure_partitioning_column_ignored!' do + it 'raises when the column is not ignored' do + expect do + Class.new(ApplicationRecord) do + include PartitionedTable + + partitioned_by :partition, strategy: :sliding_list, + next_partition_if: proc { false }, + detach_partition_if: proc { false } + end + end.to raise_error(/ignored_columns/) + end + + it 'does not raise when the column is ignored' do + expect do + Class.new(ApplicationRecord) do + include PartitionedTable + + self.ignored_columns = [:partition] + + partitioned_by :partition, strategy: :sliding_list, + next_partition_if: proc { false }, + detach_partition_if: proc { false } + end + end.not_to raise_error + end + end + context 'redirecting inserts as the active partition changes' do + let(:model) do + Class.new(ApplicationRecord) do + include PartitionedTable + + self.table_name = '_test_partitioned_test' + self.primary_key = :id + + self.ignored_columns = %w[partition] + + # method().call cannot be detected by rspec, so we add a layer of indirection here + def self.next_partition_if_wrapper(...) + next_partition?(...) + end + + def self.detach_partition_if_wrapper(...) + detach_partition?(...) + end + partitioned_by :partition, strategy: :sliding_list, + next_partition_if: method(:next_partition_if_wrapper), + detach_partition_if: method(:detach_partition_if_wrapper) + + def self.next_partition?(current_partition) + end + + def self.detach_partition?(partition) + end + end + end + + it 'redirects to the new partition', :aggregate_failures do + partition_2_model = model.create! # Goes in partition 2 + + allow(model).to receive(:next_partition?) do + model.partitioning_strategy.active_partition.value < 3 + end + + allow(model).to receive(:detach_partition?).and_return(false) + + Gitlab::Database::Partitioning::PartitionManager.new(model).sync_partitions + + partition_3_model = model.create! + + # Rails doesn't pick up on database default changes, so we need to reload + # We also want to grab the partition column to verify what it was set to. + # In normal operation we make rails ignore it so that we can use a changing default + # So we force select * to load it + all_columns = model.select(model.arel_table[Arel.star]) + partition_2_model = all_columns.find(partition_2_model.id) + partition_3_model = all_columns.find(partition_3_model.id) + + expect(partition_2_model.partition).to eq(2) + expect(partition_3_model.partition).to eq(3) + end + end +end diff --git a/spec/lib/gitlab/database/query_analyzer_spec.rb b/spec/lib/gitlab/database/query_analyzer_spec.rb index 82a1c7143d5..34c72893c53 100644 --- a/spec/lib/gitlab/database/query_analyzer_spec.rb +++ b/spec/lib/gitlab/database/query_analyzer_spec.rb @@ -128,11 +128,20 @@ RSpec.describe Gitlab::Database::QueryAnalyzer, query_analyzers: false do it 'does not call analyze on suppressed analyzers' do expect(analyzer).to receive(:suppressed?).and_return(true) + expect(analyzer).to receive(:requires_tracking?).and_return(false) expect(analyzer).not_to receive(:analyze) expect { process_sql("SELECT 1 FROM projects") }.not_to raise_error end + it 'does call analyze on suppressed analyzers if some queries require tracking' do + expect(analyzer).to receive(:suppressed?).and_return(true) + expect(analyzer).to receive(:requires_tracking?).and_return(true) + expect(analyzer).to receive(:analyze) + + expect { process_sql("SELECT 1 FROM projects") }.not_to raise_error + end + def process_sql(sql) described_class.instance.within do ApplicationRecord.load_balancer.read_write do |connection| diff --git a/spec/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics_spec.rb b/spec/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics_spec.rb index ab5f05e3ec4..86e74cf5177 100644 --- a/spec/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics_spec.rb +++ b/spec/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics_spec.rb @@ -17,7 +17,7 @@ RSpec.describe Gitlab::Database::QueryAnalyzers::GitlabSchemasMetrics, query_ana process_sql(ActiveRecord::Base, "SELECT 1 FROM projects") end - context 'properly observes all queries', :mocked_ci_connection do + context 'properly observes all queries', :add_ci_connection do using RSpec::Parameterized::TableSyntax where do diff --git a/spec/lib/gitlab/database/query_analyzers/prevent_cross_database_modification_spec.rb b/spec/lib/gitlab/database/query_analyzers/prevent_cross_database_modification_spec.rb index eb8ccb0bd89..c41b4eeea10 100644 --- a/spec/lib/gitlab/database/query_analyzers/prevent_cross_database_modification_spec.rb +++ b/spec/lib/gitlab/database/query_analyzers/prevent_cross_database_modification_spec.rb @@ -92,6 +92,23 @@ RSpec.describe Gitlab::Database::QueryAnalyzers::PreventCrossDatabaseModificatio end end end + + context 'when comments are added to the front of query strings' do + around do |example| + prepend_comment_was = Marginalia::Comment.prepend_comment + Marginalia::Comment.prepend_comment = true + + example.run + + Marginalia::Comment.prepend_comment = prepend_comment_was + end + + it 'raises error' do + Project.transaction do + expect { run_queries }.to raise_error /Cross-database data modification/ + end + end + end end context 'when executing a SELECT FOR UPDATE query' do @@ -164,4 +181,49 @@ RSpec.describe Gitlab::Database::QueryAnalyzers::PreventCrossDatabaseModificatio end.to raise_error /Cross-database data modification.*The gitlab_schema was undefined/ end end + + context 'when execution is rescued with StandardError' do + it 'raises cross-database data modification exception' do + expect do + Project.transaction do + project.touch + project.connection.execute('UPDATE foo_bars_undefined_table SET a=1 WHERE id = -1') + end + rescue StandardError + # Ensures that standard rescue does not silence errors + end.to raise_error /Cross-database data modification.*The gitlab_schema was undefined/ + end + end + + context 'when uniquiness validation is tested', type: :model do + subject { build(:ci_variable) } + + it 'does not raise exceptions' do + expect do + is_expected.to validate_uniqueness_of(:key).scoped_to(:project_id, :environment_scope).with_message(/\(\w+\) has already been taken/) + end.not_to raise_error + end + end + + context 'when doing rollback in a suppressed block' do + it 'does not raise misaligned transactions exception' do + expect do + # This is non-materialised transaction: + # 1. the transaction will be open on a write (project.touch) (in a suppressed block) + # 2. the rescue will be handled outside of suppressed block + # + # This will create misaligned boundaries since BEGIN + # of transaction will be executed within a suppressed block + Project.transaction do + described_class.with_suppressed do + project.touch + + raise 'force rollback' + end + + # the ensure of `.transaction` executes `ROLLBACK TO SAVEPOINT` + end + end.to raise_error /force rollback/ + end + end end diff --git a/spec/lib/gitlab/database/reindexing/coordinator_spec.rb b/spec/lib/gitlab/database/reindexing/coordinator_spec.rb index 085fd3061ad..0afbe46b7f1 100644 --- a/spec/lib/gitlab/database/reindexing/coordinator_spec.rb +++ b/spec/lib/gitlab/database/reindexing/coordinator_spec.rb @@ -15,10 +15,18 @@ RSpec.describe Gitlab::Database::Reindexing::Coordinator do let(:action) { create(:reindex_action, index: index) } let!(:lease) { stub_exclusive_lease(lease_key, uuid, timeout: lease_timeout) } - let(:lease_key) { 'gitlab/database/reindexing/coordinator' } + let(:lease_key) { "gitlab/database/reindexing/coordinator/#{Gitlab::Database::PRIMARY_DATABASE_NAME}" } let(:lease_timeout) { 1.day } let(:uuid) { 'uuid' } + around do |example| + model = Gitlab::Database.database_base_models[Gitlab::Database::PRIMARY_DATABASE_NAME] + + Gitlab::Database::SharedModel.using_connection(model.connection) do + example.run + end + end + before do swapout_view_for_table(:postgres_indexes) diff --git a/spec/lib/gitlab/database/reindexing_spec.rb b/spec/lib/gitlab/database/reindexing_spec.rb index 13aff343432..0c576505e07 100644 --- a/spec/lib/gitlab/database/reindexing_spec.rb +++ b/spec/lib/gitlab/database/reindexing_spec.rb @@ -6,6 +6,63 @@ RSpec.describe Gitlab::Database::Reindexing do include ExclusiveLeaseHelpers include Database::DatabaseHelpers + describe '.invoke' do + let(:databases) { Gitlab::Database.database_base_models } + let(:databases_count) { databases.count } + + it 'cleans up any leftover indexes' do + expect(described_class).to receive(:cleanup_leftovers!).exactly(databases_count).times + + described_class.invoke + end + + context 'when there is an error raised' do + it 'logs and re-raise' do + expect(described_class).to receive(:automatic_reindexing).and_raise('Unexpected!') + expect(Gitlab::AppLogger).to receive(:error) + + expect { described_class.invoke }.to raise_error('Unexpected!') + end + end + + context 'when async index creation is enabled' do + it 'executes async index creation prior to any reindexing actions' do + stub_feature_flags(database_async_index_creation: true) + + expect(Gitlab::Database::AsyncIndexes).to receive(:create_pending_indexes!).ordered.exactly(databases_count).times + expect(described_class).to receive(:automatic_reindexing).ordered.exactly(databases_count).times + + described_class.invoke + end + end + + context 'when async index creation is disabled' do + it 'does not execute async index creation' do + stub_feature_flags(database_async_index_creation: false) + + expect(Gitlab::Database::AsyncIndexes).not_to receive(:create_pending_indexes!) + + described_class.invoke + end + end + + context 'calls automatic reindexing' do + it 'uses all candidate indexes' do + expect(described_class).to receive(:automatic_reindexing).exactly(databases_count).times + + described_class.invoke + end + + context 'when explicit database is given' do + it 'skips other databases' do + expect(described_class).to receive(:automatic_reindexing).once + + described_class.invoke(Gitlab::Database::PRIMARY_DATABASE_NAME) + end + end + end + end + describe '.automatic_reindexing' do subject { described_class.automatic_reindexing(maximum_records: limit) } @@ -133,10 +190,19 @@ RSpec.describe Gitlab::Database::Reindexing do end describe '.cleanup_leftovers!' do - subject { described_class.cleanup_leftovers! } + subject(:cleanup_leftovers) { described_class.cleanup_leftovers! } + + let(:model) { Gitlab::Database.database_base_models[Gitlab::Database::PRIMARY_DATABASE_NAME] } + let(:connection) { model.connection } + + around do |example| + Gitlab::Database::SharedModel.using_connection(connection) do + example.run + end + end before do - ApplicationRecord.connection.execute(<<~SQL) + connection.execute(<<~SQL) CREATE INDEX foobar_ccnew ON users (id); CREATE INDEX foobar_ccnew1 ON users (id); SQL @@ -150,11 +216,11 @@ RSpec.describe Gitlab::Database::Reindexing do expect_query("DROP INDEX CONCURRENTLY IF EXISTS \"public\".\"foobar_ccnew1\"") expect_query("RESET idle_in_transaction_session_timeout; RESET lock_timeout") - subject + cleanup_leftovers end def expect_query(sql) - expect(ApplicationRecord.connection).to receive(:execute).ordered.with(sql).and_wrap_original do |method, sql| + expect(connection).to receive(:execute).ordered.with(sql).and_wrap_original do |method, sql| method.call(sql.sub(/CONCURRENTLY/, '')) end end diff --git a/spec/lib/gitlab/database/shared_model_spec.rb b/spec/lib/gitlab/database/shared_model_spec.rb index 94f2b5a3434..54af4a0c4dc 100644 --- a/spec/lib/gitlab/database/shared_model_spec.rb +++ b/spec/lib/gitlab/database/shared_model_spec.rb @@ -84,4 +84,16 @@ RSpec.describe Gitlab::Database::SharedModel do expect(described_class.connection).to be(original_connection) end end + + describe '#connection_db_config' do + it 'returns the class connection_db_config' do + shared_model_class = Class.new(described_class) do + self.table_name = 'postgres_async_indexes' + end + + shared_model = shared_model_class.new + + expect(shared_model.connection_db_config). to eq(described_class.connection_db_config) + end + end end diff --git a/spec/lib/gitlab/database/type/json_pg_safe_spec.rb b/spec/lib/gitlab/database/type/json_pg_safe_spec.rb new file mode 100644 index 00000000000..91dc6f39aa7 --- /dev/null +++ b/spec/lib/gitlab/database/type/json_pg_safe_spec.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::Type::JsonPgSafe do + let(:type) { described_class.new } + + describe '#serialize' do + using RSpec::Parameterized::TableSyntax + + subject { type.serialize(value) } + + where(:value, :json) do + nil | nil + 1 | '1' + 1.0 | '1.0' + "str\0ing\u0000" | '"string"' + ["\0arr", "a\u0000y"] | '["arr","ay"]' + { "key\0" => "value\u0000\0" } | '{"key":"value"}' + end + + with_them do + it { is_expected.to eq(json) } + end + end +end diff --git a/spec/lib/gitlab/diff/custom_diff_spec.rb b/spec/lib/gitlab/diff/custom_diff_spec.rb new file mode 100644 index 00000000000..246508d2e1e --- /dev/null +++ b/spec/lib/gitlab/diff/custom_diff_spec.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Diff::CustomDiff do + include RepoHelpers + + let(:project) { create(:project, :repository) } + let(:repository) { project.repository } + let(:ipynb_blob) { repository.blob_at('f6b7a707', 'files/ipython/markdown-table.ipynb') } + let(:blob) { repository.blob_at('HEAD', 'files/ruby/regex.rb') } + + describe '#preprocess_before_diff' do + context 'for ipynb files' do + it 'transforms the diff' do + expect(described_class.preprocess_before_diff(ipynb_blob.path, nil, ipynb_blob)).not_to include('cells') + end + + it 'adds the blob to the list of transformed blobs' do + described_class.preprocess_before_diff(ipynb_blob.path, nil, ipynb_blob) + + expect(described_class.transformed_for_diff?(ipynb_blob)).to be_truthy + end + end + + context 'for other files' do + it 'returns nil' do + expect(described_class.preprocess_before_diff(blob.path, nil, blob)).to be_nil + end + + it 'does not add the blob to the list of transformed blobs' do + described_class.preprocess_before_diff(blob.path, nil, blob) + + expect(described_class.transformed_for_diff?(blob)).to be_falsey + end + end + end + + describe '#transformed_blob_data' do + it 'transforms blob data if file was processed' do + described_class.preprocess_before_diff(ipynb_blob.path, nil, ipynb_blob) + + expect(described_class.transformed_blob_data(ipynb_blob)).not_to include('cells') + end + + it 'does not transform blob data if file was not processed' do + expect(described_class.transformed_blob_data(ipynb_blob)).to be_nil + end + end + + describe '#transformed_blob_language' do + it 'is md when file was preprocessed' do + described_class.preprocess_before_diff(ipynb_blob.path, nil, ipynb_blob) + + expect(described_class.transformed_blob_language(ipynb_blob)).to eq('md') + end + + it 'is nil for a .ipynb blob that was not preprocessed' do + expect(described_class.transformed_blob_language(ipynb_blob)).to be_nil + end + end +end diff --git a/spec/lib/gitlab/diff/file_spec.rb b/spec/lib/gitlab/diff/file_spec.rb index 4b437397688..45a49a36fe2 100644 --- a/spec/lib/gitlab/diff/file_spec.rb +++ b/spec/lib/gitlab/diff/file_spec.rb @@ -53,7 +53,7 @@ RSpec.describe Gitlab::Diff::File do describe 'initialize' do context 'when file is ipynb with a change after transformation' do - let(:commit) { project.commit("f6b7a707") } + let(:commit) { project.commit("532c837") } let(:diff) { commit.raw_diffs.first } let(:diff_file) { described_class.new(diff, diff_refs: commit.diff_refs, repository: project.repository) } @@ -63,7 +63,7 @@ RSpec.describe Gitlab::Diff::File do end it 'recreates the diff by transforming the files' do - expect(diff_file.diff.diff).not_to include('"| Fake') + expect(diff_file.diff.diff).not_to include('cell_type') end end @@ -73,7 +73,7 @@ RSpec.describe Gitlab::Diff::File do end it 'does not recreate the diff' do - expect(diff_file.diff.diff).to include('"| Fake') + expect(diff_file.diff.diff).to include('cell_type') end end end diff --git a/spec/lib/gitlab/diff/highlight_spec.rb b/spec/lib/gitlab/diff/highlight_spec.rb index 94b28c38fa2..624160d2f48 100644 --- a/spec/lib/gitlab/diff/highlight_spec.rb +++ b/spec/lib/gitlab/diff/highlight_spec.rb @@ -151,20 +151,6 @@ RSpec.describe Gitlab::Diff::Highlight do expect(subject[2].rich_text).to eq(%Q{ <span id="LC7" class="line" lang=""> def popen(cmd, path=nil)</span>\n}) expect(subject[2].rich_text).to be_html_safe end - - context 'when limited_diff_highlighting is disabled' do - before do - stub_feature_flags(limited_diff_highlighting: false) - stub_feature_flags(diff_line_syntax_highlighting: false) - end - - it 'blobs are highlighted as plain text with loading all data' do - expect(diff_file.blob).to receive(:load_all_data!).twice - - code = %Q{ <span id="LC7" class="line" lang=""> def popen(cmd, path=nil)</span>\n} - expect(subject[2].rich_text).to eq(code) - end - end end end end diff --git a/spec/lib/gitlab/email/handler/create_issue_handler_spec.rb b/spec/lib/gitlab/email/handler/create_issue_handler_spec.rb index bd4f1d164a8..10098a66ae9 100644 --- a/spec/lib/gitlab/email/handler/create_issue_handler_spec.rb +++ b/spec/lib/gitlab/email/handler/create_issue_handler_spec.rb @@ -101,6 +101,17 @@ RSpec.describe Gitlab::Email::Handler::CreateIssueHandler do end end + context 'when all lines of email are quotes' do + let(:email_raw) { email_fixture('emails/valid_new_issue_with_only_quotes.eml') } + + it 'creates email with correct body' do + receiver.execute + + issue = Issue.last + expect(issue.description).to include('This email has been forwarded without new content.') + end + end + context "something is wrong" do context "when the issue could not be saved" do before do diff --git a/spec/lib/gitlab/email/handler/service_desk_handler_spec.rb b/spec/lib/gitlab/email/handler/service_desk_handler_spec.rb index c579027788d..7c34fb1a926 100644 --- a/spec/lib/gitlab/email/handler/service_desk_handler_spec.rb +++ b/spec/lib/gitlab/email/handler/service_desk_handler_spec.rb @@ -12,6 +12,8 @@ RSpec.describe Gitlab::Email::Handler::ServiceDeskHandler do let(:email_raw) { email_fixture('emails/service_desk.eml') } let(:author_email) { 'jake@adventuretime.ooo' } + let(:message_id) { 'CADkmRc+rNGAGGbV2iE5p918UVy4UyJqVcXRO2=otppgzduJSg@mail.gmail.com' } + let_it_be(:group) { create(:group, :private, name: "email") } let(:expected_description) do @@ -40,6 +42,7 @@ RSpec.describe Gitlab::Email::Handler::ServiceDeskHandler do expect(new_issue.all_references.all).to be_empty expect(new_issue.title).to eq("The message subject! @all") expect(new_issue.description).to eq(expected_description.strip) + expect(new_issue.email&.email_message_id).to eq(message_id) end it 'creates an issue_email_participant' do @@ -72,6 +75,95 @@ RSpec.describe Gitlab::Email::Handler::ServiceDeskHandler do it_behaves_like 'a new issue request' end + context 'when replying to issue creation email' do + def receive_reply + reply_email_raw = email_fixture('emails/service_desk_reply.eml') + + second_receiver = Gitlab::Email::Receiver.new(reply_email_raw) + second_receiver.execute + end + + context 'when an issue with message_id has been found' do + before do + receiver.execute + end + + subject do + receive_reply + end + + it 'does not create an additional issue' do + expect { subject }.not_to change { Issue.count } + end + + it 'adds a comment to the created issue' do + subject + + notes = Issue.last.notes + new_note = notes.first + + expect(notes.count).to eq(1) + expect(new_note.note).to eq("Service desk reply!\n\n`/label ~label2`") + expect(new_note.author).to eql(User.support_bot) + end + + it 'does not send thank you email' do + expect(Notify).not_to receive(:service_desk_thank_you_email) + + subject + end + + context 'when issue_email_participants FF is enabled' do + it 'creates 2 issue_email_participants' do + subject + + expect(Issue.last.issue_email_participants.map(&:email)) + .to match_array(%w(alan@adventuretime.ooo jake@adventuretime.ooo)) + end + end + + context 'when issue_email_participants FF is disabled' do + before do + stub_feature_flags(issue_email_participants: false) + end + + it 'creates only 1 issue_email_participant' do + subject + + expect(Issue.last.issue_email_participants.map(&:email)) + .to match_array(%w(jake@adventuretime.ooo)) + end + end + end + + context 'when an issue with message_id has not been found' do + subject do + receive_reply + end + + it 'creates a new issue correctly' do + expect { subject }.to change { Issue.count }.by(1) + + issue = Issue.last + + expect(issue.description).to eq("Service desk reply!\n\n`/label ~label2`") + end + + it 'sends thank you email once' do + expect(Notify).to receive(:service_desk_thank_you_email).once.and_return(double(deliver_later: true)) + + subject + end + + it 'creates 1 issue_email_participant' do + subject + + expect(Issue.last.issue_email_participants.map(&:email)) + .to match_array(%w(alan@adventuretime.ooo)) + end + end + end + context 'when using issue templates' do let_it_be(:user) { create(:user) } @@ -270,6 +362,20 @@ RSpec.describe Gitlab::Email::Handler::ServiceDeskHandler do end end + context 'when issue email creation fails' do + before do + allow(::Issue::Email).to receive(:create!).and_raise(StandardError) + end + + it 'still creates a new issue' do + expect { receiver.execute }.to change { Issue.count }.by(1) + end + + it 'does not create issue email record' do + expect { receiver.execute }.not_to change { Issue::Email.count } + end + end + context 'when rate limiting is in effect', :freeze_time, :clean_gitlab_redis_rate_limiting do let(:receiver) { Gitlab::Email::Receiver.new(email_raw) } @@ -291,19 +397,19 @@ RSpec.describe Gitlab::Email::Handler::ServiceDeskHandler do rescue RateLimitedService::RateLimitedError end.to change { Issue.count }.by(1) end + end - context 'when requests are sent by different users' do - let(:email_raw_2) { email_fixture('emails/service_desk_forwarded.eml') } - let(:receiver2) { Gitlab::Email::Receiver.new(email_raw_2) } + context 'when requests are sent by different users' do + let(:email_raw_2) { email_fixture('emails/service_desk_forwarded.eml') } + let(:receiver2) { Gitlab::Email::Receiver.new(email_raw_2) } - subject do - receiver.execute - receiver2.execute - end + subject do + receiver.execute + receiver2.execute + end - it 'creates 2 issues' do - expect { subject }.to change { Issue.count }.by(2) - end + it 'creates 2 issues' do + expect { subject }.to change { Issue.count }.by(2) end end @@ -389,6 +495,7 @@ RSpec.describe Gitlab::Email::Handler::ServiceDeskHandler do context 'when the email is forwarded through an alias' do let(:author_email) { 'jake.g@adventuretime.ooo' } let(:email_raw) { email_fixture('emails/service_desk_forwarded.eml') } + let(:message_id) { 'CADkmRc+rNGAGGbV2iE5p918UVy4UyJqVcXRO2=fdskbsf@mail.gmail.com' } it_behaves_like 'a new issue request' end diff --git a/spec/lib/gitlab/email/hook/smime_signature_interceptor_spec.rb b/spec/lib/gitlab/email/hook/smime_signature_interceptor_spec.rb index 352eb596cd9..7dd4ee7e25d 100644 --- a/spec/lib/gitlab/email/hook/smime_signature_interceptor_spec.rb +++ b/spec/lib/gitlab/email/hook/smime_signature_interceptor_spec.rb @@ -50,6 +50,7 @@ RSpec.describe Gitlab::Email::Hook::SmimeSignatureInterceptor do expect(mail.header['To'].value).to eq('test@example.com') expect(mail.header['From'].value).to eq('info@example.com') expect(mail.header['Content-Type'].value).to match('multipart/signed').and match('protocol="application/x-pkcs7-signature"') + expect(mail.header.include?('Content-Disposition')).to eq(false) # verify signature and obtain pkcs7 encoded content p7enc = Gitlab::Email::Smime::Signer.verify_signature( diff --git a/spec/lib/gitlab/email/service_desk_receiver_spec.rb b/spec/lib/gitlab/email/service_desk_receiver_spec.rb index 6ba58ad5e93..49cbec6fffc 100644 --- a/spec/lib/gitlab/email/service_desk_receiver_spec.rb +++ b/spec/lib/gitlab/email/service_desk_receiver_spec.rb @@ -9,9 +9,7 @@ RSpec.describe Gitlab::Email::ServiceDeskReceiver do context 'when the email contains a valid email address' do before do stub_service_desk_email_setting(enabled: true, address: 'support+%{key}@example.com') - end - it 'finds the service desk key' do handler = double(execute: true, metrics_event: true, metrics_params: true) expected_params = [ an_instance_of(Mail::Message), nil, @@ -20,8 +18,38 @@ RSpec.describe Gitlab::Email::ServiceDeskReceiver do expect(Gitlab::Email::Handler::ServiceDeskHandler) .to receive(:new).with(*expected_params).and_return(handler) + end + + context 'when in a To header' do + it 'finds the service desk key' do + receiver.execute + end + end + + context 'when the email contains a valid email address in a header' do + context 'when in a Delivered-To header' do + let(:email) { fixture_file('emails/service_desk_custom_address_reply.eml') } + + it 'finds the service desk key' do + receiver.execute + end + end + + context 'when in a Envelope-To header' do + let(:email) { fixture_file('emails/service_desk_custom_address_envelope_to.eml') } + + it 'finds the service desk key' do + receiver.execute + end + end + + context 'when in a X-Envelope-To header' do + let(:email) { fixture_file('emails/service_desk_custom_address_x_envelope_to.eml') } - receiver.execute + it 'finds the service desk key' do + receiver.execute + end + end end end diff --git a/spec/lib/gitlab/empty_search_results_spec.rb b/spec/lib/gitlab/empty_search_results_spec.rb new file mode 100644 index 00000000000..e79586bef68 --- /dev/null +++ b/spec/lib/gitlab/empty_search_results_spec.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::EmptySearchResults do + subject { described_class.new } + + describe '#objects' do + it 'returns an empty array' do + expect(subject.objects).to match_array([]) + end + end + + describe '#formatted_count' do + it 'returns a zero' do + expect(subject.formatted_count).to eq('0') + end + end + + describe '#highlight_map' do + it 'returns an empty hash' do + expect(subject.highlight_map).to eq({}) + end + end + + describe '#aggregations' do + it 'returns an empty array' do + expect(subject.objects).to match_array([]) + end + end +end diff --git a/spec/lib/gitlab/error_tracking_spec.rb b/spec/lib/gitlab/error_tracking_spec.rb index 7ad1f52780a..a5d44963f4b 100644 --- a/spec/lib/gitlab/error_tracking_spec.rb +++ b/spec/lib/gitlab/error_tracking_spec.rb @@ -205,16 +205,6 @@ RSpec.describe Gitlab::ErrorTracking do expect(sentry_event.dig('extra', 'sql')).to eq('SELECT "users".* FROM "users" WHERE "users"."id" = $2 AND "users"."foo" = $1') end end - - context 'when the `ActiveRecord::StatementInvalid` is wrapped in another exception' do - it 'injects the normalized sql query into extra' do - allow(exception).to receive(:cause).and_return(ActiveRecord::StatementInvalid.new(sql: 'SELECT "users".* FROM "users" WHERE "users"."id" = 1 AND "users"."foo" = $1')) - - track_exception - - expect(sentry_event.dig('extra', 'sql')).to eq('SELECT "users".* FROM "users" WHERE "users"."id" = $2 AND "users"."foo" = $1') - end - end end context 'event processors' do diff --git a/spec/lib/gitlab/etag_caching/store_spec.rb b/spec/lib/gitlab/etag_caching/store_spec.rb index 46195e64715..6188a3fc8b3 100644 --- a/spec/lib/gitlab/etag_caching/store_spec.rb +++ b/spec/lib/gitlab/etag_caching/store_spec.rb @@ -80,5 +80,19 @@ RSpec.describe Gitlab::EtagCaching::Store, :clean_gitlab_redis_shared_state do expect(store.get(key)).to eq(etag) end end + + context 'with multiple keys' do + let(:keys) { ['/my-group/my-project/builds/234.json', '/api/graphql:pipelines/id/5'] } + + it 'stores and returns multiple values' do + etags = store.touch(*keys) + + expect(etags.size).to eq(keys.size) + + keys.each_with_index do |key, i| + expect(store.get(key)).to eq(etags[i]) + end + end + end end end diff --git a/spec/lib/gitlab/exception_log_formatter_spec.rb b/spec/lib/gitlab/exception_log_formatter_spec.rb new file mode 100644 index 00000000000..beeeeb2b64c --- /dev/null +++ b/spec/lib/gitlab/exception_log_formatter_spec.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::ExceptionLogFormatter do + describe '.format!' do + let(:exception) { RuntimeError.new('bad request') } + let(:backtrace) { caller } + + let(:payload) { {} } + + before do + allow(exception).to receive(:backtrace).and_return(backtrace) + end + + it 'adds exception data to log' do + described_class.format!(exception, payload) + + expect(payload['exception.class']).to eq('RuntimeError') + expect(payload['exception.message']).to eq('bad request') + expect(payload['exception.backtrace']).to eq(Gitlab::BacktraceCleaner.clean_backtrace(backtrace)) + expect(payload['exception.sql']).to be_nil + end + + context 'when exception is ActiveRecord::StatementInvalid' do + let(:exception) { ActiveRecord::StatementInvalid.new(sql: 'SELECT "users".* FROM "users" WHERE "users"."id" = 1 AND "users"."foo" = $1') } + + it 'adds the normalized SQL query to payload' do + described_class.format!(exception, payload) + + expect(payload['exception.sql']).to eq('SELECT "users".* FROM "users" WHERE "users"."id" = $2 AND "users"."foo" = $1') + end + end + + context 'when the ActiveRecord::StatementInvalid is wrapped in another exception' do + before do + allow(exception).to receive(:cause).and_return(ActiveRecord::StatementInvalid.new(sql: 'SELECT "users".* FROM "users" WHERE "users"."id" = 1 AND "users"."foo" = $1')) + end + + it 'adds the normalized SQL query to payload' do + described_class.format!(exception, payload) + + expect(payload['exception.sql']).to eq('SELECT "users".* FROM "users" WHERE "users"."id" = $2 AND "users"."foo" = $1') + end + end + + context 'when the ActiveRecord::StatementInvalid is a bad query' do + let(:exception) { ActiveRecord::StatementInvalid.new(sql: 'SELECT SELECT FROM SELECT') } + + it 'adds the query as-is to payload' do + described_class.format!(exception, payload) + + expect(payload['exception.sql']).to eq('SELECT SELECT FROM SELECT') + end + end + end +end diff --git a/spec/lib/gitlab/experimentation/controller_concern_spec.rb b/spec/lib/gitlab/experimentation/controller_concern_spec.rb index 1f7b7b90467..8a96771eeb8 100644 --- a/spec/lib/gitlab/experimentation/controller_concern_spec.rb +++ b/spec/lib/gitlab/experimentation/controller_concern_spec.rb @@ -97,7 +97,7 @@ RSpec.describe Gitlab::Experimentation::ControllerConcern, type: :controller do describe '#push_frontend_experiment' do it 'pushes an experiment to the frontend' do - gon = instance_double('gon') + gon = class_double('Gon') stub_experiment_for_subject(my_experiment: true) allow(controller).to receive(:gon).and_return(gon) diff --git a/spec/lib/gitlab/experimentation/experiment_spec.rb b/spec/lib/gitlab/experimentation/experiment_spec.rb index d52ab3a8983..d9bf85460b3 100644 --- a/spec/lib/gitlab/experimentation/experiment_spec.rb +++ b/spec/lib/gitlab/experimentation/experiment_spec.rb @@ -16,6 +16,7 @@ RSpec.describe Gitlab::Experimentation::Experiment do before do skip_feature_flags_yaml_validation skip_default_enabled_yaml_check + allow(Feature).to receive(:log_feature_flag_states?).and_return(false) feature = double('FeatureFlag', percentage_of_time_value: percentage, enabled?: true) allow(Feature).to receive(:get).with(:experiment_key_experiment_percentage).and_return(feature) end diff --git a/spec/lib/gitlab/git/diff_stats_collection_spec.rb b/spec/lib/gitlab/git/diff_stats_collection_spec.rb index f2fe03829be..0876a88a2ee 100644 --- a/spec/lib/gitlab/git/diff_stats_collection_spec.rb +++ b/spec/lib/gitlab/git/diff_stats_collection_spec.rb @@ -36,7 +36,7 @@ RSpec.describe Gitlab::Git::DiffStatsCollection do end it 'returns capped number when it is bigger than max_files' do - allow(::Commit).to receive(:max_diff_options).and_return(max_files: 1) + allow(::Commit).to receive(:diff_max_files).and_return(1) expect(collection.real_size).to eq('1+') end diff --git a/spec/lib/gitlab/git/rugged_impl/use_rugged_spec.rb b/spec/lib/gitlab/git/rugged_impl/use_rugged_spec.rb index 16cea1dc1a3..b2603e099e6 100644 --- a/spec/lib/gitlab/git/rugged_impl/use_rugged_spec.rb +++ b/spec/lib/gitlab/git/rugged_impl/use_rugged_spec.rb @@ -110,7 +110,7 @@ RSpec.describe Gitlab::Git::RuggedImpl::UseRugged, :seed_helper do describe '#running_puma_with_multiple_threads?' do context 'when using Puma' do before do - stub_const('::Puma', class_double('Puma')) + stub_const('::Puma', double('puma constant')) allow(Gitlab::Runtime).to receive(:puma?).and_return(true) end diff --git a/spec/lib/gitlab/git/tree_spec.rb b/spec/lib/gitlab/git/tree_spec.rb index 005f8ecaa3a..97ba177da71 100644 --- a/spec/lib/gitlab/git/tree_spec.rb +++ b/spec/lib/gitlab/git/tree_spec.rb @@ -43,7 +43,7 @@ RSpec.describe Gitlab::Git::Tree, :seed_helper do end describe '#dir?' do - let(:dir) { entries.select(&:dir?).first } + let(:dir) { entries.find(&:dir?) } it { expect(dir).to be_kind_of Gitlab::Git::Tree } it { expect(dir.id).to eq('3c122d2b7830eca25235131070602575cf8b41a1') } @@ -134,7 +134,7 @@ RSpec.describe Gitlab::Git::Tree, :seed_helper do end describe '#file?' do - let(:file) { entries.select(&:file?).first } + let(:file) { entries.find(&:file?) } it { expect(file).to be_kind_of Gitlab::Git::Tree } it { expect(file.id).to eq('dfaa3f97ca337e20154a98ac9d0be76ddd1fcc82') } @@ -143,21 +143,21 @@ RSpec.describe Gitlab::Git::Tree, :seed_helper do end describe '#readme?' do - let(:file) { entries.select(&:readme?).first } + let(:file) { entries.find(&:readme?) } it { expect(file).to be_kind_of Gitlab::Git::Tree } it { expect(file.name).to eq('README.md') } end describe '#contributing?' do - let(:file) { entries.select(&:contributing?).first } + let(:file) { entries.find(&:contributing?) } it { expect(file).to be_kind_of Gitlab::Git::Tree } it { expect(file.name).to eq('CONTRIBUTING.md') } end describe '#submodule?' do - let(:submodule) { entries.select(&:submodule?).first } + let(:submodule) { entries.find(&:submodule?) } it { expect(submodule).to be_kind_of Gitlab::Git::Tree } it { expect(submodule.id).to eq('79bceae69cb5750d6567b223597999bfa91cb3b9') } diff --git a/spec/lib/gitlab/gitaly_client/commit_service_spec.rb b/spec/lib/gitlab/gitaly_client/commit_service_spec.rb index d8e397dd6f3..8d9ab5db886 100644 --- a/spec/lib/gitlab/gitaly_client/commit_service_spec.rb +++ b/spec/lib/gitlab/gitaly_client/commit_service_spec.rb @@ -13,10 +13,6 @@ RSpec.describe Gitlab::GitalyClient::CommitService do let(:client) { described_class.new(repository) } describe '#diff_from_parent' do - before do - stub_feature_flags(increased_diff_limits: false) - end - context 'when a commit has a parent' do it 'sends an RPC request with the parent ID as left commit' do request = Gitaly::CommitDiffRequest.new( @@ -108,45 +104,6 @@ RSpec.describe Gitlab::GitalyClient::CommitService do end end - describe '#between' do - let(:from) { 'master' } - let(:to) { Gitlab::Git::EMPTY_TREE_ID } - - context 'with between_commits_via_list_commits enabled' do - before do - stub_feature_flags(between_commits_via_list_commits: true) - end - - it 'sends an RPC request' do - request = Gitaly::ListCommitsRequest.new( - repository: repository_message, revisions: ["^" + from, to], reverse: true - ) - - expect_any_instance_of(Gitaly::CommitService::Stub).to receive(:list_commits) - .with(request, kind_of(Hash)).and_return([]) - - described_class.new(repository).between(from, to) - end - end - - context 'with between_commits_via_list_commits disabled' do - before do - stub_feature_flags(between_commits_via_list_commits: false) - end - - it 'sends an RPC request' do - request = Gitaly::CommitsBetweenRequest.new( - repository: repository_message, from: from, to: to - ) - - expect_any_instance_of(Gitaly::CommitService::Stub).to receive(:commits_between) - .with(request, kind_of(Hash)).and_return([]) - - described_class.new(repository).between(from, to) - end - end - end - describe '#diff_stats' do let(:left_commit_id) { 'master' } let(:right_commit_id) { 'cfe32cf61b73a0d5e9f13e774abde7ff789b1660' } diff --git a/spec/lib/gitlab/gitaly_client/conflict_files_stitcher_spec.rb b/spec/lib/gitlab/gitaly_client/conflict_files_stitcher_spec.rb index 0eecdfcb630..d0787d8b673 100644 --- a/spec/lib/gitlab/gitaly_client/conflict_files_stitcher_spec.rb +++ b/spec/lib/gitlab/gitaly_client/conflict_files_stitcher_spec.rb @@ -43,10 +43,10 @@ RSpec.describe Gitlab::GitalyClient::ConflictFilesStitcher do messages = [ double(files: [double(header: header_1), double(header: nil, content: content_1[0..5])]), - double(files: [double(header: nil, content: content_1[6..-1])]), + double(files: [double(header: nil, content: content_1[6..])]), double(files: [double(header: header_2)]), double(files: [double(header: nil, content: content_2[0..5]), double(header: nil, content: content_2[6..10])]), - double(files: [double(header: nil, content: content_2[11..-1])]) + double(files: [double(header: nil, content: content_2[11..])]) ] conflict_files = described_class.new(messages, target_repository.gitaly_repository).to_a diff --git a/spec/lib/gitlab/gitaly_client/diff_stitcher_spec.rb b/spec/lib/gitlab/gitaly_client/diff_stitcher_spec.rb index 113c47b4f2c..54c84ddc56f 100644 --- a/spec/lib/gitlab/gitaly_client/diff_stitcher_spec.rb +++ b/spec/lib/gitlab/gitaly_client/diff_stitcher_spec.rb @@ -41,7 +41,7 @@ RSpec.describe Gitlab::GitalyClient::DiffStitcher do msg_2.raw_patch_data = diff_2.patch[0..100] msg_2.end_of_patch = false - msg_3 = OpenStruct.new(raw_patch_data: diff_2.patch[101..-1], end_of_patch: true) + msg_3 = OpenStruct.new(raw_patch_data: diff_2.patch[101..], end_of_patch: true) msg_4 = OpenStruct.new(diff_3.to_h.except(:patch)) msg_4.raw_patch_data = diff_3.patch diff --git a/spec/lib/gitlab/github_import/client_spec.rb b/spec/lib/gitlab/github_import/client_spec.rb index 194dfb228ee..c4d05e92633 100644 --- a/spec/lib/gitlab/github_import/client_spec.rb +++ b/spec/lib/gitlab/github_import/client_spec.rb @@ -221,6 +221,50 @@ RSpec.describe Gitlab::GithubImport::Client do expect(client.with_rate_limit { 10 }).to eq(10) end + + context 'when Faraday error received from octokit', :aggregate_failures do + let(:error_class) { described_class::CLIENT_CONNECTION_ERROR } + let(:info_params) { { 'error.class': error_class } } + let(:block_to_rate_limit) { -> { client.pull_request('foo/bar', 999) } } + + context 'when rate_limiting_enabled is true' do + it 'retries on error and succeeds' do + allow_retry + + expect(client).to receive(:requests_remaining?).twice.and_return(true) + expect(Gitlab::Import::Logger).to receive(:info).with(hash_including(info_params)).once + + expect(client.with_rate_limit(&block_to_rate_limit)).to be(true) + end + + it 'retries and does not succeed' do + allow(client).to receive(:requests_remaining?).and_return(true) + allow(client.octokit).to receive(:pull_request).and_raise(error_class, 'execution expired') + + expect { client.with_rate_limit(&block_to_rate_limit) }.to raise_error(error_class, 'execution expired') + end + end + + context 'when rate_limiting_enabled is false' do + before do + allow(client).to receive(:rate_limiting_enabled?).and_return(false) + end + + it 'retries on error and succeeds' do + allow_retry + + expect(Gitlab::Import::Logger).to receive(:info).with(hash_including(info_params)).once + + expect(client.with_rate_limit(&block_to_rate_limit)).to be(true) + end + + it 'retries and does not succeed' do + allow(client.octokit).to receive(:pull_request).and_raise(error_class, 'execution expired') + + expect { client.with_rate_limit(&block_to_rate_limit) }.to raise_error(error_class, 'execution expired') + end + end + end end describe '#requests_remaining?' do @@ -505,6 +549,25 @@ RSpec.describe Gitlab::GithubImport::Client do client.search_repos_by_name('test') end + + context 'when Faraday error received from octokit', :aggregate_failures do + let(:error_class) { described_class::CLIENT_CONNECTION_ERROR } + let(:info_params) { { 'error.class': error_class } } + + it 'retries on error and succeeds' do + allow_retry(:search_repositories) + + expect(Gitlab::Import::Logger).to receive(:info).with(hash_including(info_params)).once + + expect(client.search_repos_by_name('test')).to be(true) + end + + it 'retries and does not succeed' do + allow(client.octokit).to receive(:search_repositories).and_raise(error_class, 'execution expired') + + expect { client.search_repos_by_name('test') }.to raise_error(error_class, 'execution expired') + end + end end describe '#search_query' do @@ -531,4 +594,12 @@ RSpec.describe Gitlab::GithubImport::Client do end end end + + def allow_retry(method = :pull_request) + call_count = 0 + allow(client.octokit).to receive(method) do + call_count += 1 + call_count > 1 ? true : raise(described_class::CLIENT_CONNECTION_ERROR, 'execution expired') + end + end end diff --git a/spec/lib/gitlab/github_import/importer/diff_note_importer_spec.rb b/spec/lib/gitlab/github_import/importer/diff_note_importer_spec.rb index 0448ada6bca..a0e78186caa 100644 --- a/spec/lib/gitlab/github_import/importer/diff_note_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/diff_note_importer_spec.rb @@ -173,9 +173,11 @@ RSpec.describe Gitlab::GithubImport::Importer::DiffNoteImporter, :aggregate_fail EOB end - it 'imports the note as diff note' do + before do stub_user_finder(user.id, true) + end + it 'imports the note as diff note' do expect { subject.execute } .to change(DiffNote, :count) .by(1) @@ -212,6 +214,29 @@ RSpec.describe Gitlab::GithubImport::Importer::DiffNoteImporter, :aggregate_fail ``` NOTE end + + context 'when the note diff file creation fails' do + it 'falls back to the LegacyDiffNote' do + exception = ::DiffNote::NoteDiffFileCreationError.new('Failed to create diff note file') + + expect_next_instance_of(::Import::Github::Notes::CreateService) do |service| + expect(service) + .to receive(:execute) + .and_raise(exception) + end + + expect(Gitlab::GithubImport::Logger) + .to receive(:warn) + .with( + message: 'Failed to create diff note file', + 'error.class': 'DiffNote::NoteDiffFileCreationError' + ) + + expect { subject.execute } + .to change(LegacyDiffNote, :count) + .and not_change(DiffNote, :count) + end + end end end end diff --git a/spec/lib/gitlab/github_import/importer/note_importer_spec.rb b/spec/lib/gitlab/github_import/importer/note_importer_spec.rb index 96d8acbd3de..165f543525d 100644 --- a/spec/lib/gitlab/github_import/importer/note_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/note_importer_spec.rb @@ -52,6 +52,7 @@ RSpec.describe Gitlab::GithubImport::Importer::NoteImporter do project_id: project.id, author_id: user.id, note: 'This is my note', + discussion_id: match(/\A[0-9a-f]{40}\z/), system: false, created_at: created_at, updated_at: updated_at @@ -82,6 +83,7 @@ RSpec.describe Gitlab::GithubImport::Importer::NoteImporter do project_id: project.id, author_id: project.creator_id, note: "*Created by: alice*\n\nThis is my note", + discussion_id: match(/\A[0-9a-f]{40}\z/), system: false, created_at: created_at, updated_at: updated_at diff --git a/spec/lib/gitlab/github_import/parallel_importer_spec.rb b/spec/lib/gitlab/github_import/parallel_importer_spec.rb index c7b300ff043..d418e87284d 100644 --- a/spec/lib/gitlab/github_import/parallel_importer_spec.rb +++ b/spec/lib/gitlab/github_import/parallel_importer_spec.rb @@ -27,8 +27,13 @@ RSpec.describe Gitlab::GithubImport::ParallelImporter do before do create(:import_state, :started, project: project) + worker = double(:worker) expect(Gitlab::GithubImport::Stage::ImportRepositoryWorker) + .to receive(:with_status) + .and_return(worker) + + expect(worker) .to receive(:perform_async) .with(project.id) .and_return('123') diff --git a/spec/lib/gitlab/gon_helper_spec.rb b/spec/lib/gitlab/gon_helper_spec.rb index 3d3f381b6d2..b8ed4cf608d 100644 --- a/spec/lib/gitlab/gon_helper_spec.rb +++ b/spec/lib/gitlab/gon_helper_spec.rb @@ -15,7 +15,7 @@ RSpec.describe Gitlab::GonHelper do end it 'pushes a feature flag to the frontend' do - gon = instance_double('gon') + gon = class_double('Gon') thing = stub_feature_flag_gate('thing') stub_feature_flags(my_feature_flag: thing) diff --git a/spec/lib/gitlab/gpg/invalid_gpg_signature_updater_spec.rb b/spec/lib/gitlab/gpg/invalid_gpg_signature_updater_spec.rb index 771f6e1ec46..5d444775e53 100644 --- a/spec/lib/gitlab/gpg/invalid_gpg_signature_updater_spec.rb +++ b/spec/lib/gitlab/gpg/invalid_gpg_signature_updater_spec.rb @@ -192,7 +192,7 @@ RSpec.describe Gitlab::Gpg::InvalidGpgSignatureUpdater do project: project, commit_sha: commit_sha, gpg_key: nil, - gpg_key_primary_keyid: GpgHelpers::User3.subkey_fingerprints.last[24..-1], + gpg_key_primary_keyid: GpgHelpers::User3.subkey_fingerprints.last[24..], verification_status: 'unknown_key' end diff --git a/spec/lib/gitlab/grape_logging/loggers/exception_logger_spec.rb b/spec/lib/gitlab/grape_logging/loggers/exception_logger_spec.rb index 3ce09740ec8..968d938a911 100644 --- a/spec/lib/gitlab/grape_logging/loggers/exception_logger_spec.rb +++ b/spec/lib/gitlab/grape_logging/loggers/exception_logger_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::GrapeLogging::Loggers::ExceptionLogger do - let(:mock_request) { OpenStruct.new(env: {}) } + let(:mock_request) { double('env', env: {}) } let(:response_body) { nil } describe ".parameters" do @@ -76,7 +76,7 @@ RSpec.describe Gitlab::GrapeLogging::Loggers::ExceptionLogger do describe 'when an exception is available' do let(:exception) { RuntimeError.new('This is a test') } let(:mock_request) do - OpenStruct.new( + double('env', env: { ::API::Helpers::API_EXCEPTION_ENV => exception } diff --git a/spec/lib/gitlab/graphql/pagination/keyset/connection_generic_keyset_spec.rb b/spec/lib/gitlab/graphql/pagination/keyset/connection_generic_keyset_spec.rb index 0047d24a215..0741088c915 100644 --- a/spec/lib/gitlab/graphql/pagination/keyset/connection_generic_keyset_spec.rb +++ b/spec/lib/gitlab/graphql/pagination/keyset/connection_generic_keyset_spec.rb @@ -9,7 +9,7 @@ RSpec.describe Gitlab::Graphql::Pagination::Keyset::Connection do let(:arguments) { {} } let(:query_type) { GraphQL::ObjectType.new } let(:schema) { GraphQL::Schema.define(query: query_type, mutation: nil)} - let(:context) { GraphQL::Query::Context.new(query: OpenStruct.new(schema: schema), values: nil, object: nil) } + let(:context) { GraphQL::Query::Context.new(query: double('query', schema: schema), values: nil, object: nil) } let_it_be(:column_order_id) { Gitlab::Pagination::Keyset::ColumnOrderDefinition.new(attribute_name: 'id', order_expression: Project.arel_table[:id].asc) } let_it_be(:column_order_id_desc) { Gitlab::Pagination::Keyset::ColumnOrderDefinition.new(attribute_name: 'id', order_expression: Project.arel_table[:id].desc) } @@ -98,7 +98,7 @@ RSpec.describe Gitlab::Graphql::Pagination::Keyset::Connection do let(:nodes) { Project.all.order(Gitlab::Pagination::Keyset::Order.build([column_order_id_desc])) } it 'returns the correct nodes' do - expect(subject.sliced_nodes).to contain_exactly(*projects[2..-1]) + expect(subject.sliced_nodes).to contain_exactly(*projects[2..]) end end end @@ -107,7 +107,7 @@ RSpec.describe Gitlab::Graphql::Pagination::Keyset::Connection do let(:arguments) { { after: encoded_cursor(projects[1]) } } it 'only returns the project before the selected one' do - expect(subject.sliced_nodes).to contain_exactly(*projects[2..-1]) + expect(subject.sliced_nodes).to contain_exactly(*projects[2..]) end context 'when the sort order is descending' do diff --git a/spec/lib/gitlab/graphql/pagination/keyset/connection_spec.rb b/spec/lib/gitlab/graphql/pagination/keyset/connection_spec.rb index 8ef5f1147c5..b511a294f97 100644 --- a/spec/lib/gitlab/graphql/pagination/keyset/connection_spec.rb +++ b/spec/lib/gitlab/graphql/pagination/keyset/connection_spec.rb @@ -7,7 +7,7 @@ RSpec.describe Gitlab::Graphql::Pagination::Keyset::Connection do let(:arguments) { {} } let(:query_type) { GraphQL::ObjectType.new } let(:schema) { GraphQL::Schema.define(query: query_type, mutation: nil)} - let(:context) { GraphQL::Query::Context.new(query: OpenStruct.new(schema: schema), values: nil, object: nil) } + let(:context) { GraphQL::Query::Context.new(query: double('query', schema: schema), values: nil, object: nil) } subject(:connection) do described_class.new(nodes, **{ context: context, max_page_size: 3 }.merge(arguments)) @@ -120,7 +120,7 @@ RSpec.describe Gitlab::Graphql::Pagination::Keyset::Connection do let(:nodes) { Project.all.order(id: :desc) } it 'returns the correct nodes' do - expect(subject.sliced_nodes).to contain_exactly(*projects[2..-1]) + expect(subject.sliced_nodes).to contain_exactly(*projects[2..]) end end end @@ -129,7 +129,7 @@ RSpec.describe Gitlab::Graphql::Pagination::Keyset::Connection do let(:arguments) { { after: encoded_cursor(projects[1]) } } it 'only returns the project before the selected one' do - expect(subject.sliced_nodes).to contain_exactly(*projects[2..-1]) + expect(subject.sliced_nodes).to contain_exactly(*projects[2..]) end context 'when the sort order is descending' do diff --git a/spec/lib/gitlab/graphql/tracers/logger_tracer_spec.rb b/spec/lib/gitlab/graphql/tracers/logger_tracer_spec.rb index d83ac4dabc5..5bc077a963e 100644 --- a/spec/lib/gitlab/graphql/tracers/logger_tracer_spec.rb +++ b/spec/lib/gitlab/graphql/tracers/logger_tracer_spec.rb @@ -1,6 +1,5 @@ # frozen_string_literal: true -require "fast_spec_helper" -require "support/graphql/fake_query_type" +require "spec_helper" RSpec.describe Gitlab::Graphql::Tracers::LoggerTracer do let(:dummy_schema) do @@ -49,4 +48,15 @@ RSpec.describe Gitlab::Graphql::Tracers::LoggerTracer do dummy_schema.execute(query_string, variables: variables) end + + it 'logs exceptions for breaking queries' do + query_string = "query fooOperation { breakingField }" + + expect(::Gitlab::GraphqlLogger).to receive(:info).with(a_hash_including({ + 'exception.message' => 'This field is supposed to break', + 'exception.class' => 'RuntimeError' + })) + + expect { dummy_schema.execute(query_string) }.to raise_error(/This field is supposed to break/) + end end diff --git a/spec/lib/gitlab/graphql/tracers/metrics_tracer_spec.rb b/spec/lib/gitlab/graphql/tracers/metrics_tracer_spec.rb index ff6a76aa319..168f5aa529e 100644 --- a/spec/lib/gitlab/graphql/tracers/metrics_tracer_spec.rb +++ b/spec/lib/gitlab/graphql/tracers/metrics_tracer_spec.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require 'fast_spec_helper' +require 'spec_helper' require 'rspec-parameterized' require "support/graphql/fake_query_type" @@ -36,7 +36,7 @@ RSpec.describe Gitlab::Graphql::Tracers::MetricsTracer do end with_them do - it 'increments sli' do + it 'increments apdex sli' do # Trigger initialization fake_schema @@ -56,5 +56,13 @@ RSpec.describe Gitlab::Graphql::Tracers::MetricsTracer do fake_schema.execute("query lorem { helloWorld }") end end + + it "does not record apdex for failing queries" do + query_string = "query fooOperation { breakingField }" + + expect(Gitlab::Metrics::RailsSlis.graphql_query_apdex).not_to receive(:increment) + + expect { fake_schema.execute(query_string) }.to raise_error(/This field is supposed to break/) + end end end diff --git a/spec/lib/gitlab/graphql/tracers/timer_tracer_spec.rb b/spec/lib/gitlab/graphql/tracers/timer_tracer_spec.rb index 7f837e28772..986120dcd95 100644 --- a/spec/lib/gitlab/graphql/tracers/timer_tracer_spec.rb +++ b/spec/lib/gitlab/graphql/tracers/timer_tracer_spec.rb @@ -20,6 +20,7 @@ RSpec.describe Gitlab::Graphql::Tracers::TimerTracer do before do current_time = 0 + allow(tracer_spy).to receive(:trace) allow(Gitlab::Metrics::System).to receive(:monotonic_time) do current_time += expected_duration end @@ -30,6 +31,18 @@ RSpec.describe Gitlab::Graphql::Tracers::TimerTracer do dummy_schema.execute(query_string) + expect_to_have_traced(tracer_spy, expected_duration, query_string) + end + + it "adds a duration_s even if the query failed" do + query_string = "query fooOperation { breakingField }" + + expect { dummy_schema.execute(query_string) }.to raise_error(/This field is supposed to break/) + + expect_to_have_traced(tracer_spy, expected_duration, query_string) + end + + def expect_to_have_traced(tracer_spy, expected_duration, query_string) # "parse" and "execute_query" are just arbitrary trace events expect(tracer_spy).to have_received(:trace).with("parse", { duration_s: expected_duration, diff --git a/spec/lib/gitlab/hook_data/merge_request_builder_spec.rb b/spec/lib/gitlab/hook_data/merge_request_builder_spec.rb index 9e6ad35861f..ddd681f75f0 100644 --- a/spec/lib/gitlab/hook_data/merge_request_builder_spec.rb +++ b/spec/lib/gitlab/hook_data/merge_request_builder_spec.rb @@ -15,6 +15,7 @@ RSpec.describe Gitlab::HookData::MergeRequestBuilder do assignee_id assignee_ids author_id + blocking_discussions_resolved created_at description head_pipeline_id diff --git a/spec/lib/gitlab/import/import_failure_service_spec.rb b/spec/lib/gitlab/import/import_failure_service_spec.rb index c16d4a7c804..e3fec63adde 100644 --- a/spec/lib/gitlab/import/import_failure_service_spec.rb +++ b/spec/lib/gitlab/import/import_failure_service_spec.rb @@ -7,58 +7,48 @@ RSpec.describe Gitlab::Import::ImportFailureService, :aggregate_failures do let_it_be(:project) { create(:project, :import_started, import_type: import_type) } let(:exception) { StandardError.new('some error') } - let(:arguments) { { project_id: project.id } } - let(:base_arguments) { { error_source: 'SomeImporter', exception: exception }.merge(arguments) } - let(:exe_arguments) { { fail_import: false, metrics: false } } + let(:import_state) { nil } + let(:fail_import) { false } + let(:metrics) { false } + + let(:arguments) do + { + project_id: project.id, + error_source: 'SomeImporter', + exception: exception, + fail_import: fail_import, + metrics: metrics, + import_state: import_state + } + end describe '.track' do + let(:instance) { double(:failure_service) } + context 'with all arguments provided' do - let(:instance) { double(:failure_service) } - let(:instance_arguments) do + let(:arguments) do { exception: exception, import_state: '_import_state_', project_id: '_project_id_', - error_source: '_error_source_' - } - end - - let(:exe_arguments) do - { + error_source: '_error_source_', fail_import: '_fail_import_', metrics: '_metrics_' } end it 'invokes a new instance and executes' do - expect(described_class).to receive(:new).with(**instance_arguments).and_return(instance) - expect(instance).to receive(:execute).with(**exe_arguments) + expect(described_class).to receive(:new).with(**arguments).and_return(instance) + expect(instance).to receive(:execute) - described_class.track(**instance_arguments.merge(exe_arguments)) + described_class.track(**arguments) end end context 'with only necessary arguments utilizing defaults' do - let(:instance) { double(:failure_service) } - let(:instance_arguments) do - { - exception: exception, - import_state: nil, - project_id: nil, - error_source: nil - } - end - - let(:exe_arguments) do - { - fail_import: false, - metrics: false - } - end - it 'invokes a new instance and executes' do - expect(described_class).to receive(:new).with(**instance_arguments).and_return(instance) - expect(instance).to receive(:execute).with(**exe_arguments) + expect(described_class).to receive(:new).with(a_hash_including(exception: exception)).and_return(instance) + expect(instance).to receive(:execute) described_class.track(exception: exception) end @@ -66,7 +56,7 @@ RSpec.describe Gitlab::Import::ImportFailureService, :aggregate_failures do end describe '#execute' do - subject(:service) { described_class.new(**base_arguments) } + subject(:service) { described_class.new(**arguments) } shared_examples 'logs the exception and fails the import' do it 'when the failure does not abort the import' do @@ -89,13 +79,14 @@ RSpec.describe Gitlab::Import::ImportFailureService, :aggregate_failures do source: 'SomeImporter' ) - service.execute(**exe_arguments) + service.execute expect(project.import_state.reload.status).to eq('failed') expect(project.import_failures).not_to be_empty expect(project.import_failures.last.exception_class).to eq('StandardError') expect(project.import_failures.last.exception_message).to eq('some error') + expect(project.import_failures.last.retry_count).to eq(0) end end @@ -120,32 +111,36 @@ RSpec.describe Gitlab::Import::ImportFailureService, :aggregate_failures do source: 'SomeImporter' ) - service.execute(**exe_arguments) + service.execute expect(project.import_state.reload.status).to eq('started') expect(project.import_failures).not_to be_empty expect(project.import_failures.last.exception_class).to eq('StandardError') expect(project.import_failures.last.exception_message).to eq('some error') + expect(project.import_failures.last.retry_count).to eq(nil) end end context 'when tracking metrics' do - let(:exe_arguments) { { fail_import: false, metrics: true } } + let(:metrics) { true } it 'tracks the failed import' do - metrics = double(:metrics) + metrics_double = double(:metrics) - expect(Gitlab::Import::Metrics).to receive(:new).with("#{project.import_type}_importer", project).and_return(metrics) - expect(metrics).to receive(:track_failed_import) + expect(Gitlab::Import::Metrics) + .to receive(:new) + .with("#{project.import_type}_importer", project) + .and_return(metrics_double) + expect(metrics_double).to receive(:track_failed_import) - service.execute(**exe_arguments) + service.execute end end context 'when using the project as reference' do context 'when it fails the import' do - let(:exe_arguments) { { fail_import: true, metrics: false } } + let(:fail_import) { true } it_behaves_like 'logs the exception and fails the import' end @@ -156,10 +151,10 @@ RSpec.describe Gitlab::Import::ImportFailureService, :aggregate_failures do end context 'when using the import_state as reference' do - let(:arguments) { { import_state: project.import_state } } + let(:import_state) { project.import_state } context 'when it fails the import' do - let(:exe_arguments) { { fail_import: true, metrics: false } } + let(:fail_import) { true } it_behaves_like 'logs the exception and fails the import' end diff --git a/spec/lib/gitlab/import/set_async_jid_spec.rb b/spec/lib/gitlab/import/set_async_jid_spec.rb index 6931a7a953d..016f7cac61a 100644 --- a/spec/lib/gitlab/import/set_async_jid_spec.rb +++ b/spec/lib/gitlab/import/set_async_jid_spec.rb @@ -8,7 +8,7 @@ RSpec.describe Gitlab::Import::SetAsyncJid do it 'sets the JID in Redis' do expect(Gitlab::SidekiqStatus) .to receive(:set) - .with("async-import/project-import-state/#{project.id}", Gitlab::Import::StuckImportJob::IMPORT_JOBS_EXPIRATION) + .with("async-import/project-import-state/#{project.id}", Gitlab::Import::StuckImportJob::IMPORT_JOBS_EXPIRATION, value: 2) .and_call_original described_class.set_jid(project.import_state) diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index b474f5825fd..7ed80cbcf66 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -61,6 +61,7 @@ issues: - pending_escalations - customer_relations_contacts - issue_customer_relations_contacts +- email work_item_type: - issues events: @@ -197,6 +198,7 @@ merge_requests: - system_note_metadata - note_authors - cleanup_schedule +- compliance_violations external_pull_requests: - project merge_request_diff: @@ -223,6 +225,7 @@ ci_pipelines: - ci_ref - stages - statuses +- statuses_order_id_desc - latest_statuses_ordered_by_stage - builds - bridges @@ -596,6 +599,8 @@ project: - security_scans - ci_feature_usages - bulk_import_exports +- ci_project_mirror +- sync_events award_emoji: - awardable - user diff --git a/spec/lib/gitlab/import_export/attributes_permitter_spec.rb b/spec/lib/gitlab/import_export/attributes_permitter_spec.rb index 8ae387d95e3..c748f966463 100644 --- a/spec/lib/gitlab/import_export/attributes_permitter_spec.rb +++ b/spec/lib/gitlab/import_export/attributes_permitter_spec.rb @@ -140,6 +140,7 @@ RSpec.describe Gitlab::ImportExport::AttributesPermitter do :zoom_meetings | true :issues | true :group_members | true + :project | true end with_them do @@ -150,7 +151,11 @@ RSpec.describe Gitlab::ImportExport::AttributesPermitter do describe 'included_attributes for Project' do subject { described_class.new } - additional_attributes = { user: %w[id] } + # these are attributes for which either a special exception is made or are available only via included modules and not attribute introspection + additional_attributes = { + user: %w[id], + project: %w[auto_devops_deploy_strategy auto_devops_enabled issues_enabled jobs_enabled merge_requests_enabled snippets_enabled wiki_enabled build_git_strategy build_enabled security_and_compliance_enabled requirements_enabled] + } Gitlab::ImportExport::Config.new.to_h[:included_attributes].each do |relation_sym, permitted_attributes| context "for #{relation_sym}" do diff --git a/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb b/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb index 6bb6be07749..1d8b137c196 100644 --- a/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' def match_mr1_note(content_regex) - MergeRequest.find_by(title: 'MR1').notes.select { |n| n.note.match(/#{content_regex}/)}.first + MergeRequest.find_by(title: 'MR1').notes.find { |n| n.note.match(/#{content_regex}/) } end RSpec.describe Gitlab::ImportExport::Project::TreeRestorer do @@ -75,7 +75,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer do context 'for an Issue' do it 'does not import note_html' do note_content = 'Quo reprehenderit aliquam qui dicta impedit cupiditate eligendi' - issue_note = Issue.find_by(description: 'Aliquam enim illo et possimus.').notes.select { |n| n.note.match(/#{note_content}/)}.first + issue_note = Issue.find_by(description: 'Aliquam enim illo et possimus.').notes.find { |n| n.note.match(/#{note_content}/) } expect(issue_note.note_html).to match(/#{note_content}/) end @@ -552,7 +552,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer do it 'issue system note metadata restored successfully' do note_content = 'created merge request !1 to address this issue' - note = project.issues.first.notes.select { |n| n.note.match(/#{note_content}/)}.first + note = project.issues.first.notes.find { |n| n.note.match(/#{note_content}/)} expect(note.noteable_type).to eq('Issue') expect(note.system).to eq(true) diff --git a/spec/lib/gitlab/import_export/safe_model_attributes.yml b/spec/lib/gitlab/import_export/safe_model_attributes.yml index 9daa3b32fd1..6ffe2187466 100644 --- a/spec/lib/gitlab/import_export/safe_model_attributes.yml +++ b/spec/lib/gitlab/import_export/safe_model_attributes.yml @@ -33,6 +33,7 @@ Issue: - health_status - external_key - issue_type +- email_message_id Event: - id - target_type @@ -562,6 +563,7 @@ Project: - autoclose_referenced_issues - suggestion_commit_message - merge_commit_template +- squash_commit_template ProjectTracingSetting: - external_url Author: diff --git a/spec/lib/gitlab/lets_encrypt/client_spec.rb b/spec/lib/gitlab/lets_encrypt/client_spec.rb index 54b9bd3bfba..f1284318687 100644 --- a/spec/lib/gitlab/lets_encrypt/client_spec.rb +++ b/spec/lib/gitlab/lets_encrypt/client_spec.rb @@ -73,7 +73,7 @@ RSpec.describe ::Gitlab::LetsEncrypt::Client do subject(:new_order) { client.new_order('example.com') } before do - order_double = instance_double('Acme::Order') + order_double = double('Acme::Order') allow(stub_client).to receive(:new_order).and_return(order_double) end @@ -107,7 +107,7 @@ RSpec.describe ::Gitlab::LetsEncrypt::Client do subject { client.load_challenge(url) } before do - acme_challenge = instance_double('Acme::Client::Resources::Challenge') + acme_challenge = double('Acme::Client::Resources::Challenge') allow(stub_client).to receive(:challenge).with(url: url).and_return(acme_challenge) end diff --git a/spec/lib/gitlab/lograge/custom_options_spec.rb b/spec/lib/gitlab/lograge/custom_options_spec.rb index a4ae39a835a..d8f351bb8a3 100644 --- a/spec/lib/gitlab/lograge/custom_options_spec.rb +++ b/spec/lib/gitlab/lograge/custom_options_spec.rb @@ -95,5 +95,55 @@ RSpec.describe Gitlab::Lograge::CustomOptions do expect(subject[correlation_id_key]).to eq('123456') end end + + context 'when feature flags are present', :request_store do + before do + allow(Feature).to receive(:log_feature_flag_states?).and_return(false) + + definitions = {} + [:enabled_feature, :disabled_feature].each do |flag_name| + definitions[flag_name] = Feature::Definition.new("development/enabled_feature.yml", + name: flag_name, + type: 'development', + log_state_changes: true, + default_enabled: false) + + allow(Feature).to receive(:log_feature_flag_states?).with(flag_name).and_call_original + end + + allow(Feature::Definition).to receive(:definitions).and_return(definitions) + + Feature.enable(:enabled_feature) + Feature.disable(:disabled_feature) + end + + context 'and :feature_flag_log_states is enabled' do + before do + Feature.enable(:feature_flag_state_logs) + end + + it 'adds feature flag events' do + Feature.enabled?(:enabled_feature) + Feature.enabled?(:disabled_feature) + + expect(subject).to have_key(:feature_flag_states) + expect(subject[:feature_flag_states]).to match_array(%w[enabled_feature:1 disabled_feature:0]) + end + end + + context 'and :feature_flag_log_states is disabled' do + before do + Feature.disable(:feature_flag_state_logs) + end + + it 'does not track or add feature flag events' do + Feature.enabled?(:enabled_feature) + Feature.enabled?(:disabled_feature) + + expect(subject).not_to have_key(:feature_flag_states) + expect(Feature).not_to receive(:log_feature_flag_state) + end + end + end end end diff --git a/spec/lib/gitlab/merge_requests/commit_message_generator_spec.rb b/spec/lib/gitlab/merge_requests/commit_message_generator_spec.rb new file mode 100644 index 00000000000..65c76aac10c --- /dev/null +++ b/spec/lib/gitlab/merge_requests/commit_message_generator_spec.rb @@ -0,0 +1,290 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::MergeRequests::CommitMessageGenerator do + let(:merge_commit_template) { nil } + let(:squash_commit_template) { nil } + let(:project) do + create( + :project, + :public, + :repository, + merge_commit_template: merge_commit_template, + squash_commit_template: squash_commit_template + ) + end + + let(:user) { project.creator } + let(:source_branch) { 'feature' } + let(:merge_request_description) { "Merge Request Description\nNext line" } + let(:merge_request_title) { 'Bugfix' } + let(:merge_request) do + create( + :merge_request, + :simple, + source_project: project, + target_project: project, + target_branch: 'master', + source_branch: source_branch, + author: user, + description: merge_request_description, + title: merge_request_title + ) + end + + subject { described_class.new(merge_request: merge_request) } + + shared_examples_for 'commit message with template' do |message_template_name| + it 'returns nil when template is not set in target project' do + expect(result_message).to be_nil + end + + context 'when project has custom commit template' do + let(message_template_name) { <<~MSG.rstrip } + %{title} + + See merge request %{reference} + MSG + + it 'uses custom template' do + expect(result_message).to eq <<~MSG.rstrip + Bugfix + + See merge request #{merge_request.to_reference(full: true)} + MSG + end + end + + context 'when project has commit template with closed issues' do + let(message_template_name) { <<~MSG.rstrip } + Merge branch '%{source_branch}' into '%{target_branch}' + + %{title} + + %{issues} + + See merge request %{reference} + MSG + + it 'omits issues and new lines when no issues are mentioned in description' do + expect(result_message).to eq <<~MSG.rstrip + Merge branch 'feature' into 'master' + + Bugfix + + See merge request #{merge_request.to_reference(full: true)} + MSG + end + + context 'when MR closes issues' do + let(:issue_1) { create(:issue, project: project) } + let(:issue_2) { create(:issue, project: project) } + let(:merge_request_description) { "Description\n\nclosing #{issue_1.to_reference}, #{issue_2.to_reference}" } + + it 'includes them and keeps new line characters' do + expect(result_message).to eq <<~MSG.rstrip + Merge branch 'feature' into 'master' + + Bugfix + + Closes #{issue_1.to_reference} and #{issue_2.to_reference} + + See merge request #{merge_request.to_reference(full: true)} + MSG + end + end + end + + context 'when project has commit template with description' do + let(message_template_name) { <<~MSG.rstrip } + Merge branch '%{source_branch}' into '%{target_branch}' + + %{title} + + %{description} + + See merge request %{reference} + MSG + + it 'uses template' do + expect(result_message).to eq <<~MSG.rstrip + Merge branch 'feature' into 'master' + + Bugfix + + Merge Request Description + Next line + + See merge request #{merge_request.to_reference(full: true)} + MSG + end + + context 'when description is empty string' do + let(:merge_request_description) { '' } + + it 'skips description placeholder and removes new line characters before it' do + expect(result_message).to eq <<~MSG.rstrip + Merge branch 'feature' into 'master' + + Bugfix + + See merge request #{merge_request.to_reference(full: true)} + MSG + end + end + + context 'when description is nil' do + let(:merge_request_description) { nil } + + it 'skips description placeholder and removes new line characters before it' do + expect(result_message).to eq <<~MSG.rstrip + Merge branch 'feature' into 'master' + + Bugfix + + See merge request #{merge_request.to_reference(full: true)} + MSG + end + end + + context 'when description is blank string' do + let(:merge_request_description) { "\n\r \n" } + + it 'skips description placeholder and removes new line characters before it' do + expect(result_message).to eq <<~MSG.rstrip + Merge branch 'feature' into 'master' + + Bugfix + + See merge request #{merge_request.to_reference(full: true)} + MSG + end + end + end + + context 'when custom commit template contains placeholder in the middle or beginning of the line' do + let(message_template_name) { <<~MSG.rstrip } + Merge branch '%{source_branch}' into '%{target_branch}' + + %{description} %{title} + + See merge request %{reference} + MSG + + it 'uses custom template' do + expect(result_message).to eq <<~MSG.rstrip + Merge branch 'feature' into 'master' + + Merge Request Description + Next line Bugfix + + See merge request #{merge_request.to_reference(full: true)} + MSG + end + + context 'when description is empty string' do + let(:merge_request_description) { '' } + + it 'does not remove new line characters before empty placeholder' do + expect(result_message).to eq <<~MSG.rstrip + Merge branch 'feature' into 'master' + + Bugfix + + See merge request #{merge_request.to_reference(full: true)} + MSG + end + end + end + + context 'when project has template with CRLF newlines' do + let(message_template_name) do + "Merge branch '%{source_branch}' into '%{target_branch}'\r\n\r\n%{title}\r\n\r\n%{description}\r\n\r\nSee merge request %{reference}" + end + + it 'converts it to LF newlines' do + expect(result_message).to eq <<~MSG.rstrip + Merge branch 'feature' into 'master' + + Bugfix + + Merge Request Description + Next line + + See merge request #{merge_request.to_reference(full: true)} + MSG + end + + context 'when description is empty string' do + let(:merge_request_description) { '' } + + it 'skips description placeholder and removes new line characters before it' do + expect(result_message).to eq <<~MSG.rstrip + Merge branch 'feature' into 'master' + + Bugfix + + See merge request #{merge_request.to_reference(full: true)} + MSG + end + end + + context 'when project has merge commit template with first_commit' do + let(message_template_name) { <<~MSG.rstrip } + Message: %{first_commit} + MSG + + it 'uses first commit' do + expect(result_message).to eq <<~MSG.rstrip + Message: Feature added + + Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com> + MSG + end + + context 'when branch has no unmerged commits' do + let(:source_branch) { 'v1.1.0' } + + it 'is an empty string' do + expect(result_message).to eq 'Message: ' + end + end + end + + context 'when project has merge commit template with first_multiline_commit' do + let(message_template_name) { <<~MSG.rstrip } + Message: %{first_multiline_commit} + MSG + + it 'uses first multiline commit' do + expect(result_message).to eq <<~MSG.rstrip + Message: Feature added + + Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com> + MSG + end + + context 'when branch has no multiline commits' do + let(:source_branch) { 'spooky-stuff' } + + it 'is mr title' do + expect(result_message).to eq 'Message: Bugfix' + end + end + end + end + end + + describe '#merge_message' do + let(:result_message) { subject.merge_message } + + it_behaves_like 'commit message with template', :merge_commit_template + end + + describe '#squash_message' do + let(:result_message) { subject.squash_message } + + it_behaves_like 'commit message with template', :squash_commit_template + end +end diff --git a/spec/lib/gitlab/merge_requests/merge_commit_message_spec.rb b/spec/lib/gitlab/merge_requests/merge_commit_message_spec.rb deleted file mode 100644 index 884f8df5e56..00000000000 --- a/spec/lib/gitlab/merge_requests/merge_commit_message_spec.rb +++ /dev/null @@ -1,219 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::MergeRequests::MergeCommitMessage do - let(:merge_commit_template) { nil } - let(:project) { create(:project, :public, :repository, merge_commit_template: merge_commit_template) } - let(:user) { project.creator } - let(:merge_request_description) { "Merge Request Description\nNext line" } - let(:merge_request_title) { 'Bugfix' } - let(:merge_request) do - create( - :merge_request, - :simple, - source_project: project, - target_project: project, - author: user, - description: merge_request_description, - title: merge_request_title - ) - end - - subject { described_class.new(merge_request: merge_request) } - - it 'returns nil when template is not set in target project' do - expect(subject.message).to be_nil - end - - context 'when project has custom merge commit template' do - let(:merge_commit_template) { <<~MSG.rstrip } - %{title} - - See merge request %{reference} - MSG - - it 'uses custom template' do - expect(subject.message).to eq <<~MSG.rstrip - Bugfix - - See merge request #{merge_request.to_reference(full: true)} - MSG - end - end - - context 'when project has merge commit template with closed issues' do - let(:merge_commit_template) { <<~MSG.rstrip } - Merge branch '%{source_branch}' into '%{target_branch}' - - %{title} - - %{issues} - - See merge request %{reference} - MSG - - it 'omits issues and new lines when no issues are mentioned in description' do - expect(subject.message).to eq <<~MSG.rstrip - Merge branch 'feature' into 'master' - - Bugfix - - See merge request #{merge_request.to_reference(full: true)} - MSG - end - - context 'when MR closes issues' do - let(:issue_1) { create(:issue, project: project) } - let(:issue_2) { create(:issue, project: project) } - let(:merge_request_description) { "Description\n\nclosing #{issue_1.to_reference}, #{issue_2.to_reference}" } - - it 'includes them and keeps new line characters' do - expect(subject.message).to eq <<~MSG.rstrip - Merge branch 'feature' into 'master' - - Bugfix - - Closes #{issue_1.to_reference} and #{issue_2.to_reference} - - See merge request #{merge_request.to_reference(full: true)} - MSG - end - end - end - - context 'when project has merge commit template with description' do - let(:merge_commit_template) { <<~MSG.rstrip } - Merge branch '%{source_branch}' into '%{target_branch}' - - %{title} - - %{description} - - See merge request %{reference} - MSG - - it 'uses template' do - expect(subject.message).to eq <<~MSG.rstrip - Merge branch 'feature' into 'master' - - Bugfix - - Merge Request Description - Next line - - See merge request #{merge_request.to_reference(full: true)} - MSG - end - - context 'when description is empty string' do - let(:merge_request_description) { '' } - - it 'skips description placeholder and removes new line characters before it' do - expect(subject.message).to eq <<~MSG.rstrip - Merge branch 'feature' into 'master' - - Bugfix - - See merge request #{merge_request.to_reference(full: true)} - MSG - end - end - - context 'when description is nil' do - let(:merge_request_description) { nil } - - it 'skips description placeholder and removes new line characters before it' do - expect(subject.message).to eq <<~MSG.rstrip - Merge branch 'feature' into 'master' - - Bugfix - - See merge request #{merge_request.to_reference(full: true)} - MSG - end - end - - context 'when description is blank string' do - let(:merge_request_description) { "\n\r \n" } - - it 'skips description placeholder and removes new line characters before it' do - expect(subject.message).to eq <<~MSG.rstrip - Merge branch 'feature' into 'master' - - Bugfix - - See merge request #{merge_request.to_reference(full: true)} - MSG - end - end - end - - context 'when custom merge commit template contains placeholder in the middle or beginning of the line' do - let(:merge_commit_template) { <<~MSG.rstrip } - Merge branch '%{source_branch}' into '%{target_branch}' - - %{description} %{title} - - See merge request %{reference} - MSG - - it 'uses custom template' do - expect(subject.message).to eq <<~MSG.rstrip - Merge branch 'feature' into 'master' - - Merge Request Description - Next line Bugfix - - See merge request #{merge_request.to_reference(full: true)} - MSG - end - - context 'when description is empty string' do - let(:merge_request_description) { '' } - - it 'does not remove new line characters before empty placeholder' do - expect(subject.message).to eq <<~MSG.rstrip - Merge branch 'feature' into 'master' - - Bugfix - - See merge request #{merge_request.to_reference(full: true)} - MSG - end - end - end - - context 'when project has template with CRLF newlines' do - let(:merge_commit_template) do - "Merge branch '%{source_branch}' into '%{target_branch}'\r\n\r\n%{title}\r\n\r\n%{description}\r\n\r\nSee merge request %{reference}" - end - - it 'converts it to LF newlines' do - expect(subject.message).to eq <<~MSG.rstrip - Merge branch 'feature' into 'master' - - Bugfix - - Merge Request Description - Next line - - See merge request #{merge_request.to_reference(full: true)} - MSG - end - - context 'when description is empty string' do - let(:merge_request_description) { '' } - - it 'skips description placeholder and removes new line characters before it' do - expect(subject.message).to eq <<~MSG.rstrip - Merge branch 'feature' into 'master' - - Bugfix - - See merge request #{merge_request.to_reference(full: true)} - MSG - end - end - end -end diff --git a/spec/lib/gitlab/merge_requests/mergeability/redis_interface_spec.rb b/spec/lib/gitlab/merge_requests/mergeability/redis_interface_spec.rb index e5475d04d86..2471faf76b2 100644 --- a/spec/lib/gitlab/merge_requests/mergeability/redis_interface_spec.rb +++ b/spec/lib/gitlab/merge_requests/mergeability/redis_interface_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::MergeRequests::Mergeability::RedisInterface, :clean_gitlab_redis_shared_state do +RSpec.describe Gitlab::MergeRequests::Mergeability::RedisInterface, :clean_gitlab_redis_cache do subject(:redis_interface) { described_class.new } let(:merge_check) { double(cache_key: '13') } @@ -11,17 +11,17 @@ RSpec.describe Gitlab::MergeRequests::Mergeability::RedisInterface, :clean_gitla describe '#save_check' do it 'saves the hash' do - expect(Gitlab::Redis::SharedState.with { |redis| redis.get(expected_key) }).to be_nil + expect(Gitlab::Redis::Cache.with { |redis| redis.get(expected_key) }).to be_nil redis_interface.save_check(merge_check: merge_check, result_hash: result_hash) - expect(Gitlab::Redis::SharedState.with { |redis| redis.get(expected_key) }).to eq result_hash.to_json + expect(Gitlab::Redis::Cache.with { |redis| redis.get(expected_key) }).to eq result_hash.to_json end end describe '#retrieve_check' do it 'returns the hash' do - Gitlab::Redis::SharedState.with { |redis| redis.set(expected_key, result_hash.to_json) } + Gitlab::Redis::Cache.with { |redis| redis.set(expected_key, result_hash.to_json) } expect(redis_interface.retrieve_check(merge_check: merge_check)).to eq result_hash end diff --git a/spec/lib/gitlab/metrics/exporter/base_exporter_spec.rb b/spec/lib/gitlab/metrics/exporter/base_exporter_spec.rb index e4f85243528..9cd1ef4094e 100644 --- a/spec/lib/gitlab/metrics/exporter/base_exporter_spec.rb +++ b/spec/lib/gitlab/metrics/exporter/base_exporter_spec.rb @@ -3,9 +3,9 @@ require 'spec_helper' RSpec.describe Gitlab::Metrics::Exporter::BaseExporter do - let(:exporter) { described_class.new } - let(:log_filename) { File.join(Rails.root, 'log', 'sidekiq_exporter.log') } let(:settings) { double('settings') } + let(:exporter) { described_class.new(settings) } + let(:log_filename) { File.join(Rails.root, 'log', 'sidekiq_exporter.log') } before do allow_any_instance_of(described_class).to receive(:log_filename).and_return(log_filename) diff --git a/spec/lib/gitlab/metrics/exporter/sidekiq_exporter_spec.rb b/spec/lib/gitlab/metrics/exporter/sidekiq_exporter_spec.rb index 01cf47a7c58..75bc3ba9626 100644 --- a/spec/lib/gitlab/metrics/exporter/sidekiq_exporter_spec.rb +++ b/spec/lib/gitlab/metrics/exporter/sidekiq_exporter_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::Metrics::Exporter::SidekiqExporter do - let(:exporter) { described_class.new } + let(:exporter) { described_class.new(Settings.monitoring.sidekiq_exporter) } after do exporter.stop @@ -50,40 +50,4 @@ RSpec.describe Gitlab::Metrics::Exporter::SidekiqExporter do expect(exporter.log_filename).to end_with('sidekiq_exporter.log') end end - - context 'when port is already taken' do - let(:first_exporter) { described_class.new } - - before do - stub_config( - monitoring: { - sidekiq_exporter: { - enabled: true, - port: 9992, - address: '127.0.0.1' - } - } - ) - - first_exporter.start - end - - after do - first_exporter.stop - end - - it 'does print error message' do - expect(Sidekiq.logger).to receive(:error) - .with( - class: described_class.to_s, - message: 'Cannot start sidekiq_exporter', - 'exception.message' => anything) - - exporter.start - end - - it 'does not start thread' do - expect(exporter.start).to be_nil - end - end end diff --git a/spec/lib/gitlab/metrics/samplers/database_sampler_spec.rb b/spec/lib/gitlab/metrics/samplers/database_sampler_spec.rb index e97a4fdddcb..e8f8947c9e8 100644 --- a/spec/lib/gitlab/metrics/samplers/database_sampler_spec.rb +++ b/spec/lib/gitlab/metrics/samplers/database_sampler_spec.rb @@ -8,44 +8,169 @@ RSpec.describe Gitlab::Metrics::Samplers::DatabaseSampler do it_behaves_like 'metrics sampler', 'DATABASE_SAMPLER' describe '#sample' do + let(:main_labels) do + { + class: 'ActiveRecord::Base', + host: ApplicationRecord.database.config['host'], + port: ApplicationRecord.database.config['port'], + db_config_name: 'main' + } + end + + let(:ci_labels) do + { + class: 'Ci::ApplicationRecord', + host: Ci::ApplicationRecord.database.config['host'], + port: Ci::ApplicationRecord.database.config['port'], + db_config_name: 'ci' + } + end + + let(:main_replica_labels) do + { + class: 'ActiveRecord::Base', + host: 'main-replica-host', + port: 2345, + db_config_name: 'main_replica' + } + end + + let(:ci_replica_labels) do + { + class: 'Ci::ApplicationRecord', + host: 'ci-replica-host', + port: 3456, + db_config_name: 'ci_replica' + } + end + before do described_class::METRIC_DESCRIPTIONS.each_key do |metric| allow(subject.metrics[metric]).to receive(:set) end + + allow(Gitlab::Database).to receive(:database_base_models) + .and_return({ main: ActiveRecord::Base, ci: Ci::ApplicationRecord }) end - context 'for ActiveRecord::Base' do - let(:labels) do - { - class: 'ActiveRecord::Base', - host: ApplicationRecord.database.config['host'], - port: ApplicationRecord.database.config['port'] - } + context 'when all base models are connected', :add_ci_connection do + it 'samples connection pool statistics for all primaries' do + expect_metrics_with_labels(main_labels) + expect_metrics_with_labels(ci_labels) + + subject.sample end - context 'when the database is connected' do - it 'samples connection pool statistics' do - expect(subject.metrics[:size]).to receive(:set).with(labels, a_value >= 1) - expect(subject.metrics[:connections]).to receive(:set).with(labels, a_value >= 1) - expect(subject.metrics[:busy]).to receive(:set).with(labels, a_value >= 1) - expect(subject.metrics[:dead]).to receive(:set).with(labels, a_value >= 0) - expect(subject.metrics[:waiting]).to receive(:set).with(labels, a_value >= 0) + context 'when replica hosts are configured' do + let(:main_load_balancer) { ActiveRecord::Base.load_balancer } # rubocop:disable Database/MultipleDatabases + let(:main_replica_host) { main_load_balancer.host } + + let(:ci_load_balancer) { double(:load_balancer, host_list: ci_host_list, configuration: configuration) } + let(:configuration) { double(:configuration, primary_connection_specification_name: 'Ci::ApplicationRecord') } + let(:ci_host_list) { double(:host_list, hosts: [ci_replica_host]) } + let(:ci_replica_host) { double(:host, connection: ci_connection) } + let(:ci_connection) { double(:connection, pool: Ci::ApplicationRecord.connection_pool) } + + before do + allow(Gitlab::Database::LoadBalancing).to receive(:each_load_balancer) + .and_return([main_load_balancer, ci_load_balancer].to_enum) + + allow(main_load_balancer).to receive(:primary_only?).and_return(false) + allow(ci_load_balancer).to receive(:primary_only?).and_return(false) + + allow(main_replica_host).to receive(:host).and_return('main-replica-host') + allow(ci_replica_host).to receive(:host).and_return('ci-replica-host') + + allow(main_replica_host).to receive(:port).and_return(2345) + allow(ci_replica_host).to receive(:port).and_return(3456) + + allow(Gitlab::Database).to receive(:db_config_name) + .with(main_replica_host.connection) + .and_return('main_replica') + + allow(Gitlab::Database).to receive(:db_config_name) + .with(ci_replica_host.connection) + .and_return('ci_replica') + end + + it 'samples connection pool statistics for primaries and replicas' do + expect_metrics_with_labels(main_labels) + expect_metrics_with_labels(ci_labels) + expect_metrics_with_labels(main_replica_labels) + expect_metrics_with_labels(ci_replica_labels) subject.sample end end + end + + context 'when a base model is not connected', :add_ci_connection do + before do + allow(Ci::ApplicationRecord).to receive(:connected?).and_return(false) + end + + it 'records no samples for that primary' do + expect_metrics_with_labels(main_labels) + expect_no_metrics_with_labels(ci_labels) + + subject.sample + end + + context 'when the base model has replica connections' do + let(:main_load_balancer) { ActiveRecord::Base.load_balancer } # rubocop:disable Database/MultipleDatabases + let(:main_replica_host) { main_load_balancer.host } + + let(:ci_load_balancer) { double(:load_balancer, host_list: ci_host_list, configuration: configuration) } + let(:configuration) { double(:configuration, primary_connection_specification_name: 'Ci::ApplicationRecord') } + let(:ci_host_list) { double(:host_list, hosts: [ci_replica_host]) } + let(:ci_replica_host) { double(:host, connection: ci_connection) } + let(:ci_connection) { double(:connection, pool: Ci::ApplicationRecord.connection_pool) } - context 'when the database is not connected' do before do - allow(ActiveRecord::Base).to receive(:connected?).and_return(false) + allow(Gitlab::Database::LoadBalancing).to receive(:each_load_balancer) + .and_return([main_load_balancer, ci_load_balancer].to_enum) + + allow(main_load_balancer).to receive(:primary_only?).and_return(false) + allow(ci_load_balancer).to receive(:primary_only?).and_return(false) + + allow(main_replica_host).to receive(:host).and_return('main-replica-host') + allow(ci_replica_host).to receive(:host).and_return('ci-replica-host') + + allow(main_replica_host).to receive(:port).and_return(2345) + allow(ci_replica_host).to receive(:port).and_return(3456) + + allow(Gitlab::Database).to receive(:db_config_name) + .with(main_replica_host.connection) + .and_return('main_replica') + + allow(Gitlab::Database).to receive(:db_config_name) + .with(ci_replica_host.connection) + .and_return('ci_replica') end - it 'records no samples' do - expect(subject.metrics[:size]).not_to receive(:set).with(labels, anything) + it 'still records the replica metrics' do + expect_metrics_with_labels(main_labels) + expect_metrics_with_labels(main_replica_labels) + expect_no_metrics_with_labels(ci_labels) + expect_metrics_with_labels(ci_replica_labels) subject.sample end end end + + def expect_metrics_with_labels(labels) + expect(subject.metrics[:size]).to receive(:set).with(labels, a_value >= 1) + expect(subject.metrics[:connections]).to receive(:set).with(labels, a_value >= 1) + expect(subject.metrics[:busy]).to receive(:set).with(labels, a_value >= 1) + expect(subject.metrics[:dead]).to receive(:set).with(labels, a_value >= 0) + expect(subject.metrics[:waiting]).to receive(:set).with(labels, a_value >= 0) + end + + def expect_no_metrics_with_labels(labels) + described_class::METRIC_DESCRIPTIONS.each_key do |metric| + expect(subject.metrics[metric]).not_to receive(:set).with(labels, anything) + end + end end end diff --git a/spec/lib/gitlab/metrics/subscribers/active_record_spec.rb b/spec/lib/gitlab/metrics/subscribers/active_record_spec.rb index a8e4f039da4..389b0ef1044 100644 --- a/spec/lib/gitlab/metrics/subscribers/active_record_spec.rb +++ b/spec/lib/gitlab/metrics/subscribers/active_record_spec.rb @@ -198,6 +198,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::ActiveRecord do context 'query using a connection to a replica' do before do allow(Gitlab::Database::LoadBalancing).to receive(:db_role_for_connection).and_return(:replica) + allow(connection).to receive_message_chain(:pool, :db_config, :name).and_return(db_config_name) end it 'queries connection db role' do diff --git a/spec/lib/gitlab/multi_collection_paginator_spec.rb b/spec/lib/gitlab/multi_collection_paginator_spec.rb index c7c8f4f969f..080b3382684 100644 --- a/spec/lib/gitlab/multi_collection_paginator_spec.rb +++ b/spec/lib/gitlab/multi_collection_paginator_spec.rb @@ -40,7 +40,7 @@ RSpec.describe Gitlab::MultiCollectionPaginator do end it 'fils the last page with elements from the second collection' do - expected_collection = all_groups[-2..-1] + expected_collection = all_groups[-2..] expect(paginator.paginate(3)).to eq(expected_collection) end diff --git a/spec/lib/gitlab/pagination/keyset/order_spec.rb b/spec/lib/gitlab/pagination/keyset/order_spec.rb index 3c14d91fdfd..1bed8e542a2 100644 --- a/spec/lib/gitlab/pagination/keyset/order_spec.rb +++ b/spec/lib/gitlab/pagination/keyset/order_spec.rb @@ -127,7 +127,7 @@ RSpec.describe Gitlab::Pagination::Keyset::Order do end it do - expect(subject).to eq(expected.reverse[1..-1]) # removing one item because we used it to calculate cursor data for the "last" page in subject + expect(subject).to eq(expected.reverse[1..]) # removing one item because we used it to calculate cursor data for the "last" page in subject end end end diff --git a/spec/lib/gitlab/pagination/offset_pagination_spec.rb b/spec/lib/gitlab/pagination/offset_pagination_spec.rb index ffecbb06ff8..f8d50fbc517 100644 --- a/spec/lib/gitlab/pagination/offset_pagination_spec.rb +++ b/spec/lib/gitlab/pagination/offset_pagination_spec.rb @@ -82,7 +82,7 @@ RSpec.describe Gitlab::Pagination::OffsetPagination do context 'when the api_kaminari_count_with_limit feature flag is enabled' do before do - stub_feature_flags(api_kaminari_count_with_limit: true, lower_relation_max_count_limit: false) + stub_feature_flags(api_kaminari_count_with_limit: true) end context 'when resources count is less than MAX_COUNT_LIMIT' do @@ -120,41 +120,6 @@ RSpec.describe Gitlab::Pagination::OffsetPagination do end end - context 'when lower_relation_max_count_limit FF is enabled' do - before do - stub_feature_flags(lower_relation_max_count_limit: true) - end - - it_behaves_like 'paginated response' - it_behaves_like 'response with pagination headers' - - context 'when limit is met' do - before do - stub_const("::Kaminari::ActiveRecordRelationMethods::MAX_COUNT_NEW_LOWER_LIMIT", 2) - end - - it_behaves_like 'paginated response' - - it 'does not return the X-Total and X-Total-Pages headers' do - expect_no_header('X-Total') - expect_no_header('X-Total-Pages') - expect_header('X-Per-Page', '2') - expect_header('X-Page', '1') - expect_header('X-Next-Page', '2') - expect_header('X-Prev-Page', '') - - expect_header('Link', anything) do |_key, val| - expect(val).to include(%Q(<#{incoming_api_projects_url}?#{query.merge(page: 1).to_query}>; rel="first")) - expect(val).to include(%Q(<#{incoming_api_projects_url}?#{query.merge(page: 2).to_query}>; rel="next")) - expect(val).not_to include('rel="last"') - expect(val).not_to include('rel="prev"') - end - - subject.paginate(resource) - end - end - end - it 'does not return the total headers when excluding them' do expect_no_header('X-Total') expect_no_header('X-Total-Pages') diff --git a/spec/lib/gitlab/patch/legacy_database_config_spec.rb b/spec/lib/gitlab/patch/legacy_database_config_spec.rb index e6c0bdbf360..b87e16f31ae 100644 --- a/spec/lib/gitlab/patch/legacy_database_config_spec.rb +++ b/spec/lib/gitlab/patch/legacy_database_config_spec.rb @@ -11,6 +11,9 @@ RSpec.describe Gitlab::Patch::LegacyDatabaseConfig do let(:configuration) { Rails::Application::Configuration.new(Rails.root) } before do + allow(File).to receive(:exist?).and_call_original + allow(File).to receive(:exist?).with(Rails.root.join("config/database_geo.yml")).and_return(false) + # The `AS::ConfigurationFile` calls `read` in `def initialize` # thus we cannot use `expect_next_instance_of` # rubocop:disable RSpec/AnyInstanceOf diff --git a/spec/lib/gitlab/process_management_spec.rb b/spec/lib/gitlab/process_management_spec.rb new file mode 100644 index 00000000000..a71a476b540 --- /dev/null +++ b/spec/lib/gitlab/process_management_spec.rb @@ -0,0 +1,144 @@ +# frozen_string_literal: true + +require_relative '../../../lib/gitlab/process_management' + +RSpec.describe Gitlab::ProcessManagement do + describe '.trap_signals' do + it 'traps the given signals' do + expect(described_class).to receive(:trap).ordered.with(:INT) + expect(described_class).to receive(:trap).ordered.with(:HUP) + + described_class.trap_signals(%i(INT HUP)) + end + end + + describe '.modify_signals' do + it 'traps the given signals with the given command' do + expect(described_class).to receive(:trap).ordered.with(:INT, 'DEFAULT') + expect(described_class).to receive(:trap).ordered.with(:HUP, 'DEFAULT') + + described_class.modify_signals(%i(INT HUP), 'DEFAULT') + end + end + + describe '.signal_processes' do + it 'sends a signal to every given process' do + expect(described_class).to receive(:signal).with(1, :INT) + + described_class.signal_processes([1], :INT) + end + end + + describe '.signal' do + it 'sends a signal to the given process' do + allow(Process).to receive(:kill).with(:INT, 4) + expect(described_class.signal(4, :INT)).to eq(true) + end + + it 'returns false when the process does not exist' do + allow(Process).to receive(:kill).with(:INT, 4).and_raise(Errno::ESRCH) + expect(described_class.signal(4, :INT)).to eq(false) + end + end + + describe '.wait_async' do + it 'waits for a process in a separate thread' do + thread = described_class.wait_async(Process.spawn('true')) + + # Upon success Process.wait just returns the PID. + expect(thread.value).to be_a_kind_of(Numeric) + end + end + + # In the X_alive? checks, we check negative PIDs sometimes as a simple way + # to be sure the pids are definitely for non-existent processes. + # Note that -1 is special, and sends the signal to every process we have permission + # for, so we use -2, -3 etc + describe '.all_alive?' do + it 'returns true if all processes are alive' do + processes = [Process.pid] + + expect(described_class.all_alive?(processes)).to eq(true) + end + + it 'returns false when a thread was not alive' do + processes = [-2] + + expect(described_class.all_alive?(processes)).to eq(false) + end + end + + describe '.process_alive?' do + it 'returns true if the process is alive' do + process = Process.pid + + expect(described_class.process_alive?(process)).to eq(true) + end + + it 'returns false when a thread was not alive' do + process = -2 + + expect(described_class.process_alive?(process)).to eq(false) + end + + it 'returns false when no pid is given' do + process = nil + + expect(described_class.process_alive?(process)).to eq(false) + end + end + + describe '.process_died?' do + it 'returns false if the process is alive' do + process = Process.pid + + expect(described_class.process_died?(process)).to eq(false) + end + + it 'returns true when a thread was not alive' do + process = -2 + + expect(described_class.process_died?(process)).to eq(true) + end + + it 'returns true when no pid is given' do + process = nil + + expect(described_class.process_died?(process)).to eq(true) + end + end + + describe '.pids_alive' do + it 'returns the pids that are alive, from a given array' do + pids = [Process.pid, -2] + + expect(described_class.pids_alive(pids)).to match_array([Process.pid]) + end + end + + describe '.any_alive?' do + it 'returns true if at least one process is alive' do + processes = [Process.pid, -2] + + expect(described_class.any_alive?(processes)).to eq(true) + end + + it 'returns false when all threads are dead' do + processes = [-2, -3] + + expect(described_class.any_alive?(processes)).to eq(false) + end + end + + describe '.write_pid' do + it 'writes the PID of the current process to the given file' do + handle = double(:handle) + + allow(File).to receive(:open).with('/dev/null', 'w').and_yield(handle) + + expect(handle).to receive(:write).with(Process.pid.to_s) + + described_class.write_pid('/dev/null') + end + end +end diff --git a/spec/lib/gitlab/quick_actions/dsl_spec.rb b/spec/lib/gitlab/quick_actions/dsl_spec.rb index f990abfb253..942d347424f 100644 --- a/spec/lib/gitlab/quick_actions/dsl_spec.rb +++ b/spec/lib/gitlab/quick_actions/dsl_spec.rb @@ -96,8 +96,8 @@ RSpec.describe Gitlab::QuickActions::Dsl do expect(dynamic_description_def.name).to eq(:dynamic_description) expect(dynamic_description_def.aliases).to eq([]) - expect(dynamic_description_def.to_h(OpenStruct.new(noteable: 'issue'))[:description]).to eq('A dynamic description for ISSUE') - expect(dynamic_description_def.execute_message(OpenStruct.new(noteable: 'issue'), 'arg')).to eq('A dynamic execution message for ISSUE passing arg') + expect(dynamic_description_def.to_h(double('desc', noteable: 'issue'))[:description]).to eq('A dynamic description for ISSUE') + expect(dynamic_description_def.execute_message(double('desc', noteable: 'issue'), 'arg')).to eq('A dynamic execution message for ISSUE passing arg') expect(dynamic_description_def.params).to eq(['The first argument', 'The second argument']) expect(dynamic_description_def.condition_block).to be_nil expect(dynamic_description_def.types).to eq([]) diff --git a/spec/lib/gitlab/rack_attack_spec.rb b/spec/lib/gitlab/rack_attack_spec.rb index 8f03905e08d..39ea02bad8b 100644 --- a/spec/lib/gitlab/rack_attack_spec.rb +++ b/spec/lib/gitlab/rack_attack_spec.rb @@ -5,8 +5,8 @@ require 'spec_helper' RSpec.describe Gitlab::RackAttack, :aggregate_failures do describe '.configure' do let(:fake_rack_attack) { class_double("Rack::Attack") } - let(:fake_rack_attack_request) { class_double("Rack::Attack::Request") } - let(:fake_cache) { instance_double("Rack::Attack::Cache") } + let(:fake_rack_attack_request) { class_double(Rack::Attack::Request) } + let(:fake_cache) { instance_double(Rack::Attack::Cache) } let(:throttles) do { @@ -27,9 +27,6 @@ RSpec.describe Gitlab::RackAttack, :aggregate_failures do end before do - stub_const("Rack::Attack", fake_rack_attack) - stub_const("Rack::Attack::Request", fake_rack_attack_request) - allow(fake_rack_attack).to receive(:throttled_response=) allow(fake_rack_attack).to receive(:throttle) allow(fake_rack_attack).to receive(:track) @@ -37,6 +34,9 @@ RSpec.describe Gitlab::RackAttack, :aggregate_failures do allow(fake_rack_attack).to receive(:blocklist) allow(fake_rack_attack).to receive(:cache).and_return(fake_cache) allow(fake_cache).to receive(:store=) + + fake_rack_attack.const_set('Request', fake_rack_attack_request) + stub_const("Rack::Attack", fake_rack_attack) end it 'extends the request class' do @@ -78,7 +78,7 @@ RSpec.describe Gitlab::RackAttack, :aggregate_failures do it 'configures tracks and throttles with a selected set of dry-runs' do dry_run_throttles = throttles.each_key.first(2) - regular_throttles = throttles.keys[2..-1] + regular_throttles = throttles.keys[2..] stub_env('GITLAB_THROTTLE_DRY_RUN', dry_run_throttles.join(',')) described_class.configure(fake_rack_attack) diff --git a/spec/lib/gitlab/rate_limit_helpers_spec.rb b/spec/lib/gitlab/rate_limit_helpers_spec.rb deleted file mode 100644 index ad0e2de1448..00000000000 --- a/spec/lib/gitlab/rate_limit_helpers_spec.rb +++ /dev/null @@ -1,50 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::RateLimitHelpers, :clean_gitlab_redis_rate_limiting do - let(:limiter_class) do - Class.new do - include ::Gitlab::RateLimitHelpers - - attr_reader :request - - def initialize(request) - @request = request - end - end - end - - let(:request) { instance_double(ActionDispatch::Request, request_method: 'GET', ip: '127.0.0.1', fullpath: '/') } - let(:class_instance) { limiter_class.new(request) } - - let_it_be(:user) { create(:user) } - let_it_be(:project) { create(:project) } - - describe '#archive_rate_limit_reached?' do - context 'with a user' do - it 'rate limits the user properly' do - 5.times do - expect(class_instance.archive_rate_limit_reached?(user, project)).to be_falsey - end - - expect(class_instance.archive_rate_limit_reached?(user, project)).to be_truthy - end - end - - context 'with an anonymous user' do - before do - stub_const('Gitlab::RateLimitHelpers::ARCHIVE_RATE_ANONYMOUS_THRESHOLD', 2) - end - - it 'rate limits with higher limits' do - 2.times do - expect(class_instance.archive_rate_limit_reached?(nil, project)).to be_falsey - end - - expect(class_instance.archive_rate_limit_reached?(nil, project)).to be_truthy - expect(class_instance.archive_rate_limit_reached?(user, project)).to be_falsey - end - end - end -end diff --git a/spec/lib/gitlab/redis/multi_store_spec.rb b/spec/lib/gitlab/redis/multi_store_spec.rb index bf1bf65bb9b..76731bb916c 100644 --- a/spec/lib/gitlab/redis/multi_store_spec.rb +++ b/spec/lib/gitlab/redis/multi_store_spec.rb @@ -27,6 +27,11 @@ RSpec.describe Gitlab::Redis::MultiStore do subject { multi_store.send(name, *args) } + before do + skip_feature_flags_yaml_validation + skip_default_enabled_yaml_check + end + after(:all) do primary_store.flushdb secondary_store.flushdb @@ -48,6 +53,15 @@ RSpec.describe Gitlab::Redis::MultiStore do end end + context 'when instance_name is nil' do + let(:instance_name) { nil } + let(:multi_store) { described_class.new(primary_store, secondary_store, instance_name)} + + it 'fails with exception' do + expect { multi_store }.to raise_error(ArgumentError, /instance_name is required/) + end + end + context 'when primary_store is not a ::Redis instance' do before do allow(primary_store).to receive(:is_a?).with(::Redis).and_return(false) @@ -114,6 +128,12 @@ RSpec.describe Gitlab::Redis::MultiStore do end RSpec.shared_examples_for 'fallback read from the secondary store' do + let(:counter) { Gitlab::Metrics::NullMetric.instance } + + before do + allow(Gitlab::Metrics).to receive(:counter).and_return(counter) + end + it 'fallback and execute on secondary instance' do expect(secondary_store).to receive(name).with(*args).and_call_original @@ -128,7 +148,7 @@ RSpec.describe Gitlab::Redis::MultiStore do end it 'increment read fallback count metrics' do - expect(multi_store).to receive(:increment_read_fallback_count).with(name) + expect(counter).to receive(:increment).with(command: name, instance_name: instance_name) subject end @@ -169,9 +189,9 @@ RSpec.describe Gitlab::Redis::MultiStore do allow(secondary_store).to receive(name).and_call_original end - context 'with feature flag :use_multi_store enabled' do + context 'with feature flag :use_primary_and_secondary_stores_for_test_store' do before do - stub_feature_flags(use_multi_store: true) + stub_feature_flags(use_primary_and_secondary_stores_for_test_store: true) end context 'when reading from the primary is successful' do @@ -246,12 +266,38 @@ RSpec.describe Gitlab::Redis::MultiStore do end end - context 'with feature flag :use_multi_store is disabled' do + context 'with feature flag :use_primary_and_secondary_stores_for_test_store' do before do - stub_feature_flags(use_multi_store: false) + stub_feature_flags(use_primary_and_secondary_stores_for_test_store: false) end - it_behaves_like 'secondary store' + context 'with feature flag :use_primary_store_as_default_for_test_store is disabled' do + before do + stub_feature_flags(use_primary_store_as_default_for_test_store: false) + end + + it_behaves_like 'secondary store' + end + + context 'with feature flag :use_primary_store_as_default_for_test_store is enabled' do + before do + stub_feature_flags(use_primary_store_as_default_for_test_store: true) + end + + it 'execute on the primary instance' do + expect(primary_store).to receive(name).with(*args).and_call_original + + subject + end + + include_examples 'reads correct value' + + it 'does not execute on the secondary store' do + expect(secondary_store).not_to receive(name) + + subject + end + end end context 'with both primary and secondary store using same redis instance' do @@ -329,9 +375,9 @@ RSpec.describe Gitlab::Redis::MultiStore do allow(secondary_store).to receive(name).and_call_original end - context 'with feature flag :use_multi_store enabled' do + context 'with feature flag :use_primary_and_secondary_stores_for_test_store' do before do - stub_feature_flags(use_multi_store: true) + stub_feature_flags(use_primary_and_secondary_stores_for_test_store: true) end context 'when executing on primary instance is successful' do @@ -382,35 +428,57 @@ RSpec.describe Gitlab::Redis::MultiStore do end end - context 'with feature flag :use_multi_store is disabled' do + context 'with feature flag :use_primary_and_secondary_stores_for_test_store is disabled' do before do - stub_feature_flags(use_multi_store: false) + stub_feature_flags(use_primary_and_secondary_stores_for_test_store: false) end - it 'executes only on the secondary redis store', :aggregate_errors do - expect(secondary_store).to receive(name).with(*expected_args) - expect(primary_store).not_to receive(name).with(*expected_args) + context 'with feature flag :use_primary_store_as_default_for_test_store is disabled' do + before do + stub_feature_flags(use_primary_store_as_default_for_test_store: false) + end + + it 'executes only on the secondary redis store', :aggregate_errors do + expect(secondary_store).to receive(name).with(*expected_args) + expect(primary_store).not_to receive(name).with(*expected_args) + + subject + end - subject + include_examples 'verify that store contains values', :secondary_store end - include_examples 'verify that store contains values', :secondary_store + context 'with feature flag :use_primary_store_as_default_for_test_store is enabled' do + before do + stub_feature_flags(use_primary_store_as_default_for_test_store: true) + end + + it 'executes only on the primary_redis redis store', :aggregate_errors do + expect(primary_store).to receive(name).with(*expected_args) + expect(secondary_store).not_to receive(name).with(*expected_args) + + subject + end + + include_examples 'verify that store contains values', :primary_store + end end end end end context 'with unsupported command' do + let(:counter) { Gitlab::Metrics::NullMetric.instance } + before do primary_store.flushdb secondary_store.flushdb + allow(Gitlab::Metrics).to receive(:counter).and_return(counter) end let_it_be(:key) { "redis:counter" } - subject do - multi_store.incr(key) - end + subject { multi_store.incr(key) } it 'executes method missing' do expect(multi_store).to receive(:method_missing) @@ -418,31 +486,75 @@ RSpec.describe Gitlab::Redis::MultiStore do subject end - it 'logs MethodMissingError' do - expect(Gitlab::ErrorTracking).to receive(:log_exception).with(an_instance_of(Gitlab::Redis::MultiStore::MethodMissingError), - hash_including(command_name: :incr, extra: hash_including(instance_name: instance_name))) + context 'when command is not in SKIP_LOG_METHOD_MISSING_FOR_COMMANDS' do + it 'logs MethodMissingError' do + expect(Gitlab::ErrorTracking).to receive(:log_exception).with(an_instance_of(Gitlab::Redis::MultiStore::MethodMissingError), + hash_including(command_name: :incr, extra: hash_including(instance_name: instance_name))) - subject + subject + end + + it 'increments method missing counter' do + expect(counter).to receive(:increment).with(command: :incr, instance_name: instance_name) + + subject + end end - it 'increments method missing counter' do - expect(multi_store).to receive(:increment_method_missing_count).with(:incr) + context 'when command is in SKIP_LOG_METHOD_MISSING_FOR_COMMANDS' do + subject { multi_store.info } - subject + it 'does not log MethodMissingError' do + expect(Gitlab::ErrorTracking).not_to receive(:log_exception) + + subject + end + + it 'does not increment method missing counter' do + expect(counter).not_to receive(:increment) + + subject + end end - it 'fallback and executes only on the secondary store', :aggregate_errors do - expect(secondary_store).to receive(:incr).with(key).and_call_original - expect(primary_store).not_to receive(:incr) + context 'with feature flag :use_primary_store_as_default_for_test_store is enabled' do + before do + stub_feature_flags(use_primary_store_as_default_for_test_store: true) + end + + it 'fallback and executes only on the secondary store', :aggregate_errors do + expect(primary_store).to receive(:incr).with(key).and_call_original + expect(secondary_store).not_to receive(:incr) - subject + subject + end + + it 'correct value is stored on the secondary store', :aggregate_errors do + subject + + expect(secondary_store.get(key)).to be_nil + expect(primary_store.get(key)).to eq('1') + end end - it 'correct value is stored on the secondary store', :aggregate_errors do - subject + context 'with feature flag :use_primary_store_as_default_for_test_store is disabled' do + before do + stub_feature_flags(use_primary_store_as_default_for_test_store: false) + end + + it 'fallback and executes only on the secondary store', :aggregate_errors do + expect(secondary_store).to receive(:incr).with(key).and_call_original + expect(primary_store).not_to receive(:incr) + + subject + end + + it 'correct value is stored on the secondary store', :aggregate_errors do + subject - expect(primary_store.get(key)).to be_nil - expect(secondary_store.get(key)).to eq('1') + expect(primary_store.get(key)).to be_nil + expect(secondary_store.get(key)).to eq('1') + end end context 'when the command is executed within pipelined block' do @@ -468,6 +580,96 @@ RSpec.describe Gitlab::Redis::MultiStore do end end + describe '#to_s' do + subject { multi_store.to_s } + + context 'with feature flag :use_primary_and_secondary_stores_for_test_store is enabled' do + before do + stub_feature_flags(use_primary_and_secondary_stores_for_test_store: true) + end + + it 'returns same value as primary_store' do + is_expected.to eq(primary_store.to_s) + end + end + + context 'with feature flag :use_primary_and_secondary_stores_for_test_store is disabled' do + before do + stub_feature_flags(use_primary_and_secondary_stores_for_test_store: false) + end + + context 'with feature flag :use_primary_store_as_default_for_test_store is enabled' do + before do + stub_feature_flags(use_primary_store_as_default_for_test_store: true) + end + + it 'returns same value as primary_store' do + is_expected.to eq(primary_store.to_s) + end + end + + context 'with feature flag :use_primary_store_as_default_for_test_store is disabled' do + before do + stub_feature_flags(use_primary_store_as_default_for_test_store: false) + end + + it 'returns same value as primary_store' do + is_expected.to eq(secondary_store.to_s) + end + end + end + end + + describe '#is_a?' do + it 'returns true for ::Redis::Store' do + expect(multi_store.is_a?(::Redis::Store)).to be true + end + end + + describe '#use_primary_and_secondary_stores?' do + context 'with feature flag :use_primary_and_secondary_stores_for_test_store is enabled' do + before do + stub_feature_flags(use_primary_and_secondary_stores_for_test_store: true) + end + + it 'multi store is disabled' do + expect(multi_store.use_primary_and_secondary_stores?).to be true + end + end + + context 'with feature flag :use_primary_and_secondary_stores_for_test_store is disabled' do + before do + stub_feature_flags(use_primary_and_secondary_stores_for_test_store: false) + end + + it 'multi store is disabled' do + expect(multi_store.use_primary_and_secondary_stores?).to be false + end + end + end + + describe '#use_primary_store_as_default?' do + context 'with feature flag :use_primary_store_as_default_for_test_store is enabled' do + before do + stub_feature_flags(use_primary_store_as_default_for_test_store: true) + end + + it 'multi store is disabled' do + expect(multi_store.use_primary_store_as_default?).to be true + end + end + + context 'with feature flag :use_primary_store_as_default_for_test_store is disabled' do + before do + stub_feature_flags(use_primary_store_as_default_for_test_store: false) + end + + it 'multi store is disabled' do + expect(multi_store.use_primary_store_as_default?).to be false + end + end + end + def create_redis_store(options, extras = {}) ::Redis::Store.new(options.merge(extras)) end diff --git a/spec/lib/gitlab/redis/sessions_spec.rb b/spec/lib/gitlab/redis/sessions_spec.rb index 7e239c08e9f..6ecbbf3294d 100644 --- a/spec/lib/gitlab/redis/sessions_spec.rb +++ b/spec/lib/gitlab/redis/sessions_spec.rb @@ -3,5 +3,90 @@ require 'spec_helper' RSpec.describe Gitlab::Redis::Sessions do - include_examples "redis_new_instance_shared_examples", 'sessions', Gitlab::Redis::SharedState + it_behaves_like "redis_new_instance_shared_examples", 'sessions', Gitlab::Redis::SharedState + + describe 'redis instance used in connection pool' do + before do + clear_pool + end + + after do + clear_pool + end + + context 'when redis.sessions configuration is not provided' do + it 'uses ::Redis instance' do + expect(described_class).to receive(:config_fallback?).and_return(true) + + described_class.pool.with do |redis_instance| + expect(redis_instance).to be_instance_of(::Redis) + end + end + end + + context 'when redis.sessions configuration is provided' do + it 'instantiates an instance of MultiStore' do + expect(described_class).to receive(:config_fallback?).and_return(false) + + described_class.pool.with do |redis_instance| + expect(redis_instance).to be_instance_of(::Gitlab::Redis::MultiStore) + end + end + end + + def clear_pool + described_class.remove_instance_variable(:@pool) + rescue NameError + # raised if @pool was not set; ignore + end + end + + describe '#store' do + subject(:store) { described_class.store(namespace: described_class::SESSION_NAMESPACE) } + + context 'when redis.sessions configuration is NOT provided' do + it 'instantiates ::Redis instance' do + expect(described_class).to receive(:config_fallback?).and_return(true) + expect(store).to be_instance_of(::Redis::Store) + end + end + + context 'when redis.sessions configuration is provided' do + let(:config_new_format_host) { "spec/fixtures/config/redis_new_format_host.yml" } + let(:config_new_format_socket) { "spec/fixtures/config/redis_new_format_socket.yml" } + + before do + redis_clear_raw_config!(Gitlab::Redis::Sessions) + redis_clear_raw_config!(Gitlab::Redis::SharedState) + allow(described_class).to receive(:config_fallback?).and_return(false) + end + + after do + redis_clear_raw_config!(Gitlab::Redis::Sessions) + redis_clear_raw_config!(Gitlab::Redis::SharedState) + end + + # Check that Gitlab::Redis::Sessions is configured as MultiStore with proper attrs. + it 'instantiates an instance of MultiStore', :aggregate_failures do + expect(described_class).to receive(:config_file_name).and_return(config_new_format_host) + expect(::Gitlab::Redis::SharedState).to receive(:config_file_name).and_return(config_new_format_socket) + + expect(store).to be_instance_of(::Gitlab::Redis::MultiStore) + + expect(store.primary_store.to_s).to eq("Redis Client connected to test-host:6379 against DB 99 with namespace session:gitlab") + expect(store.secondary_store.to_s).to eq("Redis Client connected to /path/to/redis.sock against DB 0 with namespace session:gitlab") + + expect(store.instance_name).to eq('Sessions') + end + + context 'when MultiStore correctly configured' do + before do + allow(described_class).to receive(:config_file_name).and_return(config_new_format_host) + allow(::Gitlab::Redis::SharedState).to receive(:config_file_name).and_return(config_new_format_socket) + end + + it_behaves_like 'multi store feature flags', :use_primary_and_secondary_stores_for_sessions, :use_primary_store_as_default_for_sessions + end + end + end end diff --git a/spec/lib/gitlab/regex_spec.rb b/spec/lib/gitlab/regex_spec.rb index 05f1c88a6ab..83f85cc73d0 100644 --- a/spec/lib/gitlab/regex_spec.rb +++ b/spec/lib/gitlab/regex_spec.rb @@ -264,23 +264,37 @@ RSpec.describe Gitlab::Regex do it { is_expected.not_to match('1.2.3') } end - describe '.conan_recipe_component_regex' do - subject { described_class.conan_recipe_component_regex } - - let(:fifty_one_characters) { 'f_a' * 17} - - it { is_expected.to match('foobar') } - it { is_expected.to match('foo_bar') } - it { is_expected.to match('foo+bar') } - it { is_expected.to match('_foo+bar-baz+1.0') } - it { is_expected.to match('1.0.0') } - it { is_expected.not_to match('-foo_bar') } - it { is_expected.not_to match('+foo_bar') } - it { is_expected.not_to match('.foo_bar') } - it { is_expected.not_to match('foo@bar') } - it { is_expected.not_to match('foo/bar') } - it { is_expected.not_to match('!!()()') } - it { is_expected.not_to match(fifty_one_characters) } + context 'conan recipe components' do + shared_examples 'accepting valid recipe components values' do + let(:fifty_one_characters) { 'f_a' * 17} + + it { is_expected.to match('foobar') } + it { is_expected.to match('foo_bar') } + it { is_expected.to match('foo+bar') } + it { is_expected.to match('_foo+bar-baz+1.0') } + it { is_expected.to match('1.0.0') } + it { is_expected.not_to match('-foo_bar') } + it { is_expected.not_to match('+foo_bar') } + it { is_expected.not_to match('.foo_bar') } + it { is_expected.not_to match('foo@bar') } + it { is_expected.not_to match('foo/bar') } + it { is_expected.not_to match('!!()()') } + it { is_expected.not_to match(fifty_one_characters) } + end + + describe '.conan_recipe_component_regex' do + subject { described_class.conan_recipe_component_regex } + + it_behaves_like 'accepting valid recipe components values' + it { is_expected.not_to match('_') } + end + + describe '.conan_recipe_user_channel_regex' do + subject { described_class.conan_recipe_user_channel_regex } + + it_behaves_like 'accepting valid recipe components values' + it { is_expected.to match('_') } + end end describe '.package_name_regex' do diff --git a/spec/lib/gitlab/relative_positioning/range_spec.rb b/spec/lib/gitlab/relative_positioning/range_spec.rb index c3386336493..da1f0166d5d 100644 --- a/spec/lib/gitlab/relative_positioning/range_spec.rb +++ b/spec/lib/gitlab/relative_positioning/range_spec.rb @@ -3,8 +3,10 @@ require 'spec_helper' RSpec.describe Gitlab::RelativePositioning::Range do - item_a = OpenStruct.new(relative_position: 100, object: :x, positioned?: true) - item_b = OpenStruct.new(relative_position: 200, object: :y, positioned?: true) + position_struct = Struct.new(:relative_position, :object, :positioned?) + + item_a = position_struct.new(100, :x, true) + item_b = position_struct.new(200, :y, true) before do allow(item_a).to receive(:lhs_neighbour) { nil } @@ -90,12 +92,12 @@ RSpec.describe Gitlab::RelativePositioning::Range do end describe '#cover?' do - item_c = OpenStruct.new(relative_position: 150, object: :z, positioned?: true) - item_d = OpenStruct.new(relative_position: 050, object: :w, positioned?: true) - item_e = OpenStruct.new(relative_position: 250, object: :r, positioned?: true) - item_f = OpenStruct.new(positioned?: false) - item_ax = OpenStruct.new(relative_position: 100, object: :not_x, positioned?: true) - item_bx = OpenStruct.new(relative_position: 200, object: :not_y, positioned?: true) + item_c = position_struct.new(150, :z, true) + item_d = position_struct.new(050, :w, true) + item_e = position_struct.new(250, :r, true) + item_f = position_struct.new(positioned?: false) + item_ax = position_struct.new(100, :not_x, true) + item_bx = position_struct.new(200, :not_y, true) where(:lhs, :rhs, :item, :expected_result) do [ diff --git a/spec/lib/gitlab/repository_archive_rate_limiter_spec.rb b/spec/lib/gitlab/repository_archive_rate_limiter_spec.rb new file mode 100644 index 00000000000..49df70f3cb3 --- /dev/null +++ b/spec/lib/gitlab/repository_archive_rate_limiter_spec.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe ::Gitlab::RepositoryArchiveRateLimiter do + let(:described_class) do + Class.new do + include ::Gitlab::RepositoryArchiveRateLimiter + + def check_rate_limit!(**args) + end + end + end + + describe "#check_archive_rate_limit!" do + let(:project) { instance_double('Project') } + let(:current_user) { instance_double('User') } + let(:check) { subject.check_archive_rate_limit!(current_user, project) } + + context 'when archive_rate_limit feature flag is disabled' do + before do + stub_feature_flags(archive_rate_limit: false) + end + + it 'does not check rate limit' do + expect(subject).not_to receive(:check_rate_limit!) + + expect(check).to eq nil + end + end + + context 'when archive_rate_limit feature flag is enabled' do + before do + stub_feature_flags(archive_rate_limit: true) + end + + context 'when current user exists' do + it 'checks for project_repositories_archive rate limiting with default threshold' do + expect(subject).to receive(:check_rate_limit!) + .with(:project_repositories_archive, scope: [project, current_user], threshold: nil) + check + end + end + + context 'when current user does not exist' do + let(:current_user) { nil } + + it 'checks for project_repositories_archive rate limiting with threshold 100' do + expect(subject).to receive(:check_rate_limit!) + .with(:project_repositories_archive, scope: [project, current_user], threshold: 100) + check + end + end + end + end +end diff --git a/spec/lib/gitlab/saas_spec.rb b/spec/lib/gitlab/saas_spec.rb new file mode 100644 index 00000000000..1be36a60a97 --- /dev/null +++ b/spec/lib/gitlab/saas_spec.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Saas do + describe '.canary_toggle_com_url' do + subject { described_class.canary_toggle_com_url } + + let(:next_url) { 'https://next.gitlab.com' } + + it { is_expected.to eq(next_url) } + end +end diff --git a/spec/lib/gitlab/search/abuse_detection_spec.rb b/spec/lib/gitlab/search/abuse_detection_spec.rb new file mode 100644 index 00000000000..a18d28456cd --- /dev/null +++ b/spec/lib/gitlab/search/abuse_detection_spec.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Search::AbuseDetection do + subject { described_class.new(params) } + + let(:params) {{ query_string: 'foobar' }} + + describe 'abusive scopes validation' do + it 'allows only approved scopes' do + described_class::ALLOWED_SCOPES.each do |scope| + expect(described_class.new(scope: scope)).to be_valid + end + end + + it 'disallows anything not approved' do + expect(described_class.new(scope: 'nope')).not_to be_valid + end + end + + describe 'abusive character matching' do + refs = %w( + main + тест + maiñ + main123 + main-v123 + main-v12.3 + feature/it_works + really_important! + 测试 + ) + + refs.each do |ref| + it "does match refs permitted by git refname: #{ref}" do + [:repository_ref, :project_ref].each do |param| + validation = described_class.new(Hash[param, ref]) + expect(validation).to be_valid + end + end + + it "does NOT match refs with special characters: #{ref}" do + ['?', '\\', ' '].each do |special_character| + [:repository_ref, :project_ref].each do |param| + validation = described_class.new(Hash[param, ref + special_character]) + expect(validation).not_to be_valid + end + end + end + end + end + + describe 'numericality validation' do + it 'considers non Integers to be invalid' do + [:project_id, :group_id].each do |param| + [[1, 2, 3], 'xyz', 3.14, { foo: :bar }].each do |dtype| + expect(described_class.new(param => dtype)).not_to be_valid + end + end + end + + it 'considers Integers to be valid' do + [:project_id, :group_id].each do |param| + expect(described_class.new(param => 123)).to be_valid + end + end + end + + describe 'query_string validation' do + using ::RSpec::Parameterized::TableSyntax + + subject { described_class.new(query_string: search) } + + let(:validation_errors) do + subject.validate + subject.errors.messages + end + + where(:search, :errors) do + described_class::STOP_WORDS.each do |word| + word | { query_string: ['stopword only abusive search detected'] } + end + + 'x' | { query_string: ['abusive tiny search detected'] } + ('x' * described_class::ABUSIVE_TERM_SIZE) | { query_string: ['abusive term length detected'] } + '' | {} + '*' | {} + 'ruby' | {} + end + + with_them do + it 'validates query string for pointless search' do + expect(validation_errors).to eq(errors) + end + end + end + + describe 'abusive type coercion from string validation' do + it 'considers anything not a String invalid' do + [:query_string, :scope, :repository_ref, :project_ref].each do |param| + [[1, 2, 3], 123, 3.14, { foo: :bar }].each do |dtype| + expect(described_class.new(param => dtype)).not_to be_valid + end + end + end + + it 'considers Strings to be valid' do + [:query_string, :repository_ref, :project_ref].each do |param| + expect(described_class.new(param => "foo")).to be_valid + end + end + end +end diff --git a/spec/lib/gitlab/search/abuse_validators/no_abusive_coercion_from_string_validator_spec.rb b/spec/lib/gitlab/search/abuse_validators/no_abusive_coercion_from_string_validator_spec.rb new file mode 100644 index 00000000000..76280e65867 --- /dev/null +++ b/spec/lib/gitlab/search/abuse_validators/no_abusive_coercion_from_string_validator_spec.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Search::AbuseValidators::NoAbusiveCoercionFromStringValidator do + subject do + described_class.new({ attributes: { foo: :bar } }) + end + + let(:instance) { double(:instance) } + let(:attribute) { :attribute } + let(:validation_msg) { 'abusive coercion from string detected' } + let(:validate) { subject.validate_each(instance, attribute, attribute_value) } + + using ::RSpec::Parameterized::TableSyntax + + where(:attribute_value, :valid?) do + ['this is an arry'] | false + { 'this': 'is a hash' } | false + 123 | false + 456.78 | false + 'now this is a string' | true + end + + with_them do + it do + if valid? + expect(instance).not_to receive(:errors) + else + expect(instance).to receive_message_chain(:errors, :add).with(attribute, validation_msg) + validate + end + end + end +end diff --git a/spec/lib/gitlab/search/abuse_validators/no_abusive_term_length_validator_spec.rb b/spec/lib/gitlab/search/abuse_validators/no_abusive_term_length_validator_spec.rb new file mode 100644 index 00000000000..67409d9b628 --- /dev/null +++ b/spec/lib/gitlab/search/abuse_validators/no_abusive_term_length_validator_spec.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Search::AbuseValidators::NoAbusiveTermLengthValidator do + subject do + described_class.new({ attributes: { foo: :bar }, maximum: limit, maximum_for_url: url_limit }) + end + + let(:limit) { 100 } + let(:url_limit) { limit * 2 } + let(:instance) { double(:instance) } + let(:attribute) { :search } + let(:validation_msg) { 'abusive term length detected' } + let(:validate) { subject.validate_each(instance, attribute, search) } + + context 'when a term is over the limit' do + let(:search) { "this search is too lo#{'n' * limit}g" } + + it 'adds a validation error' do + expect(instance).to receive_message_chain(:errors, :add).with(attribute, validation_msg) + validate + end + end + + context 'when all terms are under the limit' do + let(:search) { "what is love? baby don't hurt me" } + + it 'does NOT add any validation errors' do + expect(instance).not_to receive(:errors) + validate + end + end + + context 'when a URL is detected in a search term' do + let(:double_limit) { limit * 2 } + let(:terms) do + [ + 'http://' + 'x' * (double_limit - 12) + '.com', + 'https://' + 'x' * (double_limit - 13) + '.com', + 'sftp://' + 'x' * (double_limit - 12) + '.com', + 'ftp://' + 'x' * (double_limit - 11) + '.com', + 'http://' + 'x' * (double_limit - 8) # no tld is OK + ] + end + + context 'when under twice the limit' do + let(:search) { terms.join(' ') } + + it 'does NOT add any validation errors' do + search.split.each do |term| + expect(term.length).to be < url_limit + end + + expect(instance).not_to receive(:errors) + validate + end + end + + context 'when over twice the limit' do + let(:search) do + terms.map { |t| t + 'xxxxxxxx' }.join(' ') + end + + it 'adds a validation error' do + expect(instance).to receive_message_chain(:errors, :add).with(attribute, validation_msg) + validate + end + end + end +end diff --git a/spec/lib/gitlab/search/params_spec.rb b/spec/lib/gitlab/search/params_spec.rb new file mode 100644 index 00000000000..6d15337b872 --- /dev/null +++ b/spec/lib/gitlab/search/params_spec.rb @@ -0,0 +1,136 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Search::Params do + subject { described_class.new(params, detect_abuse: detect_abuse) } + + let(:search) { 'search' } + let(:group_id) { 123 } + let(:params) { { group_id: 123, search: search } } + let(:detect_abuse) { true } + + describe 'detect_abuse conditional' do + it 'does not call AbuseDetection' do + expect(Gitlab::Search::AbuseDetection).not_to receive(:new) + described_class.new(params, detect_abuse: false) + end + + it 'uses AbuseDetection by default' do + expect(Gitlab::Search::AbuseDetection).to receive(:new).and_call_original + described_class.new(params) + end + end + + describe '#[]' do + it 'feels like regular params' do + expect(subject[:group_id]).to eq(params[:group_id]) + end + + it 'has indifferent access' do + params = described_class.new({ 'search' => search, group_id: group_id }) + expect(params['group_id']).to eq(group_id) + expect(params[:search]).to eq(search) + end + + it 'also works on attr_reader attributes' do + expect(subject[:query_string]).to eq(subject.query_string) + end + end + + describe '#query_string' do + let(:term) { 'term' } + + it "uses 'search' parameter" do + params = described_class.new({ search: search }) + expect(params.query_string).to eq(search) + end + + it "uses 'term' parameter" do + params = described_class.new({ term: term }) + expect(params.query_string).to eq(term) + end + + it "prioritizes 'search' over 'term'" do + params = described_class.new({ search: search, term: term }) + expect(params.query_string).to eq(search) + end + + it 'strips surrounding whitespace from query string' do + params = described_class.new({ search: ' ' + search + ' ' }) + expect(params.query_string).to eq(search) + end + end + + describe '#validate' do + context 'when detect_abuse is disabled' do + let(:detect_abuse) { false } + + it 'does NOT validate AbuseDetector' do + expect(Gitlab::Search::AbuseDetection).not_to receive(:new) + subject.validate + end + end + + it 'validates AbuseDetector on validation' do + expect(Gitlab::Search::AbuseDetection).to receive(:new).and_call_original + subject.validate + end + end + + describe '#valid?' do + context 'when detect_abuse is disabled' do + let(:detect_abuse) { false } + + it 'does NOT validate AbuseDetector' do + expect(Gitlab::Search::AbuseDetection).not_to receive(:new) + subject.valid? + end + end + + it 'validates AbuseDetector on validation' do + expect(Gitlab::Search::AbuseDetection).to receive(:new).and_call_original + subject.valid? + end + end + + describe 'abuse detection' do + let(:abuse_detection) { instance_double(Gitlab::Search::AbuseDetection) } + + before do + allow(subject).to receive(:abuse_detection).and_return abuse_detection + allow(abuse_detection).to receive(:errors).and_return abuse_errors + end + + context 'when there are abuse validation errors' do + let(:abuse_errors) { { foo: ['bar'] } } + + it 'is considered abusive' do + expect(subject).to be_abusive + end + end + + context 'when there are NOT any abuse validation errors' do + let(:abuse_errors) { {} } + + context 'and there are other validation errors' do + it 'is NOT considered abusive' do + allow(subject).to receive(:valid?) do + subject.errors.add :project_id, 'validation error unrelated to abuse' + false + end + + expect(subject).not_to be_abusive + end + end + + context 'and there are NO other validation errors' do + it 'is NOT considered abusive' do + allow(subject).to receive(:valid?).and_return(true) + + expect(subject).not_to be_abusive + end + end + end + end +end diff --git a/spec/lib/gitlab/security/scan_configuration_spec.rb b/spec/lib/gitlab/security/scan_configuration_spec.rb new file mode 100644 index 00000000000..0af029968e8 --- /dev/null +++ b/spec/lib/gitlab/security/scan_configuration_spec.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe ::Gitlab::Security::ScanConfiguration do + let_it_be(:project) { create(:project, :repository) } + + let(:scan) { described_class.new(project: project, type: type, configured: configured) } + + describe '#available?' do + subject { scan.available? } + + let(:configured) { true } + + context 'with a core scanner' do + let(:type) { :sast } + + it { is_expected.to be_truthy } + end + + context 'with custom scanner' do + let(:type) { :my_scanner } + + it { is_expected.to be_falsey } + end + end + + describe '#configured?' do + subject { scan.configured? } + + let(:type) { :sast } + let(:configured) { false } + + it { is_expected.to be_falsey } + end + + describe '#configuration_path' do + subject { scan.configuration_path } + + let(:configured) { true } + + context 'with a non configurable scanner' do + let(:type) { :secret_detection } + + it { is_expected.to be_nil } + end + + context 'with licensed scanner for FOSS environment' do + let(:type) { :dast } + + before do + stub_env('FOSS_ONLY', '1') + end + + it { is_expected.to be_nil } + end + + context 'with custom scanner' do + let(:type) { :my_scanner } + + it { is_expected.to be_nil } + end + end +end diff --git a/spec/lib/gitlab/sidekiq_enq_spec.rb b/spec/lib/gitlab/sidekiq_enq_spec.rb deleted file mode 100644 index 6903f01bf5f..00000000000 --- a/spec/lib/gitlab/sidekiq_enq_spec.rb +++ /dev/null @@ -1,93 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::SidekiqEnq, :clean_gitlab_redis_queues do - let(:retry_set) { Sidekiq::Scheduled::SETS.first } - let(:schedule_set) { Sidekiq::Scheduled::SETS.last } - - around do |example| - freeze_time { example.run } - end - - shared_examples 'finds jobs that are due and enqueues them' do - before do - Sidekiq.redis do |redis| - redis.zadd(retry_set, (Time.current - 1.day).to_f.to_s, '{"jid": 1}') - redis.zadd(retry_set, Time.current.to_f.to_s, '{"jid": 2}') - redis.zadd(retry_set, (Time.current + 1.day).to_f.to_s, '{"jid": 3}') - - redis.zadd(schedule_set, (Time.current - 1.day).to_f.to_s, '{"jid": 4}') - redis.zadd(schedule_set, Time.current.to_f.to_s, '{"jid": 5}') - redis.zadd(schedule_set, (Time.current + 1.day).to_f.to_s, '{"jid": 6}') - end - end - - it 'enqueues jobs that are due' do - expect(Sidekiq::Client).to receive(:push).with({ 'jid' => 1 }) - expect(Sidekiq::Client).to receive(:push).with({ 'jid' => 2 }) - expect(Sidekiq::Client).to receive(:push).with({ 'jid' => 4 }) - expect(Sidekiq::Client).to receive(:push).with({ 'jid' => 5 }) - - Gitlab::SidekiqEnq.new.enqueue_jobs - - Sidekiq.redis do |redis| - expect(redis.zscan_each(retry_set).map(&:first)).to contain_exactly('{"jid": 3}') - expect(redis.zscan_each(schedule_set).map(&:first)).to contain_exactly('{"jid": 6}') - end - end - end - - context 'when atomic_sidekiq_scheduler is disabled' do - before do - stub_feature_flags(atomic_sidekiq_scheduler: false) - end - - it_behaves_like 'finds jobs that are due and enqueues them' - - context 'when ZRANGEBYSCORE returns a job that is already removed by another process' do - before do - Sidekiq.redis do |redis| - redis.zadd(schedule_set, Time.current.to_f.to_s, '{"jid": 1}') - - allow(redis).to receive(:zrangebyscore).and_wrap_original do |m, *args, **kwargs| - m.call(*args, **kwargs).tap do |jobs| - redis.zrem(schedule_set, jobs.first) if args[0] == schedule_set && jobs.first - end - end - end - end - - it 'calls ZREM but does not enqueue the job' do - Sidekiq.redis do |redis| - expect(redis).to receive(:zrem).with(schedule_set, '{"jid": 1}').twice.and_call_original - end - expect(Sidekiq::Client).not_to receive(:push) - - Gitlab::SidekiqEnq.new.enqueue_jobs - end - end - end - - context 'when atomic_sidekiq_scheduler is enabled' do - before do - stub_feature_flags(atomic_sidekiq_scheduler: true) - end - - context 'when Lua script is not yet loaded' do - before do - Gitlab::Redis::Queues.with { |redis| redis.script(:flush) } - end - - it_behaves_like 'finds jobs that are due and enqueues them' - end - - context 'when Lua script is already loaded' do - before do - Gitlab::SidekiqEnq.new.enqueue_jobs - end - - it_behaves_like 'finds jobs that are due and enqueues them' - end - end -end diff --git a/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb b/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb index d801b84775b..210b9162be0 100644 --- a/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb +++ b/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb @@ -272,12 +272,12 @@ RSpec.describe Gitlab::SidekiqLogging::StructuredLogger do expected_end_payload.merge( 'db_duration_s' => a_value >= 0.1, 'db_count' => a_value >= 1, - "db_replica_#{db_config_name}_count" => 0, + "db_#{db_config_name}_replica_count" => 0, 'db_replica_duration_s' => a_value >= 0, 'db_primary_count' => a_value >= 1, - "db_primary_#{db_config_name}_count" => a_value >= 1, + "db_#{db_config_name}_count" => a_value >= 1, 'db_primary_duration_s' => a_value > 0, - "db_primary_#{db_config_name}_duration_s" => a_value > 0 + "db_#{db_config_name}_duration_s" => a_value > 0 ) end diff --git a/spec/lib/gitlab/sidekiq_middleware/server_metrics_spec.rb b/spec/lib/gitlab/sidekiq_middleware/server_metrics_spec.rb index 914f5a30c3a..3fbd207c2e1 100644 --- a/spec/lib/gitlab/sidekiq_middleware/server_metrics_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware/server_metrics_spec.rb @@ -239,6 +239,8 @@ RSpec.describe Gitlab::SidekiqMiddleware::ServerMetrics do shared_context 'worker declaring data consistency' do let(:worker_class) { LBTestWorker } + let(:wal_locations) { { Gitlab::Database::MAIN_DATABASE_NAME.to_sym => 'AB/12345' } } + let(:job) { { "retry" => 3, "job_id" => "a180b47c-3fd6-41b8-81e9-34da61c3400e", "wal_locations" => wal_locations } } before do stub_const('LBTestWorker', Class.new(TestWorker)) diff --git a/spec/lib/gitlab/sidekiq_status/client_middleware_spec.rb b/spec/lib/gitlab/sidekiq_status/client_middleware_spec.rb index 0cf05fb0a5c..2f2499753b9 100644 --- a/spec/lib/gitlab/sidekiq_status/client_middleware_spec.rb +++ b/spec/lib/gitlab/sidekiq_status/client_middleware_spec.rb @@ -1,14 +1,25 @@ # frozen_string_literal: true -require 'spec_helper' +require 'fast_spec_helper' RSpec.describe Gitlab::SidekiqStatus::ClientMiddleware do describe '#call' do - it 'tracks the job in Redis' do - expect(Gitlab::SidekiqStatus).to receive(:set).with('123', Gitlab::SidekiqStatus::DEFAULT_EXPIRATION) + context 'when the job has status_expiration set' do + it 'tracks the job in Redis with a value of 2' do + expect(Gitlab::SidekiqStatus).to receive(:set).with('123', 1.hour.to_i, value: 2) - described_class.new - .call('Foo', { 'jid' => '123' }, double(:queue), double(:pool)) { nil } + described_class.new + .call('Foo', { 'jid' => '123', 'status_expiration' => 1.hour.to_i }, double(:queue), double(:pool)) { nil } + end + end + + context 'when the job does not have status_expiration set' do + it 'tracks the job in Redis with a value of 1' do + expect(Gitlab::SidekiqStatus).to receive(:set).with('123', Gitlab::SidekiqStatus::DEFAULT_EXPIRATION, value: 1) + + described_class.new + .call('Foo', { 'jid' => '123' }, double(:queue), double(:pool)) { nil } + end end end end diff --git a/spec/lib/gitlab/sidekiq_status_spec.rb b/spec/lib/gitlab/sidekiq_status_spec.rb index fc2ac29a1f9..1e7b52471b0 100644 --- a/spec/lib/gitlab/sidekiq_status_spec.rb +++ b/spec/lib/gitlab/sidekiq_status_spec.rb @@ -12,6 +12,31 @@ RSpec.describe Gitlab::SidekiqStatus, :clean_gitlab_redis_queues, :clean_gitlab_ Sidekiq.redis do |redis| expect(redis.exists(key)).to eq(true) expect(redis.ttl(key) > 0).to eq(true) + expect(redis.get(key)).to eq(described_class::DEFAULT_VALUE.to_s) + end + end + + it 'allows overriding the expiration time' do + described_class.set('123', described_class::DEFAULT_EXPIRATION * 2) + + key = described_class.key_for('123') + + Sidekiq.redis do |redis| + expect(redis.exists(key)).to eq(true) + expect(redis.ttl(key) > described_class::DEFAULT_EXPIRATION).to eq(true) + expect(redis.get(key)).to eq(described_class::DEFAULT_VALUE.to_s) + end + end + + it 'allows overriding the default value' do + described_class.set('123', value: 2) + + key = described_class.key_for('123') + + Sidekiq.redis do |redis| + expect(redis.exists(key)).to eq(true) + expect(redis.ttl(key) > 0).to eq(true) + expect(redis.get(key)).to eq('2') end end end @@ -88,7 +113,7 @@ RSpec.describe Gitlab::SidekiqStatus, :clean_gitlab_redis_queues, :clean_gitlab_ end end - describe 'completed' do + describe '.completed_jids' do it 'returns the completed job' do expect(described_class.completed_jids(%w(123))).to eq(['123']) end @@ -100,4 +125,46 @@ RSpec.describe Gitlab::SidekiqStatus, :clean_gitlab_redis_queues, :clean_gitlab_ expect(described_class.completed_jids(%w(123 456 789))).to eq(['789']) end end + + describe '.job_status' do + it 'returns an array of boolean values' do + described_class.set('123') + described_class.set('456') + described_class.unset('123') + + expect(described_class.job_status(%w(123 456 789))).to eq([false, true, false]) + end + + it 'handles an empty array' do + expect(described_class.job_status([])).to eq([]) + end + + context 'when log_implicit_sidekiq_status_calls is enabled' do + it 'logs keys that contained the default value' do + described_class.set('123', value: 2) + described_class.set('456') + described_class.set('012') + + expect(Sidekiq.logger).to receive(:info).with(message: described_class::DEFAULT_VALUE_MESSAGE, + keys: [described_class.key_for('456'), described_class.key_for('012')]) + + expect(described_class.job_status(%w(123 456 789 012))).to eq([true, true, false, true]) + end + end + + context 'when log_implicit_sidekiq_status_calls is disabled' do + before do + stub_feature_flags(log_implicit_sidekiq_status_calls: false) + end + + it 'does not perform any logging' do + described_class.set('123', value: 2) + described_class.set('456') + + expect(Sidekiq.logger).not_to receive(:info) + + expect(described_class.job_status(%w(123 456 789))).to eq([true, true, false]) + end + end + end end diff --git a/spec/lib/gitlab/spamcheck/client_spec.rb b/spec/lib/gitlab/spamcheck/client_spec.rb index e542ce455bb..a6e7665569c 100644 --- a/spec/lib/gitlab/spamcheck/client_spec.rb +++ b/spec/lib/gitlab/spamcheck/client_spec.rb @@ -32,6 +32,30 @@ RSpec.describe Gitlab::Spamcheck::Client do stub_application_setting(spam_check_endpoint_url: endpoint) end + describe 'url scheme' do + let(:stub) { double(:spamcheck_stub, check_for_spam_issue: response) } + + context 'is tls ' do + let(:endpoint) { 'tls://spamcheck.example.com'} + + it 'uses secure connection' do + expect(Spamcheck::SpamcheckService::Stub).to receive(:new).with(endpoint.sub(%r{^tls://}, ''), + instance_of(GRPC::Core::ChannelCredentials), + anything).and_return(stub) + subject + end + end + + context 'is grpc' do + it 'uses insecure connection' do + expect(Spamcheck::SpamcheckService::Stub).to receive(:new).with(endpoint.sub(%r{^grpc://}, ''), + :this_channel_is_insecure, + anything).and_return(stub) + subject + end + end + end + describe '#issue_spam?' do before do allow_next_instance_of(::Spamcheck::SpamcheckService::Stub) do |instance| diff --git a/spec/lib/gitlab/subscription_portal_spec.rb b/spec/lib/gitlab/subscription_portal_spec.rb index 4be1c85f7c8..627d3bb42c7 100644 --- a/spec/lib/gitlab/subscription_portal_spec.rb +++ b/spec/lib/gitlab/subscription_portal_spec.rb @@ -56,6 +56,7 @@ RSpec.describe ::Gitlab::SubscriptionPortal do where(:method_name, :result) do :default_subscriptions_url | 'https://customers.staging.gitlab.com' :payment_form_url | 'https://customers.staging.gitlab.com/payment_forms/cc_validation' + :registration_validation_form_url | 'https://customers.staging.gitlab.com/payment_forms/cc_registration_validation' :subscriptions_graphql_url | 'https://customers.staging.gitlab.com/graphql' :subscriptions_more_minutes_url | 'https://customers.staging.gitlab.com/buy_pipeline_minutes' :subscriptions_more_storage_url | 'https://customers.staging.gitlab.com/buy_storage' @@ -63,6 +64,7 @@ RSpec.describe ::Gitlab::SubscriptionPortal do :subscriptions_plans_url | 'https://about.gitlab.com/pricing/' :subscriptions_instance_review_url | 'https://customers.staging.gitlab.com/instance_review' :subscriptions_gitlab_plans_url | 'https://customers.staging.gitlab.com/gitlab_plans' + :edit_account_url | 'https://customers.staging.gitlab.com/customers/edit' end with_them do diff --git a/spec/lib/gitlab/tracking/destinations/snowplow_micro_spec.rb b/spec/lib/gitlab/tracking/destinations/snowplow_micro_spec.rb index 6004698d092..2b94eaa2db9 100644 --- a/spec/lib/gitlab/tracking/destinations/snowplow_micro_spec.rb +++ b/spec/lib/gitlab/tracking/destinations/snowplow_micro_spec.rb @@ -48,4 +48,24 @@ RSpec.describe Gitlab::Tracking::Destinations::SnowplowMicro do end end end + + describe '#options' do + let_it_be(:group) { create :group } + + before do + stub_env('SNOWPLOW_MICRO_URI', 'http://gdk.test:9091') + end + + it 'includes protocol with the correct value' do + expect(subject.options(group)[:protocol]).to eq 'http' + end + + it 'includes port with the correct value' do + expect(subject.options(group)[:port]).to eq 9091 + end + + it 'includes forceSecureTracker with value false' do + expect(subject.options(group)[:forceSecureTracker]).to eq false + end + end end diff --git a/spec/lib/gitlab/tracking/destinations/snowplow_spec.rb b/spec/lib/gitlab/tracking/destinations/snowplow_spec.rb index f8e73a807c6..06cc2d3800c 100644 --- a/spec/lib/gitlab/tracking/destinations/snowplow_spec.rb +++ b/spec/lib/gitlab/tracking/destinations/snowplow_spec.rb @@ -29,7 +29,7 @@ RSpec.describe Gitlab::Tracking::Destinations::Snowplow do expect(SnowplowTracker::Tracker) .to receive(:new) - .with(emitter, an_instance_of(SnowplowTracker::Subject), Gitlab::Tracking::SNOWPLOW_NAMESPACE, '_abc123_') + .with(emitter, an_instance_of(SnowplowTracker::Subject), described_class::SNOWPLOW_NAMESPACE, '_abc123_') .and_return(tracker) end diff --git a/spec/lib/gitlab/tracking_spec.rb b/spec/lib/gitlab/tracking_spec.rb index 61b2c89ffa1..cd83971aef9 100644 --- a/spec/lib/gitlab/tracking_spec.rb +++ b/spec/lib/gitlab/tracking_spec.rb @@ -13,6 +13,10 @@ RSpec.describe Gitlab::Tracking do described_class.instance_variable_set("@snowplow", nil) end + after do + described_class.instance_variable_set("@snowplow", nil) + end + describe '.options' do shared_examples 'delegates to destination' do |klass| before do @@ -63,7 +67,7 @@ RSpec.describe Gitlab::Tracking do appId: '_abc123_', protocol: 'http', port: 9090, - force_secure_tracker: false, + forceSecureTracker: false, formTracking: true, linkClickTracking: true } diff --git a/spec/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll_spec.rb b/spec/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll_spec.rb index 1ae4c9414dd..59b944ac398 100644 --- a/spec/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll_spec.rb +++ b/spec/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll_spec.rb @@ -11,6 +11,7 @@ RSpec.describe Gitlab::Usage::Metrics::Aggregates::Sources::PostgresHll, :clean_ let(:metric_1) { 'metric_1' } let(:metric_2) { 'metric_2' } let(:metric_names) { [metric_1, metric_2] } + let(:error_rate) { Gitlab::Database::PostgresHll::BatchDistinctCounter::ERROR_RATE } describe 'metric calculations' do before do @@ -38,7 +39,7 @@ RSpec.describe Gitlab::Usage::Metrics::Aggregates::Sources::PostgresHll, :clean_ end it 'returns the number of unique events in the union of all metrics' do - expect(calculate_metrics_union.round(2)).to eq(3.12) + expect(calculate_metrics_union.round(2)).to be_within(error_rate).percent_of(3) end context 'when there is no aggregated data saved' do @@ -53,7 +54,7 @@ RSpec.describe Gitlab::Usage::Metrics::Aggregates::Sources::PostgresHll, :clean_ let(:metric_names) { [metric_1] } it 'returns the number of unique events for that metric' do - expect(calculate_metrics_union.round(2)).to eq(2.08) + expect(calculate_metrics_union.round(2)).to be_within(error_rate).percent_of(2) end end end @@ -64,7 +65,7 @@ RSpec.describe Gitlab::Usage::Metrics::Aggregates::Sources::PostgresHll, :clean_ end it 'returns the number of common events in the intersection of all metrics' do - expect(calculate_metrics_intersections.round(2)).to eq(1.04) + expect(calculate_metrics_intersections.round(2)).to be_within(error_rate).percent_of(1) end context 'when there is no aggregated data saved' do @@ -79,7 +80,7 @@ RSpec.describe Gitlab::Usage::Metrics::Aggregates::Sources::PostgresHll, :clean_ let(:metric_names) { [metric_1] } it 'returns the number of common/unique events for the intersection of that metric' do - expect(calculate_metrics_intersections.round(2)).to eq(2.08) + expect(calculate_metrics_intersections.round(2)).to be_within(error_rate).percent_of(2) end end end diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/snowplow_configured_to_gitlab_collector_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/snowplow_configured_to_gitlab_collector_metric_spec.rb new file mode 100644 index 00000000000..c9bc101374f --- /dev/null +++ b/spec/lib/gitlab/usage/metrics/instrumentations/snowplow_configured_to_gitlab_collector_metric_spec.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Usage::Metrics::Instrumentations::SnowplowConfiguredToGitlabCollectorMetric do + using RSpec::Parameterized::TableSyntax + + context 'for collector_hostname option' do + where(:collector_hostname, :expected_value) do + 'snowplow.trx.gitlab.net' | true + 'foo.bar.something.net' | false + end + + with_them do + before do + stub_application_setting(snowplow_collector_hostname: collector_hostname) + end + + it_behaves_like 'a correct instrumented metric value', { time_frame: 'none' } + end + end +end diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/snowplow_enabled_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/snowplow_enabled_metric_spec.rb new file mode 100644 index 00000000000..1e0cdd650fa --- /dev/null +++ b/spec/lib/gitlab/usage/metrics/instrumentations/snowplow_enabled_metric_spec.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Usage::Metrics::Instrumentations::SnowplowEnabledMetric do + using RSpec::Parameterized::TableSyntax + + context 'for snowplow enabled option' do + where(:snowplow_enabled, :expected_value) do + true | true + false | false + end + + with_them do + before do + stub_application_setting(snowplow_enabled: snowplow_enabled) + end + + it_behaves_like 'a correct instrumented metric value', { time_frame: 'none' } + end + end +end diff --git a/spec/lib/gitlab/usage/metrics/names_suggestions/generator_spec.rb b/spec/lib/gitlab/usage/metrics/names_suggestions/generator_spec.rb index dbbc718e147..0f95da74ff9 100644 --- a/spec/lib/gitlab/usage/metrics/names_suggestions/generator_spec.rb +++ b/spec/lib/gitlab/usage/metrics/names_suggestions/generator_spec.rb @@ -25,30 +25,10 @@ RSpec.describe Gitlab::Usage::Metrics::NamesSuggestions::Generator do end context 'for count with default column metrics' do - context 'with usage_data_instrumentation feature flag' do - context 'when enabled' do - before do - stub_feature_flags(usage_data_instrumentation: true) - end - - it_behaves_like 'name suggestion' do - # corresponding metric is collected with ::Gitlab::UsageDataMetrics.suggested_names - let(:key_path) { 'counts.boards' } - let(:name_suggestion) { /count_boards/ } - end - end - - context 'when disabled' do - before do - stub_feature_flags(usage_data_instrumentation: false) - end - - it_behaves_like 'name suggestion' do - # corresponding metric is collected with count(Board) - let(:key_path) { 'counts.boards' } - let(:name_suggestion) { /count_boards/ } - end - end + it_behaves_like 'name suggestion' do + # corresponding metric is collected with count(Board) + let(:key_path) { 'counts.boards' } + let(:name_suggestion) { /count_boards/ } end end diff --git a/spec/lib/gitlab/usage_data_spec.rb b/spec/lib/gitlab/usage_data_spec.rb index cf544c07195..015ecd1671e 100644 --- a/spec/lib/gitlab/usage_data_spec.rb +++ b/spec/lib/gitlab/usage_data_spec.rb @@ -80,12 +80,6 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do end end end - - it 'allows indifferent access' do - allow(::Gitlab::UsageDataCounters::HLLRedisCounter).to receive(:unique_events).and_return(1) - expect(subject[:search_unique_visits][:search_unique_visits_for_any_target_monthly]).to eq(1) - expect(subject[:search_unique_visits]['search_unique_visits_for_any_target_monthly']).to eq(1) - end end describe 'usage_activity_by_stage_package' do @@ -205,7 +199,6 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do for_defined_days_back do user = create(:user) user2 = create(:user) - create(:event, author: user) create(:group_member, user: user) create(:authentication_event, user: user, provider: :ldapmain, result: :success) create(:authentication_event, user: user2, provider: :ldapsecondary, result: :success) @@ -214,17 +207,24 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do create(:authentication_event, user: user, provider: :group_saml, result: :failed) end + for_defined_days_back(days: [31, 29, 3]) do + create(:event) + end + + stub_const('Gitlab::Database::PostgresHll::BatchDistinctCounter::DEFAULT_BATCH_SIZE', 1) + stub_const('Gitlab::Database::PostgresHll::BatchDistinctCounter::MIN_REQUIRED_BATCH_SIZE', 0) + expect(described_class.usage_activity_by_stage_manage({})).to include( events: -1, groups: 2, - users_created: 6, + users_created: 10, omniauth_providers: ['google_oauth2'], user_auth_by_provider: { 'group_saml' => 2, 'ldap' => 4, 'standard' => 0, 'two-factor' => 0, 'two-factor-via-u2f-device' => 0, "two-factor-via-webauthn-device" => 0 } ) expect(described_class.usage_activity_by_stage_manage(described_class.monthly_time_range_db_params)).to include( - events: be_within(error_rate).percent_of(1), + events: be_within(error_rate).percent_of(2), groups: 1, - users_created: 3, + users_created: 6, omniauth_providers: ['google_oauth2'], user_auth_by_provider: { 'group_saml' => 1, 'ldap' => 2, 'standard' => 0, 'two-factor' => 0, 'two-factor-via-u2f-device' => 0, "two-factor-via-webauthn-device" => 0 } ) @@ -457,42 +457,16 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do ) end - context 'with usage_data_instrumentation feature flag' do - context 'when enabled' do - it 'merges the data from instrumentation classes' do - stub_feature_flags(usage_data_instrumentation: true) - - for_defined_days_back do - user = create(:user) - project = create(:project, creator: user) - create(:issue, project: project, author: user) - create(:issue, project: project, author: User.support_bot) - end - - expect(described_class.usage_activity_by_stage_plan({})).to include(issues: Gitlab::Utils::UsageData::INSTRUMENTATION_CLASS_FALLBACK) - expect(described_class.usage_activity_by_stage_plan(described_class.monthly_time_range_db_params)).to include(issues: Gitlab::Utils::UsageData::INSTRUMENTATION_CLASS_FALLBACK) - - uncached_data = described_class.uncached_data - expect(uncached_data[:usage_activity_by_stage][:plan]).to include(issues: 3) - expect(uncached_data[:usage_activity_by_stage_monthly][:plan]).to include(issues: 2) - end + it 'does not merge the data from instrumentation classes' do + for_defined_days_back do + user = create(:user) + project = create(:project, creator: user) + create(:issue, project: project, author: user) + create(:issue, project: project, author: User.support_bot) end - context 'when disabled' do - it 'does not merge the data from instrumentation classes' do - stub_feature_flags(usage_data_instrumentation: false) - - for_defined_days_back do - user = create(:user) - project = create(:project, creator: user) - create(:issue, project: project, author: user) - create(:issue, project: project, author: User.support_bot) - end - - expect(described_class.usage_activity_by_stage_plan({})).to include(issues: 3) - expect(described_class.usage_activity_by_stage_plan(described_class.monthly_time_range_db_params)).to include(issues: 2) - end - end + expect(described_class.usage_activity_by_stage_plan({})).to include(issues: 3) + expect(described_class.usage_activity_by_stage_plan(described_class.monthly_time_range_db_params)).to include(issues: 2) end end @@ -510,53 +484,17 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do deployments: 2, failed_deployments: 2, releases: 2, - successful_deployments: 2 + successful_deployments: 2, + releases_with_milestones: 2 ) expect(described_class.usage_activity_by_stage_release(described_class.monthly_time_range_db_params)).to include( deployments: 1, failed_deployments: 1, releases: 1, - successful_deployments: 1 + successful_deployments: 1, + releases_with_milestones: 1 ) end - - context 'with usage_data_instrumentation feature flag' do - before do - for_defined_days_back do - user = create(:user) - create(:deployment, :failed, user: user) - release = create(:release, author: user) - create(:milestone, project: release.project, releases: [release]) - create(:deployment, :success, user: user) - end - end - - context 'when enabled' do - before do - stub_feature_flags(usage_data_instrumentation: true) - end - - it 'merges data from instrumentation classes' do - expect(described_class.usage_activity_by_stage_release({})).to include(releases_with_milestones: Gitlab::Utils::UsageData::INSTRUMENTATION_CLASS_FALLBACK) - expect(described_class.usage_activity_by_stage_release(described_class.monthly_time_range_db_params)).to include(releases_with_milestones: Gitlab::Utils::UsageData::INSTRUMENTATION_CLASS_FALLBACK) - - uncached_data = described_class.uncached_data - expect(uncached_data[:usage_activity_by_stage][:release]).to include(releases_with_milestones: 2) - expect(uncached_data[:usage_activity_by_stage_monthly][:release]).to include(releases_with_milestones: 1) - end - end - - context 'when disabled' do - before do - stub_feature_flags(usage_data_instrumentation: false) - end - - it 'does not merge data from instrumentation classes' do - expect(described_class.usage_activity_by_stage_release({})).to include(releases_with_milestones: 2) - expect(described_class.usage_activity_by_stage_release(described_class.monthly_time_range_db_params)).to include(releases_with_milestones: 1) - end - end - end end describe 'usage_activity_by_stage_verify' do @@ -605,16 +543,15 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do subject { described_class.data } it 'gathers usage data' do - expect(subject.keys).to include(*UsageDataHelpers::USAGE_DATA_KEYS.map(&:to_s)) + expect(subject.keys).to include(*UsageDataHelpers::USAGE_DATA_KEYS) end it 'gathers usage counts', :aggregate_failures do count_data = subject[:counts] expect(count_data[:boards]).to eq(1) expect(count_data[:projects]).to eq(4) - count_keys = UsageDataHelpers::COUNTS_KEYS.map(&:to_s) - expect(count_data.keys).to include(*count_keys) - expect(count_keys - count_data.keys).to be_empty + expect(count_data.keys).to include(*UsageDataHelpers::COUNTS_KEYS) + expect(UsageDataHelpers::COUNTS_KEYS - count_data.keys).to be_empty expect(count_data.values).to all(be_a_kind_of(Integer)) end @@ -699,7 +636,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do external_diffs: { enabled: false }, lfs: { enabled: true, object_store: { enabled: false, direct_upload: true, background_upload: false, provider: "AWS" } }, uploads: { enabled: nil, object_store: { enabled: false, direct_upload: true, background_upload: false, provider: "AWS" } }, - packages: { enabled: true, object_store: { enabled: false, direct_upload: false, background_upload: true, provider: "AWS" } } }.with_indifferent_access + packages: { enabled: true, object_store: { enabled: false, direct_upload: false, background_upload: true, provider: "AWS" } } } ) end @@ -747,23 +684,50 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do end end - it 'works when queries time out' do - allow_any_instance_of(ActiveRecord::Relation) - .to receive(:count).and_raise(ActiveRecord::StatementInvalid.new('')) + context 'when queries time out' do + let(:metric_method) { :count } + + before do + allow_any_instance_of(ActiveRecord::Relation).to receive(metric_method).and_raise(ActiveRecord::StatementInvalid) + allow(Gitlab::ErrorTracking).to receive(:should_raise_for_dev?).and_return(should_raise_for_dev) + end + + context 'with should_raise_for_dev? true' do + let(:should_raise_for_dev) { true } + + it 'raises an error' do + expect { subject }.to raise_error(ActiveRecord::StatementInvalid) + end + + context 'when metric calls find_in_batches' do + let(:metric_method) { :find_in_batches } + + it 'raises an error for jira_usage' do + expect { described_class.jira_usage }.to raise_error(ActiveRecord::StatementInvalid) + end + end + end + + context 'with should_raise_for_dev? false' do + let(:should_raise_for_dev) { false } + + it 'does not raise an error' do + expect { subject }.not_to raise_error + end - expect { subject }.not_to raise_error + context 'when metric calls find_in_batches' do + let(:metric_method) { :find_in_batches } + + it 'does not raise an error for jira_usage' do + expect { described_class.jira_usage }.not_to raise_error + end + end + end end it 'includes a recording_ce_finished_at timestamp' do expect(subject[:recording_ce_finished_at]).to be_a(Time) end - - it 'jira usage works when queries time out' do - allow_any_instance_of(ActiveRecord::Relation) - .to receive(:find_in_batches).and_raise(ActiveRecord::StatementInvalid.new('')) - - expect { described_class.jira_usage }.not_to raise_error - end end describe '.system_usage_data_monthly' do @@ -873,37 +837,12 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do subject { described_class.license_usage_data } it 'gathers license data' do + expect(subject[:uuid]).to eq(Gitlab::CurrentSettings.uuid) expect(subject[:version]).to eq(Gitlab::VERSION) expect(subject[:installation_type]).to eq('gitlab-development-kit') + expect(subject[:active_user_count]).to eq(User.active.size) expect(subject[:recorded_at]).to be_a(Time) end - - context 'with usage_data_instrumentation feature flag' do - context 'when enabled' do - it 'merges uuid and hostname data from instrumentation classes' do - stub_feature_flags(usage_data_instrumentation: true) - - expect(subject[:uuid]).to eq(Gitlab::Utils::UsageData::INSTRUMENTATION_CLASS_FALLBACK) - expect(subject[:hostname]).to eq(Gitlab::Utils::UsageData::INSTRUMENTATION_CLASS_FALLBACK) - expect(subject[:active_user_count]).to eq(Gitlab::Utils::UsageData::INSTRUMENTATION_CLASS_FALLBACK) - - uncached_data = described_class.data - expect(uncached_data[:uuid]).to eq(Gitlab::CurrentSettings.uuid) - expect(uncached_data[:hostname]).to eq(Gitlab.config.gitlab.host) - expect(uncached_data[:active_user_count]).to eq(User.active.size) - end - end - - context 'when disabled' do - it 'does not merge uuid and hostname data from instrumentation classes' do - stub_feature_flags(usage_data_instrumentation: false) - - expect(subject[:uuid]).to eq(Gitlab::CurrentSettings.uuid) - expect(subject[:hostname]).to eq(Gitlab.config.gitlab.host) - expect(subject[:active_user_count]).to eq(User.active.size) - end - end - end end context 'when not relying on database records' do @@ -1139,6 +1078,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do describe ".system_usage_data_settings" do let(:prometheus_client) { double(Gitlab::PrometheusClient) } + let(:snowplow_gitlab_host?) { Gitlab::CurrentSettings.snowplow_collector_hostname == 'snowplow.trx.gitlab.net' } before do allow(described_class).to receive(:operating_system).and_return('ubuntu-20.04') @@ -1166,51 +1106,34 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do expect(subject[:settings][:gitaly_apdex]).to be_within(0.001).of(0.95) end - context 'with usage_data_instrumentation feature flag' do - context 'when enabled' do - before do - stub_feature_flags(usage_data_instrumentation: true) - end - - it 'reports collected data categories' do - expected_value = %w[standard subscription operational optional] + it 'reports collected data categories' do + expected_value = %w[standard subscription operational optional] - allow_next_instance_of(ServicePing::PermitDataCategoriesService) do |instance| - expect(instance).to receive(:execute).and_return(expected_value) - end - - expect(described_class.data[:settings][:collected_data_categories]).to eq(expected_value) - end - - it 'gathers service_ping_features_enabled' do - expect(described_class.data[:settings][:service_ping_features_enabled]).to eq(Gitlab::CurrentSettings.usage_ping_features_enabled) - end + allow_next_instance_of(ServicePing::PermitDataCategoriesService) do |instance| + expect(instance).to receive(:execute).and_return(expected_value) end - context 'when disabled' do - before do - stub_feature_flags(usage_data_instrumentation: false) - end - - it 'reports collected data categories' do - expected_value = %w[standard subscription operational optional] - - allow_next_instance_of(ServicePing::PermitDataCategoriesService) do |instance| - expect(instance).to receive(:execute).and_return(expected_value) - end - - expect(subject[:settings][:collected_data_categories]).to eq(expected_value) - end + expect(subject[:settings][:collected_data_categories]).to eq(expected_value) + end - it 'gathers service_ping_features_enabled' do - expect(subject[:settings][:service_ping_features_enabled]).to eq(Gitlab::CurrentSettings.usage_ping_features_enabled) - end - end + it 'gathers service_ping_features_enabled' do + expect(subject[:settings][:service_ping_features_enabled]).to eq(Gitlab::CurrentSettings.usage_ping_features_enabled) end it 'gathers user_cap_feature_enabled' do expect(subject[:settings][:user_cap_feature_enabled]).to eq(Gitlab::CurrentSettings.new_user_signups_cap) end + + context 'snowplow stats' do + before do + stub_feature_flags(usage_data_instrumentation: false) + end + + it 'gathers snowplow stats' do + expect(subject[:settings][:snowplow_enabled]).to eq(Gitlab::CurrentSettings.snowplow_enabled?) + expect(subject[:settings][:snowplow_configured_to_gitlab_collector]).to eq(snowplow_gitlab_host?) + end + end end end @@ -1332,6 +1255,9 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do 'i_analytics_cohorts' => 123, 'i_analytics_dev_ops_score' => 123, 'i_analytics_instance_statistics' => 123, + 'p_analytics_ci_cd_deployment_frequency' => 123, + 'p_analytics_ci_cd_lead_time' => 123, + 'p_analytics_ci_cd_pipelines' => 123, 'p_analytics_merge_request' => 123, 'i_analytics_dev_ops_adoption' => 123, 'users_viewing_analytics_group_devops_adoption' => 123, @@ -1402,33 +1328,21 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do let(:categories) { ::Gitlab::UsageDataCounters::HLLRedisCounter.categories } - context 'with redis_hll_tracking feature enabled' do - it 'has all known_events' do - stub_feature_flags(redis_hll_tracking: true) - - expect(subject).to have_key(:redis_hll_counters) + it 'has all known_events' do + expect(subject).to have_key(:redis_hll_counters) - expect(subject[:redis_hll_counters].keys).to match_array(categories) + expect(subject[:redis_hll_counters].keys).to match_array(categories) - categories.each do |category| - keys = ::Gitlab::UsageDataCounters::HLLRedisCounter.events_for_category(category) + categories.each do |category| + keys = ::Gitlab::UsageDataCounters::HLLRedisCounter.events_for_category(category) - metrics = keys.map { |key| "#{key}_weekly" } + keys.map { |key| "#{key}_monthly" } + metrics = keys.map { |key| "#{key}_weekly" } + keys.map { |key| "#{key}_monthly" } - if ::Gitlab::UsageDataCounters::HLLRedisCounter::CATEGORIES_FOR_TOTALS.include?(category) - metrics.append("#{category}_total_unique_counts_weekly", "#{category}_total_unique_counts_monthly") - end - - expect(subject[:redis_hll_counters][category].keys).to match_array(metrics) + if ::Gitlab::UsageDataCounters::HLLRedisCounter::CATEGORIES_FOR_TOTALS.include?(category) + metrics.append("#{category}_total_unique_counts_weekly", "#{category}_total_unique_counts_monthly") end - end - end - context 'with redis_hll_tracking disabled' do - it 'does not have redis_hll_tracking key' do - stub_feature_flags(redis_hll_tracking: false) - - expect(subject).not_to have_key(:redis_hll_counters) + expect(subject[:redis_hll_counters][category].keys).to match_array(metrics) end end end @@ -1468,46 +1382,58 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do context 'when queries time out' do before do - allow_any_instance_of(ActiveRecord::Relation) - .to receive(:count).and_raise(ActiveRecord::StatementInvalid.new('')) + allow_any_instance_of(ActiveRecord::Relation).to receive(:count).and_raise(ActiveRecord::StatementInvalid) + allow(Gitlab::ErrorTracking).to receive(:should_raise_for_dev?).and_return(should_raise_for_dev) end - it 'returns -1 for email campaign data' do - expected_data = { - "in_product_marketing_email_create_0_sent" => -1, - "in_product_marketing_email_create_0_cta_clicked" => -1, - "in_product_marketing_email_create_1_sent" => -1, - "in_product_marketing_email_create_1_cta_clicked" => -1, - "in_product_marketing_email_create_2_sent" => -1, - "in_product_marketing_email_create_2_cta_clicked" => -1, - "in_product_marketing_email_team_short_0_sent" => -1, - "in_product_marketing_email_team_short_0_cta_clicked" => -1, - "in_product_marketing_email_trial_short_0_sent" => -1, - "in_product_marketing_email_trial_short_0_cta_clicked" => -1, - "in_product_marketing_email_admin_verify_0_sent" => -1, - "in_product_marketing_email_admin_verify_0_cta_clicked" => -1, - "in_product_marketing_email_verify_0_sent" => -1, - "in_product_marketing_email_verify_0_cta_clicked" => -1, - "in_product_marketing_email_verify_1_sent" => -1, - "in_product_marketing_email_verify_1_cta_clicked" => -1, - "in_product_marketing_email_verify_2_sent" => -1, - "in_product_marketing_email_verify_2_cta_clicked" => -1, - "in_product_marketing_email_trial_0_sent" => -1, - "in_product_marketing_email_trial_0_cta_clicked" => -1, - "in_product_marketing_email_trial_1_sent" => -1, - "in_product_marketing_email_trial_1_cta_clicked" => -1, - "in_product_marketing_email_trial_2_sent" => -1, - "in_product_marketing_email_trial_2_cta_clicked" => -1, - "in_product_marketing_email_team_0_sent" => -1, - "in_product_marketing_email_team_0_cta_clicked" => -1, - "in_product_marketing_email_team_1_sent" => -1, - "in_product_marketing_email_team_1_cta_clicked" => -1, - "in_product_marketing_email_team_2_sent" => -1, - "in_product_marketing_email_team_2_cta_clicked" => -1, - "in_product_marketing_email_experience_0_sent" => -1 - } + context 'with should_raise_for_dev? true' do + let(:should_raise_for_dev) { true } - expect(subject).to eq(expected_data) + it 'raises an error' do + expect { subject }.to raise_error(ActiveRecord::StatementInvalid) + end + end + + context 'with should_raise_for_dev? false' do + let(:should_raise_for_dev) { false } + + it 'returns -1 for email campaign data' do + expected_data = { + "in_product_marketing_email_create_0_sent" => -1, + "in_product_marketing_email_create_0_cta_clicked" => -1, + "in_product_marketing_email_create_1_sent" => -1, + "in_product_marketing_email_create_1_cta_clicked" => -1, + "in_product_marketing_email_create_2_sent" => -1, + "in_product_marketing_email_create_2_cta_clicked" => -1, + "in_product_marketing_email_team_short_0_sent" => -1, + "in_product_marketing_email_team_short_0_cta_clicked" => -1, + "in_product_marketing_email_trial_short_0_sent" => -1, + "in_product_marketing_email_trial_short_0_cta_clicked" => -1, + "in_product_marketing_email_admin_verify_0_sent" => -1, + "in_product_marketing_email_admin_verify_0_cta_clicked" => -1, + "in_product_marketing_email_verify_0_sent" => -1, + "in_product_marketing_email_verify_0_cta_clicked" => -1, + "in_product_marketing_email_verify_1_sent" => -1, + "in_product_marketing_email_verify_1_cta_clicked" => -1, + "in_product_marketing_email_verify_2_sent" => -1, + "in_product_marketing_email_verify_2_cta_clicked" => -1, + "in_product_marketing_email_trial_0_sent" => -1, + "in_product_marketing_email_trial_0_cta_clicked" => -1, + "in_product_marketing_email_trial_1_sent" => -1, + "in_product_marketing_email_trial_1_cta_clicked" => -1, + "in_product_marketing_email_trial_2_sent" => -1, + "in_product_marketing_email_trial_2_cta_clicked" => -1, + "in_product_marketing_email_team_0_sent" => -1, + "in_product_marketing_email_team_0_cta_clicked" => -1, + "in_product_marketing_email_team_1_sent" => -1, + "in_product_marketing_email_team_1_cta_clicked" => -1, + "in_product_marketing_email_team_2_sent" => -1, + "in_product_marketing_email_team_2_cta_clicked" => -1, + "in_product_marketing_email_experience_0_sent" => -1 + } + + expect(subject).to eq(expected_data) + end end end diff --git a/spec/lib/gitlab/utils/usage_data_spec.rb b/spec/lib/gitlab/utils/usage_data_spec.rb index e721b28ac29..325ace6fbbf 100644 --- a/spec/lib/gitlab/utils/usage_data_spec.rb +++ b/spec/lib/gitlab/utils/usage_data_spec.rb @@ -5,32 +5,38 @@ require 'spec_helper' RSpec.describe Gitlab::Utils::UsageData do include Database::DatabaseHelpers - describe '#add_metric' do - let(:metric) { 'UuidMetric'} + shared_examples 'failing hardening method' do + before do + allow(Gitlab::ErrorTracking).to receive(:should_raise_for_dev?).and_return(should_raise_for_dev) + stub_const("Gitlab::Utils::UsageData::FALLBACK", fallback) + allow(failing_class).to receive(failing_method).and_raise(ActiveRecord::StatementInvalid) + end - context 'with usage_data_instrumentation feature flag' do - context 'when enabled' do - before do - stub_feature_flags(usage_data_instrumentation: true) - end + context 'with should_raise_for_dev? false' do + let(:should_raise_for_dev) { false } - it 'returns -100 value to be overriden' do - expect(described_class.add_metric(metric)).to eq(-100) - end + it 'returns the fallback' do + expect(subject).to eq(fallback) end + end - context 'when disabled' do - before do - stub_feature_flags(usage_data_instrumentation: false) - end + context 'with should_raise_for_dev? true' do + let(:should_raise_for_dev) { true } - it 'computes the metric value for given metric' do - expect(described_class.add_metric(metric)).to eq(Gitlab::CurrentSettings.uuid) - end + it 'raises an error' do + expect { subject }.to raise_error(ActiveRecord::StatementInvalid) end end end + describe '#add_metric' do + let(:metric) { 'UuidMetric'} + + it 'computes the metric value for given metric' do + expect(described_class.add_metric(metric)).to eq(Gitlab::CurrentSettings.uuid) + end + end + describe '#count' do let(:relation) { double(:relation) } @@ -40,11 +46,14 @@ RSpec.describe Gitlab::Utils::UsageData do expect(described_class.count(relation, batch: false)).to eq(1) end - it 'returns the fallback value when counting fails' do - stub_const("Gitlab::Utils::UsageData::FALLBACK", 15) - allow(relation).to receive(:count).and_raise(ActiveRecord::StatementInvalid.new('')) + context 'when counting fails' do + subject { described_class.count(relation, batch: false) } - expect(described_class.count(relation, batch: false)).to eq(15) + let(:fallback) { 15 } + let(:failing_class) { relation } + let(:failing_method) { :count } + + it_behaves_like 'failing hardening method' end end @@ -57,11 +66,14 @@ RSpec.describe Gitlab::Utils::UsageData do expect(described_class.distinct_count(relation, batch: false)).to eq(1) end - it 'returns the fallback value when counting fails' do - stub_const("Gitlab::Utils::UsageData::FALLBACK", 15) - allow(relation).to receive(:distinct_count_by).and_raise(ActiveRecord::StatementInvalid.new('')) + context 'when counting fails' do + subject { described_class.distinct_count(relation, batch: false) } + + let(:fallback) { 15 } + let(:failing_class) { relation } + let(:failing_method) { :distinct_count_by } - expect(described_class.distinct_count(relation, batch: false)).to eq(15) + it_behaves_like 'failing hardening method' end end @@ -106,7 +118,7 @@ RSpec.describe Gitlab::Utils::UsageData do # build_needs set: ['1', '2', '3', '4', '5'] # ci_build set ['a', 'b'] # with them, current implementation is expected to consistently report - # 5.217656147118495 and 2.0809220082170614 values + # the same static values # This test suite is expected to assure, that HyperLogLog implementation # behaves consistently between changes made to other parts of codebase. # In case of fine tuning or changes to HyperLogLog algorithm implementation @@ -118,8 +130,8 @@ RSpec.describe Gitlab::Utils::UsageData do let(:model) { Ci::BuildNeed } let(:column) { :name } - let(:build_needs_estimated_cardinality) { 5.217656147118495 } - let(:ci_builds_estimated_cardinality) { 2.0809220082170614 } + let(:build_needs_estimated_cardinality) { 5.024574181542231 } + let(:ci_builds_estimated_cardinality) { 2.003916452421793 } before do allow(model.connection).to receive(:transaction_open?).and_return(false) @@ -173,14 +185,24 @@ RSpec.describe Gitlab::Utils::UsageData do stub_const("Gitlab::Utils::UsageData::DISTRIBUTED_HLL_FALLBACK", 4) end - it 'returns fallback if counter raises WRONG_CONFIGURATION_ERROR' do - expect(described_class.estimate_batch_distinct_count(relation, 'id', start: 1, finish: 0)).to eq 3 + context 'when counter raises WRONG_CONFIGURATION_ERROR' do + subject { described_class.estimate_batch_distinct_count(relation, 'id', start: 1, finish: 0) } + + let(:fallback) { 3 } + let(:failing_class) { Gitlab::Database::PostgresHll::BatchDistinctCounter } + let(:failing_method) { :new } + + it_behaves_like 'failing hardening method' end - it 'returns default fallback value when counting fails due to database error' do - allow(Gitlab::Database::PostgresHll::BatchDistinctCounter).to receive(:new).and_raise(ActiveRecord::StatementInvalid.new('')) + context 'when counting fails due to database error' do + subject { described_class.estimate_batch_distinct_count(relation) } + + let(:fallback) { 3 } + let(:failing_class) { Gitlab::Database::PostgresHll::BatchDistinctCounter } + let(:failing_method) { :new } - expect(described_class.estimate_batch_distinct_count(relation)).to eq(3) + it_behaves_like 'failing hardening method' end it 'logs error and returns DISTRIBUTED_HLL_FALLBACK value when counting raises any error', :aggregate_failures do @@ -205,13 +227,14 @@ RSpec.describe Gitlab::Utils::UsageData do expect(described_class.sum(relation, :column, batch_size: 100, start: 2, finish: 3)).to eq(1) end - it 'returns the fallback value when counting fails' do - stub_const("Gitlab::Utils::UsageData::FALLBACK", 15) - allow(Gitlab::Database::BatchCount) - .to receive(:batch_sum) - .and_raise(ActiveRecord::StatementInvalid.new('')) + context 'when counting fails' do + subject { described_class.sum(relation, :column) } - expect(described_class.sum(relation, :column)).to eq(15) + let(:fallback) { 15 } + let(:failing_class) { Gitlab::Database::BatchCount } + let(:failing_method) { :batch_sum } + + it_behaves_like 'failing hardening method' end end @@ -291,23 +314,45 @@ RSpec.describe Gitlab::Utils::UsageData do expect(histogram).to eq('2' => 1) end - it 'returns fallback and logs canceled queries' do - create(:alert_management_http_integration, :active, project: project1) + context 'when query timeout' do + subject do + with_statement_timeout(0.001) do + relation = AlertManagement::HttpIntegration.select('pg_sleep(0.002)') + described_class.histogram(relation, column, buckets: 1..100) + end + end - expect(Gitlab::AppJsonLogger).to receive(:error).with( - event: 'histogram', - relation: relation.table_name, - operation: 'histogram', - operation_args: [column, 1, 100, 99], - query: kind_of(String), - message: /PG::QueryCanceled/ - ) + before do + allow(Gitlab::ErrorTracking).to receive(:should_raise_for_dev?).and_return(should_raise_for_dev) + create(:alert_management_http_integration, :active, project: project1) + end - with_statement_timeout(0.001) do - relation = AlertManagement::HttpIntegration.select('pg_sleep(0.002)') - histogram = described_class.histogram(relation, column, buckets: 1..100) + context 'with should_raise_for_dev? false' do + let(:should_raise_for_dev) { false } + + it 'logs canceled queries' do + expect(Gitlab::AppJsonLogger).to receive(:error).with( + event: 'histogram', + relation: relation.table_name, + operation: 'histogram', + operation_args: [column, 1, 100, 99], + query: kind_of(String), + message: /PG::QueryCanceled/ + ) + subject + end - expect(histogram).to eq(fallback) + it 'returns fallback' do + expect(subject).to eq(fallback) + end + end + + context 'with should_raise_for_dev? true' do + let(:should_raise_for_dev) { true } + + it 'raises error' do + expect { subject }.to raise_error(ActiveRecord::QueryCanceled) + end end end end diff --git a/spec/lib/gitlab/utils_spec.rb b/spec/lib/gitlab/utils_spec.rb index f1601294c07..d756ec5ef83 100644 --- a/spec/lib/gitlab/utils_spec.rb +++ b/spec/lib/gitlab/utils_spec.rb @@ -249,10 +249,16 @@ RSpec.describe Gitlab::Utils do end describe '.which' do - it 'finds the full path to an executable binary' do - expect(File).to receive(:executable?).with('/bin/sh').and_return(true) + before do + stub_env('PATH', '/sbin:/usr/bin:/home/joe/bin') + end + + it 'finds the full path to an executable binary in order of appearance' do + expect(File).to receive(:executable?).with('/sbin/tool').ordered.and_return(false) + expect(File).to receive(:executable?).with('/usr/bin/tool').ordered.and_return(true) + expect(File).not_to receive(:executable?).with('/home/joe/bin/tool') - expect(which('sh', 'PATH' => '/bin')).to eq('/bin/sh') + expect(which('tool')).to eq('/usr/bin/tool') end end |