diff options
Diffstat (limited to 'spec/lib')
180 files changed, 4692 insertions, 2023 deletions
diff --git a/spec/lib/api/entities/basic_project_details_spec.rb b/spec/lib/api/entities/basic_project_details_spec.rb index 8419eb0a932..425252ea315 100644 --- a/spec/lib/api/entities/basic_project_details_spec.rb +++ b/spec/lib/api/entities/basic_project_details_spec.rb @@ -2,14 +2,16 @@ require 'spec_helper' -RSpec.describe API::Entities::BasicProjectDetails do - let_it_be(:project) { create(:project) } - - let(:current_user) { project.first_owner } +RSpec.describe API::Entities::BasicProjectDetails, feature_category: :api do + let_it_be(:project_with_repository_restriction) { create(:project, :public, :repository_private) } + let(:member_user) { project_with_repository_restriction.first_owner } subject(:output) { described_class.new(project, current_user: current_user).as_json } describe '#default_branch' do + let(:current_user) { member_user } + let(:project) { project_with_repository_restriction } + it 'delegates to Project#default_branch_or_main' do expect(project).to receive(:default_branch_or_main).twice.and_call_original @@ -20,7 +22,42 @@ RSpec.describe API::Entities::BasicProjectDetails do let(:current_user) { nil } it 'is not included' do - expect(output.keys).not_to include(:default_branch) + expect(output).not_to include(:default_branch) + end + end + end + + describe '#readme_url #forks_count' do + using RSpec::Parameterized::TableSyntax + let_it_be(:non_member_user) { create(:user) } # Creates a fresh user that is why it is not the member of the project + + context 'public project with repository is accessible by the user' do + let_it_be(:project_without_restriction) { create(:project, :public) } + + where(:current_user, :project) do + ref(:member_user) | ref(:project_without_restriction) + ref(:non_member_user) | ref(:project_without_restriction) + nil | ref(:project_without_restriction) + ref(:member_user) | ref(:project_with_repository_restriction) + end + + with_them do + it 'exposes readme_url and forks_count' do + expect(output).to include readme_url: project.readme_url, forks_count: project.forks_count + end + end + end + + context 'public project with repository is not accessible by the user' do + where(:current_user, :project) do + ref(:non_member_user) | ref(:project_with_repository_restriction) + nil | ref(:project_with_repository_restriction) + end + + with_them do + it 'does not expose readme_url and forks_count' do + expect(output).not_to include :readme_url, :forks_count + end end end end diff --git a/spec/lib/api/entities/bulk_imports/entity_spec.rb b/spec/lib/api/entities/bulk_imports/entity_spec.rb index 4de85862ab9..ba8a2ddffcb 100644 --- a/spec/lib/api/entities/bulk_imports/entity_spec.rb +++ b/spec/lib/api/entities/bulk_imports/entity_spec.rb @@ -21,7 +21,8 @@ RSpec.describe API::Entities::BulkImports::Entity do :project_id, :created_at, :updated_at, - :failures + :failures, + :migrate_projects ) end end diff --git a/spec/lib/api/entities/ml/mlflow/run_info_spec.rb b/spec/lib/api/entities/ml/mlflow/run_info_spec.rb index d5a37f53e21..db8f106c9fe 100644 --- a/spec/lib/api/entities/ml/mlflow/run_info_spec.rb +++ b/spec/lib/api/entities/ml/mlflow/run_info_spec.rb @@ -33,6 +33,20 @@ RSpec.describe API::Entities::Ml::Mlflow::RunInfo do end end + describe 'run_name' do + context 'when nil' do + it { is_expected.not_to have_key(:run_name) } + end + + context 'when not nil' do + before do + allow(candidate).to receive(:name).and_return('hello') + end + + it { expect(subject[:run_name]).to eq('hello') } + end + end + describe 'experiment_id' do it 'is the experiment iid as string' do expect(subject[:experiment_id]).to eq(candidate.experiment.iid.to_s) diff --git a/spec/lib/api/helpers/members_helpers_spec.rb b/spec/lib/api/helpers/members_helpers_spec.rb new file mode 100644 index 00000000000..987d5ba9f6c --- /dev/null +++ b/spec/lib/api/helpers/members_helpers_spec.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe API::Helpers::MembersHelpers, feature_category: :subgroups do + let(:helper) do + Class.new.include(described_class).new + end + + describe '#source_members' do + subject(:source_members) { helper.source_members(source) } + + shared_examples_for 'returns all direct members' do + specify do + expect(source_members).to match_array(direct_members) + end + end + + context 'for a group' do + let_it_be(:source) { create(:group) } + let_it_be(:direct_members) { create_list(:group_member, 2, group: source) } + + it_behaves_like 'returns all direct members' + it_behaves_like 'query with source filters' + + context 'when project_members_index_by_project_namespace feature flag is disabled' do + before do + stub_feature_flags(project_members_index_by_project_namespace: false) + end + + it_behaves_like 'returns all direct members' + it_behaves_like 'query with source filters' + end + end + + context 'for a project' do + let_it_be(:source) { create(:project, group: create(:group)) } + let_it_be(:direct_members) { create_list(:project_member, 2, project: source) } + + it_behaves_like 'returns all direct members' + it_behaves_like 'query without source filters' + + context 'when project_members_index_by_project_namespace feature flag is disabled' do + before do + stub_feature_flags(project_members_index_by_project_namespace: false) + end + + it_behaves_like 'returns all direct members' + it_behaves_like 'query with source filters' + end + end + end +end diff --git a/spec/lib/api/helpers/packages_helpers_spec.rb b/spec/lib/api/helpers/packages_helpers_spec.rb index a3b21059334..de9d139a7b6 100644 --- a/spec/lib/api/helpers/packages_helpers_spec.rb +++ b/spec/lib/api/helpers/packages_helpers_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe API::Helpers::PackagesHelpers do +RSpec.describe API::Helpers::PackagesHelpers, feature_category: :package_registry do let_it_be(:helper) { Class.new.include(API::Helpers).include(described_class).new } let_it_be(:project) { create(:project) } let_it_be(:group) { create(:group) } @@ -17,6 +17,31 @@ RSpec.describe API::Helpers::PackagesHelpers do expect(subject).to eq nil end + + context 'with an allowed required permission' do + subject { helper.authorize_packages_access!(project, :read_group) } + + it 'authorizes packages access' do + expect(helper).to receive(:require_packages_enabled!) + expect(helper).not_to receive(:authorize_read_package!) + expect(helper).to receive(:authorize!).with(:read_group, project) + + expect(subject).to eq nil + end + end + + context 'with a not allowed permission' do + subject { helper.authorize_packages_access!(project, :read_permission) } + + it 'rejects packages access' do + expect(helper).to receive(:require_packages_enabled!) + expect(helper).not_to receive(:authorize_read_package!) + expect(helper).not_to receive(:authorize!).with(:test_permission, project) + expect(helper).to receive(:forbidden!) + + expect(subject).to eq nil + end + end end describe 'authorize_read_package!' do @@ -32,7 +57,7 @@ RSpec.describe API::Helpers::PackagesHelpers do it 'calls authorize! with correct subject' do expect(helper).to receive(:authorize!).with(:read_package, have_attributes(id: subject.id, class: expected_class)) - expect(helper.send('authorize_read_package!', subject)).to eq nil + expect(helper.send(:authorize_read_package!, subject)).to eq nil end end end diff --git a/spec/lib/api/helpers/pagination_strategies_spec.rb b/spec/lib/api/helpers/pagination_strategies_spec.rb index 16cc10182b0..f6e8e3cc756 100644 --- a/spec/lib/api/helpers/pagination_strategies_spec.rb +++ b/spec/lib/api/helpers/pagination_strategies_spec.rb @@ -43,6 +43,14 @@ RSpec.describe API::Helpers::PaginationStrategies do expect(result).to eq(return_value) end + + context "with paginator_params" do + it 'correctly passes multiple parameters' do + expect(paginator).to receive(:paginate).with(relation, parameter_one: true, parameter_two: 'two') + + subject.paginate_with_strategies(relation, nil, paginator_params: { parameter_one: true, parameter_two: 'two' }) + end + end end describe '#paginator' do diff --git a/spec/lib/api/helpers/rate_limiter_spec.rb b/spec/lib/api/helpers/rate_limiter_spec.rb index 3640c7e30e7..531140a32a3 100644 --- a/spec/lib/api/helpers/rate_limiter_spec.rb +++ b/spec/lib/api/helpers/rate_limiter_spec.rb @@ -31,8 +31,8 @@ RSpec.describe API::Helpers::RateLimiter do end describe '#check_rate_limit!' do - it 'calls ApplicationRateLimiter#throttled? with the right arguments' do - expect(::Gitlab::ApplicationRateLimiter).to receive(:throttled?).with(key, scope: scope).and_return(false) + it 'calls ApplicationRateLimiter#throttled_request? with the right arguments' do + expect(::Gitlab::ApplicationRateLimiter).to receive(:throttled_request?).with(request, user, key, scope: scope).and_return(false) expect(subject).not_to receive(:render_api_error!) subject.check_rate_limit!(key, scope: scope) diff --git a/spec/lib/api/helpers_spec.rb b/spec/lib/api/helpers_spec.rb index d24a3bd13c0..a0f5ee1ea95 100644 --- a/spec/lib/api/helpers_spec.rb +++ b/spec/lib/api/helpers_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe API::Helpers do +RSpec.describe API::Helpers, feature_category: :not_owned do using RSpec::Parameterized::TableSyntax subject(:helper) { Class.new.include(described_class).new } @@ -11,7 +11,7 @@ RSpec.describe API::Helpers do include Rack::Test::Methods let(:user) { build(:user, id: 42) } - + let(:request) { instance_double(Rack::Request) } let(:helper) do Class.new(Grape::API::Instance) do helpers API::APIGuard::HelperMethods @@ -797,12 +797,13 @@ RSpec.describe API::Helpers do describe '#present_artifacts_file!' do context 'with object storage' do let(:artifact) { create(:ci_job_artifact, :zip, :remote_store) } + let(:is_head_request) { false } subject { helper.present_artifacts_file!(artifact.file) } before do allow(helper).to receive(:env).and_return({}) - + allow(helper).to receive(:request).and_return(instance_double(Rack::Request, head?: is_head_request)) stub_artifacts_object_storage(enabled: true) end @@ -814,6 +815,18 @@ RSpec.describe API::Helpers do subject end + + context 'requested with HEAD' do + let(:is_head_request) { true } + + it 'redirects to a CDN-fronted URL' do + expect(helper).to receive(:redirect) + expect(helper).to receive(:signed_head_url).and_call_original + expect(Gitlab::ApplicationContext).to receive(:push).with(artifact: artifact.file.model).and_call_original + + subject + end + end end end diff --git a/spec/lib/atlassian/jira_connect/jwt/asymmetric_spec.rb b/spec/lib/atlassian/jira_connect/jwt/asymmetric_spec.rb index 89c85489aea..c14193660e9 100644 --- a/spec/lib/atlassian/jira_connect/jwt/asymmetric_spec.rb +++ b/spec/lib/atlassian/jira_connect/jwt/asymmetric_spec.rb @@ -90,7 +90,7 @@ RSpec.describe Atlassian::JiraConnect::Jwt::Asymmetric, feature_category: :integ it { is_expected.not_to be_valid } end - context 'with jira_connect_proxy_url setting' do + context 'with jira_connect_proxy_url setting', :aggregate_failures do let(:stub_asymmetric_jwt_cdn) { 'https://example.com/-/jira_connect/public_keys' } let(:jira_connect_proxy_url_setting) { 'https://example.com' } @@ -101,6 +101,19 @@ RSpec.describe Atlassian::JiraConnect::Jwt::Asymmetric, feature_category: :integ expect(WebMock).to have_requested(:get, "https://example.com/-/jira_connect/public_keys/#{public_key_id}") end + + context 'when the setting is an empty string', :aggregate_failures do + let(:jira_connect_proxy_url_setting) { '' } + let(:stub_asymmetric_jwt_cdn) { 'https://connect-install-keys.atlassian.com' } + + it 'requests the default CDN' do + expect(JWT).to receive(:decode).twice.and_call_original + + expect(asymmetric_jwt).to be_valid + + expect(WebMock).to have_requested(:get, install_keys_url) + end + end end end diff --git a/spec/lib/banzai/filter/inline_observability_filter_spec.rb b/spec/lib/banzai/filter/inline_observability_filter_spec.rb index 341ada6d2b5..fb1ba46e76c 100644 --- a/spec/lib/banzai/filter/inline_observability_filter_spec.rb +++ b/spec/lib/banzai/filter/inline_observability_filter_spec.rb @@ -7,27 +7,49 @@ RSpec.describe Banzai::Filter::InlineObservabilityFilter do let(:input) { %(<a href="#{url}">example</a>) } let(:doc) { filter(input) } + let(:group) { create(:group) } + let(:user) { create(:user) } - context 'when the document has an external link' do - let(:url) { 'https://foo.com' } + describe '#filter?' do + context 'when the document has an external link' do + let(:url) { 'https://foo.com' } - it 'leaves regular non-observability links unchanged' do - expect(doc.to_s).to eq(input) + it 'leaves regular non-observability links unchanged' do + expect(doc.to_s).to eq(input) + end end - end - context 'when the document contains an embeddable observability link' do - let(:url) { 'https://observe.gitlab.com/12345' } + context 'when the document contains an embeddable observability link' do + let(:url) { 'https://observe.gitlab.com/12345' } + + it 'leaves the original link unchanged' do + expect(doc.at_css('a').to_s).to eq(input) + end + + it 'appends an observability charts placeholder' do + node = doc.at_css('.js-render-observability') - it 'leaves the original link unchanged' do - expect(doc.at_css('a').to_s).to eq(input) + expect(node).to be_present + expect(node.attribute('data-frame-url').to_s).to eq(url) + end end - it 'appends a observability charts placeholder' do - node = doc.at_css('.js-render-observability') + context 'when feature flag is disabled' do + let(:url) { 'https://observe.gitlab.com/12345' } + + before do + stub_feature_flags(observability_group_tab: false) + end + + it 'leaves the original link unchanged' do + expect(doc.at_css('a').to_s).to eq(input) + end + + it 'does not append an observability charts placeholder' do + node = doc.at_css('.js-render-observability') - expect(node).to be_present - expect(node.attribute('data-frame-url').to_s).to eq(url) + expect(node).not_to be_present + end end end end diff --git a/spec/lib/banzai/filter/math_filter_spec.rb b/spec/lib/banzai/filter/math_filter_spec.rb index c5d2bcd5363..374983e40a1 100644 --- a/spec/lib/banzai/filter/math_filter_spec.rb +++ b/spec/lib/banzai/filter/math_filter_spec.rb @@ -2,14 +2,15 @@ require 'spec_helper' -RSpec.describe Banzai::Filter::MathFilter do +RSpec.describe Banzai::Filter::MathFilter, feature_category: :team_planning do using RSpec::Parameterized::TableSyntax include FilterSpecHelper shared_examples 'inline math' do it 'removes surrounding dollar signs and adds class code, math and js-render-math' do - doc = filter(text) - expected = result_template.gsub('<math>', '<code class="code math js-render-math" data-math-style="inline">') + doc = pipeline_filter(text) + + expected = result_template.gsub('<math>', '<code data-math-style="inline" class="code math js-render-math">') expected.gsub!('</math>', '</code>') expect(doc.to_s).to eq expected @@ -17,12 +18,12 @@ RSpec.describe Banzai::Filter::MathFilter do end shared_examples 'display math' do - let_it_be(:template_prefix_with_pre) { '<pre class="code math js-render-math" data-math-style="display"><code>' } - let_it_be(:template_prefix_with_code) { '<code class="code math js-render-math" data-math-style="display">' } + let_it_be(:template_prefix_with_pre) { '<pre lang="math" data-math-style="display" class="js-render-math"><code>' } + let_it_be(:template_prefix_with_code) { '<code data-math-style="display" class="code math js-render-math">' } let(:use_pre_tags) { false } it 'removes surrounding dollar signs and adds class code, math and js-render-math' do - doc = filter(text) + doc = pipeline_filter(text) template_prefix = use_pre_tags ? template_prefix_with_pre : template_prefix_with_code template_suffix = "</code>#{'</pre>' if use_pre_tags}" @@ -36,36 +37,38 @@ RSpec.describe Banzai::Filter::MathFilter do describe 'inline math using $...$ syntax' do context 'with valid syntax' do where(:text, :result_template) do - '$2+2$' | '<math>2+2</math>' - '$22+1$ and $22 + a^2$' | '<math>22+1</math> and <math>22 + a^2</math>' - '$22 and $2+2$' | '$22 and <math>2+2</math>' - '$2+2$ $22 and flightjs/Flight$22 $2+2$' | '<math>2+2</math> $22 and flightjs/Flight$22 <math>2+2</math>' - '$1/2$ <b>test</b>' | '<math>1/2</math> <b>test</b>' - '$a!$' | '<math>a!</math>' - '$x$' | '<math>x</math>' + '$2+2$' | '<p><math>2+2</math></p>' + '$22+1$ and $22 + a^2$' | '<p><math>22+1</math> and <math>22 + a^2</math></p>' + '$22 and $2+2$' | '<p>$22 and <math>2+2</math></p>' + '$2+2$ $22 and flightjs/Flight$22 $2+2$' | '<p><math>2+2</math> $22 and flightjs/Flight$22 <math>2+2</math></p>' + '$1/2$ <b>test</b>' | '<p><math>1/2</math> <b>test</b></p>' + '$a!$' | '<p><math>a!</math></p>' + '$x$' | '<p><math>x</math></p>' + '$1+2\$$' | '<p><math>1+2\$</math></p>' + '$1+\$2$' | '<p><math>1+\$2</math></p>' + '$1+\%2$' | '<p><math>1+\%2</math></p>' + '$1+\#2$' | '<p><math>1+\#2</math></p>' + '$1+\&2$' | '<p><math>1+\&2</math></p>' + '$1+\{2$' | '<p><math>1+\{2</math></p>' + '$1+\}2$' | '<p><math>1+\}2</math></p>' + '$1+\_2$' | '<p><math>1+\_2</math></p>' end with_them do it_behaves_like 'inline math' end end - - it 'does not handle dollar literals properly' do - doc = filter('$20+30\$$') - expected = '<code class="code math js-render-math" data-math-style="inline">20+30\\</code>$' - - expect(doc.to_s).to eq expected - end end describe 'inline math using $`...`$ syntax' do context 'with valid syntax' do where(:text, :result_template) do - '$<code>2+2</code>$' | '<math>2+2</math>' - '$<code>22+1</code>$ and $<code>22 + a^2</code>$' | '<math>22+1</math> and <math>22 + a^2</math>' - '$22 and $<code>2+2</code>$' | '$22 and <math>2+2</math>' - '$<code>2+2</code>$ $22 and flightjs/Flight$22 $<code>2+2</code>$' | '<math>2+2</math> $22 and flightjs/Flight$22 <math>2+2</math>' - 'test $$<code>2+2</code>$$ test' | 'test $<math>2+2</math>$ test' + '$`2+2`$' | '<p><math>2+2</math></p>' + '$`22+1`$ and $`22 + a^2`$' | '<p><math>22+1</math> and <math>22 + a^2</math></p>' + '$22 and $`2+2`$' | '<p>$22 and <math>2+2</math></p>' + '$`2+2`$ $22 and flightjs/Flight$22 $`2+2`$' | '<p><math>2+2</math> $22 and flightjs/Flight$22 <math>2+2</math></p>' + 'test $$`2+2`$$ test' | '<p>test $<math>2+2</math>$ test</p>' + '$`1+\$2`$' | '<p><math>1+\$2</math></p>' end with_them do @@ -77,15 +80,15 @@ RSpec.describe Banzai::Filter::MathFilter do describe 'inline display math using $$...$$ syntax' do context 'with valid syntax' do where(:text, :result_template) do - '$$2+2$$' | '<math>2+2</math>' - '$$ 2+2 $$' | '<math>2+2</math>' - '$$22+1$$ and $$22 + a^2$$' | '<math>22+1</math> and <math>22 + a^2</math>' - '$22 and $$2+2$$' | '$22 and <math>2+2</math>' - '$$2+2$$ $22 and flightjs/Flight$22 $$2+2$$' | '<math>2+2</math> $22 and flightjs/Flight$22 <math>2+2</math>' - 'flightjs/Flight$22 and $$a^2 + b^2 = c^2$$' | 'flightjs/Flight$22 and <math>a^2 + b^2 = c^2</math>' - '$$a!$$' | '<math>a!</math>' - '$$x$$' | '<math>x</math>' - '$$20,000 and $$30,000' | '<math>20,000 and</math>30,000' + '$$2+2$$' | '<p><math>2+2</math></p>' + '$$ 2+2 $$' | '<p><math>2+2</math></p>' + '$$22+1$$ and $$22 + a^2$$' | '<p><math>22+1</math> and <math>22 + a^2</math></p>' + '$22 and $$2+2$$' | '<p>$22 and <math>2+2</math></p>' + '$$2+2$$ $22 and flightjs/Flight$22 $$2+2$$' | '<p><math>2+2</math> $22 and flightjs/Flight$22 <math>2+2</math></p>' + 'flightjs/Flight$22 and $$a^2 + b^2 = c^2$$' | '<p>flightjs/Flight$22 and <math>a^2 + b^2 = c^2</math></p>' + '$$a!$$' | '<p><math>a!</math></p>' + '$$x$$' | '<p><math>x</math></p>' + '$$20,000 and $$30,000' | '<p><math>20,000 and</math>30,000</p>' end with_them do @@ -97,8 +100,8 @@ RSpec.describe Banzai::Filter::MathFilter do describe 'block display math using $$\n...\n$$ syntax' do context 'with valid syntax' do where(:text, :result_template) do - "$$\n2+2\n$$" | "<math>2+2</math>" - "$$\n2+2\n3+4\n$$" | "<math>2+2\n3+4</math>" + "$$\n2+2\n$$" | "<math>2+2\n</math>" + "$$\n2+2\n3+4\n$$" | "<math>2+2\n3+4\n</math>" end with_them do @@ -107,72 +110,96 @@ RSpec.describe Banzai::Filter::MathFilter do end end end + + context 'when it spans multiple lines' do + let(:math) do + <<~MATH + \\begin{align*} + \\Delta t \\frac{d(b_i, a_i)}{c} + \\Delta t_{b_i} + \\end{align*} + MATH + end + + let(:text) { "$$\n#{math}$$" } + let(:result_template) { "<math>#{math}</math>" } + + it_behaves_like 'display math' do + let(:use_pre_tags) { true } + end + end + + context 'when it contains \\' do + let(:math) do + <<~MATH + E = mc^2 \\\\ + E = \\$mc^2 + MATH + end + + let(:text) { "$$\n#{math}$$" } + let(:result_template) { "<math>#{math}</math>" } + + it_behaves_like 'display math' do + let(:use_pre_tags) { true } + end + end end describe 'display math using ```math...``` syntax' do it 'adds data-math-style display attribute to display math' do - doc = filter('<pre lang="math"><code>2+2</code></pre>') + doc = pipeline_filter("```math\n2+2\n```") pre = doc.xpath('descendant-or-self::pre').first expect(pre['data-math-style']).to eq 'display' end it 'adds js-render-math class to display math' do - doc = filter('<pre lang="math"><code>2+2</code></pre>') + doc = pipeline_filter("```math\n2+2\n```") pre = doc.xpath('descendant-or-self::pre').first expect(pre[:class]).to include("js-render-math") end it 'ignores code blocks that are not math' do - input = '<pre lang="plaintext"><code>2+2</code></pre>' - doc = filter(input) + input = "```plaintext\n2+2\n```" + doc = pipeline_filter(input) - expect(doc.to_s).to eq input + expect(doc.to_s).to eq "<pre lang=\"plaintext\"><code>2+2\n</code></pre>" end it 'requires the pre to contain both code and math' do input = '<pre lang="math">something</pre>' - doc = filter(input) + doc = pipeline_filter(input) expect(doc.to_s).to eq input end - - it 'dollar signs around to display math' do - doc = filter('$<pre lang="math"><code>2+2</code></pre>$') - before = doc.xpath('descendant-or-self::text()[1]').first - after = doc.xpath('descendant-or-self::text()[3]').first - - expect(before.to_s).to eq '$' - expect(after.to_s).to eq '$' - end end describe 'unrecognized syntax' do - where(:text) do - [ - '<code>2+2</code>', - 'test $<code>2+2</code> test', - 'test <code>2+2</code>$ test', - '<em>$</em><code>2+2</code><em>$</em>', - '$20,000 and $30,000', - '$20,000 in $USD', - '$ a^2 $', - "test $$\n2+2\n$$", - "$\n$", - '$$$' - ] + where(:text, :result) do + '`2+2`' | '<p><code>2+2</code></p>' + 'test $`2+2` test' | '<p>test $<code>2+2</code> test</p>' + 'test `2+2`$ test' | '<p>test <code>2+2</code>$ test</p>' + '$20,000 and $30,000' | '<p>$20,000 and $30,000</p>' + '$20,000 in $USD' | '<p>$20,000 in $USD</p>' + '$ a^2 $' | '<p>$ a^2 $</p>' + "test $$\n2+2\n$$" | "<p>test $$\n2+2\n$$</p>" + "$\n$" | "<p>$\n$</p>" + '$$$' | '<p>$$$</p>' + '`$1+2$`' | '<p><code>$1+2$</code></p>' + '`$$1+2$$`' | '<p><code>$$1+2$$</code></p>' + '`$\$1+2$$`' | '<p><code>$\$1+2$$</code></p>' end with_them do it 'is ignored' do - expect(filter(text).to_s).to eq text + expect(pipeline_filter(text).to_s).to eq result end end end it 'handles multiple styles in one text block' do - doc = filter('$<code>2+2</code>$ + $3+3$ + $$4+4$$') + doc = pipeline_filter('$`2+2`$ + $3+3$ + $$4+4$$') expect(doc.search('.js-render-math').count).to eq(3) expect(doc.search('[data-math-style="inline"]').count).to eq(2) @@ -182,15 +209,17 @@ RSpec.describe Banzai::Filter::MathFilter do it 'limits how many elements can be marked as math' do stub_const('Banzai::Filter::MathFilter::RENDER_NODES_LIMIT', 2) - doc = filter('$<code>2+2</code>$ + $<code>3+3</code>$ + $<code>4+4</code>$') + doc = pipeline_filter('$`2+2`$ + $3+3$ + $$4+4$$') expect(doc.search('.js-render-math').count).to eq(2) end - it 'does not recognize new syntax when feature flag is off' do - stub_feature_flags(markdown_dollar_math: false) - doc = filter('$1+2$') + def pipeline_filter(text) + context = { project: nil, no_sourcepos: true } + doc = Banzai::Pipeline::PreProcessPipeline.call(text, {}) + doc = Banzai::Pipeline::PlainMarkdownPipeline.call(doc[:output], context) + doc = Banzai::Filter::SanitizationFilter.call(doc[:output], context, nil) - expect(doc.to_s).to eq '$1+2$' + filter(doc) end end diff --git a/spec/lib/banzai/filter/references/reference_filter_spec.rb b/spec/lib/banzai/filter/references/reference_filter_spec.rb index 6d7396ef216..88404f2039d 100644 --- a/spec/lib/banzai/filter/references/reference_filter_spec.rb +++ b/spec/lib/banzai/filter/references/reference_filter_spec.rb @@ -189,9 +189,9 @@ RSpec.describe Banzai::Filter::References::ReferenceFilter do let(:filter) { described_class.new(document, project: project) } it 'updates all new nodes', :aggregate_failures do - filter.instance_variable_set('@nodes', nodes) + filter.instance_variable_set(:@nodes, nodes) - expect(filter).to receive(:call) { filter.instance_variable_set('@new_nodes', new_nodes) } + expect(filter).to receive(:call) { filter.instance_variable_set(:@new_nodes, new_nodes) } expect(filter).to receive(:with_update_nodes).and_call_original expect(filter).to receive(:update_nodes!).and_call_original @@ -212,7 +212,7 @@ RSpec.describe Banzai::Filter::References::ReferenceFilter do expect_next_instance_of(described_class) do |filter| expect(filter).to receive(:call_and_update_nodes).and_call_original expect(filter).to receive(:with_update_nodes).and_call_original - expect(filter).to receive(:call) { filter.instance_variable_set('@new_nodes', new_nodes) } + expect(filter).to receive(:call) { filter.instance_variable_set(:@new_nodes, new_nodes) } expect(filter).to receive(:update_nodes!).and_call_original end diff --git a/spec/lib/banzai/filter/repository_link_filter_spec.rb b/spec/lib/banzai/filter/repository_link_filter_spec.rb index 0df680dc0c8..b2162ea2756 100644 --- a/spec/lib/banzai/filter/repository_link_filter_spec.rb +++ b/spec/lib/banzai/filter/repository_link_filter_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Banzai::Filter::RepositoryLinkFilter do +RSpec.describe Banzai::Filter::RepositoryLinkFilter, feature_category: :team_planning do include RepoHelpers def filter(doc, contexts = {}) @@ -303,6 +303,12 @@ RSpec.describe Banzai::Filter::RepositoryLinkFilter do expect(doc.at_css('img')['src']).to eq "/#{project_path}/-/raw/#{Addressable::URI.escape(ref)}/#{escaped}" end + it 'supports percent sign in filenames' do + doc = filter(link('doc/api/README%.md')) + expect(doc.at_css('a')['href']) + .to eq "/#{project_path}/-/blob/#{ref}/doc/api/README%25.md" + end + context 'when requested path is a file in the repo' do let(:requested_path) { 'doc/api/README.md' } diff --git a/spec/lib/banzai/filter/service_desk_upload_link_filter_spec.rb b/spec/lib/banzai/filter/service_desk_upload_link_filter_spec.rb new file mode 100644 index 00000000000..08d6fe03606 --- /dev/null +++ b/spec/lib/banzai/filter/service_desk_upload_link_filter_spec.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Banzai::Filter::ServiceDeskUploadLinkFilter, feature_category: :service_desk do + def filter(doc, contexts = {}) + described_class.call(doc, contexts) + end + + def link(path, text) + %(<a href="#{path}">#{text}</a>) + end + + let(:file_name) { 'test.jpg' } + let(:secret) { 'e90decf88d8f96fe9e1389afc2e4a91f' } + let(:upload_path) { "/uploads/#{secret}/#{file_name}" } + let(:html_link) { link(upload_path, file_name) } + + context 'when replace_upload_links enabled' do + context 'when it has only one attachment to replace' do + let(:contexts) { { uploads_as_attachments: ["#{secret}/#{file_name}"] } } + + context 'when filename in text is same as in link' do + it 'replaces the link with original filename in strong' do + doc = filter(html_link, contexts) + + expect(doc.at_css('a')).to be_nil + expect(doc.at_css('strong').text).to eq(file_name) + end + end + + context 'when filename in text is not same as in link' do + let(:filename_in_text) { 'Custom name' } + let(:html_link) { link(upload_path, filename_in_text) } + + it 'replaces the link with filename in text & original filename, in strong' do + doc = filter(html_link, contexts) + + expect(doc.at_css('a')).to be_nil + expect(doc.at_css('strong').text).to eq("#{filename_in_text} (#{file_name})") + end + end + end + + context 'when it has more than one attachment to replace' do + let(:file_name_1) { 'test1.jpg' } + let(:secret_1) { '17817c73e368777e6f743392e334fb8a' } + let(:upload_path_1) { "/uploads/#{secret_1}/#{file_name_1}" } + let(:html_link_1) { link(upload_path_1, file_name_1) } + + context 'when all of uploads can be replaced' do + let(:contexts) { { uploads_as_attachments: ["#{secret}/#{file_name}", "#{secret_1}/#{file_name_1}"] } } + + it 'replaces all links with original filename in strong' do + doc = filter("#{html_link} #{html_link_1}", contexts) + + expect(doc.at_css('a')).to be_nil + expect(doc.at_css("strong:contains('#{file_name}')")).not_to be_nil + expect(doc.at_css("strong:contains('#{file_name_1}')")).not_to be_nil + end + end + + context 'when not all of uploads can be replaced' do + let(:contexts) { { uploads_as_attachments: ["#{secret}/#{file_name}"] } } + + it 'replaces only specific links with original filename in strong' do + doc = filter("#{html_link} #{html_link_1}", contexts) + + expect(doc.at_css("strong:contains('#{file_name}')")).not_to be_nil + expect(doc.at_css("a:contains('#{file_name_1}')")).not_to be_nil + end + end + end + end + + context 'when uploads_as_attachments is empty' do + let(:contexts) { { uploads_as_attachments: [] } } + + it 'does not replaces the link' do + doc = filter(html_link, contexts) + + expect(doc.at_css('a')).not_to be_nil + expect(doc.at_css('a')['href']).to eq upload_path + end + end +end diff --git a/spec/lib/banzai/pipeline/full_pipeline_spec.rb b/spec/lib/banzai/pipeline/full_pipeline_spec.rb index 1a0f5a53a23..c1d5f16b562 100644 --- a/spec/lib/banzai/pipeline/full_pipeline_spec.rb +++ b/spec/lib/banzai/pipeline/full_pipeline_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Banzai::Pipeline::FullPipeline do +RSpec.describe Banzai::Pipeline::FullPipeline, feature_category: :team_planning do describe 'References' do let(:project) { create(:project, :public) } let(:issue) { create(:issue, project: project) } @@ -164,7 +164,7 @@ RSpec.describe Banzai::Pipeline::FullPipeline do markdown = '_@test\__' output = described_class.to_html(markdown, project: project) - expect(output).to include('<em>@test_</em>') + expect(output).to include('<em>@test<span>_</span></em>') end end diff --git a/spec/lib/banzai/pipeline/plain_markdown_pipeline_spec.rb b/spec/lib/banzai/pipeline/plain_markdown_pipeline_spec.rb index 536f2a67415..0e4a4e4492e 100644 --- a/spec/lib/banzai/pipeline/plain_markdown_pipeline_spec.rb +++ b/spec/lib/banzai/pipeline/plain_markdown_pipeline_spec.rb @@ -2,24 +2,25 @@ require 'spec_helper' -RSpec.describe Banzai::Pipeline::PlainMarkdownPipeline do +RSpec.describe Banzai::Pipeline::PlainMarkdownPipeline, feature_category: :team_planning do using RSpec::Parameterized::TableSyntax describe 'backslash escapes', :aggregate_failures do let_it_be(:project) { create(:project, :public) } let_it_be(:issue) { create(:issue, project: project) } - it 'converts all reference punctuation to literals' do - reference_chars = Banzai::Filter::MarkdownPreEscapeFilter::REFERENCE_CHARACTERS - markdown = reference_chars.split('').map { |char| char.prepend("\\") }.join - punctuation = Banzai::Filter::MarkdownPreEscapeFilter::REFERENCE_CHARACTERS.split('') - punctuation = punctuation.delete_if { |char| char == '&' } - punctuation << '&' + it 'converts all escapable punctuation to literals' do + markdown = Banzai::Filter::MarkdownPreEscapeFilter::ESCAPABLE_CHARS.pluck(:escaped).join result = described_class.call(markdown, project: project) output = result[:output].to_html - punctuation.each { |char| expect(output).to include("<span>#{char}</span>") } + Banzai::Filter::MarkdownPreEscapeFilter::ESCAPABLE_CHARS.pluck(:char).each do |char| + char = '&' if char == '&' + + expect(output).to include("<span>#{char}</span>") + end + expect(result[:escaped_literals]).to be_truthy end @@ -33,12 +34,12 @@ RSpec.describe Banzai::Pipeline::PlainMarkdownPipeline do end.compact reference_chars.all? do |char| - Banzai::Filter::MarkdownPreEscapeFilter::REFERENCE_CHARACTERS.include?(char) + Banzai::Filter::MarkdownPreEscapeFilter::TARGET_CHARS.include?(char) end end - it 'does not convert non-reference punctuation to spans' do - markdown = %q(\"\'\*\+\,\-\.\/\:\;\<\=\>\?\[\]\_\`\{\|\}) + %q[\(\)\\\\] + it 'does not convert non-reference/latex punctuation to spans' do + markdown = %q(\"\'\*\+\,\-\.\/\:\;\<\=\>\?\[\]\`\|) + %q[\(\)\\\\] result = described_class.call(markdown, project: project) output = result[:output].to_html @@ -55,11 +56,12 @@ RSpec.describe Banzai::Pipeline::PlainMarkdownPipeline do expect(result[:escaped_literals]).to be_falsey end - describe 'backslash escapes do not work in code blocks, code spans, autolinks, or raw HTML' do + describe 'backslash escapes are untouched in code blocks, code spans, autolinks, or raw HTML' do where(:markdown, :expected) do %q(`` \@\! ``) | %q(<code>\@\!</code>) %q( \@\!) | %Q(<code>\\@\\!\n</code>) %Q(~~~\n\\@\\!\n~~~) | %Q(<code>\\@\\!\n</code>) + %q($1+\$2$) | %q(<code data-math-style="inline">1+\\$2</code>) %q(<http://example.com?find=\@>) | %q(<a href="http://example.com?find=%5C@">http://example.com?find=\@</a>) %q[<a href="/bar\@)">] | %q[<a href="/bar%5C@)">] end diff --git a/spec/lib/banzai/pipeline/service_desk_email_pipeline_spec.rb b/spec/lib/banzai/pipeline/service_desk_email_pipeline_spec.rb new file mode 100644 index 00000000000..83541494f68 --- /dev/null +++ b/spec/lib/banzai/pipeline/service_desk_email_pipeline_spec.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Banzai::Pipeline::ServiceDeskEmailPipeline, feature_category: :service_desk do + describe '.filters' do + it 'returns the expected type' do + expect(described_class.filters).to be_kind_of(Banzai::FilterArray) + end + + it 'excludes ServiceDeskUploadLinkFilter' do + expect(described_class.filters).not_to be_empty + expect(described_class.filters).to include(Banzai::Filter::ServiceDeskUploadLinkFilter) + end + end +end diff --git a/spec/lib/bulk_imports/clients/http_spec.rb b/spec/lib/bulk_imports/clients/http_spec.rb index 4fb08fc0478..780f61f8c61 100644 --- a/spec/lib/bulk_imports/clients/http_spec.rb +++ b/spec/lib/bulk_imports/clients/http_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe BulkImports::Clients::HTTP do +RSpec.describe BulkImports::Clients::HTTP, feature_category: :importers do include ImportSpecHelper let(:url) { 'http://gitlab.example' } @@ -22,12 +22,6 @@ RSpec.describe BulkImports::Clients::HTTP do ) end - before do - allow(Gitlab::HTTP).to receive(:get) - .with('http://gitlab.example/api/v4/version', anything) - .and_return(metadata_response) - end - subject { described_class.new(url: url, token: token) } shared_examples 'performs network request' do @@ -39,7 +33,7 @@ RSpec.describe BulkImports::Clients::HTTP do context 'error handling' do context 'when error occurred' do - it 'raises BulkImports::Error' do + it 'raises BulkImports::NetworkError' do allow(Gitlab::HTTP).to receive(method).and_raise(Errno::ECONNREFUSED) expect { subject.public_send(method, resource) }.to raise_exception(BulkImports::NetworkError) @@ -47,7 +41,7 @@ RSpec.describe BulkImports::Clients::HTTP do end context 'when response is not success' do - it 'raises BulkImports::Error' do + it 'raises BulkImports::NetworkError' do response_double = double(code: 503, success?: false, parsed_response: 'Error', request: double(path: double(path: '/test'))) allow(Gitlab::HTTP).to receive(method).and_return(response_double) @@ -210,33 +204,153 @@ RSpec.describe BulkImports::Clients::HTTP do describe '#instance_version' do it 'returns version as an instance of Gitlab::VersionInfo' do + response = { version: version } + + stub_request(:get, 'http://gitlab.example/api/v4/version?private_token=token') + .to_return(status: 200, body: response.to_json, headers: { 'Content-Type' => 'application/json' }) + expect(subject.instance_version).to eq(Gitlab::VersionInfo.parse(version)) end context 'when /version endpoint is not available' do it 'requests /metadata endpoint' do - response_double = double(code: 404, success?: false, parsed_response: 'Not Found', request: double(path: double(path: '/version'))) - - allow(Gitlab::HTTP).to receive(:get) - .with('http://gitlab.example/api/v4/version', anything) - .and_return(response_double) + response = { version: version } - expect(Gitlab::HTTP).to receive(:get) - .with('http://gitlab.example/api/v4/metadata', anything) - .and_return(metadata_response) + stub_request(:get, 'http://gitlab.example/api/v4/version?private_token=token').to_return(status: 404) + stub_request(:get, 'http://gitlab.example/api/v4/metadata?private_token=token') + .to_return(status: 200, body: response.to_json, headers: { 'Content-Type' => 'application/json' }) expect(subject.instance_version).to eq(Gitlab::VersionInfo.parse(version)) end + + context 'when /metadata endpoint returns a 401' do + it 'raises a BulkImports:Error' do + stub_request(:get, 'http://gitlab.example/api/v4/version?private_token=token').to_return(status: 404) + stub_request(:get, 'http://gitlab.example/api/v4/metadata?private_token=token') + .to_return(status: 401, body: "", headers: { 'Content-Type' => 'application/json' }) + + expect { subject.instance_version }.to raise_exception(BulkImports::Error, + "Import aborted as the provided personal access token does not have the required 'api' scope or " \ + "is no longer valid.") + end + end + + context 'when /metadata endpoint returns a 403' do + it 'raises a BulkImports:Error' do + stub_request(:get, 'http://gitlab.example/api/v4/version?private_token=token').to_return(status: 404) + stub_request(:get, 'http://gitlab.example/api/v4/metadata?private_token=token') + .to_return(status: 403, body: "", headers: { 'Content-Type' => 'application/json' }) + + expect { subject.instance_version }.to raise_exception(BulkImports::Error, + "Import aborted as the provided personal access token does not have the required 'api' scope or " \ + "is no longer valid.") + end + end + + context 'when /metadata endpoint returns a 404' do + it 'raises a BulkImports:Error' do + stub_request(:get, 'http://gitlab.example/api/v4/version?private_token=token').to_return(status: 404) + stub_request(:get, 'http://gitlab.example/api/v4/metadata?private_token=token') + .to_return(status: 404, body: "", headers: { 'Content-Type' => 'application/json' }) + + expect { subject.instance_version }.to raise_exception(BulkImports::Error, 'Import aborted as it was not possible to connect to the provided GitLab instance URL.') + end + end + + context 'when /metadata endpoint returns any other BulkImports::NetworkError' do + it 'raises a BulkImports:NetworkError' do + stub_request(:get, 'http://gitlab.example/api/v4/version?private_token=token').to_return(status: 404) + stub_request(:get, 'http://gitlab.example/api/v4/metadata?private_token=token') + .to_return(status: 418, body: "", headers: { 'Content-Type' => 'application/json' }) + + expect { subject.instance_version }.to raise_exception(BulkImports::NetworkError) + end + end + end + end + + describe '#validate_instance_version!' do + before do + allow(subject).to receive(:instance_version).and_return(source_version) + end + + context 'when instance version is greater than or equal to the minimum major version' do + let(:source_version) { Gitlab::VersionInfo.new(14) } + + it { expect(subject.validate_instance_version!).to eq(true) } + end + + context 'when instance version is less than the minimum major version' do + let(:source_version) { Gitlab::VersionInfo.new(13, 10, 0) } + + it { expect { subject.validate_instance_version! }.to raise_exception(BulkImports::Error) } + end + end + + describe '#validate_import_scopes!' do + context 'when the source_version is < 15.5' do + let(:source_version) { Gitlab::VersionInfo.new(15, 0) } + + it 'skips validation' do + allow(subject).to receive(:instance_version).and_return(source_version) + + expect(subject.validate_import_scopes!).to eq(true) + end + end + + context 'when source version is 15.5 or higher' do + let(:source_version) { Gitlab::VersionInfo.new(15, 6) } + + before do + allow(subject).to receive(:instance_version).and_return(source_version) + end + + context 'when an HTTP error is raised' do + let(:response) { { enterprise: false } } + + it 'raises BulkImports::NetworkError' do + stub_request(:get, 'http://gitlab.example/api/v4/personal_access_tokens/self?private_token=token') + .to_return(status: 404) + + expect { subject.validate_import_scopes! }.to raise_exception(BulkImports::NetworkError) + end + end + + context 'when scopes are valid' do + it 'returns true' do + stub_request(:get, 'http://gitlab.example/api/v4/personal_access_tokens/self?private_token=token') + .to_return(status: 200, body: { 'scopes' => ['api'] }.to_json, headers: { 'Content-Type' => 'application/json' }) + + expect(subject.validate_import_scopes!).to eq(true) + end + end + + context 'when scopes are invalid' do + it 'raises a BulkImports error' do + stub_request(:get, 'http://gitlab.example/api/v4/personal_access_tokens/self?private_token=token') + .to_return(status: 200, body: { 'scopes' => ['read_user'] }.to_json, headers: { 'Content-Type' => 'application/json' }) + + expect(subject.instance_version).to eq(Gitlab::VersionInfo.parse(source_version)) + expect { subject.validate_import_scopes! }.to raise_exception(BulkImports::Error) + end + end end end describe '#instance_enterprise' do + let(:response) { { enterprise: false } } + + before do + stub_request(:get, 'http://gitlab.example/api/v4/version?private_token=token') + .to_return(status: 200, body: response.to_json, headers: { 'Content-Type' => 'application/json' }) + end + it 'returns source instance enterprise information' do expect(subject.instance_enterprise).to eq(false) end context 'when enterprise information is missing' do - let(:enterprise) { nil } + let(:response) { {} } it 'defaults to true' do expect(subject.instance_enterprise).to eq(true) @@ -245,14 +359,20 @@ RSpec.describe BulkImports::Clients::HTTP do end describe '#compatible_for_project_migration?' do + before do + allow(subject).to receive(:instance_version).and_return(Gitlab::VersionInfo.parse(version)) + end + context 'when instance version is lower the the expected minimum' do + let(:version) { '14.3.0' } + it 'returns false' do expect(subject.compatible_for_project_migration?).to be false end end context 'when instance version is at least the expected minimum' do - let(:version) { "14.4.4" } + let(:version) { '14.4.4' } it 'returns true' do expect(subject.compatible_for_project_migration?).to be true @@ -260,18 +380,6 @@ RSpec.describe BulkImports::Clients::HTTP do end end - context 'when source instance is incompatible' do - let(:version) { '13.0.0' } - - it 'raises an error' do - expect { subject.get(resource) } - .to raise_error( - ::BulkImports::Error, - "Unsupported GitLab Version. Minimum Supported Gitlab Version #{BulkImport::MIN_MAJOR_VERSION}." - ) - end - end - context 'when url is relative' do let(:url) { 'http://website.example/gitlab' } diff --git a/spec/lib/bulk_imports/groups/stage_spec.rb b/spec/lib/bulk_imports/groups/stage_spec.rb index 528d65615b1..cc772f07d21 100644 --- a/spec/lib/bulk_imports/groups/stage_spec.rb +++ b/spec/lib/bulk_imports/groups/stage_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe BulkImports::Groups::Stage do +RSpec.describe BulkImports::Groups::Stage, feature_category: :importers do let(:ancestor) { create(:group) } let(:group) { build(:group, parent: ancestor) } let(:bulk_import) { build(:bulk_import) } @@ -77,6 +77,28 @@ RSpec.describe BulkImports::Groups::Stage do ) end + describe 'migrate projects flag' do + context 'when true' do + it 'includes project entities pipeline' do + entity.update!(migrate_projects: true) + + expect(described_class.new(entity).pipelines).to include( + hash_including({ pipeline: BulkImports::Groups::Pipelines::ProjectEntitiesPipeline }) + ) + end + end + + context 'when false' do + it 'does not include project entities pipeline' do + entity.update!(migrate_projects: false) + + expect(described_class.new(entity).pipelines).not_to include( + hash_including({ pipeline: BulkImports::Groups::Pipelines::ProjectEntitiesPipeline }) + ) + end + end + end + context 'when feature flag is enabled on root ancestor level' do it 'includes project entities pipeline' do stub_feature_flags(bulk_import_projects: ancestor) diff --git a/spec/lib/bulk_imports/groups/transformers/subgroup_to_entity_transformer_spec.rb b/spec/lib/bulk_imports/groups/transformers/subgroup_to_entity_transformer_spec.rb index 6450d90ec0f..69cf80f92c5 100644 --- a/spec/lib/bulk_imports/groups/transformers/subgroup_to_entity_transformer_spec.rb +++ b/spec/lib/bulk_imports/groups/transformers/subgroup_to_entity_transformer_spec.rb @@ -6,7 +6,7 @@ RSpec.describe BulkImports::Groups::Transformers::SubgroupToEntityTransformer do describe "#transform" do it "transforms subgroups data in entity params" do parent = create(:group) - parent_entity = instance_double(BulkImports::Entity, group: parent, id: 1) + parent_entity = instance_double(BulkImports::Entity, group: parent, id: 1, migrate_projects: false) context = instance_double(BulkImports::Pipeline::Context, entity: parent_entity) subgroup_data = { "path" => "sub-group", @@ -18,7 +18,8 @@ RSpec.describe BulkImports::Groups::Transformers::SubgroupToEntityTransformer do source_full_path: "parent/sub-group", destination_name: "sub-group", destination_namespace: parent.full_path, - parent_id: 1 + parent_id: 1, + migrate_projects: false ) end end diff --git a/spec/lib/bulk_imports/projects/pipelines/project_attributes_pipeline_spec.rb b/spec/lib/bulk_imports/projects/pipelines/project_attributes_pipeline_spec.rb index 4320d5dc119..ecb3c8fe76d 100644 --- a/spec/lib/bulk_imports/projects/pipelines/project_attributes_pipeline_spec.rb +++ b/spec/lib/bulk_imports/projects/pipelines/project_attributes_pipeline_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe BulkImports::Projects::Pipelines::ProjectAttributesPipeline do +RSpec.describe BulkImports::Projects::Pipelines::ProjectAttributesPipeline, :with_license do let_it_be(:project) { create(:project) } let_it_be(:bulk_import) { create(:bulk_import) } let_it_be(:entity) { create(:bulk_import_entity, :project_entity, project: project, bulk_import: bulk_import) } diff --git a/spec/lib/event_filter_spec.rb b/spec/lib/event_filter_spec.rb index bab48796b8c..16612a6288c 100644 --- a/spec/lib/event_filter_spec.rb +++ b/spec/lib/event_filter_spec.rb @@ -3,6 +3,29 @@ require 'spec_helper' RSpec.describe EventFilter do + let_it_be(:public_project) { create(:project, :public) } + let_it_be(:push_event) { create(:push_event, project: public_project) } + let_it_be(:merged_event) { create(:event, :merged, project: public_project, target: public_project) } + let_it_be(:created_event) { create(:event, :created, project: public_project, target: create(:issue, project: public_project)) } + let_it_be(:updated_event) { create(:event, :updated, project: public_project, target: create(:issue, project: public_project)) } + let_it_be(:closed_event) { create(:event, :closed, project: public_project, target: create(:issue, project: public_project)) } + let_it_be(:reopened_event) { create(:event, :reopened, project: public_project, target: create(:issue, project: public_project)) } + let_it_be(:comments_event) { create(:event, :commented, project: public_project, target: public_project) } + let_it_be(:joined_event) { create(:event, :joined, project: public_project, target: public_project) } + let_it_be(:left_event) { create(:event, :left, project: public_project, target: public_project) } + let_it_be(:wiki_page_event) { create(:wiki_page_event) } + let_it_be(:wiki_page_update_event) { create(:wiki_page_event, :updated) } + let_it_be(:design_event) { create(:design_event) } + + let_it_be(:work_item_event) do + create(:event, + :created, + project: public_project, + target: create(:work_item, :task, project: public_project), + target_type: 'WorkItem' + ) + end + describe '#filter' do it 'returns "all" if given filter is nil' do expect(described_class.new(nil).filter).to eq(described_class::ALL) @@ -18,20 +41,6 @@ RSpec.describe EventFilter do end describe '#apply_filter' do - let_it_be(:public_project) { create(:project, :public) } - let_it_be(:push_event) { create(:push_event, project: public_project) } - let_it_be(:merged_event) { create(:event, :merged, project: public_project, target: public_project) } - let_it_be(:created_event) { create(:event, :created, project: public_project, target: create(:issue, project: public_project)) } - let_it_be(:updated_event) { create(:event, :updated, project: public_project, target: create(:issue, project: public_project)) } - let_it_be(:closed_event) { create(:event, :closed, project: public_project, target: create(:issue, project: public_project)) } - let_it_be(:reopened_event) { create(:event, :reopened, project: public_project, target: create(:issue, project: public_project)) } - let_it_be(:comments_event) { create(:event, :commented, project: public_project, target: public_project) } - let_it_be(:joined_event) { create(:event, :joined, project: public_project, target: public_project) } - let_it_be(:left_event) { create(:event, :left, project: public_project, target: public_project) } - let_it_be(:wiki_page_event) { create(:wiki_page_event) } - let_it_be(:wiki_page_update_event) { create(:wiki_page_event, :updated) } - let_it_be(:design_event) { create(:design_event) } - let(:filtered_events) { described_class.new(filter).apply_filter(Event.all) } context 'with the "push" filter' do @@ -53,8 +62,14 @@ RSpec.describe EventFilter do context 'with the "issue" filter' do let(:filter) { described_class::ISSUE } - it 'filters issue events only' do - expect(filtered_events).to contain_exactly(created_event, updated_event, closed_event, reopened_event) + it 'filters issue and work item events only' do + expect(filtered_events).to contain_exactly( + created_event, + updated_event, + closed_event, + reopened_event, + work_item_event + ) end end @@ -115,6 +130,31 @@ RSpec.describe EventFilter do end end + describe '#in_operator_query_builder_params' do + let(:filtered_events) { described_class.new(filter).in_operator_query_builder_params(array_data) } + let(:array_data) do + { + scope_ids: [public_project.id], + scope_model: Project, + mapping_column: 'project_id' + } + end + + context 'with the "issue" filter' do + let(:filter) { described_class::ISSUE } + + it 'also includes work item events' do + expect(filtered_events[:scope]).to contain_exactly( + created_event, + updated_event, + closed_event, + reopened_event, + work_item_event + ) + end + end + end + describe '#active?' do let(:event_filter) { described_class.new(described_class::TEAM) } diff --git a/spec/lib/gitlab/application_rate_limiter_spec.rb b/spec/lib/gitlab/application_rate_limiter_spec.rb index 41e79f811fa..c938393adce 100644 --- a/spec/lib/gitlab/application_rate_limiter_spec.rb +++ b/spec/lib/gitlab/application_rate_limiter_spec.rb @@ -214,6 +214,52 @@ RSpec.describe Gitlab::ApplicationRateLimiter, :clean_gitlab_redis_rate_limiting end end + describe '.throttled_request?', :freeze_time do + let(:request) { instance_double('Rack::Request') } + + context 'when request is not over the limit' do + it 'returns false and does not log the request' do + expect(subject).not_to receive(:log_request) + + expect(subject.throttled_request?(request, user, :test_action, scope: [user])).to eq(false) + end + end + + context 'when request is over the limit' do + before do + subject.throttled?(:test_action, scope: [user]) + end + + it 'returns true and logs the request' do + expect(subject).to receive(:log_request).with(request, :test_action_request_limit, user) + + expect(subject.throttled_request?(request, user, :test_action, scope: [user])).to eq(true) + end + + context 'when the bypass header is set' do + before do + allow(Gitlab::Throttle).to receive(:bypass_header).and_return('SOME_HEADER') + end + + it 'skips rate limit if set to "1"' do + allow(request).to receive(:get_header).with(Gitlab::Throttle.bypass_header).and_return('1') + + expect(subject).not_to receive(:log_request) + + expect(subject.throttled_request?(request, user, :test_action, scope: [user])).to eq(false) + end + + it 'does not skip rate limit if set to something else than "1"' do + allow(request).to receive(:get_header).with(Gitlab::Throttle.bypass_header).and_return('0') + + expect(subject).to receive(:log_request).with(request, :test_action_request_limit, user) + + expect(subject.throttled_request?(request, user, :test_action, scope: [user])).to eq(true) + end + end + end + end + describe '.peek' do it 'peeks at the current state without changing its value' do freeze_time do diff --git a/spec/lib/gitlab/auth/atlassian/identity_linker_spec.rb b/spec/lib/gitlab/auth/atlassian/identity_linker_spec.rb index ca6b91ac6f1..a303634d463 100644 --- a/spec/lib/gitlab/auth/atlassian/identity_linker_spec.rb +++ b/spec/lib/gitlab/auth/atlassian/identity_linker_spec.rb @@ -17,7 +17,7 @@ RSpec.describe Gitlab::Auth::Atlassian::IdentityLinker do let(:credentials) do { token: SecureRandom.alphanumeric(1254), - refresh_token: SecureRandom.alphanumeric(45), + refresh_token: SecureRandom.alphanumeric(1500), expires_at: 2.weeks.from_now.to_i, expires: true } diff --git a/spec/lib/gitlab/auth/o_auth/user_spec.rb b/spec/lib/gitlab/auth/o_auth/user_spec.rb index bb81621ec92..beeb3ca7011 100644 --- a/spec/lib/gitlab/auth/o_auth/user_spec.rb +++ b/spec/lib/gitlab/auth/o_auth/user_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Auth::OAuth::User do +RSpec.describe Gitlab::Auth::OAuth::User, feature_category: :authentication_and_authorization do include LdapHelpers let(:oauth_user) { described_class.new(auth_hash) } @@ -329,7 +329,7 @@ RSpec.describe Gitlab::Auth::OAuth::User do context "and no LDAP provider defined" do before do - stub_ldap_config(providers: []) + allow(Gitlab::Auth::Ldap::Config).to receive(:providers).at_least(:once).and_return([]) end include_examples "to verify compliance with allow_single_sign_on" @@ -509,6 +509,8 @@ RSpec.describe Gitlab::Auth::OAuth::User do context "and no corresponding LDAP person" do before do allow(Gitlab::Auth::Ldap::Person).to receive(:find_by_uid).and_return(nil) + allow(Gitlab::Auth::Ldap::Person).to receive(:find_by_email).and_return(nil) + allow(Gitlab::Auth::Ldap::Person).to receive(:find_by_dn).and_return(nil) end include_examples "to verify compliance with allow_single_sign_on" @@ -935,7 +937,7 @@ RSpec.describe Gitlab::Auth::OAuth::User do end it "does not update the user location" do - expect(gl_user.location).to be_nil + expect(gl_user.location).to be_blank expect(gl_user.user_synced_attributes_metadata.location_synced).to be(false) end end diff --git a/spec/lib/gitlab/background_migration/add_primary_email_to_emails_if_user_confirmed_spec.rb b/spec/lib/gitlab/background_migration/add_primary_email_to_emails_if_user_confirmed_spec.rb deleted file mode 100644 index b50a55a9e41..00000000000 --- a/spec/lib/gitlab/background_migration/add_primary_email_to_emails_if_user_confirmed_spec.rb +++ /dev/null @@ -1,49 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::AddPrimaryEmailToEmailsIfUserConfirmed do - let(:users) { table(:users) } - let(:emails) { table(:emails) } - - let!(:unconfirmed_user) { users.create!(name: 'unconfirmed', email: 'unconfirmed@example.com', confirmed_at: nil, projects_limit: 100) } - let!(:confirmed_user_1) { users.create!(name: 'confirmed-1', email: 'confirmed-1@example.com', confirmed_at: 1.day.ago, projects_limit: 100) } - let!(:confirmed_user_2) { users.create!(name: 'confirmed-2', email: 'confirmed-2@example.com', confirmed_at: 1.day.ago, projects_limit: 100) } - let!(:email) { emails.create!(user_id: confirmed_user_1.id, email: 'confirmed-1@example.com', confirmed_at: 1.day.ago) } - - let(:perform) { described_class.new.perform(users.first.id, users.last.id) } - - it 'adds the primary email of confirmed users to Emails, unless already added', :aggregate_failures do - expect(emails.where(email: [unconfirmed_user.email, confirmed_user_2.email])).to be_empty - - expect { perform }.not_to raise_error - - expect(emails.where(email: unconfirmed_user.email).count).to eq(0) - expect(emails.where(email: confirmed_user_1.email, user_id: confirmed_user_1.id).count).to eq(1) - expect(emails.where(email: confirmed_user_2.email, user_id: confirmed_user_2.id).count).to eq(1) - - email_2 = emails.find_by(email: confirmed_user_2.email, user_id: confirmed_user_2.id) - expect(email_2.confirmed_at).to eq(confirmed_user_2.reload.confirmed_at) - end - - it 'sets timestamps on the created Emails' do - perform - - email_2 = emails.find_by(email: confirmed_user_2.email, user_id: confirmed_user_2.id) - - expect(email_2.created_at).not_to be_nil - expect(email_2.updated_at).not_to be_nil - end - - context 'when a range of IDs is specified' do - let!(:confirmed_user_3) { users.create!(name: 'confirmed-3', email: 'confirmed-3@example.com', confirmed_at: 1.hour.ago, projects_limit: 100) } - let!(:confirmed_user_4) { users.create!(name: 'confirmed-4', email: 'confirmed-4@example.com', confirmed_at: 1.hour.ago, projects_limit: 100) } - - it 'only acts on the specified range of IDs', :aggregate_failures do - expect do - described_class.new.perform(confirmed_user_2.id, confirmed_user_3.id) - end.to change { Email.count }.by(2) - expect(emails.where(email: confirmed_user_4.email).count).to eq(0) - end - end -end diff --git a/spec/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens_spec.rb b/spec/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens_spec.rb new file mode 100644 index 00000000000..7075d4694ae --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens_spec.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillAdminModeScopeForPersonalAccessTokens, + :migration, schema: 20221228103133, feature_category: :authentication_and_authorization do + let(:users) { table(:users) } + let(:personal_access_tokens) { table(:personal_access_tokens) } + + let(:admin) { users.create!(name: 'admin', email: 'admin@example.com', projects_limit: 1, admin: true) } + let(:user) { users.create!(name: 'user', email: 'user@example.com', projects_limit: 1) } + + let!(:pat_admin_1) { personal_access_tokens.create!(name: 'admin 1', user_id: admin.id, scopes: "---\n- api\n") } + let!(:pat_user) { personal_access_tokens.create!(name: 'user 1', user_id: user.id, scopes: "---\n- api\n") } + let!(:pat_revoked) do + personal_access_tokens.create!(name: 'admin 2', user_id: admin.id, scopes: "---\n- api\n", revoked: true) + end + + let!(:pat_expired) do + personal_access_tokens.create!(name: 'admin 3', user_id: admin.id, scopes: "---\n- api\n", expires_at: 1.day.ago) + end + + let!(:pat_admin_mode) do + personal_access_tokens.create!(name: 'admin 4', user_id: admin.id, scopes: "---\n- admin_mode\n") + end + + let!(:pat_admin_2) { personal_access_tokens.create!(name: 'admin 5', user_id: admin.id, scopes: "---\n- read_api\n") } + let!(:pat_not_in_range) { personal_access_tokens.create!(name: 'admin 6', user_id: admin.id, scopes: "---\n- api\n") } + + subject do + described_class.new( + start_id: pat_admin_1.id, + end_id: pat_admin_2.id, + batch_table: :personal_access_tokens, + batch_column: :id, + sub_batch_size: 1, + pause_ms: 0, + connection: ApplicationRecord.connection + ) + end + + it "adds `admin_mode` scope to active personal access tokens of administrators" do + subject.perform + + expect(pat_admin_1.reload.scopes).to eq("---\n- api\n- admin_mode\n") + expect(pat_user.reload.scopes).to eq("---\n- api\n") + expect(pat_revoked.reload.scopes).to eq("---\n- api\n") + expect(pat_expired.reload.scopes).to eq("---\n- api\n") + expect(pat_admin_mode.reload.scopes).to eq("---\n- admin_mode\n") + expect(pat_admin_2.reload.scopes).to eq("---\n- read_api\n- admin_mode\n") + expect(pat_not_in_range.reload.scopes).to eq("---\n- api\n") + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2_spec.rb b/spec/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2_spec.rb index 8db45ac0f57..96adea03d43 100644 --- a/spec/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::BackfillJiraTrackerDeploymentType2, :migration, schema: 20210301200959 do +RSpec.describe Gitlab::BackgroundMigration::BackfillJiraTrackerDeploymentType2, :migration, schema: 20210602155110 do let!(:jira_integration_temp) { described_class::JiraServiceTemp } let!(:jira_tracker_data_temp) { described_class::JiraTrackerDataTemp } let!(:atlassian_host) { 'https://api.atlassian.net' } diff --git a/spec/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children_spec.rb b/spec/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children_spec.rb index 35928deff82..15956d2ea80 100644 --- a/spec/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::BackfillNamespaceTraversalIdsChildren, :migration, schema: 20210506065000 do +RSpec.describe Gitlab::BackgroundMigration::BackfillNamespaceTraversalIdsChildren, :migration, schema: 20210602155110 do let(:namespaces_table) { table(:namespaces) } let!(:user_namespace) { namespaces_table.create!(id: 1, name: 'user', path: 'user', type: nil) } diff --git a/spec/lib/gitlab/background_migration/backfill_namespace_traversal_ids_roots_spec.rb b/spec/lib/gitlab/background_migration/backfill_namespace_traversal_ids_roots_spec.rb index 96e43275972..019c6d54068 100644 --- a/spec/lib/gitlab/background_migration/backfill_namespace_traversal_ids_roots_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_namespace_traversal_ids_roots_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::BackfillNamespaceTraversalIdsRoots, :migration, schema: 20210506065000 do +RSpec.describe Gitlab::BackgroundMigration::BackfillNamespaceTraversalIdsRoots, :migration, schema: 20210602155110 do let(:namespaces_table) { table(:namespaces) } let!(:user_namespace) { namespaces_table.create!(id: 1, name: 'user', path: 'user', type: nil) } diff --git a/spec/lib/gitlab/background_migration/backfill_releases_author_id_spec.rb b/spec/lib/gitlab/background_migration/backfill_releases_author_id_spec.rb new file mode 100644 index 00000000000..d8ad10849f2 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_releases_author_id_spec.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillReleasesAuthorId, + :migration, schema: 20221215151822, feature_category: :release_orchestration do + let(:releases_table) { table(:releases) } + let(:user_table) { table(:users) } + let(:date_time) { DateTime.now } + + let!(:test_user) { user_table.create!(name: 'test', email: 'test@example.com', username: 'test', projects_limit: 10) } + let!(:ghost_user) do + user_table.create!(name: 'ghost', email: 'ghost@example.com', + username: 'ghost', user_type: User::USER_TYPES['ghost'], projects_limit: 100000) + end + + let(:migration) do + described_class.new(start_id: 1, end_id: 100, + batch_table: :releases, batch_column: :id, + sub_batch_size: 10, pause_ms: 0, + job_arguments: [ghost_user.id], + connection: ApplicationRecord.connection) + end + + subject(:perform_migration) { migration.perform } + + before do + releases_table.create!(tag: 'tag1', name: 'tag1', + released_at: (date_time - 1.minute), author_id: test_user.id) + releases_table.create!(tag: 'tag2', name: 'tag2', + released_at: (date_time - 2.minutes), author_id: test_user.id) + releases_table.new(tag: 'tag3', name: 'tag3', + released_at: (date_time - 3.minutes), author_id: nil).save!(validate: false) + releases_table.new(tag: 'tag4', name: 'tag4', + released_at: (date_time - 4.minutes), author_id: nil).save!(validate: false) + releases_table.new(tag: 'tag5', name: 'tag5', + released_at: (date_time - 5.minutes), author_id: nil).save!(validate: false) + releases_table.create!(tag: 'tag6', name: 'tag6', + released_at: (date_time - 6.minutes), author_id: test_user.id) + releases_table.new(tag: 'tag7', name: 'tag7', + released_at: (date_time - 7.minutes), author_id: nil).save!(validate: false) + end + + it 'backfills `author_id` for the selected records', :aggregate_failures do + expect(releases_table.where(author_id: ghost_user.id).count).to eq 0 + expect(releases_table.where(author_id: nil).count).to eq 4 + + perform_migration + + expect(releases_table.where(author_id: ghost_user.id).count).to eq 4 + expect(releases_table.where(author_id: ghost_user.id).pluck(:name)).to include('tag3', 'tag4', 'tag5', 'tag7') + expect(releases_table.where(author_id: test_user.id).count).to eq 3 + expect(releases_table.where(author_id: test_user.id).pluck(:name)).to include('tag1', 'tag2', 'tag6') + end + + it 'tracks timings of queries' do + expect(migration.batch_metrics.timings).to be_empty + + expect { perform_migration }.to change { migration.batch_metrics.timings } + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb b/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb index 1c2e0e991d9..8d5aa6236a7 100644 --- a/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::BackfillSnippetRepositories, :migration, schema: 2021_03_13_045845 do +RSpec.describe Gitlab::BackgroundMigration::BackfillSnippetRepositories, :migration, schema: 20210602155110 do let(:gitlab_shell) { Gitlab::Shell.new } let(:users) { table(:users) } let(:snippets) { table(:snippets) } diff --git a/spec/lib/gitlab/background_migration/batched_migration_job_spec.rb b/spec/lib/gitlab/background_migration/batched_migration_job_spec.rb index 7280ca0b58e..faaaccfdfaf 100644 --- a/spec/lib/gitlab/background_migration/batched_migration_job_spec.rb +++ b/spec/lib/gitlab/background_migration/batched_migration_job_spec.rb @@ -14,7 +14,7 @@ RSpec.describe Gitlab::BackgroundMigration::BatchedMigrationJob do expect(generic_instance.send(:batch_table)).to eq('projects') expect(generic_instance.send(:batch_column)).to eq('id') - expect(generic_instance.instance_variable_get('@job_arguments')).to eq(%w(x y)) + expect(generic_instance.instance_variable_get(:@job_arguments)).to eq(%w(x y)) expect(generic_instance.send(:connection)).to eq(connection) %i(start_id end_id sub_batch_size pause_ms).each do |attr| diff --git a/spec/lib/gitlab/background_migration/cleanup_orphaned_lfs_objects_projects_spec.rb b/spec/lib/gitlab/background_migration/cleanup_orphaned_lfs_objects_projects_spec.rb index dd202acc372..0d9d9eb929c 100644 --- a/spec/lib/gitlab/background_migration/cleanup_orphaned_lfs_objects_projects_spec.rb +++ b/spec/lib/gitlab/background_migration/cleanup_orphaned_lfs_objects_projects_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::CleanupOrphanedLfsObjectsProjects, schema: 20210514063252 do +RSpec.describe Gitlab::BackgroundMigration::CleanupOrphanedLfsObjectsProjects, schema: 20210602155110 do let(:lfs_objects_projects) { table(:lfs_objects_projects) } let(:lfs_objects) { table(:lfs_objects) } let(:projects) { table(:projects) } diff --git a/spec/lib/gitlab/background_migration/drop_invalid_vulnerabilities_spec.rb b/spec/lib/gitlab/background_migration/drop_invalid_vulnerabilities_spec.rb index ba04f2d20a7..66e16b16270 100644 --- a/spec/lib/gitlab/background_migration/drop_invalid_vulnerabilities_spec.rb +++ b/spec/lib/gitlab/background_migration/drop_invalid_vulnerabilities_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::DropInvalidVulnerabilities, schema: 20210301200959 do +RSpec.describe Gitlab::BackgroundMigration::DropInvalidVulnerabilities, schema: 20210602155110 do let!(:background_migration_jobs) { table(:background_migration_jobs) } let!(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') } let!(:users) { table(:users) } diff --git a/spec/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics_spec.rb b/spec/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics_spec.rb index 5495d786a48..4d7c836cff4 100644 --- a/spec/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics_spec.rb +++ b/spec/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::BackgroundMigration::MigrateProjectTaggingsContextFromTagsToTopics, - :suppress_gitlab_schemas_validate_connection, schema: 20210511095658 do + :suppress_gitlab_schemas_validate_connection, schema: 20210602155110 do it 'correctly migrates project taggings context from tags to topics' do taggings = table(:taggings) diff --git a/spec/lib/gitlab/background_migration/migrate_u2f_webauthn_spec.rb b/spec/lib/gitlab/background_migration/migrate_u2f_webauthn_spec.rb index fc957a7c425..fe45eaac3b7 100644 --- a/spec/lib/gitlab/background_migration/migrate_u2f_webauthn_spec.rb +++ b/spec/lib/gitlab/background_migration/migrate_u2f_webauthn_spec.rb @@ -4,7 +4,7 @@ require 'spec_helper' require 'webauthn/u2f_migrator' -RSpec.describe Gitlab::BackgroundMigration::MigrateU2fWebauthn, :migration, schema: 20210301200959 do +RSpec.describe Gitlab::BackgroundMigration::MigrateU2fWebauthn, :migration, schema: 20210602155110 do let(:users) { table(:users) } let(:user) { users.create!(email: 'email@email.com', name: 'foo', username: 'foo', projects_limit: 0) } diff --git a/spec/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature_spec.rb b/spec/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature_spec.rb index 79b5567f5b3..cafddb6aeaf 100644 --- a/spec/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature_spec.rb +++ b/spec/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::MoveContainerRegistryEnabledToProjectFeature, :migration, schema: 20210301200959 do +RSpec.describe Gitlab::BackgroundMigration::MoveContainerRegistryEnabledToProjectFeature, :migration, schema: 20210602155110 do let(:enabled) { 20 } let(:disabled) { 0 } diff --git a/spec/lib/gitlab/background_migration/sanitize_confidential_todos_spec.rb b/spec/lib/gitlab/background_migration/sanitize_confidential_todos_spec.rb index c58f2060001..a19a3760958 100644 --- a/spec/lib/gitlab/background_migration/sanitize_confidential_todos_spec.rb +++ b/spec/lib/gitlab/background_migration/sanitize_confidential_todos_spec.rb @@ -2,7 +2,9 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::SanitizeConfidentialTodos, :migration, schema: 20221110045406 do +RSpec.describe Gitlab::BackgroundMigration::SanitizeConfidentialTodos, :migration, feature_category: :team_planning do + let!(:issue_type_id) { table(:work_item_types).find_by(base_type: 0).id } + let(:todos) { table(:todos) } let(:notes) { table(:notes) } let(:namespaces) { table(:namespaces) } @@ -29,12 +31,16 @@ RSpec.describe Gitlab::BackgroundMigration::SanitizeConfidentialTodos, :migratio let(:issue1) do issues.create!( - project_id: project1.id, namespace_id: project_namespace1.id, issue_type: 1, title: 'issue1', author_id: user.id + project_id: project1.id, namespace_id: project_namespace1.id, issue_type: 1, title: 'issue1', author_id: user.id, + work_item_type_id: issue_type_id ) end let(:issue2) do - issues.create!(project_id: project2.id, namespace_id: project_namespace2.id, issue_type: 1, title: 'issue2') + issues.create!( + project_id: project2.id, namespace_id: project_namespace2.id, issue_type: 1, title: 'issue2', + work_item_type_id: issue_type_id + ) end let(:public_note) { notes.create!(note: 'text', project_id: project1.id) } diff --git a/spec/lib/gitlab/background_migration/truncate_overlong_vulnerability_html_titles_spec.rb b/spec/lib/gitlab/background_migration/truncate_overlong_vulnerability_html_titles_spec.rb new file mode 100644 index 00000000000..fcd88d523bc --- /dev/null +++ b/spec/lib/gitlab/background_migration/truncate_overlong_vulnerability_html_titles_spec.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +require 'spec_helper' +# rubocop:disable Layout/LineLength +RSpec.describe Gitlab::BackgroundMigration::TruncateOverlongVulnerabilityHtmlTitles, schema: 20221110100602, feature_category: :vulnerability_management do + # rubocop:enable Layout/LineLength + + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:vulnerabilities) { table(:vulnerabilities) } + let(:users) { table(:users) } + let(:namespace) { namespaces.create!(name: 'name', path: 'path') } + + let(:project) do + projects + .create!(name: "project", path: "project", namespace_id: namespace.id, project_namespace_id: namespace.id) + end + + let!(:user) { create_user! } + + let!(:vulnerability_1) { create_vulnerability!(title_html: 'a' * 900, project_id: project.id, author_id: user.id) } + let!(:vulnerability_2) { create_vulnerability!(title_html: 'a' * 801, project_id: project.id, author_id: user.id) } + let!(:vulnerability_3) { create_vulnerability!(title_html: 'a' * 800, project_id: project.id, author_id: user.id) } + let!(:vulnerability_4) { create_vulnerability!(title_html: 'a' * 544, project_id: project.id, author_id: user.id) } + + subject do + described_class.new( + start_id: vulnerabilities.minimum(:id), + end_id: vulnerabilities.maximum(:id), + batch_table: :vulnerabilities, + batch_column: :id, + sub_batch_size: 200, + pause_ms: 2.minutes, + connection: ApplicationRecord.connection + ) + end + + describe '#perform' do + it 'truncates the vulnerability html title when longer than 800 characters' do + subject.perform + + expect(vulnerability_1.reload.title_html.length).to eq(800) + expect(vulnerability_2.reload.title_html.length).to eq(800) + expect(vulnerability_3.reload.title_html.length).to eq(800) + expect(vulnerability_4.reload.title_html.length).to eq(544) + end + end + + private + + # rubocop:disable Metrics/ParameterLists + def create_vulnerability!( + project_id:, author_id:, title: 'test', title_html: 'test', severity: 7, confidence: 7, report_type: 0, state: 1, + dismissed_at: nil + ) + vulnerabilities.create!( + project_id: project_id, + author_id: author_id, + title: title, + title_html: title_html, + severity: severity, + confidence: confidence, + report_type: report_type, + state: state, + dismissed_at: dismissed_at + ) + end + # rubocop:enable Metrics/ParameterLists + + def create_user!(name: "Example User", email: "user@example.com", user_type: nil) + users.create!( + name: name, + email: email, + username: name, + projects_limit: 10 + ) + end +end diff --git a/spec/lib/gitlab/background_migration/update_timelogs_project_id_spec.rb b/spec/lib/gitlab/background_migration/update_timelogs_project_id_spec.rb index fc4d776b8be..7261758e010 100644 --- a/spec/lib/gitlab/background_migration/update_timelogs_project_id_spec.rb +++ b/spec/lib/gitlab/background_migration/update_timelogs_project_id_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::UpdateTimelogsProjectId, schema: 20210427212034 do +RSpec.describe Gitlab::BackgroundMigration::UpdateTimelogsProjectId, schema: 20210602155110 do let!(:namespace) { table(:namespaces).create!(name: 'namespace', path: 'namespace') } let!(:project1) { table(:projects).create!(namespace_id: namespace.id) } let!(:project2) { table(:projects).create!(namespace_id: namespace.id) } diff --git a/spec/lib/gitlab/background_migration/update_users_where_two_factor_auth_required_from_group_spec.rb b/spec/lib/gitlab/background_migration/update_users_where_two_factor_auth_required_from_group_spec.rb index e14328b6150..4599491b580 100644 --- a/spec/lib/gitlab/background_migration/update_users_where_two_factor_auth_required_from_group_spec.rb +++ b/spec/lib/gitlab/background_migration/update_users_where_two_factor_auth_required_from_group_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::UpdateUsersWhereTwoFactorAuthRequiredFromGroup, :migration, schema: 20210519154058 do +RSpec.describe Gitlab::BackgroundMigration::UpdateUsersWhereTwoFactorAuthRequiredFromGroup, :migration, schema: 20210602155110 do include MigrationHelpers::NamespacesHelpers let(:group_with_2fa_parent) { create_namespace('parent', Gitlab::VisibilityLevel::PRIVATE, require_two_factor_authentication: true) } diff --git a/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb b/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb index c78140a70b3..2dea0aef4cf 100644 --- a/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb +++ b/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb @@ -24,7 +24,7 @@ RSpec.describe Gitlab::Cache::Ci::ProjectPipelineStatus, :clean_gitlab_redis_cac described_class.load_in_batch_for_projects([project]) # Don't call the accessor that would lazy load the variable - project_pipeline_status = project.instance_variable_get('@pipeline_status') + project_pipeline_status = project.instance_variable_get(:@pipeline_status) expect(project_pipeline_status).to be_a(described_class) expect(project_pipeline_status).to be_loaded diff --git a/spec/lib/gitlab/ci/config/entry/cache_spec.rb b/spec/lib/gitlab/ci/config/entry/cache_spec.rb index 414cbb169b9..67252eed938 100644 --- a/spec/lib/gitlab/ci/config/entry/cache_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/cache_spec.rb @@ -16,12 +16,14 @@ RSpec.describe Gitlab::Ci::Config::Entry::Cache do let(:policy) { nil } let(:key) { 'some key' } let(:when_config) { nil } + let(:unprotect) { false } let(:config) do { key: key, untracked: true, - paths: ['some/path/'] + paths: ['some/path/'], + unprotect: unprotect }.tap do |config| config[:policy] = policy if policy config[:when] = when_config if when_config @@ -31,7 +33,7 @@ RSpec.describe Gitlab::Ci::Config::Entry::Cache do describe '#value' do shared_examples 'hash key value' do it 'returns hash value' do - expect(entry.value).to eq(key: key, untracked: true, paths: ['some/path/'], policy: 'pull-push', when: 'on_success') + expect(entry.value).to eq(key: key, untracked: true, paths: ['some/path/'], policy: 'pull-push', when: 'on_success', unprotect: false) end end @@ -57,6 +59,14 @@ RSpec.describe Gitlab::Ci::Config::Entry::Cache do end end + context 'with option `unprotect` specified' do + let(:unprotect) { true } + + it 'returns true' do + expect(entry.value).to match(a_hash_including(unprotect: true)) + end + end + context 'with `policy`' do where(:policy, :result) do 'pull-push' | 'pull-push' diff --git a/spec/lib/gitlab/ci/config/entry/job_spec.rb b/spec/lib/gitlab/ci/config/entry/job_spec.rb index becb46ac2e7..c1b9bd58d98 100644 --- a/spec/lib/gitlab/ci/config/entry/job_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/job_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::Entry::Job do +RSpec.describe Gitlab::Ci::Config::Entry::Job, feature_category: :pipeline_authoring do let(:entry) { described_class.new(config, name: :rspec) } it_behaves_like 'with inheritable CI config' do @@ -337,100 +337,6 @@ RSpec.describe Gitlab::Ci::Config::Entry::Job do end end - context 'when only: is used with rules:' do - let(:config) { { only: ['merge_requests'], rules: [{ if: '$THIS' }] } } - - it 'returns error about mixing only: with rules:' do - expect(entry).not_to be_valid - expect(entry.errors).to include /may not be used with `rules`/ - end - - context 'and only: is blank' do - let(:config) { { only: nil, rules: [{ if: '$THIS' }] } } - - it 'returns error about mixing only: with rules:' do - expect(entry).not_to be_valid - expect(entry.errors).to include /may not be used with `rules`/ - end - end - - context 'and rules: is blank' do - let(:config) { { only: ['merge_requests'], rules: nil } } - - it 'returns error about mixing only: with rules:' do - expect(entry).not_to be_valid - expect(entry.errors).to include /may not be used with `rules`/ - end - end - end - - context 'when except: is used with rules:' do - let(:config) { { except: { refs: %w[master] }, rules: [{ if: '$THIS' }] } } - - it 'returns error about mixing except: with rules:' do - expect(entry).not_to be_valid - expect(entry.errors).to include /may not be used with `rules`/ - end - - context 'and except: is blank' do - let(:config) { { except: nil, rules: [{ if: '$THIS' }] } } - - it 'returns error about mixing except: with rules:' do - expect(entry).not_to be_valid - expect(entry.errors).to include /may not be used with `rules`/ - end - end - - context 'and rules: is blank' do - let(:config) { { except: { refs: %w[master] }, rules: nil } } - - it 'returns error about mixing except: with rules:' do - expect(entry).not_to be_valid - expect(entry.errors).to include /may not be used with `rules`/ - end - end - end - - context 'when only: and except: are both used with rules:' do - let(:config) do - { - only: %w[merge_requests], - except: { refs: %w[master] }, - rules: [{ if: '$THIS' }] - } - end - - it 'returns errors about mixing both only: and except: with rules:' do - expect(entry).not_to be_valid - expect(entry.errors).to include /may not be used with `rules`/ - expect(entry.errors).to include /may not be used with `rules`/ - end - - context 'when only: and except: as both blank' do - let(:config) do - { only: nil, except: nil, rules: [{ if: '$THIS' }] } - end - - it 'returns errors about mixing both only: and except: with rules:' do - expect(entry).not_to be_valid - expect(entry.errors).to include /may not be used with `rules`/ - expect(entry.errors).to include /may not be used with `rules`/ - end - end - - context 'when rules: is blank' do - let(:config) do - { only: %w[merge_requests], except: { refs: %w[master] }, rules: nil } - end - - it 'returns errors about mixing both only: and except: with rules:' do - expect(entry).not_to be_valid - expect(entry.errors).to include /may not be used with `rules`/ - expect(entry.errors).to include /may not be used with `rules`/ - end - end - end - context 'when start_in specified without delayed specification' do let(:config) { { start_in: '1 day' } } @@ -603,6 +509,92 @@ RSpec.describe Gitlab::Ci::Config::Entry::Job do end end end + + context 'when only: is used with rules:' do + let(:config) { { only: ['merge_requests'], rules: [{ if: '$THIS' }], script: 'echo' } } + + it 'returns error about mixing only: with rules:' do + expect(entry).not_to be_valid + expect(entry.errors).to include /may not be used with `rules`: only/ + end + + context 'and only: is blank' do + let(:config) { { only: nil, rules: [{ if: '$THIS' }], script: 'echo' } } + + it 'is valid:' do + expect(entry).to be_valid + end + end + + context 'and rules: is blank' do + let(:config) { { only: ['merge_requests'], rules: nil, script: 'echo' } } + + it 'is valid' do + expect(entry).to be_valid + end + end + end + + context 'when except: is used with rules:' do + let(:config) { { except: { refs: %w[master] }, rules: [{ if: '$THIS' }], script: 'echo' } } + + it 'returns error about mixing except: with rules:' do + expect(entry).not_to be_valid + expect(entry.errors).to include /may not be used with `rules`: except/ + end + + context 'and except: is blank' do + let(:config) { { except: nil, rules: [{ if: '$THIS' }], script: 'echo' } } + + it 'is valid' do + expect(entry).to be_valid + end + end + + context 'and rules: is blank' do + let(:config) { { except: { refs: %w[master] }, rules: nil, script: 'echo' } } + + it 'is valid' do + expect(entry).to be_valid + end + end + end + + context 'when only: and except: are both used with rules:' do + let(:config) do + { + only: %w[merge_requests], + except: { refs: %w[master] }, + rules: [{ if: '$THIS' }], + script: 'echo' + } + end + + it 'returns errors about mixing both only: and except: with rules:' do + expect(entry).not_to be_valid + expect(entry.errors).to include /may not be used with `rules`: only, except/ + end + + context 'when only: and except: as both blank' do + let(:config) do + { only: nil, except: nil, rules: [{ if: '$THIS' }], script: 'echo' } + end + + it 'is valid' do + expect(entry).to be_valid + end + end + + context 'when rules: is blank' do + let(:config) do + { only: %w[merge_requests], except: { refs: %w[master] }, rules: nil, script: 'echo' } + end + + it 'is valid' do + expect(entry).to be_valid + end + end + end end describe '#relevant?' do @@ -639,7 +631,7 @@ RSpec.describe Gitlab::Ci::Config::Entry::Job do it 'overrides default config' do expect(entry[:image].value).to eq(name: 'some_image') - expect(entry[:cache].value).to eq([key: 'test', policy: 'pull-push', when: 'on_success']) + expect(entry[:cache].value).to eq([key: 'test', policy: 'pull-push', when: 'on_success', unprotect: false]) end end @@ -654,7 +646,7 @@ RSpec.describe Gitlab::Ci::Config::Entry::Job do it 'uses config from default entry' do expect(entry[:image].value).to eq 'specified' - expect(entry[:cache].value).to eq([key: 'test', policy: 'pull-push', when: 'on_success']) + expect(entry[:cache].value).to eq([key: 'test', policy: 'pull-push', when: 'on_success', unprotect: false]) end end diff --git a/spec/lib/gitlab/ci/config/entry/processable_spec.rb b/spec/lib/gitlab/ci/config/entry/processable_spec.rb index f1578a068b9..b28562ba2ea 100644 --- a/spec/lib/gitlab/ci/config/entry/processable_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/processable_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::Entry::Processable do +RSpec.describe Gitlab::Ci::Config::Entry::Processable, feature_category: :pipeline_authoring do let(:node_class) do Class.new(::Gitlab::Config::Entry::Node) do include Gitlab::Ci::Config::Entry::Processable @@ -104,111 +104,102 @@ RSpec.describe Gitlab::Ci::Config::Entry::Processable do end end - context 'when only: is used with rules:' do - let(:config) { { only: ['merge_requests'], rules: [{ if: '$THIS' }] } } + context 'when a variable has an invalid data attribute' do + let(:config) do + { + script: 'echo', + variables: { 'VAR1' => 'val 1', 'VAR2' => { value: 'val 2', description: 'hello var 2' } } + } + end - it 'returns error about mixing only: with rules:' do - expect(entry).not_to be_valid - expect(entry.errors).to include /may not be used with `rules`/ + it 'reports error about variable' do + expect(entry.errors) + .to include 'variables:var2 config uses invalid data keys: description' end + end + end - context 'and only: is blank' do - let(:config) { { only: nil, rules: [{ if: '$THIS' }] } } + context 'when only: is used with rules:' do + let(:config) { { only: ['merge_requests'], rules: [{ if: '$THIS' }] } } - it 'returns error about mixing only: with rules:' do - expect(entry).not_to be_valid - expect(entry.errors).to include /may not be used with `rules`/ - end - end + it 'returns error about mixing only: with rules:' do + expect(entry).not_to be_valid + expect(entry.errors).to include /may not be used with `rules`: only/ + end - context 'and rules: is blank' do - let(:config) { { only: ['merge_requests'], rules: nil } } + context 'and only: is blank' do + let(:config) { { only: nil, rules: [{ if: '$THIS' }] } } - it 'returns error about mixing only: with rules:' do - expect(entry).not_to be_valid - expect(entry.errors).to include /may not be used with `rules`/ - end + it 'is valid' do + expect(entry).to be_valid end end - context 'when except: is used with rules:' do - let(:config) { { except: { refs: %w[master] }, rules: [{ if: '$THIS' }] } } + context 'and rules: is blank' do + let(:config) { { only: ['merge_requests'], rules: nil } } - it 'returns error about mixing except: with rules:' do - expect(entry).not_to be_valid - expect(entry.errors).to include /may not be used with `rules`/ + it 'is valid' do + expect(entry).to be_valid end + end + end - context 'and except: is blank' do - let(:config) { { except: nil, rules: [{ if: '$THIS' }] } } + context 'when except: is used with rules:' do + let(:config) { { except: { refs: %w[master] }, rules: [{ if: '$THIS' }] } } - it 'returns error about mixing except: with rules:' do - expect(entry).not_to be_valid - expect(entry.errors).to include /may not be used with `rules`/ - end - end + it 'returns error about mixing except: with rules:' do + expect(entry).not_to be_valid + expect(entry.errors).to include /may not be used with `rules`: except/ + end - context 'and rules: is blank' do - let(:config) { { except: { refs: %w[master] }, rules: nil } } + context 'and except: is blank' do + let(:config) { { except: nil, rules: [{ if: '$THIS' }] } } - it 'returns error about mixing except: with rules:' do - expect(entry).not_to be_valid - expect(entry.errors).to include /may not be used with `rules`/ - end + it 'is valid' do + expect(entry).to be_valid end end - context 'when only: and except: are both used with rules:' do - let(:config) do - { - only: %w[merge_requests], - except: { refs: %w[master] }, - rules: [{ if: '$THIS' }] - } - end + context 'and rules: is blank' do + let(:config) { { except: { refs: %w[master] }, rules: nil } } - it 'returns errors about mixing both only: and except: with rules:' do - expect(entry).not_to be_valid - expect(entry.errors).to include /may not be used with `rules`/ - expect(entry.errors).to include /may not be used with `rules`/ + it 'is valid' do + expect(entry).to be_valid end + end + end - context 'when only: and except: as both blank' do - let(:config) do - { only: nil, except: nil, rules: [{ if: '$THIS' }] } - end + context 'when only: and except: are both used with rules:' do + let(:config) do + { + only: %w[merge_requests], + except: { refs: %w[master] }, + rules: [{ if: '$THIS' }] + } + end - it 'returns errors about mixing both only: and except: with rules:' do - expect(entry).not_to be_valid - expect(entry.errors).to include /may not be used with `rules`/ - expect(entry.errors).to include /may not be used with `rules`/ - end - end + it 'returns errors about mixing both only: and except: with rules:' do + expect(entry).not_to be_valid + expect(entry.errors).to include /may not be used with `rules`: only, except/ + end - context 'when rules: is blank' do - let(:config) do - { only: %w[merge_requests], except: { refs: %w[master] }, rules: nil } - end + context 'when only: and except: as both blank' do + let(:config) do + { only: nil, except: nil, rules: [{ if: '$THIS' }] } + end - it 'returns errors about mixing both only: and except: with rules:' do - expect(entry).not_to be_valid - expect(entry.errors).to include /may not be used with `rules`/ - expect(entry.errors).to include /may not be used with `rules`/ - end + it 'is valid' do + expect(entry).to be_valid end end - context 'when a variable has an invalid data attribute' do + context 'when rules: is blank' do let(:config) do - { - script: 'echo', - variables: { 'VAR1' => 'val 1', 'VAR2' => { value: 'val 2', description: 'hello var 2' } } - } + { only: %w[merge_requests], except: { refs: %w[master] }, rules: nil } end - it 'reports error about variable' do - expect(entry.errors) - .to include 'variables:var2 config uses invalid data keys: description' + it 'is valid' do + expect(entry).to be_valid end end end diff --git a/spec/lib/gitlab/ci/config/entry/product/matrix_spec.rb b/spec/lib/gitlab/ci/config/entry/product/matrix_spec.rb index 394d91466bf..cbd3109522c 100644 --- a/spec/lib/gitlab/ci/config/entry/product/matrix_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/product/matrix_spec.rb @@ -29,7 +29,7 @@ RSpec.describe ::Gitlab::Ci::Config::Entry::Product::Matrix do [ { 'VAR_1' => (1..10).to_a, - 'VAR_2' => (11..20).to_a + 'VAR_2' => (11..31).to_a } ] end @@ -41,7 +41,7 @@ RSpec.describe ::Gitlab::Ci::Config::Entry::Product::Matrix do describe '#errors' do it 'returns error about too many jobs' do expect(matrix.errors) - .to include('matrix config generates too many jobs (maximum is 50)') + .to include('matrix config generates too many jobs (maximum is 200)') end end end diff --git a/spec/lib/gitlab/ci/config/entry/product/parallel_spec.rb b/spec/lib/gitlab/ci/config/entry/product/parallel_spec.rb index a16f1cf9e43..ec21519a8f6 100644 --- a/spec/lib/gitlab/ci/config/entry/product/parallel_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/product/parallel_spec.rb @@ -33,10 +33,10 @@ RSpec.describe ::Gitlab::Ci::Config::Entry::Product::Parallel do it_behaves_like 'invalid config', /must be greater than or equal to 2/ end - context 'when it is bigger than 50' do - let(:config) { 51 } + context 'when it is bigger than 200' do + let(:config) { 201 } - it_behaves_like 'invalid config', /must be less than or equal to 50/ + it_behaves_like 'invalid config', /must be less than or equal to 200/ end context 'when it is not an integer' do diff --git a/spec/lib/gitlab/ci/config/entry/reports/coverage_report_spec.rb b/spec/lib/gitlab/ci/config/entry/reports/coverage_report_spec.rb index 0fd9a83a4fa..ccd6f6ab427 100644 --- a/spec/lib/gitlab/ci/config/entry/reports/coverage_report_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/reports/coverage_report_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::Entry::Reports::CoverageReport do +RSpec.describe Gitlab::Ci::Config::Entry::Reports::CoverageReport, feature_category: :pipeline_authoring do let(:entry) { described_class.new(config) } describe 'validations' do @@ -14,6 +14,16 @@ RSpec.describe Gitlab::Ci::Config::Entry::Reports::CoverageReport do it { expect(entry.value).to eq(config) } end + context 'when it is not a hash' do + where(:config) { ['string', true, []] } + + with_them do + it { expect(entry).not_to be_valid } + + it { expect(entry.errors).to include /should be a hash/ } + end + end + context 'with unsupported coverage format' do let(:config) { { coverage_format: 'jacoco', path: 'jacoco.xml' } } diff --git a/spec/lib/gitlab/ci/config/entry/reports_spec.rb b/spec/lib/gitlab/ci/config/entry/reports_spec.rb index 45aa859a356..715cb18fb92 100644 --- a/spec/lib/gitlab/ci/config/entry/reports_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/reports_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::Entry::Reports do +RSpec.describe Gitlab::Ci::Config::Entry::Reports, feature_category: :pipeline_authoring do let(:entry) { described_class.new(config) } describe 'validates ALLOWED_KEYS' do @@ -90,6 +90,18 @@ RSpec.describe Gitlab::Ci::Config::Entry::Reports do end end end + + context 'when coverage_report is nil' do + let(:config) { { coverage_report: nil } } + + it 'is valid' do + expect(entry).to be_valid + end + + it 'returns artifacts configuration as an empty hash' do + expect(entry.value).to eq({}) + end + end end context 'when entry value is not correct' do diff --git a/spec/lib/gitlab/ci/config/entry/root_spec.rb b/spec/lib/gitlab/ci/config/entry/root_spec.rb index c40589104cd..9722609aef6 100644 --- a/spec/lib/gitlab/ci/config/entry/root_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/root_spec.rb @@ -127,7 +127,8 @@ RSpec.describe Gitlab::Ci::Config::Entry::Root do image: { name: 'image:1.0' }, services: [{ name: 'postgres:9.1' }, { name: 'mysql:5.5' }], stage: 'test', - cache: [{ key: 'k', untracked: true, paths: ['public/'], policy: 'pull-push', when: 'on_success' }], + cache: [{ key: 'k', untracked: true, paths: ['public/'], policy: 'pull-push', when: 'on_success', + unprotect: false }], job_variables: {}, root_variables_inheritance: true, ignore: false, @@ -142,7 +143,8 @@ RSpec.describe Gitlab::Ci::Config::Entry::Root do image: { name: 'image:1.0' }, services: [{ name: 'postgres:9.1' }, { name: 'mysql:5.5' }], stage: 'test', - cache: [{ key: 'k', untracked: true, paths: ['public/'], policy: 'pull-push', when: 'on_success' }], + cache: [{ key: 'k', untracked: true, paths: ['public/'], policy: 'pull-push', when: 'on_success', + unprotect: false }], job_variables: {}, root_variables_inheritance: true, ignore: false, @@ -158,7 +160,8 @@ RSpec.describe Gitlab::Ci::Config::Entry::Root do release: { name: "Release $CI_TAG_NAME", tag_name: 'v0.06', description: "./release_changelog.txt" }, image: { name: "image:1.0" }, services: [{ name: "postgres:9.1" }, { name: "mysql:5.5" }], - cache: [{ key: "k", untracked: true, paths: ["public/"], policy: "pull-push", when: 'on_success' }], + cache: [{ key: "k", untracked: true, paths: ["public/"], policy: "pull-push", when: 'on_success', + unprotect: false }], only: { refs: %w(branches tags) }, job_variables: { 'VAR' => { value: 'job' } }, root_variables_inheritance: true, @@ -206,7 +209,7 @@ RSpec.describe Gitlab::Ci::Config::Entry::Root do image: { name: 'image:1.0' }, services: [{ name: 'postgres:9.1' }, { name: 'mysql:5.5' }], stage: 'test', - cache: [{ key: 'k', untracked: true, paths: ['public/'], policy: 'pull-push', when: 'on_success' }], + cache: [{ key: 'k', untracked: true, paths: ['public/'], policy: 'pull-push', when: 'on_success', unprotect: false }], job_variables: {}, root_variables_inheritance: true, ignore: false, @@ -219,7 +222,7 @@ RSpec.describe Gitlab::Ci::Config::Entry::Root do image: { name: 'image:1.0' }, services: [{ name: 'postgres:9.1' }, { name: 'mysql:5.5' }], stage: 'test', - cache: [{ key: 'k', untracked: true, paths: ['public/'], policy: 'pull-push', when: 'on_success' }], + cache: [{ key: 'k', untracked: true, paths: ['public/'], policy: 'pull-push', when: 'on_success', unprotect: false }], job_variables: { 'VAR' => { value: 'job' } }, root_variables_inheritance: true, ignore: false, @@ -274,7 +277,7 @@ RSpec.describe Gitlab::Ci::Config::Entry::Root do describe '#cache_value' do it 'returns correct cache definition' do - expect(root.cache_value).to eq([key: 'a', policy: 'pull-push', when: 'on_success']) + expect(root.cache_value).to eq([key: 'a', policy: 'pull-push', when: 'on_success', unprotect: false]) end end end diff --git a/spec/lib/gitlab/ci/config/entry/variable_spec.rb b/spec/lib/gitlab/ci/config/entry/variable_spec.rb index 97b06c8b1a5..1067db6d124 100644 --- a/spec/lib/gitlab/ci/config/entry/variable_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/variable_spec.rb @@ -257,14 +257,6 @@ RSpec.describe Gitlab::Ci::Config::Entry::Variable do subject(:value_with_data) { entry.value_with_data } it { is_expected.to eq(value: 'value', raw: true) } - - context 'when the FF ci_raw_variables_in_yaml_config is disabled' do - before do - stub_feature_flags(ci_raw_variables_in_yaml_config: false) - end - - it { is_expected.to eq(value: 'value') } - end end context 'when config expand is true' do diff --git a/spec/lib/gitlab/ci/config/external/file/local_spec.rb b/spec/lib/gitlab/ci/config/external/file/local_spec.rb index f5b36ebfa45..a77acb45978 100644 --- a/spec/lib/gitlab/ci/config/external/file/local_spec.rb +++ b/spec/lib/gitlab/ci/config/external/file/local_spec.rb @@ -2,11 +2,13 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::External::File::Local do +RSpec.describe Gitlab::Ci::Config::External::File::Local, feature_category: :pipeline_authoring do + include RepoHelpers + let_it_be(:project) { create(:project, :repository) } let_it_be(:user) { create(:user) } - let(:sha) { '12345' } + let(:sha) { project.commit.sha } let(:variables) { project.predefined_variables.to_runner_variables } let(:context) { Gitlab::Ci::Config::External::Context.new(**context_params) } let(:params) { { local: location } } @@ -172,14 +174,17 @@ RSpec.describe Gitlab::Ci::Config::External::File::Local do let(:another_location) { 'another-config.yml' } let(:another_content) { 'rspec: JOB' } - before do - allow(project.repository).to receive(:blob_data_at).with(sha, location) - .and_return(content) - - allow(project.repository).to receive(:blob_data_at).with(sha, another_location) - .and_return(another_content) + let(:project_files) do + { + location => content, + another_location => another_content + } + end - local_file.validate! + around(:all) do |example| + create_and_delete_files(project, project_files) do + example.run + end end it 'does expand hash to include the template' do @@ -196,11 +201,11 @@ RSpec.describe Gitlab::Ci::Config::External::File::Local do it { is_expected.to eq( context_project: project.full_path, - context_sha: '12345', + context_sha: sha, type: :local, - location: location, - blob: "http://localhost/#{project.full_path}/-/blob/12345/lib/gitlab/ci/templates/existent-file.yml", - raw: "http://localhost/#{project.full_path}/-/raw/12345/lib/gitlab/ci/templates/existent-file.yml", + location: '/lib/gitlab/ci/templates/existent-file.yml', + blob: "http://localhost/#{project.full_path}/-/blob/#{sha}/lib/gitlab/ci/templates/existent-file.yml", + raw: "http://localhost/#{project.full_path}/-/raw/#{sha}/lib/gitlab/ci/templates/existent-file.yml", extra: {} ) } diff --git a/spec/lib/gitlab/ci/config/external/mapper_spec.rb b/spec/lib/gitlab/ci/config/external/mapper_spec.rb index b7e58d4dfa1..9d0e57d4292 100644 --- a/spec/lib/gitlab/ci/config/external/mapper_spec.rb +++ b/spec/lib/gitlab/ci/config/external/mapper_spec.rb @@ -2,7 +2,8 @@ require 'spec_helper' -# This will be removed with FF ci_refactoring_external_mapper and moved to below. +# This will be use with the FF ci_refactoring_external_mapper_verifier in the next MR. +# It can be removed when the FF is removed. RSpec.shared_context 'gitlab_ci_config_external_mapper' do include StubRequests include RepoHelpers @@ -466,12 +467,4 @@ end RSpec.describe Gitlab::Ci::Config::External::Mapper, feature_category: :pipeline_authoring do it_behaves_like 'gitlab_ci_config_external_mapper' - - context 'when the FF ci_refactoring_external_mapper is disabled' do - before do - stub_feature_flags(ci_refactoring_external_mapper: false) - end - - it_behaves_like 'gitlab_ci_config_external_mapper' - end end diff --git a/spec/lib/gitlab/ci/config_spec.rb b/spec/lib/gitlab/ci/config_spec.rb index b48a89059bf..5cdc9c21561 100644 --- a/spec/lib/gitlab/ci/config_spec.rb +++ b/spec/lib/gitlab/ci/config_spec.rb @@ -4,6 +4,7 @@ require 'spec_helper' RSpec.describe Gitlab::Ci::Config, feature_category: :pipeline_authoring do include StubRequests + include RepoHelpers let_it_be(:user) { create(:user) } @@ -313,9 +314,12 @@ RSpec.describe Gitlab::Ci::Config, feature_category: :pipeline_authoring do context "when using 'include' directive" do let_it_be(:group) { create(:group) } + let_it_be(:project) { create(:project, :repository, group: group) } + let_it_be(:main_project) { create(:project, :repository, :public, group: group) } + + let(:project_sha) { project.commit.id } + let(:main_project_sha) { main_project.commit.id } - let(:project) { create(:project, :repository, group: group) } - let(:main_project) { create(:project, :repository, :public, group: group) } let(:pipeline) { build(:ci_pipeline, project: project) } let(:remote_location) { 'https://gitlab.com/gitlab-org/gitlab-foss/blob/1234/.gitlab-ci-1.yml' } @@ -356,36 +360,38 @@ RSpec.describe Gitlab::Ci::Config, feature_category: :pipeline_authoring do end let(:config) do - described_class.new(gitlab_ci_yml, project: project, pipeline: pipeline, sha: '12345', user: user) + described_class.new(gitlab_ci_yml, project: project, pipeline: pipeline, sha: project_sha, user: user) end - before do - stub_full_request(remote_location).to_return(body: remote_file_content) - - allow(project.repository) - .to receive(:blob_data_at).and_return(local_file_content) + let(:project_files) do + { + local_location => local_file_content + } + end - main_project.repository.create_file( - main_project.creator, - '.gitlab-ci.yml', - local_file_content, - message: 'Add README.md', - branch_name: 'master' - ) + let(:main_project_files) do + { + '.gitlab-ci.yml' => local_file_content, + '.another-ci-file.yml' => local_file_content + } + end - main_project.repository.create_file( - main_project.creator, - '.another-ci-file.yml', - local_file_content, - message: 'Add README.md', - branch_name: 'master' - ) + before do + stub_full_request(remote_location).to_return(body: remote_file_content) create(:ci_variable, project: project, key: "REF", value: "HEAD") create(:ci_group_variable, group: group, key: "FILENAME", value: ".gitlab-ci.yml") create(:ci_instance_variable, key: 'MAIN_PROJECT', value: main_project.full_path) end + around do |example| + create_and_delete_files(project, project_files) do + create_and_delete_files(main_project, main_project_files) do + example.run + end + end + end + context "when gitlab_ci_yml has valid 'include' defined" do it 'returns a composed hash' do composed_hash = { @@ -434,25 +440,25 @@ RSpec.describe Gitlab::Ci::Config, feature_category: :pipeline_authoring do expect(config.metadata[:includes]).to contain_exactly( { type: :local, location: local_location, - blob: "http://localhost/#{project.full_path}/-/blob/12345/#{local_location}", - raw: "http://localhost/#{project.full_path}/-/raw/12345/#{local_location}", + blob: "http://localhost/#{project.full_path}/-/blob/#{project_sha}/#{local_location}", + raw: "http://localhost/#{project.full_path}/-/raw/#{project_sha}/#{local_location}", extra: {}, context_project: project.full_path, - context_sha: '12345' }, + context_sha: project_sha }, { type: :remote, location: remote_location, blob: nil, raw: remote_location, extra: {}, context_project: project.full_path, - context_sha: '12345' }, + context_sha: project_sha }, { type: :file, location: '.gitlab-ci.yml', - blob: "http://localhost/#{main_project.full_path}/-/blob/#{main_project.commit.sha}/.gitlab-ci.yml", - raw: "http://localhost/#{main_project.full_path}/-/raw/#{main_project.commit.sha}/.gitlab-ci.yml", + blob: "http://localhost/#{main_project.full_path}/-/blob/#{main_project_sha}/.gitlab-ci.yml", + raw: "http://localhost/#{main_project.full_path}/-/raw/#{main_project_sha}/.gitlab-ci.yml", extra: { project: main_project.full_path, ref: 'HEAD' }, context_project: project.full_path, - context_sha: '12345' } + context_sha: project_sha } ) end end @@ -511,16 +517,13 @@ RSpec.describe Gitlab::Ci::Config, feature_category: :pipeline_authoring do describe 'external file version' do context 'when external local file SHA is defined' do it 'is using a defined value' do - expect(project.repository).to receive(:blob_data_at) - .with('eeff1122', local_location) - - described_class.new(gitlab_ci_yml, project: project, sha: 'eeff1122', user: user, pipeline: pipeline) + described_class.new(gitlab_ci_yml, project: project, sha: project_sha, user: user, pipeline: pipeline) end end context 'when external local file SHA is not defined' do it 'is using latest SHA on the default branch' do - expect(project.repository).to receive(:root_ref_sha) + expect(project.repository).to receive(:root_ref_sha).and_call_original described_class.new(gitlab_ci_yml, project: project, sha: nil, user: user, pipeline: pipeline) end @@ -757,13 +760,11 @@ RSpec.describe Gitlab::Ci::Config, feature_category: :pipeline_authoring do before do project.add_developer(user) + end - allow_next_instance_of(Repository) do |repository| - allow(repository).to receive(:blob_data_at).with(an_instance_of(String), local_location) - .and_return(local_file_content) - - allow(repository).to receive(:blob_data_at).with(an_instance_of(String), other_file_location) - .and_return(other_file_content) + around do |example| + create_and_delete_files(project, { other_file_location => other_file_content }) do + example.run end end @@ -819,14 +820,10 @@ RSpec.describe Gitlab::Ci::Config, feature_category: :pipeline_authoring do HEREDOC end - before do - project.repository.create_file( - project.creator, - 'my_builds.yml', - local_file_content, - message: 'Add my_builds.yml', - branch_name: '12345' - ) + around do |example| + create_and_delete_files(project, { 'my_builds.yml' => local_file_content }) do + example.run + end end context 'when the exists file does not exist' do @@ -853,7 +850,7 @@ RSpec.describe Gitlab::Ci::Config, feature_category: :pipeline_authoring do include: - local: #{local_location} rules: - - if: $CI_COMMIT_SHA == "#{project.commit.sha}" + - if: $CI_COMMIT_REF_NAME == "master" HEREDOC end diff --git a/spec/lib/gitlab/ci/parsers/sbom/validators/cyclonedx_schema_validator_spec.rb b/spec/lib/gitlab/ci/parsers/sbom/validators/cyclonedx_schema_validator_spec.rb index 712dc00ec7a..acb7c122bcd 100644 --- a/spec/lib/gitlab/ci/parsers/sbom/validators/cyclonedx_schema_validator_spec.rb +++ b/spec/lib/gitlab/ci/parsers/sbom/validators/cyclonedx_schema_validator_spec.rb @@ -62,6 +62,47 @@ RSpec.describe Gitlab::Ci::Parsers::Sbom::Validators::CyclonedxSchemaValidator, it { is_expected.to be_valid } end + context 'when components have licenses' do + let(:components) do + [ + { + "type" => "library", + "name" => "activesupport", + "version" => "5.1.4", + "licenses" => [ + { "license" => { "id" => "MIT" } } + ] + } + ] + end + + it { is_expected.to be_valid } + end + + context 'when components have a signature' do + let(:components) do + [ + { + "type" => "library", + "name" => "activesupport", + "version" => "5.1.4", + "signature" => { + "algorithm" => "ES256", + "publicKey" => { + "kty" => "EC", + "crv" => "P-256", + "x" => "6BKxpty8cI-exDzCkh-goU6dXq3MbcY0cd1LaAxiNrU", + "y" => "mCbcvUzm44j3Lt2b5BPyQloQ91tf2D2V-gzeUxWaUdg" + }, + "value" => "ybT1qz5zHNi4Ndc6y7Zhamuf51IqXkPkZwjH1XcC-KSuBiaQplTw6Jasf2MbCLg3CF7PAdnMO__WSLwvI5r2jA" + } + } + ] + end + + it { is_expected.to be_valid } + end + context "when components are not valid" do let(:components) do [ diff --git a/spec/lib/gitlab/ci/parsers/security/validators/schema_validator_spec.rb b/spec/lib/gitlab/ci/parsers/security/validators/schema_validator_spec.rb index c94ed1f8d6d..12886c79d7d 100644 --- a/spec/lib/gitlab/ci/parsers/security/validators/schema_validator_spec.rb +++ b/spec/lib/gitlab/ci/parsers/security/validators/schema_validator_spec.rb @@ -2,9 +2,10 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator do +RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator, feature_category: :vulnerability_management do let_it_be(:project) { create(:project) } + let(:current_dast_versions) { described_class::CURRENT_VERSIONS[:dast].join(', ') } let(:supported_dast_versions) { described_class::SUPPORTED_VERSIONS[:dast].join(', ') } let(:deprecated_schema_version_message) {} let(:missing_schema_version_message) do @@ -19,6 +20,14 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator do } end + let(:analyzer_vendor) do + { 'name' => 'A DAST analyzer' } + end + + let(:scanner_vendor) do + { 'name' => 'A DAST scanner' } + end + let(:report_data) do { 'scan' => { @@ -26,7 +35,7 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator do 'id' => 'my-dast-analyzer', 'name' => 'My DAST analyzer', 'version' => '0.1.0', - 'vendor' => { 'name' => 'A DAST analyzer' } + 'vendor' => analyzer_vendor }, 'end_time' => '2020-01-28T03:26:02', 'scanned_resources' => [], @@ -34,7 +43,7 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator do 'id' => 'my-dast-scanner', 'name' => 'My DAST scanner', 'version' => '0.2.0', - 'vendor' => { 'name' => 'A DAST scanner' } + 'vendor' => scanner_vendor }, 'start_time' => '2020-01-28T03:26:01', 'status' => 'success', @@ -458,8 +467,9 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator do let(:report_version) { described_class::DEPRECATED_VERSIONS[report_type].last } let(:expected_deprecation_message) do - "Version #{report_version} for report type #{report_type} has been deprecated, supported versions for this "\ - "report type are: #{supported_dast_versions}. GitLab will attempt to parse and ingest this report if valid." + "version #{report_version} for report type #{report_type} is deprecated. "\ + "However, GitLab will still attempt to parse and ingest this report. "\ + "Upgrade the security report to one of the following versions: #{current_dast_versions}." end let(:expected_deprecation_warnings) do @@ -492,6 +502,22 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator do it_behaves_like 'report with expected warnings' end + + context 'and the report passes schema validation as a GitLab-vendored analyzer' do + let(:analyzer_vendor) do + { 'name' => 'GitLab' } + end + + it { is_expected.to be_empty } + end + + context 'and the report passes schema validation as a GitLab-vendored scanner' do + let(:scanner_vendor) do + { 'name' => 'GitLab' } + end + + it { is_expected.to be_empty } + end end context 'when given an unsupported schema version' do diff --git a/spec/lib/gitlab/ci/pipeline/chain/create_deployments_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/create_deployments_spec.rb index be5d3a96126..bec80a43a76 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/create_deployments_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/create_deployments_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Pipeline::Chain::CreateDeployments do +RSpec.describe Gitlab::Ci::Pipeline::Chain::CreateDeployments, feature_category: :continuous_integration do let_it_be(:project) { create(:project, :repository) } let_it_be(:user) { create(:user) } @@ -19,6 +19,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::CreateDeployments do subject { step.perform! } before do + stub_feature_flags(move_create_deployments_to_worker: false) job.pipeline = pipeline end diff --git a/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb index eba0db0adfb..e13e78d0db8 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb @@ -63,11 +63,11 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Create do end let(:job) do - build(:ci_build, stage: stage, pipeline: pipeline, project: project) + build(:ci_build, ci_stage: stage, pipeline: pipeline, project: project) end let(:bridge) do - build(:ci_bridge, stage: stage, pipeline: pipeline, project: project) + build(:ci_bridge, ci_stage: stage, pipeline: pipeline, project: project) end before do diff --git a/spec/lib/gitlab/ci/pipeline/chain/populate_metadata_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/populate_metadata_spec.rb index 35e1c48a942..00200b57b1e 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/populate_metadata_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/populate_metadata_spec.rb @@ -54,94 +54,76 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::PopulateMetadata do expect(step.break?).to be false end - context 'with feature flag disabled' do - before do - stub_feature_flags(pipeline_name: false) - end - - it 'does not build pipeline_metadata' do - run_chain + it 'builds pipeline_metadata' do + run_chain - expect(pipeline.pipeline_metadata).to be_nil - end + expect(pipeline.pipeline_metadata.name).to eq('Pipeline name') + expect(pipeline.pipeline_metadata.project).to eq(pipeline.project) + expect(pipeline.pipeline_metadata).not_to be_persisted end - context 'with feature flag enabled' do - before do - stub_feature_flags(pipeline_name: true) + context 'with empty name' do + let(:config) do + { workflow: { name: ' ' }, rspec: { script: 'rspec' } } end - it 'builds pipeline_metadata' do + it 'strips whitespace from name' do run_chain - expect(pipeline.pipeline_metadata.name).to eq('Pipeline name') - expect(pipeline.pipeline_metadata.project).to eq(pipeline.project) - expect(pipeline.pipeline_metadata).not_to be_persisted + expect(pipeline.pipeline_metadata).to be_nil end - context 'with empty name' do + context 'with empty name after variable substitution' do let(:config) do - { workflow: { name: ' ' }, rspec: { script: 'rspec' } } + { workflow: { name: '$VAR1' }, rspec: { script: 'rspec' } } end - it 'strips whitespace from name' do + it 'does not save empty name' do run_chain expect(pipeline.pipeline_metadata).to be_nil end - - context 'with empty name after variable substitution' do - let(:config) do - { workflow: { name: '$VAR1' }, rspec: { script: 'rspec' } } - end - - it 'does not save empty name' do - run_chain - - expect(pipeline.pipeline_metadata).to be_nil - end - end end + end - context 'with variables' do - let(:config) do - { - variables: { ROOT_VAR: 'value $WORKFLOW_VAR1' }, - workflow: { - name: 'Pipeline $ROOT_VAR $WORKFLOW_VAR2 $UNKNOWN_VAR', - rules: [{ variables: { WORKFLOW_VAR1: 'value1', WORKFLOW_VAR2: 'value2' } }] - }, - rspec: { script: 'rspec' } - } - end + context 'with variables' do + let(:config) do + { + variables: { ROOT_VAR: 'value $WORKFLOW_VAR1' }, + workflow: { + name: 'Pipeline $ROOT_VAR $WORKFLOW_VAR2 $UNKNOWN_VAR', + rules: [{ variables: { WORKFLOW_VAR1: 'value1', WORKFLOW_VAR2: 'value2' } }] + }, + rspec: { script: 'rspec' } + } + end - it 'substitutes variables' do - run_chain + it 'substitutes variables' do + run_chain - expect(pipeline.pipeline_metadata.name).to eq('Pipeline value value1 value2') - end + expect(pipeline.pipeline_metadata.name).to eq('Pipeline value value1 value2') end + end - context 'with invalid name' do - let(:config) do - { - variables: { ROOT_VAR: 'a' * 256 }, - workflow: { - name: 'Pipeline $ROOT_VAR' - }, - rspec: { script: 'rspec' } - } - end + context 'with invalid name' do + let(:config) do + { + variables: { ROOT_VAR: 'a' * 256 }, + workflow: { + name: 'Pipeline $ROOT_VAR' + }, + rspec: { script: 'rspec' } + } + end - it 'returns error and breaks chain' do - ret = run_chain + it 'returns error and breaks chain' do + ret = run_chain - expect(ret) - .to match_array(["Failed to build pipeline metadata! Name is too long (maximum is 255 characters)"]) - expect(pipeline.pipeline_metadata.errors.full_messages) - .to match_array(['Name is too long (maximum is 255 characters)']) - expect(step.break?).to be true - end + expect(ret) + .to match_array(["Failed to build pipeline metadata! Name is too long (maximum is 255 characters)"]) + expect(pipeline.pipeline_metadata.errors.full_messages) + .to match_array(['Name is too long (maximum is 255 characters)']) + expect(step.break?).to be true end end end diff --git a/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb index 62de4d2e96d..91bb94bbb11 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Pipeline::Chain::Populate do +RSpec.describe Gitlab::Ci::Pipeline::Chain::Populate, feature_category: :continuous_integration do let_it_be(:project) { create(:project, :repository) } let_it_be(:user) { create(:user) } @@ -90,7 +90,8 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Populate do it 'appends an error about missing stages' do expect(pipeline.errors.to_a) - .to include 'No stages / jobs for this pipeline.' + .to include 'Pipeline will not run for the selected trigger. ' \ + 'The rules configuration prevented any jobs from being added to the pipeline.' end it 'wastes pipeline iid' do diff --git a/spec/lib/gitlab/ci/pipeline/seed/build/cache_spec.rb b/spec/lib/gitlab/ci/pipeline/seed/build/cache_spec.rb index fb8020bf43e..c264ea3bece 100644 --- a/spec/lib/gitlab/ci/pipeline/seed/build/cache_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/seed/build/cache_spec.rb @@ -212,6 +212,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build::Cache do paths: ['vendor/ruby'], untracked: true, policy: 'push', + unprotect: true, when: 'on_success' } end diff --git a/spec/lib/gitlab/ci/status/bridge/factory_spec.rb b/spec/lib/gitlab/ci/status/bridge/factory_spec.rb index 6081f104e42..c13901a4776 100644 --- a/spec/lib/gitlab/ci/status/bridge/factory_spec.rb +++ b/spec/lib/gitlab/ci/status/bridge/factory_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Status::Bridge::Factory do +RSpec.describe Gitlab::Ci::Status::Bridge::Factory, feature_category: :continuous_integration do let(:user) { create(:user) } let(:project) { bridge.project } let(:status) { factory.fabricate! } @@ -59,13 +59,15 @@ RSpec.describe Gitlab::Ci::Status::Bridge::Factory do context 'failed with downstream_pipeline_creation_failed' do before do - bridge.options = { downstream_errors: ['No stages / jobs for this pipeline.', 'other error'] } + bridge.options = { downstream_errors: ['Pipeline will not run for the selected trigger. ' \ + 'The rules configuration prevented any jobs from being added to the pipeline.', 'other error'] } bridge.failure_reason = 'downstream_pipeline_creation_failed' end it 'fabricates correct status_tooltip' do expect(status.status_tooltip).to eq( - "#{s_('CiStatusText|failed')} - (downstream pipeline can not be created, No stages / jobs for this pipeline., other error)" + "#{s_('CiStatusText|failed')} - (downstream pipeline can not be created, Pipeline will not run for the selected trigger. " \ + "The rules configuration prevented any jobs from being added to the pipeline., other error)" ) end end diff --git a/spec/lib/gitlab/ci/status/build/manual_spec.rb b/spec/lib/gitlab/ci/status/build/manual_spec.rb index a1152cb77e3..8f5d1558314 100644 --- a/spec/lib/gitlab/ci/status/build/manual_spec.rb +++ b/spec/lib/gitlab/ci/status/build/manual_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Status::Build::Manual do +RSpec.describe Gitlab::Ci::Status::Build::Manual, feature_category: :continuous_integration do let_it_be(:user) { create(:user) } let_it_be(:job) { create(:ci_build, :manual) } @@ -18,11 +18,44 @@ RSpec.describe Gitlab::Ci::Status::Build::Manual do job.project.add_maintainer(user) end - it { expect(subject.illustration[:content]).to match /This job requires manual intervention to start/ } + context 'when the job has not been played' do + it 'instructs the user about possible actions' do + expect(subject.illustration[:content]).to eq( + _( + 'This job does not start automatically and must be started manually. ' \ + 'You can add CI/CD variables below for last-minute configuration changes before starting the job.' + ) + ) + end + end + + context 'when the job is retryable' do + before do + job.update!(status: :failed) + end + + it 'instructs the user about possible actions' do + expect(subject.illustration[:content]).to eq( + _("You can modify this job's CI/CD variables before running it again.") + ) + end + end + end + + context 'when the user can not trigger the job because of outdated deployment' do + before do + allow(job).to receive(:outdated_deployment?).and_return(true) + end + + it { expect(subject.illustration[:content]).to match /This deployment job does not run automatically and must be started manually, but it's older than the latest deployment, and therefore can't run/ } end - context 'when the user can not trigger the job' do - it { expect(subject.illustration[:content]).to match /This job does not run automatically and must be started manually/ } + context 'when the user can not trigger the job due to another reason' do + it 'informs the user' do + expect(subject.illustration[:content]).to eq( + _("This job does not run automatically and must be started manually, but you do not have access to it.") + ) + end end end diff --git a/spec/lib/gitlab/ci/templates/Jobs/code_quality_gitlab_ci_yaml_spec.rb b/spec/lib/gitlab/ci/templates/Jobs/code_quality_gitlab_ci_yaml_spec.rb index 16c5d7a4b6d..286f3d10b7f 100644 --- a/spec/lib/gitlab/ci/templates/Jobs/code_quality_gitlab_ci_yaml_spec.rb +++ b/spec/lib/gitlab/ci/templates/Jobs/code_quality_gitlab_ci_yaml_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe 'Jobs/Code-Quality.gitlab-ci.yml' do +RSpec.describe 'Jobs/Code-Quality.gitlab-ci.yml', feature_category: :continuous_integration do subject(:template) { Gitlab::Template::GitlabCiYmlTemplate.find('Jobs/Code-Quality') } describe 'the created pipeline' do @@ -63,7 +63,8 @@ RSpec.describe 'Jobs/Code-Quality.gitlab-ci.yml' do context 'on master' do it 'has no jobs' do expect(build_names).to be_empty - expect(pipeline.errors.full_messages).to match_array(["No stages / jobs for this pipeline."]) + expect(pipeline.errors.full_messages).to match_array(['Pipeline will not run for the selected trigger. ' \ + 'The rules configuration prevented any jobs from being added to the pipeline.']) end end @@ -72,7 +73,8 @@ RSpec.describe 'Jobs/Code-Quality.gitlab-ci.yml' do it 'has no jobs' do expect(build_names).to be_empty - expect(pipeline.errors.full_messages).to match_array(["No stages / jobs for this pipeline."]) + expect(pipeline.errors.full_messages).to match_array(['Pipeline will not run for the selected trigger. ' \ + 'The rules configuration prevented any jobs from being added to the pipeline.']) end end @@ -81,7 +83,8 @@ RSpec.describe 'Jobs/Code-Quality.gitlab-ci.yml' do it 'has no jobs' do expect(build_names).to be_empty - expect(pipeline.errors.full_messages).to match_array(["No stages / jobs for this pipeline."]) + expect(pipeline.errors.full_messages).to match_array(['Pipeline will not run for the selected trigger. ' \ + 'The rules configuration prevented any jobs from being added to the pipeline.']) end end end diff --git a/spec/lib/gitlab/ci/templates/Jobs/sast_iac_gitlab_ci_yaml_spec.rb b/spec/lib/gitlab/ci/templates/Jobs/sast_iac_gitlab_ci_yaml_spec.rb index 8a5aea7c0f0..68d249e31f9 100644 --- a/spec/lib/gitlab/ci/templates/Jobs/sast_iac_gitlab_ci_yaml_spec.rb +++ b/spec/lib/gitlab/ci/templates/Jobs/sast_iac_gitlab_ci_yaml_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe 'Jobs/SAST-IaC.gitlab-ci.yml' do +RSpec.describe 'Jobs/SAST-IaC.gitlab-ci.yml', feature_category: :continuous_integration do subject(:template) { Gitlab::Template::GitlabCiYmlTemplate.find('Jobs/SAST-IaC') } describe 'the created pipeline' do @@ -50,7 +50,8 @@ RSpec.describe 'Jobs/SAST-IaC.gitlab-ci.yml' do context 'on default branch' do it 'has no jobs' do expect(build_names).to be_empty - expect(pipeline.errors.full_messages).to match_array(["No stages / jobs for this pipeline."]) + expect(pipeline.errors.full_messages).to match_array(['Pipeline will not run for the selected trigger. ' \ + 'The rules configuration prevented any jobs from being added to the pipeline.']) end end @@ -59,7 +60,8 @@ RSpec.describe 'Jobs/SAST-IaC.gitlab-ci.yml' do it 'has no jobs' do expect(build_names).to be_empty - expect(pipeline.errors.full_messages).to match_array(["No stages / jobs for this pipeline."]) + expect(pipeline.errors.full_messages).to match_array(['Pipeline will not run for the selected trigger. ' \ + 'The rules configuration prevented any jobs from being added to the pipeline.']) end end end diff --git a/spec/lib/gitlab/ci/templates/Jobs/sast_iac_latest_gitlab_ci_yaml_spec.rb b/spec/lib/gitlab/ci/templates/Jobs/sast_iac_latest_gitlab_ci_yaml_spec.rb index d540b035f81..039a6a739dd 100644 --- a/spec/lib/gitlab/ci/templates/Jobs/sast_iac_latest_gitlab_ci_yaml_spec.rb +++ b/spec/lib/gitlab/ci/templates/Jobs/sast_iac_latest_gitlab_ci_yaml_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe 'Jobs/SAST-IaC.latest.gitlab-ci.yml' do +RSpec.describe 'Jobs/SAST-IaC.latest.gitlab-ci.yml', feature_category: :continuous_integration do subject(:template) { Gitlab::Template::GitlabCiYmlTemplate.find('Jobs/SAST-IaC.latest') } describe 'the created pipeline' do @@ -51,7 +51,8 @@ RSpec.describe 'Jobs/SAST-IaC.latest.gitlab-ci.yml' do context 'on default branch' do it 'has no jobs' do expect(build_names).to be_empty - expect(pipeline.errors.full_messages).to match_array(["No stages / jobs for this pipeline."]) + expect(pipeline.errors.full_messages).to match_array(['Pipeline will not run for the selected trigger. ' \ + 'The rules configuration prevented any jobs from being added to the pipeline.']) end end @@ -60,7 +61,8 @@ RSpec.describe 'Jobs/SAST-IaC.latest.gitlab-ci.yml' do it 'has no jobs' do expect(build_names).to be_empty - expect(pipeline.errors.full_messages).to match_array(["No stages / jobs for this pipeline."]) + expect(pipeline.errors.full_messages).to match_array(['Pipeline will not run for the selected trigger. ' \ + 'The rules configuration prevented any jobs from being added to the pipeline.']) end end end diff --git a/spec/lib/gitlab/ci/templates/Jobs/test_gitlab_ci_yaml_spec.rb b/spec/lib/gitlab/ci/templates/Jobs/test_gitlab_ci_yaml_spec.rb index 7cf0cf3ed33..d73d8a15feb 100644 --- a/spec/lib/gitlab/ci/templates/Jobs/test_gitlab_ci_yaml_spec.rb +++ b/spec/lib/gitlab/ci/templates/Jobs/test_gitlab_ci_yaml_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe 'Jobs/Test.gitlab-ci.yml' do +RSpec.describe 'Jobs/Test.gitlab-ci.yml', feature_category: :continuous_integration do subject(:template) { Gitlab::Template::GitlabCiYmlTemplate.find('Jobs/Test') } describe 'the created pipeline' do @@ -63,7 +63,8 @@ RSpec.describe 'Jobs/Test.gitlab-ci.yml' do context 'on master' do it 'has no jobs' do expect(build_names).to be_empty - expect(pipeline.errors.full_messages).to match_array(["No stages / jobs for this pipeline."]) + expect(pipeline.errors.full_messages).to match_array(['Pipeline will not run for the selected trigger. ' \ + 'The rules configuration prevented any jobs from being added to the pipeline.']) end end @@ -72,7 +73,8 @@ RSpec.describe 'Jobs/Test.gitlab-ci.yml' do it 'has no jobs' do expect(build_names).to be_empty - expect(pipeline.errors.full_messages).to match_array(["No stages / jobs for this pipeline."]) + expect(pipeline.errors.full_messages).to match_array(['Pipeline will not run for the selected trigger. ' \ + 'The rules configuration prevented any jobs from being added to the pipeline.']) end end @@ -81,7 +83,8 @@ RSpec.describe 'Jobs/Test.gitlab-ci.yml' do it 'has no jobs' do expect(build_names).to be_empty - expect(pipeline.errors.full_messages).to match_array(["No stages / jobs for this pipeline."]) + expect(pipeline.errors.full_messages).to match_array(['Pipeline will not run for the selected trigger. ' \ + 'The rules configuration prevented any jobs from being added to the pipeline.']) end end end diff --git a/spec/lib/gitlab/ci/templates/auto_devops_gitlab_ci_yaml_spec.rb b/spec/lib/gitlab/ci/templates/auto_devops_gitlab_ci_yaml_spec.rb index b2ca906e172..09ca2678de5 100644 --- a/spec/lib/gitlab/ci/templates/auto_devops_gitlab_ci_yaml_spec.rb +++ b/spec/lib/gitlab/ci/templates/auto_devops_gitlab_ci_yaml_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe 'Auto-DevOps.gitlab-ci.yml' do +RSpec.describe 'Auto-DevOps.gitlab-ci.yml', feature_category: :auto_devops do using RSpec::Parameterized::TableSyntax subject(:template) { Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps') } diff --git a/spec/lib/gitlab/ci/templates/npm_spec.rb b/spec/lib/gitlab/ci/templates/npm_spec.rb index 55fd4675f11..a949a7ccfb1 100644 --- a/spec/lib/gitlab/ci/templates/npm_spec.rb +++ b/spec/lib/gitlab/ci/templates/npm_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe 'npm.gitlab-ci.yml' do +RSpec.describe 'npm.gitlab-ci.yml', feature_category: :continuous_integration do subject(:template) { Gitlab::Template::GitlabCiYmlTemplate.find('npm') } describe 'the created pipeline' do @@ -43,7 +43,8 @@ RSpec.describe 'npm.gitlab-ci.yml' do shared_examples 'no pipeline created' do it 'does not create a pipeline because the only job (publish) is not created' do expect(build_names).to be_empty - expect(pipeline.errors.full_messages).to match_array(["No stages / jobs for this pipeline."]) + expect(pipeline.errors.full_messages).to match_array(['Pipeline will not run for the selected trigger. ' \ + 'The rules configuration prevented any jobs from being added to the pipeline.']) end end diff --git a/spec/lib/gitlab/ci/templates/terraform_latest_gitlab_ci_yaml_spec.rb b/spec/lib/gitlab/ci/templates/terraform_latest_gitlab_ci_yaml_spec.rb index 6ae51f9783b..a81f29d0d01 100644 --- a/spec/lib/gitlab/ci/templates/terraform_latest_gitlab_ci_yaml_spec.rb +++ b/spec/lib/gitlab/ci/templates/terraform_latest_gitlab_ci_yaml_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe 'Terraform.latest.gitlab-ci.yml' do +RSpec.describe 'Terraform.latest.gitlab-ci.yml', feature_category: :continuous_integration do before do allow(Gitlab::Template::GitlabCiYmlTemplate).to receive(:excluded_patterns).and_return([]) end @@ -66,7 +66,12 @@ RSpec.describe 'Terraform.latest.gitlab-ci.yml' do it 'does not create a branch pipeline', :aggregate_failures do expect(branch_build_names).to be_empty - expect(branch_pipeline.errors.full_messages).to match_array(["No stages / jobs for this pipeline."]) + expect(branch_pipeline.errors.full_messages).to match_array( + [ + 'Pipeline will not run for the selected trigger. ' \ + 'The rules configuration prevented any jobs from being added to the pipeline.' + ] + ) end end end diff --git a/spec/lib/gitlab/ci/templates/themekit_gitlab_ci_yaml_spec.rb b/spec/lib/gitlab/ci/templates/themekit_gitlab_ci_yaml_spec.rb index 157fd39f1cc..607db33f61a 100644 --- a/spec/lib/gitlab/ci/templates/themekit_gitlab_ci_yaml_spec.rb +++ b/spec/lib/gitlab/ci/templates/themekit_gitlab_ci_yaml_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe 'ThemeKit.gitlab-ci.yml' do +RSpec.describe 'ThemeKit.gitlab-ci.yml', feature_category: :continuous_integration do before do allow(Gitlab::Template::GitlabCiYmlTemplate).to receive(:excluded_patterns).and_return([]) end @@ -52,7 +52,8 @@ RSpec.describe 'ThemeKit.gitlab-ci.yml' do it 'has no jobs' do expect(build_names).to be_empty - expect(pipeline.errors.full_messages).to match_array(["No stages / jobs for this pipeline."]) + expect(pipeline.errors.full_messages).to match_array(['Pipeline will not run for the selected trigger. ' \ + 'The rules configuration prevented any jobs from being added to the pipeline.']) end end end diff --git a/spec/lib/gitlab/ci/variables/collection_spec.rb b/spec/lib/gitlab/ci/variables/collection_spec.rb index 10b8f0065d9..4ee122cc607 100644 --- a/spec/lib/gitlab/ci/variables/collection_spec.rb +++ b/spec/lib/gitlab/ci/variables/collection_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Variables::Collection do +RSpec.describe Gitlab::Ci::Variables::Collection, feature_category: :pipeline_authoring do describe '.new' do it 'can be initialized with an array' do variable = { key: 'VAR', value: 'value', public: true, masked: false } @@ -295,69 +295,6 @@ RSpec.describe Gitlab::Ci::Variables::Collection do end end - describe '#expand_value' do - let(:collection) do - Gitlab::Ci::Variables::Collection.new - .append(key: 'CI_JOB_NAME', value: 'test-1') - .append(key: 'CI_BUILD_ID', value: '1') - .append(key: 'TEST1', value: 'test-3') - .append(key: 'FILEVAR1', value: 'file value 1', file: true) - end - - context 'table tests' do - using RSpec::Parameterized::TableSyntax - - where do - { - "empty value": { - value: '', - result: '' - }, - "simple expansions": { - value: 'key$TEST1-$CI_BUILD_ID', - result: 'keytest-3-1' - }, - "complex expansion": { - value: 'key${TEST1}-${CI_JOB_NAME}', - result: 'keytest-3-test-1' - }, - "missing variable not keeping original": { - value: 'key${MISSING_VAR}-${CI_JOB_NAME}', - result: 'key-test-1' - }, - "missing variable keeping original": { - value: 'key${MISSING_VAR}-${CI_JOB_NAME}', - result: 'key${MISSING_VAR}-test-1', - keep_undefined: true - }, - "escaped characters are kept intact": { - value: 'key-$TEST1-%%HOME%%-$${HOME}', - result: 'key-test-3-%%HOME%%-$${HOME}' - }, - "file variable with expand_file_refs: true": { - value: 'key-$FILEVAR1-$TEST1', - result: 'key-file value 1-test-3' - }, - "file variable with expand_file_refs: false": { - value: 'key-$FILEVAR1-$TEST1', - result: 'key-$FILEVAR1-test-3', - expand_file_refs: false - } - } - end - - with_them do - let(:options) { { keep_undefined: keep_undefined, expand_file_refs: expand_file_refs }.compact } - - subject(:expanded_result) { collection.expand_value(value, **options) } - - it 'matches expected expansion' do - is_expected.to eq(result) - end - end - end - end - describe '#sort_and_expand_all' do context 'table tests' do using RSpec::Parameterized::TableSyntax @@ -369,6 +306,14 @@ RSpec.describe Gitlab::Ci::Variables::Collection do keep_undefined: false, result: [] }, + "empty string": { + variables: [ + { key: 'variable', value: '' } + ], + result: [ + { key: 'variable', value: '' } + ] + }, "simple expansions": { variables: [ { key: 'variable', value: 'value' }, @@ -560,13 +505,42 @@ RSpec.describe Gitlab::Ci::Variables::Collection do { key: 'variable2', value: '$variable3' }, { key: 'variable3', value: 'key$variable$variable2' } ] + }, + "file variables with expand_file_refs: true": { + variables: [ + { key: 'file_var', value: 'secret content', file: true }, + { key: 'variable1', value: 'var one' }, + { key: 'variable2', value: 'var two $variable1 $file_var' } + ], + result: [ + { key: 'file_var', value: 'secret content' }, + { key: 'variable1', value: 'var one' }, + { key: 'variable2', value: 'var two var one secret content' } + ] + }, + "file variables with expand_file_refs: false": { + variables: [ + { key: 'file_var', value: 'secret content', file: true }, + { key: 'variable1', value: 'var one' }, + { key: 'variable2', value: 'var two $variable1 $file_var' } + ], + expand_file_refs: false, + result: [ + { key: 'file_var', value: 'secret content' }, + { key: 'variable1', value: 'var one' }, + { key: 'variable2', value: 'var two var one $file_var' } + ] } } end with_them do let(:collection) { Gitlab::Ci::Variables::Collection.new(variables) } - let(:options) { { keep_undefined: keep_undefined, expand_raw_refs: expand_raw_refs }.compact } + let(:options) do + { keep_undefined: keep_undefined, + expand_raw_refs: expand_raw_refs, + expand_file_refs: expand_file_refs }.compact + end subject(:expanded_result) { collection.sort_and_expand_all(**options) } @@ -585,43 +559,21 @@ RSpec.describe Gitlab::Ci::Variables::Collection do end end end + end - context 'with the file_variable_is_referenced_in_another_variable logging' do - let(:collection) do - Gitlab::Ci::Variables::Collection.new - .append(key: 'VAR1', value: 'test-1') - .append(key: 'VAR2', value: '$VAR1') - .append(key: 'VAR3', value: '$VAR1', raw: true) - .append(key: 'FILEVAR4', value: 'file-test-4', file: true) - .append(key: 'VAR5', value: '$FILEVAR4') - .append(key: 'VAR6', value: '$FILEVAR4', raw: true) - end - - subject(:sort_and_expand_all) { collection.sort_and_expand_all(project: project) } - - context 'when a project is not passed' do - let(:project) {} - - it 'does not log anything' do - expect(Gitlab::AppJsonLogger).not_to receive(:info) - - sort_and_expand_all - end - end + describe '#to_s' do + let(:variables) do + [ + { key: 'VAR', value: 'value', public: true }, + { key: 'VAR2', value: 'value2', public: false } + ] + end - context 'when a project is passed' do - let(:project) { create(:project) } + let(:errors) { 'circular variable reference detected' } + let(:collection) { Gitlab::Ci::Variables::Collection.new(variables, errors) } - it 'logs file_variable_is_referenced_in_another_variable once for VAR5' do - expect(Gitlab::AppJsonLogger).to receive(:info).with( - event: 'file_variable_is_referenced_in_another_variable', - project_id: project.id, - variable: 'FILEVAR4' - ).once + subject(:result) { collection.to_s } - sort_and_expand_all - end - end - end + it { is_expected.to eq("[\"VAR\", \"VAR2\"], @errors='circular variable reference detected'") } end end diff --git a/spec/lib/gitlab/ci/yaml_processor_spec.rb b/spec/lib/gitlab/ci/yaml_processor_spec.rb index ae98d2e0cad..b9f65ff749d 100644 --- a/spec/lib/gitlab/ci/yaml_processor_spec.rb +++ b/spec/lib/gitlab/ci/yaml_processor_spec.rb @@ -4,8 +4,9 @@ require 'spec_helper' module Gitlab module Ci - RSpec.describe YamlProcessor do + RSpec.describe YamlProcessor, feature_category: :pipeline_authoring do include StubRequests + include RepoHelpers subject(:processor) { described_class.new(config, user: nil).execute } @@ -1302,32 +1303,6 @@ module Gitlab 'VAR3' => { value: 'value3', raw: true } ) end - - context 'when the FF ci_raw_variables_in_yaml_config is disabled' do - before do - stub_feature_flags(ci_raw_variables_in_yaml_config: false) - end - - it 'returns variables without description and raw' do - expect(job_variables).to contain_exactly( - { key: 'VAR4', value: 'value4' }, - { key: 'VAR5', value: 'value5' }, - { key: 'VAR6', value: 'value6' } - ) - - expect(execute.root_variables).to contain_exactly( - { key: 'VAR1', value: 'value1' }, - { key: 'VAR2', value: 'value2' }, - { key: 'VAR3', value: 'value3' } - ) - - expect(execute.root_variables_with_prefill_data).to eq( - 'VAR1' => { value: 'value1' }, - 'VAR2' => { value: 'value2', description: 'description2' }, - 'VAR3' => { value: 'value3' } - ) - end - end end end @@ -1505,9 +1480,19 @@ module Gitlab let(:opts) { { project: project, sha: project.commit.sha } } context "when the included internal file is present" do - before do - expect(project.repository).to receive(:blob_data_at) - .and_return(YAML.dump({ job1: { script: 'hello' } })) + let(:project_files) do + { + 'local.gitlab-ci.yml' => <<~YAML + job1: + script: hello + YAML + } + end + + around do |example| + create_and_delete_files(project, project_files) do + example.run + end end it { is_expected.to be_valid } @@ -1699,7 +1684,8 @@ module Gitlab untracked: true, key: 'key', policy: 'pull-push', - when: 'on_success' + when: 'on_success', + unprotect: false ]) end @@ -1723,7 +1709,8 @@ module Gitlab untracked: true, key: { files: ['file'] }, policy: 'pull-push', - when: 'on_success' + when: 'on_success', + unprotect: false ]) end @@ -1749,14 +1736,16 @@ module Gitlab untracked: true, key: 'keya', policy: 'pull-push', - when: 'on_success' + when: 'on_success', + unprotect: false }, { paths: ['logs/', 'binaries/'], untracked: true, key: 'key', policy: 'pull-push', - when: 'on_success' + when: 'on_success', + unprotect: false } ] ) @@ -1783,7 +1772,8 @@ module Gitlab untracked: true, key: { files: ['file'] }, policy: 'pull-push', - when: 'on_success' + when: 'on_success', + unprotect: false ]) end @@ -1808,7 +1798,8 @@ module Gitlab untracked: true, key: { files: ['file'], prefix: 'prefix' }, policy: 'pull-push', - when: 'on_success' + when: 'on_success', + unprotect: false ]) end @@ -1831,7 +1822,8 @@ module Gitlab untracked: false, key: 'local', policy: 'pull-push', - when: 'on_success' + when: 'on_success', + unprotect: false ]) end end diff --git a/spec/lib/gitlab/config/entry/validators_spec.rb b/spec/lib/gitlab/config/entry/validators_spec.rb index 0458bcd6354..54a2adbefd2 100644 --- a/spec/lib/gitlab/config/entry/validators_spec.rb +++ b/spec/lib/gitlab/config/entry/validators_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Config::Entry::Validators do +RSpec.describe Gitlab::Config::Entry::Validators, feature_category: :pipeline_authoring do let(:klass) do Class.new do include ActiveModel::Validations @@ -40,4 +40,66 @@ RSpec.describe Gitlab::Config::Entry::Validators do end end end + + describe described_class::DisallowedKeysValidator do + using RSpec::Parameterized::TableSyntax + + where(:config, :disallowed_keys, :ignore_nil, :valid_result) do + { foo: '1' } | 'foo' | false | false + { foo: '1', bar: '2', baz: '3' } | 'foo, bar' | false | false + { baz: '1', qux: '2' } | '' | false | true + { foo: nil } | 'foo' | false | false + { foo: nil, bar: '2', baz: '3' } | 'foo, bar' | false | false + { foo: nil, bar: nil, baz: '3' } | 'foo, bar' | false | false + { baz: nil, qux: nil } | '' | false | true + { foo: '1' } | 'foo' | true | false + { foo: '1', bar: '2', baz: '3' } | 'foo, bar' | true | false + { baz: '1', qux: '2' } | '' | true | true + { foo: nil } | '' | true | true + { foo: nil, bar: '2', baz: '3' } | 'bar' | true | false + { foo: nil, bar: nil, baz: '3' } | '' | true | true + { baz: nil, qux: nil } | '' | true | true + end + + with_them do + before do + klass.instance_variable_set(:@ignore_nil, ignore_nil) + + klass.instance_eval do + validates :config, disallowed_keys: { + in: %i[foo bar], + ignore_nil: @ignore_nil # rubocop:disable RSpec/InstanceVariable + } + end + + allow(instance).to receive(:config).and_return(config) + end + + it 'validates the instance' do + expect(instance.valid?).to be(valid_result) + + unless valid_result + expect(instance.errors.messages_for(:config)).to include "contains disallowed keys: #{disallowed_keys}" + end + end + end + + context 'when custom message is provided' do + before do + klass.instance_eval do + validates :config, disallowed_keys: { + in: %i[foo bar], + message: 'custom message' + } + end + + allow(instance).to receive(:config).and_return({ foo: '1' }) + end + + it 'returns the custom message when invalid' do + expect(instance).not_to be_valid + expect(instance.errors.messages_for(:config)).to include "custom message: foo" + end + end + end end diff --git a/spec/lib/gitlab/counters/buffered_counter_spec.rb b/spec/lib/gitlab/counters/buffered_counter_spec.rb index a1fd97768ea..2d5209161d9 100644 --- a/spec/lib/gitlab/counters/buffered_counter_spec.rb +++ b/spec/lib/gitlab/counters/buffered_counter_spec.rb @@ -7,7 +7,8 @@ RSpec.describe Gitlab::Counters::BufferedCounter, :clean_gitlab_redis_shared_sta subject(:counter) { described_class.new(counter_record, attribute) } - let(:counter_record) { create(:project_statistics) } + let_it_be(:counter_record) { create(:project_statistics) } + let(:attribute) { :build_artifacts_size } describe '#get' do @@ -25,42 +26,447 @@ RSpec.describe Gitlab::Counters::BufferedCounter, :clean_gitlab_redis_shared_sta end describe '#increment' do - it 'sets a new key by the given value' do - counter.increment(123) + let(:increment) { Gitlab::Counters::Increment.new(amount: 123, ref: 1) } + let(:other_increment) { Gitlab::Counters::Increment.new(amount: 100, ref: 2) } + + context 'when the counter is not undergoing refresh' do + it 'sets a new key by the given value' do + counter.increment(increment) + + expect(counter.get).to eq(increment.amount) + end + + it 'increments an existing key by the given value' do + counter.increment(other_increment) + counter.increment(increment) + + expect(counter.get).to eq(other_increment.amount + increment.amount) + end + + it 'returns the value of the key after the increment' do + counter.increment(increment) + result = counter.increment(other_increment) + + expect(result).to eq(increment.amount + other_increment.amount) + end + + it 'schedules a worker to commit the counter key into database' do + expect(FlushCounterIncrementsWorker).to receive(:perform_in) + .with(described_class::WORKER_DELAY, counter_record.class.to_s, counter_record.id, attribute) + + counter.increment(increment) + end + end + + context 'when the counter is undergoing refresh' do + let(:increment_1) { Gitlab::Counters::Increment.new(amount: 123, ref: 1) } + let(:decrement_1) { Gitlab::Counters::Increment.new(amount: -increment_1.amount, ref: increment_1.ref) } + + let(:increment_2) { Gitlab::Counters::Increment.new(amount: 100, ref: 2) } + let(:decrement_2) { Gitlab::Counters::Increment.new(amount: -increment_2.amount, ref: increment_2.ref) } + + before do + counter.initiate_refresh! + end + + it 'does not increment the counter key' do + expect { counter.increment(increment) }.not_to change { counter.get }.from(0) + end + + it 'increments the amount in the refresh key' do + counter.increment(increment) + + expect(redis_get_key(counter.refresh_key).to_i).to eq(increment.amount) + end + + it 'schedules a worker to commit the counter key into database' do + expect(FlushCounterIncrementsWorker).to receive(:perform_in) + .with(described_class::WORKER_DELAY, counter_record.class.to_s, counter_record.id, attribute) + + counter.increment(increment) + end + + shared_examples 'changing the counter refresh key by the given amount' do + it 'changes the refresh counter key by the given value' do + expect { counter.increment(increment) } + .to change { redis_get_key(counter.refresh_key).to_i }.by(increment.amount) + end + + it 'returns the value of the key after the increment' do + expect(counter.increment(increment)).to eq(expected_counter_value) + end + end + + shared_examples 'not changing the counter refresh key' do + it 'does not change the counter' do + expect { counter.increment(increment) }.not_to change { redis_get_key(counter.refresh_key).to_i } + end + + it 'returns the unchanged value of the key' do + expect(counter.increment(increment)).to eq(expected_counter_value) + end + end + + context 'when it is an increment (positive amount)' do + let(:increment) { increment_1 } + + context 'when it is the first increment on the ref' do + let(:expected_counter_value) { increment.amount } + + it_behaves_like 'changing the counter refresh key by the given amount' + end + + context 'when it follows an existing increment on the same ref' do + before do + counter.increment(increment) + end + + let(:expected_counter_value) { increment.amount } + + it_behaves_like 'not changing the counter refresh key' + end + + context 'when it follows an existing decrement on the same ref' do + before do + counter.increment(decrement_1) + end + + let(:expected_counter_value) { 0 } + + it_behaves_like 'not changing the counter refresh key' + end + + context 'when there has been an existing increment on another ref' do + before do + counter.increment(increment_2) + end + + let(:expected_counter_value) { increment.amount + increment_2.amount } + + it_behaves_like 'changing the counter refresh key by the given amount' + end + + context 'when there has been an existing decrement on another ref' do + before do + counter.increment(decrement_2) + end + + let(:expected_counter_value) { increment.amount } + + it_behaves_like 'changing the counter refresh key by the given amount' + end + end - expect(counter.get).to eq(123) + context 'when it is a decrement (negative amount)' do + let(:increment) { decrement_1 } + + context 'when it is the first decrement on the same ref' do + let(:expected_counter_value) { 0 } + + it_behaves_like 'not changing the counter refresh key' + end + + context 'when it follows an existing decrement on the ref' do + before do + counter.increment(decrement_1) + end + + let(:expected_counter_value) { 0 } + + it_behaves_like 'not changing the counter refresh key' + end + + context 'when it follows an existing increment on the ref' do + before do + counter.increment(increment_1) + end + + let(:expected_counter_value) { 0 } + + it_behaves_like 'changing the counter refresh key by the given amount' + end + + context 'when there has been an existing increment on another ref' do + before do + counter.increment(increment_2) + end + + let(:expected_counter_value) { increment_2.amount } + + it_behaves_like 'not changing the counter refresh key' + end + + context 'when there has been an existing decrement on another ref' do + before do + counter.increment(decrement_2) + end + + let(:expected_counter_value) { 0 } + + it_behaves_like 'not changing the counter refresh key' + end + end + + context 'when the amount is 0' do + let(:increment) { Gitlab::Counters::Increment.new(amount: 0, ref: 1) } + + context 'when it is the first increment on the ref' do + let(:expected_counter_value) { 0 } + + it_behaves_like 'not changing the counter refresh key' + end + + context 'when it follows the another increment on the ref' do + let(:expected_counter_value) { 0 } + + before do + counter.increment(increment) + end + + it_behaves_like 'not changing the counter refresh key' + end + end + + context 'when the ref is greater than 67108863 (8MB)' do + let(:increment) { Gitlab::Counters::Increment.new(amount: 123, ref: 67108864) } + + let(:increment_2) { Gitlab::Counters::Increment.new(amount: 123, ref: 267108863) } + let(:decrement_2) { Gitlab::Counters::Increment.new(amount: -increment_2.amount, ref: increment_2.ref) } + + let(:expected_counter_value) { increment.amount } + + it 'deduplicates increments correctly' do + counter.increment(decrement_2) + counter.increment(increment) + counter.increment(increment_2) + + expect(redis_get_key(counter.refresh_key).to_i).to eq(increment.amount) + end + end + end + + context 'when feature flag is disabled' do + before do + stub_feature_flags(project_statistics_bulk_increment: false) + end + + context 'when the counter is not undergoing refresh' do + it 'sets a new key by the given value' do + counter.increment(increment) + + expect(counter.get).to eq(increment.amount) + end + + it 'increments an existing key by the given value' do + counter.increment(other_increment) + counter.increment(increment) + + expect(counter.get).to eq(other_increment.amount + increment.amount) + end + end + + context 'when the counter is undergoing refresh' do + before do + counter.initiate_refresh! + end + + context 'when it is a decrement (negative amount)' do + let(:decrement) { Gitlab::Counters::Increment.new(amount: -123, ref: 3) } + + it 'immediately decrements the counter key to negative' do + counter.increment(decrement) + + expect(counter.get).to eq(decrement.amount) + end + end + end end + end + + describe '#bulk_increment' do + let(:other_increment) { Gitlab::Counters::Increment.new(amount: 1) } + let(:increments) { [Gitlab::Counters::Increment.new(amount: 123), Gitlab::Counters::Increment.new(amount: 456)] } - it 'increments an existing key by the given value' do - counter.increment(100) - counter.increment(123) + context 'when the counter is not undergoing refresh' do + it 'increments the key by the given values' do + counter.bulk_increment(increments) + + expect(counter.get).to eq(increments.sum(&:amount)) + end + + it 'returns the value of the key after the increment' do + counter.increment(other_increment) + + result = counter.bulk_increment(increments) + + expect(result).to eq(other_increment.amount + increments.sum(&:amount)) + end - expect(counter.get).to eq(100 + 123) + it 'schedules a worker to commit the counter into database' do + expect(FlushCounterIncrementsWorker).to receive(:perform_in) + .with(described_class::WORKER_DELAY, counter_record.class.to_s, counter_record.id, attribute) + + counter.bulk_increment(increments) + end end - it 'returns the new value' do - counter.increment(123) + context 'when the counter is undergoing refresh' do + let(:increment_1) { Gitlab::Counters::Increment.new(amount: 123, ref: 1) } + let(:decrement_1) { Gitlab::Counters::Increment.new(amount: -increment_1.amount, ref: increment_1.ref) } + + let(:increment_2) { Gitlab::Counters::Increment.new(amount: 100, ref: 2) } + let(:decrement_2) { Gitlab::Counters::Increment.new(amount: -increment_2.amount, ref: increment_2.ref) } + + let(:increment_3) { Gitlab::Counters::Increment.new(amount: 100, ref: 3) } + let(:decrement_3) { Gitlab::Counters::Increment.new(amount: -increment_3.amount, ref: increment_3.ref) } + + before do + counter.initiate_refresh! + end + + shared_examples 'changing the counter refresh key by the expected amount' do + it 'changes the counter refresh key by the net change' do + expect { counter.bulk_increment(increments) } + .to change { redis_get_key(counter.refresh_key).to_i }.by(expected_change) + end + + it 'returns the value of the key after the increment' do + expect(counter.bulk_increment(increments)).to eq(expected_counter_value) + end + end + + context 'when there are 2 increments on different ref' do + let(:increments) { [increment_1, increment_2] } + let(:expected_change) { increments.sum(&:amount) } + let(:expected_counter_value) { increments.sum(&:amount) } + + it_behaves_like 'changing the counter refresh key by the expected amount' + + context 'when there has been previous decrements' do + before do + counter.increment(decrement_1) + counter.increment(decrement_3) + end + + let(:expected_change) { increment_2.amount } + let(:expected_counter_value) { increment_2.amount } + + it_behaves_like 'changing the counter refresh key by the expected amount' + end + + context 'when one of the increment is repeated' do + before do + counter.increment(increment_2) + end + + let(:expected_change) { increment_1.amount } + let(:expected_counter_value) { increment_2.amount + increment_1.amount } - expect(counter.increment(23)).to eq(146) + it_behaves_like 'changing the counter refresh key by the expected amount' + end + end + + context 'when there are 2 decrements on different ref' do + let(:increments) { [decrement_1, decrement_2] } + let(:expected_change) { 0 } + let(:expected_counter_value) { 0 } + + it_behaves_like 'changing the counter refresh key by the expected amount' + + context 'when there has been previous increments' do + before do + counter.increment(increment_1) + counter.increment(increment_3) + end + + let(:expected_change) { decrement_1.amount } + let(:expected_counter_value) { increment_3.amount } + + it_behaves_like 'changing the counter refresh key by the expected amount' + end + end + + context 'when there is a mixture of increment and decrement on different refs' do + let(:increments) { [increment_1, decrement_2] } + let(:expected_change) { increment_1.amount } + let(:expected_counter_value) { increment_1.amount } + + it_behaves_like 'changing the counter refresh key by the expected amount' + + context 'when the increment ref has been decremented' do + before do + counter.increment(decrement_1) + end + + let(:expected_change) { 0 } + let(:expected_counter_value) { 0 } + + it_behaves_like 'changing the counter refresh key by the expected amount' + end + + context 'when the decrement ref has been incremented' do + before do + counter.increment(increment_2) + end + + let(:expected_change) { increments.sum(&:amount) } + let(:expected_counter_value) { increment_1.amount } + + it_behaves_like 'changing the counter refresh key by the expected amount' + end + end end - it 'schedules a worker to commit the counter into database' do - expect(FlushCounterIncrementsWorker).to receive(:perform_in) - .with(described_class::WORKER_DELAY, counter_record.class.to_s, counter_record.id, attribute) + context 'when feature flag is disabled' do + before do + stub_feature_flags(project_statistics_bulk_increment: false) + end + + context 'when the counter is not undergoing refresh' do + it 'sets a new key by the given value' do + counter.bulk_increment(increments) + + expect(counter.get).to eq(increments.sum(&:amount)) + end + + it 'increments an existing key by the given value' do + counter.increment(other_increment) + + result = counter.bulk_increment(increments) - counter.increment(123) + expect(result).to eq(other_increment.amount + increments.sum(&:amount)) + end + end + + context 'when the counter is undergoing refresh' do + before do + counter.initiate_refresh! + end + + context 'when it is a decrement (negative amount)' do + let(:decrement) { Gitlab::Counters::Increment.new(amount: -123, ref: 3) } + + it 'immediately decrements the counter key to negative' do + counter.bulk_increment([decrement]) + + expect(counter.get).to eq(decrement.amount) + end + end + end end end - describe '#reset!' do + describe '#initiate_refresh!' do + let(:increment) { Gitlab::Counters::Increment.new(amount: 123) } + before do allow(counter_record).to receive(:update!) - counter.increment(123) + counter.increment(increment) end it 'removes the key from Redis' do - counter.reset! + counter.initiate_refresh! Gitlab::Redis::SharedState.with do |redis| expect(redis.exists?(counter.key)).to eq(false) @@ -68,7 +474,7 @@ RSpec.describe Gitlab::Counters::BufferedCounter, :clean_gitlab_redis_shared_sta end it 'resets the counter to 0' do - counter.reset! + counter.initiate_refresh! expect(counter.get).to eq(0) end @@ -76,7 +482,91 @@ RSpec.describe Gitlab::Counters::BufferedCounter, :clean_gitlab_redis_shared_sta it 'resets the record to 0' do expect(counter_record).to receive(:update!).with(attribute => 0) - counter.reset! + counter.initiate_refresh! + end + + it 'sets a refresh indicator with a long expiry' do + counter.initiate_refresh! + + expect(redis_exists_key(counter.refresh_indicator_key)).to eq(true) + expect(redis_key_ttl(counter.refresh_indicator_key)).to eq(described_class::REFRESH_KEYS_TTL) + end + end + + describe '#finalize_refresh' do + before do + counter.initiate_refresh! + end + + context 'with existing amount in refresh key' do + let(:increment) { Gitlab::Counters::Increment.new(amount: 123, ref: 1) } + let(:other_increment) { Gitlab::Counters::Increment.new(amount: 100, ref: 2) } + let(:other_decrement) { Gitlab::Counters::Increment.new(amount: -100, ref: 2) } + + before do + counter.bulk_increment([other_decrement, increment, other_increment]) + end + + it 'moves the deduplicated amount in the refresh key into the counter key' do + expect { counter.finalize_refresh } + .to change { counter.get }.by(increment.amount) + end + + it 'removes the refresh counter key and the refresh indicator' do + expect { counter.finalize_refresh } + .to change { redis_exists_key(counter.refresh_key) }.from(true).to(false) + .and change { redis_exists_key(counter.refresh_indicator_key) }.from(true).to(false) + end + + it 'schedules a worker to clean up the refresh tracking keys' do + expect(Counters::CleanupRefreshWorker).to receive(:perform_async) + .with(counter_record.class.to_s, counter_record.id, attribute) + + counter.finalize_refresh + end + end + + context 'without existing amount in refresh key' do + it 'does not change the counter key' do + expect { counter.finalize_refresh }.not_to change { counter.get } + end + + it 'removes the refresh indicator key' do + expect { counter.finalize_refresh } + .to change { redis_exists_key(counter.refresh_indicator_key) }.from(true).to(false) + end + + it 'schedules a worker to commit the counter key into database' do + expect(FlushCounterIncrementsWorker).to receive(:perform_in) + .with(described_class::WORKER_DELAY, counter_record.class.to_s, counter_record.id, attribute) + + counter.finalize_refresh + end + end + end + + describe '#cleanup_refresh' do + let(:increment) { Gitlab::Counters::Increment.new(amount: 123, ref: 67108864) } + let(:increment_2) { Gitlab::Counters::Increment.new(amount: 123, ref: 267108864) } + let(:decrement_2) { Gitlab::Counters::Increment.new(amount: -increment_2.amount, ref: increment_2.ref) } + let(:increment_3) { Gitlab::Counters::Increment.new(amount: 123, ref: 534217728) } + + before do + stub_const("#{described_class}::CLEANUP_BATCH_SIZE", 2) + stub_const("#{described_class}::CLEANUP_INTERVAL_SECONDS", 0.001) + + counter.initiate_refresh! + counter.increment(decrement_2) + counter.increment(increment) + counter.increment(increment_2) + counter.finalize_refresh + end + + it 'removes all tracking keys' do + Gitlab::Redis::SharedState.with do |redis| + expect { counter.cleanup_refresh } + .to change { redis.scan_each(match: "#{counter.refresh_key}*").to_a.count }.from(4).to(0) + end end end @@ -88,7 +578,7 @@ RSpec.describe Gitlab::Counters::BufferedCounter, :clean_gitlab_redis_shared_sta end context 'when there is an amount to commit' do - let(:increments) { [10, -3] } + let(:increments) { [10, -3].map { |amt| Gitlab::Counters::Increment.new(amount: amt) } } before do increments.each { |i| counter.increment(i) } @@ -96,21 +586,11 @@ RSpec.describe Gitlab::Counters::BufferedCounter, :clean_gitlab_redis_shared_sta it 'commits the increment into the database' do expect { counter.commit_increment! } - .to change { counter_record.reset.read_attribute(attribute) }.by(increments.sum) + .to change { counter_record.reset.read_attribute(attribute) }.by(increments.sum(&:amount)) end it 'removes the increment entry from Redis' do - Gitlab::Redis::SharedState.with do |redis| - key_exists = redis.exists?(counter.key) - expect(key_exists).to be_truthy - end - - counter.commit_increment! - - Gitlab::Redis::SharedState.with do |redis| - key_exists = redis.exists?(counter.key) - expect(key_exists).to be_falsey - end + expect { counter.commit_increment! }.to change { redis_exists_key(counter.key) }.from(true).to(false) end end @@ -171,7 +651,7 @@ RSpec.describe Gitlab::Counters::BufferedCounter, :clean_gitlab_redis_shared_sta context 'when there are increments to flush' do before do - counter.increment(10) + counter.increment(Gitlab::Counters::Increment.new(amount: 10)) end it 'executes the callbacks' do @@ -223,11 +703,27 @@ RSpec.describe Gitlab::Counters::BufferedCounter, :clean_gitlab_redis_shared_sta it 'drops the increment key and creates the flushed key if it does not exist' do counter.amount_to_be_flushed - Gitlab::Redis::SharedState.with do |redis| - expect(redis.exists?(increment_key)).to eq(false) - expect(redis.exists?(flushed_key)).to eq(flushed_key_present) - end + expect(redis_exists_key(increment_key)).to eq(false) + expect(redis_exists_key(flushed_key)).to eq(flushed_key_present) end end end + + def redis_get_key(key) + Gitlab::Redis::SharedState.with do |redis| + redis.get(key) + end + end + + def redis_exists_key(key) + Gitlab::Redis::SharedState.with do |redis| + redis.exists?(key) + end + end + + def redis_key_ttl(key) + Gitlab::Redis::SharedState.with do |redis| + redis.ttl(key) + end + end end diff --git a/spec/lib/gitlab/counters/legacy_counter_spec.rb b/spec/lib/gitlab/counters/legacy_counter_spec.rb index e66b1ce08c4..9b0ffafff67 100644 --- a/spec/lib/gitlab/counters/legacy_counter_spec.rb +++ b/spec/lib/gitlab/counters/legacy_counter_spec.rb @@ -5,37 +5,50 @@ require 'spec_helper' RSpec.describe Gitlab::Counters::LegacyCounter do subject(:counter) { described_class.new(counter_record, attribute) } - let(:counter_record) { create(:project_statistics) } + let_it_be(:counter_record, reload: true) { create(:project_statistics) } + let(:attribute) { :snippets_size } - let(:amount) { 123 } + + let(:increment) { Gitlab::Counters::Increment.new(amount: 123) } + let(:other_increment) { Gitlab::Counters::Increment.new(amount: 100) } describe '#increment' do it 'increments the attribute in the counter record' do - expect { counter.increment(amount) }.to change { counter_record.reload.method(attribute).call }.by(amount) + expect { counter.increment(increment) } + .to change { counter_record.reload.method(attribute).call }.by(increment.amount) end it 'returns the value after the increment' do - counter.increment(100) + counter.increment(other_increment) - expect(counter.increment(amount)).to eq(100 + amount) + expect(counter.increment(increment)).to eq(other_increment.amount + increment.amount) end it 'executes after counter_record after commit callback' do expect(counter_record).to receive(:execute_after_commit_callbacks).and_call_original - counter.increment(amount) + counter.increment(increment) end end - describe '#reset!' do - before do - allow(counter_record).to receive(:update!) + describe '#bulk_increment' do + let(:increments) { [Gitlab::Counters::Increment.new(amount: 123), Gitlab::Counters::Increment.new(amount: 456)] } + + it 'increments the attribute in the counter record' do + expect { counter.bulk_increment(increments) } + .to change { counter_record.reload.method(attribute).call }.by(increments.sum(&:amount)) + end + + it 'returns the value after the increment' do + counter.increment(other_increment) + + expect(counter.bulk_increment(increments)).to eq(other_increment.amount + increments.sum(&:amount)) end - it 'resets the record to 0' do - expect(counter_record).to receive(:update!).with(attribute => 0) + it 'executes after counter_record after commit callback' do + expect(counter_record).to receive(:execute_after_commit_callbacks).and_call_original - counter.reset! + counter.bulk_increment(increments) end end end diff --git a/spec/lib/gitlab/data_builder/build_spec.rb b/spec/lib/gitlab/data_builder/build_spec.rb index 544b210651b..92fef93bddb 100644 --- a/spec/lib/gitlab/data_builder/build_spec.rb +++ b/spec/lib/gitlab/data_builder/build_spec.rb @@ -2,10 +2,11 @@ require 'spec_helper' -RSpec.describe Gitlab::DataBuilder::Build do +RSpec.describe Gitlab::DataBuilder::Build, feature_category: :integrations do let_it_be(:runner) { create(:ci_runner, :instance, :tagged_only) } let_it_be(:user) { create(:user, :public_email) } - let_it_be(:ci_build) { create(:ci_build, :running, runner: runner, user: user) } + let_it_be(:pipeline) { create(:ci_pipeline, name: 'Build pipeline') } + let_it_be(:ci_build) { create(:ci_build, :running, pipeline: pipeline, runner: runner, user: user) } describe '.build' do around do |example| @@ -33,6 +34,7 @@ RSpec.describe Gitlab::DataBuilder::Build do it { expect(data[:project_name]).to eq(ci_build.project.full_name) } it { expect(data[:pipeline_id]).to eq(ci_build.pipeline.id) } it { expect(data[:retries_count]).to eq(ci_build.retries_count) } + it { expect(data[:commit][:name]).to eq(pipeline.name) } it { expect(data[:user]).to eq( @@ -61,10 +63,10 @@ RSpec.describe Gitlab::DataBuilder::Build do described_class.build(b) # Don't use ci_build variable here since it has all associations loaded into memory end - expect(control.count).to eq(13) + expect(control.count).to eq(14) end - context 'when feature flag is disabled' do + context 'when job_webhook_retries_count feature flag is disabled' do before do stub_feature_flags(job_webhook_retries_count: false) end @@ -79,7 +81,7 @@ RSpec.describe Gitlab::DataBuilder::Build do described_class.build(b) # Don't use ci_build variable here since it has all associations loaded into memory end - expect(control.count).to eq(12) + expect(control.count).to eq(13) end end diff --git a/spec/lib/gitlab/database/async_indexes/index_creator_spec.rb b/spec/lib/gitlab/database/async_indexes/index_creator_spec.rb index 7ad3eb395a9..207aedd1a38 100644 --- a/spec/lib/gitlab/database/async_indexes/index_creator_spec.rb +++ b/spec/lib/gitlab/database/async_indexes/index_creator_spec.rb @@ -16,7 +16,7 @@ RSpec.describe Gitlab::Database::AsyncIndexes::IndexCreator do let(:connection) { model.connection } let!(:lease) { stub_exclusive_lease(lease_key, :uuid, timeout: lease_timeout) } - let(:lease_key) { "gitlab/database/async_indexes/index_creator/#{Gitlab::Database::PRIMARY_DATABASE_NAME}" } + let(:lease_key) { "gitlab/database/indexing/actions/#{Gitlab::Database::PRIMARY_DATABASE_NAME}" } let(:lease_timeout) { described_class::TIMEOUT_PER_ACTION } around do |example| @@ -39,7 +39,7 @@ RSpec.describe Gitlab::Database::AsyncIndexes::IndexCreator do it 'creates the index while controlling statement timeout' do allow(connection).to receive(:execute).and_call_original - expect(connection).to receive(:execute).with("SET statement_timeout TO '32400s'").ordered.and_call_original + expect(connection).to receive(:execute).with("SET statement_timeout TO '72000s'").ordered.and_call_original expect(connection).to receive(:execute).with(async_index.definition).ordered.and_call_original expect(connection).to receive(:execute).with("RESET statement_timeout").ordered.and_call_original diff --git a/spec/lib/gitlab/database/async_indexes/index_destructor_spec.rb b/spec/lib/gitlab/database/async_indexes/index_destructor_spec.rb index adb0f45706d..11039ad4f7e 100644 --- a/spec/lib/gitlab/database/async_indexes/index_destructor_spec.rb +++ b/spec/lib/gitlab/database/async_indexes/index_destructor_spec.rb @@ -16,7 +16,7 @@ RSpec.describe Gitlab::Database::AsyncIndexes::IndexDestructor do let(:connection) { model.connection } let!(:lease) { stub_exclusive_lease(lease_key, :uuid, timeout: lease_timeout) } - let(:lease_key) { "gitlab/database/async_indexes/index_destructor/#{Gitlab::Database::PRIMARY_DATABASE_NAME}" } + let(:lease_key) { "gitlab/database/indexing/actions/#{Gitlab::Database::PRIMARY_DATABASE_NAME}" } let(:lease_timeout) { described_class::TIMEOUT_PER_ACTION } before do diff --git a/spec/lib/gitlab/database/background_migration/batched_migration_wrapper_spec.rb b/spec/lib/gitlab/database/background_migration/batched_migration_wrapper_spec.rb index 983f482d464..f3a292abbae 100644 --- a/spec/lib/gitlab/database/background_migration/batched_migration_wrapper_spec.rb +++ b/spec/lib/gitlab/database/background_migration/batched_migration_wrapper_spec.rb @@ -152,6 +152,7 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper, ' it 'runs the job with the correct arguments' do expect(job_class).to receive(:new).with(no_args).and_return(job_instance) + expect(Gitlab::ApplicationContext).to receive(:push).with(feature_category: :database) expect(job_instance).to receive(:perform).with(1, 10, 'events', 'id', 1, pause_ms, 'id', 'other_id') perform diff --git a/spec/lib/gitlab/database/background_migration/health_status/indicators/autovacuum_active_on_table_spec.rb b/spec/lib/gitlab/database/background_migration/health_status/indicators/autovacuum_active_on_table_spec.rb index db4383a79d4..1c0f5a0c420 100644 --- a/spec/lib/gitlab/database/background_migration/health_status/indicators/autovacuum_active_on_table_spec.rb +++ b/spec/lib/gitlab/database/background_migration/health_status/indicators/autovacuum_active_on_table_spec.rb @@ -2,7 +2,8 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::BackgroundMigration::HealthStatus::Indicators::AutovacuumActiveOnTable do +RSpec.describe Gitlab::Database::BackgroundMigration::HealthStatus::Indicators::AutovacuumActiveOnTable, + feature_category: :database do include Database::DatabaseHelpers let(:connection) { Gitlab::Database.database_base_models[:main].connection } @@ -17,7 +18,7 @@ RSpec.describe Gitlab::Database::BackgroundMigration::HealthStatus::Indicators:: subject { described_class.new(context).evaluate } before do - swapout_view_for_table(:postgres_autovacuum_activity) + swapout_view_for_table(:postgres_autovacuum_activity, connection: connection) end let(:tables) { [table] } diff --git a/spec/lib/gitlab/database/consistency_checker_spec.rb b/spec/lib/gitlab/database/consistency_checker_spec.rb index 2ff79d20786..c0f0c349ddd 100644 --- a/spec/lib/gitlab/database/consistency_checker_spec.rb +++ b/spec/lib/gitlab/database/consistency_checker_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::ConsistencyChecker do +RSpec.describe Gitlab::Database::ConsistencyChecker, feature_category: :pods do let(:batch_size) { 10 } let(:max_batches) { 4 } let(:max_runtime) { described_class::MAX_RUNTIME } diff --git a/spec/lib/gitlab/database/gitlab_schema_spec.rb b/spec/lib/gitlab/database/gitlab_schema_spec.rb index 4b37cbda047..28a087d5401 100644 --- a/spec/lib/gitlab/database/gitlab_schema_spec.rb +++ b/spec/lib/gitlab/database/gitlab_schema_spec.rb @@ -8,13 +8,40 @@ RSpec.shared_examples 'validate path globs' do |path_globs| end end +RSpec.shared_examples 'validate schema data' do |tables_and_views| + it 'all tables and views have assigned a known gitlab_schema' do + expect(tables_and_views).to all( + match([be_a(String), be_in(Gitlab::Database.schemas_to_base_models.keys.map(&:to_sym))]) + ) + end +end + RSpec.describe Gitlab::Database::GitlabSchema do - describe '.views_and_tables_to_schema' do - it 'all tables and views have assigned a known gitlab_schema' do - expect(described_class.views_and_tables_to_schema).to all( - match([be_a(String), be_in(Gitlab::Database.schemas_to_base_models.keys.map(&:to_sym))]) - ) + shared_examples 'maps table name to table schema' do + using RSpec::Parameterized::TableSyntax + + where(:name, :classification) do + 'ci_builds' | :gitlab_ci + 'my_schema.ci_builds' | :gitlab_ci + 'information_schema.columns' | :gitlab_internal + 'audit_events_part_5fc467ac26' | :gitlab_main + '_test_gitlab_main_table' | :gitlab_main + '_test_gitlab_ci_table' | :gitlab_ci + '_test_my_table' | :gitlab_shared + 'pg_attribute' | :gitlab_internal + end + + with_them do + it { is_expected.to eq(classification) } end + end + + describe '.deleted_views_and_tables_to_schema' do + include_examples 'validate schema data', described_class.deleted_views_and_tables_to_schema + end + + describe '.views_and_tables_to_schema' do + include_examples 'validate schema data', described_class.views_and_tables_to_schema # This being run across different databases indirectly also tests # a general consistency of structure across databases @@ -55,6 +82,14 @@ RSpec.describe Gitlab::Database::GitlabSchema do include_examples 'validate path globs', described_class.view_path_globs end + describe '.deleted_tables_path_globs' do + include_examples 'validate path globs', described_class.deleted_tables_path_globs + end + + describe '.deleted_views_path_globs' do + include_examples 'validate path globs', described_class.deleted_views_path_globs + end + describe '.tables_to_schema' do let(:database_models) { Gitlab::Database.database_base_models.except(:geo) } let(:views) { database_models.flat_map { |_, m| m.connection.views }.sort.uniq } @@ -81,25 +116,85 @@ RSpec.describe Gitlab::Database::GitlabSchema do end end + describe '.table_schemas' do + let(:tables) { %w[users projects ci_builds] } + + subject { described_class.table_schemas(tables) } + + it 'returns the matched schemas' do + expect(subject).to match_array %i[gitlab_main gitlab_ci].to_set + end + + context 'when one of the tables does not have a matching table schema' do + let(:tables) { %w[users projects unknown ci_builds] } + + context 'and undefined parameter is false' do + subject { described_class.table_schemas(tables, undefined: false) } + + it 'includes a nil value' do + is_expected.to match_array [:gitlab_main, nil, :gitlab_ci].to_set + end + end + + context 'and undefined parameter is true' do + subject { described_class.table_schemas(tables, undefined: true) } + + it 'includes "undefined_<table_name>"' do + is_expected.to match_array [:gitlab_main, :undefined_unknown, :gitlab_ci].to_set + end + end + + context 'and undefined parameter is not specified' do + it 'includes a nil value' do + is_expected.to match_array [:gitlab_main, :undefined_unknown, :gitlab_ci].to_set + end + end + end + end + describe '.table_schema' do - using RSpec::Parameterized::TableSyntax + subject { described_class.table_schema(name) } - where(:name, :classification) do - 'ci_builds' | :gitlab_ci - 'my_schema.ci_builds' | :gitlab_ci - 'information_schema.columns' | :gitlab_internal - 'audit_events_part_5fc467ac26' | :gitlab_main - '_test_gitlab_main_table' | :gitlab_main - '_test_gitlab_ci_table' | :gitlab_ci - '_test_my_table' | :gitlab_shared - 'pg_attribute' | :gitlab_internal - 'my_other_table' | :undefined_my_other_table + it_behaves_like 'maps table name to table schema' + + context 'when mapping fails' do + let(:name) { 'unknown_table' } + + context "and parameter 'undefined' is set to true" do + subject { described_class.table_schema(name, undefined: true) } + + it { is_expected.to eq(:undefined_unknown_table) } + end + + context "and parameter 'undefined' is set to false" do + subject { described_class.table_schema(name, undefined: false) } + + it { is_expected.to be_nil } + end + + context "and parameter 'undefined' is not set" do + subject { described_class.table_schema(name) } + + it { is_expected.to eq(:undefined_unknown_table) } + end end + end - with_them do - subject { described_class.table_schema(name) } + describe '.table_schema!' do + subject { described_class.table_schema!(name) } - it { is_expected.to eq(classification) } + it_behaves_like 'maps table name to table schema' + + context 'when mapping fails' do + let(:name) { 'non_existing_table' } + + it "raises error" do + expect { subject }.to raise_error( + Gitlab::Database::GitlabSchema::UnknownSchemaError, + "Could not find gitlab schema for table #{name}: " \ + "Any new tables must be added to the database dictionary" + ) + end end end end diff --git a/spec/lib/gitlab/database/indexing_exclusive_lease_guard_spec.rb b/spec/lib/gitlab/database/indexing_exclusive_lease_guard_spec.rb new file mode 100644 index 00000000000..ddc9cdee92f --- /dev/null +++ b/spec/lib/gitlab/database/indexing_exclusive_lease_guard_spec.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::IndexingExclusiveLeaseGuard, feature_category: :database do + let(:helper_class) do + Class.new do + include Gitlab::Database::IndexingExclusiveLeaseGuard + + attr_reader :connection + + def initialize(connection) + @connection = connection + end + end + end + + describe '#lease_key' do + let(:helper) { helper_class.new(connection) } + let(:lease_key) { "gitlab/database/indexing/actions/#{database_name}" } + + context 'with CI database connection' do + let(:connection) { Ci::ApplicationRecord.connection } + let(:database_name) { Gitlab::Database::CI_DATABASE_NAME } + + before do + skip_if_multiple_databases_not_setup + end + + it { expect(helper.lease_key).to eq(lease_key) } + end + + context 'with MAIN database connection' do + let(:connection) { ApplicationRecord.connection } + let(:database_name) { Gitlab::Database::MAIN_DATABASE_NAME } + + it { expect(helper.lease_key).to eq(lease_key) } + end + end +end diff --git a/spec/lib/gitlab/database/load_balancing/resolver_spec.rb b/spec/lib/gitlab/database/load_balancing/resolver_spec.rb index 0051cf50255..4af36693383 100644 --- a/spec/lib/gitlab/database/load_balancing/resolver_spec.rb +++ b/spec/lib/gitlab/database/load_balancing/resolver_spec.rb @@ -2,15 +2,16 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::LoadBalancing::Resolver do +RSpec.describe Gitlab::Database::LoadBalancing::Resolver, :freeze_time, feature_category: :database do describe '#resolve' do let(:ip_addr) { IPAddr.new('127.0.0.2') } context 'when nameserver is an IP' do it 'returns an IPAddr object' do service = described_class.new('127.0.0.2') + response = service.resolve - expect(service.resolve).to eq(ip_addr) + expect(response.address).to eq(ip_addr) end end @@ -22,12 +23,14 @@ RSpec.describe Gitlab::Database::LoadBalancing::Resolver do allow(instance).to receive(:getaddress).with('localhost').and_return('127.0.0.2') end - expect(subject).to eq(ip_addr) + expect(subject.address).to eq(ip_addr) end context 'when nameserver is not in the hosts file' do + let(:raw_ttl) { 10 } + it 'looks the nameserver up in DNS' do - resource = double(:resource, address: ip_addr) + resource = double(:resource, address: ip_addr, ttl: raw_ttl) packet = double(:packet, answer: [resource]) allow_next_instance_of(Resolv::Hosts) do |instance| @@ -38,7 +41,8 @@ RSpec.describe Gitlab::Database::LoadBalancing::Resolver do .with('localhost', Net::DNS::A) .and_return(packet) - expect(subject).to eq(ip_addr) + expect(subject.address).to eq(ip_addr) + expect(subject.ttl).to eq(raw_ttl.seconds.from_now) end context 'when nameserver is not in DNS' do diff --git a/spec/lib/gitlab/database/load_balancing/service_discovery_spec.rb b/spec/lib/gitlab/database/load_balancing/service_discovery_spec.rb index 984d60e9962..bfd9c644ffa 100644 --- a/spec/lib/gitlab/database/load_balancing/service_discovery_spec.rb +++ b/spec/lib/gitlab/database/load_balancing/service_discovery_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::LoadBalancing::ServiceDiscovery do +RSpec.describe Gitlab::Database::LoadBalancing::ServiceDiscovery, feature_category: :database do let(:load_balancer) do configuration = Gitlab::Database::LoadBalancing::Configuration.new(ActiveRecord::Base) configuration.service_discovery[:record] = 'localhost' @@ -23,6 +23,8 @@ RSpec.describe Gitlab::Database::LoadBalancing::ServiceDiscovery do resource = double(:resource, address: IPAddr.new('127.0.0.1')) packet = double(:packet, answer: [resource]) + service.instance_variable_set(:@nameserver_ttl, Gitlab::Database::LoadBalancing::Resolver::FAR_FUTURE_TTL) + allow(Net::DNS::Resolver).to receive(:start) .with('localhost', Net::DNS::A) .and_return(packet) @@ -362,4 +364,52 @@ RSpec.describe Gitlab::Database::LoadBalancing::ServiceDiscovery do expect(service.addresses_from_load_balancer).to eq(addresses) end end + + describe '#resolver', :freeze_time do + context 'without predefined resolver' do + it 'fetches a new resolver and assigns it to the instance variable' do + expect(service.instance_variable_get(:@resolver)).not_to be_present + + service_resolver = service.resolver + + expect(service.instance_variable_get(:@resolver)).to be_present + expect(service_resolver).to be_present + end + end + + context 'with predefined resolver' do + let(:resolver) do + Net::DNS::Resolver.new( + nameservers: 'localhost', + port: 8600 + ) + end + + before do + service.instance_variable_set(:@resolver, resolver) + end + + context "when nameserver's TTL is in the future" do + it 'returns the existing resolver' do + expect(service.resolver).to eq(resolver) + end + end + + context "when nameserver's TTL is in the past" do + before do + service.instance_variable_set( + :@nameserver_ttl, + 1.minute.ago + ) + end + + it 'fetches new resolver' do + service_resolver = service.resolver + + expect(service_resolver).to be_present + expect(service_resolver).not_to eq(resolver) + end + end + end + end end diff --git a/spec/lib/gitlab/database/lock_writes_manager_spec.rb b/spec/lib/gitlab/database/lock_writes_manager_spec.rb index 242b2040eaa..c06c463d918 100644 --- a/spec/lib/gitlab/database/lock_writes_manager_spec.rb +++ b/spec/lib/gitlab/database/lock_writes_manager_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::LockWritesManager do +RSpec.describe Gitlab::Database::LockWritesManager, :delete, feature_category: :pods do let(:connection) { ApplicationRecord.connection } let(:test_table) { '_test_table' } let(:logger) { instance_double(Logger) } @@ -13,12 +13,14 @@ RSpec.describe Gitlab::Database::LockWritesManager do table_name: test_table, connection: connection, database_name: 'main', + with_retries: true, logger: logger, dry_run: dry_run ) end before do + allow(connection).to receive(:execute).and_call_original allow(logger).to receive(:info) connection.execute(<<~SQL) @@ -29,20 +31,24 @@ RSpec.describe Gitlab::Database::LockWritesManager do SQL end + after do + ApplicationRecord.connection.execute("DROP TABLE IF EXISTS #{test_table}") + end + describe "#table_locked_for_writes?" do it 'returns false for a table that is not locked for writes' do - expect(subject.table_locked_for_writes?(test_table)).to eq(false) + expect(subject.table_locked_for_writes?).to eq(false) end it 'returns true for a table that is locked for writes' do - expect { subject.lock_writes }.to change { subject.table_locked_for_writes?(test_table) }.from(false).to(true) + expect { subject.lock_writes }.to change { subject.table_locked_for_writes? }.from(false).to(true) end context 'for detached partition tables in another schema' do let(:test_table) { 'gitlab_partitions_dynamic._test_table_20220101' } it 'returns true for a table that is locked for writes' do - expect { subject.lock_writes }.to change { subject.table_locked_for_writes?(test_table) }.from(false).to(true) + expect { subject.lock_writes }.to change { subject.table_locked_for_writes? }.from(false).to(true) end end end @@ -83,21 +89,19 @@ RSpec.describe Gitlab::Database::LockWritesManager do it 'retries again if it receives a statement_timeout a few number of times' do error_message = "PG::QueryCanceled: ERROR: canceling statement due to statement timeout" call_count = 0 - allow(connection).to receive(:execute) do |statement| - if statement.include?("CREATE TRIGGER") - call_count += 1 - raise(ActiveRecord::QueryCanceled, error_message) if call_count.even? - end + expect(connection).to receive(:execute).twice.with(/^CREATE TRIGGER gitlab_schema_write_trigger_for_/) do + call_count += 1 + raise(ActiveRecord::QueryCanceled, error_message) if call_count.odd? end subject.lock_writes + + expect(call_count).to eq(2) # The first call fails, the 2nd call succeeds end it 'raises the exception if it happened many times' do error_message = "PG::QueryCanceled: ERROR: canceling statement due to statement timeout" - allow(connection).to receive(:execute) do |statement| - if statement.include?("CREATE TRIGGER") - raise(ActiveRecord::QueryCanceled, error_message) - end + allow(connection).to receive(:execute).with(/^CREATE TRIGGER gitlab_schema_write_trigger_for_/) do + raise(ActiveRecord::QueryCanceled, error_message) end expect do @@ -152,6 +156,7 @@ RSpec.describe Gitlab::Database::LockWritesManager do table_name: test_table, connection: connection, database_name: 'main', + with_retries: true, logger: logger, dry_run: false ).lock_writes diff --git a/spec/lib/gitlab/database/loose_foreign_keys_spec.rb b/spec/lib/gitlab/database/loose_foreign_keys_spec.rb index ff99f681b0c..3c2d9ca82f2 100644 --- a/spec/lib/gitlab/database/loose_foreign_keys_spec.rb +++ b/spec/lib/gitlab/database/loose_foreign_keys_spec.rb @@ -112,4 +112,31 @@ RSpec.describe Gitlab::Database::LooseForeignKeys do end end end + + describe '.build_definition' do + context 'when child table schema is not defined' do + let(:loose_foreign_keys_yaml) do + { + 'ci_unknown_table' => [ + { + 'table' => 'projects', + 'column' => 'project_id', + 'on_delete' => 'async_delete' + } + ] + } + end + + subject { described_class.definitions } + + before do + described_class.instance_variable_set(:@definitions, nil) + described_class.instance_variable_set(:@loose_foreign_keys_yaml, loose_foreign_keys_yaml) + end + + it 'raises Gitlab::Database::GitlabSchema::UnknownSchemaError error' do + expect { subject }.to raise_error(Gitlab::Database::GitlabSchema::UnknownSchemaError) + end + end + end end diff --git a/spec/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables_spec.rb b/spec/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables_spec.rb index 9fd49b312eb..089c7a779f2 100644 --- a/spec/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables_spec.rb +++ b/spec/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables_spec.rb @@ -3,27 +3,39 @@ require 'spec_helper' RSpec.describe Gitlab::Database::MigrationHelpers::AutomaticLockWritesOnTables, - :reestablished_active_record_base, query_analyzers: false do + :reestablished_active_record_base, :delete, query_analyzers: false, feature_category: :pods do using RSpec::Parameterized::TableSyntax let(:schema_class) { Class.new(Gitlab::Database::Migration[2.1]) } + let(:skip_automatic_lock_on_writes) { false } let(:gitlab_main_table_name) { :_test_gitlab_main_table } let(:gitlab_ci_table_name) { :_test_gitlab_ci_table } let(:gitlab_geo_table_name) { :_test_gitlab_geo_table } let(:gitlab_shared_table_name) { :_test_table } + let(:renamed_gitlab_main_table_name) { :_test_gitlab_main_new_table } + let(:renamed_gitlab_ci_table_name) { :_test_gitlab_ci_new_table } + before do stub_feature_flags(automatic_lock_writes_on_table: true) reconfigure_db_connection(model: ActiveRecord::Base, config_model: config_model) end + # Drop the created test tables, because we use non-transactional tests + after do + drop_table_if_exists(gitlab_main_table_name) + drop_table_if_exists(gitlab_ci_table_name) + drop_table_if_exists(gitlab_geo_table_name) + drop_table_if_exists(gitlab_shared_table_name) + drop_table_if_exists(renamed_gitlab_main_table_name) + drop_table_if_exists(renamed_gitlab_ci_table_name) + end + shared_examples 'does not lock writes on table' do |config_model| let(:config_model) { config_model } it 'allows deleting records from the table' do - allow_next_instance_of(Gitlab::Database::LockWritesManager) do |instance| - expect(instance).not_to receive(:lock_writes) - end + expect(Gitlab::Database::LockWritesManager).not_to receive(:new) run_migration @@ -37,9 +49,10 @@ RSpec.describe Gitlab::Database::MigrationHelpers::AutomaticLockWritesOnTables, let(:config_model) { config_model } it 'errors on deleting' do - allow_next_instance_of(Gitlab::Database::LockWritesManager) do |instance| + expect_next_instance_of(Gitlab::Database::LockWritesManager) do |instance| expect(instance).to receive(:lock_writes).and_call_original end + expect(Gitlab::Database::WithLockRetries).not_to receive(:new) run_migration @@ -49,22 +62,35 @@ RSpec.describe Gitlab::Database::MigrationHelpers::AutomaticLockWritesOnTables, end end - context 'when executing create_table migrations' do - let(:create_gitlab_main_table_migration_class) { create_table_migration(gitlab_main_table_name) } - let(:create_gitlab_ci_table_migration_class) { create_table_migration(gitlab_ci_table_name) } - let(:create_gitlab_shared_table_migration_class) { create_table_migration(gitlab_shared_table_name) } + shared_examples 'locks writes on table using WithLockRetries' do |config_model| + let(:config_model) { config_model } + + it 'locks the writes on the table using WithLockRetries' do + expect_next_instance_of(Gitlab::Database::WithLockRetries) do |instance| + expect(instance).to receive(:run).and_call_original + end + run_migration + + expect do + migration_class.connection.execute("DELETE FROM #{table_name}") + end.to raise_error(ActiveRecord::StatementInvalid, /is write protected/) + end + end + + context 'when executing create_table migrations' do context 'when single database' do let(:config_model) { Gitlab::Database.database_base_models[:main] } + let(:create_gitlab_main_table_migration_class) { create_table_migration(gitlab_main_table_name) } + let(:create_gitlab_ci_table_migration_class) { create_table_migration(gitlab_ci_table_name) } + let(:create_gitlab_shared_table_migration_class) { create_table_migration(gitlab_shared_table_name) } before do skip_if_multiple_databases_are_setup end it 'does not lock any newly created tables' do - allow_next_instance_of(Gitlab::Database::LockWritesManager) do |instance| - expect(instance).not_to receive(:lock_writes) - end + expect(Gitlab::Database::LockWritesManager).not_to receive(:new) create_gitlab_main_table_migration_class.migrate(:up) create_gitlab_ci_table_migration_class.migrate(:up) @@ -83,9 +109,12 @@ RSpec.describe Gitlab::Database::MigrationHelpers::AutomaticLockWritesOnTables, skip_if_multiple_databases_not_setup end - let(:skip_automatic_lock_on_writes) { false } let(:migration_class) { create_table_migration(table_name, skip_automatic_lock_on_writes) } - let(:run_migration) { migration_class.migrate(:up) } + let(:run_migration) do + migration_class.connection.transaction do + migration_class.migrate(:up) + end + end context 'for creating a gitlab_main table' do let(:table_name) { gitlab_main_table_name } @@ -95,7 +124,9 @@ RSpec.describe Gitlab::Database::MigrationHelpers::AutomaticLockWritesOnTables, context 'when table listed as a deleted table' do before do - stub_const("Gitlab::Database::GitlabSchema::DELETED_TABLES", { table_name.to_s => :gitlab_main }) + allow(Gitlab::Database::GitlabSchema).to receive(:deleted_tables_to_schema).and_return( + { table_name.to_s => :gitlab_main } + ) end it_behaves_like 'does not lock writes on table', Gitlab::Database.database_base_models[:ci] @@ -107,6 +138,14 @@ RSpec.describe Gitlab::Database::MigrationHelpers::AutomaticLockWritesOnTables, it_behaves_like 'does not lock writes on table', Gitlab::Database.database_base_models[:ci] end + context 'when migration does not run within a transaction' do + let(:run_migration) do + migration_class.migrate(:up) + end + + it_behaves_like 'locks writes on table using WithLockRetries', Gitlab::Database.database_base_models[:ci] + end + context 'when the SKIP_AUTOMATIC_LOCK_ON_WRITES feature flag is set' do before do stub_env('SKIP_AUTOMATIC_LOCK_ON_WRITES' => 'true') @@ -132,7 +171,9 @@ RSpec.describe Gitlab::Database::MigrationHelpers::AutomaticLockWritesOnTables, context 'when table listed as a deleted table' do before do - stub_const("Gitlab::Database::GitlabSchema::DELETED_TABLES", { table_name.to_s => :gitlab_ci }) + allow(Gitlab::Database::GitlabSchema).to receive(:deleted_tables_to_schema).and_return( + { table_name.to_s => :gitlab_ci } + ) end it_behaves_like 'does not lock writes on table', Gitlab::Database.database_base_models[:main] @@ -202,11 +243,15 @@ RSpec.describe Gitlab::Database::MigrationHelpers::AutomaticLockWritesOnTables, end let(:migration_class) { rename_table_migration(old_table_name, table_name) } - let(:run_migration) { migration_class.migrate(:up) } + let(:run_migration) do + migration_class.connection.transaction do + migration_class.migrate(:up) + end + end context 'when a gitlab_main table' do let(:old_table_name) { gitlab_main_table_name } - let(:table_name) { :_test_gitlab_main_new_table } + let(:table_name) { renamed_gitlab_main_table_name } let(:database_base_model) { Gitlab::Database.database_base_models[:main] } it_behaves_like 'does not lock writes on table', Gitlab::Database.database_base_models[:main] @@ -215,7 +260,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers::AutomaticLockWritesOnTables, context 'when a gitlab_ci table' do let(:old_table_name) { gitlab_ci_table_name } - let(:table_name) { :_test_gitlab_ci_new_table } + let(:table_name) { renamed_gitlab_ci_table_name } let(:database_base_model) { Gitlab::Database.database_base_models[:ci] } it_behaves_like 'does not lock writes on table', Gitlab::Database.database_base_models[:ci] @@ -236,9 +281,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers::AutomaticLockWritesOnTables, end it 'does not lock any newly created tables' do - allow_next_instance_of(Gitlab::Database::LockWritesManager) do |instance| - expect(instance).not_to receive(:lock_writes) - end + expect(Gitlab::Database::LockWritesManager).not_to receive(:new) drop_gitlab_main_table_migration_class.connection.execute("CREATE TABLE #{gitlab_main_table_name}()") drop_gitlab_ci_table_migration_class.connection.execute("CREATE TABLE #{gitlab_ci_table_name}()") @@ -268,7 +311,11 @@ RSpec.describe Gitlab::Database::MigrationHelpers::AutomaticLockWritesOnTables, end let(:migration_class) { drop_table_migration(table_name) } - let(:run_migration) { migration_class.migrate(:down) } + let(:run_migration) do + migration_class.connection.transaction do + migration_class.migrate(:down) + end + end context 'for re-creating a gitlab_main table' do let(:table_name) { gitlab_main_table_name } @@ -293,14 +340,14 @@ RSpec.describe Gitlab::Database::MigrationHelpers::AutomaticLockWritesOnTables, end end - def create_table_migration(table_name, skip_lock_on_writes = false) + def create_table_migration(table_name, skip_automatic_lock_on_writes = false) migration_class = Class.new(schema_class) do class << self; attr_accessor :table_name; end def change create_table self.class.table_name end end - migration_class.skip_automatic_lock_on_writes = skip_lock_on_writes + migration_class.skip_automatic_lock_on_writes = skip_automatic_lock_on_writes migration_class.tap { |klass| klass.table_name = table_name } end @@ -331,4 +378,11 @@ RSpec.describe Gitlab::Database::MigrationHelpers::AutomaticLockWritesOnTables, def geo_configured? !!ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, name: 'geo') end + + # To drop the test tables that have been created in the test migrations + def drop_table_if_exists(table_name) + Gitlab::Database.database_base_models.each_value do |model| + model.connection.execute("DROP TABLE IF EXISTS #{table_name}") + end + end end diff --git a/spec/lib/gitlab/database/migration_helpers/restrict_gitlab_schema_spec.rb b/spec/lib/gitlab/database/migration_helpers/restrict_gitlab_schema_spec.rb index e8045f5afec..714fbab5aff 100644 --- a/spec/lib/gitlab/database/migration_helpers/restrict_gitlab_schema_spec.rb +++ b/spec/lib/gitlab/database/migration_helpers/restrict_gitlab_schema_spec.rb @@ -2,7 +2,8 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::MigrationHelpers::RestrictGitlabSchema, query_analyzers: false, stub_feature_flags: false do +RSpec.describe Gitlab::Database::MigrationHelpers::RestrictGitlabSchema, query_analyzers: false, + stub_feature_flags: false, feature_category: :pods do let(:schema_class) { Class.new(Gitlab::Database::Migration[1.0]).include(described_class) } # We keep only the GitlabSchemasValidateConnection analyzer running @@ -125,8 +126,9 @@ RSpec.describe Gitlab::Database::MigrationHelpers::RestrictGitlabSchema, query_a "does add index to ci_builds in gitlab_main and gitlab_ci" => { migration: ->(klass) do def change - # Due to running in transactin we cannot use `add_concurrent_index` - add_index :ci_builds, :tag, where: "type = 'Ci::Build'", name: 'index_ci_builds_on_tag_and_type_eq_ci_build' + # Due to running in transaction we cannot use `add_concurrent_index` + index_name = 'index_ci_builds_on_tag_and_type_eq_ci_build' + add_index :ci_builds, :tag, where: "type = 'Ci::Build'", name: index_name end end, query_matcher: /CREATE INDEX/, diff --git a/spec/lib/gitlab/database/migration_helpers_spec.rb b/spec/lib/gitlab/database/migration_helpers_spec.rb index 30eeff31326..12fa115cc4e 100644 --- a/spec/lib/gitlab/database/migration_helpers_spec.rb +++ b/spec/lib/gitlab/database/migration_helpers_spec.rb @@ -743,6 +743,75 @@ RSpec.describe Gitlab::Database::MigrationHelpers do end end + context 'ON UPDATE statements' do + context 'on_update: :nullify' do + it 'appends ON UPDATE SET NULL statement' do + expect(model).to receive(:with_lock_retries).and_call_original + expect(model).to receive(:disable_statement_timeout).and_call_original + expect(model).to receive(:statement_timeout_disabled?).and_return(false) + expect(model).to receive(:execute).with(/SET statement_timeout TO/) + expect(model).to receive(:execute).ordered.with(/VALIDATE CONSTRAINT/) + expect(model).to receive(:execute).ordered.with(/RESET statement_timeout/) + + expect(model).to receive(:execute).with(/ON UPDATE SET NULL/) + + model.add_concurrent_foreign_key(:projects, :users, + column: :user_id, + on_update: :nullify) + end + end + + context 'on_update: :cascade' do + it 'appends ON UPDATE CASCADE statement' do + expect(model).to receive(:with_lock_retries).and_call_original + expect(model).to receive(:disable_statement_timeout).and_call_original + expect(model).to receive(:statement_timeout_disabled?).and_return(false) + expect(model).to receive(:execute).with(/SET statement_timeout TO/) + expect(model).to receive(:execute).ordered.with(/VALIDATE CONSTRAINT/) + expect(model).to receive(:execute).ordered.with(/RESET statement_timeout/) + + expect(model).to receive(:execute).with(/ON UPDATE CASCADE/) + + model.add_concurrent_foreign_key(:projects, :users, + column: :user_id, + on_update: :cascade) + end + end + + context 'on_update: nil' do + it 'appends no ON UPDATE statement' do + expect(model).to receive(:with_lock_retries).and_call_original + expect(model).to receive(:disable_statement_timeout).and_call_original + expect(model).to receive(:statement_timeout_disabled?).and_return(false) + expect(model).to receive(:execute).with(/SET statement_timeout TO/) + expect(model).to receive(:execute).ordered.with(/VALIDATE CONSTRAINT/) + expect(model).to receive(:execute).ordered.with(/RESET statement_timeout/) + + expect(model).not_to receive(:execute).with(/ON UPDATE/) + + model.add_concurrent_foreign_key(:projects, :users, + column: :user_id, + on_update: nil) + end + end + + context 'when on_update is not provided' do + it 'appends no ON UPDATE statement' do + expect(model).to receive(:with_lock_retries).and_call_original + expect(model).to receive(:disable_statement_timeout).and_call_original + expect(model).to receive(:statement_timeout_disabled?).and_return(false) + expect(model).to receive(:execute).with(/SET statement_timeout TO/) + expect(model).to receive(:execute).ordered.with(/VALIDATE CONSTRAINT/) + expect(model).to receive(:execute).ordered.with(/RESET statement_timeout/) + + expect(model).not_to receive(:execute).with(/ON UPDATE/) + + model.add_concurrent_foreign_key(:projects, :users, + column: :user_id) + end + end + end + context 'when no custom key name is supplied' do it 'creates a concurrent foreign key and validates it' do expect(model).to receive(:with_lock_retries).and_call_original @@ -760,6 +829,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers do name = model.concurrent_foreign_key_name(:projects, :user_id) expect(model).to receive(:foreign_key_exists?).with(:projects, :users, column: :user_id, + on_update: nil, on_delete: :cascade, name: name, primary_key: :id).and_return(true) @@ -792,6 +862,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers do expect(model).to receive(:foreign_key_exists?).with(:projects, :users, name: :foo, primary_key: :id, + on_update: nil, on_delete: :cascade, column: :user_id).and_return(true) @@ -861,6 +932,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers do "ADD CONSTRAINT fk_multiple_columns\n" \ "FOREIGN KEY \(partition_number, user_id\)\n" \ "REFERENCES users \(partition_number, id\)\n" \ + "ON UPDATE CASCADE\n" \ "ON DELETE CASCADE\n" \ "NOT VALID;\n" ) @@ -871,7 +943,8 @@ RSpec.describe Gitlab::Database::MigrationHelpers do column: [:partition_number, :user_id], target_column: [:partition_number, :id], validate: false, - name: :fk_multiple_columns + name: :fk_multiple_columns, + on_update: :cascade ) end @@ -883,6 +956,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers do { column: [:partition_number, :user_id], name: :fk_multiple_columns, + on_update: :cascade, on_delete: :cascade, primary_key: [:partition_number, :id] } @@ -898,6 +972,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers do :users, column: [:partition_number, :user_id], target_column: [:partition_number, :id], + on_update: :cascade, validate: false, name: :fk_multiple_columns ) @@ -973,58 +1048,58 @@ RSpec.describe Gitlab::Database::MigrationHelpers do describe '#foreign_key_exists?' do before do - key = ActiveRecord::ConnectionAdapters::ForeignKeyDefinition.new( - :projects, :users, - { - column: :non_standard_id, - name: :fk_projects_users_non_standard_id, - on_delete: :cascade, - primary_key: :id - } - ) - allow(model).to receive(:foreign_keys).with(:projects).and_return([key]) + model.connection.execute(<<~SQL) + create table referenced ( + id bigserial primary key not null + ); + create table referencing ( + id bigserial primary key not null, + non_standard_id bigint not null, + constraint fk_referenced foreign key (non_standard_id) references referenced(id) on delete cascade + ); + SQL end shared_examples_for 'foreign key checks' do it 'finds existing foreign keys by column' do - expect(model.foreign_key_exists?(:projects, target_table, column: :non_standard_id)).to be_truthy + expect(model.foreign_key_exists?(:referencing, target_table, column: :non_standard_id)).to be_truthy end it 'finds existing foreign keys by name' do - expect(model.foreign_key_exists?(:projects, target_table, name: :fk_projects_users_non_standard_id)).to be_truthy + expect(model.foreign_key_exists?(:referencing, target_table, name: :fk_referenced)).to be_truthy end it 'finds existing foreign_keys by name and column' do - expect(model.foreign_key_exists?(:projects, target_table, name: :fk_projects_users_non_standard_id, column: :non_standard_id)).to be_truthy + expect(model.foreign_key_exists?(:referencing, target_table, name: :fk_referenced, column: :non_standard_id)).to be_truthy end it 'finds existing foreign_keys by name, column and on_delete' do - expect(model.foreign_key_exists?(:projects, target_table, name: :fk_projects_users_non_standard_id, column: :non_standard_id, on_delete: :cascade)).to be_truthy + expect(model.foreign_key_exists?(:referencing, target_table, name: :fk_referenced, column: :non_standard_id, on_delete: :cascade)).to be_truthy end it 'finds existing foreign keys by target table only' do - expect(model.foreign_key_exists?(:projects, target_table)).to be_truthy + expect(model.foreign_key_exists?(:referencing, target_table)).to be_truthy end it 'compares by column name if given' do - expect(model.foreign_key_exists?(:projects, target_table, column: :user_id)).to be_falsey + expect(model.foreign_key_exists?(:referencing, target_table, column: :user_id)).to be_falsey end it 'compares by target column name if given' do - expect(model.foreign_key_exists?(:projects, target_table, primary_key: :user_id)).to be_falsey - expect(model.foreign_key_exists?(:projects, target_table, primary_key: :id)).to be_truthy + expect(model.foreign_key_exists?(:referencing, target_table, primary_key: :user_id)).to be_falsey + expect(model.foreign_key_exists?(:referencing, target_table, primary_key: :id)).to be_truthy end it 'compares by foreign key name if given' do - expect(model.foreign_key_exists?(:projects, target_table, name: :non_existent_foreign_key_name)).to be_falsey + expect(model.foreign_key_exists?(:referencing, target_table, name: :non_existent_foreign_key_name)).to be_falsey end it 'compares by foreign key name and column if given' do - expect(model.foreign_key_exists?(:projects, target_table, name: :non_existent_foreign_key_name, column: :non_standard_id)).to be_falsey + expect(model.foreign_key_exists?(:referencing, target_table, name: :non_existent_foreign_key_name, column: :non_standard_id)).to be_falsey end it 'compares by foreign key name, column and on_delete if given' do - expect(model.foreign_key_exists?(:projects, target_table, name: :fk_projects_users_non_standard_id, column: :non_standard_id, on_delete: :nullify)).to be_falsey + expect(model.foreign_key_exists?(:referencing, target_table, name: :fk_referenced, column: :non_standard_id, on_delete: :nullify)).to be_falsey end end @@ -1035,7 +1110,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers do end context 'specifying a target table' do - let(:target_table) { :users } + let(:target_table) { :referenced } it_behaves_like 'foreign key checks' end @@ -1044,59 +1119,66 @@ RSpec.describe Gitlab::Database::MigrationHelpers do expect(model.foreign_key_exists?(:projects, :other_table)).to be_falsey end + it 'raises an error if an invalid on_delete is specified' do + # The correct on_delete key is "nullify" + expect { model.foreign_key_exists?(:referenced, on_delete: :set_null) }.to raise_error(ArgumentError) + end + context 'with foreign key using multiple columns' do before do - key = ActiveRecord::ConnectionAdapters::ForeignKeyDefinition.new( - :projects, :users, - { - column: [:partition_number, :id], - name: :fk_projects_users_partition_number_id, - on_delete: :cascade, - primary_key: [:partition_number, :id] - } - ) - allow(model).to receive(:foreign_keys).with(:projects).and_return([key]) + model.connection.execute(<<~SQL) + create table p_referenced ( + id bigserial not null, + partition_number bigint not null default 100, + primary key (partition_number, id) + ); + create table p_referencing ( + id bigserial primary key not null, + partition_number bigint not null, + constraint fk_partitioning foreign key (partition_number, id) references p_referenced(partition_number, id) on delete cascade + ); + SQL end it 'finds existing foreign keys by columns' do - expect(model.foreign_key_exists?(:projects, :users, column: [:partition_number, :id])).to be_truthy + expect(model.foreign_key_exists?(:p_referencing, :p_referenced, column: [:partition_number, :id])).to be_truthy end it 'finds existing foreign keys by name' do - expect(model.foreign_key_exists?(:projects, :users, name: :fk_projects_users_partition_number_id)).to be_truthy + expect(model.foreign_key_exists?(:p_referencing, :p_referenced, name: :fk_partitioning)).to be_truthy end it 'finds existing foreign_keys by name and column' do - expect(model.foreign_key_exists?(:projects, :users, name: :fk_projects_users_partition_number_id, column: [:partition_number, :id])).to be_truthy + expect(model.foreign_key_exists?(:p_referencing, :p_referenced, name: :fk_partitioning, column: [:partition_number, :id])).to be_truthy end it 'finds existing foreign_keys by name, column and on_delete' do - expect(model.foreign_key_exists?(:projects, :users, name: :fk_projects_users_partition_number_id, column: [:partition_number, :id], on_delete: :cascade)).to be_truthy + expect(model.foreign_key_exists?(:p_referencing, :p_referenced, name: :fk_partitioning, column: [:partition_number, :id], on_delete: :cascade)).to be_truthy end it 'finds existing foreign keys by target table only' do - expect(model.foreign_key_exists?(:projects, :users)).to be_truthy + expect(model.foreign_key_exists?(:p_referencing, :p_referenced)).to be_truthy end it 'compares by column name if given' do - expect(model.foreign_key_exists?(:projects, :users, column: :id)).to be_falsey + expect(model.foreign_key_exists?(:p_referencing, :p_referenced, column: :id)).to be_falsey end it 'compares by target column name if given' do - expect(model.foreign_key_exists?(:projects, :users, primary_key: :user_id)).to be_falsey - expect(model.foreign_key_exists?(:projects, :users, primary_key: [:partition_number, :id])).to be_truthy + expect(model.foreign_key_exists?(:p_referencing, :p_referenced, primary_key: :user_id)).to be_falsey + expect(model.foreign_key_exists?(:p_referencing, :p_referenced, primary_key: [:partition_number, :id])).to be_truthy end it 'compares by foreign key name if given' do - expect(model.foreign_key_exists?(:projects, :users, name: :non_existent_foreign_key_name)).to be_falsey + expect(model.foreign_key_exists?(:p_referencing, :p_referenced, name: :non_existent_foreign_key_name)).to be_falsey end it 'compares by foreign key name and column if given' do - expect(model.foreign_key_exists?(:projects, :users, name: :non_existent_foreign_key_name, column: [:partition_number, :id])).to be_falsey + expect(model.foreign_key_exists?(:p_referencing, :p_referenced, name: :non_existent_foreign_key_name, column: [:partition_number, :id])).to be_falsey end it 'compares by foreign key name, column and on_delete if given' do - expect(model.foreign_key_exists?(:projects, :users, name: :fk_projects_users_partition_number_id, column: [:partition_number, :id], on_delete: :nullify)).to be_falsey + expect(model.foreign_key_exists?(:p_referencing, :p_referenced, name: :fk_partitioning, column: [:partition_number, :id], on_delete: :nullify)).to be_falsey end end end @@ -1159,7 +1241,8 @@ RSpec.describe Gitlab::Database::MigrationHelpers do Gitlab::Database::LockWritesManager.new( table_name: test_table, connection: model.connection, - database_name: 'main' + database_name: 'main', + with_retries: false ) end @@ -1340,7 +1423,8 @@ RSpec.describe Gitlab::Database::MigrationHelpers do Gitlab::Database::LockWritesManager.new( table_name: test_table, connection: model.connection, - database_name: 'main' + database_name: 'main', + with_retries: false ) end diff --git a/spec/lib/gitlab/database/migrations/instrumentation_spec.rb b/spec/lib/gitlab/database/migrations/instrumentation_spec.rb index 3540a120b8f..b0bdbf5c371 100644 --- a/spec/lib/gitlab/database/migrations/instrumentation_spec.rb +++ b/spec/lib/gitlab/database/migrations/instrumentation_spec.rb @@ -2,6 +2,8 @@ require 'spec_helper' RSpec.describe Gitlab::Database::Migrations::Instrumentation do + subject(:instrumentation) { described_class.new(result_dir: result_dir) } + let(:result_dir) { Dir.mktmpdir } let(:connection) { ActiveRecord::Migration.connection } @@ -9,17 +11,18 @@ RSpec.describe Gitlab::Database::Migrations::Instrumentation do FileUtils.rm_rf(result_dir) end describe '#observe' do - subject { described_class.new(result_dir: result_dir) } - def load_observation(result_dir, migration_name) Gitlab::Json.parse(File.read(File.join(result_dir, migration_name, described_class::STATS_FILENAME))) end let(:migration_name) { 'test' } let(:migration_version) { '12345' } + let(:migration_meta) { { 'max_batch_size' => 1, 'total_tuple_count' => 10, 'interval' => 60 } } it 'executes the given block' do - expect { |b| subject.observe(version: migration_version, name: migration_name, connection: connection, &b) }.to yield_control + expect do |b| + instrumentation.observe(version: migration_version, name: migration_name, connection: connection, meta: migration_meta, &b) + end.to yield_control end context 'behavior with observers' do @@ -68,13 +71,17 @@ RSpec.describe Gitlab::Database::Migrations::Instrumentation do end context 'on successful execution' do - subject { described_class.new(result_dir: result_dir).observe(version: migration_version, name: migration_name, connection: connection) {} } + subject do + instrumentation.observe(version: migration_version, name: migration_name, + connection: connection, meta: migration_meta) {} + end it 'records a valid observation', :aggregate_failures do expect(subject.walltime).not_to be_nil expect(subject.success).to be_truthy expect(subject.version).to eq(migration_version) expect(subject.name).to eq(migration_name) + expect(subject.meta).to eq(migration_meta) end end @@ -82,9 +89,10 @@ RSpec.describe Gitlab::Database::Migrations::Instrumentation do where(exception: ['something went wrong', SystemStackError, Interrupt]) with_them do - let(:instance) { described_class.new(result_dir: result_dir) } - - subject(:observe) { instance.observe(version: migration_version, name: migration_name, connection: connection) { raise exception } } + subject(:observe) do + instrumentation.observe(version: migration_version, name: migration_name, + connection: connection, meta: migration_meta) { raise exception } + end it 'raises the exception' do expect { observe }.to raise_error(exception) @@ -106,14 +114,13 @@ RSpec.describe Gitlab::Database::Migrations::Instrumentation do expect(subject['success']).to be_falsey expect(subject['version']).to eq(migration_version) expect(subject['name']).to eq(migration_name) + expect(subject['meta']).to include(migration_meta) end end end end context 'sequence of migrations with failures' do - subject { described_class.new(result_dir: result_dir) } - let(:migration1) { double('migration1', call: nil) } let(:migration2) { double('migration2', call: nil) } @@ -121,9 +128,9 @@ RSpec.describe Gitlab::Database::Migrations::Instrumentation do let(:migration_version_2) { '98765' } it 'records observations for all migrations' do - subject.observe(version: migration_version, name: migration_name, connection: connection) {} + instrumentation.observe(version: migration_version, name: migration_name, connection: connection) {} begin - subject.observe(version: migration_version_2, name: migration_name_2, connection: connection) { raise 'something went wrong' } + instrumentation.observe(version: migration_version_2, name: migration_name_2, connection: connection) { raise 'something went wrong' } rescue StandardError nil end diff --git a/spec/lib/gitlab/database/migrations/test_batched_background_runner_spec.rb b/spec/lib/gitlab/database/migrations/test_batched_background_runner_spec.rb index 73d69d55e5a..0b048617ce1 100644 --- a/spec/lib/gitlab/database/migrations/test_batched_background_runner_spec.rb +++ b/spec/lib/gitlab/database/migrations/test_batched_background_runner_spec.rb @@ -69,12 +69,27 @@ RSpec.describe Gitlab::Database::Migrations::TestBatchedBackgroundRunner, :freez end context 'running a real background migration' do + let(:interval) { 5.minutes } + let(:meta) { { "max_batch_size" => nil, "total_tuple_count" => nil, "interval" => interval } } + + let(:params) do + { + version: nil, + connection: connection, + meta: { + interval: 300, + max_batch_size: nil, + total_tuple_count: nil + } + } + end + before do queue_migration('CopyColumnUsingBackgroundMigrationJob', table_name, :id, :id, :data, batch_size: 100, - job_interval: 5.minutes) # job_interval is skipped when testing + job_interval: interval) # job_interval is skipped when testing end subject(:sample_migration) do @@ -91,10 +106,9 @@ RSpec.describe Gitlab::Database::Migrations::TestBatchedBackgroundRunner, :freez }.by_at_most(-1) end - it 'uses the correct connection to instrument the background migration' do + it 'uses the correct params to instrument the background migration' do expect_next_instance_of(Gitlab::Database::Migrations::Instrumentation) do |instrumentation| - expect(instrumentation).to receive(:observe).with(hash_including(connection: connection)) - .at_least(:once).and_call_original + expect(instrumentation).to receive(:observe).with(hash_including(params)).at_least(:once).and_call_original end subject diff --git a/spec/lib/gitlab/database/partitioning_spec.rb b/spec/lib/gitlab/database/partitioning_spec.rb index db5ca890155..855d0bc46a4 100644 --- a/spec/lib/gitlab/database/partitioning_spec.rb +++ b/spec/lib/gitlab/database/partitioning_spec.rb @@ -10,15 +10,15 @@ RSpec.describe Gitlab::Database::Partitioning do around do |example| previously_registered_models = described_class.registered_models.dup - described_class.instance_variable_set('@registered_models', Set.new) + described_class.instance_variable_set(:@registered_models, Set.new) previously_registered_tables = described_class.registered_tables.dup - described_class.instance_variable_set('@registered_tables', Set.new) + described_class.instance_variable_set(:@registered_tables, Set.new) example.run - described_class.instance_variable_set('@registered_models', previously_registered_models) - described_class.instance_variable_set('@registered_tables', previously_registered_tables) + described_class.instance_variable_set(:@registered_models, previously_registered_models) + described_class.instance_variable_set(:@registered_tables, previously_registered_tables) end describe '.register_models' do diff --git a/spec/lib/gitlab/database/postgres_autovacuum_activity_spec.rb b/spec/lib/gitlab/database/postgres_autovacuum_activity_spec.rb index c1ac8f0c9cd..f24c4559349 100644 --- a/spec/lib/gitlab/database/postgres_autovacuum_activity_spec.rb +++ b/spec/lib/gitlab/database/postgres_autovacuum_activity_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::PostgresAutovacuumActivity, type: :model do +RSpec.describe Gitlab::Database::PostgresAutovacuumActivity, type: :model, feature_category: :database do include Database::DatabaseHelpers it { is_expected.to be_a Gitlab::Database::SharedModel } @@ -13,7 +13,7 @@ RSpec.describe Gitlab::Database::PostgresAutovacuumActivity, type: :model do let(:tables) { %w[foo test] } before do - swapout_view_for_table(:postgres_autovacuum_activity) + swapout_view_for_table(:postgres_autovacuum_activity, connection: ApplicationRecord.connection) # unrelated create(:postgres_autovacuum_activity, table: 'bar') diff --git a/spec/lib/gitlab/database/postgres_foreign_key_spec.rb b/spec/lib/gitlab/database/postgres_foreign_key_spec.rb index b0e08ca1e67..a8dbc4be16f 100644 --- a/spec/lib/gitlab/database/postgres_foreign_key_spec.rb +++ b/spec/lib/gitlab/database/postgres_foreign_key_spec.rb @@ -2,28 +2,32 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::PostgresForeignKey, type: :model do +RSpec.describe Gitlab::Database::PostgresForeignKey, type: :model, feature_category: :database do # PostgresForeignKey does not `behaves_like 'a postgres model'` because it does not correspond 1-1 with a single entry # in pg_class before do - ActiveRecord::Base.connection.execute(<<~SQL) - CREATE TABLE public.referenced_table ( - id bigserial primary key not null - ); - - CREATE TABLE public.other_referenced_table ( - id bigserial primary key not null - ); - - CREATE TABLE public.constrained_table ( - id bigserial primary key not null, - referenced_table_id bigint not null, - other_referenced_table_id bigint not null, - CONSTRAINT fk_constrained_to_referenced FOREIGN KEY(referenced_table_id) REFERENCES referenced_table(id), - CONSTRAINT fk_constrained_to_other_referenced FOREIGN KEY(other_referenced_table_id) - REFERENCES other_referenced_table(id) - ); + ApplicationRecord.connection.execute(<<~SQL) + CREATE TABLE public.referenced_table ( + id bigserial primary key not null, + id_b bigserial not null, + UNIQUE (id, id_b) + ); + + CREATE TABLE public.other_referenced_table ( + id bigserial primary key not null + ); + + CREATE TABLE public.constrained_table ( + id bigserial primary key not null, + referenced_table_id bigint not null, + referenced_table_id_b bigint not null, + other_referenced_table_id bigint not null, + CONSTRAINT fk_constrained_to_referenced FOREIGN KEY(referenced_table_id, referenced_table_id_b) REFERENCES referenced_table(id, id_b) on delete restrict, + CONSTRAINT fk_constrained_to_other_referenced FOREIGN KEY(other_referenced_table_id) + REFERENCES other_referenced_table(id) + ); + SQL end @@ -39,6 +43,14 @@ RSpec.describe Gitlab::Database::PostgresForeignKey, type: :model do end end + describe '#by_referenced_table_name' do + it 'finds the foreign keys for the referenced table' do + expected = described_class.find_by!(name: 'fk_constrained_to_referenced') + + expect(described_class.by_referenced_table_name('referenced_table')).to contain_exactly(expected) + end + end + describe '#by_constrained_table_identifier' do it 'throws an error when the identifier name is not fully qualified' do expect { described_class.by_constrained_table_identifier('constrained_table') }.to raise_error(ArgumentError, /not fully qualified/) @@ -50,4 +62,147 @@ RSpec.describe Gitlab::Database::PostgresForeignKey, type: :model do expect(described_class.by_constrained_table_identifier('public.constrained_table')).to match_array(expected) end end + + describe '#by_constrained_table_name' do + it 'finds the foreign keys for the constrained table' do + expected = described_class.where(name: %w[fk_constrained_to_referenced fk_constrained_to_other_referenced]).to_a + + expect(described_class.by_constrained_table_name('constrained_table')).to match_array(expected) + end + end + + describe '#by_name' do + it 'finds foreign keys by name' do + expect(described_class.by_name('fk_constrained_to_referenced').pluck(:name)).to contain_exactly('fk_constrained_to_referenced') + end + end + + context 'when finding columns for foreign keys' do + using RSpec::Parameterized::TableSyntax + + let(:fks) { described_class.by_constrained_table_name('constrained_table') } + + where(:fk, :expected_constrained, :expected_referenced) do + lazy { described_class.find_by(name: 'fk_constrained_to_referenced') } | %w[referenced_table_id referenced_table_id_b] | %w[id id_b] + lazy { described_class.find_by(name: 'fk_constrained_to_other_referenced') } | %w[other_referenced_table_id] | %w[id] + end + + with_them do + it 'finds the correct constrained column names' do + expect(fk.constrained_columns).to eq(expected_constrained) + end + + it 'finds the correct referenced column names' do + expect(fk.referenced_columns).to eq(expected_referenced) + end + + describe '#by_constrained_columns' do + it 'finds the correct foreign key' do + expect(fks.by_constrained_columns(expected_constrained)).to contain_exactly(fk) + end + end + + describe '#by_referenced_columns' do + it 'finds the correct foreign key' do + expect(fks.by_referenced_columns(expected_referenced)).to contain_exactly(fk) + end + end + end + end + + describe '#on_delete_action' do + before do + ApplicationRecord.connection.execute(<<~SQL) + create table public.referenced_table_all_on_delete_actions ( + id bigserial primary key not null + ); + + create table public.constrained_table_all_on_delete_actions ( + id bigserial primary key not null, + ref_id_no_action bigint not null constraint fk_no_action references referenced_table_all_on_delete_actions(id), + ref_id_restrict bigint not null constraint fk_restrict references referenced_table_all_on_delete_actions(id) on delete restrict, + ref_id_nullify bigint not null constraint fk_nullify references referenced_table_all_on_delete_actions(id) on delete set null, + ref_id_cascade bigint not null constraint fk_cascade references referenced_table_all_on_delete_actions(id) on delete cascade, + ref_id_set_default bigint not null constraint fk_set_default references referenced_table_all_on_delete_actions(id) on delete set default + ) + SQL + end + + let(:fks) { described_class.by_constrained_table_name('constrained_table_all_on_delete_actions') } + + context 'with an invalid on_delete_action' do + it 'raises an error' do + # the correct value is :nullify, not :set_null + expect { fks.by_on_delete_action(:set_null) }.to raise_error(ArgumentError) + end + end + + where(:fk_name, :expected_on_delete_action) do + [ + %w[fk_no_action no_action], + %w[fk_restrict restrict], + %w[fk_nullify nullify], + %w[fk_cascade cascade], + %w[fk_set_default set_default] + ] + end + + with_them do + subject(:fk) { fks.find_by(name: fk_name) } + + it 'has the appropriate on delete action' do + expect(fk.on_delete_action).to eq(expected_on_delete_action) + end + + describe '#by_on_delete_action' do + it 'finds the key by on delete action' do + expect(fks.by_on_delete_action(expected_on_delete_action)).to contain_exactly(fk) + end + end + end + end + + context 'when supporting foreign keys to inherited tables in postgres 12' do + before do + skip('not supported before postgres 12') if ApplicationRecord.database.version.to_f < 12 + + ApplicationRecord.connection.execute(<<~SQL) + create table public.parent ( + id bigserial primary key not null + ) partition by hash(id); + + create table public.child partition of parent for values with (modulus 2, remainder 1); + + create table public.referencing_partitioned ( + id bigserial not null primary key, + constraint fk_inherited foreign key (id) references parent(id) + ) + SQL + end + + describe '#is_inherited' do + using RSpec::Parameterized::TableSyntax + + where(:fk, :inherited) do + lazy { described_class.find_by(name: 'fk_inherited') } | false + lazy { described_class.by_referenced_table_identifier('public.child').first! } | true + lazy { described_class.find_by(name: 'fk_constrained_to_referenced') } | false + end + + with_them do + it 'has the appropriate inheritance value' do + expect(fk.is_inherited).to eq(inherited) + end + end + end + + describe '#not_inherited' do + let(:fks) { described_class.by_constrained_table_identifier('public.referencing_partitioned') } + + it 'lists all non-inherited foreign keys' do + expect(fks.pluck(:referenced_table_name)).to contain_exactly('parent', 'child') + expect(fks.not_inherited.pluck(:referenced_table_name)).to contain_exactly('parent') + end + end + end end diff --git a/spec/lib/gitlab/database/query_analyzer_spec.rb b/spec/lib/gitlab/database/query_analyzer_spec.rb index 6dc9ffc4aba..0b849063562 100644 --- a/spec/lib/gitlab/database/query_analyzer_spec.rb +++ b/spec/lib/gitlab/database/query_analyzer_spec.rb @@ -10,7 +10,6 @@ RSpec.describe Gitlab::Database::QueryAnalyzer, query_analyzers: false do before do allow(described_class.instance).to receive(:all_analyzers).and_return([analyzer, disabled_analyzer]) allow(analyzer).to receive(:enabled?).and_return(true) - allow(analyzer).to receive(:raw?).and_return(false) allow(analyzer).to receive(:suppressed?).and_return(false) allow(analyzer).to receive(:begin!) allow(analyzer).to receive(:end!) @@ -182,13 +181,6 @@ RSpec.describe Gitlab::Database::QueryAnalyzer, query_analyzers: false do expect { process_sql("SELECT 1 FROM projects") }.not_to raise_error end - it 'does call analyze with raw sql when raw? is true' do - expect(analyzer).to receive(:raw?).and_return(true) - expect(analyzer).to receive(:analyze).with('SELECT 1 FROM projects') - - expect { process_sql("SELECT 1 FROM projects") }.not_to raise_error - end - def process_sql(sql) described_class.instance.within do ApplicationRecord.load_balancer.read_write do |connection| diff --git a/spec/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics_spec.rb b/spec/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics_spec.rb index 62c5ead855a..3a92f35d585 100644 --- a/spec/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics_spec.rb +++ b/spec/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics_spec.rb @@ -53,6 +53,14 @@ RSpec.describe Gitlab::Database::QueryAnalyzers::GitlabSchemasMetrics, query_ana gitlab_schemas: "gitlab_ci", db_config_name: "ci" } + }, + "for query accessing gitlab_main and unknown schema" => { + model: ApplicationRecord, + sql: "SELECT 1 FROM projects LEFT JOIN not_in_schema ON not_in_schema.project_id=projects.id", + expectations: { + gitlab_schemas: "gitlab_main,undefined_not_in_schema", + db_config_name: "main" + } } } end diff --git a/spec/lib/gitlab/database/query_analyzers/gitlab_schemas_validate_connection_spec.rb b/spec/lib/gitlab/database/query_analyzers/gitlab_schemas_validate_connection_spec.rb index ddf5793049d..47038bbd138 100644 --- a/spec/lib/gitlab/database/query_analyzers/gitlab_schemas_validate_connection_spec.rb +++ b/spec/lib/gitlab/database/query_analyzers/gitlab_schemas_validate_connection_spec.rb @@ -2,7 +2,8 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::QueryAnalyzers::GitlabSchemasValidateConnection, query_analyzers: false do +RSpec.describe Gitlab::Database::QueryAnalyzers::GitlabSchemasValidateConnection, query_analyzers: false, + feature_category: :pods do let(:analyzer) { described_class } # We keep only the GitlabSchemasValidateConnection analyzer running @@ -51,6 +52,12 @@ RSpec.describe Gitlab::Database::QueryAnalyzers::GitlabSchemasValidateConnection sql: "SELECT 1 FROM ci_builds", expect_error: /The query tried to access \["ci_builds"\]/, setup: -> (_) { skip_if_multiple_databases_not_setup } + }, + "for query accessing unknown gitlab_schema" => { + model: ::ApplicationRecord, + sql: "SELECT 1 FROM new_table", + expect_error: /The query tried to access \["new_table"\] \(of undefined_new_table\)/, + setup: -> (_) { skip_if_multiple_databases_not_setup } } } end diff --git a/spec/lib/gitlab/database/query_analyzers/prevent_cross_database_modification_spec.rb b/spec/lib/gitlab/database/query_analyzers/prevent_cross_database_modification_spec.rb index 22a70dc7df0..a4322689bf9 100644 --- a/spec/lib/gitlab/database/query_analyzers/prevent_cross_database_modification_spec.rb +++ b/spec/lib/gitlab/database/query_analyzers/prevent_cross_database_modification_spec.rb @@ -2,7 +2,8 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::QueryAnalyzers::PreventCrossDatabaseModification, query_analyzers: false do +RSpec.describe Gitlab::Database::QueryAnalyzers::PreventCrossDatabaseModification, query_analyzers: false, + feature_category: :pods do let_it_be(:pipeline, refind: true) { create(:ci_pipeline) } let_it_be(:project, refind: true) { create(:project) } diff --git a/spec/lib/gitlab/database/query_analyzers/query_recorder_spec.rb b/spec/lib/gitlab/database/query_analyzers/query_recorder_spec.rb index bcc39c0c3db..22ff66ff55e 100644 --- a/spec/lib/gitlab/database/query_analyzers/query_recorder_spec.rb +++ b/spec/lib/gitlab/database/query_analyzers/query_recorder_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::QueryAnalyzers::QueryRecorder, query_analyzers: false do +RSpec.describe Gitlab::Database::QueryAnalyzers::QueryRecorder, feature_category: :database, query_analyzers: false do # We keep only the QueryRecorder analyzer running around do |example| described_class.with_suppressed(false) do @@ -11,7 +11,6 @@ RSpec.describe Gitlab::Database::QueryAnalyzers::QueryRecorder, query_analyzers: end context 'with query analyzer' do - let(:query) { 'SELECT 1 FROM projects' } let(:log_path) { Rails.root.join(described_class::LOG_PATH) } let(:log_file) { described_class.log_file } @@ -20,14 +19,44 @@ RSpec.describe Gitlab::Database::QueryAnalyzers::QueryRecorder, query_analyzers: end shared_examples_for 'an enabled query recorder' do - it 'logs queries to a file' do - allow(FileUtils).to receive(:mkdir_p) - .with(log_path) - expect(File).to receive(:write) - .with(log_file, /^{"sql":"#{query}/, mode: 'a') - expect(described_class).to receive(:analyze).with(/^#{query}/).and_call_original - - expect { ApplicationRecord.connection.execute(query) }.not_to raise_error + using RSpec::Parameterized::TableSyntax + + normalized_query = <<~SQL.strip.tr("\n", ' ') + SELECT \\\\"projects\\\\".\\\\"id\\\\" + FROM \\\\"projects\\\\" + WHERE \\\\"projects\\\\".\\\\"namespace_id\\\\" = \\? + AND \\\\"projects\\\\".\\\\"id\\\\" IN \\(\\?,\\?,\\?\\); + SQL + + where(:list_parameter, :bind_parameters) do + '$2, $3' | [1, 2, 3] + '$2, $3, $4' | [1, 2, 3, 4] + '$2 ,$3 ,$4 ,$5' | [1, 2, 3, 4, 5] + '$2 , $3 , $4 , $5, $6' | [1, 2, 3, 4, 5, 6] + '$2, $3 ,$4 , $5,$6,$7' | [1, 2, 3, 4, 5, 6, 7] + '$2,$3,$4,$5,$6,$7,$8' | [1, 2, 3, 4, 5, 6, 7, 8] + end + + with_them do + before do + allow(described_class).to receive(:analyze).and_call_original + allow(FileUtils).to receive(:mkdir_p) + .with(log_path) + end + + it 'logs normalized queries to a file' do + expect(File).to receive(:write) + .with(log_file, /^{"normalized":"#{normalized_query}/, mode: 'a') + + expect do + ApplicationRecord.connection.exec_query(<<~SQL.strip.tr("\n", ' '), 'SQL', bind_parameters) + SELECT "projects"."id" + FROM "projects" + WHERE "projects"."namespace_id" = $1 + AND "projects"."id" IN (#{list_parameter}); + SQL + end.not_to raise_error + end end end diff --git a/spec/lib/gitlab/database/reindexing/coordinator_spec.rb b/spec/lib/gitlab/database/reindexing/coordinator_spec.rb index bb91617714a..bf993e85cb8 100644 --- a/spec/lib/gitlab/database/reindexing/coordinator_spec.rb +++ b/spec/lib/gitlab/database/reindexing/coordinator_spec.rb @@ -2,16 +2,18 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::Reindexing::Coordinator do +RSpec.describe Gitlab::Database::Reindexing::Coordinator, feature_category: :database do include Database::DatabaseHelpers include ExclusiveLeaseHelpers - let(:notifier) { instance_double(Gitlab::Database::Reindexing::GrafanaNotifier, notify_start: nil, notify_end: nil) } let(:index) { create(:postgres_index) } let(:connection) { index.connection } + let(:notifier) do + instance_double(Gitlab::Database::Reindexing::GrafanaNotifier, notify_start: nil, notify_end: nil) + end let!(:lease) { stub_exclusive_lease(lease_key, uuid, timeout: lease_timeout) } - let(:lease_key) { "gitlab/database/reindexing/coordinator/#{Gitlab::Database::PRIMARY_DATABASE_NAME}" } + let(:lease_key) { "gitlab/database/indexing/actions/#{Gitlab::Database::PRIMARY_DATABASE_NAME}" } let(:lease_timeout) { 1.day } let(:uuid) { 'uuid' } @@ -19,75 +21,83 @@ RSpec.describe Gitlab::Database::Reindexing::Coordinator do model = Gitlab::Database.database_base_models[Gitlab::Database::PRIMARY_DATABASE_NAME] Gitlab::Database::SharedModel.using_connection(model.connection) do + swapout_view_for_table(:postgres_indexes, connection: model.connection) example.run end end - before do - swapout_view_for_table(:postgres_indexes) - end - describe '#perform' do subject { described_class.new(index, notifier).perform } let(:reindexer) { instance_double(Gitlab::Database::Reindexing::ReindexConcurrently, perform: nil) } let(:action) { create(:reindex_action, index: index) } - before do - allow(Gitlab::Database::Reindexing::ReindexConcurrently).to receive(:new).with(index).and_return(reindexer) - allow(Gitlab::Database::Reindexing::ReindexAction).to receive(:create_for).with(index).and_return(action) - end + context 'when executed during the weekend', time_travel_to: '2023-01-07T09:44:07Z' do + before do + allow(Gitlab::Database::Reindexing::ReindexConcurrently).to receive(:new).with(index).and_return(reindexer) + allow(Gitlab::Database::Reindexing::ReindexAction).to receive(:create_for).with(index).and_return(action) + end - context 'locking' do - it 'acquires a lock while reindexing' do - expect(lease).to receive(:try_obtain).ordered.and_return(uuid) + context 'locking' do + it 'acquires a lock while reindexing' do + expect(lease).to receive(:try_obtain).ordered.and_return(uuid) - expect(reindexer).to receive(:perform).ordered + expect(reindexer).to receive(:perform).ordered - expect(Gitlab::ExclusiveLease).to receive(:cancel).ordered.with(lease_key, uuid) + expect(Gitlab::ExclusiveLease).to receive(:cancel).ordered.with(lease_key, uuid) - subject - end + subject + end - it 'does not perform reindexing actions if lease is not granted' do - expect(lease).to receive(:try_obtain).ordered.and_return(false) - expect(Gitlab::Database::Reindexing::ReindexConcurrently).not_to receive(:new) + it 'does not perform reindexing actions if lease is not granted' do + expect(lease).to receive(:try_obtain).ordered.and_return(false) + expect(Gitlab::Database::Reindexing::ReindexConcurrently).not_to receive(:new) - subject + subject + end end - end - context 'notifications' do - it 'sends #notify_start before reindexing' do - expect(notifier).to receive(:notify_start).with(action).ordered - expect(reindexer).to receive(:perform).ordered + context 'notifications' do + it 'sends #notify_start before reindexing' do + expect(notifier).to receive(:notify_start).with(action).ordered + expect(reindexer).to receive(:perform).ordered - subject - end + subject + end - it 'sends #notify_end after reindexing and updating the action is done' do - expect(action).to receive(:finish).ordered - expect(notifier).to receive(:notify_end).with(action).ordered + it 'sends #notify_end after reindexing and updating the action is done' do + expect(action).to receive(:finish).ordered + expect(notifier).to receive(:notify_end).with(action).ordered - subject + subject + end end - end - context 'action tracking' do - it 'calls #finish on the action' do - expect(reindexer).to receive(:perform).ordered - expect(action).to receive(:finish).ordered + context 'action tracking' do + it 'calls #finish on the action' do + expect(reindexer).to receive(:perform).ordered + expect(action).to receive(:finish).ordered - subject - end + subject + end - it 'upon error, it still calls finish and raises the error' do - expect(reindexer).to receive(:perform).ordered.and_raise('something went wrong') - expect(action).to receive(:finish).ordered + it 'upon error, it still calls finish and raises the error' do + expect(reindexer).to receive(:perform).ordered.and_raise('something went wrong') + expect(action).to receive(:finish).ordered - expect { subject }.to raise_error(/something went wrong/) + expect { subject }.to raise_error(/something went wrong/) - expect(action).to be_failed + expect(action).to be_failed + end + end + end + + context 'when executed during the week', time_travel_to: '2023-01-09T09:44:07Z' do + it 'does not start reindexing' do + expect(lease).not_to receive(:try_obtain) + expect(Gitlab::Database::Reindexing::ReindexConcurrently).not_to receive(:new) + + expect(subject).to be_nil end end end @@ -97,33 +107,45 @@ RSpec.describe Gitlab::Database::Reindexing::Coordinator do subject(:drop) { described_class.new(index, notifier).drop } - context 'when exclusive lease is granted' do - it 'drops the index with lock retries' do - expect(lease).to receive(:try_obtain).ordered.and_return(uuid) + context 'when executed during the weekend', time_travel_to: '2023-01-07T09:44:07Z' do + context 'when exclusive lease is granted' do + it 'drops the index with lock retries' do + expect(lease).to receive(:try_obtain).ordered.and_return(uuid) + + expect_query("SET lock_timeout TO '60000ms'") + expect_query("DROP INDEX CONCURRENTLY IF EXISTS \"public\".\"#{index.name}\"") + expect_query("RESET idle_in_transaction_session_timeout; RESET lock_timeout") - expect_query("SET lock_timeout TO '60000ms'") - expect_query("DROP INDEX CONCURRENTLY IF EXISTS \"public\".\"#{index.name}\"") - expect_query("RESET idle_in_transaction_session_timeout; RESET lock_timeout") + expect(Gitlab::ExclusiveLease).to receive(:cancel).ordered.with(lease_key, uuid) - expect(Gitlab::ExclusiveLease).to receive(:cancel).ordered.with(lease_key, uuid) + drop + end - drop + def expect_query(sql) + expect(connection).to receive(:execute).ordered.with(sql).and_wrap_original do |method, sql| + method.call(sql.sub(/CONCURRENTLY/, '')) + end + end end - def expect_query(sql) - expect(connection).to receive(:execute).ordered.with(sql).and_wrap_original do |method, sql| - method.call(sql.sub(/CONCURRENTLY/, '')) + context 'when exclusive lease is not granted' do + it 'does not drop the index' do + expect(lease).to receive(:try_obtain).ordered.and_return(false) + expect(Gitlab::Database::WithLockRetriesOutsideTransaction).not_to receive(:new) + expect(connection).not_to receive(:execute) + + drop end end end - context 'when exclusive lease is not granted' do - it 'does not drop the index' do - expect(lease).to receive(:try_obtain).ordered.and_return(false) + context 'when executed during the week', time_travel_to: '2023-01-09T09:44:07Z' do + it 'does not start reindexing' do + expect(lease).not_to receive(:try_obtain) expect(Gitlab::Database::WithLockRetriesOutsideTransaction).not_to receive(:new) expect(connection).not_to receive(:execute) - drop + expect(drop).to be_nil end end end diff --git a/spec/lib/gitlab/database/reindexing/grafana_notifier_spec.rb b/spec/lib/gitlab/database/reindexing/grafana_notifier_spec.rb index 1bccdda3be1..e67c97cbf9c 100644 --- a/spec/lib/gitlab/database/reindexing/grafana_notifier_spec.rb +++ b/spec/lib/gitlab/database/reindexing/grafana_notifier_spec.rb @@ -12,7 +12,7 @@ RSpec.describe Gitlab::Database::Reindexing::GrafanaNotifier do let(:action) { create(:reindex_action) } before do - swapout_view_for_table(:postgres_indexes) + swapout_view_for_table(:postgres_indexes, connection: ApplicationRecord.connection) end let(:headers) do @@ -25,7 +25,9 @@ RSpec.describe Gitlab::Database::Reindexing::GrafanaNotifier do let(:response) { double('response', success?: true) } def expect_api_call(payload) - expect(Gitlab::HTTP).to receive(:post).with("#{api_url}/api/annotations", body: payload.to_json, headers: headers, allow_local_requests: true).and_return(response) + expect(Gitlab::HTTP).to receive(:post).with( + "#{api_url}/api/annotations", body: payload.to_json, headers: headers, allow_local_requests: true + ).and_return(response) end shared_examples_for 'interacting with Grafana annotations API' do @@ -109,7 +111,9 @@ RSpec.describe Gitlab::Database::Reindexing::GrafanaNotifier do end context 'additional tag is provided' do - subject { described_class.new(api_key: api_key, api_url: api_url, additional_tag: additional_tag).notify_start(action) } + subject do + described_class.new(api_key: api_key, api_url: api_url, additional_tag: additional_tag).notify_start(action) + end let(:payload) do { @@ -163,7 +167,9 @@ RSpec.describe Gitlab::Database::Reindexing::GrafanaNotifier do end context 'additional tag is provided' do - subject { described_class.new(api_key: api_key, api_url: api_url, additional_tag: additional_tag).notify_end(action) } + subject do + described_class.new(api_key: api_key, api_url: api_url, additional_tag: additional_tag).notify_end(action) + end let(:payload) do { diff --git a/spec/lib/gitlab/database/reindexing/index_selection_spec.rb b/spec/lib/gitlab/database/reindexing/index_selection_spec.rb index 2ae9037959d..e82a2ab467d 100644 --- a/spec/lib/gitlab/database/reindexing/index_selection_spec.rb +++ b/spec/lib/gitlab/database/reindexing/index_selection_spec.rb @@ -2,14 +2,16 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::Reindexing::IndexSelection do +RSpec.describe Gitlab::Database::Reindexing::IndexSelection, feature_category: :database do include Database::DatabaseHelpers subject { described_class.new(Gitlab::Database::PostgresIndex.all).to_a } + let(:connection) { ApplicationRecord.connection } + before do - swapout_view_for_table(:postgres_index_bloat_estimates) - swapout_view_for_table(:postgres_indexes) + swapout_view_for_table(:postgres_index_bloat_estimates, connection: connection) + swapout_view_for_table(:postgres_indexes, connection: connection) create_list(:postgres_index, 10, ondisk_size_bytes: 10.gigabytes).each_with_index do |index, i| create(:postgres_index_bloat_estimate, index: index, bloat_size_bytes: 2.gigabyte * (i + 1)) @@ -17,7 +19,7 @@ RSpec.describe Gitlab::Database::Reindexing::IndexSelection do end def execute(sql) - ActiveRecord::Base.connection.execute(sql) + connection.execute(sql) end it 'orders by highest relative bloat first' do @@ -74,4 +76,30 @@ RSpec.describe Gitlab::Database::Reindexing::IndexSelection do expect(subject.map(&:name).sort).to eq(not_recently_reindexed.map(&:name).sort) end end + + context 'with restricted tables' do + let!(:ci_builds) do + create( + :postgres_index_bloat_estimate, + index: create(:postgres_index, ondisk_size_bytes: 100.gigabytes, tablename: 'ci_builds'), + bloat_size_bytes: 20.gigabyte + ) + end + + context 'when executed on Fridays', time_travel_to: '2022-12-16T09:44:07Z' do + it { expect(subject).not_to include(ci_builds.index) } + end + + context 'when executed on Saturdays', time_travel_to: '2022-12-17T09:44:07Z' do + it { expect(subject).to include(ci_builds.index) } + end + + context 'when executed on Sundays', time_travel_to: '2022-12-18T09:44:07Z' do + it { expect(subject).not_to include(ci_builds.index) } + end + + context 'when executed on Mondays', time_travel_to: '2022-12-19T09:44:07Z' do + it { expect(subject).not_to include(ci_builds.index) } + end + end end diff --git a/spec/lib/gitlab/database/reindexing/reindex_action_spec.rb b/spec/lib/gitlab/database/reindexing/reindex_action_spec.rb index 1b409924acc..06b89e08737 100644 --- a/spec/lib/gitlab/database/reindexing/reindex_action_spec.rb +++ b/spec/lib/gitlab/database/reindexing/reindex_action_spec.rb @@ -2,13 +2,13 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::Reindexing::ReindexAction do +RSpec.describe Gitlab::Database::Reindexing::ReindexAction, feature_category: :database do include Database::DatabaseHelpers let(:index) { create(:postgres_index) } before_all do - swapout_view_for_table(:postgres_indexes) + swapout_view_for_table(:postgres_indexes, connection: ApplicationRecord.connection) end it { is_expected.to be_a Gitlab::Database::SharedModel } diff --git a/spec/lib/gitlab/database/reindexing_spec.rb b/spec/lib/gitlab/database/reindexing_spec.rb index fa26aa59329..6575c92e313 100644 --- a/spec/lib/gitlab/database/reindexing_spec.rb +++ b/spec/lib/gitlab/database/reindexing_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::Reindexing, feature_category: :database do +RSpec.describe Gitlab::Database::Reindexing, feature_category: :database, time_travel_to: '2023-01-07T09:44:07Z' do include ExclusiveLeaseHelpers include Database::DatabaseHelpers @@ -76,7 +76,7 @@ RSpec.describe Gitlab::Database::Reindexing, feature_category: :database do let(:limit) { 5 } before_all do - swapout_view_for_table(:postgres_indexes) + swapout_view_for_table(:postgres_indexes, connection: ApplicationRecord.connection) end before do @@ -147,7 +147,7 @@ RSpec.describe Gitlab::Database::Reindexing, feature_category: :database do subject { described_class.perform_from_queue(maximum_records: limit) } before_all do - swapout_view_for_table(:postgres_indexes) + swapout_view_for_table(:postgres_indexes, connection: ApplicationRecord.connection) end let(:limit) { 2 } diff --git a/spec/lib/gitlab/database/tables_truncate_spec.rb b/spec/lib/gitlab/database/tables_truncate_spec.rb index 4d04bd67a1e..9af0b964221 100644 --- a/spec/lib/gitlab/database/tables_truncate_spec.rb +++ b/spec/lib/gitlab/database/tables_truncate_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::Database::TablesTruncate, :reestablished_active_record_base, - :suppress_gitlab_schemas_validate_connection do + :suppress_gitlab_schemas_validate_connection, feature_category: :pods do include MigrationsHelpers let(:min_batch_size) { 1 } @@ -18,7 +18,7 @@ RSpec.describe Gitlab::Database::TablesTruncate, :reestablished_active_record_ba let(:main_db_shared_item_model) { table("_test_gitlab_shared_items", database: "main") } let(:main_db_partitioned_item) { table("_test_gitlab_hook_logs", database: "main") } let(:main_db_partitioned_item_detached) do - table("gitlab_partitions_dynamic._test_gitlab_hook_logs_20220101", database: "main") + table("gitlab_partitions_dynamic._test_gitlab_hook_logs_202201", database: "main") end # CI Database @@ -29,7 +29,7 @@ RSpec.describe Gitlab::Database::TablesTruncate, :reestablished_active_record_ba let(:ci_db_shared_item_model) { table("_test_gitlab_shared_items", database: "ci") } let(:ci_db_partitioned_item) { table("_test_gitlab_hook_logs", database: "ci") } let(:ci_db_partitioned_item_detached) do - table("gitlab_partitions_dynamic._test_gitlab_hook_logs_20220101", database: "ci") + table("gitlab_partitions_dynamic._test_gitlab_hook_logs_202201", database: "ci") end shared_examples 'truncating legacy tables on a database' do @@ -64,19 +64,19 @@ RSpec.describe Gitlab::Database::TablesTruncate, :reestablished_active_record_ba id bigserial not null, created_at timestamptz not null, item_id BIGINT NOT NULL, - primary key (id, created_at), + PRIMARY KEY (id, created_at), CONSTRAINT fk_constrained_1 FOREIGN KEY(item_id) REFERENCES _test_gitlab_main_items(id) ) PARTITION BY RANGE(created_at); - CREATE TABLE gitlab_partitions_dynamic._test_gitlab_hook_logs_20220101 + CREATE TABLE gitlab_partitions_dynamic._test_gitlab_hook_logs_202201 PARTITION OF _test_gitlab_hook_logs FOR VALUES FROM ('20220101') TO ('20220131'); - CREATE TABLE gitlab_partitions_dynamic._test_gitlab_hook_logs_20220201 + CREATE TABLE gitlab_partitions_dynamic._test_gitlab_hook_logs_202202 PARTITION OF _test_gitlab_hook_logs FOR VALUES FROM ('20220201') TO ('20220228'); - ALTER TABLE _test_gitlab_hook_logs DETACH PARTITION gitlab_partitions_dynamic._test_gitlab_hook_logs_20220101; + ALTER TABLE _test_gitlab_hook_logs DETACH PARTITION gitlab_partitions_dynamic._test_gitlab_hook_logs_202201; SQL main_connection.execute(main_tables_sql) @@ -124,14 +124,14 @@ RSpec.describe Gitlab::Database::TablesTruncate, :reestablished_active_record_ba Gitlab::Database::SharedModel.using_connection(main_connection) do Postgresql::DetachedPartition.create!( - table_name: '_test_gitlab_hook_logs_20220101', + table_name: '_test_gitlab_hook_logs_202201', drop_after: Time.current ) end Gitlab::Database::SharedModel.using_connection(ci_connection) do Postgresql::DetachedPartition.create!( - table_name: '_test_gitlab_hook_logs_20220101', + table_name: '_test_gitlab_hook_logs_202201', drop_after: Time.current ) end @@ -176,7 +176,8 @@ RSpec.describe Gitlab::Database::TablesTruncate, :reestablished_active_record_ba Gitlab::Database::LockWritesManager.new( table_name: table, connection: connection, - database_name: connection.pool.db_config.name + database_name: connection.pool.db_config.name, + with_retries: false ).lock_writes end end @@ -236,6 +237,25 @@ RSpec.describe Gitlab::Database::TablesTruncate, :reestablished_active_record_ba end end + context 'when one of the attached partitions happened to be locked for writes' do + before do + skip if connection.pool.db_config.name != 'ci' + + Gitlab::Database::LockWritesManager.new( + table_name: "#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}._test_gitlab_hook_logs_202202", + connection: connection, + database_name: connection.pool.db_config.name, + with_retries: false + ).lock_writes + end + + it 'truncates the locked partition successfully' do + expect do + truncate_legacy_tables + end.to change { ci_db_partitioned_item.count }.from(5).to(0) + end + end + context 'with geo configured' do let(:geo_connection) { Gitlab::Database.database_base_models[:geo].connection } diff --git a/spec/lib/gitlab/database_spec.rb b/spec/lib/gitlab/database_spec.rb index 1a482b33a92..86bc8e71fd7 100644 --- a/spec/lib/gitlab/database_spec.rb +++ b/spec/lib/gitlab/database_spec.rb @@ -302,6 +302,26 @@ RSpec.describe Gitlab::Database do end end + describe '.database_base_models_with_gitlab_shared' do + before do + Gitlab::Database.instance_variable_set(:@database_base_models_with_gitlab_shared, nil) + end + + it 'memoizes the models' do + expect { Gitlab::Database.database_base_models_with_gitlab_shared }.to change { Gitlab::Database.instance_variable_get(:@database_base_models_with_gitlab_shared) }.from(nil) + end + end + + describe '.database_base_models_using_load_balancing' do + before do + Gitlab::Database.instance_variable_set(:@database_base_models_using_load_balancing, nil) + end + + it 'memoizes the models' do + expect { Gitlab::Database.database_base_models_using_load_balancing }.to change { Gitlab::Database.instance_variable_get(:@database_base_models_using_load_balancing) }.from(nil) + end + end + describe '#true_value' do it 'returns correct value' do expect(described_class.true_value).to eq "'t'" diff --git a/spec/lib/gitlab/diff/file_collection/merge_request_diff_base_spec.rb b/spec/lib/gitlab/diff/file_collection/merge_request_diff_base_spec.rb index 51bee6d45e4..861852d8f0b 100644 --- a/spec/lib/gitlab/diff/file_collection/merge_request_diff_base_spec.rb +++ b/spec/lib/gitlab/diff/file_collection/merge_request_diff_base_spec.rb @@ -26,6 +26,17 @@ RSpec.describe Gitlab::Diff::FileCollection::MergeRequestDiffBase do end end + describe '#diff_files' do + subject(:diff_files) { described_class.new(diffable, diff_options: nil).diff_files } + + it 'measures diffs_highlight_cache_decorate' do + allow(Gitlab::Metrics).to receive(:measure).and_call_original + expect(Gitlab::Metrics).to receive(:measure).with(:diffs_highlight_cache_decorate).and_call_original + + diff_files + end + end + describe '#cache_key' do subject(:cache_key) { described_class.new(diffable, diff_options: nil).cache_key } diff --git a/spec/lib/gitlab/diff/file_collection/merge_request_diff_batch_spec.rb b/spec/lib/gitlab/diff/file_collection/merge_request_diff_batch_spec.rb index 9ac242459bf..8e14f48ae29 100644 --- a/spec/lib/gitlab/diff/file_collection/merge_request_diff_batch_spec.rb +++ b/spec/lib/gitlab/diff/file_collection/merge_request_diff_batch_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Diff::FileCollection::MergeRequestDiffBatch, feature_category: :code_review do +RSpec.describe Gitlab::Diff::FileCollection::MergeRequestDiffBatch, feature_category: :code_review_workflow do let(:merge_request) { create(:merge_request) } let(:batch_page) { 0 } let(:batch_size) { 10 } diff --git a/spec/lib/gitlab/diff/file_collection/paginated_merge_request_diff_spec.rb b/spec/lib/gitlab/diff/file_collection/paginated_merge_request_diff_spec.rb index 74e5e667702..ee956d04325 100644 --- a/spec/lib/gitlab/diff/file_collection/paginated_merge_request_diff_spec.rb +++ b/spec/lib/gitlab/diff/file_collection/paginated_merge_request_diff_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Diff::FileCollection::PaginatedMergeRequestDiff, feature_category: :code_review do +RSpec.describe Gitlab::Diff::FileCollection::PaginatedMergeRequestDiff, feature_category: :code_review_workflow do let(:merge_request) { create(:merge_request) } let(:page) { 1 } let(:per_page) { 10 } diff --git a/spec/lib/gitlab/error_tracking_spec.rb b/spec/lib/gitlab/error_tracking_spec.rb index 4900547e9e9..5eedd716a4a 100644 --- a/spec/lib/gitlab/error_tracking_spec.rb +++ b/spec/lib/gitlab/error_tracking_spec.rb @@ -154,6 +154,32 @@ RSpec.describe Gitlab::ErrorTracking do end end + describe '.log_and_raise_exception' do + subject(:log_and_raise_exception) do + described_class.log_and_raise_exception(exception, extra) + end + + it 'only logs and raises the exception' do + expect(Raven).not_to receive(:capture_exception) + expect(Sentry).not_to receive(:capture_exception) + expect(Gitlab::ErrorTracking::Logger).to receive(:error).with(logger_payload) + + expect { log_and_raise_exception }.to raise_error(RuntimeError) + end + + context 'when extra details are provided' do + let(:extra) { { test: 1, my_token: 'test' } } + + it 'filters parameters' do + expect(Gitlab::ErrorTracking::Logger).to receive(:error).with( + hash_including({ 'extra.test' => 1, 'extra.my_token' => '[FILTERED]' }) + ) + + expect { log_and_raise_exception }.to raise_error(RuntimeError) + end + end + end + describe '.track_exception' do subject(:track_exception) do described_class.track_exception(exception, extra) diff --git a/spec/lib/gitlab/gitaly_client/ref_service_spec.rb b/spec/lib/gitlab/gitaly_client/ref_service_spec.rb index ae2e343377d..14d5cef103b 100644 --- a/spec/lib/gitlab/gitaly_client/ref_service_spec.rb +++ b/spec/lib/gitlab/gitaly_client/ref_service_spec.rb @@ -409,17 +409,6 @@ RSpec.describe Gitlab::GitalyClient::RefService do end end - describe '#pack_refs' do - it 'sends a pack_refs message' do - expect_any_instance_of(Gitaly::RefService::Stub) - .to receive(:pack_refs) - .with(gitaly_request_with_path(storage_name, relative_path), kind_of(Hash)) - .and_return(double(:pack_refs_response)) - - client.pack_refs - end - end - describe '#find_refs_by_oid' do let(:oid) { project.repository.commit.id } diff --git a/spec/lib/gitlab/gitaly_client/repository_service_spec.rb b/spec/lib/gitlab/gitaly_client/repository_service_spec.rb index 5aef250afac..5eb60d2caa5 100644 --- a/spec/lib/gitlab/gitaly_client/repository_service_spec.rb +++ b/spec/lib/gitlab/gitaly_client/repository_service_spec.rb @@ -21,39 +21,6 @@ RSpec.describe Gitlab::GitalyClient::RepositoryService do end end - describe '#garbage_collect' do - it 'sends a garbage_collect message' do - expect_any_instance_of(Gitaly::RepositoryService::Stub) - .to receive(:garbage_collect) - .with(gitaly_request_with_path(storage_name, relative_path), kind_of(Hash)) - .and_return(double(:garbage_collect_response)) - - client.garbage_collect(true, prune: true) - end - end - - describe '#repack_full' do - it 'sends a repack_full message' do - expect_any_instance_of(Gitaly::RepositoryService::Stub) - .to receive(:repack_full) - .with(gitaly_request_with_path(storage_name, relative_path), kind_of(Hash)) - .and_return(double(:repack_full_response)) - - client.repack_full(true) - end - end - - describe '#repack_incremental' do - it 'sends a repack_incremental message' do - expect_any_instance_of(Gitaly::RepositoryService::Stub) - .to receive(:repack_incremental) - .with(gitaly_request_with_path(storage_name, relative_path), kind_of(Hash)) - .and_return(double(:repack_incremental_response)) - - client.repack_incremental - end - end - describe '#optimize_repository' do it 'sends a optimize_repository message' do expect_any_instance_of(Gitaly::RepositoryService::Stub) diff --git a/spec/lib/gitlab/gitaly_client_spec.rb b/spec/lib/gitlab/gitaly_client_spec.rb index 3d33bf93c23..f5e75242f40 100644 --- a/spec/lib/gitlab/gitaly_client_spec.rb +++ b/spec/lib/gitlab/gitaly_client_spec.rb @@ -4,13 +4,19 @@ require 'spec_helper' # We stub Gitaly in `spec/support/gitaly.rb` for other tests. We don't want # those stubs while testing the GitalyClient itself. -RSpec.describe Gitlab::GitalyClient do +RSpec.describe Gitlab::GitalyClient, feature_category: :gitaly do def stub_repos_storages(address) allow(Gitlab.config.repositories).to receive(:storages).and_return({ 'default' => { 'gitaly_address' => address } }) end + around do |example| + described_class.clear_stubs! + example.run + described_class.clear_stubs! + end + describe '.query_time', :request_store do it 'increments query times' do subject.add_query_time(0.4510004) @@ -157,45 +163,131 @@ RSpec.describe Gitlab::GitalyClient do end end + describe '.create_channel' do + where(:storage, :address, :expected_target) do + [ + ['default', 'unix:tmp/gitaly.sock', 'unix:tmp/gitaly.sock'], + ['default', 'tcp://localhost:9876', 'localhost:9876'], + ['default', 'tls://localhost:9876', 'localhost:9876'] + ] + end + + with_them do + before do + allow(Gitlab.config.repositories).to receive(:storages).and_return( + 'default' => { 'gitaly_address' => address }, + 'other' => { 'gitaly_address' => address } + ) + end + + it 'creates channel based on storage' do + channel = described_class.create_channel(storage) + + expect(channel).to be_a(GRPC::Core::Channel) + expect(channel.target).to eql(expected_target) + end + + it 'caches channel based on storage' do + channel_1 = described_class.create_channel(storage) + channel_2 = described_class.create_channel(storage) + + expect(channel_1).to equal(channel_2) + end + + it 'returns different channels for different storages' do + channel_1 = described_class.create_channel(storage) + channel_2 = described_class.create_channel('other') + + expect(channel_1).not_to equal(channel_2) + end + end + end + describe '.stub' do - # Notice that this is referring to gRPC "stubs", not rspec stubs - before do - described_class.clear_stubs! + matcher :be_a_grpc_channel do |expected_address| + match { |actual| actual.is_a?(::GRPC::Core::Channel) && actual.target == expected_address } + end + + matcher :have_same_channel do |expected| + match do |actual| + # gRPC client stub does not expose the underlying channel. We need a way + # to verify two stubs have the same channel. So, no way around. + expected_channel = expected.instance_variable_get(:@ch) + actual_channel = actual.instance_variable_get(:@ch) + expected_channel.is_a?(GRPC::Core::Channel) && + actual_channel.is_a?(GRPC::Core::Channel) && + expected_channel == actual_channel + end end context 'when passed a UNIX socket address' do - it 'passes the address as-is to GRPC' do - address = 'unix:/tmp/gitaly.sock' - stub_repos_storages address + let(:address) { 'unix:/tmp/gitaly.sock' } - expect(Gitaly::CommitService::Stub).to receive(:new).with(address, any_args) + before do + stub_repos_storages address + end + it 'passes the address as-is to GRPC' do + expect(Gitaly::CommitService::Stub).to receive(:new).with( + address, nil, channel_override: be_a_grpc_channel(address), interceptors: [] + ) described_class.stub(:commit_service, 'default') end + + it 'shares the same channel object with other stub' do + stub_commit = described_class.stub(:commit_service, 'default') + stub_blob = described_class.stub(:blob_service, 'default') + + expect(stub_commit).to have_same_channel(stub_blob) + end end context 'when passed a TLS address' do - it 'strips tls:// prefix before passing it to GRPC::Core::Channel initializer' do - address = 'localhost:9876' + let(:address) { 'localhost:9876' } + + before do prefixed_address = "tls://#{address}" stub_repos_storages prefixed_address + end - expect(Gitaly::CommitService::Stub).to receive(:new).with(address, any_args) + it 'strips tls:// prefix before passing it to GRPC::Core::Channel initializer' do + expect(Gitaly::CommitService::Stub).to receive(:new).with( + address, nil, channel_override: be_a(GRPC::Core::Channel), interceptors: [] + ) described_class.stub(:commit_service, 'default') end + + it 'shares the same channel object with other stub' do + stub_commit = described_class.stub(:commit_service, 'default') + stub_blob = described_class.stub(:blob_service, 'default') + + expect(stub_commit).to have_same_channel(stub_blob) + end end context 'when passed a TCP address' do - it 'strips tcp:// prefix before passing it to GRPC::Core::Channel initializer' do - address = 'localhost:9876' + let(:address) { 'localhost:9876' } + + before do prefixed_address = "tcp://#{address}" stub_repos_storages prefixed_address + end - expect(Gitaly::CommitService::Stub).to receive(:new).with(address, any_args) + it 'strips tcp:// prefix before passing it to GRPC::Core::Channel initializer' do + expect(Gitaly::CommitService::Stub).to receive(:new).with( + address, nil, channel_override: be_a(GRPC::Core::Channel), interceptors: [] + ) described_class.stub(:commit_service, 'default') end + + it 'shares the same channel object with other stub' do + stub_commit = described_class.stub(:commit_service, 'default') + stub_blob = described_class.stub(:blob_service, 'default') + + expect(stub_commit).to have_same_channel(stub_blob) + end end end diff --git a/spec/lib/gitlab/github_gists_import/importer/gist_importer_spec.rb b/spec/lib/gitlab/github_gists_import/importer/gist_importer_spec.rb index 69a4d646562..6bfbfbdeddf 100644 --- a/spec/lib/gitlab/github_gists_import/importer/gist_importer_spec.rb +++ b/spec/lib/gitlab/github_gists_import/importer/gist_importer_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::GithubGistsImport::Importer::GistImporter, feature_category: :importer do +RSpec.describe Gitlab::GithubGistsImport::Importer::GistImporter, feature_category: :importers do subject { described_class.new(gist_object, user.id).execute } let_it_be(:user) { create(:user) } @@ -63,7 +63,7 @@ RSpec.describe Gitlab::GithubGistsImport::Importer::GistImporter, feature_catego expect(user.snippets.count).to eq(0) expect(result.error?).to eq(true) - expect(result.errors).to match_array(['Snippet max file count exceeded']) + expect(result.errors).to match_array(['Snippet maximum file count exceeded']) end end diff --git a/spec/lib/gitlab/github_gists_import/importer/gists_importer_spec.rb b/spec/lib/gitlab/github_gists_import/importer/gists_importer_spec.rb index 704999a99a9..d555a847ea5 100644 --- a/spec/lib/gitlab/github_gists_import/importer/gists_importer_spec.rb +++ b/spec/lib/gitlab/github_gists_import/importer/gists_importer_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::GithubGistsImport::Importer::GistsImporter, feature_category: :importer do +RSpec.describe Gitlab::GithubGistsImport::Importer::GistsImporter, feature_category: :importers do subject(:result) { described_class.new(user, token).execute } let_it_be(:user) { create(:user) } diff --git a/spec/lib/gitlab/github_gists_import/representation/gist_spec.rb b/spec/lib/gitlab/github_gists_import/representation/gist_spec.rb index 480aefb2c74..d6b47a1e837 100644 --- a/spec/lib/gitlab/github_gists_import/representation/gist_spec.rb +++ b/spec/lib/gitlab/github_gists_import/representation/gist_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::GithubGistsImport::Representation::Gist, feature_category: :importer do +RSpec.describe Gitlab::GithubGistsImport::Representation::Gist, feature_category: :importers do shared_examples 'a Gist' do it 'returns an instance of Gist' do expect(gist).to be_an_instance_of(described_class) diff --git a/spec/lib/gitlab/github_gists_import/status_spec.rb b/spec/lib/gitlab/github_gists_import/status_spec.rb index 4cbbbd430eb..d2016ef0248 100644 --- a/spec/lib/gitlab/github_gists_import/status_spec.rb +++ b/spec/lib/gitlab/github_gists_import/status_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::GithubGistsImport::Status, :clean_gitlab_redis_cache, feature_category: :importer do +RSpec.describe Gitlab::GithubGistsImport::Status, :clean_gitlab_redis_cache, feature_category: :importers do subject(:import_status) { described_class.new(user.id) } let_it_be(:user) { create(:user) } diff --git a/spec/lib/gitlab/github_import/bulk_importing_spec.rb b/spec/lib/gitlab/github_import/bulk_importing_spec.rb index af31cb6c873..136ddb566aa 100644 --- a/spec/lib/gitlab/github_import/bulk_importing_spec.rb +++ b/spec/lib/gitlab/github_import/bulk_importing_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::GithubImport::BulkImporting, feature_category: :importer do +RSpec.describe Gitlab::GithubImport::BulkImporting, feature_category: :importers do let(:project) { instance_double(Project, id: 1) } let(:importer) { MyImporter.new(project, double) } let(:importer_class) do diff --git a/spec/lib/gitlab/github_import/client_spec.rb b/spec/lib/gitlab/github_import/client_spec.rb index 526a8721ff3..d69bc4d60ee 100644 --- a/spec/lib/gitlab/github_import/client_spec.rb +++ b/spec/lib/gitlab/github_import/client_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::GithubImport::Client do +RSpec.describe Gitlab::GithubImport::Client, feature_category: :importer do subject(:client) { described_class.new('foo', parallel: parallel) } let(:parallel) { true } @@ -614,6 +614,46 @@ RSpec.describe Gitlab::GithubImport::Client do client.search_repos_by_name_graphql('test') end + context 'when relation type option present' do + context 'when relation type is owned' do + let(:expected_query) { 'test in:name is:public,private user:user' } + + it 'searches for repositories within the organization based on name' do + expect(client.octokit).to receive(:post).with( + '/graphql', { query: expected_graphql }.to_json + ) + + client.search_repos_by_name_graphql('test', relation_type: 'owned') + end + end + + context 'when relation type is organization' do + let(:expected_query) { 'test in:name is:public,private org:test-login' } + + it 'searches for repositories within the organization based on name' do + expect(client.octokit).to receive(:post).with( + '/graphql', { query: expected_graphql }.to_json + ) + + client.search_repos_by_name_graphql( + 'test', relation_type: 'organization', organization_login: 'test-login' + ) + end + end + + context 'when relation type is collaborated' do + let(:expected_query) { 'test in:name is:public,private repo:repo1 repo:repo2' } + + it 'searches for collaborated repositories based on name' do + expect(client.octokit).to receive(:post).with( + '/graphql', { query: expected_graphql }.to_json + ) + + client.search_repos_by_name_graphql('test', relation_type: 'collaborated') + end + end + end + context 'when pagination options present' do context 'with "first" option' do let(:expected_graphql_params) do diff --git a/spec/lib/gitlab/github_import/importer/labels_importer_spec.rb b/spec/lib/gitlab/github_import/importer/labels_importer_spec.rb index ad9ef4afddd..9e295ab215a 100644 --- a/spec/lib/gitlab/github_import/importer/labels_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/labels_importer_spec.rb @@ -2,7 +2,8 @@ require 'spec_helper' -RSpec.describe Gitlab::GithubImport::Importer::LabelsImporter, :clean_gitlab_redis_cache, feature_category: :importer do +RSpec.describe Gitlab::GithubImport::Importer::LabelsImporter, :clean_gitlab_redis_cache, +feature_category: :importers do let(:project) { create(:project, import_source: 'foo/bar') } let(:client) { double(:client) } let(:importer) { described_class.new(project, client) } diff --git a/spec/lib/gitlab/github_import/importer/milestones_importer_spec.rb b/spec/lib/gitlab/github_import/importer/milestones_importer_spec.rb index 8667729d79b..47b9a41c364 100644 --- a/spec/lib/gitlab/github_import/importer/milestones_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/milestones_importer_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::GithubImport::Importer::MilestonesImporter, :clean_gitlab_redis_cache, - feature_category: :importer do + feature_category: :importers do let(:project) { create(:project, import_source: 'foo/bar') } let(:client) { double(:client) } let(:importer) { described_class.new(project, client) } diff --git a/spec/lib/gitlab/github_import/importer/protected_branch_importer_spec.rb b/spec/lib/gitlab/github_import/importer/protected_branch_importer_spec.rb index d6b7411e640..d999bb3a3a3 100644 --- a/spec/lib/gitlab/github_import/importer/protected_branch_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/protected_branch_importer_spec.rb @@ -15,6 +15,9 @@ RSpec.describe Gitlab::GithubImport::Importer::ProtectedBranchImporter do let(:expected_merge_access_level) { Gitlab::Access::MAINTAINER } let(:expected_allow_force_push) { false } let(:expected_code_owner_approval_required) { false } + let(:allowed_to_push_users) { [] } + let(:push_access_levels_number) { 1 } + let(:push_access_levels_attributes) { [{ access_level: expected_push_access_level }] } let(:github_protected_branch) do Gitlab::GithubImport::Representation::ProtectedBranch.new( id: branch_name, @@ -22,7 +25,8 @@ RSpec.describe Gitlab::GithubImport::Importer::ProtectedBranchImporter do required_conversation_resolution: required_conversation_resolution, required_signatures: required_signatures, required_pull_request_reviews: required_pull_request_reviews, - require_code_owner_reviews: require_code_owner_reviews_on_github + require_code_owner_reviews: require_code_owner_reviews_on_github, + allowed_to_push_users: allowed_to_push_users ) end @@ -36,7 +40,7 @@ RSpec.describe Gitlab::GithubImport::Importer::ProtectedBranchImporter do let(:expected_ruleset) do { name: 'protection', - push_access_levels_attributes: [{ access_level: expected_push_access_level }], + push_access_levels_attributes: push_access_levels_attributes, merge_access_levels_attributes: [{ access_level: expected_merge_access_level }], allow_force_push: expected_allow_force_push, code_owner_approval_required: expected_code_owner_approval_required @@ -56,7 +60,7 @@ RSpec.describe Gitlab::GithubImport::Importer::ProtectedBranchImporter do it 'creates protected branch and access levels for given github rule' do expect { importer.execute }.to change(ProtectedBranch, :count).by(1) - .and change(ProtectedBranch::PushAccessLevel, :count).by(1) + .and change(ProtectedBranch::PushAccessLevel, :count).by(push_access_levels_number) .and change(ProtectedBranch::MergeAccessLevel, :count).by(1) end end @@ -220,10 +224,97 @@ RSpec.describe Gitlab::GithubImport::Importer::ProtectedBranchImporter do context 'when required_pull_request_reviews rule is enabled on GitHub' do let(:required_pull_request_reviews) { true } - let(:expected_push_access_level) { Gitlab::Access::NO_ACCESS } - let(:expected_merge_access_level) { Gitlab::Access::MAINTAINER } - it_behaves_like 'create branch protection by the strictest ruleset' + context 'when no user is allowed to bypass push restrictions' do + let(:expected_push_access_level) { Gitlab::Access::NO_ACCESS } + let(:expected_merge_access_level) { Gitlab::Access::MAINTAINER } + + it_behaves_like 'create branch protection by the strictest ruleset' + end + + context 'when there are users who are allowed to bypass push restrictions' do + let(:owner_id) { project.owner.id } + let(:owner_username) { project.owner.username } + let(:other_user) { create(:user) } + let(:other_user_id) { other_user.id } + let(:other_user_username) { other_user.username } + let(:allowed_to_push_users) do + [ + { id: owner_id, login: owner_username }, + { id: other_user_id, login: other_user_username } + ] + end + + context 'when the protected_refs_for_users feature is available', if: Gitlab.ee? do + let(:expected_merge_access_level) { Gitlab::Access::MAINTAINER } + + before do + stub_licensed_features(protected_refs_for_users: true) + end + + context 'when the users are found on GitLab' do + let(:push_access_levels_number) { 2 } + let(:push_access_levels_attributes) do + [ + { user_id: owner_id }, + { user_id: other_user_id } + ] + end + + before do + project.add_member(other_user, :maintainer) + allow_next_instance_of(Gitlab::GithubImport::UserFinder) do |finder| + allow(finder).to receive(:find).with(owner_id, owner_username).and_return(owner_id) + allow(finder).to receive(:find).with(other_user_id, other_user_username).and_return(other_user_id) + end + end + + it_behaves_like 'create branch protection by the strictest ruleset' + end + + context 'when one of found users is not a member of the imported project' do + let(:push_access_levels_number) { 1 } + let(:push_access_levels_attributes) do + [ + { user_id: owner_id } + ] + end + + before do + allow_next_instance_of(Gitlab::GithubImport::UserFinder) do |finder| + allow(finder).to receive(:find).with(owner_id, owner_username).and_return(owner_id) + allow(finder).to receive(:find).with(other_user_id, other_user_username).and_return(other_user_id) + end + end + + it_behaves_like 'create branch protection by the strictest ruleset' + end + + context 'when the user are not found on GitLab' do + before do + allow_next_instance_of(Gitlab::GithubImport::UserFinder) do |finder| + allow(finder).to receive(:find).and_return(nil) + end + end + + let(:expected_push_access_level) { Gitlab::Access::NO_ACCESS } + let(:expected_merge_access_level) { Gitlab::Access::MAINTAINER } + + it_behaves_like 'create branch protection by the strictest ruleset' + end + end + + context 'when the protected_refs_for_users feature is not available' do + before do + stub_licensed_features(protected_refs_for_users: false) + end + + let(:expected_push_access_level) { Gitlab::Access::NO_ACCESS } + let(:expected_merge_access_level) { Gitlab::Access::MAINTAINER } + + it_behaves_like 'create branch protection by the strictest ruleset' + end + end end context 'when required_pull_request_reviews rule is disabled on GitHub' do diff --git a/spec/lib/gitlab/github_import/importer/releases_importer_spec.rb b/spec/lib/gitlab/github_import/importer/releases_importer_spec.rb index ccbe5b5fc50..fe4d3e9d90b 100644 --- a/spec/lib/gitlab/github_import/importer/releases_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/releases_importer_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::GithubImport::Importer::ReleasesImporter, feature_category: :importer do +RSpec.describe Gitlab::GithubImport::Importer::ReleasesImporter, feature_category: :importers do let(:project) { create(:project) } let(:client) { double(:client) } let(:importer) { described_class.new(project, client) } diff --git a/spec/lib/gitlab/github_import/page_counter_spec.rb b/spec/lib/gitlab/github_import/page_counter_spec.rb index 511b19c00e5..ddb62cc8fad 100644 --- a/spec/lib/gitlab/github_import/page_counter_spec.rb +++ b/spec/lib/gitlab/github_import/page_counter_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::GithubImport::PageCounter, :clean_gitlab_redis_cache, feature_category: :importer do +RSpec.describe Gitlab::GithubImport::PageCounter, :clean_gitlab_redis_cache, feature_category: :importers do let(:project) { double(:project, id: 1) } let(:counter) { described_class.new(project, :issues) } diff --git a/spec/lib/gitlab/github_import/representation/protected_branch_spec.rb b/spec/lib/gitlab/github_import/representation/protected_branch_spec.rb index 60cae79459e..e57ea31d1d2 100644 --- a/spec/lib/gitlab/github_import/representation/protected_branch_spec.rb +++ b/spec/lib/gitlab/github_import/representation/protected_branch_spec.rb @@ -28,6 +28,14 @@ RSpec.describe Gitlab::GithubImport::Representation::ProtectedBranch do it 'includes the protected branch require_code_owner_reviews' do expect(protected_branch.require_code_owner_reviews).to eq true end + + it 'includes the protected branch allowed_to_push_users' do + expect(protected_branch.allowed_to_push_users[0]) + .to be_an_instance_of(Gitlab::GithubImport::Representation::User) + + expect(protected_branch.allowed_to_push_users[0].id).to eq(4) + expect(protected_branch.allowed_to_push_users[0].login).to eq('alice') + end end end @@ -40,7 +48,7 @@ RSpec.describe Gitlab::GithubImport::Representation::ProtectedBranch do ) enabled_setting = Struct.new(:enabled, keyword_init: true) required_pull_request_reviews = Struct.new( - :url, :dismissal_restrictions, :require_code_owner_reviews, + :url, :dismissal_restrictions, :require_code_owner_reviews, :bypass_pull_request_allowances, keyword_init: true ) response.new( @@ -57,7 +65,17 @@ RSpec.describe Gitlab::GithubImport::Representation::ProtectedBranch do required_pull_request_reviews: required_pull_request_reviews.new( url: 'https://example.com/branches/main/protection/required_pull_request_reviews', dismissal_restrictions: {}, - require_code_owner_reviews: true + require_code_owner_reviews: true, + bypass_pull_request_allowances: { + users: [ + { + login: 'alice', + id: 4, + url: 'https://api.github.com/users/cervols', + type: 'User' + } + ] + } ) ) end @@ -76,7 +94,8 @@ RSpec.describe Gitlab::GithubImport::Representation::ProtectedBranch do 'required_conversation_resolution' => true, 'required_signatures' => true, 'required_pull_request_reviews' => true, - 'require_code_owner_reviews' => true + 'require_code_owner_reviews' => true, + 'allowed_to_push_users' => [{ 'id' => 4, 'login' => 'alice' }] } end diff --git a/spec/lib/gitlab/http_spec.rb b/spec/lib/gitlab/http_spec.rb index 929fd37ee40..57e4b4fc74b 100644 --- a/spec/lib/gitlab/http_spec.rb +++ b/spec/lib/gitlab/http_spec.rb @@ -51,10 +51,10 @@ RSpec.describe Gitlab::HTTP do end @original_net_http = Net.send(:remove_const, :HTTP) - @webmock_net_http = WebMock::HttpLibAdapters::NetHttpAdapter.instance_variable_get('@webMockNetHTTP') + @webmock_net_http = WebMock::HttpLibAdapters::NetHttpAdapter.instance_variable_get(:@webMockNetHTTP) Net.send(:const_set, :HTTP, mocked_http) - WebMock::HttpLibAdapters::NetHttpAdapter.instance_variable_set('@webMockNetHTTP', mocked_http) + WebMock::HttpLibAdapters::NetHttpAdapter.instance_variable_set(:@webMockNetHTTP, mocked_http) # Reload Gitlab::NetHttpAdapter Gitlab.send(:remove_const, :NetHttpAdapter) @@ -72,7 +72,7 @@ RSpec.describe Gitlab::HTTP do after(:all) do Net.send(:remove_const, :HTTP) Net.send(:const_set, :HTTP, @original_net_http) - WebMock::HttpLibAdapters::NetHttpAdapter.instance_variable_set('@webMockNetHTTP', @webmock_net_http) + WebMock::HttpLibAdapters::NetHttpAdapter.instance_variable_set(:@webMockNetHTTP, @webmock_net_http) # Reload Gitlab::NetHttpAdapter Gitlab.send(:remove_const, :NetHttpAdapter) diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index b34399d20f1..8750bf4387c 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -422,7 +422,9 @@ project: - wiki_page_hooks_integrations - deployment_hooks_integrations - alert_hooks_integrations +- incident_hooks_integrations - vulnerability_hooks_integrations +- apple_app_store_integration - campfire_integration - confluence_integration - datadog_integration @@ -482,6 +484,8 @@ project: - project_repository - users - requesters +- namespace_members +- namespace_requesters - deploy_keys_projects - deploy_keys - users_star_projects @@ -664,6 +668,7 @@ project: - pipeline_metadata - disable_download_button - dependency_list_exports +- sbom_occurrences award_emoji: - awardable - user @@ -679,8 +684,6 @@ timelogs: - note push_event_payload: - event -issuable_severity: -- issue issue_assignees: - issue - assignee @@ -705,6 +708,7 @@ metrics: resource_label_events: - user - issue +- work_item - merge_request - epic - label @@ -857,11 +861,13 @@ approvals: resource_milestone_events: - user - issue + - work_item - merge_request - milestone resource_state_events: - user - issue + - work_item - merge_request - source_merge_request - epic @@ -874,6 +880,7 @@ iteration: resource_iteration_events: - user - issue + - work_item - merge_request - iteration iterations_cadence: diff --git a/spec/lib/gitlab/import_export/base/relation_object_saver_spec.rb b/spec/lib/gitlab/import_export/base/relation_object_saver_spec.rb index 4ee825c71b6..a8b4b9a6f05 100644 --- a/spec/lib/gitlab/import_export/base/relation_object_saver_spec.rb +++ b/spec/lib/gitlab/import_export/base/relation_object_saver_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::ImportExport::Base::RelationObjectSaver do +RSpec.describe Gitlab::ImportExport::Base::RelationObjectSaver, feature_category: :importers do let(:project) { create(:project) } let(:relation_object) { build(:issue, project: project) } let(:relation_definition) { {} } @@ -34,6 +34,7 @@ RSpec.describe Gitlab::ImportExport::Base::RelationObjectSaver do it 'saves relation object with subrelations' do expect(relation_object.notes).to receive(:<<).and_call_original + expect(relation_object).to receive(:save).and_call_original saver.execute @@ -80,6 +81,7 @@ RSpec.describe Gitlab::ImportExport::Base::RelationObjectSaver do it 'saves valid subrelations and logs invalid subrelation' do expect(relation_object.notes).to receive(:<<).twice.and_call_original + expect(relation_object).to receive(:save).and_call_original expect(Gitlab::Import::Logger) .to receive(:info) .with( diff --git a/spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb b/spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb index ce888b71d5e..f18d9e64f52 100644 --- a/spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb +++ b/spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::ImportExport::FastHashSerializer do +RSpec.describe Gitlab::ImportExport::FastHashSerializer, :with_license do # FastHashSerializer#execute generates the hash which is not easily accessible # and includes `JSONBatchRelation` items which are serialized at this point. # Wrapping the result into JSON generating/parsing is for making diff --git a/spec/lib/gitlab/import_export/group/legacy_tree_restorer_spec.rb b/spec/lib/gitlab/import_export/group/legacy_tree_restorer_spec.rb deleted file mode 100644 index a5b03974bc0..00000000000 --- a/spec/lib/gitlab/import_export/group/legacy_tree_restorer_spec.rb +++ /dev/null @@ -1,153 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::ImportExport::Group::LegacyTreeRestorer do - include ImportExport::CommonUtil - - let(:shared) { Gitlab::ImportExport::Shared.new(group) } - - describe 'restore group tree' do - before_all do - # Using an admin for import, so we can check assignment of existing members - user = create(:admin, email: 'root@gitlabexample.com') - create(:user, email: 'adriene.mcclure@gitlabexample.com') - create(:user, email: 'gwendolyn_robel@gitlabexample.com') - - RSpec::Mocks.with_temporary_scope do - @group = create(:group, name: 'group', path: 'group') - @shared = Gitlab::ImportExport::Shared.new(@group) - - setup_import_export_config('group_exports/complex') - - group_tree_restorer = described_class.new(user: user, shared: @shared, group: @group, group_hash: nil) - - @restored_group_json = group_tree_restorer.restore - end - end - - context 'JSON' do - it 'restores models based on JSON' do - expect(@restored_group_json).to be_truthy - end - - it 'has the group description' do - expect(Group.find_by_path('group').description).to eq('Group Description') - end - - it 'has group labels' do - expect(@group.labels.count).to eq(10) - end - - context 'issue boards' do - it 'has issue boards' do - expect(@group.boards.count).to eq(1) - end - - it 'has board label lists' do - lists = @group.boards.find_by(name: 'first board').lists - - expect(lists.count).to eq(3) - expect(lists.first.label.title).to eq('TSL') - expect(lists.second.label.title).to eq('Sosync') - end - end - - it 'has badges' do - expect(@group.badges.count).to eq(1) - end - - it 'has milestones' do - expect(@group.milestones.count).to eq(5) - end - - it 'has group children' do - expect(@group.children.count).to eq(2) - end - - it 'has group members' do - expect(@group.members.map(&:user).map(&:email)).to contain_exactly('root@gitlabexample.com', 'adriene.mcclure@gitlabexample.com', 'gwendolyn_robel@gitlabexample.com') - end - end - end - - context 'excluded attributes' do - let!(:source_user) { create(:user, id: 123) } - let!(:importer_user) { create(:user) } - let(:group) { create(:group) } - let(:shared) { Gitlab::ImportExport::Shared.new(group) } - let(:group_tree_restorer) { described_class.new(user: importer_user, shared: shared, group: group, group_hash: nil) } - let(:group_json) { Gitlab::Json.parse(File.read(File.join(shared.export_path, 'group.json'))) } - - shared_examples 'excluded attributes' do - excluded_attributes = %w[ - id - owner_id - parent_id - created_at - updated_at - runners_token - runners_token_encrypted - saml_discovery_token - ] - - before do - group.add_owner(importer_user) - - setup_import_export_config('group_exports/complex') - end - - excluded_attributes.each do |excluded_attribute| - it 'does not allow override of excluded attributes' do - expect(group_json[excluded_attribute]).not_to eq(group.public_send(excluded_attribute)) - end - end - end - - include_examples 'excluded attributes' - end - - context 'group.json file access check' do - let(:user) { create(:user) } - let!(:group) { create(:group, name: 'group2', path: 'group2') } - let(:group_tree_restorer) { described_class.new(user: user, shared: shared, group: group, group_hash: nil) } - let(:restored_group_json) { group_tree_restorer.restore } - - it 'does not read a symlink' do - Dir.mktmpdir do |tmpdir| - setup_symlink(tmpdir, 'group.json') - allow(shared).to receive(:export_path).and_call_original - - expect(group_tree_restorer.restore).to eq(false) - expect(shared.errors).to include('Incorrect JSON format') - end - end - end - - context 'group visibility levels' do - let(:user) { create(:user) } - let(:shared) { Gitlab::ImportExport::Shared.new(group) } - let(:group_tree_restorer) { described_class.new(user: user, shared: shared, group: group, group_hash: nil) } - - before do - setup_import_export_config(filepath) - - group_tree_restorer.restore - end - - shared_examples 'with visibility level' do |visibility_level, expected_visibilities| - context "when visibility level is #{visibility_level}" do - let(:group) { create(:group, visibility_level) } - let(:filepath) { "group_exports/visibility_levels/#{visibility_level}" } - - it "imports all subgroups as #{visibility_level}" do - expect(group.children.map(&:visibility_level)).to match_array(expected_visibilities) - end - end - end - - include_examples 'with visibility level', :public, [20, 10, 0] - include_examples 'with visibility level', :private, [0, 0, 0] - include_examples 'with visibility level', :internal, [10, 10, 0] - end -end diff --git a/spec/lib/gitlab/import_export/group/legacy_tree_saver_spec.rb b/spec/lib/gitlab/import_export/group/legacy_tree_saver_spec.rb deleted file mode 100644 index f5a4fc79c90..00000000000 --- a/spec/lib/gitlab/import_export/group/legacy_tree_saver_spec.rb +++ /dev/null @@ -1,159 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::ImportExport::Group::LegacyTreeSaver do - describe 'saves the group tree into a json object' do - let(:shared) { Gitlab::ImportExport::Shared.new(group) } - let(:group_tree_saver) { described_class.new(group: group, current_user: user, shared: shared) } - let(:export_path) { "#{Dir.tmpdir}/group_tree_saver_spec" } - let(:user) { create(:user) } - let!(:group) { setup_group } - - before do - group.add_maintainer(user) - allow(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) - end - - after do - FileUtils.rm_rf(export_path) - end - - it 'saves group successfully' do - expect(group_tree_saver.save).to be true - end - - # It is mostly duplicated in - # `spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb` - # except: - # context 'with description override' do - # context 'group members' do - # ^ These are specific for the Group::LegacyTreeSaver - context 'JSON' do - let(:saved_group_json) do - group_tree_saver.save # rubocop:disable Rails/SaveBang - group_json(group_tree_saver.full_path) - end - - it 'saves the correct json' do - expect(saved_group_json).to include({ 'description' => 'description' }) - end - - it 'has milestones' do - expect(saved_group_json['milestones']).not_to be_empty - end - - it 'has labels' do - expect(saved_group_json['labels']).not_to be_empty - end - - it 'has boards' do - expect(saved_group_json['boards']).not_to be_empty - end - - it 'has board label list' do - expect(saved_group_json['boards'].first['lists']).not_to be_empty - end - - it 'has group members' do - expect(saved_group_json['members']).not_to be_empty - end - - it 'has priorities associated to labels' do - expect(saved_group_json['labels'].first['priorities']).not_to be_empty - end - - it 'has badges' do - expect(saved_group_json['badges']).not_to be_empty - end - - context 'group children' do - let(:children) { group.children } - - it 'exports group children' do - expect(saved_group_json['children'].length).to eq(children.count) - end - - it 'exports group children of children' do - expect(saved_group_json['children'].first['children'].length).to eq(children.first.children.count) - end - end - - context 'group members' do - let(:user2) { create(:user, email: 'group@member.com') } - let(:member_emails) do - saved_group_json['members'].map do |pm| - pm['user']['public_email'] - end - end - - before do - user2.update!(public_email: user2.email) - group.add_developer(user2) - end - - it 'exports group members as group owner' do - group.add_owner(user) - - expect(member_emails).to include('group@member.com') - end - - context 'as admin' do - let(:user) { create(:admin) } - - it 'exports group members as admin' do - expect(member_emails).to include('group@member.com') - end - - it 'exports group members' do - member_types = saved_group_json['members'].map { |pm| pm['source_type'] } - - expect(member_types).to all(eq('Namespace')) - end - end - end - - context 'group attributes' do - shared_examples 'excluded attributes' do - excluded_attributes = %w[ - id - owner_id - parent_id - created_at - updated_at - runners_token - runners_token_encrypted - saml_discovery_token - ] - - excluded_attributes.each do |excluded_attribute| - it 'does not contain excluded attribute' do - expect(saved_group_json).not_to include(excluded_attribute => group.public_send(excluded_attribute)) - end - end - end - - include_examples 'excluded attributes' - end - end - end - - def setup_group - group = create(:group, description: 'description') - sub_group = create(:group, description: 'description', parent: group) - create(:group, description: 'description', parent: sub_group) - create(:milestone, group: group) - create(:group_badge, group: group) - group_label = create(:group_label, group: group) - create(:label_priority, label: group_label, priority: 1) - board = create(:board, group: group, milestone_id: Milestone::Upcoming.id) - create(:list, board: board, label: group_label) - create(:group_badge, group: group) - - group - end - - def group_json(filename) - ::JSON.parse(File.read(filename)) - end -end diff --git a/spec/lib/gitlab/import_export/project/tree_saver_spec.rb b/spec/lib/gitlab/import_export/project/tree_saver_spec.rb index 15108d28bf2..74b6e039601 100644 --- a/spec/lib/gitlab/import_export/project/tree_saver_spec.rb +++ b/spec/lib/gitlab/import_export/project/tree_saver_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::ImportExport::Project::TreeSaver do +RSpec.describe Gitlab::ImportExport::Project::TreeSaver, :with_license do let_it_be(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" } let_it_be(:exportable_path) { 'project' } let_it_be(:user) { create(:user) } diff --git a/spec/lib/gitlab/import_export/safe_model_attributes.yml b/spec/lib/gitlab/import_export/safe_model_attributes.yml index 75d980cd5f4..e14e929faf3 100644 --- a/spec/lib/gitlab/import_export/safe_model_attributes.yml +++ b/spec/lib/gitlab/import_export/safe_model_attributes.yml @@ -702,7 +702,9 @@ Badge: ProjectCiCdSetting: - group_runners_enabled - runner_token_expiration_interval +- default_git_depth ProjectSetting: +- squash_option - allow_merge_on_skipped_pipeline - only_allow_merge_if_all_status_checks_passed - has_confluence @@ -916,6 +918,7 @@ PushRule: - reject_unsigned_commits - commit_committer_check - regexp_uses_re2 + - reject_non_dco_commits MergeRequest::CleanupSchedule: - id - scheduled_at diff --git a/spec/lib/gitlab/import_export/snippets_repo_restorer_spec.rb b/spec/lib/gitlab/import_export/snippets_repo_restorer_spec.rb index ebb0d62afa0..e348e8f7991 100644 --- a/spec/lib/gitlab/import_export/snippets_repo_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/snippets_repo_restorer_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::ImportExport::SnippetsRepoRestorer do +RSpec.describe Gitlab::ImportExport::SnippetsRepoRestorer, :clean_gitlab_redis_repository_cache, feature_category: :importers do describe 'bundle a snippet Git repo' do let_it_be(:user) { create(:user) } let_it_be(:project) { create(:project, namespace: user.namespace) } @@ -26,9 +26,18 @@ RSpec.describe Gitlab::ImportExport::SnippetsRepoRestorer do shared_examples 'imports snippet repositories' do before do snippet1.snippet_repository&.delete + # We need to explicitly invalidate repository.exists? from cache by calling repository.expire_exists_cache. + # Previously, we didn't have to do this because snippet1.repository_exists? would hit Rails.cache, which is a + # NullStore, thus cache.read would always be false. + # Now, since we are using a separate instance of Redis, ie Gitlab::Redis::RepositoryCache, + # snippet.repository_exists? would still be true because snippet.repository.remove doesn't invalidate the + # cache (snippet.repository.remove only makes gRPC call to Gitaly). + # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/107232#note_1214358593 for more. + snippet1.repository.expire_exists_cache snippet1.repository.remove snippet2.snippet_repository&.delete + snippet2.repository.expire_exists_cache snippet2.repository.remove end diff --git a/spec/lib/gitlab/import_export/version_checker_spec.rb b/spec/lib/gitlab/import_export/version_checker_spec.rb index 14c62edb786..b3730d85f13 100644 --- a/spec/lib/gitlab/import_export/version_checker_spec.rb +++ b/spec/lib/gitlab/import_export/version_checker_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::ImportExport::VersionChecker do +RSpec.describe Gitlab::ImportExport::VersionChecker, feature_category: :import do include ImportExport::CommonUtil let!(:shared) { Gitlab::ImportExport::Shared.new(nil) } diff --git a/spec/lib/gitlab/instrumentation_helper_spec.rb b/spec/lib/gitlab/instrumentation_helper_spec.rb index 7d78d25f18e..ce67d1d0297 100644 --- a/spec/lib/gitlab/instrumentation_helper_spec.rb +++ b/spec/lib/gitlab/instrumentation_helper_spec.rb @@ -4,7 +4,8 @@ require 'spec_helper' require 'rspec-parameterized' require 'support/helpers/rails_helpers' -RSpec.describe Gitlab::InstrumentationHelper do +RSpec.describe Gitlab::InstrumentationHelper, :clean_gitlab_redis_repository_cache, :clean_gitlab_redis_cache, + feature_category: :scalability do using RSpec::Parameterized::TableSyntax describe '.add_instrumentation_data', :request_store do @@ -22,19 +23,42 @@ RSpec.describe Gitlab::InstrumentationHelper do expect(payload).to include(db_count: 0, db_cached_count: 0, db_write_count: 0) end - context 'when Gitaly calls are made' do - it 'adds Gitaly data and omits Redis data' do - project = create(:project) - RequestStore.clear! - project.repository.exists? + shared_examples 'make Gitaly calls' do + context 'when Gitaly calls are made' do + it 'adds Gitaly and Redis data' do + project = create(:project) + RequestStore.clear! + project.repository.exists? - subject + subject - expect(payload[:gitaly_calls]).to eq(1) - expect(payload[:gitaly_duration_s]).to be >= 0 - expect(payload[:redis_calls]).to be_nil - expect(payload[:redis_duration_ms]).to be_nil + expect(payload[:gitaly_calls]).to eq(1) + expect(payload[:gitaly_duration_s]).to be >= 0 + # With MultiStore, the number of `redis_calls` depends on whether primary_store + # (Gitlab::Redis::Repositorycache) and secondary_store (Gitlab::Redis::Cache) are of the same instance. + # In GitLab.com CI, primary and secondary are the same instance, thus only 1 call being made. If primary + # and secondary are different instances, an additional fallback read to secondary_store will be made because + # the first `get` call is a cache miss. Then, the following expect will fail. + expect(payload[:redis_calls]).to eq(1) + expect(payload[:redis_duration_ms]).to be_nil + end + end + end + + context 'when multistore ff use_primary_and_secondary_stores_for_repository_cache is enabled' do + before do + stub_feature_flags(use_primary_and_secondary_stores_for_repository_cache: true) end + + it_behaves_like 'make Gitaly calls' + end + + context 'when multistore ff use_primary_and_secondary_stores_for_repository_cache is disabled' do + before do + stub_feature_flags(use_primary_and_secondary_stores_for_repository_cache: false) + end + + it_behaves_like 'make Gitaly calls' end context 'when Redis calls are made' do diff --git a/spec/lib/gitlab/memory/reporter_spec.rb b/spec/lib/gitlab/memory/reporter_spec.rb index 924397ceb4f..64ae740a5d7 100644 --- a/spec/lib/gitlab/memory/reporter_spec.rb +++ b/spec/lib/gitlab/memory/reporter_spec.rb @@ -26,15 +26,15 @@ RSpec.describe Gitlab::Memory::Reporter, :aggregate_failures, feature_category: FileUtils.rm_rf(reports_path) end - describe '#run_report', time_travel_to: '2020-02-02 10:30:45 0000' do + describe '#run_report', time_travel_to: '2020-02-02 10:30:45 +0000' do let(:report_duration_counter) { instance_double(::Prometheus::Client::Counter) } let(:file_size) { 1_000_000 } let(:report_file) { "#{reports_path}/fake_report.2020-02-02.10:30:45:000.worker_1.abc123.gz" } - - let(:input) { StringIO.new } - let(:output) { StringIO.new } + let(:output) { File.read(report_file) } before do + stub_const('Gitlab::Memory::Reporter::COMPRESS_CMD', %w[cat]) + allow(SecureRandom).to receive(:uuid).and_return('abc123') allow(Gitlab::Metrics).to receive(:counter).and_return(report_duration_counter) @@ -44,22 +44,13 @@ RSpec.describe Gitlab::Memory::Reporter, :aggregate_failures, feature_category: allow(File).to receive(:size).with(report_file).and_return(file_size) allow(logger).to receive(:info) - - stub_gzip end shared_examples 'runs and stores reports' do it 'runs the given report and returns true' do expect(reporter.run_report(report)).to be(true) - expect(output.string).to eq('I ran') - end - - it 'closes read and write streams' do - expect(input).to receive(:close).ordered.at_least(:once) - expect(output).to receive(:close).ordered.at_least(:once) - - reporter.run_report(report) + expect(output).to eq('I ran') end it 'logs start and finish event' do @@ -111,39 +102,47 @@ RSpec.describe Gitlab::Memory::Reporter, :aggregate_failures, feature_category: end context 'when an error occurs' do - before do - allow(report).to receive(:run).and_raise(RuntimeError.new('report failed')) - end + shared_examples 'handles errors gracefully' do + it 'logs the error and returns false' do + expect(logger).to receive(:info).ordered.with(hash_including(message: 'started')) + expect(logger).to receive(:error).ordered.with( + hash_including( + message: 'failed', error: match(error_message) + )) + + expect(reporter.run_report(report)).to be(false) + end + + context 'when compression process is still running' do + it 'terminates the process' do + allow(logger).to receive(:info) + allow(logger).to receive(:error) - it 'logs the error and returns false' do - expect(logger).to receive(:info).ordered.with(hash_including(message: 'started')) - expect(logger).to receive(:error).ordered.with( - hash_including( - message: 'failed', error: '#<RuntimeError: report failed>' - )) + expect(Gitlab::ProcessManagement).to receive(:signal).with(an_instance_of(Integer), :KILL) - expect(reporter.run_report(report)).to be(false) + reporter.run_report(report) + end + end end - it 'closes read and write streams' do - allow(logger).to receive(:info) - allow(logger).to receive(:error) + context 'when cause was an error being raised' do + let(:error_message) { 'report failed' } - expect(input).to receive(:close).ordered.at_least(:once) - expect(output).to receive(:close).ordered.at_least(:once) + before do + allow(report).to receive(:run).and_raise(RuntimeError.new('report failed')) + end - reporter.run_report(report) + it_behaves_like 'handles errors gracefully' end - context 'when compression process is still running' do - it 'terminates the process' do - allow(logger).to receive(:info) - allow(logger).to receive(:error) + context 'when cause was compression command failing' do + let(:error_message) { "StandardError: exit 1: cat:" } - expect(Gitlab::ProcessManagement).to receive(:signal).with(an_instance_of(Integer), :KILL) - - reporter.run_report(report) + before do + stub_const('Gitlab::Memory::Reporter::COMPRESS_CMD', %w[cat --bad-flag]) end + + it_behaves_like 'handles errors gracefully' end end @@ -191,16 +190,4 @@ RSpec.describe Gitlab::Memory::Reporter, :aggregate_failures, feature_category: it_behaves_like 'runs and stores reports' end end - - # We need to stub out the call into gzip. We do this by intercepting the write - # end of the pipe and replacing it with a StringIO instead, which we can - # easily inspect for contents. - def stub_gzip - pid = 42 - allow(IO).to receive(:pipe).and_return([input, output]) - allow(Process).to receive(:spawn).with( - "gzip", "--fast", in: input, out: an_instance_of(File), err: an_instance_of(IO) - ).and_return(pid) - allow(Process).to receive(:waitpid).with(pid) - end end diff --git a/spec/lib/gitlab/memory/watchdog_spec.rb b/spec/lib/gitlab/memory/watchdog_spec.rb index 1603dda0c39..0b2f24476d9 100644 --- a/spec/lib/gitlab/memory/watchdog_spec.rb +++ b/spec/lib/gitlab/memory/watchdog_spec.rb @@ -98,7 +98,8 @@ RSpec.describe Gitlab::Memory::Watchdog, :aggregate_failures, feature_category: expect(reporter).to receive(:stopped).once .with( memwd_handler_class: handler.class.name, - memwd_sleep_time_s: sleep_time_seconds + memwd_sleep_time_s: sleep_time_seconds, + memwd_reason: 'background task stopped' ) watchdog.call diff --git a/spec/lib/gitlab/merge_requests/message_generator_spec.rb b/spec/lib/gitlab/merge_requests/message_generator_spec.rb index 59aaffc4377..ac9a9aa2897 100644 --- a/spec/lib/gitlab/merge_requests/message_generator_spec.rb +++ b/spec/lib/gitlab/merge_requests/message_generator_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::MergeRequests::MessageGenerator, feature_category: :code_review do +RSpec.describe Gitlab::MergeRequests::MessageGenerator, feature_category: :code_review_workflow do let(:merge_commit_template) { nil } let(:squash_commit_template) { nil } let(:project) do diff --git a/spec/lib/gitlab/observability_spec.rb b/spec/lib/gitlab/observability_spec.rb index 2b1d22d9019..8068d2f8ec9 100644 --- a/spec/lib/gitlab/observability_spec.rb +++ b/spec/lib/gitlab/observability_spec.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require 'fast_spec_helper' +require 'spec_helper' RSpec.describe Gitlab::Observability do describe '.observability_url' do @@ -30,4 +30,39 @@ RSpec.describe Gitlab::Observability do it { is_expected.to eq(observe_url) } end end + + describe '.observability_enabled?' do + let_it_be(:group) { build(:user) } + let_it_be(:user) { build(:group) } + + subject do + described_class.observability_enabled?(user, group) + end + + it 'checks if read_observability ability is allowed for the given user and group' do + allow(Ability).to receive(:allowed?).and_return(true) + + subject + + expect(Ability).to have_received(:allowed?).with(user, :read_observability, group) + end + + it 'returns true if the read_observability ability is allowed' do + allow(Ability).to receive(:allowed?).and_return(true) + + expect(subject).to eq(true) + end + + it 'returns false if the read_observability ability is not allowed' do + allow(Ability).to receive(:allowed?).and_return(false) + + expect(subject).to eq(false) + end + + it 'returns false if observability url is missing' do + allow(described_class).to receive(:observability_url).and_return("") + + expect(subject).to eq(false) + end + end end diff --git a/spec/lib/gitlab/pages/cache_control_spec.rb b/spec/lib/gitlab/pages/cache_control_spec.rb index d46124e0e16..dd15aa87441 100644 --- a/spec/lib/gitlab/pages/cache_control_spec.rb +++ b/spec/lib/gitlab/pages/cache_control_spec.rb @@ -3,20 +3,23 @@ require 'spec_helper' RSpec.describe Gitlab::Pages::CacheControl, feature_category: :pages do - describe '.for_namespace' do - subject(:cache_control) { described_class.for_namespace(1) } + RSpec.shared_examples 'cache_control' do |type| + it { expect(subject.cache_key).to match(/pages_domain_for_#{type}_1_*/) } - it { expect(subject.cache_key).to match(/pages_domain_for_namespace_1_*/) } + describe '#clear_cache', :use_clean_rails_redis_caching do + before do + Rails.cache.write("pages_domain_for_#{type}_1", ['settings-hash']) + Rails.cache.write("pages_domain_for_#{type}_1_settings-hash", 'payload') + end - describe '#clear_cache' do it 'clears the cache' do expect(Rails.cache) .to receive(:delete_multi) .with( array_including( [ - "pages_domain_for_namespace_1", - /pages_domain_for_namespace_1_*/ + "pages_domain_for_#{type}_1", + "pages_domain_for_#{type}_1_settings-hash" ] )) @@ -25,63 +28,53 @@ RSpec.describe Gitlab::Pages::CacheControl, feature_category: :pages do end end - describe '.for_project' do - subject(:cache_control) { described_class.for_project(1) } + describe '.for_namespace' do + subject(:cache_control) { described_class.for_namespace(1) } - it { expect(subject.cache_key).to match(/pages_domain_for_project_1_*/) } + it_behaves_like 'cache_control', 'namespace' + end - describe '#clear_cache' do - it 'clears the cache' do - expect(Rails.cache) - .to receive(:delete_multi) - .with( - array_including( - [ - "pages_domain_for_project_1", - /pages_domain_for_project_1_*/ - ] - )) + describe '.for_domain' do + subject(:cache_control) { described_class.for_domain(1) } - subject.clear_cache - end - end + it_behaves_like 'cache_control', 'domain' end describe '#cache_key' do it 'does not change the pages config' do - expect { described_class.new(type: :project, id: 1).cache_key } + expect { described_class.new(type: :domain, id: 1).cache_key } .not_to change(Gitlab.config, :pages) end it 'is based on pages settings' do access_control = Gitlab.config.pages.access_control - cache_key = described_class.new(type: :project, id: 1).cache_key + cache_key = described_class.new(type: :domain, id: 1).cache_key stub_config(pages: { access_control: !access_control }) - expect(described_class.new(type: :project, id: 1).cache_key).not_to eq(cache_key) + expect(described_class.new(type: :domain, id: 1).cache_key).not_to eq(cache_key) end it 'is based on the force_pages_access_control settings' do force_pages_access_control = ::Gitlab::CurrentSettings.force_pages_access_control - cache_key = described_class.new(type: :project, id: 1).cache_key + cache_key = described_class.new(type: :domain, id: 1).cache_key ::Gitlab::CurrentSettings.force_pages_access_control = !force_pages_access_control - expect(described_class.new(type: :project, id: 1).cache_key).not_to eq(cache_key) + expect(described_class.new(type: :domain, id: 1).cache_key).not_to eq(cache_key) end it 'caches the application settings hash' do expect(Rails.cache) .to receive(:write) - .with("pages_domain_for_project_1", kind_of(Set)) + .with('pages_domain_for_domain_1', kind_of(Set)) - described_class.new(type: :project, id: 1).cache_key + described_class.new(type: :domain, id: 1).cache_key end end it 'fails with invalid type' do expect { described_class.new(type: :unknown, id: nil) } - .to raise_error(ArgumentError, "type must be :namespace or :project") + .to raise_error(ArgumentError, 'type must be :namespace or :domain') end end diff --git a/spec/lib/gitlab/pagination/cursor_based_keyset_spec.rb b/spec/lib/gitlab/pagination/cursor_based_keyset_spec.rb index 879c874b134..dc62fcb4478 100644 --- a/spec/lib/gitlab/pagination/cursor_based_keyset_spec.rb +++ b/spec/lib/gitlab/pagination/cursor_based_keyset_spec.rb @@ -10,6 +10,10 @@ RSpec.describe Gitlab::Pagination::CursorBasedKeyset do expect(subject.available_for_type?(Group.all)).to be_truthy end + it 'returns true for Ci::Build' do + expect(subject.available_for_type?(Ci::Build.all)).to be_truthy + end + it 'return false for other types of relations' do expect(subject.available_for_type?(User.all)).to be_falsey end @@ -29,6 +33,12 @@ RSpec.describe Gitlab::Pagination::CursorBasedKeyset do it { is_expected.to be false } end + + context 'when relation is Ci::Build' do + let(:relation) { Ci::Build.all } + + it { is_expected.to be false } + end end describe '.available?' do @@ -45,6 +55,20 @@ RSpec.describe Gitlab::Pagination::CursorBasedKeyset do it 'return false for other types of relations' do expect(subject.available?(cursor_based_request_context, User.all)).to be_falsey + expect(subject.available?(cursor_based_request_context, Ci::Build.all)).to be_falsey + end + end + + context 'with order-by id desc' do + let(:order_by) { :id } + let(:sort) { :desc } + + it 'returns true for Ci::Build' do + expect(subject.available?(cursor_based_request_context, Ci::Build.all)).to be_truthy + end + + it 'returns true for AuditEvent' do + expect(subject.available?(cursor_based_request_context, AuditEvent.all)).to be_truthy end end diff --git a/spec/lib/gitlab/pagination/keyset/simple_order_builder_spec.rb b/spec/lib/gitlab/pagination/keyset/simple_order_builder_spec.rb index 4f1d380ab0a..e85b0354ff6 100644 --- a/spec/lib/gitlab/pagination/keyset/simple_order_builder_spec.rb +++ b/spec/lib/gitlab/pagination/keyset/simple_order_builder_spec.rb @@ -92,34 +92,6 @@ RSpec.describe Gitlab::Pagination::Keyset::SimpleOrderBuilder do end end - context "NULLS order given as as an Arel literal" do - context 'when NULLS LAST order is given without a tie-breaker' do - let(:scope) { Project.order(Project.arel_table[:created_at].asc.nulls_last) } - - it 'sets the column definition for created_at appropriately' do - expect(column_definition.attribute_name).to eq('created_at') - end - - it 'orders by primary key' do - expect(sql_with_order) - .to end_with('ORDER BY "projects"."created_at" ASC NULLS LAST, "projects"."id" DESC') - end - end - - context 'when NULLS FIRST order is given with a tie-breaker' do - let(:scope) { Issue.order(Issue.arel_table[:relative_position].desc.nulls_first).order(id: :asc) } - - it 'sets the column definition for created_at appropriately' do - expect(column_definition.attribute_name).to eq('relative_position') - end - - it 'orders by the given primary key' do - expect(sql_with_order) - .to end_with('ORDER BY "issues"."relative_position" DESC NULLS FIRST, "issues"."id" ASC') - end - end - end - context "NULLS order given as as an Arel node" do context 'when NULLS LAST order is given without a tie-breaker' do let(:scope) { Project.order(Project.arel_table[:created_at].asc.nulls_last) } diff --git a/spec/lib/gitlab/rack_attack_spec.rb b/spec/lib/gitlab/rack_attack_spec.rb index 7ba4eab50c7..960a81b8c9d 100644 --- a/spec/lib/gitlab/rack_attack_spec.rb +++ b/spec/lib/gitlab/rack_attack_spec.rb @@ -35,7 +35,7 @@ RSpec.describe Gitlab::RackAttack, :aggregate_failures do allow(fake_rack_attack).to receive(:cache).and_return(fake_cache) allow(fake_cache).to receive(:store=) - fake_rack_attack.const_set('Request', fake_rack_attack_request) + fake_rack_attack.const_set(:Request, fake_rack_attack_request) stub_const("Rack::Attack", fake_rack_attack) end diff --git a/spec/lib/gitlab/redis/duplicate_jobs_spec.rb b/spec/lib/gitlab/redis/duplicate_jobs_spec.rb index be20e6dcdaf..4d46a567032 100644 --- a/spec/lib/gitlab/redis/duplicate_jobs_spec.rb +++ b/spec/lib/gitlab/redis/duplicate_jobs_spec.rb @@ -14,16 +14,6 @@ RSpec.describe Gitlab::Redis::DuplicateJobs do describe '#pool' do subject { described_class.pool } - before do - redis_clear_raw_config!(Gitlab::Redis::SharedState) - redis_clear_raw_config!(Gitlab::Redis::Queues) - end - - after do - redis_clear_raw_config!(Gitlab::Redis::SharedState) - redis_clear_raw_config!(Gitlab::Redis::Queues) - end - around do |example| clear_pool example.run diff --git a/spec/lib/gitlab/redis/multi_store_spec.rb b/spec/lib/gitlab/redis/multi_store_spec.rb index 0e7eedf66b1..f198ba90d0a 100644 --- a/spec/lib/gitlab/redis/multi_store_spec.rb +++ b/spec/lib/gitlab/redis/multi_store_spec.rb @@ -25,7 +25,9 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do let_it_be(:instance_name) { 'TestStore' } let_it_be(:multi_store) { described_class.new(primary_store, secondary_store, instance_name) } - subject { multi_store.send(name, *args) } + subject do + multi_store.send(name, *args) + end before do skip_feature_flags_yaml_validation @@ -108,34 +110,93 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do end end + # rubocop:disable RSpec/MultipleMemoizedHelpers context 'with READ redis commands' do + subject do + multi_store.send(name, *args, **kwargs) + end + let_it_be(:key1) { "redis:{1}:key_a" } let_it_be(:key2) { "redis:{1}:key_b" } let_it_be(:value1) { "redis_value1" } let_it_be(:value2) { "redis_value2" } let_it_be(:skey) { "redis:set:key" } + let_it_be(:skey2) { "redis:set:key2" } + let_it_be(:smemberargs) { [skey, value1] } + let_it_be(:hkey) { "redis:hash:key" } + let_it_be(:hkey2) { "redis:hash:key2" } + let_it_be(:zkey) { "redis:sortedset:key" } + let_it_be(:zkey2) { "redis:sortedset:key2" } + let_it_be(:hitem1) { "item1" } + let_it_be(:hitem2) { "item2" } let_it_be(:keys) { [key1, key2] } let_it_be(:values) { [value1, value2] } let_it_be(:svalues) { [value2, value1] } - - where(:case_name, :name, :args, :value, :block) do - 'execute :get command' | :get | ref(:key1) | ref(:value1) | nil - 'execute :mget command' | :mget | ref(:keys) | ref(:values) | nil - 'execute :mget with block' | :mget | ref(:keys) | ref(:values) | ->(value) { value } - 'execute :smembers command' | :smembers | ref(:skey) | ref(:svalues) | nil - 'execute :scard command' | :scard | ref(:skey) | 2 | nil + let_it_be(:hgetargs) { [hkey, hitem1] } + let_it_be(:hmgetval) { [value1] } + let_it_be(:mhmgetargs) { [hkey, hitem1] } + let_it_be(:hvalmapped) { { "item1" => value1 } } + let_it_be(:sscanargs) { [skey2, 0] } + let_it_be(:sscanval) { ["0", [value1]] } + let_it_be(:sscan_eachval) { [value1] } + let_it_be(:sscan_each_arg) { { match: '*1*' } } + let_it_be(:hscan_eachval) { [[hitem1, value1]] } + let_it_be(:zscan_eachval) { [[value1, 1.0]] } + let_it_be(:scan_each_arg) { { match: 'redis*' } } + let_it_be(:scan_each_val) { [key1, key2, skey, skey2, hkey, hkey2, zkey, zkey2] } + + # rubocop:disable Layout/LineLength + where(:case_name, :name, :args, :value, :kwargs, :block) do + 'execute :get command' | :get | ref(:key1) | ref(:value1) | {} | nil + 'execute :mget command' | :mget | ref(:keys) | ref(:values) | {} | nil + 'execute :mget with block' | :mget | ref(:keys) | ref(:values) | {} | ->(value) { value } + 'execute :smembers command' | :smembers | ref(:skey) | ref(:svalues) | {} | nil + 'execute :scard command' | :scard | ref(:skey) | 2 | {} | nil + 'execute :sismember command' | :sismember | ref(:smemberargs) | true | {} | nil + 'execute :exists command' | :exists | ref(:key1) | 1 | {} | nil + 'execute :exists? command' | :exists? | ref(:key1) | true | {} | nil + 'execute :hget command' | :hget | ref(:hgetargs) | ref(:value1) | {} | nil + 'execute :hlen command' | :hlen | ref(:hkey) | 1 | {} | nil + 'execute :hgetall command' | :hgetall | ref(:hkey) | ref(:hvalmapped) | {} | nil + 'execute :hexists command' | :hexists | ref(:hgetargs) | true | {} | nil + 'execute :hmget command' | :hmget | ref(:hgetargs) | ref(:hmgetval) | {} | nil + 'execute :mapped_hmget command' | :mapped_hmget | ref(:mhmgetargs) | ref(:hvalmapped) | {} | nil + 'execute :sscan command' | :sscan | ref(:sscanargs) | ref(:sscanval) | {} | nil + + # we run *scan_each here as they are reads too + 'execute :scan_each command' | :scan_each | nil | ref(:scan_each_val) | ref(:scan_each_arg) | nil + 'execute :sscan_each command' | :sscan_each | ref(:skey2) | ref(:sscan_eachval) | {} | nil + 'execute :sscan_each w block' | :sscan_each | ref(:skey) | ref(:sscan_eachval) | ref(:sscan_each_arg) | nil + 'execute :hscan_each command' | :hscan_each | ref(:hkey) | ref(:hscan_eachval) | {} | nil + 'execute :hscan_each w block' | :hscan_each | ref(:hkey2) | ref(:hscan_eachval) | ref(:sscan_each_arg) | nil + 'execute :zscan_each command' | :zscan_each | ref(:zkey) | ref(:zscan_eachval) | {} | nil + 'execute :zscan_each w block' | :zscan_each | ref(:zkey2) | ref(:zscan_eachval) | ref(:sscan_each_arg) | nil end + # rubocop:enable Layout/LineLength - before(:all) do + before do primary_store.set(key1, value1) primary_store.set(key2, value2) - primary_store.sadd?(skey, value1) - primary_store.sadd?(skey, value2) + primary_store.sadd?(skey, [value1, value2]) + primary_store.sadd?(skey2, [value1]) + primary_store.hset(hkey, hitem1, value1) + primary_store.hset(hkey2, hitem1, value1, hitem2, value2) + primary_store.zadd(zkey, 1, value1) + primary_store.zadd(zkey2, [[1, value1], [2, value2]]) secondary_store.set(key1, value1) secondary_store.set(key2, value2) - secondary_store.sadd?(skey, value1) - secondary_store.sadd?(skey, value2) + secondary_store.sadd?(skey, [value1, value2]) + secondary_store.sadd?(skey2, [value1]) + secondary_store.hset(hkey, hitem1, value1) + secondary_store.hset(hkey2, hitem1, value1, hitem2, value2) + secondary_store.zadd(zkey, 1, value1) + secondary_store.zadd(zkey2, [[1, value1], [2, value2]]) + end + + after do + primary_store.flushdb + secondary_store.flushdb end RSpec.shared_examples_for 'reads correct value' do @@ -157,7 +218,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do end it 'fallback and execute on secondary instance' do - expect(secondary_store).to receive(name).with(*args).and_call_original + expect(secondary_store).to receive(name).with(*expected_args).and_call_original subject end @@ -181,7 +242,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do context 'when fallback read from the secondary instance raises an exception' do before do - allow(secondary_store).to receive(name).with(*args).and_raise(StandardError) + allow(secondary_store).to receive(name).with(*expected_args).and_raise(StandardError) end it 'fails with exception' do @@ -192,7 +253,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do RSpec.shared_examples_for 'secondary store' do it 'execute on the secondary instance' do - expect(secondary_store).to receive(name).with(*args).and_call_original + expect(secondary_store).to receive(name).with(*expected_args).and_call_original subject end @@ -208,6 +269,8 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do with_them do describe name.to_s do + let(:expected_args) { kwargs&.present? ? [*args, { **kwargs }] : Array(args) } + before do allow(primary_store).to receive(name).and_call_original allow(secondary_store).to receive(name).and_call_original @@ -215,7 +278,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do context 'when reading from the primary is successful' do it 'returns the correct value' do - expect(primary_store).to receive(name).with(*args).and_call_original + expect(primary_store).to receive(name).with(*expected_args).and_call_original subject end @@ -231,7 +294,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do context 'when reading from primary instance is raising an exception' do before do - allow(primary_store).to receive(name).with(*args).and_raise(StandardError) + allow(primary_store).to receive(name).with(*expected_args).and_raise(StandardError) allow(Gitlab::ErrorTracking).to receive(:log_exception) end @@ -245,9 +308,10 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do include_examples 'fallback read from the secondary store' end - context 'when reading from primary instance return no value' do + context 'when reading from empty primary instance' do before do - allow(primary_store).to receive(name).and_return(nil) + # this ensures a cache miss without having to stub primary store + primary_store.flushdb end include_examples 'fallback read from the secondary store' @@ -256,7 +320,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do context 'when the command is executed within pipelined block' do subject do multi_store.pipelined do |pipeline| - pipeline.send(name, *args) + pipeline.send(name, *args, **kwargs) end end @@ -266,7 +330,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do 2.times do expect_next_instance_of(Redis::PipelinedConnection) do |pipeline| - expect(pipeline).to receive(name).with(*args).once.and_call_original + expect(pipeline).to receive(name).with(*expected_args).once.and_call_original end end @@ -276,7 +340,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do if params[:block] subject do - multi_store.send(name, *args, &block) + multi_store.send(name, *expected_args, &block) end context 'when block is provided' do @@ -297,6 +361,115 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do it_behaves_like 'secondary store' end + + context 'when use_primary_and_secondary_stores feature flag is disabled' do + before do + stub_feature_flags(use_primary_and_secondary_stores_for_test_store: false) + end + + context 'when using secondary store as default' do + before do + stub_feature_flags(use_primary_store_as_default_for_test_store: false) + end + + it 'executes only on secondary redis store', :aggregate_errors do + expect(secondary_store).to receive(name).with(*expected_args).and_call_original + expect(primary_store).not_to receive(name).with(*expected_args).and_call_original + + subject + end + end + + context 'when using primary store as default' do + it 'executes only on primary redis store', :aggregate_errors do + expect(primary_store).to receive(name).with(*expected_args).and_call_original + expect(secondary_store).not_to receive(name).with(*expected_args).and_call_original + + subject + end + end + end + end + end + end + # rubocop:enable RSpec/MultipleMemoizedHelpers + + context 'with nested command in block' do + let(:skey) { "test_set" } + let(:values) { %w[{x}a {x}b {x}c] } + + before do + primary_store.set('{x}a', 1) + primary_store.set('{x}b', 2) + primary_store.set('{x}c', 3) + + secondary_store.set('{x}a', 10) + secondary_store.set('{x}b', 20) + secondary_store.set('{x}c', 30) + end + + subject do + multi_store.mget(values) do |v| + multi_store.sadd(skey, v) + multi_store.scard(skey) + v # mget receiving block returns the last line of the block for cache-hit check + end + end + + RSpec.shared_examples_for 'primary instance executes block' do + it 'ensures primary instance is executing the block' do + expect(primary_store).to receive(:send).with(:mget, values).and_call_original + expect(primary_store).to receive(:send).with(:sadd, skey, %w[1 2 3]).and_call_original + expect(primary_store).to receive(:send).with(:scard, skey).and_call_original + + expect(secondary_store).not_to receive(:send).with(:mget, values).and_call_original + expect(secondary_store).not_to receive(:send).with(:sadd, skey, %w[1 2 3]).and_call_original + expect(secondary_store).not_to receive(:send).with(:scard, skey).and_call_original + + subject + end + end + + context 'when using both stores' do + context 'when primary instance is default store' do + it_behaves_like 'primary instance executes block' + end + + context 'when secondary instance is default store' do + before do + stub_feature_flags(use_primary_store_as_default_for_test_store: false) + end + + # multistore read still favours the primary store + it_behaves_like 'primary instance executes block' + end + end + + context 'when using 1 store only' do + before do + stub_feature_flags(use_primary_and_secondary_stores_for_test_store: false) + end + + context 'when primary instance is default store' do + it_behaves_like 'primary instance executes block' + end + + context 'when secondary instance is default store' do + before do + stub_feature_flags(use_primary_store_as_default_for_test_store: false) + end + + it 'ensures only secondary instance is executing the block' do + expect(secondary_store).to receive(:send).with(:mget, values).and_call_original + expect(secondary_store).to receive(:send).with(:sadd, skey, %w[10 20 30]).and_call_original + expect(secondary_store).to receive(:send).with(:scard, skey).and_call_original + + expect(primary_store).not_to receive(:send).with(:mget, values).and_call_original + expect(primary_store).not_to receive(:send).with(:sadd, skey, %w[10 20 30]).and_call_original + expect(primary_store).not_to receive(:send).with(:scard, skey).and_call_original + + subject + end end end end @@ -316,9 +489,17 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do end end + # rubocop:disable RSpec/MultipleMemoizedHelpers context 'with WRITE redis commands' do + let_it_be(:ikey1) { "counter1" } + let_it_be(:ikey2) { "counter2" } + let_it_be(:iargs) { [ikey2, 3] } + let_it_be(:ivalue1) { "1" } + let_it_be(:ivalue2) { "3" } let_it_be(:key1) { "redis:{1}:key_a" } let_it_be(:key2) { "redis:{1}:key_b" } + let_it_be(:key3) { "redis:{1}:key_c" } + let_it_be(:key4) { "redis:{1}:key_d" } let_it_be(:value1) { "redis_value1" } let_it_be(:value2) { "redis_value2" } let_it_be(:key1_value1) { [key1, value1] } @@ -331,27 +512,50 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do let_it_be(:skey_value1) { [skey, [value1]] } let_it_be(:skey_value2) { [skey, [value2]] } let_it_be(:script) { %(redis.call("set", "#{key1}", "#{value1}")) } - + let_it_be(:hkey1) { "redis:{1}:hash_a" } + let_it_be(:hkey2) { "redis:{1}:hash_b" } + let_it_be(:item) { "item" } + let_it_be(:hdelarg) { [hkey1, item] } + let_it_be(:hsetarg) { [hkey2, item, value1] } + let_it_be(:mhsetarg) { [hkey2, { "item" => value1 }] } + let_it_be(:hgetarg) { [hkey2, item] } + let_it_be(:expireargs) { [key3, ttl] } + + # rubocop:disable Layout/LineLength where(:case_name, :name, :args, :expected_value, :verification_name, :verification_args) do - 'execute :set command' | :set | ref(:key1_value1) | ref(:value1) | :get | ref(:key1) - 'execute :setnx command' | :setnx | ref(:key1_value2) | ref(:value1) | :get | ref(:key2) - 'execute :setex command' | :setex | ref(:key1_ttl_value1) | ref(:ttl) | :ttl | ref(:key1) - 'execute :sadd command' | :sadd | ref(:skey_value2) | ref(:svalues1) | :smembers | ref(:skey) - 'execute :srem command' | :srem | ref(:skey_value1) | [] | :smembers | ref(:skey) - 'execute :del command' | :del | ref(:key2) | nil | :get | ref(:key2) - 'execute :flushdb command' | :flushdb | nil | 0 | :dbsize | nil - 'execute :eval command' | :eval | ref(:script) | ref(:value1) | :get | ref(:key1) + 'execute :set command' | :set | ref(:key1_value1) | ref(:value1) | :get | ref(:key1) + 'execute :setnx command' | :setnx | ref(:key1_value2) | ref(:value1) | :get | ref(:key2) + 'execute :setex command' | :setex | ref(:key1_ttl_value1) | ref(:ttl) | :ttl | ref(:key1) + 'execute :sadd command' | :sadd | ref(:skey_value2) | ref(:svalues1) | :smembers | ref(:skey) + 'execute :srem command' | :srem | ref(:skey_value1) | [] | :smembers | ref(:skey) + 'execute :del command' | :del | ref(:key2) | nil | :get | ref(:key2) + 'execute :unlink command' | :unlink | ref(:key3) | nil | :get | ref(:key3) + 'execute :flushdb command' | :flushdb | nil | 0 | :dbsize | nil + 'execute :eval command' | :eval | ref(:script) | ref(:value1) | :get | ref(:key1) + 'execute :incr command' | :incr | ref(:ikey1) | ref(:ivalue1) | :get | ref(:ikey1) + 'execute :incrby command' | :incrby | ref(:iargs) | ref(:ivalue2) | :get | ref(:ikey2) + 'execute :hset command' | :hset | ref(:hsetarg) | ref(:value1) | :hget | ref(:hgetarg) + 'execute :hdel command' | :hdel | ref(:hdelarg) | nil | :hget | ref(:hdelarg) + 'execute :expire command' | :expire | ref(:expireargs) | ref(:ttl) | :ttl | ref(:key3) + 'execute :mapped_hmset command' | :mapped_hmset | ref(:mhsetarg) | ref(:value1) | :hget | ref(:hgetarg) end + # rubocop:enable Layout/LineLength before do primary_store.flushdb secondary_store.flushdb primary_store.set(key2, value1) + primary_store.set(key3, value1) + primary_store.set(key4, value1) primary_store.sadd?(skey, value1) + primary_store.hset(hkey2, item, value1) secondary_store.set(key2, value1) + secondary_store.set(key3, value1) + secondary_store.set(key4, value1) secondary_store.sadd?(skey, value1) + secondary_store.hset(hkey2, item, value1) end with_them do @@ -375,6 +579,34 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do include_examples 'verify that store contains values', :secondary_store end + context 'when use_primary_and_secondary_stores feature flag is disabled' do + before do + stub_feature_flags(use_primary_and_secondary_stores_for_test_store: false) + end + + context 'when using secondary store as default' do + before do + stub_feature_flags(use_primary_store_as_default_for_test_store: false) + end + + it 'executes only on secondary redis store', :aggregate_errors do + expect(secondary_store).to receive(name).with(*expected_args).and_call_original + expect(primary_store).not_to receive(name).with(*expected_args).and_call_original + + subject + end + end + + context 'when using primary store as default' do + it 'executes only on primary redis store', :aggregate_errors do + expect(primary_store).to receive(name).with(*expected_args).and_call_original + expect(secondary_store).not_to receive(name).with(*expected_args).and_call_original + + subject + end + end + end + context 'when executing on the primary instance is raising an exception' do before do allow(primary_store).to receive(name).with(*expected_args).and_raise(StandardError) @@ -419,6 +651,121 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do end end end + # rubocop:enable RSpec/MultipleMemoizedHelpers + + context 'with ENUMERATOR_COMMANDS redis commands' do + let_it_be(:hkey) { "redis:hash" } + let_it_be(:skey) { "redis:set" } + let_it_be(:zkey) { "redis:sortedset" } + let_it_be(:rvalue) { "value1" } + let_it_be(:scan_kwargs) { { match: 'redis:hash' } } + + where(:case_name, :name, :args, :kwargs) do + 'execute :scan_each command' | :scan_each | nil | ref(:scan_kwargs) + 'execute :sscan_each command' | :sscan_each | ref(:skey) | {} + 'execute :hscan_each command' | :hscan_each | ref(:hkey) | {} + 'execute :zscan_each command' | :zscan_each | ref(:zkey) | {} + end + + before(:all) do + primary_store.hset(hkey, rvalue, 1) + primary_store.sadd?(skey, rvalue) + primary_store.zadd(zkey, 1, rvalue) + + secondary_store.hset(hkey, rvalue, 1) + secondary_store.sadd?(skey, rvalue) + secondary_store.zadd(zkey, 1, rvalue) + end + + RSpec.shared_examples_for 'enumerator commands execution' do |both_stores, default_primary| + context 'without block passed in' do + subject do + multi_store.send(name, *args, **kwargs) + end + + it 'returns an enumerator' do + expect(subject).to be_instance_of(Enumerator) + end + end + + context 'with block passed in' do + subject do + multi_store.send(name, *args, **kwargs) { |key| multi_store.incr(rvalue) } + end + + it 'returns nil' do + expect(subject).to eq(nil) + end + + it 'runs block on correct Redis instance' do + if both_stores + expect(primary_store).to receive(name).with(*expected_args).and_call_original + expect(secondary_store).not_to receive(name) + + expect(primary_store).to receive(:incr).with(rvalue) + expect(secondary_store).to receive(:incr).with(rvalue) + elsif default_primary + expect(primary_store).to receive(name).with(*expected_args).and_call_original + expect(primary_store).to receive(:incr).with(rvalue) + + expect(secondary_store).not_to receive(name) + expect(secondary_store).not_to receive(:incr).with(rvalue) + else + expect(secondary_store).to receive(name).with(*expected_args).and_call_original + expect(secondary_store).to receive(:incr).with(rvalue) + + expect(primary_store).not_to receive(name) + expect(primary_store).not_to receive(:incr).with(rvalue) + end + + subject + end + end + end + + with_them do + describe name.to_s do + let(:expected_args) { kwargs.present? ? [*args, { **kwargs }] : Array(args) } + + before do + allow(primary_store).to receive(name).and_call_original + allow(secondary_store).to receive(name).and_call_original + end + + context 'when only using 1 store' do + before do + stub_feature_flags(use_primary_and_secondary_stores_for_test_store: false) + end + + context 'when using secondary store as default' do + before do + stub_feature_flags(use_primary_store_as_default_for_test_store: false) + end + + it_behaves_like 'enumerator commands execution', false, false + end + + context 'when using primary store as default' do + it_behaves_like 'enumerator commands execution', false, true + end + end + + context 'when using both stores' do + context 'when using secondary store as default' do + before do + stub_feature_flags(use_primary_store_as_default_for_test_store: false) + end + + it_behaves_like 'enumerator commands execution', true, false + end + + context 'when using primary store as default' do + it_behaves_like 'enumerator commands execution', true, true + end + end + end + end + end RSpec.shared_examples_for 'pipelined command' do |name| let_it_be(:key1) { "redis:{1}:key_a" } @@ -554,6 +901,34 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do end end end + + context 'when use_primary_and_secondary_stores feature flag is disabled' do + before do + stub_feature_flags(use_primary_and_secondary_stores_for_test_store: false) + end + + context 'when using secondary store as default' do + before do + stub_feature_flags(use_primary_store_as_default_for_test_store: false) + end + + it 'executes on secondary store', :aggregate_errors do + expect(primary_store).not_to receive(:send).and_call_original + expect(secondary_store).to receive(:send).and_call_original + + subject + end + end + + context 'when using primary store as default' do + it 'executes on primary store', :aggregate_errors do + expect(secondary_store).not_to receive(:send).and_call_original + expect(primary_store).to receive(:send).and_call_original + + subject + end + end + end end end @@ -565,129 +940,211 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do include_examples 'pipelined command', :pipelined end - context 'with unsupported command' do - let(:counter) { Gitlab::Metrics::NullMetric.instance } - - before do - primary_store.flushdb - secondary_store.flushdb - allow(Gitlab::Metrics).to receive(:counter).and_return(counter) - end - - let_it_be(:key) { "redis:counter" } + describe '#ping' do + subject { multi_store.ping } - subject { multi_store.incr(key) } + context 'when using both stores' do + before do + allow(multi_store).to receive(:use_primary_and_secondary_stores?).and_return(true) + end - it 'responds to missing method' do - expect(multi_store).to receive(:respond_to_missing?).and_call_original + context 'without message' do + it 'returns PONG' do + expect(subject).to eq('PONG') + end + end - expect(multi_store.respond_to?(:incr)).to be(true) - end + context 'with message' do + it 'returns the same message' do + expect(multi_store.ping('hello world')).to eq('hello world') + end + end - it 'executes method missing' do - expect(multi_store).to receive(:method_missing) + shared_examples 'returns an error' do + before do + allow(store).to receive(:ping).and_raise('boom') + end - subject - end + it 'returns the error' do + expect { subject }.to raise_error('boom') + end + end - context 'when command is not in SKIP_LOG_METHOD_MISSING_FOR_COMMANDS' do - it 'logs MethodMissingError' do - expect(Gitlab::ErrorTracking).to receive(:log_exception).with( - an_instance_of(Gitlab::Redis::MultiStore::MethodMissingError), - hash_including(command_name: :incr, instance_name: instance_name) - ) + context 'when primary store returns an error' do + let(:store) { primary_store } - subject + it_behaves_like 'returns an error' end - it 'increments method missing counter' do - expect(counter).to receive(:increment).with(command: :incr, instance_name: instance_name) + context 'when secondary store returns an error' do + let(:store) { secondary_store } - subject + it_behaves_like 'returns an error' end end - context 'when command is in SKIP_LOG_METHOD_MISSING_FOR_COMMANDS' do - subject { multi_store.info } + shared_examples 'single store as default store' do + context 'when the store retuns success' do + it 'returns response from the respective store' do + expect(store).to receive(:ping).and_return('PONG') - it 'does not log MethodMissingError' do - expect(Gitlab::ErrorTracking).not_to receive(:log_exception) + subject - subject + expect(subject).to eq('PONG') + end end - it 'does not increment method missing counter' do - expect(counter).not_to receive(:increment) + context 'when the store returns an error' do + before do + allow(store).to receive(:ping).and_raise('boom') + end - subject + it 'returns the error' do + expect { subject }.to raise_error('boom') + end end end - context 'with feature flag :use_primary_store_as_default_for_test_store is enabled' do + context 'when using only one store' do before do - stub_feature_flags(use_primary_store_as_default_for_test_store: true) + allow(multi_store).to receive(:use_primary_and_secondary_stores?).and_return(false) end - it 'fallback and executes only on the secondary store', :aggregate_errors do - expect(primary_store).to receive(:incr).with(key).and_call_original - expect(secondary_store).not_to receive(:incr) + context 'when using primary_store as default store' do + let(:store) { primary_store } - subject + before do + allow(multi_store).to receive(:use_primary_store_as_default?).and_return(true) + end + + it_behaves_like 'single store as default store' end - it 'correct value is stored on the secondary store', :aggregate_errors do - subject + context 'when using secondary_store as default store' do + let(:store) { secondary_store } - expect(secondary_store.get(key)).to be_nil - expect(primary_store.get(key)).to eq('1') + before do + allow(multi_store).to receive(:use_primary_store_as_default?).and_return(false) + end + + it_behaves_like 'single store as default store' end end + end - context 'with feature flag :use_primary_store_as_default_for_test_store is disabled' do + context 'with unsupported command' do + let(:counter) { Gitlab::Metrics::NullMetric.instance } + + before do + primary_store.flushdb + secondary_store.flushdb + allow(Gitlab::Metrics).to receive(:counter).and_return(counter) + end + + subject { multi_store.command } + + context 'when in test environment' do + it 'raises error' do + expect { subject }.to raise_error(instance_of(Gitlab::Redis::MultiStore::MethodMissingError)) + end + end + + context 'when not in test environment' do before do - stub_feature_flags(use_primary_store_as_default_for_test_store: false) + stub_rails_env('production') end - it 'fallback and executes only on the secondary store', :aggregate_errors do - expect(secondary_store).to receive(:incr).with(key).and_call_original - expect(primary_store).not_to receive(:incr) + it 'responds to missing method' do + expect(multi_store).to receive(:respond_to_missing?).and_call_original - subject + expect(multi_store.respond_to?(:command)).to be(true) end - it 'correct value is stored on the secondary store', :aggregate_errors do + it 'executes method missing' do + expect(multi_store).to receive(:method_missing) + subject + end + + context 'when command is not in SKIP_LOG_METHOD_MISSING_FOR_COMMANDS' do + it 'logs MethodMissingError' do + expect(Gitlab::ErrorTracking).to receive(:log_exception).with( + an_instance_of(Gitlab::Redis::MultiStore::MethodMissingError), + hash_including(command_name: :command, instance_name: instance_name) + ) + + subject + end + + it 'increments method missing counter' do + expect(counter).to receive(:increment).with(command: :command, instance_name: instance_name) + + subject + end - expect(primary_store.get(key)).to be_nil - expect(secondary_store.get(key)).to eq('1') + it 'fallback and executes only on the secondary store', :aggregate_errors do + expect(primary_store).to receive(:command).and_call_original + expect(secondary_store).not_to receive(:command) + + subject + end end - end - context 'when the command is executed within pipelined block' do - subject do - multi_store.pipelined do |pipeline| - pipeline.incr(key) + context 'when command is in SKIP_LOG_METHOD_MISSING_FOR_COMMANDS' do + subject { multi_store.info } + + it 'does not log MethodMissingError' do + expect(Gitlab::ErrorTracking).not_to receive(:log_exception) + + subject + end + + it 'does not increment method missing counter' do + expect(counter).not_to receive(:increment) + + subject end end - it 'is executed only 1 time on each instance', :aggregate_errors do - expect(primary_store).to receive(:pipelined).once.and_call_original - expect(secondary_store).to receive(:pipelined).once.and_call_original + context 'with feature flag :use_primary_store_as_default_for_test_store is enabled' do + it 'fallback and executes only on the secondary store', :aggregate_errors do + expect(primary_store).to receive(:command).and_call_original + expect(secondary_store).not_to receive(:command) - 2.times do - expect_next_instance_of(Redis::PipelinedConnection) do |pipeline| - expect(pipeline).to receive(:incr).with(key).once - end + subject end + end - subject + context 'with feature flag :use_primary_store_as_default_for_test_store is disabled' do + before do + stub_feature_flags(use_primary_store_as_default_for_test_store: false) + end + + it 'fallback and executes only on the secondary store', :aggregate_errors do + expect(secondary_store).to receive(:command).and_call_original + expect(primary_store).not_to receive(:command) + + subject + end end - it "both redis stores are containing correct values", :aggregate_errors do - subject + context 'when the command is executed within pipelined block' do + subject do + multi_store.pipelined(&:command) + end + + it 'is executed only 1 time on each instance', :aggregate_errors do + expect(primary_store).to receive(:pipelined).once.and_call_original + expect(secondary_store).to receive(:pipelined).once.and_call_original + + 2.times do + expect_next_instance_of(Redis::PipelinedConnection) do |pipeline| + expect(pipeline).to receive(:command).once + end + end - expect(primary_store.get(key)).to eq('1') - expect(secondary_store.get(key)).to eq('1') + subject + end end end end diff --git a/spec/lib/gitlab/redis/repository_cache_spec.rb b/spec/lib/gitlab/redis/repository_cache_spec.rb new file mode 100644 index 00000000000..b11e9ebf1f3 --- /dev/null +++ b/spec/lib/gitlab/redis/repository_cache_spec.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Redis::RepositoryCache, feature_category: :scalability do + include_examples "redis_new_instance_shared_examples", 'repository_cache', Gitlab::Redis::Cache + include_examples "redis_shared_examples" + + describe '#pool' do + let(:config_new_format_host) { "spec/fixtures/config/redis_new_format_host.yml" } + let(:config_new_format_socket) { "spec/fixtures/config/redis_new_format_socket.yml" } + + subject { described_class.pool } + + before do + allow(described_class).to receive(:config_file_name).and_return(config_new_format_host) + allow(Gitlab::Redis::Cache).to receive(:config_file_name).and_return(config_new_format_socket) + end + + around do |example| + clear_pool + example.run + ensure + clear_pool + end + + it 'instantiates an instance of MultiStore' do + subject.with do |redis_instance| + expect(redis_instance).to be_instance_of(::Gitlab::Redis::MultiStore) + + expect(redis_instance.primary_store.connection[:id]).to eq("redis://test-host:6379/99") + expect(redis_instance.secondary_store.connection[:id]).to eq("unix:///path/to/redis.sock/0") + + expect(redis_instance.instance_name).to eq('RepositoryCache') + end + end + + it_behaves_like 'multi store feature flags', :use_primary_and_secondary_stores_for_repository_cache, + :use_primary_store_as_default_for_repository_cache + end + + describe '#raw_config_hash' do + it 'has a legacy default URL' do + expect(subject).to receive(:fetch_config).and_return(false) + + expect(subject.send(:raw_config_hash)).to eq(url: 'redis://localhost:6380') + end + end +end diff --git a/spec/lib/gitlab/redis/sidekiq_status_spec.rb b/spec/lib/gitlab/redis/sidekiq_status_spec.rb index 76d130d67f7..e7cf229b494 100644 --- a/spec/lib/gitlab/redis/sidekiq_status_spec.rb +++ b/spec/lib/gitlab/redis/sidekiq_status_spec.rb @@ -18,18 +18,10 @@ RSpec.describe Gitlab::Redis::SidekiqStatus do subject { described_class.pool } before do - redis_clear_raw_config!(Gitlab::Redis::SharedState) - redis_clear_raw_config!(Gitlab::Redis::Queues) - allow(Gitlab::Redis::SharedState).to receive(:config_file_name).and_return(config_new_format_host) allow(Gitlab::Redis::Queues).to receive(:config_file_name).and_return(config_new_format_socket) end - after do - redis_clear_raw_config!(Gitlab::Redis::SharedState) - redis_clear_raw_config!(Gitlab::Redis::Queues) - end - around do |example| clear_pool example.run diff --git a/spec/lib/gitlab/regex_spec.rb b/spec/lib/gitlab/regex_spec.rb index 89ef76d246e..9532a30144f 100644 --- a/spec/lib/gitlab/regex_spec.rb +++ b/spec/lib/gitlab/regex_spec.rb @@ -7,7 +7,7 @@ require_relative '../../support/shared_examples/lib/gitlab/regex_shared_examples # All specs that can be run with fast_spec_helper only # See regex_requires_app_spec for tests that require the full spec_helper -RSpec.describe Gitlab::Regex do +RSpec.describe Gitlab::Regex, feature_category: :tooling do shared_examples_for 'project/group name chars regex' do it { is_expected.to match('gitlab-ce') } it { is_expected.to match('GitLab CE') } @@ -72,6 +72,59 @@ RSpec.describe Gitlab::Regex do it { is_expected.to eq("can contain only letters, digits, emojis, '_', '.', dash, space, parenthesis. It must start with letter, digit, emoji or '_'.") } end + describe '.bulk_import_namespace_path_regex' do + subject { described_class.bulk_import_namespace_path_regex } + + it { is_expected.not_to match('?gitlab') } + it { is_expected.not_to match("Users's something") } + it { is_expected.not_to match('/source') } + it { is_expected.not_to match('http:') } + it { is_expected.not_to match('https:') } + it { is_expected.not_to match('example.com/?stuff=true') } + it { is_expected.not_to match('example.com:5000/?stuff=true') } + it { is_expected.not_to match('http://gitlab.example/gitlab-org/manage/import/gitlab-migration-test') } + it { is_expected.not_to match('_good_for_me!') } + it { is_expected.not_to match('good_for+you') } + it { is_expected.not_to match('source/') } + it { is_expected.not_to match('.source/full./path') } + + it { is_expected.to match('source') } + it { is_expected.to match('.source') } + it { is_expected.to match('_source') } + it { is_expected.to match('source/full') } + it { is_expected.to match('source/full/path') } + it { is_expected.to match('.source/.full/.path') } + it { is_expected.to match('domain_namespace') } + it { is_expected.to match('gitlab-migration-test') } + end + + describe '.group_path_regex' do + subject { described_class.group_path_regex } + + it { is_expected.not_to match('?gitlab') } + it { is_expected.not_to match("Users's something") } + it { is_expected.not_to match('/source') } + it { is_expected.not_to match('http:') } + it { is_expected.not_to match('https:') } + it { is_expected.not_to match('example.com/?stuff=true') } + it { is_expected.not_to match('example.com:5000/?stuff=true') } + it { is_expected.not_to match('http://gitlab.example/gitlab-org/manage/import/gitlab-migration-test') } + it { is_expected.not_to match('_good_for_me!') } + it { is_expected.not_to match('good_for+you') } + it { is_expected.not_to match('source/') } + it { is_expected.not_to match('.source/full./path') } + + it { is_expected.not_to match('source/full') } + it { is_expected.not_to match('source/full/path') } + it { is_expected.not_to match('.source/.full/.path') } + + it { is_expected.to match('source') } + it { is_expected.to match('.source') } + it { is_expected.to match('_source') } + it { is_expected.to match('domain_namespace') } + it { is_expected.to match('gitlab-migration-test') } + end + describe '.environment_name_regex' do subject { described_class.environment_name_regex } diff --git a/spec/lib/gitlab/relative_positioning/mover_spec.rb b/spec/lib/gitlab/relative_positioning/mover_spec.rb index cbb15ae876d..85e985b1b6f 100644 --- a/spec/lib/gitlab/relative_positioning/mover_spec.rb +++ b/spec/lib/gitlab/relative_positioning/mover_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe RelativePositioning::Mover do +RSpec.describe RelativePositioning::Mover, feature_category: :portfolio_management do let_it_be(:user) { create(:user) } let_it_be(:one_sibling, reload: true) { create(:project, creator: user, namespace: user.namespace) } let_it_be(:one_free_space, reload: true) { create(:project, creator: user, namespace: user.namespace) } diff --git a/spec/lib/gitlab/repository_cache/preloader_spec.rb b/spec/lib/gitlab/repository_cache/preloader_spec.rb index 8c6618c9f8f..71244dd41ed 100644 --- a/spec/lib/gitlab/repository_cache/preloader_spec.rb +++ b/spec/lib/gitlab/repository_cache/preloader_spec.rb @@ -2,53 +2,80 @@ require 'spec_helper' -RSpec.describe Gitlab::RepositoryCache::Preloader, :use_clean_rails_redis_caching do +RSpec.describe Gitlab::RepositoryCache::Preloader, :use_clean_rails_redis_caching, + feature_category: :source_code_management do let(:projects) { create_list(:project, 2, :repository) } let(:repositories) { projects.map(&:repository) } - describe '#preload' do - context 'when the values are already cached' do - before do - # Warm the cache but use a different model so they are not memoized - repos = Project.id_in(projects).order(:id).map(&:repository) + before do + stub_feature_flags(use_primary_store_as_default_for_repository_cache: false) + end - allow(repos[0].head_tree).to receive(:readme_path).and_return('README.txt') - allow(repos[1].head_tree).to receive(:readme_path).and_return('README.md') + shared_examples 'preload' do + describe '#preload' do + context 'when the values are already cached' do + before do + # Warm the cache but use a different model so they are not memoized + repos = Project.id_in(projects).order(:id).map(&:repository) - repos.map(&:exists?) - repos.map(&:readme_path) - end + allow(repos[0]).to receive(:readme_path_gitaly).and_return('README.txt') + allow(repos[1]).to receive(:readme_path_gitaly).and_return('README.md') - it 'prevents individual cache reads for cached methods' do - expect(Rails.cache).to receive(:read_multi).once.and_call_original + repos.map(&:exists?) + repos.map(&:readme_path) + end - described_class.new(repositories).preload( - %i[exists? readme_path] - ) + it 'prevents individual cache reads for cached methods' do + expect(cache).to receive(:read_multi).once.and_call_original - expect(Rails.cache).not_to receive(:read) - expect(Rails.cache).not_to receive(:write) + described_class.new(repositories).preload( + %i[exists? readme_path] + ) - expect(repositories[0].exists?).to eq(true) - expect(repositories[0].readme_path).to eq('README.txt') + expect(cache).not_to receive(:read) + expect(cache).not_to receive(:write) - expect(repositories[1].exists?).to eq(true) - expect(repositories[1].readme_path).to eq('README.md') + expect(repositories[0].exists?).to eq(true) + expect(repositories[0].readme_path).to eq('README.txt') + + expect(repositories[1].exists?).to eq(true) + expect(repositories[1].readme_path).to eq('README.md') + end end - end - context 'when values are not cached' do - it 'reads and writes from cache individually' do - described_class.new(repositories).preload( - %i[exists? has_visible_content?] - ) + context 'when values are not cached' do + it 'reads and writes from cache individually' do + described_class.new(repositories).preload( + %i[exists? has_visible_content?] + ) - expect(Rails.cache).to receive(:read).exactly(4).times - expect(Rails.cache).to receive(:write).exactly(4).times + expect(cache).to receive(:read).exactly(4).times + expect(cache).to receive(:write).exactly(4).times - repositories.each(&:exists?) - repositories.each(&:has_visible_content?) + repositories.each(&:exists?) + repositories.each(&:has_visible_content?) + end end end end + + context 'when use_primary_and_secondary_stores_for_repository_cache feature flag is enabled' do + let(:cache) { Gitlab::RepositoryCache.store } + + before do + stub_feature_flags(use_primary_and_secondary_stores_for_repository_cache: true) + end + + it_behaves_like 'preload' + end + + context 'when use_primary_and_secondary_stores_for_repository_cache feature flag is disabled' do + let(:cache) { Rails.cache } + + before do + stub_feature_flags(use_primary_and_secondary_stores_for_repository_cache: false) + end + + it_behaves_like 'preload' + end end diff --git a/spec/lib/gitlab/repository_hash_cache_spec.rb b/spec/lib/gitlab/repository_hash_cache_spec.rb index 6b52c315a70..d41bf45f72e 100644 --- a/spec/lib/gitlab/repository_hash_cache_spec.rb +++ b/spec/lib/gitlab/repository_hash_cache_spec.rb @@ -69,20 +69,35 @@ RSpec.describe Gitlab::RepositoryHashCache, :clean_gitlab_redis_cache do end end - describe "#key?" do - subject { cache.key?(:example, "test") } + shared_examples "key?" do + describe "#key?" do + subject { cache.key?(:example, "test") } - context "key exists" do - before do - cache.write(:example, test_hash) + context "key exists" do + before do + cache.write(:example, test_hash) + end + + it { is_expected.to be(true) } end - it { is_expected.to be(true) } + context "key doesn't exist" do + it { is_expected.to be(false) } + end end + end - context "key doesn't exist" do - it { is_expected.to be(false) } + context "when both multistore FF is enabled" do + it_behaves_like "key?" + end + + context "when both multistore FF is disabled" do + before do + stub_feature_flags(use_primary_and_secondary_stores_for_repository_cache: false) + stub_feature_flags(use_primary_store_as_default_for_repository_cache: false) end + + it_behaves_like "key?" end describe "#read_members" do diff --git a/spec/lib/gitlab/seeders/ci/runner/runner_fleet_pipeline_seeder_spec.rb b/spec/lib/gitlab/seeders/ci/runner/runner_fleet_pipeline_seeder_spec.rb new file mode 100644 index 00000000000..2862bcc9719 --- /dev/null +++ b/spec/lib/gitlab/seeders/ci/runner/runner_fleet_pipeline_seeder_spec.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +require 'spec_helper' + +NULL_LOGGER = Gitlab::JsonLogger.new('/dev/null') +TAG_LIST = Gitlab::Seeders::Ci::Runner::RunnerFleetSeeder::TAG_LIST.to_set + +RSpec.describe ::Gitlab::Seeders::Ci::Runner::RunnerFleetPipelineSeeder, feature_category: :runner_fleet do + subject(:seeder) do + described_class.new(NULL_LOGGER, projects_to_runners: projects_to_runners, job_count: job_count) + end + + def runner_ids_for_project(runner_count, project) + create_list(:ci_runner, runner_count, :project, projects: [project], tag_list: TAG_LIST.to_a.sample(5)).map(&:id) + end + + let_it_be(:projects) { create_list(:project, 4) } + let_it_be(:projects_to_runners) do + [ + { project_id: projects[0].id, runner_ids: runner_ids_for_project(2, projects[0]) }, + { project_id: projects[1].id, runner_ids: runner_ids_for_project(1, projects[1]) }, + { project_id: projects[2].id, runner_ids: runner_ids_for_project(2, projects[2]) }, + { project_id: projects[3].id, runner_ids: runner_ids_for_project(1, projects[3]) } + ] + end + + describe '#seed' do + context 'with job_count specified' do + let(:job_count) { 20 } + + it 'creates expected jobs', :aggregate_failures do + expect { seeder.seed }.to change { Ci::Build.count }.by(job_count) + .and change { Ci::Pipeline.count }.by(4) + + expect(Ci::Pipeline.where.not(started_at: nil).map(&:queued_duration)).to all(be < 5.minutes) + expect(Ci::Build.where.not(started_at: nil).map(&:queued_duration)).to all(be < 5.minutes) + + projects_to_runners.first(3).each do |project| + expect(Ci::Build.where(runner_id: project[:runner_ids])).not_to be_empty + end + end + end + + context 'with nil job_count' do + let(:job_count) { nil } + + before do + stub_const('Gitlab::Seeders::Ci::Runner::RunnerFleetPipelineSeeder::DEFAULT_JOB_COUNT', 2) + end + + it 'creates expected jobs', :aggregate_failures do + expect { seeder.seed }.to change { Ci::Build.count }.by(2) + .and change { Ci::Pipeline.count }.by(2) + expect(Ci::Build.last(2).map(&:tag_list).map(&:to_set)).to all satisfy { |r| r.subset?(TAG_LIST) } + end + end + end +end diff --git a/spec/lib/gitlab/seeders/ci/runner/runner_fleet_seeder_spec.rb b/spec/lib/gitlab/seeders/ci/runner/runner_fleet_seeder_spec.rb new file mode 100644 index 00000000000..fe52b586d49 --- /dev/null +++ b/spec/lib/gitlab/seeders/ci/runner/runner_fleet_seeder_spec.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +require 'spec_helper' + +NULL_LOGGER = Gitlab::JsonLogger.new('/dev/null') + +RSpec.describe ::Gitlab::Seeders::Ci::Runner::RunnerFleetSeeder, feature_category: :runner_fleet do + let_it_be(:user) { create(:user, :admin, username: 'test-admin') } + + subject(:seeder) do + described_class.new(NULL_LOGGER, + username: user.username, + registration_prefix: registration_prefix, + runner_count: runner_count) + end + + describe '#seed', :enable_admin_mode do + subject(:seed) { seeder.seed } + + let(:runner_count) { 20 } + let(:registration_prefix) { 'prefix-' } + let(:runner_releases_url) do + ::Gitlab::CurrentSettings.current_application_settings.public_runner_releases_url + end + + before do + WebMock.stub_request(:get, runner_releases_url).to_return( + body: '[]', + status: 200, + headers: { 'Content-Type' => 'application/json' } + ) + end + + it 'creates expected hierarchy', :aggregate_failures do + expect { seed }.to change { Ci::Runner.count }.by(runner_count) + .and change { Ci::Runner.instance_type.count }.by(1) + .and change { Project.count }.by(3) + .and change { Group.count }.by(6) + + expect(Group.search(registration_prefix)).to contain_exactly( + an_object_having_attributes(name: "#{registration_prefix}top-level group 1"), + an_object_having_attributes(name: "#{registration_prefix}top-level group 2"), + an_object_having_attributes(name: "#{registration_prefix}group 1.1"), + an_object_having_attributes(name: "#{registration_prefix}group 1.1.1"), + an_object_having_attributes(name: "#{registration_prefix}group 1.1.2"), + an_object_having_attributes(name: "#{registration_prefix}group 2.1") + ) + + expect(Project.search(registration_prefix)).to contain_exactly( + an_object_having_attributes(name: "#{registration_prefix}project 1.1.1.1"), + an_object_having_attributes(name: "#{registration_prefix}project 1.1.2.1"), + an_object_having_attributes(name: "#{registration_prefix}project 2.1.1") + ) + + project_1_1_1_1 = Project.find_by_name("#{registration_prefix}project 1.1.1.1") + project_1_1_2_1 = Project.find_by_name("#{registration_prefix}project 1.1.2.1") + project_2_1_1 = Project.find_by_name("#{registration_prefix}project 2.1.1") + expect(seed).to contain_exactly( + { project_id: project_1_1_1_1.id, runner_ids: an_instance_of(Array) }, + { project_id: project_1_1_2_1.id, runner_ids: an_instance_of(Array) }, + { project_id: project_2_1_1.id, runner_ids: an_instance_of(Array) } + ) + seed.each do |project| + expect(project[:runner_ids].length).to be_between(0, 5) + expect(Project.find(project[:project_id]).all_available_runners.ids).to include(*project[:runner_ids]) + expect(::Ci::Pipeline.for_project(project[:runner_ids])).to be_empty + expect(::Ci::Build.where(runner_id: project[:runner_ids])).to be_empty + end + end + end +end diff --git a/spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb b/spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb index 5baeec93036..6f46a5aea3b 100644 --- a/spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb +++ b/spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb @@ -307,10 +307,10 @@ RSpec.describe Gitlab::SidekiqDaemon::MemoryKiller do end describe '#signal_and_wait' do - let(:time) { 0 } + let(:time) { 0.1 } let(:signal) { 'my-signal' } let(:explanation) { 'my-explanation' } - let(:check_interval_seconds) { 2 } + let(:check_interval_seconds) { 0.1 } subject { memory_killer.send(:signal_and_wait, time, signal, explanation) } @@ -318,37 +318,19 @@ RSpec.describe Gitlab::SidekiqDaemon::MemoryKiller do stub_const("#{described_class}::CHECK_INTERVAL_SECONDS", check_interval_seconds) end - context 'when all jobs are finished' do - let(:running_jobs) { {} } - - it 'send signal and return when all jobs finished' do - expect(Process).to receive(:kill).with(signal, pid).ordered - expect(Gitlab::Metrics::System).to receive(:monotonic_time).and_call_original - - expect(memory_killer).to receive(:enabled?).and_return(true) - - expect(memory_killer).not_to receive(:sleep) - - subject - end - end + it 'send signal and wait till deadline' do + expect(Process).to receive(:kill) + .with(signal, pid) + .ordered - context 'when there are still running jobs' do - let(:running_jobs) { { 'jid1' => { worker_class: DummyWorker } } } - - it 'send signal and wait till deadline if any job not finished' do - expect(Process).to receive(:kill) - .with(signal, pid) - .ordered - - expect(Gitlab::Metrics::System).to receive(:monotonic_time) - .and_call_original - .at_least(:once) + expect(Gitlab::Metrics::System).to receive(:monotonic_time) + .and_call_original + .at_least(3) - expect(memory_killer).to receive(:enabled?).and_return(true).at_least(:once) + expect(memory_killer).to receive(:enabled?).and_return(true).at_least(:twice) + expect(memory_killer).to receive(:sleep).at_least(:once).and_call_original - subject - end + subject end end diff --git a/spec/lib/gitlab/ssh/commit_spec.rb b/spec/lib/gitlab/ssh/commit_spec.rb index cc977a80f95..77f37857c82 100644 --- a/spec/lib/gitlab/ssh/commit_spec.rb +++ b/spec/lib/gitlab/ssh/commit_spec.rb @@ -1,9 +1,10 @@ # frozen_string_literal: true require 'spec_helper' -RSpec.describe Gitlab::Ssh::Commit do +RSpec.describe Gitlab::Ssh::Commit, feature_category: :source_code_management do let_it_be(:project) { create(:project, :repository) } let_it_be(:signed_by_key) { create(:key) } + let_it_be(:fingerprint) { signed_by_key.fingerprint_sha256 } let(:commit) { create(:commit, project: project) } let(:signature_text) { 'signature_text' } @@ -19,8 +20,11 @@ RSpec.describe Gitlab::Ssh::Commit do .with(Gitlab::Git::Repository, commit.sha) .and_return(signature_data) - allow(verifier).to receive(:verification_status).and_return(verification_status) - allow(verifier).to receive(:signed_by_key).and_return(signed_by_key) + allow(verifier).to receive_messages({ + verification_status: verification_status, + signed_by_key: signed_by_key, + key_fingerprint: fingerprint + }) allow(Gitlab::Ssh::Signature).to receive(:new) .with(signature_text, signed_text, commit.committer_email) @@ -44,6 +48,8 @@ RSpec.describe Gitlab::Ssh::Commit do commit_sha: commit.sha, project: project, key_id: signed_by_key.id, + key_fingerprint_sha256: signed_by_key.fingerprint_sha256, + user_id: signed_by_key.user_id, verification_status: 'verified' ) end @@ -51,6 +57,7 @@ RSpec.describe Gitlab::Ssh::Commit do context 'when signed_by_key is nil' do let_it_be(:signed_by_key) { nil } + let_it_be(:fingerprint) { nil } let(:verification_status) { :unknown_key } @@ -59,6 +66,8 @@ RSpec.describe Gitlab::Ssh::Commit do commit_sha: commit.sha, project: project, key_id: nil, + key_fingerprint_sha256: nil, + user_id: nil, verification_status: 'unknown_key' ) end diff --git a/spec/lib/gitlab/ssh/signature_spec.rb b/spec/lib/gitlab/ssh/signature_spec.rb index 5149972dbf9..ee9b38cae7d 100644 --- a/spec/lib/gitlab/ssh/signature_spec.rb +++ b/spec/lib/gitlab/ssh/signature_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ssh::Signature do +RSpec.describe Gitlab::Ssh::Signature, feature_category: :source_code_management do # ssh-keygen -t ed25519 let_it_be(:committer_email) { 'ssh-commit-test@example.com' } let_it_be(:public_key_text) { 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHZ8NHEnCIpC4mnot+BRxv6L+fq+TnN1CgsRrHWLmfwb' } @@ -267,4 +267,10 @@ RSpec.describe Gitlab::Ssh::Signature do end end end + + describe '#key_fingerprint' do + it 'returns the pubkey sha256 fingerprint' do + expect(signature.key_fingerprint).to eq('dw7gPSvYtkCBU+BbTolbbckUEX3sL6NsGIJTQ4PYEnM') + end + end end diff --git a/spec/lib/gitlab/submodule_links_spec.rb b/spec/lib/gitlab/submodule_links_spec.rb index e2bbda81780..12c322ea914 100644 --- a/spec/lib/gitlab/submodule_links_spec.rb +++ b/spec/lib/gitlab/submodule_links_spec.rb @@ -51,7 +51,7 @@ RSpec.describe Gitlab::SubmoduleLinks do expect(subject.compare).to be_nil end - cache_store = links.instance_variable_get("@cache_store") + cache_store = links.instance_variable_get(:@cache_store) expect(cache_store[ref]).to eq({ "gitlab-foss" => "git@gitlab.com:gitlab-org/gitlab-foss.git" }) end diff --git a/spec/lib/gitlab/tracking_spec.rb b/spec/lib/gitlab/tracking_spec.rb index 99ca402616a..e79bb2ef129 100644 --- a/spec/lib/gitlab/tracking_spec.rb +++ b/spec/lib/gitlab/tracking_spec.rb @@ -10,11 +10,11 @@ RSpec.describe Gitlab::Tracking do stub_application_setting(snowplow_cookie_domain: '.gitfoo.com') stub_application_setting(snowplow_app_id: '_abc123_') - described_class.instance_variable_set("@tracker", nil) + described_class.instance_variable_set(:@tracker, nil) end after do - described_class.instance_variable_set("@tracker", nil) + described_class.instance_variable_set(:@tracker, nil) end describe '.options' do diff --git a/spec/lib/gitlab/usage/metrics/aggregates/aggregate_spec.rb b/spec/lib/gitlab/usage/metrics/aggregates/aggregate_spec.rb index 10e336e9235..8be0769a379 100644 --- a/spec/lib/gitlab/usage/metrics/aggregates/aggregate_spec.rb +++ b/spec/lib/gitlab/usage/metrics/aggregates/aggregate_spec.rb @@ -58,6 +58,50 @@ RSpec.describe Gitlab::Usage::Metrics::Aggregates::Aggregate, :clean_gitlab_redi end end + # EE version has validation that doesn't allow undefined events + # On CE, we detect EE events as undefined + context 'when configuration includes undefined events', unless: Gitlab.ee? do + let(:number_of_days) { 28 } + + before do + allow(Gitlab::UsageDataCounters::HLLRedisCounter).to receive(:known_event?).with('event3').and_return(false) + end + + where(:operator, :datasource, :expected_method, :expected_events) do + 'AND' | 'redis_hll' | :calculate_metrics_intersections | %w[event1 event2] + 'AND' | 'database' | :calculate_metrics_intersections | %w[event1 event2 event3] + 'OR' | 'redis_hll' | :calculate_metrics_union | %w[event1 event2] + 'OR' | 'database' | :calculate_metrics_union | %w[event1 event2 event3] + end + + with_them do + let(:time_frame) { "#{number_of_days}d" } + let(:start_date) { number_of_days.days.ago.to_date } + let(:params) { { start_date: start_date, end_date: end_date, recorded_at: recorded_at } } + let(:aggregate) do + { + source: datasource, + operator: operator, + events: %w[event1 event2 event3] + } + end + + subject(:calculate_count_for_aggregation) do + described_class + .new(recorded_at) + .calculate_count_for_aggregation(aggregation: aggregate, time_frame: time_frame) + end + + it 'returns the number of unique events for aggregation', :aggregate_failures do + expect(namespace::SOURCES[datasource]) + .to receive(expected_method) + .with(params.merge(metric_names: expected_events)) + .and_return(5) + expect(calculate_count_for_aggregation).to eq(5) + end + end + end + context 'with invalid configuration' do where(:time_frame, :operator, :datasource, :expected_error) do '28d' | 'SUM' | 'redis_hll' | namespace::UnknownAggregationOperator diff --git a/spec/lib/gitlab/usage/service_ping/legacy_metric_timing_decorator_spec.rb b/spec/lib/gitlab/usage/service_ping/legacy_metric_metadata_decorator_spec.rb index 46592379b3d..c8c2feda234 100644 --- a/spec/lib/gitlab/usage/service_ping/legacy_metric_timing_decorator_spec.rb +++ b/spec/lib/gitlab/usage/service_ping/legacy_metric_metadata_decorator_spec.rb @@ -2,26 +2,31 @@ require 'spec_helper' -RSpec.describe Gitlab::Usage::ServicePing::LegacyMetricTimingDecorator do +RSpec.describe Gitlab::Usage::ServicePing::LegacyMetricMetadataDecorator, feature_category: :service_ping do using RSpec::Parameterized::TableSyntax let(:duration) { 123 } - where(:metric_value, :metric_class) do - 1 | Integer - "value" | String - true | TrueClass - false | FalseClass - nil | NilClass + where(:metric_value, :error, :metric_class) do + 1 | nil | Integer + "value" | nil | String + true | nil | TrueClass + false | nil | FalseClass + nil | nil | NilClass + nil | StandardError.new | NilClass end with_them do - let(:decorated_object) { described_class.new(metric_value, duration) } + let(:decorated_object) { described_class.new(metric_value, duration, error: error) } it 'exposes a duration with the correct value' do expect(decorated_object.duration).to eq(duration) end + it 'exposes error with the correct value' do + expect(decorated_object.error).to eq(error) + end + it 'imitates wrapped class', :aggregate_failures do expect(decorated_object).to eq metric_value expect(decorated_object.class).to eq metric_class diff --git a/spec/lib/gitlab/usage_data_metrics_spec.rb b/spec/lib/gitlab/usage_data_metrics_spec.rb index 5d58933f1fd..34f8e5b2a2f 100644 --- a/spec/lib/gitlab/usage_data_metrics_spec.rb +++ b/spec/lib/gitlab/usage_data_metrics_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::UsageDataMetrics do +RSpec.describe Gitlab::UsageDataMetrics, :with_license do describe '.uncached_data' do subject { described_class.uncached_data } diff --git a/spec/lib/gitlab/usage_data_queries_spec.rb b/spec/lib/gitlab/usage_data_queries_spec.rb index 2fe43c11d27..30588324adf 100644 --- a/spec/lib/gitlab/usage_data_queries_spec.rb +++ b/spec/lib/gitlab/usage_data_queries_spec.rb @@ -11,9 +11,9 @@ RSpec.describe Gitlab::UsageDataQueries do end end - describe '.with_duration' do + describe '.with_metadata' do it 'yields passed block' do - expect { |block| described_class.with_duration(&block) }.to yield_with_no_args + expect { |block| described_class.with_metadata(&block) }.to yield_with_no_args end end diff --git a/spec/lib/gitlab/usage_data_spec.rb b/spec/lib/gitlab/usage_data_spec.rb index 214331e15e8..592ac280d32 100644 --- a/spec/lib/gitlab/usage_data_spec.rb +++ b/spec/lib/gitlab/usage_data_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::UsageData, :aggregate_failures do +RSpec.describe Gitlab::UsageData, :aggregate_failures, feature_category: :service_ping do include UsageDataHelpers before do @@ -1122,12 +1122,20 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do end end - describe ".with_duration" do + describe ".with_metadata" do it 'records duration' do - expect(::Gitlab::Usage::ServicePing::LegacyMetricTimingDecorator) - .to receive(:new).with(2, kind_of(Float)) + result = described_class.with_metadata { 1 + 1 } - described_class.with_duration { 1 + 1 } + expect(result.duration).to be_an(Float) + end + + it 'records error and returns nil', :aggregated_errors do + allow(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception) + + result = described_class.with_metadata { raise } + + expect(result.error).to be_an(StandardError) + expect(result).to be_nil end end diff --git a/spec/lib/gitlab/utils/lazy_attributes_spec.rb b/spec/lib/gitlab/utils/lazy_attributes_spec.rb index 1ebc9b0d711..430b79c3063 100644 --- a/spec/lib/gitlab/utils/lazy_attributes_spec.rb +++ b/spec/lib/gitlab/utils/lazy_attributes_spec.rb @@ -47,9 +47,9 @@ RSpec.describe Gitlab::Utils::LazyAttributes do end it 'only calls the block once even if it returned `nil`', :aggregate_failures do - expect(instance.instance_variable_get('@number')).to receive(:call).once.and_call_original - expect(instance.instance_variable_get('@accessor_2')).to receive(:call).once.and_call_original - expect(instance.instance_variable_get('@incorrect_type')).to receive(:call).once.and_call_original + expect(instance.instance_variable_get(:@number)).to receive(:call).once.and_call_original + expect(instance.instance_variable_get(:@accessor_2)).to receive(:call).once.and_call_original + expect(instance.instance_variable_get(:@incorrect_type)).to receive(:call).once.and_call_original 2.times do instance.number diff --git a/spec/lib/gitlab/utils/strong_memoize_spec.rb b/spec/lib/gitlab/utils/strong_memoize_spec.rb index 287858579d6..71f2502b91c 100644 --- a/spec/lib/gitlab/utils/strong_memoize_spec.rb +++ b/spec/lib/gitlab/utils/strong_memoize_spec.rb @@ -2,12 +2,13 @@ require 'fast_spec_helper' require 'rspec-benchmark' +require 'rspec-parameterized' RSpec.configure do |config| config.include RSpec::Benchmark::Matchers end -RSpec.describe Gitlab::Utils::StrongMemoize do +RSpec.describe Gitlab::Utils::StrongMemoize, feature_category: :not_owned do let(:klass) do strong_memoize_class = described_class @@ -35,15 +36,10 @@ RSpec.describe Gitlab::Utils::StrongMemoize do end strong_memoize_attr :method_name_attr - def different_method_name_attr + def enabled? trace << value value end - strong_memoize_attr :different_method_name_attr, :different_member_name_attr - - def enabled? - true - end strong_memoize_attr :enabled? def method_name_with_args(*args) @@ -80,6 +76,8 @@ RSpec.describe Gitlab::Utils::StrongMemoize do subject(:object) { klass.new(value) } shared_examples 'caching the value' do + let(:member_name) { described_class.normalize_key(method_name) } + it 'only calls the block once' do value0 = object.send(method_name) value1 = object.send(method_name) @@ -103,7 +101,6 @@ RSpec.describe Gitlab::Utils::StrongMemoize do context "with value #{value}" do let(:value) { value } let(:method_name) { :method_name } - let(:member_name) { :method_name } it_behaves_like 'caching the value' @@ -176,31 +173,44 @@ RSpec.describe Gitlab::Utils::StrongMemoize do end describe '#strong_memoized?' do - let(:value) { :anything } + shared_examples 'memoization check' do |method_name| + context "for #{method_name}" do + let(:value) { :anything } - subject { object.strong_memoized?(:method_name) } + subject { object.strong_memoized?(method_name) } - it 'returns false if the value is uncached' do - is_expected.to be(false) - end + it 'returns false if the value is uncached' do + is_expected.to be(false) + end - it 'returns true if the value is cached' do - object.method_name + it 'returns true if the value is cached' do + object.public_send(method_name) - is_expected.to be(true) + is_expected.to be(true) + end + end end + + it_behaves_like 'memoization check', :method_name + it_behaves_like 'memoization check', :enabled? end describe '#clear_memoization' do - let(:value) { 'mepmep' } + shared_examples 'clearing memoization' do |method_name| + let(:member_name) { described_class.normalize_key(method_name) } + let(:value) { 'mepmep' } - it 'removes the instance variable' do - object.method_name + it 'removes the instance variable' do + object.public_send(method_name) - object.clear_memoization(:method_name) + object.clear_memoization(method_name) - expect(object.instance_variable_defined?(:@method_name)).to be(false) + expect(object.instance_variable_defined?(:"@#{member_name}")).to be(false) + end end + + it_behaves_like 'clearing memoization', :method_name + it_behaves_like 'clearing memoization', :enabled? end describe '.strong_memoize_attr' do @@ -209,7 +219,6 @@ RSpec.describe Gitlab::Utils::StrongMemoize do context "memoized after method definition with value #{value}" do let(:method_name) { :method_name_attr } - let(:member_name) { :method_name_attr } it_behaves_like 'caching the value' @@ -218,30 +227,7 @@ RSpec.describe Gitlab::Utils::StrongMemoize do end it 'retains method arity' do - expect(klass.instance_method(member_name).arity).to eq(0) - end - end - - context "memoized before method definition with different member name and value #{value}" do - let(:method_name) { :different_method_name_attr } - let(:member_name) { :different_member_name_attr } - - it_behaves_like 'caching the value' - - it 'calls the existing .method_added' do - expect(klass.method_added_list).to include(:different_method_name_attr) - end - end - - context 'with valid method name' do - let(:method_name) { :enabled? } - - context 'with invalid member name' do - let(:member_name) { :enabled? } - - it 'is invalid' do - expect { object.send(method_name) { value } }.to raise_error /is not allowed as an instance variable name/ - end + expect(klass.instance_method(method_name).arity).to eq(0) end end end @@ -299,4 +285,41 @@ RSpec.describe Gitlab::Utils::StrongMemoize do end end end + + describe '.normalize_key' do + using RSpec::Parameterized::TableSyntax + + subject { described_class.normalize_key(input) } + + where(:input, :output, :valid) do + :key | :key | true + "key" | "key" | true + :key? | "key?" | true + "key?" | "key?" | true + :key! | "key!" | true + "key!" | "key!" | true + # invalid cases caught elsewhere + :"ke?y" | :"ke?y" | false + "ke?y" | "ke?y" | false + :"ke!y" | :"ke!y" | false + "ke!y" | "ke!y" | false + end + + with_them do + let(:ivar) { "@#{output}" } + + it { is_expected.to eq(output) } + + if params[:valid] + it 'is a valid ivar name' do + expect { instance_variable_defined?(ivar) }.not_to raise_error + end + else + it 'raises a NameError error' do + expect { instance_variable_defined?(ivar) } + .to raise_error(NameError, /not allowed as an instance/) + end + end + end + end end diff --git a/spec/lib/gitlab/utils/usage_data_spec.rb b/spec/lib/gitlab/utils/usage_data_spec.rb index 13d046b0816..2925ceef256 100644 --- a/spec/lib/gitlab/utils/usage_data_spec.rb +++ b/spec/lib/gitlab/utils/usage_data_spec.rb @@ -31,9 +31,9 @@ RSpec.describe Gitlab::Utils::UsageData do end end - describe '.with_duration' do + describe '.with_metadata' do it 'yields passed block' do - expect { |block| described_class.with_duration(&block) }.to yield_with_no_args + expect { |block| described_class.with_metadata(&block) }.to yield_with_no_args end end @@ -55,7 +55,7 @@ RSpec.describe Gitlab::Utils::UsageData do end it 'records duration' do - expect(described_class).to receive(:with_duration) + expect(described_class).to receive(:with_metadata) allow(relation).to receive(:count).and_return(1) described_class.count(relation, batch: false) @@ -82,7 +82,7 @@ RSpec.describe Gitlab::Utils::UsageData do end it 'records duration' do - expect(described_class).to receive(:with_duration) + expect(described_class).to receive(:with_metadata) allow(relation).to receive(:distinct_count_by).and_return(1) described_class.distinct_count(relation, batch: false) @@ -242,7 +242,7 @@ RSpec.describe Gitlab::Utils::UsageData do end it 'records duration' do - expect(described_class).to receive(:with_duration) + expect(described_class).to receive(:with_metadata) allow(Gitlab::Database::BatchCount).to receive(:batch_sum).and_return(1) described_class.sum(relation, :column) @@ -272,7 +272,7 @@ RSpec.describe Gitlab::Utils::UsageData do end it 'records duration' do - expect(described_class).to receive(:with_duration) + expect(described_class).to receive(:with_metadata) allow(Gitlab::Database::BatchCount).to receive(:batch_average).and_return(1) @@ -367,14 +367,14 @@ RSpec.describe Gitlab::Utils::UsageData do end it 'records duration' do - expect(described_class).to receive(:with_duration) + expect(described_class).to receive(:with_metadata) described_class.histogram(relation, column, buckets: 1..100) end context 'when query timeout' do subject do - with_statement_timeout(0.001) do + with_statement_timeout(0.001, connection: ApplicationRecord.connection) do relation = AlertManagement::HttpIntegration.select('pg_sleep(0.002)') described_class.histogram(relation, column, buckets: 1..100) end @@ -425,7 +425,7 @@ RSpec.describe Gitlab::Utils::UsageData do end it 'records duration' do - expect(described_class).to receive(:with_duration) + expect(described_class).to receive(:with_metadata) described_class.add end @@ -455,7 +455,7 @@ RSpec.describe Gitlab::Utils::UsageData do end it 'records duration' do - expect(described_class).to receive(:with_duration) + expect(described_class).to receive(:with_metadata) described_class.alt_usage_data end @@ -471,7 +471,7 @@ RSpec.describe Gitlab::Utils::UsageData do describe '#redis_usage_data' do it 'records duration' do - expect(described_class).to receive(:with_duration) + expect(described_class).to receive(:with_metadata) described_class.redis_usage_data end @@ -520,7 +520,7 @@ RSpec.describe Gitlab::Utils::UsageData do describe '#with_prometheus_client' do it 'records duration' do - expect(described_class).to receive(:with_duration) + expect(described_class).to receive(:with_metadata) described_class.with_prometheus_client { |client| client } end diff --git a/spec/lib/gitlab/version_info_spec.rb b/spec/lib/gitlab/version_info_spec.rb index 078f952afad..99c7a762392 100644 --- a/spec/lib/gitlab/version_info_spec.rb +++ b/spec/lib/gitlab/version_info_spec.rb @@ -92,6 +92,8 @@ RSpec.describe Gitlab::VersionInfo do it { expect(described_class.parse("1.0.0-rc1-ee")).to eq(@v1_0_0) } it { expect(described_class.parse("git 1.0.0b1")).to eq(@v1_0_0) } it { expect(described_class.parse("git 1.0b1")).not_to be_valid } + it { expect(described_class.parse("1.1.#{'1' * described_class::MAX_VERSION_LENGTH}")).not_to be_valid } + it { expect(described_class.parse(nil)).not_to be_valid } context 'with parse_suffix: true' do let(:versions) do @@ -182,4 +184,10 @@ RSpec.describe Gitlab::VersionInfo do it { expect(@v1_0_1.without_patch).to eq(@v1_0_0) } it { expect(@v1_0_1_rc1.without_patch).to eq(@v1_0_0) } end + + describe 'MAX_VERSION_LENGTH' do + subject { described_class::MAX_VERSION_LENGTH } + + it { is_expected.to eq(128) } + end end diff --git a/spec/lib/google_api/cloud_platform/client_spec.rb b/spec/lib/google_api/cloud_platform/client_spec.rb index 82ab6c089da..4ea395830ad 100644 --- a/spec/lib/google_api/cloud_platform/client_spec.rb +++ b/spec/lib/google_api/cloud_platform/client_spec.rb @@ -6,7 +6,6 @@ require 'google/apis/sqladmin_v1beta4' RSpec.describe GoogleApi::CloudPlatform::Client do let(:token) { 'token' } let(:client) { described_class.new(token, nil) } - let(:user_agent_options) { client.instance_eval { user_agent_header } } let(:gcp_project_id) { String('gcp_proj_id') } let(:operation) { true } let(:database_instance) { Google::Apis::SqladminV1beta4::DatabaseInstance.new(state: 'RUNNABLE') } @@ -77,150 +76,6 @@ RSpec.describe GoogleApi::CloudPlatform::Client do end end - describe '#projects_zones_clusters_get' do - subject { client.projects_zones_clusters_get(spy, spy, spy) } - - let(:gke_cluster) { double } - - before do - allow_any_instance_of(Google::Apis::ContainerV1::ContainerService) - .to receive(:get_zone_cluster).with(any_args, options: user_agent_options) - .and_return(gke_cluster) - end - - it { is_expected.to eq(gke_cluster) } - end - - describe '#projects_zones_clusters_create' do - subject do - client.projects_zones_clusters_create( - project_id, zone, cluster_name, cluster_size, machine_type: machine_type, legacy_abac: legacy_abac, enable_addons: enable_addons) - end - - let(:project_id) { 'project-123' } - let(:zone) { 'us-central1-a' } - let(:cluster_name) { 'test-cluster' } - let(:cluster_size) { 1 } - let(:machine_type) { 'n1-standard-2' } - let(:legacy_abac) { true } - let(:enable_addons) { [] } - - let(:addons_config) do - enable_addons.index_with do - { disabled: false } - end - end - - let(:cluster_options) do - { - cluster: { - name: cluster_name, - initial_node_count: cluster_size, - node_config: { - machine_type: machine_type, - oauth_scopes: [ - "https://www.googleapis.com/auth/devstorage.read_only", - "https://www.googleapis.com/auth/logging.write", - "https://www.googleapis.com/auth/monitoring" - ] - }, - master_auth: { - client_certificate_config: { - issue_client_certificate: true - } - }, - legacy_abac: { - enabled: legacy_abac - }, - ip_allocation_policy: { - use_ip_aliases: true, - cluster_ipv4_cidr_block: '/16' - }, - addons_config: addons_config - } - } - end - - let(:create_cluster_request_body) { double('Google::Apis::ContainerV1beta1::CreateClusterRequest') } - let(:operation) { double } - - before do - allow_any_instance_of(Google::Apis::ContainerV1beta1::ContainerService) - .to receive(:create_cluster).with(any_args) - .and_return(operation) - end - - it 'sets corresponded parameters' do - expect_any_instance_of(Google::Apis::ContainerV1beta1::ContainerService) - .to receive(:create_cluster).with(project_id, zone, create_cluster_request_body, options: user_agent_options) - - expect(Google::Apis::ContainerV1beta1::CreateClusterRequest) - .to receive(:new).with(cluster_options).and_return(create_cluster_request_body) - - expect(subject).to eq operation - end - - context 'create without legacy_abac' do - let(:legacy_abac) { false } - - it 'sets corresponded parameters' do - expect_any_instance_of(Google::Apis::ContainerV1beta1::ContainerService) - .to receive(:create_cluster).with(project_id, zone, create_cluster_request_body, options: user_agent_options) - - expect(Google::Apis::ContainerV1beta1::CreateClusterRequest) - .to receive(:new).with(cluster_options).and_return(create_cluster_request_body) - - expect(subject).to eq operation - end - end - - context 'create with enable_addons for cloud_run' do - let(:enable_addons) { [:http_load_balancing, :istio_config, :cloud_run_config] } - - it 'sets corresponded parameters' do - expect_any_instance_of(Google::Apis::ContainerV1beta1::ContainerService) - .to receive(:create_cluster).with(project_id, zone, create_cluster_request_body, options: user_agent_options) - - expect(Google::Apis::ContainerV1beta1::CreateClusterRequest) - .to receive(:new).with(cluster_options).and_return(create_cluster_request_body) - - expect(subject).to eq operation - end - end - end - - describe '#projects_zones_operations' do - subject { client.projects_zones_operations(spy, spy, spy) } - - let(:operation) { double } - - before do - allow_any_instance_of(Google::Apis::ContainerV1::ContainerService) - .to receive(:get_zone_operation).with(any_args, options: user_agent_options) - .and_return(operation) - end - - it { is_expected.to eq(operation) } - end - - describe '#parse_operation_id' do - subject { client.parse_operation_id(self_link) } - - context 'when expected url' do - let(:self_link) do - 'projects/gcp-project-12345/zones/us-central1-a/operations/ope-123' - end - - it { is_expected.to eq('ope-123') } - end - - context 'when unexpected url' do - let(:self_link) { '???' } - - it { is_expected.to be_nil } - end - end - describe '#user_agent_header' do subject { client.instance_eval { user_agent_header } } diff --git a/spec/lib/object_storage/direct_upload_spec.rb b/spec/lib/object_storage/direct_upload_spec.rb index c2201fb60ac..569e6a3a7c6 100644 --- a/spec/lib/object_storage/direct_upload_spec.rb +++ b/spec/lib/object_storage/direct_upload_spec.rb @@ -234,6 +234,7 @@ RSpec.describe ObjectStorage::DirectUpload do expect(subject[:GetURL]).to start_with(storage_url) expect(subject[:StoreURL]).to start_with(storage_url) expect(subject[:DeleteURL]).to start_with(storage_url) + expect(subject[:SkipDelete]).to eq(false) expect(subject[:CustomPutHeaders]).to be_truthy expect(subject[:PutHeaders]).to eq({}) end diff --git a/spec/lib/sidebars/groups/menus/observability_menu_spec.rb b/spec/lib/sidebars/groups/menus/observability_menu_spec.rb index 3a91b1aea2f..5b993cd6f28 100644 --- a/spec/lib/sidebars/groups/menus/observability_menu_spec.rb +++ b/spec/lib/sidebars/groups/menus/observability_menu_spec.rb @@ -20,23 +20,25 @@ RSpec.describe Sidebars::Groups::Menus::ObservabilityMenu do allow(menu).to receive(:can?).and_call_original end - context 'when user can :read_observability' do + context 'when observability is enabled' do before do - allow(menu).to receive(:can?).with(user, :read_observability, group).and_return(true) + allow(Gitlab::Observability).to receive(:observability_enabled?).and_return(true) end it 'returns true' do expect(menu.render?).to eq true + expect(Gitlab::Observability).to have_received(:observability_enabled?).with(user, group) end end - context 'when user cannot :read_observability' do + context 'when observability is disabled' do before do - allow(menu).to receive(:can?).with(user, :read_observability, group).and_return(false) + allow(Gitlab::Observability).to receive(:observability_enabled?).and_return(false) end it 'returns false' do expect(menu.render?).to eq false + expect(Gitlab::Observability).to have_received(:observability_enabled?).with(user, group) end end end diff --git a/spec/lib/sidebars/groups/menus/settings_menu_spec.rb b/spec/lib/sidebars/groups/menus/settings_menu_spec.rb index 4e3c639672b..c5246fe93dd 100644 --- a/spec/lib/sidebars/groups/menus/settings_menu_spec.rb +++ b/spec/lib/sidebars/groups/menus/settings_menu_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Sidebars::Groups::Menus::SettingsMenu do +RSpec.describe Sidebars::Groups::Menus::SettingsMenu, :with_license do let_it_be(:owner) { create(:user) } let_it_be_with_refind(:group) do diff --git a/spec/lib/sidebars/projects/menus/settings_menu_spec.rb b/spec/lib/sidebars/projects/menus/settings_menu_spec.rb index 0733e0c6521..c7aca0fb97e 100644 --- a/spec/lib/sidebars/projects/menus/settings_menu_spec.rb +++ b/spec/lib/sidebars/projects/menus/settings_menu_spec.rb @@ -10,6 +10,10 @@ RSpec.describe Sidebars::Projects::Menus::SettingsMenu do subject { described_class.new(context) } + before do + stub_feature_flags(show_pages_in_deployments_menu: false) + end + describe '#render?' do it 'returns false when menu does not have any menu items' do allow(subject).to receive(:has_renderable_items?).and_return(false) diff --git a/spec/lib/sidebars/your_work/menus/issues_menu_spec.rb b/spec/lib/sidebars/your_work/menus/issues_menu_spec.rb new file mode 100644 index 00000000000..a1206c0bc1c --- /dev/null +++ b/spec/lib/sidebars/your_work/menus/issues_menu_spec.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Sidebars::YourWork::Menus::IssuesMenu, feature_category: :navigation do + let(:user) { create(:user) } + let(:context) { Sidebars::Context.new(current_user: user, container: nil) } + + subject { described_class.new(context) } + + include_examples 'menu item shows pill based on count', :assigned_open_issues_count +end diff --git a/spec/lib/sidebars/your_work/menus/merge_requests_menu_spec.rb b/spec/lib/sidebars/your_work/menus/merge_requests_menu_spec.rb new file mode 100644 index 00000000000..b3251a54178 --- /dev/null +++ b/spec/lib/sidebars/your_work/menus/merge_requests_menu_spec.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Sidebars::YourWork::Menus::MergeRequestsMenu, feature_category: :navigation do + let(:user) { create(:user) } + let(:context) { Sidebars::Context.new(current_user: user, container: nil) } + + subject { described_class.new(context) } + + include_examples 'menu item shows pill based on count', :assigned_open_merge_requests_count +end diff --git a/spec/lib/sidebars/your_work/menus/todos_menu_spec.rb b/spec/lib/sidebars/your_work/menus/todos_menu_spec.rb new file mode 100644 index 00000000000..a8177a6a01b --- /dev/null +++ b/spec/lib/sidebars/your_work/menus/todos_menu_spec.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Sidebars::YourWork::Menus::TodosMenu, feature_category: :navigation do + let(:user) { create(:user) } + let(:context) { Sidebars::Context.new(current_user: user, container: nil) } + + subject { described_class.new(context) } + + include_examples 'menu item shows pill based on count', :todos_pending_count +end diff --git a/spec/lib/unnested_in_filters/rewriter_spec.rb b/spec/lib/unnested_in_filters/rewriter_spec.rb index fe34fba579b..bba27276037 100644 --- a/spec/lib/unnested_in_filters/rewriter_spec.rb +++ b/spec/lib/unnested_in_filters/rewriter_spec.rb @@ -69,15 +69,21 @@ RSpec.describe UnnestedInFilters::Rewriter do let(:recorded_queries) { ActiveRecord::QueryRecorder.new { rewriter.rewrite.load } } let(:relation) { User.where(state: :active, user_type: %i(support_bot alert_bot)).limit(2) } + let(:users_default_select_fields) do + User.default_select_columns + .map { |field| "\"users\".\"#{field.name}\"" } + .join(',') + end + let(:expected_query) do <<~SQL SELECT - "users".* + #{users_default_select_fields} FROM unnest('{1,2}'::smallint[]) AS "user_types"("user_type"), LATERAL ( SELECT - "users".* + #{users_default_select_fields} FROM "users" WHERE @@ -101,13 +107,13 @@ RSpec.describe UnnestedInFilters::Rewriter do let(:expected_query) do <<~SQL SELECT - "users".* + #{users_default_select_fields} FROM unnest(ARRAY(SELECT "users"."state" FROM "users")::character varying[]) AS "states"("state"), unnest('{1,2}'::smallint[]) AS "user_types"("user_type"), LATERAL ( SELECT - "users".* + #{users_default_select_fields} FROM "users" WHERE @@ -129,12 +135,12 @@ RSpec.describe UnnestedInFilters::Rewriter do let(:expected_query) do <<~SQL SELECT - "users".* + #{users_default_select_fields} FROM unnest('{active,blocked,banned}'::charactervarying[]) AS "states"("state"), LATERAL ( SELECT - "users".* + #{users_default_select_fields} FROM "users" WHERE @@ -181,8 +187,6 @@ RSpec.describe UnnestedInFilters::Rewriter do let(:expected_query) do <<~SQL - SELECT - "users".* FROM "users" WHERE @@ -217,7 +221,7 @@ RSpec.describe UnnestedInFilters::Rewriter do end it 'changes the query' do - expect(issued_query.gsub(/\s/, '')).to start_with(expected_query.gsub(/\s/, '')) + expect(issued_query.gsub(/\s/, '')).to include(expected_query.gsub(/\s/, '')) end end @@ -226,8 +230,6 @@ RSpec.describe UnnestedInFilters::Rewriter do let(:expected_query) do <<~SQL - SELECT - "users".* FROM "users" WHERE @@ -257,7 +259,7 @@ RSpec.describe UnnestedInFilters::Rewriter do end it 'does not rewrite the in statement for the joined table' do - expect(issued_query.gsub(/\s/, '')).to start_with(expected_query.gsub(/\s/, '')) + expect(issued_query.gsub(/\s/, '')).to include(expected_query.gsub(/\s/, '')) end end |