diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-05-17 19:05:49 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-05-17 19:05:49 +0300 |
commit | 43a25d93ebdabea52f99b05e15b06250cd8f07d7 (patch) | |
tree | dceebdc68925362117480a5d672bcff122fb625b /spec/lib/gitlab | |
parent | 20c84b99005abd1c82101dfeff264ac50d2df211 (diff) |
Add latest changes from gitlab-org/gitlab@16-0-stable-eev16.0.0-rc42
Diffstat (limited to 'spec/lib/gitlab')
599 files changed, 17302 insertions, 12391 deletions
diff --git a/spec/lib/gitlab/analytics/cycle_analytics/average_spec.rb b/spec/lib/gitlab/analytics/cycle_analytics/average_spec.rb index de325454b34..122a94a39c2 100644 --- a/spec/lib/gitlab/analytics/cycle_analytics/average_spec.rb +++ b/spec/lib/gitlab/analytics/cycle_analytics/average_spec.rb @@ -40,7 +40,7 @@ RSpec.describe Gitlab::Analytics::CycleAnalytics::Average do subject(:average_duration_in_seconds) { average.seconds } context 'when no results' do - let(:query) { Issue.none } + let(:query) { Issue.joins(:metrics).none } it { is_expected.to eq(nil) } end @@ -54,7 +54,7 @@ RSpec.describe Gitlab::Analytics::CycleAnalytics::Average do subject(:average_duration_in_days) { average.days } context 'when no results' do - let(:query) { Issue.none } + let(:query) { Issue.joins(:metrics).none } it { is_expected.to eq(nil) } end diff --git a/spec/lib/gitlab/analytics/cycle_analytics/request_params_spec.rb b/spec/lib/gitlab/analytics/cycle_analytics/request_params_spec.rb new file mode 100644 index 00000000000..9b362debb10 --- /dev/null +++ b/spec/lib/gitlab/analytics/cycle_analytics/request_params_spec.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Analytics::CycleAnalytics::RequestParams, feature_category: :value_stream_management do + it_behaves_like 'unlicensed cycle analytics request params' do + let_it_be(:user) { create(:user) } + let_it_be(:root_group) { create(:group) } + let_it_be_with_refind(:project) { create(:project, group: root_group) } + + let(:namespace) { project.project_namespace } + + describe 'project-level data attributes' do + subject(:attributes) { described_class.new(params).to_data_attributes } + + it 'includes the namespace attribute' do + expect(attributes).to match(hash_including({ + namespace: { + name: project.name, + full_path: project.full_path, + type: "Project" + } + })) + end + + context 'with a subgroup project' do + let_it_be(:sub_group) { create(:group, parent: root_group) } + let_it_be_with_refind(:subgroup_project) { create(:project, group: sub_group) } + let(:namespace) { subgroup_project.project_namespace } + + it 'includes the correct group_path' do + expect(attributes).to match(hash_including({ + group_path: "groups/#{subgroup_project.namespace.full_path}", + full_path: subgroup_project.full_path + })) + end + end + end + end +end diff --git a/spec/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event_spec.rb b/spec/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event_spec.rb index 1e0034e386e..24248c557bd 100644 --- a/spec/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event_spec.rb +++ b/spec/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event_spec.rb @@ -1,8 +1,8 @@ # frozen_string_literal: true -require 'fast_spec_helper' +require 'spec_helper' -RSpec.describe Gitlab::Analytics::CycleAnalytics::StageEvents::StageEvent do +RSpec.describe Gitlab::Analytics::CycleAnalytics::StageEvents::StageEvent, feature_category: :product_analytics do let(:instance) { described_class.new({}) } it { expect(described_class).to respond_to(:name) } diff --git a/spec/lib/gitlab/api_authentication/token_resolver_spec.rb b/spec/lib/gitlab/api_authentication/token_resolver_spec.rb index c0c8e7aba63..48cae42dcd2 100644 --- a/spec/lib/gitlab/api_authentication/token_resolver_spec.rb +++ b/spec/lib/gitlab/api_authentication/token_resolver_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::APIAuthentication::TokenResolver, feature_category: :authentication_and_authorization do +RSpec.describe Gitlab::APIAuthentication::TokenResolver, feature_category: :system_access do let_it_be(:user) { create(:user) } let_it_be(:project, reload: true) { create(:project, :public) } let_it_be(:personal_access_token) { create(:personal_access_token, user: user) } diff --git a/spec/lib/gitlab/app_logger_spec.rb b/spec/lib/gitlab/app_logger_spec.rb index 85ca60d539f..149c3d1f19f 100644 --- a/spec/lib/gitlab/app_logger_spec.rb +++ b/spec/lib/gitlab/app_logger_spec.rb @@ -2,26 +2,12 @@ require 'spec_helper' -RSpec.describe Gitlab::AppLogger do +RSpec.describe Gitlab::AppLogger, feature_category: :shared do subject { described_class } - it 'builds two Logger instances' do - expect(Gitlab::Logger).to receive(:new).and_call_original - expect(Gitlab::JsonLogger).to receive(:new).and_call_original + specify { expect(described_class.primary_logger).to be Gitlab::AppJsonLogger } - subject.info('Hello World!') - end - - it 'logs info to AppLogger and AppJsonLogger' do - expect_any_instance_of(Gitlab::AppTextLogger).to receive(:info).and_call_original - expect_any_instance_of(Gitlab::AppJsonLogger).to receive(:info).and_call_original - - subject.info('Hello World!') - end - - it 'logs info to only the AppJsonLogger when unstructured logs are disabled' do - stub_env('UNSTRUCTURED_RAILS_LOG', 'false') - expect_any_instance_of(Gitlab::AppTextLogger).not_to receive(:info).and_call_original + it 'logs to AppJsonLogger' do expect_any_instance_of(Gitlab::AppJsonLogger).to receive(:info).and_call_original subject.info('Hello World!') diff --git a/spec/lib/gitlab/asciidoc_spec.rb b/spec/lib/gitlab/asciidoc_spec.rb index cb9d1e9eae8..31e575e0466 100644 --- a/spec/lib/gitlab/asciidoc_spec.rb +++ b/spec/lib/gitlab/asciidoc_spec.rb @@ -4,7 +4,7 @@ require 'spec_helper' require 'nokogiri' module Gitlab - RSpec.describe Asciidoc do + RSpec.describe Asciidoc, feature_category: :wiki do include FakeBlobHelpers before do @@ -97,8 +97,8 @@ module Gitlab output = <<~HTML <div> <div> - <div class=\"gl-relative markdown-code-block js-markdown-code\"> - <pre lang=\"plaintext\" class=\"code highlight js-syntax-highlight language-plaintext\" data-canonical-lang=\"mypre\" v-pre=\"true\"><code></code></pre> + <div class="gl-relative markdown-code-block js-markdown-code"> + <pre data-canonical-lang="mypre" class="code highlight js-syntax-highlight language-plaintext" lang="plaintext" v-pre="true"><code></code></pre> <copy-code></copy-code> </div> </div> @@ -369,7 +369,7 @@ module Gitlab <div> <div> <div class="gl-relative markdown-code-block js-markdown-code"> - <pre lang="javascript" class="code highlight js-syntax-highlight language-javascript" data-canonical-lang="js" v-pre="true"><code><span id="LC1" class="line" lang="javascript"><span class="nx">console</span><span class="p">.</span><span class="nx">log</span><span class="p">(</span><span class="dl">'</span><span class="s1">hello world</span><span class="dl">'</span><span class="p">)</span></span></code></pre> + <pre data-canonical-lang="js" class="code highlight js-syntax-highlight language-javascript" lang="javascript" v-pre="true"><code><span id="LC1" class="line" lang="javascript"><span class="nx">console</span><span class="p">.</span><span class="nf">log</span><span class="p">(</span><span class="dl">'</span><span class="s1">hello world</span><span class="dl">'</span><span class="p">)</span></span></code></pre> <copy-code></copy-code> </div> </div> @@ -399,7 +399,7 @@ module Gitlab <div>class.cpp</div> <div> <div class="gl-relative markdown-code-block js-markdown-code"> - <pre lang="cpp" class="code highlight js-syntax-highlight language-cpp" data-canonical-lang="c++" v-pre="true"><code><span id="LC1" class="line" lang="cpp"><span class="cp">#include</span> <span class="cpf"><stdio.h></span></span> + <pre data-canonical-lang="c++" class="code highlight js-syntax-highlight language-cpp" lang="cpp" v-pre="true"><code><span id="LC1" class="line" lang="cpp"><span class="cp">#include</span> <span class="cpf"><stdio.h></span></span> <span id="LC2" class="line" lang="cpp"></span> <span id="LC3" class="line" lang="cpp"><span class="k">for</span> <span class="p">(</span><span class="kt">int</span> <span class="n">i</span> <span class="o">=</span> <span class="mi">0</span><span class="p">;</span> <span class="n">i</span> <span class="o"><</span> <span class="mi">5</span><span class="p">;</span> <span class="n">i</span><span class="o">++</span><span class="p">)</span> <span class="p">{</span></span> <span id="LC4" class="line" lang="cpp"> <span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="o"><<</span><span class="s">"*"</span><span class="o"><<</span><span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span></span> @@ -457,7 +457,7 @@ module Gitlab stem:[2+2] is 4 MD - expect(render(input, context)).to include('<pre data-math-style="display" lang="plaintext" class="code math js-render-math" data-canonical-lang="" v-pre="true"><code><span id="LC1" class="line" lang="plaintext">eta_x gamma</span></code></pre>') + expect(render(input, context)).to include('<pre data-math-style="display" class="code math js-render-math" lang="plaintext" v-pre="true"><code><span id="LC1" class="line" lang="plaintext">eta_x gamma</span></code></pre>') expect(render(input, context)).to include('<p><code data-math-style="inline" class="code math js-render-math">2+2</code> is 4</p>') end end diff --git a/spec/lib/gitlab/audit/auditor_spec.rb b/spec/lib/gitlab/audit/auditor_spec.rb index 4b16333d913..2b3c8506440 100644 --- a/spec/lib/gitlab/audit/auditor_spec.rb +++ b/spec/lib/gitlab/audit/auditor_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Audit::Auditor do +RSpec.describe Gitlab::Audit::Auditor, feature_category: :audit_events do let(:name) { 'audit_operation' } let(:author) { create(:user, :with_sign_ins) } let(:group) { create(:group) } @@ -22,9 +22,9 @@ RSpec.describe Gitlab::Audit::Auditor do subject(:auditor) { described_class } describe '.audit' do - context 'when authentication event' do - let(:audit!) { auditor.audit(context) } + let(:audit!) { auditor.audit(context) } + context 'when authentication event' do it 'creates an authentication event' do expect(AuthenticationEvent).to receive(:new).with( { @@ -210,19 +210,38 @@ RSpec.describe Gitlab::Audit::Auditor do end context 'when authentication event is false' do + let(:target) { group } let(:context) do { name: name, author: author, scope: group, - target: group, authentication_event: false, message: "sample message" } + target: target, authentication_event: false, message: "sample message" } end it 'does not create an authentication event' do expect { auditor.audit(context) }.not_to change(AuthenticationEvent, :count) end + + context 'with permitted target' do + { feature_flag: :operations_feature_flag }.each do |target_type, factory_name| + context "with #{target_type}" do + let(:target) { build_stubbed factory_name } + + it 'logs audit events to database', :aggregate_failures, :freeze_time do + audit! + audit_event = AuditEvent.last + + expect(audit_event.author_id).to eq(author.id) + expect(audit_event.entity_id).to eq(group.id) + expect(audit_event.entity_type).to eq(group.class.name) + expect(audit_event.created_at).to eq(Time.zone.now) + expect(audit_event.details[:target_id]).to eq(target.id) + expect(audit_event.details[:target_type]).to eq(target.class.name) + end + end + end + end end context 'when authentication event is invalid' do - let(:audit!) { auditor.audit(context) } - before do allow(AuthenticationEvent).to receive(:new).and_raise(ActiveRecord::RecordInvalid) allow(Gitlab::ErrorTracking).to receive(:track_exception) @@ -243,8 +262,6 @@ RSpec.describe Gitlab::Audit::Auditor do end context 'when audit events are invalid' do - let(:audit!) { auditor.audit(context) } - before do expect_next_instance_of(AuditEvent) do |instance| allow(instance).to receive(:save!).and_raise(ActiveRecord::RecordInvalid) diff --git a/spec/lib/gitlab/auth/auth_finders_spec.rb b/spec/lib/gitlab/auth/auth_finders_spec.rb index 6aedd0a0a23..4498e369695 100644 --- a/spec/lib/gitlab/auth/auth_finders_spec.rb +++ b/spec/lib/gitlab/auth/auth_finders_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Auth::AuthFinders, feature_category: :authentication_and_authorization do +RSpec.describe Gitlab::Auth::AuthFinders, feature_category: :system_access do include described_class include HttpBasicAuthHelpers @@ -409,6 +409,17 @@ RSpec.describe Gitlab::Auth::AuthFinders, feature_category: :authentication_and_ expect(find_user_from_access_token).to be_nil end + context 'when run for kubernetes internal API endpoint' do + before do + set_bearer_token('AgentToken') + set_header('SCRIPT_NAME', '/api/v4/internal/kubernetes/modules/starboard_vulnerability/policies_configuration') + end + + it 'returns nil' do + expect(find_user_from_access_token).to be_nil + end + end + context 'when validate_access_token! returns valid' do it 'returns user' do set_header(described_class::PRIVATE_TOKEN_HEADER, personal_access_token.token) diff --git a/spec/lib/gitlab/auth/o_auth/auth_hash_spec.rb b/spec/lib/gitlab/auth/o_auth/auth_hash_spec.rb index c94f962ee93..8c50b2acac6 100644 --- a/spec/lib/gitlab/auth/o_auth/auth_hash_spec.rb +++ b/spec/lib/gitlab/auth/o_auth/auth_hash_spec.rb @@ -2,14 +2,19 @@ require 'spec_helper' -RSpec.describe Gitlab::Auth::OAuth::AuthHash do +RSpec.describe Gitlab::Auth::OAuth::AuthHash, feature_category: :user_management do let(:provider) { 'ldap' } let(:auth_hash) do described_class.new( OmniAuth::AuthHash.new( provider: provider, uid: uid_ascii, - info: info_hash + info: info_hash, + extra: { + raw_info: { + 'https://example.com/claims/username': username_claim_utf8 + } + } ) ) end @@ -24,6 +29,7 @@ RSpec.describe Gitlab::Auth::OAuth::AuthHash do let(:first_name_raw) { +'Onur' } let(:last_name_raw) { +"K\xC3\xBC\xC3\xA7\xC3\xBCk" } let(:name_raw) { +"Onur K\xC3\xBC\xC3\xA7\xC3\xBCk" } + let(:username_claim_raw) { +'onur.partner' } let(:uid_ascii) { uid_raw.force_encoding(Encoding::ASCII_8BIT) } let(:email_ascii) { email_raw.force_encoding(Encoding::ASCII_8BIT) } @@ -37,6 +43,7 @@ RSpec.describe Gitlab::Auth::OAuth::AuthHash do let(:nickname_utf8) { nickname_ascii.force_encoding(Encoding::UTF_8) } let(:name_utf8) { name_ascii.force_encoding(Encoding::UTF_8) } let(:first_name_utf8) { first_name_ascii.force_encoding(Encoding::UTF_8) } + let(:username_claim_utf8) { username_claim_raw.force_encoding(Encoding::ASCII_8BIT) } let(:info_hash) do { @@ -98,10 +105,16 @@ RSpec.describe Gitlab::Auth::OAuth::AuthHash do allow(Gitlab::Auth::OAuth::Provider).to receive(:config_for).and_return(provider_config) end - it 'uses the custom field for the username' do + it 'uses the custom field for the username within info' do expect(auth_hash.username).to eql first_name_utf8 end + it 'uses the custom field for the username within extra.raw_info' do + provider_config['args']['gitlab_username_claim'] = 'https://example.com/claims/username' + + expect(auth_hash.username).to eql username_claim_utf8 + end + it 'uses the default claim for the username when the custom claim is not found' do provider_config['args']['gitlab_username_claim'] = 'nonexistent' @@ -146,4 +159,66 @@ RSpec.describe Gitlab::Auth::OAuth::AuthHash do expect(auth_hash.password.encoding).to eql Encoding::UTF_8 end end + + describe '#get_from_auth_hash_or_info' do + context 'for a key not within auth_hash' do + let(:auth_hash) do + described_class.new( + OmniAuth::AuthHash.new( + provider: provider, + uid: uid_ascii, + info: info_hash + ) + ) + end + + let(:info_hash) { { nickname: nickname_ascii } } + + it 'provides username from info_hash' do + expect(auth_hash.username).to eql nickname_utf8 + end + end + + context 'for a key within auth_hash' do + let(:auth_hash) do + described_class.new( + OmniAuth::AuthHash.new( + provider: provider, + uid: uid_ascii, + info: info_hash, + username: nickname_ascii + ) + ) + end + + let(:info_hash) { { something: nickname_ascii } } + + it 'provides username from auth_hash' do + expect(auth_hash.username).to eql nickname_utf8 + end + end + + context 'for a key within auth_hash extra' do + let(:auth_hash) do + described_class.new( + OmniAuth::AuthHash.new( + provider: provider, + uid: uid_ascii, + info: info_hash, + extra: { + raw_info: { + nickname: nickname_ascii + } + } + ) + ) + end + + let(:info_hash) { { something: nickname_ascii } } + + it 'provides username from auth_hash extra' do + expect(auth_hash.username).to eql nickname_utf8 + end + end + end end diff --git a/spec/lib/gitlab/auth/o_auth/provider_spec.rb b/spec/lib/gitlab/auth/o_auth/provider_spec.rb index 96a31c50989..226669bab33 100644 --- a/spec/lib/gitlab/auth/o_auth/provider_spec.rb +++ b/spec/lib/gitlab/auth/o_auth/provider_spec.rb @@ -49,7 +49,7 @@ RSpec.describe Gitlab::Auth::OAuth::Provider do context 'for an LDAP provider' do context 'when the provider exists' do it 'returns the config' do - expect(described_class.config_for('ldapmain')).to be_a(Hash) + expect(described_class.config_for('ldapmain')).to be_a(GitlabSettings::Options) end end diff --git a/spec/lib/gitlab/auth/o_auth/user_spec.rb b/spec/lib/gitlab/auth/o_auth/user_spec.rb index 04fbbff3559..78e0df91103 100644 --- a/spec/lib/gitlab/auth/o_auth/user_spec.rb +++ b/spec/lib/gitlab/auth/o_auth/user_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Auth::OAuth::User, feature_category: :authentication_and_authorization do +RSpec.describe Gitlab::Auth::OAuth::User, feature_category: :system_access do include LdapHelpers let(:oauth_user) { described_class.new(auth_hash) } @@ -320,6 +320,38 @@ RSpec.describe Gitlab::Auth::OAuth::User, feature_category: :authentication_and_ end include_examples "to verify compliance with allow_single_sign_on" + + context 'and other providers' do + context 'when sync_name is disabled' do + before do + stub_ldap_config(sync_name: false) + end + + let!(:existing_user) { create(:omniauth_user, name: 'John Swift', email: 'john@example.com', extern_uid: dn, provider: 'twitter', username: 'john') } + + it "updates the gl_user name" do + oauth_user.save # rubocop:disable Rails/SaveBang + + expect(gl_user).to be_valid + expect(gl_user.name).to eql 'John' + end + end + + context 'when sync_name is enabled' do + before do + stub_ldap_config(sync_name: true) + end + + let!(:existing_user) { create(:omniauth_user, name: 'John Swift', email: 'john@example.com', extern_uid: dn, provider: 'twitter', username: 'john') } + + it "updates the gl_user name" do + oauth_user.save # rubocop:disable Rails/SaveBang + + expect(gl_user).to be_valid + expect(gl_user.name).to eql 'John' + end + end + end end context "with auto_link_ldap_user enabled" do @@ -418,54 +450,41 @@ RSpec.describe Gitlab::Auth::OAuth::User, feature_category: :authentication_and_ end context "and LDAP user has an account already" do + let(:provider) { 'ldapmain' } + + before do + allow(Gitlab::Auth::Ldap::Person).to receive(:find_by_uid).and_return(ldap_user) + stub_omniauth_config(sync_profile_attributes: true) + allow(Gitlab.config.ldap).to receive(:enabled).and_return(true) + end + context 'when sync_name is disabled' do before do - allow(Gitlab.config.ldap).to receive(:enabled).and_return(true) - allow(Gitlab.config.ldap).to receive(:sync_name).and_return(false) + stub_ldap_config(sync_name: false) end - let!(:existing_user) { create(:omniauth_user, name: 'John Doe', email: 'john@example.com', extern_uid: dn, provider: 'ldapmain', username: 'john') } - - it "adds the omniauth identity to the LDAP account" do - allow(Gitlab::Auth::Ldap::Person).to receive(:find_by_uid).and_return(ldap_user) + let!(:existing_user) { create(:omniauth_user, name: 'John Deo', email: 'john@example.com', extern_uid: dn, provider: 'ldapmain', username: 'john') } + it "does not update the user name" do oauth_user.save # rubocop:disable Rails/SaveBang expect(gl_user).to be_valid - expect(gl_user.username).to eql 'john' - expect(gl_user.name).to eql 'John Doe' - expect(gl_user.email).to eql 'john@example.com' - expect(gl_user.identities.length).to be 2 - identities_as_hash = gl_user.identities.map { |id| { provider: id.provider, extern_uid: id.extern_uid } } - expect(identities_as_hash).to match_array( - [ - { provider: 'ldapmain', extern_uid: dn }, - { provider: 'twitter', extern_uid: uid } - ] - ) + expect(gl_user.name).to eql 'John Deo' end end context 'when sync_name is enabled' do - let!(:existing_user) { create(:omniauth_user, name: 'John Swift', email: 'john@example.com', extern_uid: dn, provider: 'ldapmain', username: 'john') } + before do + stub_ldap_config(sync_name: true) + end - it "adds the omniauth identity to the LDAP account" do - allow(Gitlab::Auth::Ldap::Person).to receive(:find_by_uid).and_return(ldap_user) + let!(:existing_user) { create(:omniauth_user, name: 'John Swift', email: 'john@example.com', extern_uid: dn, provider: 'ldapmain', username: 'john') } + it "updates the user name" do oauth_user.save # rubocop:disable Rails/SaveBang expect(gl_user).to be_valid - expect(gl_user.username).to eql 'john' - expect(gl_user.name).to eql 'John Swift' - expect(gl_user.email).to eql 'john@example.com' - expect(gl_user.identities.length).to be 2 - identities_as_hash = gl_user.identities.map { |id| { provider: id.provider, extern_uid: id.extern_uid } } - expect(identities_as_hash).to match_array( - [ - { provider: 'ldapmain', extern_uid: dn }, - { provider: 'twitter', extern_uid: uid } - ] - ) + expect(gl_user.name).to eql 'John' end end end diff --git a/spec/lib/gitlab/auth/otp/strategies/duo_auth/manual_otp_spec.rb b/spec/lib/gitlab/auth/otp/strategies/duo_auth/manual_otp_spec.rb new file mode 100644 index 00000000000..d04e0ad9fb4 --- /dev/null +++ b/spec/lib/gitlab/auth/otp/strategies/duo_auth/manual_otp_spec.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Auth::Otp::Strategies::DuoAuth::ManualOtp, feature_category: :system_access do + let_it_be(:user) { create(:user) } + + let_it_be(:otp_code) { 42 } + + let_it_be(:hostname) { 'duo_auth.example.com' } + let_it_be(:integration_key) { 'int3gr4t1on' } + let_it_be(:secret_key) { 's3cr3t' } + + let_it_be(:duo_response_builder) { Struct.new(:body) } + + let_it_be(:response_status) { 200 } + + let_it_be(:duo_auth_url) { "https://#{hostname}/auth/v2/auth/" } + let_it_be(:params) do + { username: user.username, + factor: "passcode", + passcode: otp_code } + end + + let_it_be(:manual_otp) { described_class.new(user) } + + subject(:response) { manual_otp.validate(otp_code) } + + before do + stub_duo_auth_config( + enabled: true, + hostname: hostname, + secret_key: secret_key, + integration_key: integration_key + ) + end + + context 'when successful validation' do + before do + allow(duo_client).to receive(:request) + .with("POST", "/auth/v2/auth", params) + .and_return(duo_response_builder.new('{ "response": { "result": "allow" }}')) + + allow(manual_otp).to receive(:duo_client).and_return(duo_client) + end + + it 'returns success' do + response + + expect(response[:status]).to eq(:success) + end + end + + context 'when unsuccessful validation' do + before do + allow(duo_client).to receive(:request) + .with("POST", "/auth/v2/auth", params) + .and_return(duo_response_builder.new('{ "response": { "result": "deny" }}')) + + allow(manual_otp).to receive(:duo_client).and_return(duo_client) + end + + it 'returns error' do + response + + expect(response[:status]).to eq(:error) + end + end + + context 'when unexpected error' do + before do + allow(duo_client).to receive(:request) + .with("POST", "/auth/v2/auth", params) + .and_return(duo_response_builder.new('aaa')) + + allow(manual_otp).to receive(:duo_client).and_return(duo_client) + end + + it 'returns error' do + response + + expect(response[:status]).to eq(:error) + expect(response[:message]).to match(/unexpected character/) + end + end + + def stub_duo_auth_config(duo_auth_settings) + allow(::Gitlab.config.duo_auth).to(receive_messages(duo_auth_settings)) + end + + def duo_client + manual_otp.send(:duo_client) + end +end diff --git a/spec/lib/gitlab/auth/u2f_webauthn_converter_spec.rb b/spec/lib/gitlab/auth/u2f_webauthn_converter_spec.rb deleted file mode 100644 index deddc7f5294..00000000000 --- a/spec/lib/gitlab/auth/u2f_webauthn_converter_spec.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Auth::U2fWebauthnConverter do - let_it_be(:u2f_registration) do - device = U2F::FakeU2F.new(FFaker::BaconIpsum.characters(5)) - create(:u2f_registration, name: 'u2f_device', - certificate: Base64.strict_encode64(device.cert_raw), - key_handle: U2F.urlsafe_encode64(device.key_handle_raw), - public_key: Base64.strict_encode64(device.origin_public_key_raw)) - end - - it 'converts u2f registration' do - webauthn_credential = WebAuthn::U2fMigrator.new( - app_id: Gitlab.config.gitlab.url, - certificate: u2f_registration.certificate, - key_handle: u2f_registration.key_handle, - public_key: u2f_registration.public_key, - counter: u2f_registration.counter - ).credential - - converted_webauthn = described_class.new(u2f_registration).convert - - expect(converted_webauthn).to( - include(user_id: u2f_registration.user_id, - credential_xid: Base64.strict_encode64(webauthn_credential.id))) - end -end diff --git a/spec/lib/gitlab/auth_spec.rb b/spec/lib/gitlab/auth_spec.rb index a5f46aa1f35..36c87fb4557 100644 --- a/spec/lib/gitlab/auth_spec.rb +++ b/spec/lib/gitlab/auth_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching, feature_category: :authentication_and_authorization do +RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching, feature_category: :system_access do let_it_be(:project) { create(:project) } let(:auth_failure) { { actor: nil, project: nil, type: nil, authentication_abilities: nil } } @@ -21,6 +21,10 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching, feature_cate expect(subject::REPOSITORY_SCOPES).to match_array %i[read_repository write_repository] end + it 'OBSERVABILITY_SCOPES contains all scopes for Observability access' do + expect(subject::OBSERVABILITY_SCOPES).to match_array %i[read_observability write_observability] + end + it 'OPENID_SCOPES contains all scopes for OpenID Connect' do expect(subject::OPENID_SCOPES).to match_array [:openid] end @@ -31,54 +35,103 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching, feature_cate end context 'available_scopes' do - it 'contains all non-default scopes' do + before do stub_container_registry_config(enabled: true) + end - expect(subject.all_available_scopes).to match_array %i[api read_user read_api read_repository write_repository read_registry write_registry sudo admin_mode] + it 'contains all non-default scopes' do + expect(subject.all_available_scopes).to match_array %i[api read_user read_api read_repository write_repository read_registry write_registry sudo admin_mode read_observability write_observability] end - it 'contains for non-admin user all non-default scopes without ADMIN access' do - stub_container_registry_config(enabled: true) - user = create(:user, admin: false) + it 'contains for non-admin user all non-default scopes without ADMIN access and without observability scopes' do + user = build_stubbed(:user, admin: false) expect(subject.available_scopes_for(user)).to match_array %i[api read_user read_api read_repository write_repository read_registry write_registry] end - it 'contains for admin user all non-default scopes with ADMIN access' do - stub_container_registry_config(enabled: true) - user = create(:user, admin: true) + it 'contains for admin user all non-default scopes with ADMIN access and without observability scopes' do + user = build_stubbed(:user, admin: true) expect(subject.available_scopes_for(user)).to match_array %i[api read_user read_api read_repository write_repository read_registry write_registry sudo admin_mode] end + it 'contains for project all resource bot scopes without observability scopes' do + expect(subject.available_scopes_for(project)).to match_array %i[api read_api read_repository write_repository read_registry write_registry] + end + + it 'contains for group all resource bot scopes' do + group = build_stubbed(:group) + + expect(subject.available_scopes_for(group)).to match_array %i[api read_api read_repository write_repository read_registry write_registry read_observability write_observability] + end + + it 'contains for unsupported type no scopes' do + expect(subject.available_scopes_for(:something)).to be_empty + end + it 'optional_scopes contains all non-default scopes' do - stub_container_registry_config(enabled: true) + expect(subject.optional_scopes).to match_array %i[read_user read_api read_repository write_repository read_registry write_registry sudo admin_mode openid profile email read_observability write_observability] + end + + context 'with observability_group_tab feature flag' do + context 'when disabled' do + before do + stub_feature_flags(observability_group_tab: false) + end + + it 'contains for group all resource bot scopes without observability scopes' do + group = build_stubbed(:group) - expect(subject.optional_scopes).to match_array %i[read_user read_api read_repository write_repository read_registry write_registry sudo admin_mode openid profile email] + expect(subject.available_scopes_for(group)).to match_array %i[api read_api read_repository write_repository read_registry write_registry] + end + end + + context 'when enabled for specific group' do + let(:group) { build_stubbed(:group) } + + before do + stub_feature_flags(observability_group_tab: group) + end + + it 'contains for other group all resource bot scopes including observability scopes' do + expect(subject.available_scopes_for(group)).to match_array %i[api read_api read_repository write_repository read_registry write_registry read_observability write_observability] + end + + it 'contains for admin user all non-default scopes with ADMIN access and without observability scopes' do + user = build_stubbed(:user, admin: true) + + expect(subject.available_scopes_for(user)).to match_array %i[api read_user read_api read_repository write_repository read_registry write_registry sudo admin_mode] + end + + it 'contains for project all resource bot scopes without observability scopes' do + expect(subject.available_scopes_for(project)).to match_array %i[api read_api read_repository write_repository read_registry write_registry] + end + + it 'contains for other group all resource bot scopes without observability scopes' do + other_group = build_stubbed(:group) + + expect(subject.available_scopes_for(other_group)).to match_array %i[api read_api read_repository write_repository read_registry write_registry] + end + end end - context 'with feature flag disabled' do + context 'with admin_mode_for_api feature flag disabled' do before do stub_feature_flags(admin_mode_for_api: false) end it 'contains all non-default scopes' do - stub_container_registry_config(enabled: true) - - expect(subject.all_available_scopes).to match_array %i[api read_user read_api read_repository write_repository read_registry write_registry sudo admin_mode] + expect(subject.all_available_scopes).to match_array %i[api read_user read_api read_repository write_repository read_registry write_registry sudo admin_mode read_observability write_observability] end - it 'contains for admin user all non-default scopes with ADMIN access' do - stub_container_registry_config(enabled: true) - user = create(:user, admin: true) + it 'contains for admin user all non-default scopes with ADMIN access and without observability scopes' do + user = build_stubbed(:user, admin: true) expect(subject.available_scopes_for(user)).to match_array %i[api read_user read_api read_repository write_repository read_registry write_registry sudo] end it 'optional_scopes contains all non-default scopes' do - stub_container_registry_config(enabled: true) - - expect(subject.optional_scopes).to match_array %i[read_user read_api read_repository write_repository read_registry write_registry sudo admin_mode openid profile email] + expect(subject.optional_scopes).to match_array %i[read_user read_api read_repository write_repository read_registry write_registry sudo admin_mode openid profile email read_observability write_observability] end end @@ -120,8 +173,8 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching, feature_cate end end - it 'raises an IpBlacklisted exception' do - expect { subject }.to raise_error(Gitlab::Auth::IpBlacklisted) + it 'raises an IpBlocked exception' do + expect { subject }.to raise_error(Gitlab::Auth::IpBlocked) end end @@ -314,15 +367,17 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching, feature_cate using RSpec::Parameterized::TableSyntax where(:scopes, :abilities) do - 'api' | described_class.full_authentication_abilities - 'read_api' | described_class.read_only_authentication_abilities - 'read_repository' | [:download_code] - 'write_repository' | [:download_code, :push_code] - 'read_user' | [] - 'sudo' | [] - 'openid' | [] - 'profile' | [] - 'email' | [] + 'api' | described_class.full_authentication_abilities + 'read_api' | described_class.read_only_authentication_abilities + 'read_repository' | [:download_code] + 'write_repository' | [:download_code, :push_code] + 'read_user' | [] + 'sudo' | [] + 'openid' | [] + 'profile' | [] + 'email' | [] + 'read_observability' | [] + 'write_observability' | [] end with_them do @@ -1024,6 +1079,7 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching, feature_cate it { is_expected.to include(*described_class::API_SCOPES - [:read_user]) } it { is_expected.to include(*described_class::REPOSITORY_SCOPES) } it { is_expected.to include(*described_class.registry_scopes) } + it { is_expected.to include(*described_class::OBSERVABILITY_SCOPES) } end private diff --git a/spec/lib/gitlab/avatar_cache_spec.rb b/spec/lib/gitlab/avatar_cache_spec.rb index ffe6f81b6e7..a57d811edaf 100644 --- a/spec/lib/gitlab/avatar_cache_spec.rb +++ b/spec/lib/gitlab/avatar_cache_spec.rb @@ -62,40 +62,52 @@ RSpec.describe Gitlab::AvatarCache, :clean_gitlab_redis_cache do end describe "#delete_by_email" do - subject { described_class.delete_by_email(*emails) } + shared_examples 'delete emails' do + subject { described_class.delete_by_email(*emails) } - before do - perform_fetch - end + before do + perform_fetch + end - context "no emails, somehow" do - let(:emails) { [] } + context "no emails, somehow" do + let(:emails) { [] } - it { is_expected.to eq(0) } - end + it { is_expected.to eq(0) } + end - context "single email" do - let(:emails) { "foo@bar.com" } + context "single email" do + let(:emails) { "foo@bar.com" } - it "removes the email" do - expect(read(key, "20:2:true")).to eq(avatar_path) + it "removes the email" do + expect(read(key, "20:2:true")).to eq(avatar_path) - expect(subject).to eq(1) + expect(subject).to eq(1) - expect(read(key, "20:2:true")).to eq(nil) + expect(read(key, "20:2:true")).to eq(nil) + end end - end - context "multiple emails" do - let(:emails) { ["foo@bar.com", "missing@baz.com"] } + context "multiple emails" do + let(:emails) { ["foo@bar.com", "missing@baz.com"] } - it "removes the emails it finds" do - expect(read(key, "20:2:true")).to eq(avatar_path) + it "removes the emails it finds" do + expect(read(key, "20:2:true")).to eq(avatar_path) - expect(subject).to eq(1) + expect(subject).to eq(1) - expect(read(key, "20:2:true")).to eq(nil) + expect(read(key, "20:2:true")).to eq(nil) + end + end + end + + context 'when feature flag disabled' do + before do + stub_feature_flags(use_pipeline_over_multikey: false) end + + it_behaves_like 'delete emails' end + + it_behaves_like 'delete emails' end end diff --git a/spec/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens_spec.rb b/spec/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens_spec.rb index 7075d4694ae..92fec48454c 100644 --- a/spec/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::BackgroundMigration::BackfillAdminModeScopeForPersonalAccessTokens, - :migration, schema: 20221228103133, feature_category: :authentication_and_authorization do + :migration, schema: 20221228103133, feature_category: :system_access do let(:users) { table(:users) } let(:personal_access_tokens) { table(:personal_access_tokens) } @@ -24,8 +24,12 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillAdminModeScopeForPersonalAcc personal_access_tokens.create!(name: 'admin 4', user_id: admin.id, scopes: "---\n- admin_mode\n") end - let!(:pat_admin_2) { personal_access_tokens.create!(name: 'admin 5', user_id: admin.id, scopes: "---\n- read_api\n") } - let!(:pat_not_in_range) { personal_access_tokens.create!(name: 'admin 6', user_id: admin.id, scopes: "---\n- api\n") } + let!(:pat_with_symbol_in_scopes) do + personal_access_tokens.create!(name: 'admin 5', user_id: admin.id, scopes: "---\n- :api\n") + end + + let!(:pat_admin_2) { personal_access_tokens.create!(name: 'admin 6', user_id: admin.id, scopes: "---\n- read_api\n") } + let!(:pat_not_in_range) { personal_access_tokens.create!(name: 'admin 7', user_id: admin.id, scopes: "---\n- api\n") } subject do described_class.new( @@ -47,6 +51,7 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillAdminModeScopeForPersonalAcc expect(pat_revoked.reload.scopes).to eq("---\n- api\n") expect(pat_expired.reload.scopes).to eq("---\n- api\n") expect(pat_admin_mode.reload.scopes).to eq("---\n- admin_mode\n") + expect(pat_with_symbol_in_scopes.reload.scopes).to eq("---\n- api\n- admin_mode\n") expect(pat_admin_2.reload.scopes).to eq("---\n- read_api\n- admin_mode\n") expect(pat_not_in_range.reload.scopes).to eq("---\n- api\n") end diff --git a/spec/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities_spec.rb b/spec/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities_spec.rb index 3aab0cdf54b..edb6ff59340 100644 --- a/spec/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities_spec.rb @@ -4,10 +4,12 @@ require 'spec_helper' RSpec.describe Gitlab::BackgroundMigration::BackfillClusterAgentsHasVulnerabilities, :migration do # rubocop:disable Layout/LineLength let(:migration) do - described_class.new(start_id: 1, end_id: 10, - batch_table: table_name, batch_column: batch_column, - sub_batch_size: sub_batch_size, pause_ms: pause_ms, - connection: ApplicationRecord.connection) + described_class.new( + start_id: 1, end_id: 10, + batch_table: table_name, batch_column: batch_column, + sub_batch_size: sub_batch_size, pause_ms: pause_ms, + connection: ApplicationRecord.connection + ) end let(:users_table) { table(:users) } diff --git a/spec/lib/gitlab/background_migration/backfill_design_management_repositories_spec.rb b/spec/lib/gitlab/background_migration/backfill_design_management_repositories_spec.rb new file mode 100644 index 00000000000..0cabdc78db8 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_design_management_repositories_spec.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe( + Gitlab::BackgroundMigration::BackfillDesignManagementRepositories, + schema: 20230406121544, + feature_category: :geo_replication +) do + let!(:namespaces) { table(:namespaces) } + let!(:projects) { table(:projects) } + let!(:design_management_repositories) { table(:design_management_repositories) } + + subject(:migration) do + described_class.new( + start_id: projects.minimum(:id), + end_id: projects.maximum(:id), + batch_table: :projects, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ) + end + + describe '#perform' do + it 'creates design_management_repositories entries for all projects in range' do + namespace1 = create_namespace('test1') + namespace2 = create_namespace('test2') + project1 = create_project(namespace1, 'test1') + project2 = create_project(namespace2, 'test2') + design_management_repositories.create!(project_id: project2.id) + + expect { migration.perform } + .to change { design_management_repositories.pluck(:project_id) } + .from([project2.id]) + .to match_array([project1.id, project2.id]) + end + + context 'when project_id already exists in design_management_repositories' do + it "doesn't duplicate project_id" do + namespace = create_namespace('test1') + project = create_project(namespace, 'test1') + design_management_repositories.create!(project_id: project.id) + + expect { migration.perform } + .not_to change { design_management_repositories.pluck(:project_id) } + end + end + + def create_namespace(name) + namespaces.create!( + name: name, + path: name, + type: 'Project' + ) + end + + def create_project(namespace, name) + projects.create!( + namespace_id: namespace.id, + project_namespace_id: namespace.id, + name: name, + path: name + ) + end + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_environment_tiers_spec.rb b/spec/lib/gitlab/background_migration/backfill_environment_tiers_spec.rb index 788ed40b61e..9026c327e3c 100644 --- a/spec/lib/gitlab/background_migration/backfill_environment_tiers_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_environment_tiers_spec.rb @@ -8,10 +8,12 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillEnvironmentTiers, let!(:project) { table(:projects).create!(namespace_id: namespace.id, project_namespace_id: namespace.id) } let(:migration) do - described_class.new(start_id: 1, end_id: 1000, - batch_table: :environments, batch_column: :id, - sub_batch_size: 10, pause_ms: 0, - connection: ApplicationRecord.connection) + described_class.new( + start_id: 1, end_id: 1000, + batch_table: :environments, batch_column: :id, + sub_batch_size: 10, pause_ms: 0, + connection: ApplicationRecord.connection + ) end describe '#perform' do diff --git a/spec/lib/gitlab/background_migration/backfill_group_features_spec.rb b/spec/lib/gitlab/background_migration/backfill_group_features_spec.rb index e0be5a785b8..023d4b04e63 100644 --- a/spec/lib/gitlab/background_migration/backfill_group_features_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_group_features_spec.rb @@ -7,14 +7,16 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillGroupFeatures, :migration, s let(:namespaces) { table(:namespaces) } subject do - described_class.new(start_id: 1, - end_id: 4, - batch_table: :namespaces, - batch_column: :id, - sub_batch_size: 10, - pause_ms: 0, - job_arguments: [4], - connection: ActiveRecord::Base.connection) + described_class.new( + start_id: 1, + end_id: 4, + batch_table: :namespaces, + batch_column: :id, + sub_batch_size: 10, + pause_ms: 0, + job_arguments: [4], + connection: ActiveRecord::Base.connection + ) end describe '#perform' do diff --git a/spec/lib/gitlab/background_migration/backfill_imported_issue_search_data_spec.rb b/spec/lib/gitlab/background_migration/backfill_imported_issue_search_data_spec.rb index 479afb56210..b3f04055e0a 100644 --- a/spec/lib/gitlab/background_migration/backfill_imported_issue_search_data_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_imported_issue_search_data_spec.rb @@ -30,13 +30,15 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillImportedIssueSearchData, end let(:migration) do - described_class.new(start_id: issue.id, - end_id: issue.id + 30, - batch_table: :issues, - batch_column: :id, - sub_batch_size: 2, - pause_ms: 0, - connection: ApplicationRecord.connection) + described_class.new( + start_id: issue.id, + end_id: issue.id + 30, + batch_table: :issues, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: ApplicationRecord.connection + ) end let(:perform_migration) { migration.perform } diff --git a/spec/lib/gitlab/background_migration/backfill_namespace_details_spec.rb b/spec/lib/gitlab/background_migration/backfill_namespace_details_spec.rb index b6282de0da6..39ad60fb13b 100644 --- a/spec/lib/gitlab/background_migration/backfill_namespace_details_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_namespace_details_spec.rb @@ -7,27 +7,36 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillNamespaceDetails, :migration let(:namespace_details) { table(:namespace_details) } subject(:perform_migration) do - described_class.new(start_id: namespaces.minimum(:id), - end_id: namespaces.maximum(:id), - batch_table: :namespaces, - batch_column: :id, - sub_batch_size: 2, - pause_ms: 0, - connection: ActiveRecord::Base.connection) - .perform + described_class.new( + start_id: namespaces.minimum(:id), + end_id: namespaces.maximum(:id), + batch_table: :namespaces, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ).perform end describe '#perform' do it 'creates details for all namespaces in range' do - namespace1 = namespaces.create!(id: 5, name: 'test1', path: 'test1', description: "Some description1", - description_html: "Some description html1", cached_markdown_version: 4) - namespaces.create!(id: 6, name: 'test2', path: 'test2', type: 'Project', - description: "Some description2", description_html: "Some description html2", - cached_markdown_version: 4) - namespace3 = namespaces.create!(id: 7, name: 'test3', path: 'test3', description: "Some description3", - description_html: "Some description html3", cached_markdown_version: 4) - namespace4 = namespaces.create!(id: 8, name: 'test4', path: 'test4', description: "Some description3", - description_html: "Some description html4", cached_markdown_version: 4) + namespace1 = namespaces.create!( + id: 5, name: 'test1', path: 'test1', description: "Some description1", + description_html: "Some description html1", cached_markdown_version: 4 + ) + namespaces.create!( + id: 6, name: 'test2', path: 'test2', type: 'Project', + description: "Some description2", description_html: "Some description html2", + cached_markdown_version: 4 + ) + namespace3 = namespaces.create!( + id: 7, name: 'test3', path: 'test3', description: "Some description3", + description_html: "Some description html3", cached_markdown_version: 4 + ) + namespace4 = namespaces.create!( + id: 8, name: 'test4', path: 'test4', description: "Some description3", + description_html: "Some description html4", cached_markdown_version: 4 + ) namespace_details.delete_all expect(namespace_details.pluck(:namespace_id)).to eql [] diff --git a/spec/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route_spec.rb b/spec/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route_spec.rb index b821efcadb0..3a8a327550b 100644 --- a/spec/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route_spec.rb @@ -22,18 +22,29 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillNamespaceIdForNamespaceRoute subject(:perform_migration) { migration.perform(1, 10, table_name, batch_column, sub_batch_size, pause_ms) } before do - routes_table.create!(id: 1, name: 'test1', path: 'test1', source_id: namespace1.id, - source_type: namespace1.class.sti_name) - routes_table.create!(id: 2, name: 'test2', path: 'test2', source_id: namespace2.id, - source_type: namespace2.class.sti_name) - routes_table.create!(id: 5, name: 'test3', path: 'test3', source_id: project1.id, - source_type: project1.class.sti_name) # should be ignored - project route - routes_table.create!(id: 6, name: 'test4', path: 'test4', source_id: non_existing_record_id, - source_type: namespace3.class.sti_name) # should be ignored - invalid source_id - routes_table.create!(id: 10, name: 'test5', path: 'test5', source_id: namespace3.id, - source_type: namespace3.class.sti_name) - routes_table.create!(id: 11, name: 'test6', path: 'test6', source_id: namespace4.id, - source_type: namespace4.class.sti_name) # should be ignored - outside the scope + routes_table.create!( + id: 1, name: 'test1', path: 'test1', source_id: namespace1.id, source_type: namespace1.class.sti_name + ) + + routes_table.create!( + id: 2, name: 'test2', path: 'test2', source_id: namespace2.id, source_type: namespace2.class.sti_name + ) + + routes_table.create!( + id: 5, name: 'test3', path: 'test3', source_id: project1.id, source_type: project1.class.sti_name + ) # should be ignored - project route + + routes_table.create!( + id: 6, name: 'test4', path: 'test4', source_id: non_existing_record_id, source_type: namespace3.class.sti_name + ) # should be ignored - invalid source_id + + routes_table.create!( + id: 10, name: 'test5', path: 'test5', source_id: namespace3.id, source_type: namespace3.class.sti_name + ) + + routes_table.create!( + id: 11, name: 'test6', path: 'test6', source_id: namespace4.id, source_type: namespace4.class.sti_name + ) # should be ignored - outside the scope end it 'backfills `type` for the selected records', :aggregate_failures do diff --git a/spec/lib/gitlab/background_migration/backfill_namespace_id_of_vulnerability_reads_spec.rb b/spec/lib/gitlab/background_migration/backfill_namespace_id_of_vulnerability_reads_spec.rb index 564aa3b8c01..6a55c6951d5 100644 --- a/spec/lib/gitlab/background_migration/backfill_namespace_id_of_vulnerability_reads_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_namespace_id_of_vulnerability_reads_spec.rb @@ -38,14 +38,15 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillNamespaceIdOfVulnerabilityRe end subject(:perform_migration) do - described_class.new(start_id: vulnerability_read.vulnerability_id, - end_id: vulnerability_read.vulnerability_id, - batch_table: :vulnerability_reads, - batch_column: :vulnerability_id, - sub_batch_size: 1, - pause_ms: 0, - connection: ActiveRecord::Base.connection) - .perform + described_class.new( + start_id: vulnerability_read.vulnerability_id, + end_id: vulnerability_read.vulnerability_id, + batch_table: :vulnerability_reads, + batch_column: :vulnerability_id, + sub_batch_size: 1, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ).perform end it 'sets the namespace_id of existing record' do diff --git a/spec/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children_spec.rb b/spec/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children_spec.rb deleted file mode 100644 index 876eb070745..00000000000 --- a/spec/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children_spec.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::BackfillNamespaceTraversalIdsChildren, :migration, schema: 20210826171758 do - let(:namespaces_table) { table(:namespaces) } - - let!(:user_namespace) { namespaces_table.create!(id: 1, name: 'user', path: 'user', type: nil) } - let!(:root_group) { namespaces_table.create!(id: 2, name: 'group', path: 'group', type: 'Group', parent_id: nil) } - let!(:sub_group) { namespaces_table.create!(id: 3, name: 'subgroup', path: 'subgroup', type: 'Group', parent_id: 2) } - - describe '#perform' do - it 'backfills traversal_ids for child namespaces' do - described_class.new.perform(1, 3, 5) - - expect(user_namespace.reload.traversal_ids).to eq([]) - expect(root_group.reload.traversal_ids).to eq([]) - expect(sub_group.reload.traversal_ids).to eq([root_group.id, sub_group.id]) - end - end -end diff --git a/spec/lib/gitlab/background_migration/backfill_namespace_traversal_ids_roots_spec.rb b/spec/lib/gitlab/background_migration/backfill_namespace_traversal_ids_roots_spec.rb deleted file mode 100644 index ad9b54608c6..00000000000 --- a/spec/lib/gitlab/background_migration/backfill_namespace_traversal_ids_roots_spec.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::BackfillNamespaceTraversalIdsRoots, :migration, schema: 20210826171758 do - let(:namespaces_table) { table(:namespaces) } - - let!(:user_namespace) { namespaces_table.create!(id: 1, name: 'user', path: 'user', type: nil) } - let!(:root_group) { namespaces_table.create!(id: 2, name: 'group', path: 'group', type: 'Group', parent_id: nil) } - let!(:sub_group) { namespaces_table.create!(id: 3, name: 'subgroup', path: 'subgroup', type: 'Group', parent_id: 2) } - - describe '#perform' do - it 'backfills traversal_ids for root namespaces' do - described_class.new.perform(1, 3, 5) - - expect(user_namespace.reload.traversal_ids).to eq([user_namespace.id]) - expect(root_group.reload.traversal_ids).to eq([root_group.id]) - expect(sub_group.reload.traversal_ids).to eq([]) - end - end -end diff --git a/spec/lib/gitlab/background_migration/backfill_partitioned_table_spec.rb b/spec/lib/gitlab/background_migration/backfill_partitioned_table_spec.rb new file mode 100644 index 00000000000..53216cc780b --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_partitioned_table_spec.rb @@ -0,0 +1,140 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillPartitionedTable, feature_category: :database do + subject(:backfill_job) do + described_class.new( + start_id: 1, + end_id: 3, + batch_table: source_table, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + job_arguments: [destination_table], + connection: connection + ) + end + + let(:connection) { ApplicationRecord.connection } + let(:source_table) { '_test_source_table' } + let(:destination_table) { "#{source_table}_partitioned" } + let(:source_model) { Class.new(ApplicationRecord) } + let(:destination_model) { Class.new(ApplicationRecord) } + + describe '#perform' do + context 'without the destination table' do + let(:expected_error_message) do + "exiting backfill migration because partitioned table #{destination_table} does not exist. " \ + "This could be due to rollback of the migration which created the partitioned table." + end + + it 'raises an exception' do + expect { backfill_job.perform }.to raise_error(expected_error_message) + end + end + + context 'with destination table being not partitioned' do + before do + connection.execute(<<~SQL) + CREATE TABLE #{destination_table} ( + id serial NOT NULL, + col1 int NOT NULL, + col2 text NOT NULL, + created_at timestamptz NOT NULL, + PRIMARY KEY (id, created_at) + ) + SQL + end + + after do + connection.drop_table destination_table + end + + let(:expected_error_message) do + "exiting backfill migration because the given destination table is not partitioned." + end + + it 'raises an exception' do + expect { backfill_job.perform }.to raise_error(expected_error_message) + end + end + + context 'when the destination table exists' do + before do + connection.execute(<<~SQL) + CREATE TABLE #{source_table} ( + id serial NOT NULL PRIMARY KEY, + col1 int NOT NULL, + col2 text NOT NULL, + created_at timestamptz NOT NULL + ) + SQL + + connection.execute(<<~SQL) + CREATE TABLE #{destination_table} ( + id serial NOT NULL, + col1 int NOT NULL, + col2 text NOT NULL, + created_at timestamptz NOT NULL, + PRIMARY KEY (id, created_at) + ) PARTITION BY RANGE (created_at) + SQL + + connection.execute(<<~SQL) + CREATE TABLE #{destination_table}_202001 PARTITION OF #{destination_table} + FOR VALUES FROM ('2020-01-01') TO ('2020-02-01') + SQL + + connection.execute(<<~SQL) + CREATE TABLE #{destination_table}_202002 PARTITION OF #{destination_table} + FOR VALUES FROM ('2020-02-01') TO ('2020-03-01') + SQL + + source_model.table_name = source_table + destination_model.table_name = destination_table + end + + after do + connection.drop_table source_table + connection.drop_table destination_table + end + + let(:timestamp) { Time.utc(2020, 1, 2).round } + let!(:source1) { create_source_record(timestamp) } + let!(:source2) { create_source_record(timestamp + 1.day) } + let!(:source3) { create_source_record(timestamp + 1.month) } + + it 'copies data into the destination table idempotently' do + expect(destination_model.count).to eq(0) + + backfill_job.perform + + expect(destination_model.count).to eq(3) + + source_model.find_each do |source_record| + destination_record = destination_model.find_by_id(source_record.id) + + expect(destination_record.attributes).to eq(source_record.attributes) + end + + backfill_job.perform + + expect(destination_model.count).to eq(3) + end + + it 'breaks the assigned batch into smaller sub batches' do + expect_next_instance_of(Gitlab::Database::PartitioningMigrationHelpers::BulkCopy) do |bulk_copy| + expect(bulk_copy).to receive(:copy_between).with(source1.id, source2.id) + expect(bulk_copy).to receive(:copy_between).with(source3.id, source3.id) + end + + backfill_job.perform + end + end + end + + def create_source_record(timestamp) + source_model.create!(col1: 123, col2: 'original value', created_at: timestamp) + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_prepared_at_merge_requests_spec.rb b/spec/lib/gitlab/background_migration/backfill_prepared_at_merge_requests_spec.rb new file mode 100644 index 00000000000..28ecfae1bd4 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_prepared_at_merge_requests_spec.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillPreparedAtMergeRequests, :migration, + feature_category: :code_review_workflow, schema: 20230202135758 do + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:mr_table) { table(:merge_requests) } + + let(:namespace) { namespaces.create!(name: 'batchtest1', type: 'Group', path: 'space1') } + let(:proj_namespace) { namespaces.create!(name: 'proj1', path: 'proj1', type: 'Project', parent_id: namespace.id) } + let(:project) do + projects.create!(name: 'proj1', path: 'proj1', namespace_id: namespace.id, project_namespace_id: proj_namespace.id) + end + + it 'updates merge requests with prepared_at nil' do + time = Time.current + + mr_1 = mr_table.create!(target_project_id: project.id, source_branch: 'master', target_branch: 'feature', + prepared_at: nil, merge_status: 'checking') + mr_2 = mr_table.create!(target_project_id: project.id, source_branch: 'master', target_branch: 'feature', + prepared_at: nil, merge_status: 'preparing') + mr_3 = mr_table.create!(target_project_id: project.id, source_branch: 'master', target_branch: 'feature', + prepared_at: time) + mr_4 = mr_table.create!(target_project_id: project.id, source_branch: 'master', target_branch: 'feature', + prepared_at: time, merge_status: 'checking') + mr_5 = mr_table.create!(target_project_id: project.id, source_branch: 'master', target_branch: 'feature', + prepared_at: time, merge_status: 'preparing') + + test_worker = described_class.new( + start_id: mr_1.id, + end_id: [(mr_5.id + 1), 100].max, + batch_table: :merge_requests, + batch_column: :id, + sub_batch_size: 10, + pause_ms: 0, + connection: ApplicationRecord.connection + ) + + expect(mr_1.prepared_at).to be_nil + expect(mr_2.prepared_at).to be_nil + expect(mr_3.prepared_at.to_i).to eq(time.to_i) + expect(mr_4.prepared_at.to_i).to eq(time.to_i) + expect(mr_5.prepared_at.to_i).to eq(time.to_i) + + test_worker.perform + + expect(mr_1.reload.prepared_at.to_i).to eq(mr_1.created_at.to_i) + expect(mr_2.reload.prepared_at).to be_nil + expect(mr_3.reload.prepared_at.to_i).to eq(time.to_i) + expect(mr_4.reload.prepared_at.to_i).to eq(time.to_i) + expect(mr_5.reload.prepared_at.to_i).to eq(time.to_i) + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level_spec.rb b/spec/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level_spec.rb index fd6c055b9f6..47ff2883fb2 100644 --- a/spec/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level_spec.rb @@ -101,14 +101,15 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillProjectFeaturePackageRegistr end subject(:perform_migration) do - described_class.new(start_id: project1.id, - end_id: project5.id, - batch_table: :projects, - batch_column: :id, - sub_batch_size: 2, - pause_ms: 0, - connection: ActiveRecord::Base.connection) - .perform + described_class.new( + start_id: project1.id, + end_id: project5.id, + batch_table: :projects, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ).perform end it 'backfills project_features.package_registry_access_level', :aggregate_failures do diff --git a/spec/lib/gitlab/background_migration/backfill_project_member_namespace_id_spec.rb b/spec/lib/gitlab/background_migration/backfill_project_member_namespace_id_spec.rb index ca7ca41a33e..96f49624d22 100644 --- a/spec/lib/gitlab/background_migration/backfill_project_member_namespace_id_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_project_member_namespace_id_spec.rb @@ -4,10 +4,12 @@ require 'spec_helper' RSpec.describe Gitlab::BackgroundMigration::BackfillProjectMemberNamespaceId, :migration, schema: 20220516054011 do let(:migration) do - described_class.new(start_id: 1, end_id: 10, - batch_table: table_name, batch_column: batch_column, - sub_batch_size: sub_batch_size, pause_ms: pause_ms, - connection: ApplicationRecord.connection) + described_class.new( + start_id: 1, end_id: 10, + batch_table: table_name, batch_column: batch_column, + sub_batch_size: sub_batch_size, pause_ms: pause_ms, + connection: ApplicationRecord.connection + ) end let(:members_table) { table(:members) } @@ -35,37 +37,55 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillProjectMemberNamespaceId, :m projects_table.create!(id: 102, name: 'project3', path: 'project3', namespace_id: 202, project_namespace_id: 302) # project1, no member namespace (fill in) - members_table.create!(id: 1, source_id: 100, - source_type: 'Project', type: 'ProjectMember', - member_namespace_id: nil, access_level: 10, notification_level: 3) + members_table.create!( + id: 1, source_id: 100, + source_type: 'Project', type: 'ProjectMember', + member_namespace_id: nil, access_level: 10, notification_level: 3 + ) + # bogus source id, no member namespace id (do nothing) - members_table.create!(id: 2, source_id: non_existing_record_id, - source_type: 'Project', type: 'ProjectMember', - member_namespace_id: nil, access_level: 10, notification_level: 3) + members_table.create!( + id: 2, source_id: non_existing_record_id, + source_type: 'Project', type: 'ProjectMember', + member_namespace_id: nil, access_level: 10, notification_level: 3 + ) + # project3, existing member namespace id (do nothing) - members_table.create!(id: 3, source_id: 102, - source_type: 'Project', type: 'ProjectMember', - member_namespace_id: 300, access_level: 10, notification_level: 3) + members_table.create!( + id: 3, source_id: 102, + source_type: 'Project', type: 'ProjectMember', + member_namespace_id: 300, access_level: 10, notification_level: 3 + ) # Group memberships (do not change) # group1, no member namespace (do nothing) - members_table.create!(id: 4, source_id: 201, - source_type: 'Namespace', type: 'GroupMember', - member_namespace_id: nil, access_level: 10, notification_level: 3) + members_table.create!( + id: 4, source_id: 201, + source_type: 'Namespace', type: 'GroupMember', + member_namespace_id: nil, access_level: 10, notification_level: 3 + ) + # group2, existing member namespace (do nothing) - members_table.create!(id: 5, source_id: 202, - source_type: 'Namespace', type: 'GroupMember', - member_namespace_id: 201, access_level: 10, notification_level: 3) + members_table.create!( + id: 5, source_id: 202, + source_type: 'Namespace', type: 'GroupMember', + member_namespace_id: 201, access_level: 10, notification_level: 3 + ) # Project Namespace memberships (do not change) # project namespace, existing member namespace (do nothing) - members_table.create!(id: 6, source_id: 300, - source_type: 'Namespace', type: 'ProjectNamespaceMember', - member_namespace_id: 201, access_level: 10, notification_level: 3) + members_table.create!( + id: 6, source_id: 300, + source_type: 'Namespace', type: 'ProjectNamespaceMember', + member_namespace_id: 201, access_level: 10, notification_level: 3 + ) + # project namespace, not member namespace (do nothing) - members_table.create!(id: 7, source_id: 301, - source_type: 'Namespace', type: 'ProjectNamespaceMember', - member_namespace_id: 201, access_level: 10, notification_level: 3) + members_table.create!( + id: 7, source_id: 301, + source_type: 'Namespace', type: 'ProjectNamespaceMember', + member_namespace_id: 201, access_level: 10, notification_level: 3 + ) end it 'backfills `member_namespace_id` for the selected records', :aggregate_failures do diff --git a/spec/lib/gitlab/background_migration/backfill_project_namespace_details_spec.rb b/spec/lib/gitlab/background_migration/backfill_project_namespace_details_spec.rb index 01daf16d10c..aac17a426b5 100644 --- a/spec/lib/gitlab/background_migration/backfill_project_namespace_details_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_project_namespace_details_spec.rb @@ -8,32 +8,41 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillProjectNamespaceDetails, :mi let!(:projects) { table(:projects) } subject(:perform_migration) do - described_class.new(start_id: projects.minimum(:id), - end_id: projects.maximum(:id), - batch_table: :projects, - batch_column: :id, - sub_batch_size: 2, - pause_ms: 0, - connection: ActiveRecord::Base.connection) - .perform + described_class.new( + start_id: projects.minimum(:id), + end_id: projects.maximum(:id), + batch_table: :projects, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ).perform end describe '#perform' do it 'creates details for all project namespaces in range' do - namespaces.create!(id: 5, name: 'test1', path: 'test1', description: "Some description1", - description_html: "Some description html1", cached_markdown_version: 4) + namespaces.create!( + id: 5, name: 'test1', path: 'test1', description: "Some description1", + description_html: "Some description html1", cached_markdown_version: 4 + ) project_namespace1 = namespaces.create!(id: 6, name: 'test2', path: 'test2', type: 'Project') - namespaces.create!(id: 7, name: 'test3', path: 'test3', description: "Some description3", - description_html: "Some description html3", cached_markdown_version: 4) + namespaces.create!( + id: 7, name: 'test3', path: 'test3', description: "Some description3", + description_html: "Some description html3", cached_markdown_version: 4 + ) project_namespace2 = namespaces.create!(id: 8, name: 'test4', path: 'test4', type: 'Project') - project1 = projects.create!(namespace_id: project_namespace1.id, name: 'gitlab1', path: 'gitlab1', - project_namespace_id: project_namespace1.id, description: "Some description2", - description_html: "Some description html2", cached_markdown_version: 4) - project2 = projects.create!(namespace_id: project_namespace2.id, name: 'gitlab2', path: 'gitlab2', - project_namespace_id: project_namespace2.id, - description: "Some description3", - description_html: "Some description html4", cached_markdown_version: 4) + project1 = projects.create!( + namespace_id: project_namespace1.id, name: 'gitlab1', path: 'gitlab1', + project_namespace_id: project_namespace1.id, description: "Some description2", + description_html: "Some description html2", cached_markdown_version: 4 + ) + project2 = projects.create!( + namespace_id: project_namespace2.id, name: 'gitlab2', path: 'gitlab2', + project_namespace_id: project_namespace2.id, + description: "Some description3", + description_html: "Some description html4", cached_markdown_version: 4 + ) namespace_details.delete_all diff --git a/spec/lib/gitlab/background_migration/backfill_project_wiki_repositories_spec.rb b/spec/lib/gitlab/background_migration/backfill_project_wiki_repositories_spec.rb new file mode 100644 index 00000000000..e81bd0604e6 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_project_wiki_repositories_spec.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe( + Gitlab::BackgroundMigration::BackfillProjectWikiRepositories, + schema: 20230306195007, + feature_category: :geo_replication) do + let!(:namespaces) { table(:namespaces) } + let!(:projects) { table(:projects) } + let!(:project_wiki_repositories) { table(:project_wiki_repositories) } + + subject(:migration) do + described_class.new( + start_id: projects.minimum(:id), + end_id: projects.maximum(:id), + batch_table: :projects, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ) + end + + describe '#perform' do + it 'creates project_wiki_repositories entries for all projects in range' do + namespace1 = create_namespace('test1') + namespace2 = create_namespace('test2') + project1 = create_project(namespace1, 'test1') + project2 = create_project(namespace2, 'test2') + project_wiki_repositories.create!(project_id: project2.id) + + expect { migration.perform } + .to change { project_wiki_repositories.pluck(:project_id) } + .from([project2.id]) + .to match_array([project1.id, project2.id]) + end + + it 'does nothing if project_id already exist in project_wiki_repositories' do + namespace = create_namespace('test1') + project = create_project(namespace, 'test1') + project_wiki_repositories.create!(project_id: project.id) + + expect { migration.perform } + .not_to change { project_wiki_repositories.pluck(:project_id) } + end + + def create_namespace(name) + namespaces.create!( + name: name, + path: name, + type: 'Project' + ) + end + + def create_project(namespace, name) + projects.create!( + namespace_id: namespace.id, + project_namespace_id: namespace.id, + name: name, + path: name + ) + end + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_releases_author_id_spec.rb b/spec/lib/gitlab/background_migration/backfill_releases_author_id_spec.rb index d8ad10849f2..898f241a930 100644 --- a/spec/lib/gitlab/background_migration/backfill_releases_author_id_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_releases_author_id_spec.rb @@ -10,35 +10,52 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillReleasesAuthorId, let!(:test_user) { user_table.create!(name: 'test', email: 'test@example.com', username: 'test', projects_limit: 10) } let!(:ghost_user) do - user_table.create!(name: 'ghost', email: 'ghost@example.com', - username: 'ghost', user_type: User::USER_TYPES['ghost'], projects_limit: 100000) + user_table.create!( + name: 'ghost', email: 'ghost@example.com', + username: 'ghost', user_type: User::USER_TYPES['ghost'], projects_limit: 100000 + ) end let(:migration) do - described_class.new(start_id: 1, end_id: 100, - batch_table: :releases, batch_column: :id, - sub_batch_size: 10, pause_ms: 0, - job_arguments: [ghost_user.id], - connection: ApplicationRecord.connection) + described_class.new( + start_id: 1, end_id: 100, + batch_table: :releases, batch_column: :id, + sub_batch_size: 10, pause_ms: 0, + job_arguments: [ghost_user.id], + connection: ApplicationRecord.connection + ) end subject(:perform_migration) { migration.perform } before do - releases_table.create!(tag: 'tag1', name: 'tag1', - released_at: (date_time - 1.minute), author_id: test_user.id) - releases_table.create!(tag: 'tag2', name: 'tag2', - released_at: (date_time - 2.minutes), author_id: test_user.id) - releases_table.new(tag: 'tag3', name: 'tag3', - released_at: (date_time - 3.minutes), author_id: nil).save!(validate: false) - releases_table.new(tag: 'tag4', name: 'tag4', - released_at: (date_time - 4.minutes), author_id: nil).save!(validate: false) - releases_table.new(tag: 'tag5', name: 'tag5', - released_at: (date_time - 5.minutes), author_id: nil).save!(validate: false) - releases_table.create!(tag: 'tag6', name: 'tag6', - released_at: (date_time - 6.minutes), author_id: test_user.id) - releases_table.new(tag: 'tag7', name: 'tag7', - released_at: (date_time - 7.minutes), author_id: nil).save!(validate: false) + releases_table.create!( + tag: 'tag1', name: 'tag1', released_at: (date_time - 1.minute), author_id: test_user.id + ) + + releases_table.create!( + tag: 'tag2', name: 'tag2', released_at: (date_time - 2.minutes), author_id: test_user.id + ) + + releases_table.new( + tag: 'tag3', name: 'tag3', released_at: (date_time - 3.minutes), author_id: nil + ).save!(validate: false) + + releases_table.new( + tag: 'tag4', name: 'tag4', released_at: (date_time - 4.minutes), author_id: nil + ).save!(validate: false) + + releases_table.new( + tag: 'tag5', name: 'tag5', released_at: (date_time - 5.minutes), author_id: nil + ).save!(validate: false) + + releases_table.create!( + tag: 'tag6', name: 'tag6', released_at: (date_time - 6.minutes), author_id: test_user.id + ) + + releases_table.new( + tag: 'tag7', name: 'tag7', released_at: (date_time - 7.minutes), author_id: nil + ).save!(validate: false) end it 'backfills `author_id` for the selected records', :aggregate_failures do diff --git a/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb b/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb index 80fd86e90bb..d8874cb811b 100644 --- a/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb @@ -2,8 +2,8 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::BackfillSnippetRepositories, :migration, schema: 20210826171758, -feature_category: :source_code_management do +RSpec.describe Gitlab::BackgroundMigration::BackfillSnippetRepositories, :migration, schema: 20211202041233, + feature_category: :source_code_management do let(:gitlab_shell) { Gitlab::Shell.new } let(:users) { table(:users) } let(:snippets) { table(:snippets) } @@ -14,24 +14,28 @@ feature_category: :source_code_management do let(:user_name) { 'Test' } let!(:user) do - users.create!(id: 1, - email: 'user@example.com', - projects_limit: 10, - username: 'test', - name: user_name, - state: user_state, - last_activity_on: 1.minute.ago, - user_type: user_type, - confirmed_at: 1.day.ago) + users.create!( + id: 1, + email: 'user@example.com', + projects_limit: 10, + username: 'test', + name: user_name, + state: user_state, + last_activity_on: 1.minute.ago, + user_type: user_type, + confirmed_at: 1.day.ago + ) end let!(:migration_bot) do - users.create!(id: 100, - email: "noreply+gitlab-migration-bot%s@#{Settings.gitlab.host}", - user_type: HasUserType::USER_TYPES[:migration_bot], - name: 'GitLab Migration Bot', - projects_limit: 10, - username: 'bot') + users.create!( + id: 100, + email: "noreply+gitlab-migration-bot%s@#{Settings.gitlab.host}", + user_type: HasUserType::USER_TYPES[:migration_bot], + name: 'GitLab Migration Bot', + projects_limit: 10, + username: 'bot' + ) end let!(:snippet_with_repo) { snippets.create!(id: 1, type: 'PersonalSnippet', author_id: user.id, file_name: file_name, content: content) } @@ -260,15 +264,17 @@ feature_category: :source_code_management do context 'when both user name and snippet file_name are invalid' do let(:user_name) { '.' } let!(:other_user) do - users.create!(id: 2, - email: 'user2@example.com', - projects_limit: 10, - username: 'test2', - name: 'Test2', - state: user_state, - last_activity_on: 1.minute.ago, - user_type: user_type, - confirmed_at: 1.day.ago) + users.create!( + id: 2, + email: 'user2@example.com', + projects_limit: 10, + username: 'test2', + name: 'Test2', + state: user_state, + last_activity_on: 1.minute.ago, + user_type: user_type, + confirmed_at: 1.day.ago + ) end let!(:invalid_snippet) { snippets.create!(id: 4, type: 'PersonalSnippet', author_id: user.id, file_name: '.', content: content) } @@ -322,10 +328,12 @@ feature_category: :source_code_management do end def raw_repository(snippet) - Gitlab::Git::Repository.new('default', - "#{disk_path(snippet)}.git", - Gitlab::GlRepository::SNIPPET.identifier_for_container(snippet), - "@snippets/#{snippet.id}") + Gitlab::Git::Repository.new( + 'default', + "#{disk_path(snippet)}.git", + Gitlab::GlRepository::SNIPPET.identifier_for_container(snippet), + "@snippets/#{snippet.id}" + ) end def hashed_repository(snippet) diff --git a/spec/lib/gitlab/background_migration/backfill_upvotes_count_on_issues_spec.rb b/spec/lib/gitlab/background_migration/backfill_upvotes_count_on_issues_spec.rb deleted file mode 100644 index 7142aea3ab2..00000000000 --- a/spec/lib/gitlab/background_migration/backfill_upvotes_count_on_issues_spec.rb +++ /dev/null @@ -1,46 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::BackfillUpvotesCountOnIssues, schema: 20210826171758 do - let(:award_emoji) { table(:award_emoji) } - - let!(:namespace) { table(:namespaces).create!(name: 'namespace', path: 'namespace') } - let!(:project1) { table(:projects).create!(namespace_id: namespace.id) } - let!(:project2) { table(:projects).create!(namespace_id: namespace.id) } - let!(:issue1) { table(:issues).create!(project_id: project1.id) } - let!(:issue2) { table(:issues).create!(project_id: project2.id) } - let!(:issue3) { table(:issues).create!(project_id: project2.id) } - let!(:issue4) { table(:issues).create!(project_id: project2.id) } - - describe '#perform' do - before do - add_upvotes(issue1, :thumbsdown, 1) - add_upvotes(issue2, :thumbsup, 2) - add_upvotes(issue2, :thumbsdown, 1) - add_upvotes(issue3, :thumbsup, 3) - add_upvotes(issue4, :thumbsup, 4) - end - - it 'updates upvotes_count' do - subject.perform(issue1.id, issue4.id) - - expect(issue1.reload.upvotes_count).to eq(0) - expect(issue2.reload.upvotes_count).to eq(2) - expect(issue3.reload.upvotes_count).to eq(3) - expect(issue4.reload.upvotes_count).to eq(4) - end - end - - private - - def add_upvotes(issue, name, count) - count.times do - award_emoji.create!( - name: name.to_s, - awardable_type: 'Issue', - awardable_id: issue.id - ) - end - end -end diff --git a/spec/lib/gitlab/background_migration/backfill_user_namespace_spec.rb b/spec/lib/gitlab/background_migration/backfill_user_namespace_spec.rb deleted file mode 100644 index 395248b786d..00000000000 --- a/spec/lib/gitlab/background_migration/backfill_user_namespace_spec.rb +++ /dev/null @@ -1,39 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::BackfillUserNamespace, :migration, schema: 20210930211936 do - let(:migration) { described_class.new } - let(:namespaces_table) { table(:namespaces) } - - let(:table_name) { 'namespaces' } - let(:batch_column) { :id } - let(:sub_batch_size) { 100 } - let(:pause_ms) { 0 } - - subject(:perform_migration) { migration.perform(1, 10, table_name, batch_column, sub_batch_size, pause_ms) } - - before do - namespaces_table.create!(id: 1, name: 'test1', path: 'test1', type: nil) - namespaces_table.create!(id: 2, name: 'test2', path: 'test2', type: 'User') - namespaces_table.create!(id: 3, name: 'test3', path: 'test3', type: 'Group') - namespaces_table.create!(id: 4, name: 'test4', path: 'test4', type: nil) - namespaces_table.create!(id: 11, name: 'test11', path: 'test11', type: nil) - end - - it 'backfills `type` for the selected records', :aggregate_failures do - queries = ActiveRecord::QueryRecorder.new do - perform_migration - end - - expect(queries.count).to eq(3) - expect(namespaces_table.where(type: 'User').count).to eq 3 - expect(namespaces_table.where(type: 'User').pluck(:id)).to match_array([1, 2, 4]) - end - - it 'tracks timings of queries' do - expect(migration.batch_metrics.timings).to be_empty - - expect { perform_migration }.to change { migration.batch_metrics.timings } - end -end diff --git a/spec/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent_spec.rb b/spec/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent_spec.rb index f642ec8c20d..3f1a57434a7 100644 --- a/spec/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent_spec.rb @@ -4,10 +4,12 @@ require 'spec_helper' RSpec.describe Gitlab::BackgroundMigration::BackfillVulnerabilityReadsClusterAgent, :migration, schema: 20220525221133 do # rubocop:disable Layout/LineLength let(:migration) do - described_class.new(start_id: 1, end_id: 10, - batch_table: table_name, batch_column: batch_column, - sub_batch_size: sub_batch_size, pause_ms: pause_ms, - connection: ApplicationRecord.connection) + described_class.new( + start_id: 1, end_id: 10, + batch_table: table_name, batch_column: batch_column, + sub_batch_size: sub_batch_size, pause_ms: pause_ms, + connection: ApplicationRecord.connection + ) end let(:users_table) { table(:users) } diff --git a/spec/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues_spec.rb b/spec/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues_spec.rb index 5f93424faf6..c7e4095a488 100644 --- a/spec/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues_spec.rb @@ -2,7 +2,10 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::BackfillWorkItemTypeIdForIssues, :migration, schema: 20220825142324 do +RSpec.describe Gitlab::BackgroundMigration::BackfillWorkItemTypeIdForIssues, + :migration, + schema: 20220825142324, + feature_category: :team_planning do let(:batch_column) { 'id' } let(:sub_batch_size) { 2 } let(:pause_ms) { 0 } @@ -13,6 +16,7 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillWorkItemTypeIdForIssues, :mi let(:project) { table(:projects).create!(namespace_id: namespace.id, project_namespace_id: namespace.id) } let(:issues_table) { table(:issues) } let(:issue_type) { table(:work_item_types).find_by!(namespace_id: nil, base_type: issue_type_enum[:issue]) } + let(:task_type) { table(:work_item_types).find_by!(namespace_id: nil, base_type: issue_type_enum[:task]) } let(:issue1) { issues_table.create!(project_id: project.id, issue_type: issue_type_enum[:issue]) } let(:issue2) { issues_table.create!(project_id: project.id, issue_type: issue_type_enum[:issue]) } @@ -25,7 +29,7 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillWorkItemTypeIdForIssues, :mi let(:start_id) { issue1.id } let(:end_id) { requirement1.id } - let(:all_issues) { [issue1, issue2, issue3, incident1, test_case1, requirement1] } + let!(:all_issues) { [issue1, issue2, issue3, incident1, test_case1, requirement1] } let(:migration) do described_class.new( @@ -52,6 +56,27 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillWorkItemTypeIdForIssues, :mi expect(all_issues - [issue1, issue2, issue3]).to all(have_attributes(work_item_type_id: nil)) end + context 'when a record already had a work_item_type_id assigned' do + let!(:issue4) do + issues_table.create!( + project_id: project.id, + issue_type: issue_type_enum[:issue], + work_item_type_id: task_type.id + ) + end + + let(:end_id) { issue4.id } + + it 'ovewrites the work_item_type_id' do + # creating with the wrong issue_type/work_item_type_id on purpose so we can test + # that the migration is capable of fixing such inconsistencies + expect do + migrate + issue4.reload + end.to change { issue4.work_item_type_id }.from(task_type.id).to(issue_type.id) + end + end + it 'tracks timings of queries' do expect(migration.batch_metrics.timings).to be_empty diff --git a/spec/lib/gitlab/background_migration/batched_migration_job_spec.rb b/spec/lib/gitlab/background_migration/batched_migration_job_spec.rb index faaaccfdfaf..781bf93dd85 100644 --- a/spec/lib/gitlab/background_migration/batched_migration_job_spec.rb +++ b/spec/lib/gitlab/background_migration/batched_migration_job_spec.rb @@ -301,6 +301,28 @@ RSpec.describe Gitlab::BackgroundMigration::BatchedMigrationJob do perform_job end + context 'when using a sub batch exception for timeouts' do + let(:job_class) do + Class.new(described_class) do + operation_name :update + + def perform(*_) + each_sub_batch { raise ActiveRecord::StatementTimeout } # rubocop:disable Lint/UnreachableLoop + end + end + end + + let(:job_instance) do + job_class.new(start_id: 1, end_id: 10, batch_table: '_test_table', batch_column: 'id', + sub_batch_size: 2, pause_ms: 1000, connection: connection, + sub_batch_exception: StandardError) + end + + it 'raises the expected error type' do + expect { job_instance.perform }.to raise_error(StandardError) + end + end + context 'when batching_arguments are given' do it 'forwards them for batching' do expect(job_instance).to receive(:base_relation).and_return(test_table) diff --git a/spec/lib/gitlab/background_migration/cleanup_orphaned_lfs_objects_projects_spec.rb b/spec/lib/gitlab/background_migration/cleanup_orphaned_lfs_objects_projects_spec.rb deleted file mode 100644 index 5ffe665f0ad..00000000000 --- a/spec/lib/gitlab/background_migration/cleanup_orphaned_lfs_objects_projects_spec.rb +++ /dev/null @@ -1,85 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::CleanupOrphanedLfsObjectsProjects, schema: 20210826171758 do - let(:lfs_objects_projects) { table(:lfs_objects_projects) } - let(:lfs_objects) { table(:lfs_objects) } - let(:projects) { table(:projects) } - let(:namespaces) { table(:namespaces) } - - let(:namespace) { namespaces.create!(name: 'namespace', path: 'namespace') } - let(:project) { projects.create!(namespace_id: namespace.id) } - let(:another_project) { projects.create!(namespace_id: namespace.id) } - let(:lfs_object) { lfs_objects.create!(oid: 'abcdef', size: 1) } - let(:another_lfs_object) { lfs_objects.create!(oid: '1abcde', size: 2) } - - let!(:without_object1) { create_object(project_id: project.id) } - let!(:without_object2) { create_object(project_id: another_project.id) } - let!(:without_object3) { create_object(project_id: another_project.id) } - let!(:with_project_and_object1) { create_object(project_id: project.id, lfs_object_id: lfs_object.id) } - let!(:with_project_and_object2) { create_object(project_id: project.id, lfs_object_id: another_lfs_object.id) } - let!(:with_project_and_object3) { create_object(project_id: another_project.id, lfs_object_id: another_lfs_object.id) } - let!(:without_project1) { create_object(lfs_object_id: lfs_object.id) } - let!(:without_project2) { create_object(lfs_object_id: another_lfs_object.id) } - let!(:without_project_and_object) { create_object } - - def create_object(project_id: non_existing_record_id, lfs_object_id: non_existing_record_id) - lfs_objects_project = nil - - ActiveRecord::Base.connection.disable_referential_integrity do - lfs_objects_project = lfs_objects_projects.create!(project_id: project_id, lfs_object_id: lfs_object_id) - end - - lfs_objects_project - end - - subject { described_class.new } - - describe '#perform' do - it 'lfs_objects_projects without an existing lfs object or project are removed' do - subject.perform(without_object1.id, without_object3.id) - - expect(lfs_objects_projects.all).to match_array( - [ - with_project_and_object1, with_project_and_object2, with_project_and_object3, - without_project1, without_project2, without_project_and_object - ]) - - subject.perform(with_project_and_object1.id, with_project_and_object3.id) - - expect(lfs_objects_projects.all).to match_array( - [ - with_project_and_object1, with_project_and_object2, with_project_and_object3, - without_project1, without_project2, without_project_and_object - ]) - - subject.perform(without_project1.id, without_project_and_object.id) - - expect(lfs_objects_projects.all).to match_array( - [ - with_project_and_object1, with_project_and_object2, with_project_and_object3 - ]) - - expect(lfs_objects.ids).to contain_exactly(lfs_object.id, another_lfs_object.id) - expect(projects.ids).to contain_exactly(project.id, another_project.id) - end - - it 'cache for affected projects is being reset' do - expect(ProjectCacheWorker).to receive(:bulk_perform_in) do |delay, args| - expect(delay).to eq(1.minute) - expect(args).to match_array([[project.id, [], [:lfs_objects_size]], [another_project.id, [], [:lfs_objects_size]]]) - end - - subject.perform(without_object1.id, with_project_and_object1.id) - - expect(ProjectCacheWorker).not_to receive(:bulk_perform_in) - - subject.perform(with_project_and_object1.id, with_project_and_object3.id) - - expect(ProjectCacheWorker).not_to receive(:bulk_perform_in) - - subject.perform(without_project1.id, without_project_and_object.id) - end - end -end diff --git a/spec/lib/gitlab/background_migration/cleanup_personal_access_tokens_with_nil_expires_at_spec.rb b/spec/lib/gitlab/background_migration/cleanup_personal_access_tokens_with_nil_expires_at_spec.rb new file mode 100644 index 00000000000..ade16c0a780 --- /dev/null +++ b/spec/lib/gitlab/background_migration/cleanup_personal_access_tokens_with_nil_expires_at_spec.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::CleanupPersonalAccessTokensWithNilExpiresAt, schema: 20230510062503, feature_category: :system_access do # rubocop:disable Layout/LineLength + let(:personal_access_tokens_table) { table(:personal_access_tokens) } + let(:users_table) { table(:users) } + let(:expires_at_default) { described_class::EXPIRES_AT_DEFAULT } + + subject(:perform_migration) do + described_class.new( + start_id: 1, + end_id: 30, + batch_table: :personal_access_tokens, + batch_column: :id, + sub_batch_size: 3, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ).perform + end + + before do + user = users_table.create!(name: 'PAT_USER', email: 'pat_user@gmail.com', username: "pat_user1", projects_limit: 0) + personal_access_tokens_table.create!(user_id: user.id, name: "PAT#1", expires_at: expires_at_default + 1.day) + personal_access_tokens_table.create!(user_id: user.id, name: "PAT#2", expires_at: nil) + personal_access_tokens_table.create!(user_id: user.id, name: "PAT#3", expires_at: Time.zone.now + 2.days) + end + + it 'adds expiry to personal access tokens', :aggregate_failures do + freeze_time do + expect(ActiveRecord::QueryRecorder.new { perform_migration }.count).to eq(3) + + expect(personal_access_tokens_table.find_by_name("PAT#1").expires_at).to eq(expires_at_default.to_date + 1.day) + expect(personal_access_tokens_table.find_by_name("PAT#2").expires_at).to eq(expires_at_default.to_date) + expect(personal_access_tokens_table.find_by_name("PAT#3").expires_at).to eq(Time.zone.now.to_date + 2.days) + end + end +end diff --git a/spec/lib/gitlab/background_migration/delete_orphaned_deployments_spec.rb b/spec/lib/gitlab/background_migration/delete_orphaned_deployments_spec.rb deleted file mode 100644 index 8f058c875a2..00000000000 --- a/spec/lib/gitlab/background_migration/delete_orphaned_deployments_spec.rb +++ /dev/null @@ -1,54 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::DeleteOrphanedDeployments, :migration, schema: 20210826171758 do - let!(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') } - let!(:project) { table(:projects).create!(namespace_id: namespace.id) } - let!(:environment) { table(:environments).create!(name: 'production', slug: 'production', project_id: project.id) } - let(:background_migration_jobs) { table(:background_migration_jobs) } - - before do - create_deployment!(environment.id, project.id) - end - - it 'deletes only orphaned deployments' do - expect(valid_deployments.pluck(:id)).not_to be_empty - - subject.perform(table(:deployments).minimum(:id), table(:deployments).maximum(:id)) - - expect(valid_deployments.pluck(:id)).not_to be_empty - end - - it 'marks jobs as done' do - first_job = background_migration_jobs.create!( - class_name: 'DeleteOrphanedDeployments', - arguments: [table(:deployments).minimum(:id), table(:deployments).minimum(:id)] - ) - - subject.perform(table(:deployments).minimum(:id), table(:deployments).minimum(:id)) - - expect(first_job.reload.status).to eq(Gitlab::Database::BackgroundMigrationJob.statuses[:succeeded]) - end - - private - - def valid_deployments - table(:deployments).where('EXISTS (SELECT 1 FROM environments WHERE deployments.environment_id = environments.id)') - end - - def orphaned_deployments - table(:deployments).where('NOT EXISTS (SELECT 1 FROM environments WHERE deployments.environment_id = environments.id)') - end - - def create_deployment!(environment_id, project_id) - table(:deployments).create!( - environment_id: environment_id, - project_id: project_id, - ref: 'master', - tag: false, - sha: 'x', - status: 1, - iid: table(:deployments).count + 1) - end -end diff --git a/spec/lib/gitlab/background_migration/delete_orphaned_packages_dependencies_spec.rb b/spec/lib/gitlab/background_migration/delete_orphaned_packages_dependencies_spec.rb new file mode 100644 index 00000000000..0d82717c7de --- /dev/null +++ b/spec/lib/gitlab/background_migration/delete_orphaned_packages_dependencies_spec.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::DeleteOrphanedPackagesDependencies, schema: 20230303105806, + feature_category: :package_registry do + let!(:migration_attrs) do + { + start_id: 1, + end_id: 1000, + batch_table: :packages_dependencies, + batch_column: :id, + sub_batch_size: 500, + pause_ms: 0, + connection: ApplicationRecord.connection + } + end + + let!(:migration) { described_class.new(**migration_attrs) } + + let(:packages_dependencies) { table(:packages_dependencies) } + + let!(:namespace) { table(:namespaces).create!(name: 'project', path: 'project', type: 'Project') } + let!(:project) do + table(:projects).create!(name: 'project', path: 'project', project_namespace_id: namespace.id, + namespace_id: namespace.id) + end + + let!(:package) do + table(:packages_packages).create!(name: 'test', version: '1.2.3', package_type: 2, project_id: project.id) + end + + let!(:orphan_dependency_1) { packages_dependencies.create!(name: 'dependency 1', version_pattern: '~0.0.1') } + let!(:orphan_dependency_2) { packages_dependencies.create!(name: 'dependency 2', version_pattern: '~0.0.2') } + let!(:orphan_dependency_3) { packages_dependencies.create!(name: 'dependency 3', version_pattern: '~0.0.3') } + let!(:linked_dependency) do + packages_dependencies.create!(name: 'dependency 4', version_pattern: '~0.0.4').tap do |dependency| + table(:packages_dependency_links).create!(package_id: package.id, dependency_id: dependency.id, + dependency_type: 'dependencies') + end + end + + subject(:perform_migration) { migration.perform } + + it 'executes 3 queries' do + queries = ActiveRecord::QueryRecorder.new do + perform_migration + end + + expect(queries.count).to eq(3) + end + + it 'deletes only orphaned dependencies' do + expect { perform_migration }.to change { packages_dependencies.count }.by(-3) + expect(packages_dependencies.all).to eq([linked_dependency]) + end +end diff --git a/spec/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images_spec.rb b/spec/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images_spec.rb deleted file mode 100644 index e7b0471810d..00000000000 --- a/spec/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images_spec.rb +++ /dev/null @@ -1,142 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::DisableExpirationPoliciesLinkedToNoContainerImages, :migration, schema: 20220326161803 do # rubocop:disable Layout/LineLength - let!(:projects) { table(:projects) } - let!(:container_expiration_policies) { table(:container_expiration_policies) } - let!(:container_repositories) { table(:container_repositories) } - let!(:namespaces) { table(:namespaces) } - - let!(:namespace) { namespaces.create!(name: 'test', path: 'test') } - - let!(:policy1) { create_expiration_policy(project_id: 1, enabled: true) } - let!(:policy2) { create_expiration_policy(project_id: 2, enabled: false) } - let!(:policy3) { create_expiration_policy(project_id: 3, enabled: false) } - let!(:policy4) { create_expiration_policy(project_id: 4, enabled: true, with_images: true) } - let!(:policy5) { create_expiration_policy(project_id: 5, enabled: false, with_images: true) } - let!(:policy6) { create_expiration_policy(project_id: 6, enabled: false) } - let!(:policy7) { create_expiration_policy(project_id: 7, enabled: true) } - let!(:policy8) { create_expiration_policy(project_id: 8, enabled: true, with_images: true) } - let!(:policy9) { create_expiration_policy(project_id: 9, enabled: true) } - - describe '#perform' do - subject { described_class.new.perform(from_id, to_id) } - - shared_examples 'disabling policies with no images' do - it 'disables the proper policies' do - subject - - rows = container_expiration_policies.order(:project_id).to_h do |row| - [row.project_id, row.enabled] - end - expect(rows).to eq(expected_rows) - end - end - - context 'the whole range' do - let(:from_id) { 1 } - let(:to_id) { 9 } - - it_behaves_like 'disabling policies with no images' do - let(:expected_rows) do - { - 1 => false, - 2 => false, - 3 => false, - 4 => true, - 5 => false, - 6 => false, - 7 => false, - 8 => true, - 9 => false - } - end - end - end - - context 'a range with no policies to disable' do - let(:from_id) { 2 } - let(:to_id) { 6 } - - it_behaves_like 'disabling policies with no images' do - let(:expected_rows) do - { - 1 => true, - 2 => false, - 3 => false, - 4 => true, - 5 => false, - 6 => false, - 7 => true, - 8 => true, - 9 => true - } - end - end - end - - context 'a range with only images' do - let(:from_id) { 4 } - let(:to_id) { 5 } - - it_behaves_like 'disabling policies with no images' do - let(:expected_rows) do - { - 1 => true, - 2 => false, - 3 => false, - 4 => true, - 5 => false, - 6 => false, - 7 => true, - 8 => true, - 9 => true - } - end - end - end - - context 'a range with a single element' do - let(:from_id) { 9 } - let(:to_id) { 9 } - - it_behaves_like 'disabling policies with no images' do - let(:expected_rows) do - { - 1 => true, - 2 => false, - 3 => false, - 4 => true, - 5 => false, - 6 => false, - 7 => true, - 8 => true, - 9 => false - } - end - end - end - end - - def create_expiration_policy(project_id:, enabled:, with_images: false) - projects.create!(id: project_id, namespace_id: namespace.id, name: "gitlab-#{project_id}") - - if with_images - container_repositories.create!(project_id: project_id, name: "image-#{project_id}") - end - - container_expiration_policies.create!( - enabled: enabled, - project_id: project_id - ) - end - - def enabled_policies - container_expiration_policies.where(enabled: true) - end - - def disabled_policies - container_expiration_policies.where(enabled: false) - end -end diff --git a/spec/lib/gitlab/background_migration/drop_invalid_security_findings_spec.rb b/spec/lib/gitlab/background_migration/drop_invalid_security_findings_spec.rb deleted file mode 100644 index 5fdd8683d06..00000000000 --- a/spec/lib/gitlab/background_migration/drop_invalid_security_findings_spec.rb +++ /dev/null @@ -1,57 +0,0 @@ -# frozen_string_literal: true -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::DropInvalidSecurityFindings, :suppress_gitlab_schemas_validate_connection, - schema: 20211108211434 do - let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user', type: Namespaces::UserNamespace.sti_name) } - let(:project) { table(:projects).create!(namespace_id: namespace.id) } - - let(:pipelines) { table(:ci_pipelines) } - let!(:pipeline) { pipelines.create!(project_id: project.id) } - - let(:ci_builds) { table(:ci_builds) } - let!(:ci_build) { ci_builds.create! } - - let(:security_scans) { table(:security_scans) } - let!(:security_scan) do - security_scans.create!( - scan_type: 1, - status: 1, - build_id: ci_build.id, - project_id: project.id, - pipeline_id: pipeline.id - ) - end - - let(:vulnerability_scanners) { table(:vulnerability_scanners) } - let!(:vulnerability_scanner) { vulnerability_scanners.create!(project_id: project.id, external_id: 'test 1', name: 'test scanner 1') } - - let(:security_findings) { table(:security_findings) } - let!(:security_finding_without_uuid) do - security_findings.create!( - severity: 1, - confidence: 1, - scan_id: security_scan.id, - scanner_id: vulnerability_scanner.id, - uuid: nil - ) - end - - let!(:security_finding_with_uuid) do - security_findings.create!( - severity: 1, - confidence: 1, - scan_id: security_scan.id, - scanner_id: vulnerability_scanner.id, - uuid: 'bd95c085-71aa-51d7-9bb6-08ae669c262e' - ) - end - - let(:sub_batch_size) { 10_000 } - - subject { described_class.new.perform(security_finding_without_uuid.id, security_finding_with_uuid.id, sub_batch_size) } - - it 'drops Security::Finding objects with no UUID' do - expect { subject }.to change(security_findings, :count).from(2).to(1) - end -end diff --git a/spec/lib/gitlab/background_migration/drop_invalid_vulnerabilities_spec.rb b/spec/lib/gitlab/background_migration/drop_invalid_vulnerabilities_spec.rb deleted file mode 100644 index 8f3ef44e00c..00000000000 --- a/spec/lib/gitlab/background_migration/drop_invalid_vulnerabilities_spec.rb +++ /dev/null @@ -1,126 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::DropInvalidVulnerabilities, schema: 20210826171758 do - let!(:background_migration_jobs) { table(:background_migration_jobs) } - let!(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') } - let!(:users) { table(:users) } - let!(:user) { create_user! } - let!(:project) { table(:projects).create!(id: 123, namespace_id: namespace.id) } - - let!(:scanners) { table(:vulnerability_scanners) } - let!(:scanner) { scanners.create!(project_id: project.id, external_id: 'test 1', name: 'test scanner 1') } - let!(:different_scanner) { scanners.create!(project_id: project.id, external_id: 'test 2', name: 'test scanner 2') } - - let!(:vulnerabilities) { table(:vulnerabilities) } - let!(:vulnerability_with_finding) do - create_vulnerability!( - project_id: project.id, - author_id: user.id - ) - end - - let!(:vulnerability_without_finding) do - create_vulnerability!( - project_id: project.id, - author_id: user.id - ) - end - - let!(:vulnerability_identifiers) { table(:vulnerability_identifiers) } - let!(:primary_identifier) do - vulnerability_identifiers.create!( - project_id: project.id, - external_type: 'uuid-v5', - external_id: 'uuid-v5', - fingerprint: '7e394d1b1eb461a7406d7b1e08f057a1cf11287a', - name: 'Identifier for UUIDv5') - end - - let!(:vulnerabilities_findings) { table(:vulnerability_occurrences) } - let!(:finding) do - create_finding!( - vulnerability_id: vulnerability_with_finding.id, - project_id: project.id, - scanner_id: scanner.id, - primary_identifier_id: primary_identifier.id - ) - end - - let(:succeeded_status) { 1 } - let(:pending_status) { 0 } - - it 'drops Vulnerabilities without any Findings' do - expect(vulnerabilities.pluck(:id)).to eq([vulnerability_with_finding.id, vulnerability_without_finding.id]) - - expect { subject.perform(vulnerability_with_finding.id, vulnerability_without_finding.id) }.to change(vulnerabilities, :count).by(-1) - - expect(vulnerabilities.pluck(:id)).to eq([vulnerability_with_finding.id]) - end - - it 'marks jobs as done' do - background_migration_jobs.create!( - class_name: 'DropInvalidVulnerabilities', - arguments: [vulnerability_with_finding.id, vulnerability_with_finding.id] - ) - - background_migration_jobs.create!( - class_name: 'DropInvalidVulnerabilities', - arguments: [vulnerability_without_finding.id, vulnerability_without_finding.id] - ) - - subject.perform(vulnerability_with_finding.id, vulnerability_with_finding.id) - - expect(background_migration_jobs.first.status).to eq(succeeded_status) - expect(background_migration_jobs.second.status).to eq(pending_status) - end - - private - - def create_vulnerability!(project_id:, author_id:, title: 'test', severity: 7, confidence: 7, report_type: 0) - vulnerabilities.create!( - project_id: project_id, - author_id: author_id, - title: title, - severity: severity, - confidence: confidence, - report_type: report_type - ) - end - - # rubocop:disable Metrics/ParameterLists - def create_finding!( - vulnerability_id:, project_id:, scanner_id:, primary_identifier_id:, - name: "test", severity: 7, confidence: 7, report_type: 0, - project_fingerprint: '123qweasdzxc', location_fingerprint: 'test', - metadata_version: 'test', raw_metadata: 'test', uuid: SecureRandom.uuid) - vulnerabilities_findings.create!( - vulnerability_id: vulnerability_id, - project_id: project_id, - name: name, - severity: severity, - confidence: confidence, - report_type: report_type, - project_fingerprint: project_fingerprint, - scanner_id: scanner_id, - primary_identifier_id: primary_identifier_id, - location_fingerprint: location_fingerprint, - metadata_version: metadata_version, - raw_metadata: raw_metadata, - uuid: uuid - ) - end - # rubocop:enable Metrics/ParameterLists - - def create_user!(name: "Example User", email: "user@example.com", user_type: nil) - users.create!( - name: name, - email: email, - username: name, - projects_limit: 0, - user_type: user_type, - confirmed_at: Time.current - ) - end -end diff --git a/spec/lib/gitlab/background_migration/encrypt_ci_trigger_token_spec.rb b/spec/lib/gitlab/background_migration/encrypt_ci_trigger_token_spec.rb index b52f30a5e21..dd3e7877f8a 100644 --- a/spec/lib/gitlab/background_migration/encrypt_ci_trigger_token_spec.rb +++ b/spec/lib/gitlab/background_migration/encrypt_ci_trigger_token_spec.rb @@ -10,8 +10,7 @@ RSpec.describe Gitlab::BackgroundMigration::EncryptCiTriggerToken, feature_categ mode: :per_attribute_iv, key: ::Settings.attr_encrypted_db_key_base_32, algorithm: 'aes-256-gcm', - encode: false, - encode_iv: false + encode: false end end @@ -52,6 +51,7 @@ RSpec.describe Gitlab::BackgroundMigration::EncryptCiTriggerToken, feature_categ already_encrypted_token = Ci::Trigger.find(with_encryption.id) expect(already_encrypted_token.encrypted_token).to eq(with_encryption.encrypted_token) expect(already_encrypted_token.encrypted_token_iv).to eq(with_encryption.encrypted_token_iv) + expect(already_encrypted_token.token).to eq(already_encrypted_token.encrypted_token_tmp) expect(with_encryption.token).to eq(with_encryption.encrypted_token_tmp) end end diff --git a/spec/lib/gitlab/background_migration/extract_project_topics_into_separate_table_spec.rb b/spec/lib/gitlab/background_migration/extract_project_topics_into_separate_table_spec.rb deleted file mode 100644 index 586e75ffb37..00000000000 --- a/spec/lib/gitlab/background_migration/extract_project_topics_into_separate_table_spec.rb +++ /dev/null @@ -1,46 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::ExtractProjectTopicsIntoSeparateTable, - :suppress_gitlab_schemas_validate_connection, schema: 20210826171758 do - it 'correctly extracts project topics into separate table' do - namespaces = table(:namespaces) - projects = table(:projects) - taggings = table(:taggings) - tags = table(:tags) - project_topics = table(:project_topics) - topics = table(:topics) - - namespace = namespaces.create!(name: 'foo', path: 'foo') - project = projects.create!(namespace_id: namespace.id) - tag_1 = tags.create!(name: 'Topic1') - tag_2 = tags.create!(name: 'Topic2') - tag_3 = tags.create!(name: 'Topic3') - topic_3 = topics.create!(name: 'Topic3') - tagging_1 = taggings.create!(taggable_type: 'Project', taggable_id: project.id, context: 'topics', tag_id: tag_1.id) - tagging_2 = taggings.create!(taggable_type: 'Project', taggable_id: project.id, context: 'topics', tag_id: tag_2.id) - other_tagging = taggings.create!(taggable_type: 'Other', taggable_id: project.id, context: 'topics', tag_id: tag_1.id) - tagging_3 = taggings.create!(taggable_type: 'Project', taggable_id: project.id, context: 'topics', tag_id: tag_3.id) - tagging_4 = taggings.create!(taggable_type: 'Project', taggable_id: -1, context: 'topics', tag_id: tag_1.id) - tagging_5 = taggings.create!(taggable_type: 'Project', taggable_id: project.id, context: 'topics', tag_id: -1) - - subject.perform(tagging_1.id, tagging_5.id) - - # Tagging records - expect { tagging_1.reload }.to raise_error(ActiveRecord::RecordNotFound) - expect { tagging_2.reload }.to raise_error(ActiveRecord::RecordNotFound) - expect { other_tagging.reload }.not_to raise_error - expect { tagging_3.reload }.to raise_error(ActiveRecord::RecordNotFound) - expect { tagging_4.reload }.to raise_error(ActiveRecord::RecordNotFound) - expect { tagging_5.reload }.to raise_error(ActiveRecord::RecordNotFound) - - # Topic records - topic_1 = topics.find_by(name: 'Topic1') - topic_2 = topics.find_by(name: 'Topic2') - expect(topics.all).to contain_exactly(topic_1, topic_2, topic_3) - - # ProjectTopic records - expect(project_topics.all.map(&:topic_id)).to contain_exactly(topic_1.id, topic_2.id, topic_3.id) - end -end diff --git a/spec/lib/gitlab/background_migration/fix_first_mentioned_in_commit_at_spec.rb b/spec/lib/gitlab/background_migration/fix_first_mentioned_in_commit_at_spec.rb deleted file mode 100644 index 7f15aceca42..00000000000 --- a/spec/lib/gitlab/background_migration/fix_first_mentioned_in_commit_at_spec.rb +++ /dev/null @@ -1,166 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require Rails.root.join('db', 'post_migrate', '20211004110500_add_temporary_index_to_issue_metrics.rb') - -RSpec.describe Gitlab::BackgroundMigration::FixFirstMentionedInCommitAt, :migration, schema: 20211004110500 do - let(:namespaces) { table(:namespaces) } - let(:projects) { table(:projects) } - let(:users) { table(:users) } - let(:merge_requests) { table(:merge_requests) } - let(:issues) { table(:issues) } - let(:issue_metrics) { table(:issue_metrics) } - let(:merge_requests_closing_issues) { table(:merge_requests_closing_issues) } - let(:diffs) { table(:merge_request_diffs) } - let(:ten_days_ago) { 10.days.ago } - let(:commits) do - table(:merge_request_diff_commits).tap do |t| - t.extend(SuppressCompositePrimaryKeyWarning) - end - end - - let(:namespace) { namespaces.create!(name: 'ns', path: 'ns') } - let(:project) { projects.create!(namespace_id: namespace.id) } - - let!(:issue1) do - issues.create!( - title: 'issue', - description: 'description', - project_id: project.id - ) - end - - let!(:issue2) do - issues.create!( - title: 'issue', - description: 'description', - project_id: project.id - ) - end - - let!(:merge_request1) do - merge_requests.create!( - source_branch: 'a', - target_branch: 'master', - target_project_id: project.id - ) - end - - let!(:merge_request2) do - merge_requests.create!( - source_branch: 'b', - target_branch: 'master', - target_project_id: project.id - ) - end - - let!(:merge_request_closing_issue1) do - merge_requests_closing_issues.create!(issue_id: issue1.id, merge_request_id: merge_request1.id) - end - - let!(:merge_request_closing_issue2) do - merge_requests_closing_issues.create!(issue_id: issue2.id, merge_request_id: merge_request2.id) - end - - let!(:diff1) { diffs.create!(merge_request_id: merge_request1.id) } - let!(:diff2) { diffs.create!(merge_request_id: merge_request1.id) } - - let!(:other_diff) { diffs.create!(merge_request_id: merge_request2.id) } - - let!(:commit1) do - commits.create!( - merge_request_diff_id: diff2.id, - relative_order: 0, - sha: Gitlab::Database::ShaAttribute.serialize('aaa'), - authored_date: 5.days.ago - ) - end - - let!(:commit2) do - commits.create!( - merge_request_diff_id: diff2.id, - relative_order: 1, - sha: Gitlab::Database::ShaAttribute.serialize('aaa'), - authored_date: 10.days.ago - ) - end - - let!(:commit3) do - commits.create!( - merge_request_diff_id: other_diff.id, - relative_order: 1, - sha: Gitlab::Database::ShaAttribute.serialize('aaa'), - authored_date: 5.days.ago - ) - end - - def run_migration - described_class - .new - .perform(issue_metrics.minimum(:issue_id), issue_metrics.maximum(:issue_id)) - end - - shared_examples 'fixes first_mentioned_in_commit_at' do - it "marks successful slices as completed" do - min_issue_id = issue_metrics.minimum(:issue_id) - max_issue_id = issue_metrics.maximum(:issue_id) - - expect(subject).to receive(:mark_job_as_succeeded).with(min_issue_id, max_issue_id) - - subject.perform(min_issue_id, max_issue_id) - end - - context 'when the persisted first_mentioned_in_commit_at is later than the first commit authored_date' do - it 'updates the issue_metrics record' do - record1 = issue_metrics.create!(issue_id: issue1.id, first_mentioned_in_commit_at: Time.current) - record2 = issue_metrics.create!(issue_id: issue2.id, first_mentioned_in_commit_at: Time.current) - - run_migration - record1.reload - record2.reload - - expect(record1.first_mentioned_in_commit_at).to be_within(2.seconds).of(commit2.authored_date) - expect(record2.first_mentioned_in_commit_at).to be_within(2.seconds).of(commit3.authored_date) - end - end - - context 'when the persisted first_mentioned_in_commit_at is earlier than the first commit authored_date' do - it 'does not update the issue_metrics record' do - record = issue_metrics.create!(issue_id: issue1.id, first_mentioned_in_commit_at: 20.days.ago) - - expect { run_migration }.not_to change { record.reload.first_mentioned_in_commit_at } - end - end - - context 'when the first_mentioned_in_commit_at is null' do - it 'does nothing' do - record = issue_metrics.create!(issue_id: issue1.id, first_mentioned_in_commit_at: nil) - - expect { run_migration }.not_to change { record.reload.first_mentioned_in_commit_at } - end - end - end - - describe 'running the migration when first_mentioned_in_commit_at is timestamp without time zone' do - it_behaves_like 'fixes first_mentioned_in_commit_at' - end - - describe 'running the migration when first_mentioned_in_commit_at is timestamp with time zone' do - around do |example| - AddTemporaryIndexToIssueMetrics.new.down - - ActiveRecord::Base.connection.execute "ALTER TABLE issue_metrics ALTER first_mentioned_in_commit_at type timestamp with time zone" - Gitlab::BackgroundMigration::FixFirstMentionedInCommitAt::TmpIssueMetrics.reset_column_information - AddTemporaryIndexToIssueMetrics.new.up - - example.run - - AddTemporaryIndexToIssueMetrics.new.down - ActiveRecord::Base.connection.execute "ALTER TABLE issue_metrics ALTER first_mentioned_in_commit_at type timestamp without time zone" - Gitlab::BackgroundMigration::FixFirstMentionedInCommitAt::TmpIssueMetrics.reset_column_information - AddTemporaryIndexToIssueMetrics.new.up - end - - it_behaves_like 'fixes first_mentioned_in_commit_at' - end -end diff --git a/spec/lib/gitlab/background_migration/fix_merge_request_diff_commit_users_spec.rb b/spec/lib/gitlab/background_migration/fix_merge_request_diff_commit_users_spec.rb deleted file mode 100644 index 99df21562b0..00000000000 --- a/spec/lib/gitlab/background_migration/fix_merge_request_diff_commit_users_spec.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -# rubocop: disable RSpec/FactoriesInMigrationSpecs -RSpec.describe Gitlab::BackgroundMigration::FixMergeRequestDiffCommitUsers do - let(:migration) { described_class.new } - - describe '#perform' do - context 'when the project exists' do - it 'does nothing' do - project = create(:project) - - expect { migration.perform(project.id) }.not_to raise_error - end - end - - context 'when the project does not exist' do - it 'does nothing' do - expect { migration.perform(-1) }.not_to raise_error - end - end - end -end -# rubocop: enable RSpec/FactoriesInMigrationSpecs diff --git a/spec/lib/gitlab/background_migration/fix_vulnerability_reads_has_issues_spec.rb b/spec/lib/gitlab/background_migration/fix_vulnerability_reads_has_issues_spec.rb new file mode 100644 index 00000000000..9f431c43f39 --- /dev/null +++ b/spec/lib/gitlab/background_migration/fix_vulnerability_reads_has_issues_spec.rb @@ -0,0 +1,100 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::FixVulnerabilityReadsHasIssues, schema: 20230302185739, feature_category: :vulnerability_management do # rubocop:disable Layout/LineLength + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:users) { table(:users) } + let(:scanners) { table(:vulnerability_scanners) } + let(:vulnerabilities) { table(:vulnerabilities) } + let(:vulnerability_reads) { table(:vulnerability_reads) } + let(:work_item_types) { table(:work_item_types) } + let(:issues) { table(:issues) } + let(:vulnerability_issue_links) { table(:vulnerability_issue_links) } + + let(:namespace) { namespaces.create!(name: 'user', path: 'user') } + let(:project) { projects.create!(namespace_id: namespace.id, project_namespace_id: namespace.id) } + let(:user) { users.create!(username: 'john_doe', email: 'johndoe@gitlab.com', projects_limit: 10) } + let(:scanner) { scanners.create!(project_id: project.id, external_id: 'external_id', name: 'Test Scanner') } + let(:work_item_type) { work_item_types.create!(name: 'test') } + + let(:vulnerability_records) do + Array.new(4).map do |_, n| + vulnerabilities.create!( + project_id: project.id, + author_id: user.id, + title: "vulnerability #{n}", + severity: 1, + confidence: 1, + report_type: 1 + ) + end + end + + let(:vulnerabilities_with_issues) { [vulnerability_records.first, vulnerability_records.third] } + let(:vulnerabilities_without_issues) { vulnerability_records - vulnerabilities_with_issues } + + let(:vulnerability_read_records) do + vulnerability_records.map do |vulnerability| + vulnerability_reads.create!( + project_id: project.id, + vulnerability_id: vulnerability.id, + scanner_id: scanner.id, + has_issues: false, + severity: 1, + report_type: 1, + state: 1, + uuid: SecureRandom.uuid + ) + end + end + + let!(:issue_links) do + vulnerabilities_with_issues.map do |vulnerability| + issue = issues.create!( + title: vulnerability.title, + author_id: user.id, + project_id: project.id, + confidential: true, + work_item_type_id: work_item_type.id, + namespace_id: namespace.id + ) + + vulnerability_issue_links.create!( + vulnerability_id: vulnerability.id, + issue_id: issue.id + ) + end + end + + def vulnerability_read_for(vulnerability) + vulnerability_read_records.find { |read| read.vulnerability_id == vulnerability.id } + end + + subject(:perform_migration) do + described_class.new( + start_id: issue_links.first.vulnerability_id, + end_id: issue_links.last.vulnerability_id, + batch_table: :vulnerability_issue_links, + batch_column: :vulnerability_id, + sub_batch_size: issue_links.size, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ).perform + end + + it 'only changes records with issue links' do + expect(vulnerability_read_records).to all(have_attributes(has_issues: false)) + + perform_migration + + vulnerabilities_with_issues.each do |vulnerability| + expect(vulnerability_read_for(vulnerability).reload.has_issues).to eq(true) + end + + vulnerabilities_without_issues.each do |vulnerability| + expect(vulnerability_read_for(vulnerability).reload.has_issues).to eq(false) + end + end +end diff --git a/spec/lib/gitlab/background_migration/issues_internal_id_scope_updater_spec.rb b/spec/lib/gitlab/background_migration/issues_internal_id_scope_updater_spec.rb new file mode 100644 index 00000000000..1adff322b41 --- /dev/null +++ b/spec/lib/gitlab/background_migration/issues_internal_id_scope_updater_spec.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +require 'spec_helper' +# this needs the schema to be before we introduce the not null constraint on routes#namespace_id +# rubocop:disable RSpec/MultipleMemoizedHelpers +RSpec.describe Gitlab::BackgroundMigration::IssuesInternalIdScopeUpdater, feature_category: :team_planning do + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:internal_ids) { table(:internal_ids) } + + let(:gr1) { namespaces.create!(name: 'batchtest1', type: 'Group', path: 'space1') } + let(:gr2) { namespaces.create!(name: 'batchtest2', type: 'Group', parent_id: gr1.id, path: 'space2') } + + let(:pr_nmsp1) { namespaces.create!(name: 'proj1', path: 'proj1', type: 'Project', parent_id: gr1.id) } + let(:pr_nmsp2) { namespaces.create!(name: 'proj2', path: 'proj2', type: 'Project', parent_id: gr1.id) } + let(:pr_nmsp3) { namespaces.create!(name: 'proj3', path: 'proj3', type: 'Project', parent_id: gr2.id) } + let(:pr_nmsp4) { namespaces.create!(name: 'proj4', path: 'proj4', type: 'Project', parent_id: gr2.id) } + let(:pr_nmsp5) { namespaces.create!(name: 'proj5', path: 'proj5', type: 'Project', parent_id: gr2.id) } + let(:pr_nmsp6) { namespaces.create!(name: 'proj6', path: 'proj6', type: 'Project', parent_id: gr2.id) } + + # rubocop:disable Layout/LineLength + let(:p1) { projects.create!(name: 'proj1', path: 'proj1', namespace_id: gr1.id, project_namespace_id: pr_nmsp1.id) } + let(:p2) { projects.create!(name: 'proj2', path: 'proj2', namespace_id: gr1.id, project_namespace_id: pr_nmsp2.id) } + let(:p3) { projects.create!(name: 'proj3', path: 'proj3', namespace_id: gr2.id, project_namespace_id: pr_nmsp3.id) } + let(:p4) { projects.create!(name: 'proj4', path: 'proj4', namespace_id: gr2.id, project_namespace_id: pr_nmsp4.id) } + let(:p5) { projects.create!(name: 'proj5', path: 'proj5', namespace_id: gr2.id, project_namespace_id: pr_nmsp5.id) } + let(:p6) { projects.create!(name: 'proj6', path: 'proj6', namespace_id: gr2.id, project_namespace_id: pr_nmsp6.id) } + # rubocop:enable Layout/LineLength + + # a project that already is covered by a record for its namespace. This should result in no new record added and + # project related record deleted + let!(:issues_internal_ids_p1) { internal_ids.create!(project_id: p1.id, usage: 0, last_value: 100) } + let!(:issues_internal_ids_pr_nmsp1) { internal_ids.create!(namespace_id: pr_nmsp1.id, usage: 0, last_value: 111) } + + # project records that do not have a corresponding namespace record. This should result 2 new records + # scoped to corresponding project namespaces being added and the project related records being deleted. + let!(:issues_internal_ids_p2) { internal_ids.create!(project_id: p2.id, usage: 0, last_value: 200) } + let!(:issues_internal_ids_p3) { internal_ids.create!(project_id: p3.id, usage: 0, last_value: 300) } + + # a project record on a different usage, should not be affected by the migration and + # no new record should be created for this case + let!(:issues_internal_ids_p4) { internal_ids.create!(project_id: p4.id, usage: 4, last_value: 400) } + + # a project namespace scoped record without a corresponding project record, should not affect anything. + let!(:issues_internal_ids_pr_nmsp5) { internal_ids.create!(namespace_id: pr_nmsp5.id, usage: 0, last_value: 500) } + + # a record scoped to a group, should not affect anything. + let!(:issues_internal_ids_gr1) { internal_ids.create!(namespace_id: gr1.id, usage: 0, last_value: 600) } + + # a project that is covered by a record for its namespace, but has a higher last_value, due to updates during rolling + # deploy for instance, see https://gitlab.com/gitlab-com/gl-infra/production/-/issues/8548 + let!(:issues_internal_ids_p6) { internal_ids.create!(project_id: p6.id, usage: 0, last_value: 111) } + let!(:issues_internal_ids_pr_nmsp6) { internal_ids.create!(namespace_id: pr_nmsp6.id, usage: 0, last_value: 100) } + + subject(:perform_migration) do + described_class.new( + start_id: internal_ids.minimum(:id), + end_id: internal_ids.maximum(:id), + batch_table: :internal_ids, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ).perform + end + + it 'backfills internal_ids records and removes related project records', :aggregate_failures do + perform_migration + + expected_recs = [pr_nmsp1.id, pr_nmsp2.id, pr_nmsp3.id, pr_nmsp5.id, gr1.id, pr_nmsp6.id] + + # all namespace scoped records for issues(0) usage + expect(internal_ids.where.not(namespace_id: nil).where(usage: 0).count).to eq(6) + # all namespace_ids for issues(0) usage + expect(internal_ids.where.not(namespace_id: nil).where(usage: 0).pluck(:namespace_id)).to match_array(expected_recs) + # this is the record with usage: 4 + expect(internal_ids.where.not(project_id: nil).count).to eq(1) + # no project scoped records for issues usage left + expect(internal_ids.where.not(project_id: nil).where(usage: 0).count).to eq(0) + + # the case when the project_id scoped record had the higher last_value, + # see `issues_internal_ids_p6` and issues_internal_ids_pr_nmsp6 definitions above + expect(internal_ids.where(namespace_id: pr_nmsp6.id).first.last_value).to eq(111) + + # the case when the namespace_id scoped record had the higher last_value, + # see `issues_internal_ids_p1` and issues_internal_ids_pr_nmsp1 definitions above. + expect(internal_ids.where(namespace_id: pr_nmsp1.id).first.last_value).to eq(111) + end +end +# rubocop:enable RSpec/MultipleMemoizedHelpers diff --git a/spec/lib/gitlab/background_migration/migrate_evidences_for_vulnerability_findings_spec.rb b/spec/lib/gitlab/background_migration/migrate_evidences_for_vulnerability_findings_spec.rb new file mode 100644 index 00000000000..ba2f571f5aa --- /dev/null +++ b/spec/lib/gitlab/background_migration/migrate_evidences_for_vulnerability_findings_spec.rb @@ -0,0 +1,119 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::MigrateEvidencesForVulnerabilityFindings, + feature_category: :vulnerability_management do + let(:vulnerability_occurrences) { table(:vulnerability_occurrences) } + let(:vulnerability_finding_evidences) { table(:vulnerability_finding_evidences) } + let(:evidence_hash) { { url: 'http://test.com' } } + let(:namespace1) { table(:namespaces).create!(name: 'namespace 1', path: 'namespace1') } + let(:project1) { table(:projects).create!(namespace_id: namespace1.id, project_namespace_id: namespace1.id) } + let(:user) { table(:users).create!(email: 'test1@example.com', projects_limit: 5) } + + let(:scanner1) do + table(:vulnerability_scanners).create!(project_id: project1.id, external_id: 'test 1', name: 'test scanner 1') + end + + let(:stating_id) { vulnerability_occurrences.pluck(:id).min } + let(:end_id) { vulnerability_occurrences.pluck(:id).max } + + let(:migration) do + described_class.new( + start_id: stating_id, + end_id: end_id, + batch_table: :vulnerability_occurrences, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 2, + connection: ApplicationRecord.connection + ) + end + + subject(:perform_migration) { migration.perform } + + context 'without the presence of evidence key' do + before do + create_finding!(project1.id, scanner1.id, { other_keys: 'test' }) + end + + it 'does not create any evidence' do + expect { perform_migration }.not_to change { vulnerability_finding_evidences.count } + end + end + + context 'with evidence equals to nil' do + before do + create_finding!(project1.id, scanner1.id, { evidence: nil }) + end + + it 'does not create any evidence' do + expect { perform_migration }.not_to change { vulnerability_finding_evidences.count } + end + end + + context 'with existing evidence within raw_metadata' do + let!(:finding1) { create_finding!(project1.id, scanner1.id, { evidence: evidence_hash }) } + let!(:finding2) { create_finding!(project1.id, scanner1.id, { evidence: evidence_hash }) } + + it 'creates new evidence for each finding' do + expect { perform_migration }.to change { vulnerability_finding_evidences.count }.by(2) + end + + context 'when parse throws exception JSON::ParserError' do + before do + allow(Gitlab::Json).to receive(:parse).and_raise(JSON::ParserError) + end + + it 'does not create new records' do + expect { perform_migration }.not_to change { vulnerability_finding_evidences.count } + end + end + end + + context 'with unsupported Unicode escape sequence' do + let!(:finding1) { create_finding!(project1.id, scanner1.id, { evidence: { 'summary' => "\u0000" } }) } + + it 'does not create new evidence' do + expect { perform_migration }.not_to change { vulnerability_finding_evidences.count } + end + end + + context 'with existing evidence records' do + let!(:finding) { create_finding!(project1.id, scanner1.id, { evidence: evidence_hash }) } + + before do + vulnerability_finding_evidences.create!(vulnerability_occurrence_id: finding.id, data: evidence_hash) + end + + it 'does not create new evidence' do + expect { perform_migration }.not_to change { vulnerability_finding_evidences.count } + end + + context 'with non-existing evidence' do + let!(:finding3) { create_finding!(project1.id, scanner1.id, { evidence: { url: 'http://secondary.com' } }) } + + it 'creates a new evidence only to the non-existing evidence' do + expect { perform_migration }.to change { vulnerability_finding_evidences.count }.by(1) + end + end + end + + private + + def create_finding!(project_id, scanner_id, raw_metadata) + vulnerability = table(:vulnerabilities).create!(project_id: project_id, author_id: user.id, title: 'test', + severity: 4, confidence: 4, report_type: 0) + + identifier = table(:vulnerability_identifiers).create!(project_id: project_id, external_type: 'uuid-v5', + external_id: 'uuid-v5', fingerprint: OpenSSL::Digest::SHA256.hexdigest(vulnerability.id.to_s), + name: 'Identifier for UUIDv5 2 2') + + table(:vulnerability_occurrences).create!( + vulnerability_id: vulnerability.id, project_id: project_id, scanner_id: scanner_id, + primary_identifier_id: identifier.id, name: 'test', severity: 4, confidence: 4, report_type: 0, + uuid: SecureRandom.uuid, project_fingerprint: '123qweasdzxc', location: { "image" => "alpine:3.4" }, + location_fingerprint: 'test', metadata_version: 'test', + raw_metadata: raw_metadata.to_json) + end +end diff --git a/spec/lib/gitlab/background_migration/migrate_human_user_type_spec.rb b/spec/lib/gitlab/background_migration/migrate_human_user_type_spec.rb new file mode 100644 index 00000000000..7edeaed5794 --- /dev/null +++ b/spec/lib/gitlab/background_migration/migrate_human_user_type_spec.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::MigrateHumanUserType, schema: 20230327103401, feature_category: :user_management do # rubocop:disable Layout/LineLength + let!(:valid_users) do + # 13 is the max value we have at the moment. + (0..13).map do |type| + table(:users).create!(username: "user#{type}", email: "user#{type}@test.com", user_type: type, projects_limit: 0) + end + end + + let!(:user_to_update) do + table(:users).create!(username: "user_nil", email: "user_nil@test.com", user_type: nil, projects_limit: 0) + end + + let(:starting_id) { table(:users).pluck(:id).min } + let(:end_id) { table(:users).pluck(:id).max } + + let(:migration) do + described_class.new( + start_id: starting_id, + end_id: end_id, + batch_table: :users, + batch_column: :id, + sub_batch_size: 100, + pause_ms: 2, + connection: ::ApplicationRecord.connection + ) + end + + describe 'perform' do + it 'updates user with `nil` user type only' do + expect do + migration.perform + valid_users.map(&:reload) + user_to_update.reload + end.not_to change { valid_users.map(&:user_type) } + + expect(user_to_update.user_type).to eq(0) + end + end +end diff --git a/spec/lib/gitlab/background_migration/migrate_links_for_vulnerability_findings_spec.rb b/spec/lib/gitlab/background_migration/migrate_links_for_vulnerability_findings_spec.rb new file mode 100644 index 00000000000..b973f9d4350 --- /dev/null +++ b/spec/lib/gitlab/background_migration/migrate_links_for_vulnerability_findings_spec.rb @@ -0,0 +1,192 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::MigrateLinksForVulnerabilityFindings, + feature_category: :vulnerability_management do + let(:vulnerability_occurrences) { table(:vulnerability_occurrences) } + let(:vulnerability_finding_links) { table(:vulnerability_finding_links) } + let(:link_hash) { { url: 'http://test.com' } } + let(:namespace1) { table(:namespaces).create!(name: 'namespace 1', path: 'namespace1') } + let(:project1) { table(:projects).create!(namespace_id: namespace1.id, project_namespace_id: namespace1.id) } + let(:user) { table(:users).create!(email: 'test1@example.com', projects_limit: 5) } + + let(:scanner1) do + table(:vulnerability_scanners).create!(project_id: project1.id, external_id: 'test 1', name: 'test scanner 1') + end + + let(:stating_id) { vulnerability_occurrences.pluck(:id).min } + let(:end_id) { vulnerability_occurrences.pluck(:id).max } + + let(:migration) do + described_class.new( + start_id: stating_id, + end_id: end_id, + batch_table: :vulnerability_occurrences, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 2, + connection: ApplicationRecord.connection + ) + end + + subject(:perform_migration) { migration.perform } + + context 'without the presence of links key' do + before do + create_finding!(project1.id, scanner1.id, { other_keys: 'test' }) + end + + it 'does not create any link' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_finding_links.count } + end + end + + context 'with links equals to an array of nil element' do + before do + create_finding!(project1.id, scanner1.id, { links: [nil] }) + end + + it 'does not create any link' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_finding_links.count } + end + end + + context 'with links equals to a string' do + before do + create_finding!(project1.id, scanner1.id, { links: "wrong format" }) + end + + it 'does not create any link' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_finding_links.count } + end + end + + context 'with some elements which do not contain the key url' do + let!(:finding) do + create_finding!(project1.id, scanner1.id, { links: [link_hash, "wrong format", {}] }) + end + + it 'creates links only to valid elements' do + expect(Gitlab::AppLogger).not_to receive(:error) + + perform_migration + + expect(vulnerability_finding_links.all).to contain_exactly(have_attributes( + url: link_hash[:url], + vulnerability_occurrence_id: finding.id)) + end + end + + context 'when link name is too long' do + let!(:finding) do + create_finding!(project1.id, scanner1.id, { links: [{ name: 'A' * 300, url: 'https://foo' }] }) + end + + it 'skips creation of link and logs error' do + expect(Gitlab::AppLogger).to receive(:error).with({ + class: described_class.name, + message: /check_55f0a95439/, + model_id: finding.id + }) + expect { perform_migration }.not_to change { vulnerability_finding_links.count } + end + end + + context 'when link url is too long' do + let!(:finding) do + create_finding!(project1.id, scanner1.id, { links: [{ url: "https://f#{'o' * 2050}" }] }) + end + + it 'skips creation of link and logs error' do + expect(Gitlab::AppLogger).to receive(:error).with({ + class: described_class.name, + message: /check_b7fe886df6/, + model_id: finding.id + }) + expect { perform_migration }.not_to change { vulnerability_finding_links.count } + end + end + + context 'with links equals to an array of duplicated elements' do + let!(:finding) do + create_finding!(project1.id, scanner1.id, { links: [link_hash, link_hash] }) + end + + it 'creates one new link' do + expect(Gitlab::AppLogger).not_to receive(:error) + + perform_migration + + expect(vulnerability_finding_links.all).to contain_exactly(have_attributes( + url: link_hash[:url], + vulnerability_occurrence_id: finding.id)) + end + end + + context 'with existing links within raw_metadata' do + let!(:finding1) { create_finding!(project1.id, scanner1.id, { links: [link_hash] }) } + let!(:finding2) { create_finding!(project1.id, scanner1.id, { links: [link_hash] }) } + + it 'creates new link for each finding' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.to change { vulnerability_finding_links.count }.by(2) + end + end + + context 'when Gitlab::Json throws exception JSON::ParserError' do + before do + create_finding!(project1.id, scanner1.id, { links: [link_hash] }) + allow(Gitlab::Json).to receive(:parse).and_raise(JSON::ParserError) + end + + it 'does not log this error nor create new records' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_finding_links.count } + end + end + + context 'with existing link records' do + let!(:finding) { create_finding!(project1.id, scanner1.id, { links: [link_hash] }) } + + before do + vulnerability_finding_links.create!(vulnerability_occurrence_id: finding.id, url: link_hash[:url]) + end + + it 'does not create new link' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_finding_links.count } + end + + it 'does not raise ActiveRecord::RecordNotUnique' do + expect { perform_migration }.not_to raise_error + end + end + + private + + def create_finding!(project_id, scanner_id, raw_metadata) + vulnerability = table(:vulnerabilities).create!(project_id: project_id, author_id: user.id, title: 'test', + severity: 4, confidence: 4, report_type: 0) + + identifier = table(:vulnerability_identifiers).create!(project_id: project_id, external_type: 'uuid-v5', + external_id: 'uuid-v5', fingerprint: OpenSSL::Digest::SHA256.hexdigest(vulnerability.id.to_s), + name: 'Identifier for UUIDv5 2 2') + + table(:vulnerability_occurrences).create!( + vulnerability_id: vulnerability.id, project_id: project_id, scanner_id: scanner_id, + primary_identifier_id: identifier.id, name: 'test', severity: 4, confidence: 4, report_type: 0, + uuid: SecureRandom.uuid, project_fingerprint: '123qweasdzxc', location: { "image" => "alpine:3.4" }, + location_fingerprint: 'test', metadata_version: 'test', + raw_metadata: raw_metadata.to_json) + end +end diff --git a/spec/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users_spec.rb b/spec/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users_spec.rb deleted file mode 100644 index c3ae2cc060c..00000000000 --- a/spec/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users_spec.rb +++ /dev/null @@ -1,413 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::MigrateMergeRequestDiffCommitUsers, schema: 20211012134316 do - let(:namespaces) { table(:namespaces) } - let(:projects) { table(:projects) } - let(:users) { table(:users) } - let(:merge_requests) { table(:merge_requests) } - let(:diffs) { table(:merge_request_diffs) } - let(:commits) do - table(:merge_request_diff_commits).tap do |t| - t.extend(SuppressCompositePrimaryKeyWarning) - end - end - - let(:commit_users) { described_class::MergeRequestDiffCommitUser } - - let(:namespace) { namespaces.create!(name: 'foo', path: 'foo') } - let(:project) { projects.create!(namespace_id: namespace.id) } - let(:merge_request) do - merge_requests.create!( - source_branch: 'x', - target_branch: 'master', - target_project_id: project.id - ) - end - - let(:diff) { diffs.create!(merge_request_id: merge_request.id) } - let(:migration) { described_class.new } - - describe 'MergeRequestDiffCommit' do - describe '.each_row_to_migrate' do - it 'yields the rows to migrate for a given range' do - commit1 = commits.create!( - merge_request_diff_id: diff.id, - relative_order: 0, - sha: Gitlab::Database::ShaAttribute.serialize('123abc'), - author_name: 'bob', - author_email: 'bob@example.com', - committer_name: 'bob', - committer_email: 'bob@example.com' - ) - - commit2 = commits.create!( - merge_request_diff_id: diff.id, - relative_order: 1, - sha: Gitlab::Database::ShaAttribute.serialize('123abc'), - author_name: 'Alice', - author_email: 'alice@example.com', - committer_name: 'Alice', - committer_email: 'alice@example.com' - ) - - # We stub this constant to make sure we run at least two pagination - # queries for getting the data. This way we can test if the pagination - # is actually working properly. - stub_const( - 'Gitlab::BackgroundMigration::MigrateMergeRequestDiffCommitUsers::COMMIT_ROWS_PER_QUERY', - 1 - ) - - rows = [] - - described_class::MergeRequestDiffCommit.each_row_to_migrate(diff.id, diff.id + 1) do |row| - rows << row - end - - expect(rows.length).to eq(2) - - expect(rows[0].author_name).to eq(commit1.author_name) - expect(rows[1].author_name).to eq(commit2.author_name) - end - end - end - - describe 'MergeRequestDiffCommitUser' do - describe '.union' do - it 'produces a union of the given queries' do - alice = commit_users.create!(name: 'Alice', email: 'alice@example.com') - bob = commit_users.create!(name: 'Bob', email: 'bob@example.com') - users = commit_users.union( - [ - commit_users.where(name: 'Alice').to_sql, - commit_users.where(name: 'Bob').to_sql - ]) - - expect(users).to include(alice) - expect(users).to include(bob) - end - end - end - - describe '#perform' do - it 'skips jobs that have already been completed' do - Gitlab::Database::BackgroundMigrationJob.create!( - class_name: 'MigrateMergeRequestDiffCommitUsers', - arguments: [1, 10], - status: :succeeded - ) - - expect(migration).not_to receive(:get_data_to_update) - - migration.perform(1, 10) - end - - it 'migrates the data in the range' do - commits.create!( - merge_request_diff_id: diff.id, - relative_order: 0, - sha: Gitlab::Database::ShaAttribute.serialize('123abc'), - author_name: 'bob', - author_email: 'bob@example.com', - committer_name: 'bob', - committer_email: 'bob@example.com' - ) - - migration.perform(diff.id, diff.id + 1) - - bob = commit_users.find_by(name: 'bob') - commit = commits.first - - expect(commit.commit_author_id).to eq(bob.id) - expect(commit.committer_id).to eq(bob.id) - end - - it 'treats empty names and Emails the same as NULL values' do - commits.create!( - merge_request_diff_id: diff.id, - relative_order: 0, - sha: Gitlab::Database::ShaAttribute.serialize('123abc'), - author_name: 'bob', - author_email: 'bob@example.com', - committer_name: '', - committer_email: '' - ) - - migration.perform(diff.id, diff.id + 1) - - bob = commit_users.find_by(name: 'bob') - commit = commits.first - - expect(commit.commit_author_id).to eq(bob.id) - expect(commit.committer_id).to be_nil - end - - it 'does not update rows without a committer and author' do - commits.create!( - merge_request_diff_id: diff.id, - relative_order: 0, - sha: Gitlab::Database::ShaAttribute.serialize('123abc') - ) - - migration.perform(diff.id, diff.id + 1) - - commit = commits.first - - expect(commit_users.count).to eq(0) - expect(commit.commit_author_id).to be_nil - expect(commit.committer_id).to be_nil - end - - it 'marks the background job as done' do - Gitlab::Database::BackgroundMigrationJob.create!( - class_name: 'MigrateMergeRequestDiffCommitUsers', - arguments: [diff.id, diff.id + 1] - ) - - migration.perform(diff.id, diff.id + 1) - - job = Gitlab::Database::BackgroundMigrationJob.first - - expect(job.status).to eq('succeeded') - end - end - - describe '#get_data_to_update' do - it 'returns the users and commit rows to update' do - commits.create!( - merge_request_diff_id: diff.id, - relative_order: 0, - sha: Gitlab::Database::ShaAttribute.serialize('123abc'), - author_name: 'bob' + ('a' * 510), - author_email: 'bob@example.com', - committer_name: 'bob' + ('a' * 510), - committer_email: 'bob@example.com' - ) - - commits.create!( - merge_request_diff_id: diff.id, - relative_order: 1, - sha: Gitlab::Database::ShaAttribute.serialize('456abc'), - author_name: 'alice', - author_email: 'alice@example.com', - committer_name: 'alice', - committer_email: 'alice@example.com' - ) - - users, to_update = migration.get_data_to_update(diff.id, diff.id + 1) - - bob_name = 'bob' + ('a' * 509) - - expect(users).to include(%w[alice alice@example.com]) - expect(users).to include([bob_name, 'bob@example.com']) - - expect(to_update[[diff.id, 0]]) - .to eq([[bob_name, 'bob@example.com'], [bob_name, 'bob@example.com']]) - - expect(to_update[[diff.id, 1]]) - .to eq([%w[alice alice@example.com], %w[alice alice@example.com]]) - end - - it 'does not include a user if both the name and Email are missing' do - commits.create!( - merge_request_diff_id: diff.id, - relative_order: 0, - sha: Gitlab::Database::ShaAttribute.serialize('123abc'), - author_name: nil, - author_email: nil, - committer_name: 'bob', - committer_email: 'bob@example.com' - ) - - users, _ = migration.get_data_to_update(diff.id, diff.id + 1) - - expect(users).to eq([%w[bob bob@example.com]].to_set) - end - end - - describe '#get_user_rows_in_batches' do - it 'retrieves all existing users' do - alice = commit_users.create!(name: 'alice', email: 'alice@example.com') - bob = commit_users.create!(name: 'bob', email: 'bob@example.com') - - users = [[alice.name, alice.email], [bob.name, bob.email]] - mapping = {} - - migration.get_user_rows_in_batches(users, mapping) - - expect(mapping[%w[alice alice@example.com]]).to eq(alice) - expect(mapping[%w[bob bob@example.com]]).to eq(bob) - end - end - - describe '#create_missing_users' do - it 'creates merge request diff commit users that are missing' do - alice = commit_users.create!(name: 'alice', email: 'alice@example.com') - users = [%w[alice alice@example.com], %w[bob bob@example.com]] - mapping = { %w[alice alice@example.com] => alice } - - migration.create_missing_users(users, mapping) - - expect(mapping[%w[alice alice@example.com]]).to eq(alice) - expect(mapping[%w[bob bob@example.com]].name).to eq('bob') - expect(mapping[%w[bob bob@example.com]].email).to eq('bob@example.com') - end - end - - describe '#update_commit_rows' do - it 'updates the merge request diff commit rows' do - to_update = { [42, 0] => [%w[alice alice@example.com], []] } - user_mapping = { %w[alice alice@example.com] => double(:user, id: 1) } - - expect(migration) - .to receive(:bulk_update_commit_rows) - .with({ [42, 0] => [1, nil] }) - - migration.update_commit_rows(to_update, user_mapping) - end - end - - describe '#bulk_update_commit_rows' do - context 'when there are no authors and committers' do - it 'does not update any rows' do - migration.bulk_update_commit_rows({ [1, 0] => [] }) - - expect(described_class::MergeRequestDiffCommit.connection) - .not_to receive(:execute) - end - end - - context 'when there are only authors' do - it 'only updates the author IDs' do - author = commit_users.create!(name: 'Alice', email: 'alice@example.com') - commit = commits.create!( - merge_request_diff_id: diff.id, - relative_order: 0, - sha: Gitlab::Database::ShaAttribute.serialize('123abc') - ) - - mapping = { - [commit.merge_request_diff_id, commit.relative_order] => - [author.id, nil] - } - - migration.bulk_update_commit_rows(mapping) - - commit = commits.first - - expect(commit.commit_author_id).to eq(author.id) - expect(commit.committer_id).to be_nil - end - end - - context 'when there are only committers' do - it 'only updates the committer IDs' do - committer = - commit_users.create!(name: 'Alice', email: 'alice@example.com') - - commit = commits.create!( - merge_request_diff_id: diff.id, - relative_order: 0, - sha: Gitlab::Database::ShaAttribute.serialize('123abc') - ) - - mapping = { - [commit.merge_request_diff_id, commit.relative_order] => - [nil, committer.id] - } - - migration.bulk_update_commit_rows(mapping) - - commit = commits.first - - expect(commit.committer_id).to eq(committer.id) - expect(commit.commit_author_id).to be_nil - end - end - - context 'when there are both authors and committers' do - it 'updates both the author and committer IDs' do - author = commit_users.create!(name: 'Bob', email: 'bob@example.com') - committer = - commit_users.create!(name: 'Alice', email: 'alice@example.com') - - commit = commits.create!( - merge_request_diff_id: diff.id, - relative_order: 0, - sha: Gitlab::Database::ShaAttribute.serialize('123abc') - ) - - mapping = { - [commit.merge_request_diff_id, commit.relative_order] => - [author.id, committer.id] - } - - migration.bulk_update_commit_rows(mapping) - - commit = commits.first - - expect(commit.commit_author_id).to eq(author.id) - expect(commit.committer_id).to eq(committer.id) - end - end - - context 'when there are multiple commit rows to update' do - it 'updates all the rows' do - author = commit_users.create!(name: 'Bob', email: 'bob@example.com') - committer = - commit_users.create!(name: 'Alice', email: 'alice@example.com') - - commit1 = commits.create!( - merge_request_diff_id: diff.id, - relative_order: 0, - sha: Gitlab::Database::ShaAttribute.serialize('123abc') - ) - - commit2 = commits.create!( - merge_request_diff_id: diff.id, - relative_order: 1, - sha: Gitlab::Database::ShaAttribute.serialize('456abc') - ) - - mapping = { - [commit1.merge_request_diff_id, commit1.relative_order] => - [author.id, committer.id], - - [commit2.merge_request_diff_id, commit2.relative_order] => - [author.id, nil] - } - - migration.bulk_update_commit_rows(mapping) - - commit1 = commits.find_by(relative_order: 0) - commit2 = commits.find_by(relative_order: 1) - - expect(commit1.commit_author_id).to eq(author.id) - expect(commit1.committer_id).to eq(committer.id) - - expect(commit2.commit_author_id).to eq(author.id) - expect(commit2.committer_id).to be_nil - end - end - end - - describe '#primary_key' do - it 'returns the primary key for the commits table' do - key = migration.primary_key - - expect(key.to_sql).to eq('("merge_request_diff_commits"."merge_request_diff_id", "merge_request_diff_commits"."relative_order")') - end - end - - describe '#prepare' do - it 'trims a value to at most 512 characters' do - expect(migration.prepare('€' * 1_000)).to eq('€' * 512) - end - - it 'returns nil if the value is an empty string' do - expect(migration.prepare('')).to be_nil - end - end -end diff --git a/spec/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics_spec.rb b/spec/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics_spec.rb deleted file mode 100644 index b252df4ecff..00000000000 --- a/spec/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics_spec.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::MigrateProjectTaggingsContextFromTagsToTopics, - :suppress_gitlab_schemas_validate_connection, schema: 20210826171758 do - it 'correctly migrates project taggings context from tags to topics' do - taggings = table(:taggings) - - project_old_tagging_1 = taggings.create!(taggable_type: 'Project', context: 'tags') - project_new_tagging_1 = taggings.create!(taggable_type: 'Project', context: 'topics') - project_other_context_tagging_1 = taggings.create!(taggable_type: 'Project', context: 'other') - project_old_tagging_2 = taggings.create!(taggable_type: 'Project', context: 'tags') - project_old_tagging_3 = taggings.create!(taggable_type: 'Project', context: 'tags') - - subject.perform(project_old_tagging_1.id, project_old_tagging_2.id) - - project_old_tagging_1.reload - project_new_tagging_1.reload - project_other_context_tagging_1.reload - project_old_tagging_2.reload - project_old_tagging_3.reload - - expect(project_old_tagging_1.context).to eq('topics') - expect(project_new_tagging_1.context).to eq('topics') - expect(project_other_context_tagging_1.context).to eq('other') - expect(project_old_tagging_2.context).to eq('topics') - expect(project_old_tagging_3.context).to eq('tags') - end -end diff --git a/spec/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings_spec.rb b/spec/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings_spec.rb new file mode 100644 index 00000000000..b75c0e61b19 --- /dev/null +++ b/spec/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings_spec.rb @@ -0,0 +1,173 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::MigrateRemediationsForVulnerabilityFindings, + feature_category: :vulnerability_management do + let(:vulnerability_occurrences) { table(:vulnerability_occurrences) } + let(:vulnerability_findings_remediations) { table(:vulnerability_findings_remediations) } + let(:vulnerability_remediations) { table(:vulnerability_remediations) } + let(:remediation_hash) { { summary: 'summary', diff: "ZGlmZiAtLWdp" } } + let(:namespace1) { table(:namespaces).create!(name: 'namespace 1', path: 'namespace1') } + let(:project1) { table(:projects).create!(namespace_id: namespace1.id, project_namespace_id: namespace1.id) } + let(:user) { table(:users).create!(email: 'test1@example.com', projects_limit: 5) } + + let(:scanner1) do + table(:vulnerability_scanners).create!(project_id: project1.id, external_id: 'test 1', name: 'test scanner 1') + end + + let(:stating_id) { vulnerability_occurrences.pluck(:id).min } + let(:end_id) { vulnerability_occurrences.pluck(:id).max } + + let(:migration) do + described_class.new( + start_id: stating_id, + end_id: end_id, + batch_table: :vulnerability_occurrences, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 2, + connection: ApplicationRecord.connection + ) + end + + subject(:perform_migration) { migration.perform } + + context 'without the presence of remediation key' do + before do + create_finding!(project1.id, scanner1.id, { other_keys: 'test' }) + end + + it 'does not create any remediation' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_remediations.count } + end + end + + context 'with remediation equals to an array of nil element' do + before do + create_finding!(project1.id, scanner1.id, { remediations: [nil] }) + end + + it 'does not create any remediation' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_remediations.count } + end + end + + context 'with remediation with empty string as the diff key' do + let!(:finding) do + create_finding!(project1.id, scanner1.id, { remediations: [{ summary: 'summary', diff: '' }] }) + end + + it 'does not create any remediation' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_remediations.count } + end + end + + context 'with remediation equals to an array of duplicated elements' do + let!(:finding) do + create_finding!(project1.id, scanner1.id, { remediations: [remediation_hash, remediation_hash] }) + end + + it 'creates new remediation' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.to change { vulnerability_remediations.count }.by(1) + expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding.id).length).to eq(1) + end + end + + context 'with existing remediations within raw_metadata' do + let!(:finding1) { create_finding!(project1.id, scanner1.id, { remediations: [remediation_hash] }) } + let!(:finding2) { create_finding!(project1.id, scanner1.id, { remediations: [remediation_hash] }) } + + it 'creates new remediation' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.to change { vulnerability_remediations.count }.by(1) + expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding1.id).length).to eq(1) + expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding2.id).length).to eq(1) + end + + context 'when create throws exception other than ActiveRecord::RecordNotUnique' do + before do + allow(migration).to receive(:create_finding_remediations).and_raise(StandardError) + end + + it 'rolls back all related transactions' do + expect(Gitlab::AppLogger).to receive(:error).with({ + class: described_class.name, message: StandardError.to_s, model_id: finding1.id + }) + expect(Gitlab::AppLogger).to receive(:error).with({ + class: described_class.name, message: StandardError.to_s, model_id: finding2.id + }) + expect { perform_migration }.not_to change { vulnerability_remediations.count } + expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding1.id).length).to eq(0) + expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding2.id).length).to eq(0) + end + end + end + + context 'with existing remediation records' do + let!(:finding) { create_finding!(project1.id, scanner1.id, { remediations: [remediation_hash] }) } + + before do + vulnerability_remediations.create!(project_id: project1.id, summary: remediation_hash[:summary], + checksum: checksum(remediation_hash[:diff]), file: Tempfile.new.path) + end + + it 'does not create new remediation' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_remediations.count } + expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding.id).length).to eq(1) + end + end + + context 'with same raw_metadata for different projects' do + let(:namespace2) { table(:namespaces).create!(name: 'namespace 2', path: 'namespace2') } + let(:project2) { table(:projects).create!(namespace_id: namespace2.id, project_namespace_id: namespace2.id) } + let(:scanner2) do + table(:vulnerability_scanners).create!(project_id: project2.id, external_id: 'test 2', name: 'test scanner 2') + end + + let!(:finding1) { create_finding!(project1.id, scanner1.id, { remediations: [remediation_hash] }) } + let!(:finding2) { create_finding!(project2.id, scanner2.id, { remediations: [remediation_hash] }) } + + it 'creates new remediation for each project' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.to change { vulnerability_remediations.count }.by(2) + expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding1.id).length).to eq(1) + expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding2.id).length).to eq(1) + end + end + + private + + def create_finding!(project_id, scanner_id, raw_metadata) + vulnerability = table(:vulnerabilities).create!(project_id: project_id, author_id: user.id, title: 'test', + severity: 4, confidence: 4, report_type: 0) + + identifier = table(:vulnerability_identifiers).create!(project_id: project_id, external_type: 'uuid-v5', + external_id: 'uuid-v5', fingerprint: OpenSSL::Digest::SHA256.hexdigest(vulnerability.id.to_s), + name: 'Identifier for UUIDv5 2 2') + + table(:vulnerability_occurrences).create!( + vulnerability_id: vulnerability.id, project_id: project_id, scanner_id: scanner_id, + primary_identifier_id: identifier.id, name: 'test', severity: 4, confidence: 4, report_type: 0, + uuid: SecureRandom.uuid, project_fingerprint: '123qweasdzxc', location: { "image" => "alpine:3.4" }, + location_fingerprint: 'test', metadata_version: 'test', + raw_metadata: raw_metadata.to_json) + end + + def checksum(value) + sha = Digest::SHA256.hexdigest(value) + Gitlab::Database::ShaAttribute.new.serialize(sha) + end +end diff --git a/spec/lib/gitlab/background_migration/migrate_u2f_webauthn_spec.rb b/spec/lib/gitlab/background_migration/migrate_u2f_webauthn_spec.rb deleted file mode 100644 index 08fde0d0ff4..00000000000 --- a/spec/lib/gitlab/background_migration/migrate_u2f_webauthn_spec.rb +++ /dev/null @@ -1,67 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -require 'webauthn/u2f_migrator' - -RSpec.describe Gitlab::BackgroundMigration::MigrateU2fWebauthn, :migration, schema: 20210826171758 do - let(:users) { table(:users) } - - let(:user) { users.create!(email: 'email@email.com', name: 'foo', username: 'foo', projects_limit: 0) } - - let(:u2f_registrations) { table(:u2f_registrations) } - let(:webauthn_registrations) { table(:webauthn_registrations) } - - let!(:u2f_registration_not_migrated) { create_u2f_registration(1, 'reg1') } - let!(:u2f_registration_not_migrated_no_name) { create_u2f_registration(2, nil, 2) } - let!(:u2f_registration_migrated) { create_u2f_registration(3, 'reg3') } - - subject { described_class.new.perform(1, 3) } - - before do - converted_credential = convert_credential_for(u2f_registration_migrated) - webauthn_registrations.create!(converted_credential) - end - - it 'migrates all records' do - expect { subject }.to change { webauthn_registrations.count }.from(1).to(3) - - all_webauthn_registrations = webauthn_registrations.all.map(&:attributes) - - [u2f_registration_not_migrated, u2f_registration_not_migrated_no_name].each do |u2f_registration| - expected_credential = convert_credential_for(u2f_registration).except(:created_at).stringify_keys - expect(all_webauthn_registrations).to include(a_hash_including(expected_credential)) - end - end - - def create_u2f_registration(id, name, counter = 5) - device = U2F::FakeU2F.new(FFaker::BaconIpsum.characters(5)) - u2f_registrations.create!({ id: id, - certificate: Base64.strict_encode64(device.cert_raw), - key_handle: U2F.urlsafe_encode64(device.key_handle_raw), - public_key: Base64.strict_encode64(device.origin_public_key_raw), - counter: counter, - name: name, - user_id: user.id }) - end - - def convert_credential_for(u2f_registration) - converted_credential = WebAuthn::U2fMigrator.new( - app_id: Gitlab.config.gitlab.url, - certificate: u2f_registration.certificate, - key_handle: u2f_registration.key_handle, - public_key: u2f_registration.public_key, - counter: u2f_registration.counter - ).credential - - { - credential_xid: Base64.strict_encode64(converted_credential.id), - public_key: Base64.strict_encode64(converted_credential.public_key), - counter: u2f_registration.counter, - name: u2f_registration.name || '', - user_id: u2f_registration.user_id, - u2f_registration_id: u2f_registration.id, - created_at: u2f_registration.created_at - } - end -end diff --git a/spec/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature_spec.rb b/spec/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature_spec.rb deleted file mode 100644 index 71cf58a933f..00000000000 --- a/spec/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature_spec.rb +++ /dev/null @@ -1,98 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::MoveContainerRegistryEnabledToProjectFeature, :migration, schema: 20210826171758 do - let(:enabled) { 20 } - let(:disabled) { 0 } - - let(:namespaces) { table(:namespaces) } - let(:project_features) { table(:project_features) } - let(:projects) { table(:projects) } - - let(:namespace) { namespaces.create!(name: 'user', path: 'user') } - let!(:project1) { projects.create!(namespace_id: namespace.id) } - let!(:project2) { projects.create!(namespace_id: namespace.id) } - let!(:project3) { projects.create!(namespace_id: namespace.id) } - let!(:project4) { projects.create!(namespace_id: namespace.id) } - - # pages_access_level cannot be null. - let(:non_null_project_features) { { pages_access_level: enabled } } - let!(:project_feature1) { project_features.create!(project_id: project1.id, **non_null_project_features) } - let!(:project_feature2) { project_features.create!(project_id: project2.id, **non_null_project_features) } - let!(:project_feature3) { project_features.create!(project_id: project3.id, **non_null_project_features) } - - describe '#perform' do - before do - project1.update!(container_registry_enabled: true) - project2.update!(container_registry_enabled: false) - project3.update!(container_registry_enabled: nil) - project4.update!(container_registry_enabled: true) - end - - it 'copies values to project_features' do - table(:background_migration_jobs).create!( - class_name: 'MoveContainerRegistryEnabledToProjectFeature', - arguments: [project1.id, project4.id] - ) - table(:background_migration_jobs).create!( - class_name: 'MoveContainerRegistryEnabledToProjectFeature', - arguments: [-1, -3] - ) - - expect(project1.container_registry_enabled).to eq(true) - expect(project2.container_registry_enabled).to eq(false) - expect(project3.container_registry_enabled).to eq(nil) - expect(project4.container_registry_enabled).to eq(true) - - expect(project_feature1.container_registry_access_level).to eq(disabled) - expect(project_feature2.container_registry_access_level).to eq(disabled) - expect(project_feature3.container_registry_access_level).to eq(disabled) - - expect_next_instance_of(Gitlab::BackgroundMigration::Logger) do |logger| - expect(logger).to receive(:info) - .with(message: "#{described_class}: Copied container_registry_enabled values for projects with IDs between #{project1.id}..#{project4.id}") - - expect(logger).not_to receive(:info) - end - - subject.perform(project1.id, project4.id) - - expect(project1.reload.container_registry_enabled).to eq(true) - expect(project2.reload.container_registry_enabled).to eq(false) - expect(project3.reload.container_registry_enabled).to eq(nil) - expect(project4.container_registry_enabled).to eq(true) - - expect(project_feature1.reload.container_registry_access_level).to eq(enabled) - expect(project_feature2.reload.container_registry_access_level).to eq(disabled) - expect(project_feature3.reload.container_registry_access_level).to eq(disabled) - - expect(table(:background_migration_jobs).first.status).to eq(1) # succeeded - expect(table(:background_migration_jobs).second.status).to eq(0) # pending - end - - context 'when no projects exist in range' do - it 'does not fail' do - expect(project1.container_registry_enabled).to eq(true) - expect(project_feature1.container_registry_access_level).to eq(disabled) - - expect { subject.perform(-1, -2) }.not_to raise_error - - expect(project1.container_registry_enabled).to eq(true) - expect(project_feature1.container_registry_access_level).to eq(disabled) - end - end - - context 'when projects in range all have nil container_registry_enabled' do - it 'does not fail' do - expect(project3.container_registry_enabled).to eq(nil) - expect(project_feature3.container_registry_access_level).to eq(disabled) - - expect { subject.perform(project3.id, project3.id) }.not_to raise_error - - expect(project3.container_registry_enabled).to eq(nil) - expect(project_feature3.container_registry_access_level).to eq(disabled) - end - end - end -end diff --git a/spec/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects_spec.rb b/spec/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects_spec.rb index a8574411957..f671a673a08 100644 --- a/spec/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects_spec.rb +++ b/spec/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects_spec.rb @@ -2,7 +2,8 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::NullifyCreatorIdColumnOfOrphanedProjects, feature_category: :projects do +RSpec.describe Gitlab::BackgroundMigration::NullifyCreatorIdColumnOfOrphanedProjects, feature_category: :projects, + schema: 20230130073109 do let(:users) { table(:users) } let(:projects) { table(:projects) } let(:namespaces) { table(:namespaces) } diff --git a/spec/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds_spec.rb b/spec/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds_spec.rb index 2f0eef3c399..5b234679e22 100644 --- a/spec/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds_spec.rb +++ b/spec/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::BackgroundMigration::NullifyOrphanRunnerIdOnCiBuilds, - :suppress_gitlab_schemas_validate_connection, migration: :gitlab_ci, schema: 20220223112304 do + :suppress_gitlab_schemas_validate_connection, migration: :gitlab_ci, schema: 20220223112304 do let(:namespaces) { table(:namespaces) } let(:projects) { table(:projects) } let(:ci_runners) { table(:ci_runners) } diff --git a/spec/lib/gitlab/background_migration/populate_topics_total_projects_count_cache_spec.rb b/spec/lib/gitlab/background_migration/populate_topics_total_projects_count_cache_spec.rb deleted file mode 100644 index 8e07b43f5b9..00000000000 --- a/spec/lib/gitlab/background_migration/populate_topics_total_projects_count_cache_spec.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::PopulateTopicsTotalProjectsCountCache, schema: 20211006060436 do - it 'correctly populates total projects count cache' do - namespaces = table(:namespaces) - projects = table(:projects) - topics = table(:topics) - project_topics = table(:project_topics) - - group = namespaces.create!(name: 'group', path: 'group') - project_1 = projects.create!(namespace_id: group.id) - project_2 = projects.create!(namespace_id: group.id) - project_3 = projects.create!(namespace_id: group.id) - topic_1 = topics.create!(name: 'Topic1') - topic_2 = topics.create!(name: 'Topic2') - topic_3 = topics.create!(name: 'Topic3') - topic_4 = topics.create!(name: 'Topic4') - - project_topics.create!(project_id: project_1.id, topic_id: topic_1.id) - project_topics.create!(project_id: project_1.id, topic_id: topic_3.id) - project_topics.create!(project_id: project_2.id, topic_id: topic_3.id) - project_topics.create!(project_id: project_1.id, topic_id: topic_4.id) - project_topics.create!(project_id: project_2.id, topic_id: topic_4.id) - project_topics.create!(project_id: project_3.id, topic_id: topic_4.id) - - subject.perform(topic_1.id, topic_4.id) - - expect(topic_1.reload.total_projects_count).to eq(1) - expect(topic_2.reload.total_projects_count).to eq(0) - expect(topic_3.reload.total_projects_count).to eq(2) - expect(topic_4.reload.total_projects_count).to eq(3) - end -end diff --git a/spec/lib/gitlab/background_migration/populate_vulnerability_dismissal_fields_spec.rb b/spec/lib/gitlab/background_migration/populate_vulnerability_dismissal_fields_spec.rb new file mode 100644 index 00000000000..50380247c9f --- /dev/null +++ b/spec/lib/gitlab/background_migration/populate_vulnerability_dismissal_fields_spec.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::PopulateVulnerabilityDismissalFields, schema: 20230412185837, feature_category: :vulnerability_management do # rubocop:disable Layout/LineLength + let(:users) { table(:users) } + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:vulnerabilities) { table(:vulnerabilities) } + let(:findings) { table(:vulnerability_occurrences) } + let(:scanners) { table(:vulnerability_scanners) } + let(:identifiers) { table(:vulnerability_identifiers) } + let(:feedback) { table(:vulnerability_feedback) } + + let(:user) { users.create!(name: 'test', email: 'test@example.com', projects_limit: 5) } + let(:namespace) { namespaces.create!(name: 'gitlab', path: 'gitlab-org') } + let(:project) { projects.create!(namespace_id: namespace.id, name: 'foo', project_namespace_id: namespace.id) } + let(:vulnerability_1) do + vulnerabilities.create!(title: 'title', state: 2, severity: 0, + confidence: 5, report_type: 2, project_id: project.id, author_id: user.id + ) + end + + let(:vulnerability_2) do + vulnerabilities.create!(title: 'title', state: 2, severity: 0, + confidence: 5, report_type: 2, project_id: project.id, author_id: user.id + ) + end + + let(:scanner) { scanners.create!(project_id: project.id, external_id: 'foo', name: 'bar') } + let(:identifier) do + identifiers.create!(project_id: project.id, fingerprint: 'foo', + external_type: 'bar', external_id: 'zoo', name: 'identifier' + ) + end + + let(:uuid) { SecureRandom.uuid } + + before do + feedback.create!(feedback_type: 0, + category: 'sast', + project_fingerprint: '418291a26024a1445b23fe64de9380cdcdfd1fa8', + project_id: project.id, + author_id: user.id, + created_at: Time.current, + finding_uuid: uuid + ) + + findings.create!(name: 'Finding', + report_type: 'sast', + project_fingerprint: '418291a26024a1445b23fe64de9380cdcdfd1f98', + location_fingerprint: 'bar', + severity: 1, + confidence: 1, + metadata_version: 1, + raw_metadata: '', + details: {}, + uuid: uuid, + project_id: project.id, + vulnerability_id: vulnerability_1.id, + scanner_id: scanner.id, + primary_identifier_id: identifier.id + ) + + allow(::Gitlab::BackgroundMigration::Logger).to receive_messages(info: true, warn: true, error: true) + end + + subject do + described_class.new( + start_id: vulnerability_1.id, + end_id: vulnerability_2.id, + batch_table: :vulnerabilities, + batch_column: :id, + sub_batch_size: 200, + pause_ms: 2.minutes, + connection: ApplicationRecord.connection + ) + end + + describe '#perform' do + it 'updates the missing dismissal information of the vulnerability' do + expect { subject.perform }.to change { vulnerability_1.reload.dismissed_at } + .from(nil) + .and change { vulnerability_1.reload.dismissed_by_id }.from(nil).to(user.id) + end + + it 'writes log messages', :aggregate_failures do + subject.perform + + expect(::Gitlab::BackgroundMigration::Logger).to have_received(:info).with(migrator: described_class.name, + message: 'Dismissal information has been copied', + count: 2 + ) + expect(::Gitlab::BackgroundMigration::Logger).to have_received(:warn).with(migrator: described_class.name, + message: 'Could not update vulnerability!', + vulnerability_id: vulnerability_2.id + ) + end + + context 'when logger throws exception StandardError' do + before do + allow(::Gitlab::BackgroundMigration::Logger).to receive(:warn).and_raise(StandardError) + end + + it 'logs StandardError' do + expect(::Gitlab::BackgroundMigration::Logger).to receive(:error).with({ + migrator: described_class.name, message: StandardError.to_s, vulnerability_id: vulnerability_2.id + }) + + subject.perform + end + end + end +end diff --git a/spec/lib/gitlab/background_migration/prune_stale_project_export_jobs_spec.rb b/spec/lib/gitlab/background_migration/prune_stale_project_export_jobs_spec.rb index 5150d0ea4b0..3446b9f0676 100644 --- a/spec/lib/gitlab/background_migration/prune_stale_project_export_jobs_spec.rb +++ b/spec/lib/gitlab/background_migration/prune_stale_project_export_jobs_spec.rb @@ -10,14 +10,15 @@ RSpec.describe Gitlab::BackgroundMigration::PruneStaleProjectExportJobs, feature let(:uploads) { table(:project_relation_export_uploads) } subject(:perform_migration) do - described_class.new(start_id: 1, - end_id: 300, - batch_table: :project_export_jobs, - batch_column: :id, - sub_batch_size: 2, - pause_ms: 0, - connection: ActiveRecord::Base.connection) - .perform + described_class.new( + start_id: 1, + end_id: 300, + batch_table: :project_export_jobs, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ).perform end it 'removes export jobs and associated relations older than 7 days' do diff --git a/spec/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid_spec.rb b/spec/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid_spec.rb deleted file mode 100644 index 2271bbfb2f3..00000000000 --- a/spec/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid_spec.rb +++ /dev/null @@ -1,530 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -def create_background_migration_job(ids, status) - proper_status = case status - when :pending - Gitlab::Database::BackgroundMigrationJob.statuses['pending'] - when :succeeded - Gitlab::Database::BackgroundMigrationJob.statuses['succeeded'] - else - raise ArgumentError - end - - background_migration_jobs.create!( - class_name: 'RecalculateVulnerabilitiesOccurrencesUuid', - arguments: Array(ids), - status: proper_status, - created_at: Time.now.utc - ) -end - -RSpec.describe Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid, :suppress_gitlab_schemas_validate_connection, schema: 20211124132705 do - let(:background_migration_jobs) { table(:background_migration_jobs) } - let(:pending_jobs) { background_migration_jobs.where(status: Gitlab::Database::BackgroundMigrationJob.statuses['pending']) } - let(:succeeded_jobs) { background_migration_jobs.where(status: Gitlab::Database::BackgroundMigrationJob.statuses['succeeded']) } - let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') } - let(:users) { table(:users) } - let(:user) { create_user! } - let(:project) { table(:projects).create!(id: 123, namespace_id: namespace.id) } - let(:scanners) { table(:vulnerability_scanners) } - let(:scanner) { scanners.create!(project_id: project.id, external_id: 'test 1', name: 'test scanner 1') } - let(:scanner2) { scanners.create!(project_id: project.id, external_id: 'test 2', name: 'test scanner 2') } - let(:vulnerabilities) { table(:vulnerabilities) } - let(:vulnerability_findings) { table(:vulnerability_occurrences) } - let(:vulnerability_finding_pipelines) { table(:vulnerability_occurrence_pipelines) } - let(:vulnerability_finding_signatures) { table(:vulnerability_finding_signatures) } - let(:vulnerability_identifiers) { table(:vulnerability_identifiers) } - - let(:identifier_1) { 'identifier-1' } - let!(:vulnerability_identifier) do - vulnerability_identifiers.create!( - project_id: project.id, - external_type: identifier_1, - external_id: identifier_1, - fingerprint: Gitlab::Database::ShaAttribute.serialize('ff9ef548a6e30a0462795d916f3f00d1e2b082ca'), - name: 'Identifier 1') - end - - let(:identifier_2) { 'identifier-2' } - let!(:vulnerability_identfier2) do - vulnerability_identifiers.create!( - project_id: project.id, - external_type: identifier_2, - external_id: identifier_2, - fingerprint: Gitlab::Database::ShaAttribute.serialize('4299e8ddd819f9bde9cfacf45716724c17b5ddf7'), - name: 'Identifier 2') - end - - let(:identifier_3) { 'identifier-3' } - let!(:vulnerability_identifier3) do - vulnerability_identifiers.create!( - project_id: project.id, - external_type: identifier_3, - external_id: identifier_3, - fingerprint: Gitlab::Database::ShaAttribute.serialize('8e91632f9c6671e951834a723ee221c44cc0d844'), - name: 'Identifier 3') - end - - let(:known_uuid_v4) { "b3cc2518-5446-4dea-871c-89d5e999c1ac" } - let(:known_uuid_v5) { "05377088-dc26-5161-920e-52a7159fdaa1" } - let(:desired_uuid_v5) { "f3e9a23f-9181-54bf-a5ab-c5bc7a9b881a" } - - subject { described_class.new.perform(start_id, end_id) } - - context "when finding has a UUIDv4" do - before do - @uuid_v4 = create_finding!( - vulnerability_id: nil, - project_id: project.id, - scanner_id: scanner2.id, - primary_identifier_id: vulnerability_identfier2.id, - report_type: 0, # "sast" - location_fingerprint: Gitlab::Database::ShaAttribute.serialize("fa18f432f1d56675f4098d318739c3cd5b14eb3e"), - uuid: known_uuid_v4 - ) - end - - let(:start_id) { @uuid_v4.id } - let(:end_id) { @uuid_v4.id } - - it "replaces it with UUIDv5" do - expect(vulnerability_findings.pluck(:uuid)).to match_array([known_uuid_v4]) - - subject - - expect(vulnerability_findings.pluck(:uuid)).to match_array([desired_uuid_v5]) - end - - it 'logs recalculation' do - expect_next_instance_of(Gitlab::BackgroundMigration::Logger) do |instance| - expect(instance).to receive(:info).twice - end - - subject - end - end - - context "when finding has a UUIDv5" do - before do - @uuid_v5 = create_finding!( - vulnerability_id: nil, - project_id: project.id, - scanner_id: scanner.id, - primary_identifier_id: vulnerability_identifier.id, - report_type: 0, # "sast" - location_fingerprint: Gitlab::Database::ShaAttribute.serialize("838574be0210968bf6b9f569df9c2576242cbf0a"), - uuid: known_uuid_v5 - ) - end - - let(:start_id) { @uuid_v5.id } - let(:end_id) { @uuid_v5.id } - - it "stays the same" do - expect(vulnerability_findings.pluck(:uuid)).to match_array([known_uuid_v5]) - - subject - - expect(vulnerability_findings.pluck(:uuid)).to match_array([known_uuid_v5]) - end - end - - context 'if a duplicate UUID would be generated' do # rubocop: disable RSpec/MultipleMemoizedHelpers - let(:v1) do - create_vulnerability!( - project_id: project.id, - author_id: user.id - ) - end - - let!(:finding_with_incorrect_uuid) do - create_finding!( - vulnerability_id: v1.id, - project_id: project.id, - scanner_id: scanner.id, - primary_identifier_id: vulnerability_identifier.id, - report_type: 0, # "sast" - location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis') - uuid: 'bd95c085-71aa-51d7-9bb6-08ae669c262e' - ) - end - - let(:v2) do - create_vulnerability!( - project_id: project.id, - author_id: user.id - ) - end - - let!(:finding_with_correct_uuid) do - create_finding!( - vulnerability_id: v2.id, - project_id: project.id, - primary_identifier_id: vulnerability_identifier.id, - scanner_id: scanner2.id, - report_type: 0, # "sast" - location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis') - uuid: '91984483-5efe-5215-b471-d524ac5792b1' - ) - end - - let(:v3) do - create_vulnerability!( - project_id: project.id, - author_id: user.id - ) - end - - let!(:finding_with_incorrect_uuid2) do - create_finding!( - vulnerability_id: v3.id, - project_id: project.id, - scanner_id: scanner.id, - primary_identifier_id: vulnerability_identfier2.id, - report_type: 0, # "sast" - location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis') - uuid: '00000000-1111-2222-3333-444444444444' - ) - end - - let(:v4) do - create_vulnerability!( - project_id: project.id, - author_id: user.id - ) - end - - let!(:finding_with_correct_uuid2) do - create_finding!( - vulnerability_id: v4.id, - project_id: project.id, - scanner_id: scanner2.id, - primary_identifier_id: vulnerability_identfier2.id, - report_type: 0, # "sast" - location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis') - uuid: '1edd751e-ef9a-5391-94db-a832c8635bfc' - ) - end - - let!(:finding_with_incorrect_uuid3) do - create_finding!( - vulnerability_id: nil, - project_id: project.id, - scanner_id: scanner.id, - primary_identifier_id: vulnerability_identifier3.id, - report_type: 0, # "sast" - location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis') - uuid: '22222222-3333-4444-5555-666666666666' - ) - end - - let!(:duplicate_not_in_the_same_batch) do - create_finding!( - id: 99999, - vulnerability_id: nil, - project_id: project.id, - scanner_id: scanner2.id, - primary_identifier_id: vulnerability_identifier3.id, - report_type: 0, # "sast" - location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis') - uuid: '4564f9d5-3c6b-5cc3-af8c-7c25285362a7' - ) - end - - let(:start_id) { finding_with_incorrect_uuid.id } - let(:end_id) { finding_with_incorrect_uuid3.id } - - before do - 4.times do - create_finding_pipeline!(project_id: project.id, finding_id: finding_with_incorrect_uuid.id) - create_finding_pipeline!(project_id: project.id, finding_id: finding_with_correct_uuid.id) - create_finding_pipeline!(project_id: project.id, finding_id: finding_with_incorrect_uuid2.id) - create_finding_pipeline!(project_id: project.id, finding_id: finding_with_correct_uuid2.id) - end - end - - it 'drops duplicates and related records', :aggregate_failures do - expect(vulnerability_findings.pluck(:id)).to match_array( - [ - finding_with_correct_uuid.id, - finding_with_incorrect_uuid.id, - finding_with_correct_uuid2.id, - finding_with_incorrect_uuid2.id, - finding_with_incorrect_uuid3.id, - duplicate_not_in_the_same_batch.id - ]) - - expect { subject }.to change(vulnerability_finding_pipelines, :count).from(16).to(8) - .and change(vulnerability_findings, :count).from(6).to(3) - .and change(vulnerabilities, :count).from(4).to(2) - - expect(vulnerability_findings.pluck(:id)).to match_array([finding_with_incorrect_uuid.id, finding_with_incorrect_uuid2.id, finding_with_incorrect_uuid3.id]) - end - - context 'if there are conflicting UUID values within the batch' do # rubocop: disable RSpec/MultipleMemoizedHelpers - let(:end_id) { finding_with_broken_data_integrity.id } - let(:vulnerability_5) { create_vulnerability!(project_id: project.id, author_id: user.id) } - let(:different_project) { table(:projects).create!(namespace_id: namespace.id) } - let!(:identifier_with_broken_data_integrity) do - vulnerability_identifiers.create!( - project_id: different_project.id, - external_type: identifier_2, - external_id: identifier_2, - fingerprint: Gitlab::Database::ShaAttribute.serialize('4299e8ddd819f9bde9cfacf45716724c17b5ddf7'), - name: 'Identifier 2') - end - - let(:finding_with_broken_data_integrity) do - create_finding!( - vulnerability_id: vulnerability_5, - project_id: project.id, - scanner_id: scanner.id, - primary_identifier_id: identifier_with_broken_data_integrity.id, - report_type: 0, # "sast" - location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis') - uuid: SecureRandom.uuid - ) - end - - it 'deletes the conflicting record' do - expect { subject }.to change { vulnerability_findings.find_by_id(finding_with_broken_data_integrity.id) }.to(nil) - end - end - - context 'if a conflicting UUID is found during the migration' do # rubocop:disable RSpec/MultipleMemoizedHelpers - let(:finding_class) { Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid::VulnerabilitiesFinding } - let(:uuid) { '4564f9d5-3c6b-5cc3-af8c-7c25285362a7' } - - before do - exception = ActiveRecord::RecordNotUnique.new("(uuid)=(#{uuid})") - - call_count = 0 - allow(::Gitlab::Database::BulkUpdate).to receive(:execute) do - call_count += 1 - call_count.eql?(1) ? raise(exception) : {} - end - - allow(finding_class).to receive(:find_by).with(uuid: uuid).and_return(duplicate_not_in_the_same_batch) - end - - it 'retries the recalculation' do - subject - - expect(Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid::VulnerabilitiesFinding) - .to have_received(:find_by).with(uuid: uuid).once - end - - it 'logs the conflict' do - expect_next_instance_of(Gitlab::BackgroundMigration::Logger) do |instance| - expect(instance).to receive(:info).exactly(6).times - end - - subject - end - - it 'marks the job as done' do - create_background_migration_job([start_id, end_id], :pending) - - subject - - expect(pending_jobs.count).to eq(0) - expect(succeeded_jobs.count).to eq(1) - end - end - - it 'logs an exception if a different uniquness problem was found' do - exception = ActiveRecord::RecordNotUnique.new("Totally not an UUID uniqueness problem") - allow(::Gitlab::Database::BulkUpdate).to receive(:execute).and_raise(exception) - allow(Gitlab::ErrorTracking).to receive(:track_and_raise_exception) - - subject - - expect(Gitlab::ErrorTracking).to have_received(:track_and_raise_exception).with(exception).once - end - - it 'logs a duplicate found message' do - expect_next_instance_of(Gitlab::BackgroundMigration::Logger) do |instance| - expect(instance).to receive(:info).exactly(3).times - end - - subject - end - end - - context 'when finding has a signature' do - before do - @f1 = create_finding!( - vulnerability_id: nil, - project_id: project.id, - scanner_id: scanner.id, - primary_identifier_id: vulnerability_identifier.id, - report_type: 0, # "sast" - location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis') - uuid: 'd15d774d-e4b1-5a1b-929b-19f2a53e35ec' - ) - - vulnerability_finding_signatures.create!( - finding_id: @f1.id, - algorithm_type: 2, # location - signature_sha: Gitlab::Database::ShaAttribute.serialize('57d4e05205f6462a73f039a5b2751aa1ab344e6e') # sha1('youshouldusethis') - ) - - vulnerability_finding_signatures.create!( - finding_id: @f1.id, - algorithm_type: 1, # hash - signature_sha: Gitlab::Database::ShaAttribute.serialize('c554d8d8df1a7a14319eafdaae24af421bf5b587') # sha1('andnotthis') - ) - - @f2 = create_finding!( - vulnerability_id: nil, - project_id: project.id, - scanner_id: scanner.id, - primary_identifier_id: vulnerability_identfier2.id, - report_type: 0, # "sast" - location_fingerprint: Gitlab::Database::ShaAttribute.serialize('ca41a2544e941a007a73a666cb0592b255316ab8'), # sha1('youshouldntusethis') - uuid: '4be029b5-75e5-5ac0-81a2-50ab41726135' - ) - - vulnerability_finding_signatures.create!( - finding_id: @f2.id, - algorithm_type: 2, # location - signature_sha: Gitlab::Database::ShaAttribute.serialize('57d4e05205f6462a73f039a5b2751aa1ab344e6e') # sha1('youshouldusethis') - ) - - vulnerability_finding_signatures.create!( - finding_id: @f2.id, - algorithm_type: 1, # hash - signature_sha: Gitlab::Database::ShaAttribute.serialize('c554d8d8df1a7a14319eafdaae24af421bf5b587') # sha1('andnotthis') - ) - end - - let(:start_id) { @f1.id } - let(:end_id) { @f2.id } - - let(:uuids_before) { [@f1.uuid, @f2.uuid] } - let(:uuids_after) { %w[d3b60ddd-d312-5606-b4d3-ad058eebeacb 349d9bec-c677-5530-a8ac-5e58889c3b1a] } - - it 'is recalculated using signature' do - expect(vulnerability_findings.pluck(:uuid)).to match_array(uuids_before) - - subject - - expect(vulnerability_findings.pluck(:uuid)).to match_array(uuids_after) - end - end - - context 'if all records are removed before the job ran' do - let(:start_id) { 1 } - let(:end_id) { 9 } - - before do - create_background_migration_job([start_id, end_id], :pending) - end - - it 'does not error out' do - expect { subject }.not_to raise_error - end - - it 'marks the job as done' do - subject - - expect(pending_jobs.count).to eq(0) - expect(succeeded_jobs.count).to eq(1) - end - end - - context 'when recalculation fails' do - before do - @uuid_v4 = create_finding!( - vulnerability_id: nil, - project_id: project.id, - scanner_id: scanner2.id, - primary_identifier_id: vulnerability_identfier2.id, - report_type: 0, # "sast" - location_fingerprint: Gitlab::Database::ShaAttribute.serialize("fa18f432f1d56675f4098d318739c3cd5b14eb3e"), - uuid: known_uuid_v4 - ) - - allow(Gitlab::ErrorTracking).to receive(:track_and_raise_exception) - allow(::Gitlab::Database::BulkUpdate).to receive(:execute).and_raise(expected_error) - end - - let(:start_id) { @uuid_v4.id } - let(:end_id) { @uuid_v4.id } - let(:expected_error) { RuntimeError.new } - - it 'captures the errors and does not crash entirely' do - expect { subject }.not_to raise_error - - allow(Gitlab::ErrorTracking).to receive(:track_and_raise_exception) - expect(Gitlab::ErrorTracking).to have_received(:track_and_raise_exception).with(expected_error).once - end - - it_behaves_like 'marks background migration job records' do - let(:arguments) { [1, 4] } - subject { described_class.new } - end - end - - it_behaves_like 'marks background migration job records' do - let(:arguments) { [1, 4] } - subject { described_class.new } - end - - private - - def create_vulnerability!(project_id:, author_id:, title: 'test', severity: 7, confidence: 7, report_type: 0) - vulnerabilities.create!( - project_id: project_id, - author_id: author_id, - title: title, - severity: severity, - confidence: confidence, - report_type: report_type - ) - end - - # rubocop:disable Metrics/ParameterLists - def create_finding!( - vulnerability_id:, project_id:, scanner_id:, primary_identifier_id:, id: nil, - name: "test", severity: 7, confidence: 7, report_type: 0, - project_fingerprint: '123qweasdzxc', location_fingerprint: 'test', - metadata_version: 'test', raw_metadata: 'test', uuid: SecureRandom.uuid) - vulnerability_findings.create!({ - id: id, - vulnerability_id: vulnerability_id, - project_id: project_id, - name: name, - severity: severity, - confidence: confidence, - report_type: report_type, - project_fingerprint: project_fingerprint, - scanner_id: scanner_id, - primary_identifier_id: primary_identifier_id, - location_fingerprint: location_fingerprint, - metadata_version: metadata_version, - raw_metadata: raw_metadata, - uuid: uuid - }.compact - ) - end - # rubocop:enable Metrics/ParameterLists - - def create_user!(name: "Example User", email: "user@example.com", user_type: nil, created_at: Time.zone.now, confirmed_at: Time.zone.now) - users.create!( - name: name, - email: email, - username: name, - projects_limit: 0, - user_type: user_type, - confirmed_at: confirmed_at - ) - end - - def create_finding_pipeline!(project_id:, finding_id:) - pipeline = table(:ci_pipelines).create!(project_id: project_id) - vulnerability_finding_pipelines.create!(pipeline_id: pipeline.id, occurrence_id: finding_id) - end -end diff --git a/spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb b/spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb index 5fede892463..582c0fe1b1b 100644 --- a/spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb +++ b/spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb @@ -86,8 +86,10 @@ RSpec.describe Gitlab::BackgroundMigration::RemoveBackfilledJobArtifactsExpireAt def create_job_artifact(id:, file_type:, expire_at:) job = table(:ci_builds, database: :ci).create!(id: id, partition_id: 100) - job_artifact.create!(id: id, job_id: job.id, expire_at: expire_at, project_id: project.id, - file_type: file_type, partition_id: 100) + job_artifact.create!( + id: id, job_id: job.id, expire_at: expire_at, project_id: project.id, + file_type: file_type, partition_id: 100 + ) end end end diff --git a/spec/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings_spec.rb b/spec/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings_spec.rb deleted file mode 100644 index ed08ae22245..00000000000 --- a/spec/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings_spec.rb +++ /dev/null @@ -1,171 +0,0 @@ -# frozen_string_literal: true -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::RemoveDuplicateVulnerabilitiesFindings, :migration, schema: 20220326161803 do - let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') } - let(:users) { table(:users) } - let(:user) { create_user! } - let(:project) { table(:projects).create!(id: 14219619, namespace_id: namespace.id) } - let(:scanners) { table(:vulnerability_scanners) } - let!(:scanner1) { scanners.create!(project_id: project.id, external_id: 'test 1', name: 'test scanner 1') } - let!(:scanner2) { scanners.create!(project_id: project.id, external_id: 'test 2', name: 'test scanner 2') } - let!(:scanner3) { scanners.create!(project_id: project.id, external_id: 'test 3', name: 'test scanner 3') } - let!(:unrelated_scanner) { scanners.create!(project_id: project.id, external_id: 'unreleated_scanner', name: 'unrelated scanner') } - let(:vulnerabilities) { table(:vulnerabilities) } - let(:vulnerability_findings) { table(:vulnerability_occurrences) } - let(:vulnerability_identifiers) { table(:vulnerability_identifiers) } - let(:vulnerability_identifier) do - vulnerability_identifiers.create!( - id: 1244459, - project_id: project.id, - external_type: 'vulnerability-identifier', - external_id: 'vulnerability-identifier', - fingerprint: '0a203e8cd5260a1948edbedc76c7cb91ad6a2e45', - name: 'vulnerability identifier') - end - - let!(:vulnerability_for_first_duplicate) do - create_vulnerability!( - project_id: project.id, - author_id: user.id - ) - end - - let!(:first_finding_duplicate) do - create_finding!( - id: 5606961, - uuid: "bd95c085-71aa-51d7-9bb6-08ae669c262e", - vulnerability_id: vulnerability_for_first_duplicate.id, - report_type: 0, - location_fingerprint: '00049d5119c2cb3bfb3d1ee1f6e031fe925aed75', - primary_identifier_id: vulnerability_identifier.id, - scanner_id: scanner1.id, - project_id: project.id - ) - end - - let!(:vulnerability_for_second_duplicate) do - create_vulnerability!( - project_id: project.id, - author_id: user.id - ) - end - - let!(:second_finding_duplicate) do - create_finding!( - id: 8765432, - uuid: "5b714f58-1176-5b26-8fd5-e11dfcb031b5", - vulnerability_id: vulnerability_for_second_duplicate.id, - report_type: 0, - location_fingerprint: '00049d5119c2cb3bfb3d1ee1f6e031fe925aed75', - primary_identifier_id: vulnerability_identifier.id, - scanner_id: scanner2.id, - project_id: project.id - ) - end - - let!(:vulnerability_for_third_duplicate) do - create_vulnerability!( - project_id: project.id, - author_id: user.id - ) - end - - let!(:third_finding_duplicate) do - create_finding!( - id: 8832995, - uuid: "cfe435fa-b25b-5199-a56d-7b007cc9e2d4", - vulnerability_id: vulnerability_for_third_duplicate.id, - report_type: 0, - location_fingerprint: '00049d5119c2cb3bfb3d1ee1f6e031fe925aed75', - primary_identifier_id: vulnerability_identifier.id, - scanner_id: scanner3.id, - project_id: project.id - ) - end - - let!(:unrelated_finding) do - create_finding!( - id: 9999999, - uuid: Gitlab::UUID.v5(SecureRandom.hex), - vulnerability_id: nil, - report_type: 1, - location_fingerprint: 'random_location_fingerprint', - primary_identifier_id: vulnerability_identifier.id, - scanner_id: unrelated_scanner.id, - project_id: project.id - ) - end - - subject { described_class.new.perform(first_finding_duplicate.id, unrelated_finding.id) } - - before do - stub_const("#{described_class}::DELETE_BATCH_SIZE", 1) - end - - it "removes entries which would result in duplicate UUIDv5" do - expect(vulnerability_findings.count).to eq(4) - - expect { subject }.to change { vulnerability_findings.count }.from(4).to(2) - - expect(vulnerability_findings.pluck(:id)).to match_array([third_finding_duplicate.id, unrelated_finding.id]) - end - - it "removes vulnerabilites without findings" do - expect(vulnerabilities.count).to eq(3) - - expect { subject }.to change { vulnerabilities.count }.from(3).to(1) - - expect(vulnerabilities.pluck(:id)).to match_array([vulnerability_for_third_duplicate.id]) - end - - private - - def create_vulnerability!(project_id:, author_id:, title: 'test', severity: 7, confidence: 7, report_type: 0) - vulnerabilities.create!( - project_id: project_id, - author_id: author_id, - title: title, - severity: severity, - confidence: confidence, - report_type: report_type - ) - end - - # rubocop:disable Metrics/ParameterLists - def create_finding!( - vulnerability_id:, project_id:, scanner_id:, primary_identifier_id:, id: nil, - name: "test", severity: 7, confidence: 7, report_type: 0, - project_fingerprint: '123qweasdzxc', location_fingerprint: 'test', - metadata_version: 'test', raw_metadata: 'test', uuid: SecureRandom.uuid) - params = { - vulnerability_id: vulnerability_id, - project_id: project_id, - name: name, - severity: severity, - confidence: confidence, - report_type: report_type, - project_fingerprint: project_fingerprint, - scanner_id: scanner_id, - primary_identifier_id: vulnerability_identifier.id, - location_fingerprint: location_fingerprint, - metadata_version: metadata_version, - raw_metadata: raw_metadata, - uuid: uuid - } - params[:id] = id unless id.nil? - vulnerability_findings.create!(params) - end - # rubocop:enable Metrics/ParameterLists - - def create_user!(name: "Example User", email: "user@example.com", user_type: nil, created_at: Time.zone.now, confirmed_at: Time.zone.now) - users.create!( - name: name, - email: email, - username: name, - projects_limit: 0, - user_type: user_type, - confirmed_at: confirmed_at - ) - end -end diff --git a/spec/lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings_spec.rb b/spec/lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings_spec.rb index 1844347f4a9..60ee61cf50a 100644 --- a/spec/lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings_spec.rb +++ b/spec/lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' RSpec.describe Gitlab::BackgroundMigration::RemoveOccurrencePipelinesAndDuplicateVulnerabilitiesFindings, :migration, - :suppress_gitlab_schemas_validate_connection, schema: 20220326161803 do + :suppress_gitlab_schemas_validate_connection, schema: 20220326161803 do let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') } let(:users) { table(:users) } let(:user) { create_user! } diff --git a/spec/lib/gitlab/background_migration/remove_project_group_link_with_missing_groups_spec.rb b/spec/lib/gitlab/background_migration/remove_project_group_link_with_missing_groups_spec.rb new file mode 100644 index 00000000000..c45c402ab9d --- /dev/null +++ b/spec/lib/gitlab/background_migration/remove_project_group_link_with_missing_groups_spec.rb @@ -0,0 +1,124 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::RemoveProjectGroupLinkWithMissingGroups, :migration, + feature_category: :subgroups, schema: 20230206172702 do + let(:projects) { table(:projects) } + let(:namespaces) { table(:namespaces) } + let(:project_group_links) { table(:project_group_links) } + + let!(:group) do + namespaces.create!( + name: 'Group0', type: 'Group', path: 'space0' + ) + end + + let!(:group_1) do + namespaces.create!( + name: 'Group1', type: 'Group', path: 'space1' + ) + end + + let!(:group_2) do + namespaces.create!( + name: 'Group2', type: 'Group', path: 'space2' + ) + end + + let!(:group_3) do + namespaces.create!( + name: 'Group3', type: 'Group', path: 'space3' + ) + end + + let!(:project_namespace_1) do + namespaces.create!( + name: 'project_1', path: 'project_1', type: 'Project' + ) + end + + let!(:project_namespace_2) do + namespaces.create!( + name: 'project_2', path: 'project_2', type: 'Project' + ) + end + + let!(:project_namespace_3) do + namespaces.create!( + name: 'project_3', path: 'project_3', type: 'Project' + ) + end + + let!(:project_1) do + projects.create!( + name: 'project_1', path: 'project_1', namespace_id: group.id, project_namespace_id: project_namespace_1.id + ) + end + + let!(:project_2) do + projects.create!( + name: 'project_2', path: 'project_2', namespace_id: group.id, project_namespace_id: project_namespace_2.id + ) + end + + let!(:project_3) do + projects.create!( + name: 'project_3', path: 'project_3', namespace_id: group.id, project_namespace_id: project_namespace_3.id + ) + end + + let!(:project_group_link_1) do + project_group_links.create!( + project_id: project_1.id, group_id: group_1.id, group_access: Gitlab::Access::DEVELOPER + ) + end + + let!(:project_group_link_2) do + project_group_links.create!( + project_id: project_2.id, group_id: group_2.id, group_access: Gitlab::Access::DEVELOPER + ) + end + + let!(:project_group_link_3) do + project_group_links.create!( + project_id: project_3.id, group_id: group_3.id, group_access: Gitlab::Access::DEVELOPER + ) + end + + let!(:project_group_link_4) do + project_group_links.create!( + project_id: project_3.id, group_id: group_2.id, group_access: Gitlab::Access::DEVELOPER + ) + end + + subject do + described_class.new( + start_id: project_group_link_1.id, + end_id: project_group_link_4.id, + batch_table: :project_group_links, + batch_column: :id, + sub_batch_size: 1, + pause_ms: 0, + connection: ApplicationRecord.connection + ).perform + end + + it 'removes the `project_group_links` records whose associated group does not exist anymore' do + group_2.delete + + # Schema is fixed to `20230206172702` on this spec. + # This expectation is needed to make sure that the orphaned records are indeed deleted via the migration + # and not via the foreign_key relationship introduced after `20230206172702`, in `20230207002330` + expect(project_group_links.count).to eq(4) + + expect { subject } + .to change { project_group_links.count }.from(4).to(2) + .and change { + project_group_links.where(project_id: project_2.id, group_id: group_2.id).present? + }.from(true).to(false) + .and change { + project_group_links.where(project_id: project_3.id, group_id: group_2.id).present? + }.from(true).to(false) + end +end diff --git a/spec/lib/gitlab/background_migration/remove_self_managed_wiki_notes_spec.rb b/spec/lib/gitlab/background_migration/remove_self_managed_wiki_notes_spec.rb index 81927100562..59d5d56ebe8 100644 --- a/spec/lib/gitlab/background_migration/remove_self_managed_wiki_notes_spec.rb +++ b/spec/lib/gitlab/background_migration/remove_self_managed_wiki_notes_spec.rb @@ -6,14 +6,15 @@ RSpec.describe Gitlab::BackgroundMigration::RemoveSelfManagedWikiNotes, :migrati let(:notes) { table(:notes) } subject(:perform_migration) do - described_class.new(start_id: 1, - end_id: 30, - batch_table: :notes, - batch_column: :id, - sub_batch_size: 2, - pause_ms: 0, - connection: ActiveRecord::Base.connection) - .perform + described_class.new( + start_id: 1, + end_id: 30, + batch_table: :notes, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ).perform end it 'removes all wiki notes' do diff --git a/spec/lib/gitlab/background_migration/remove_vulnerability_finding_links_spec.rb b/spec/lib/gitlab/background_migration/remove_vulnerability_finding_links_spec.rb index 918df8f4442..32134b99e37 100644 --- a/spec/lib/gitlab/background_migration/remove_vulnerability_finding_links_spec.rb +++ b/spec/lib/gitlab/background_migration/remove_vulnerability_finding_links_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::RemoveVulnerabilityFindingLinks, :migration, schema: 20211104165220 do +RSpec.describe Gitlab::BackgroundMigration::RemoveVulnerabilityFindingLinks, :migration, schema: 20211202041233 do let(:vulnerability_findings) { table(:vulnerability_occurrences) } let(:finding_links) { table(:vulnerability_finding_links) } diff --git a/spec/lib/gitlab/background_migration/reset_too_many_tags_skipped_registry_imports_spec.rb b/spec/lib/gitlab/background_migration/reset_too_many_tags_skipped_registry_imports_spec.rb index 3f59b0a24a3..afdd855c5a8 100644 --- a/spec/lib/gitlab/background_migration/reset_too_many_tags_skipped_registry_imports_spec.rb +++ b/spec/lib/gitlab/background_migration/reset_too_many_tags_skipped_registry_imports_spec.rb @@ -3,8 +3,8 @@ require 'spec_helper' RSpec.describe Gitlab::BackgroundMigration::ResetTooManyTagsSkippedRegistryImports, :migration, - :aggregate_failures, - schema: 20220502173045 do + :aggregate_failures, + schema: 20220502173045 do let(:namespaces) { table(:namespaces) } let(:projects) { table(:projects) } let(:container_repositories) { table(:container_repositories) } @@ -15,46 +15,54 @@ RSpec.describe Gitlab::BackgroundMigration::ResetTooManyTagsSkippedRegistryImpor let!(:project) { projects.create!(id: 1, project_namespace_id: 1, namespace_id: 1, path: 'bar', name: 'bar') } let!(:container_repository1) do - container_repositories.create!(id: 1, - project_id: 1, - name: 'a', - migration_state: 'import_skipped', - migration_skipped_at: Time.zone.now, - migration_skipped_reason: 2, - migration_pre_import_started_at: Time.zone.now, - migration_pre_import_done_at: Time.zone.now, - migration_import_started_at: Time.zone.now, - migration_import_done_at: Time.zone.now, - migration_aborted_at: Time.zone.now, - migration_retries_count: 2, - migration_aborted_in_state: 'importing') + container_repositories.create!( + id: 1, + project_id: 1, + name: 'a', + migration_state: 'import_skipped', + migration_skipped_at: Time.zone.now, + migration_skipped_reason: 2, + migration_pre_import_started_at: Time.zone.now, + migration_pre_import_done_at: Time.zone.now, + migration_import_started_at: Time.zone.now, + migration_import_done_at: Time.zone.now, + migration_aborted_at: Time.zone.now, + migration_retries_count: 2, + migration_aborted_in_state: 'importing' + ) end let!(:container_repository2) do - container_repositories.create!(id: 2, - project_id: 1, - name: 'b', - migration_state: 'import_skipped', - migration_skipped_at: Time.zone.now, - migration_skipped_reason: 2) + container_repositories.create!( + id: 2, + project_id: 1, + name: 'b', + migration_state: 'import_skipped', + migration_skipped_at: Time.zone.now, + migration_skipped_reason: 2 + ) end let!(:container_repository3) do - container_repositories.create!(id: 3, - project_id: 1, - name: 'c', - migration_state: 'import_skipped', - migration_skipped_at: Time.zone.now, - migration_skipped_reason: 1) + container_repositories.create!( + id: 3, + project_id: 1, + name: 'c', + migration_state: 'import_skipped', + migration_skipped_at: Time.zone.now, + migration_skipped_reason: 1 + ) end # This is an unlikely state, but included here to test the edge case let!(:container_repository4) do - container_repositories.create!(id: 4, - project_id: 1, - name: 'd', - migration_state: 'default', - migration_skipped_reason: 2) + container_repositories.create!( + id: 4, + project_id: 1, + name: 'd', + migration_state: 'default', + migration_skipped_reason: 2 + ) end describe '#up' do diff --git a/spec/lib/gitlab/background_migration/set_correct_vulnerability_state_spec.rb b/spec/lib/gitlab/background_migration/set_correct_vulnerability_state_spec.rb index 2372ce21c4c..df1ee494987 100644 --- a/spec/lib/gitlab/background_migration/set_correct_vulnerability_state_spec.rb +++ b/spec/lib/gitlab/background_migration/set_correct_vulnerability_state_spec.rb @@ -35,13 +35,15 @@ RSpec.describe Gitlab::BackgroundMigration::SetCorrectVulnerabilityState do let(:dismissed_state) { 2 } let(:migration_job) do - described_class.new(start_id: vulnerability_with_dismissed_at.id, - end_id: vulnerability_without_dismissed_at.id, - batch_table: :vulnerabilities, - batch_column: :id, - sub_batch_size: 1, - pause_ms: 0, - connection: ActiveRecord::Base.connection) + described_class.new( + start_id: vulnerability_with_dismissed_at.id, + end_id: vulnerability_without_dismissed_at.id, + batch_table: :vulnerabilities, + batch_column: :id, + sub_batch_size: 1, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ) end describe '#filter_batch' do diff --git a/spec/lib/gitlab/background_migration/set_legacy_open_source_license_available_for_non_public_projects_spec.rb b/spec/lib/gitlab/background_migration/set_legacy_open_source_license_available_for_non_public_projects_spec.rb index e9f73672144..5109c3ec0c2 100644 --- a/spec/lib/gitlab/background_migration/set_legacy_open_source_license_available_for_non_public_projects_spec.rb +++ b/spec/lib/gitlab/background_migration/set_legacy_open_source_license_available_for_non_public_projects_spec.rb @@ -3,21 +3,22 @@ require 'spec_helper' RSpec.describe Gitlab::BackgroundMigration::SetLegacyOpenSourceLicenseAvailableForNonPublicProjects, - :migration, - schema: 20220722110026 do + :migration, + schema: 20220722110026 do let(:namespaces_table) { table(:namespaces) } let(:projects_table) { table(:projects) } let(:project_settings_table) { table(:project_settings) } subject(:perform_migration) do - described_class.new(start_id: projects_table.minimum(:id), - end_id: projects_table.maximum(:id), - batch_table: :projects, - batch_column: :id, - sub_batch_size: 2, - pause_ms: 0, - connection: ActiveRecord::Base.connection) - .perform + described_class.new( + start_id: projects_table.minimum(:id), + end_id: projects_table.maximum(:id), + batch_table: :projects, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ).perform end it 'sets `legacy_open_source_license_available` attribute to false for non-public projects', :aggregate_failures do @@ -37,11 +38,13 @@ RSpec.describe Gitlab::BackgroundMigration::SetLegacyOpenSourceLicenseAvailableF def create_legacy_license_project(path, visibility_level:) namespace = namespaces_table.create!(name: "namespace-#{path}", path: "namespace-#{path}") project_namespace = namespaces_table.create!(name: "project-namespace-#{path}", path: path, type: 'Project') - project = projects_table.create!(name: path, - path: path, - namespace_id: namespace.id, - project_namespace_id: project_namespace.id, - visibility_level: visibility_level) + project = projects_table.create!( + name: path, + path: path, + namespace_id: namespace.id, + project_namespace_id: project_namespace.id, + visibility_level: visibility_level + ) project_settings_table.create!(project_id: project.id, legacy_open_source_license_available: true) project diff --git a/spec/lib/gitlab/background_migration/steal_migrate_merge_request_diff_commit_users_spec.rb b/spec/lib/gitlab/background_migration/steal_migrate_merge_request_diff_commit_users_spec.rb deleted file mode 100644 index 841a7f306d7..00000000000 --- a/spec/lib/gitlab/background_migration/steal_migrate_merge_request_diff_commit_users_spec.rb +++ /dev/null @@ -1,50 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::StealMigrateMergeRequestDiffCommitUsers, schema: 20211012134316 do - let(:migration) { described_class.new } - - describe '#perform' do - it 'processes the background migration' do - spy = instance_spy( - Gitlab::BackgroundMigration::MigrateMergeRequestDiffCommitUsers - ) - - allow(Gitlab::BackgroundMigration::MigrateMergeRequestDiffCommitUsers) - .to receive(:new) - .and_return(spy) - - expect(spy).to receive(:perform).with(1, 4) - expect(migration).to receive(:schedule_next_job) - - migration.perform(1, 4) - end - end - - describe '#schedule_next_job' do - it 'schedules the next job in ascending order' do - Gitlab::Database::BackgroundMigrationJob.create!( - class_name: 'MigrateMergeRequestDiffCommitUsers', - arguments: [10, 20] - ) - - Gitlab::Database::BackgroundMigrationJob.create!( - class_name: 'MigrateMergeRequestDiffCommitUsers', - arguments: [40, 50] - ) - - expect(BackgroundMigrationWorker) - .to receive(:perform_in) - .with(5.minutes, 'StealMigrateMergeRequestDiffCommitUsers', [10, 20]) - - migration.schedule_next_job - end - - it 'does not schedule any new jobs when there are none' do - expect(BackgroundMigrationWorker).not_to receive(:perform_in) - - migration.schedule_next_job - end - end -end diff --git a/spec/lib/gitlab/background_migration/update_delayed_project_removal_to_null_for_user_namespaces_spec.rb b/spec/lib/gitlab/background_migration/update_delayed_project_removal_to_null_for_user_namespaces_spec.rb index 980a7771f4c..0579a299c74 100644 --- a/spec/lib/gitlab/background_migration/update_delayed_project_removal_to_null_for_user_namespaces_spec.rb +++ b/spec/lib/gitlab/background_migration/update_delayed_project_removal_to_null_for_user_namespaces_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::BackgroundMigration::UpdateDelayedProjectRemovalToNullForUserNamespaces, - :migration do + :migration do let(:namespaces_table) { table(:namespaces) } let(:namespace_settings_table) { table(:namespace_settings) } diff --git a/spec/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url_spec.rb b/spec/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url_spec.rb index c090c1df424..75fe5699986 100644 --- a/spec/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url_spec.rb +++ b/spec/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url_spec.rb @@ -13,10 +13,12 @@ RSpec.describe Gitlab::BackgroundMigration::UpdateJiraTrackerDataDeploymentTypeB let(:sub_batch_size) { 1 } let(:pause_ms) { 0 } let(:migration) do - described_class.new(start_id: 1, end_id: 10, - batch_table: table_name, batch_column: batch_column, - sub_batch_size: sub_batch_size, pause_ms: pause_ms, - connection: ApplicationRecord.connection) + described_class.new( + start_id: 1, end_id: 10, + batch_table: table_name, batch_column: batch_column, + sub_batch_size: sub_batch_size, pause_ms: pause_ms, + connection: ApplicationRecord.connection + ) end subject(:perform_migration) do diff --git a/spec/lib/gitlab/background_migration/update_timelogs_project_id_spec.rb b/spec/lib/gitlab/background_migration/update_timelogs_project_id_spec.rb deleted file mode 100644 index b8c3bf8f3ac..00000000000 --- a/spec/lib/gitlab/background_migration/update_timelogs_project_id_spec.rb +++ /dev/null @@ -1,52 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::UpdateTimelogsProjectId, schema: 20210826171758 do - let!(:namespace) { table(:namespaces).create!(name: 'namespace', path: 'namespace') } - let!(:project1) { table(:projects).create!(namespace_id: namespace.id) } - let!(:project2) { table(:projects).create!(namespace_id: namespace.id) } - let!(:issue1) { table(:issues).create!(project_id: project1.id) } - let!(:issue2) { table(:issues).create!(project_id: project2.id) } - let!(:merge_request1) { table(:merge_requests).create!(target_project_id: project1.id, source_branch: 'master', target_branch: 'feature') } - let!(:merge_request2) { table(:merge_requests).create!(target_project_id: project2.id, source_branch: 'master', target_branch: 'feature') } - let!(:timelog1) { table(:timelogs).create!(issue_id: issue1.id, time_spent: 60) } - let!(:timelog2) { table(:timelogs).create!(issue_id: issue1.id, time_spent: 60) } - let!(:timelog3) { table(:timelogs).create!(issue_id: issue2.id, time_spent: 60) } - let!(:timelog4) { table(:timelogs).create!(merge_request_id: merge_request1.id, time_spent: 600) } - let!(:timelog5) { table(:timelogs).create!(merge_request_id: merge_request1.id, time_spent: 600) } - let!(:timelog6) { table(:timelogs).create!(merge_request_id: merge_request2.id, time_spent: 600) } - let!(:timelog7) { table(:timelogs).create!(issue_id: issue2.id, time_spent: 60, project_id: project1.id) } - let!(:timelog8) { table(:timelogs).create!(merge_request_id: merge_request2.id, time_spent: 600, project_id: project1.id) } - - describe '#perform' do - context 'when timelogs belong to issues' do - it 'sets correct project_id' do - subject.perform(timelog1.id, timelog3.id) - - expect(timelog1.reload.project_id).to eq(issue1.project_id) - expect(timelog2.reload.project_id).to eq(issue1.project_id) - expect(timelog3.reload.project_id).to eq(issue2.project_id) - end - end - - context 'when timelogs belong to merge requests' do - it 'sets correct project ids' do - subject.perform(timelog4.id, timelog6.id) - - expect(timelog4.reload.project_id).to eq(merge_request1.target_project_id) - expect(timelog5.reload.project_id).to eq(merge_request1.target_project_id) - expect(timelog6.reload.project_id).to eq(merge_request2.target_project_id) - end - end - - context 'when timelogs already belong to projects' do - it 'does not update the project id' do - subject.perform(timelog7.id, timelog8.id) - - expect(timelog7.reload.project_id).to eq(project1.id) - expect(timelog8.reload.project_id).to eq(project1.id) - end - end - end -end diff --git a/spec/lib/gitlab/background_migration/update_users_where_two_factor_auth_required_from_group_spec.rb b/spec/lib/gitlab/background_migration/update_users_where_two_factor_auth_required_from_group_spec.rb deleted file mode 100644 index f16ae489b78..00000000000 --- a/spec/lib/gitlab/background_migration/update_users_where_two_factor_auth_required_from_group_spec.rb +++ /dev/null @@ -1,84 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::UpdateUsersWhereTwoFactorAuthRequiredFromGroup, :migration, schema: 20210826171758 do - include MigrationHelpers::NamespacesHelpers - - let(:group_with_2fa_parent) { create_namespace('parent', Gitlab::VisibilityLevel::PRIVATE, require_two_factor_authentication: true) } - let(:group_with_2fa_child) { create_namespace('child', Gitlab::VisibilityLevel::PRIVATE, parent_id: group_with_2fa_parent.id) } - let(:members_table) { table(:members) } - let(:users_table) { table(:users) } - - subject { described_class.new } - - describe '#perform' do - context 'with group members' do - let(:user_1) { create_user('user@example.com') } - let!(:member) { create_group_member(user_1, group_with_2fa_parent) } - let!(:user_without_group) { create_user('user_without@example.com') } - let(:user_other) { create_user('user_other@example.com') } - let!(:member_other) { create_group_member(user_other, group_with_2fa_parent) } - - it 'updates user when user should be required to establish two factor authentication' do - subject.perform(user_1.id, user_without_group.id) - - expect(user_1.reload.require_two_factor_authentication_from_group).to eq(true) - end - - it 'does not update user who is not in current batch' do - subject.perform(user_1.id, user_without_group.id) - - expect(user_other.reload.require_two_factor_authentication_from_group).to eq(false) - end - - it 'updates all users in current batch' do - subject.perform(user_1.id, user_other.id) - - expect(user_other.reload.require_two_factor_authentication_from_group).to eq(true) - end - - it 'updates user when user is member of group in which parent group requires two factor authentication' do - member.destroy! - - subgroup = create_namespace('subgroup', Gitlab::VisibilityLevel::PRIVATE, require_two_factor_authentication: false, parent_id: group_with_2fa_child.id) - create_group_member(user_1, subgroup) - - subject.perform(user_1.id, user_other.id) - - expect(user_1.reload.require_two_factor_authentication_from_group).to eq(true) - end - - it 'updates user when user is member of a group and the subgroup requires two factor authentication' do - member.destroy! - - parent = create_namespace('other_parent', Gitlab::VisibilityLevel::PRIVATE, require_two_factor_authentication: false) - create_namespace('other_subgroup', Gitlab::VisibilityLevel::PRIVATE, require_two_factor_authentication: true, parent_id: parent.id) - create_group_member(user_1, parent) - - subject.perform(user_1.id, user_other.id) - - expect(user_1.reload.require_two_factor_authentication_from_group).to eq(true) - end - - it 'does not update user when not a member of a group that requires two factor authentication' do - member_other.destroy! - - other_group = create_namespace('other_group', Gitlab::VisibilityLevel::PRIVATE, require_two_factor_authentication: false) - create_group_member(user_other, other_group) - - subject.perform(user_1.id, user_other.id) - - expect(user_other.reload.require_two_factor_authentication_from_group).to eq(false) - end - end - end - - def create_user(email, require_2fa: false) - users_table.create!(email: email, projects_limit: 10, require_two_factor_authentication_from_group: require_2fa) - end - - def create_group_member(user, group) - members_table.create!(user_id: user.id, source_id: group.id, access_level: GroupMember::MAINTAINER, source_type: "Namespace", type: "GroupMember", notification_level: 3) - end -end diff --git a/spec/lib/gitlab/background_task_spec.rb b/spec/lib/gitlab/background_task_spec.rb index 102556b6b2f..da92fc9e765 100644 --- a/spec/lib/gitlab/background_task_spec.rb +++ b/spec/lib/gitlab/background_task_spec.rb @@ -1,10 +1,10 @@ # frozen_string_literal: true -require 'fast_spec_helper' +require 'spec_helper' # We need to capture task state from a closure, which requires instance variables. # rubocop: disable RSpec/InstanceVariable -RSpec.describe Gitlab::BackgroundTask do +RSpec.describe Gitlab::BackgroundTask, feature_category: :build do let(:options) { {} } let(:task) do proc do diff --git a/spec/lib/gitlab/bare_repository_import/importer_spec.rb b/spec/lib/gitlab/bare_repository_import/importer_spec.rb deleted file mode 100644 index 3a885d70eb4..00000000000 --- a/spec/lib/gitlab/bare_repository_import/importer_spec.rb +++ /dev/null @@ -1,197 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BareRepositoryImport::Importer do - let!(:admin) { create(:admin) } - let!(:base_dir) { Dir.mktmpdir + '/' } - let(:bare_repository) { Gitlab::BareRepositoryImport::Repository.new(base_dir, File.join(base_dir, "#{project_path}.git")) } - let(:gitlab_shell) { Gitlab::Shell.new } - let(:source_project) { TestEnv.factory_repo_bundle_path } - - subject(:importer) { described_class.new(admin, bare_repository) } - - before do - allow(described_class).to receive(:log) - end - - after do - FileUtils.rm_rf(base_dir) - end - - shared_examples 'importing a repository' do - describe '.execute' do - it 'creates a project for a repository in storage' do - FileUtils.mkdir_p(File.join(base_dir, "#{project_path}.git")) - fake_importer = double - - expect(described_class).to receive(:new).and_return(fake_importer) - expect(fake_importer).to receive(:create_project_if_needed) - - described_class.execute(base_dir) - end - - it 'skips wiki repos' do - repo_dir = File.join(base_dir, 'the-group', 'the-project.wiki.git') - FileUtils.mkdir_p(File.join(repo_dir)) - - expect(described_class).to receive(:log).with(" * Skipping repo #{repo_dir}") - expect(described_class).not_to receive(:new) - - described_class.execute(base_dir) - end - - context 'without admin users' do - let(:admin) { nil } - - it 'raises an error' do - expect { described_class.execute(base_dir) }.to raise_error(Gitlab::BareRepositoryImport::Importer::NoAdminError) - end - end - end - - describe '#create_project_if_needed' do - it 'starts an import for a project that did not exist' do - expect(importer).to receive(:create_project) - - importer.create_project_if_needed - end - - it 'skips importing when the project already exists' do - project = create(:project, path: 'a-project', namespace: existing_group) - - expect(importer).not_to receive(:create_project) - expect(importer).to receive(:log).with(" * #{project.name} (#{project_path}) exists") - - importer.create_project_if_needed - end - - it 'creates a project with the correct path in the database' do - importer.create_project_if_needed - - expect(Project.find_by_full_path(project_path)).not_to be_nil - end - - it 'does not schedule an import' do - expect_next_instance_of(Project) do |instance| - expect(instance).not_to receive(:import_schedule) - end - - importer.create_project_if_needed - end - - it 'creates the Git repo on disk' do - prepare_repository("#{project_path}.git", source_project) - - importer.create_project_if_needed - - project = Project.find_by_full_path(project_path) - repo_path = "#{project.disk_path}.git" - - expect(gitlab_shell.repository_exists?(project.repository_storage, repo_path)).to be(true) - end - - context 'hashed storage enabled' do - it 'creates a project with the correct path in the database' do - stub_application_setting(hashed_storage_enabled: true) - - importer.create_project_if_needed - - expect(Project.find_by_full_path(project_path)).not_to be_nil - end - end - end - end - - context 'with subgroups' do - let(:project_path) { 'a-group/a-sub-group/a-project' } - - let(:existing_group) do - group = create(:group, path: 'a-group') - create(:group, path: 'a-sub-group', parent: group) - end - - it_behaves_like 'importing a repository' - end - - context 'without subgroups' do - let(:project_path) { 'a-group/a-project' } - let(:existing_group) { create(:group, path: 'a-group') } - - it_behaves_like 'importing a repository' - end - - context 'without groups' do - let(:project_path) { 'a-project' } - - it 'starts an import for a project that did not exist' do - expect(importer).to receive(:create_project) - - importer.create_project_if_needed - end - - it 'creates a project with the correct path in the database' do - importer.create_project_if_needed - - expect(Project.find_by_full_path("#{admin.full_path}/#{project_path}")).not_to be_nil - end - - it 'creates the Git repo in disk' do - prepare_repository("#{project_path}.git", source_project) - - importer.create_project_if_needed - - project = Project.find_by_full_path("#{admin.full_path}/#{project_path}") - - expect(gitlab_shell.repository_exists?(project.repository_storage, project.disk_path + '.git')).to be(true) - expect(gitlab_shell.repository_exists?(project.repository_storage, project.disk_path + '.wiki.git')).to be(true) - end - - context 'with a repository already on disk' do - # This is a quick way to get a valid repository instead of copying an - # existing one. Since it's not persisted, the importer will try to - # create the project. - let(:project) { build(:project, :legacy_storage, :repository) } - let(:project_path) { project.full_path } - - it 'moves an existing project to the correct path' do - original_commit_count = project.repository.commit_count - - expect(importer).to receive(:create_project).and_call_original - - new_project = importer.create_project_if_needed - - expect(new_project.repository.commit_count).to eq(original_commit_count) - end - end - end - - context 'with Wiki' do - let(:project_path) { 'a-group/a-project' } - let(:existing_group) { create(:group, path: 'a-group') } - - it_behaves_like 'importing a repository' - - it 'creates the Wiki git repo in disk' do - prepare_repository("#{project_path}.git", source_project) - prepare_repository("#{project_path}.wiki.git", source_project) - - expect(Projects::CreateService).to receive(:new).with(admin, hash_including(skip_wiki: true, - import_type: 'bare_repository')).and_call_original - - importer.create_project_if_needed - - project = Project.find_by_full_path(project_path) - - expect(gitlab_shell.repository_exists?(project.repository_storage, project.disk_path + '.wiki.git')).to be(true) - end - end - - def prepare_repository(project_path, source_project) - repo_path = File.join(base_dir, project_path) - - cmd = %W(#{Gitlab.config.git.bin_path} clone --bare #{source_project} #{repo_path}) - - system(git_env, *cmd, chdir: base_dir, out: '/dev/null', err: '/dev/null') - end -end diff --git a/spec/lib/gitlab/bare_repository_import/repository_spec.rb b/spec/lib/gitlab/bare_repository_import/repository_spec.rb deleted file mode 100644 index a9778e0e8a7..00000000000 --- a/spec/lib/gitlab/bare_repository_import/repository_spec.rb +++ /dev/null @@ -1,123 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe ::Gitlab::BareRepositoryImport::Repository do - context 'legacy storage' do - subject { described_class.new('/full/path/', '/full/path/to/repo.git') } - - it 'stores the repo path' do - expect(subject.repo_path).to eq('/full/path/to/repo.git') - end - - it 'stores the group path' do - expect(subject.group_path).to eq('to') - end - - it 'stores the project name' do - expect(subject.project_name).to eq('repo') - end - - it 'stores the wiki path' do - expect(subject.wiki_path).to eq('/full/path/to/repo.wiki.git') - end - - describe '#processable?' do - it 'returns false if it is a wiki' do - subject = described_class.new('/full/path/', '/full/path/to/a/b/my.wiki.git') - - expect(subject).not_to be_processable - end - - it 'returns true if group path is missing' do - subject = described_class.new('/full/path/', '/full/path/repo.git') - - expect(subject).to be_processable - end - - it 'returns true when group path and project name are present' do - expect(subject).to be_processable - end - end - - describe '#project_full_path' do - it 'returns the project full path with trailing slash in the root path' do - expect(subject.project_full_path).to eq('to/repo') - end - - it 'returns the project full path with no trailing slash in the root path' do - subject = described_class.new('/full/path', '/full/path/to/repo.git') - - expect(subject.project_full_path).to eq('to/repo') - end - end - end - - context 'hashed storage' do - let(:hashed_path) { "@hashed/6b/86/6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b" } - let(:root_path) { Gitlab::GitalyClient::StorageSettings.allow_disk_access { TestEnv.repos_path } } - let(:repo_path) { File.join(root_path, "#{hashed_path}.git") } - let(:wiki_path) { File.join(root_path, "#{hashed_path}.wiki.git") } - let(:raw_repository) { Gitlab::Git::Repository.new('default', "#{hashed_path}.git", nil, nil) } - let(:full_path) { 'to/repo' } - - before do - raw_repository.create_repository - raw_repository.set_full_path(full_path: full_path) if full_path - end - - after do - raw_repository.remove - end - - subject { described_class.new(root_path, repo_path) } - - it 'stores the repo path' do - expect(subject.repo_path).to eq(repo_path) - end - - it 'stores the wiki path' do - expect(subject.wiki_path).to eq(wiki_path) - end - - it 'reads the group path from .git/config' do - expect(subject.group_path).to eq('to') - end - - it 'reads the project name from .git/config' do - expect(subject.project_name).to eq('repo') - end - - describe '#processable?' do - it 'returns false if it is a wiki' do - subject = described_class.new(root_path, wiki_path) - - expect(subject).not_to be_processable - end - - it 'returns true when group path and project name are present' do - expect(subject).to be_processable - end - - context 'group and project name are missing' do - let(:full_path) { nil } - - it 'returns false' do - expect(subject).not_to be_processable - end - end - end - - describe '#project_full_path' do - it 'returns the project full path with trailing slash in the root path' do - expect(subject.project_full_path).to eq('to/repo') - end - - it 'returns the project full path with no trailing slash in the root path' do - subject = described_class.new(root_path[0...-1], repo_path) - - expect(subject.project_full_path).to eq('to/repo') - end - end - end -end diff --git a/spec/lib/gitlab/bitbucket_import/importer_spec.rb b/spec/lib/gitlab/bitbucket_import/importer_spec.rb index 1526a1a9f2d..48ceda9e8d8 100644 --- a/spec/lib/gitlab/bitbucket_import/importer_spec.rb +++ b/spec/lib/gitlab/bitbucket_import/importer_spec.rb @@ -358,7 +358,7 @@ RSpec.describe Gitlab::BitbucketImport::Importer, feature_category: :integration describe 'issue import' do it 'allocates internal ids' do - expect(Issue).to receive(:track_project_iid!).with(project, 6) + expect(Issue).to receive(:track_namespace_iid!).with(project.project_namespace, 6) importer.execute end diff --git a/spec/lib/gitlab/bitbucket_import/project_creator_spec.rb b/spec/lib/gitlab/bitbucket_import/project_creator_spec.rb index 236e04a041b..7ecdc5d25ea 100644 --- a/spec/lib/gitlab/bitbucket_import/project_creator_spec.rb +++ b/spec/lib/gitlab/bitbucket_import/project_creator_spec.rb @@ -27,8 +27,8 @@ RSpec.describe Gitlab::BitbucketImport::ProjectCreator do end it 'creates project' do - expect_next_instance_of(Project) do |project| - expect(project).to receive(:add_import_job) + allow_next_instance_of(Project) do |project| + allow(project).to receive(:add_import_job) end project_creator = described_class.new(repo, 'vim', namespace, user, access_params) diff --git a/spec/lib/gitlab/bullet/exclusions_spec.rb b/spec/lib/gitlab/bullet/exclusions_spec.rb index 325b0167f58..ccedfee28c7 100644 --- a/spec/lib/gitlab/bullet/exclusions_spec.rb +++ b/spec/lib/gitlab/bullet/exclusions_spec.rb @@ -3,7 +3,7 @@ require 'fast_spec_helper' require 'tempfile' -RSpec.describe Gitlab::Bullet::Exclusions do +RSpec.describe Gitlab::Bullet::Exclusions, feature_category: :application_performance do let(:config_file) do file = Tempfile.new('bullet.yml') File.basename(file) @@ -78,6 +78,19 @@ RSpec.describe Gitlab::Bullet::Exclusions do expect(described_class.new('_some_bogus_file_').execute).to match([]) end end + + context 'with a Symbol' do + let(:exclude) { [] } + let(:config) { { exclusions: { abc: { exclude: exclude } } } } + + before do + File.write(config_file, YAML.dump(config)) + end + + it 'raises an exception' do + expect { executor }.to raise_error(Psych::DisallowedClass) + end + end end describe '#validate_paths!' do diff --git a/spec/lib/gitlab/cache/client_spec.rb b/spec/lib/gitlab/cache/client_spec.rb new file mode 100644 index 00000000000..638fed1a905 --- /dev/null +++ b/spec/lib/gitlab/cache/client_spec.rb @@ -0,0 +1,162 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Cache::Client, feature_category: :source_code_management do + subject(:client) { described_class.new(metadata, backend: backend) } + + let(:backend) { Rails.cache } + let(:metadata) do + Gitlab::Cache::Metadata.new( + cache_identifier: cache_identifier, + feature_category: feature_category, + backing_resource: backing_resource + ) + end + + let(:cache_identifier) { 'MyClass#cache' } + let(:feature_category) { :source_code_management } + let(:backing_resource) { :cpu } + + let(:metadata_mock) do + Gitlab::Cache::Metadata.new( + cache_identifier: cache_identifier, + feature_category: feature_category + ) + end + + let(:metrics_mock) { Gitlab::Cache::Metrics.new(metadata_mock) } + + describe '.build_with_metadata' do + it 'builds a cache client with metrics support' do + attributes = { + cache_identifier: cache_identifier, + feature_category: feature_category, + backing_resource: backing_resource + } + + instance = described_class.build_with_metadata(**attributes) + + expect(instance).to be_a(described_class) + expect(instance.metadata).to have_attributes(**attributes) + end + end + + describe 'Methods', :use_clean_rails_memory_store_caching do + let(:expected_key) { 'key' } + + before do + allow(Gitlab::Cache::Metrics).to receive(:new).and_return(metrics_mock) + end + + describe '#read' do + context 'when key does not exist' do + it 'returns nil' do + expect(client.read('key')).to be_nil + end + + it 'increments cache miss' do + expect(metrics_mock).to receive(:increment_cache_miss) + + client.read('key') + end + end + + context 'when key exists' do + before do + backend.write(expected_key, 'value') + end + + it 'returns key value' do + expect(client.read('key')).to eq('value') + end + + it 'increments cache hit' do + expect(metrics_mock).to receive(:increment_cache_hit) + + client.read('key') + end + end + end + + describe '#write' do + it 'calls backend "#write" method with the expected key' do + expect(backend).to receive(:write).with(expected_key, 'value') + + client.write('key', 'value') + end + end + + describe '#exist?' do + it 'calls backend "#exist?" method with the expected key' do + expect(backend).to receive(:exist?).with(expected_key) + + client.exist?('key') + end + end + + describe '#delete' do + it 'calls backend "#delete" method with the expected key' do + expect(backend).to receive(:delete).with(expected_key) + + client.delete('key') + end + end + + # rubocop:disable Style/RedundantFetchBlock + describe '#fetch' do + it 'creates key in the specific format' do + client.fetch('key') { 'value' } + + expect(backend.fetch(expected_key)).to eq('value') + end + + it 'yields the block once' do + expect { |b| client.fetch('key', &b) }.to yield_control.once + end + + context 'when key already exists' do + before do + backend.write(expected_key, 'value') + end + + it 'does not redefine the value' do + expect(client.fetch('key') { 'new-value' }).to eq('value') + end + + it 'increments a cache hit' do + expect(metrics_mock).to receive(:increment_cache_hit) + + client.fetch('key') + end + + it 'does not measure the cache generation time' do + expect(metrics_mock).not_to receive(:observe_cache_generation) + + client.fetch('key') { 'new-value' } + end + end + + context 'when key does not exist' do + it 'caches the key' do + expect(client.fetch('key') { 'value' }).to eq('value') + + expect(client.fetch('key')).to eq('value') + end + + it 'increments a cache miss' do + expect(metrics_mock).to receive(:increment_cache_miss) + + client.fetch('key') + end + + it 'measures the cache generation time' do + expect(metrics_mock).to receive(:observe_cache_generation) + + client.fetch('key') { 'value' } + end + end + end + end + # rubocop:enable Style/RedundantFetchBlock +end diff --git a/spec/lib/gitlab/cache/metadata_spec.rb b/spec/lib/gitlab/cache/metadata_spec.rb index 2e8af7a9c44..d2b79fb8b08 100644 --- a/spec/lib/gitlab/cache/metadata_spec.rb +++ b/spec/lib/gitlab/cache/metadata_spec.rb @@ -5,24 +5,18 @@ require 'spec_helper' RSpec.describe Gitlab::Cache::Metadata, feature_category: :source_code_management do subject(:attributes) do described_class.new( - caller_id: caller_id, cache_identifier: cache_identifier, feature_category: feature_category, backing_resource: backing_resource ) end - let(:caller_id) { 'caller-id' } let(:cache_identifier) { 'ApplicationController#show' } let(:feature_category) { :source_code_management } let(:backing_resource) { :unknown } describe '#initialize' do context 'when optional arguments are not set' do - before do - Gitlab::ApplicationContext.push(caller_id: 'context-id') - end - it 'sets default value for them' do attributes = described_class.new( cache_identifier: cache_identifier, @@ -30,7 +24,6 @@ RSpec.describe Gitlab::Cache::Metadata, feature_category: :source_code_managemen ) expect(attributes.backing_resource).to eq(:unknown) - expect(attributes.caller_id).to eq('context-id') end end @@ -68,12 +61,6 @@ RSpec.describe Gitlab::Cache::Metadata, feature_category: :source_code_managemen end end - describe '#caller_id' do - subject { attributes.caller_id } - - it { is_expected.to eq caller_id } - end - describe '#cache_identifier' do subject { attributes.cache_identifier } diff --git a/spec/lib/gitlab/cache/metrics_spec.rb b/spec/lib/gitlab/cache/metrics_spec.rb index 24b274f4209..76ec0dbfa0b 100644 --- a/spec/lib/gitlab/cache/metrics_spec.rb +++ b/spec/lib/gitlab/cache/metrics_spec.rb @@ -7,14 +7,12 @@ RSpec.describe Gitlab::Cache::Metrics do let(:metadata) do Gitlab::Cache::Metadata.new( - caller_id: caller_id, cache_identifier: cache_identifier, feature_category: feature_category, backing_resource: backing_resource ) end - let(:caller_id) { 'caller-id' } let(:cache_identifier) { 'ApplicationController#show' } let(:feature_category) { :source_code_management } let(:backing_resource) { :unknown } @@ -37,7 +35,6 @@ RSpec.describe Gitlab::Cache::Metrics do .to receive(:increment) .with( { - caller_id: caller_id, cache_identifier: cache_identifier, feature_category: feature_category, backing_resource: backing_resource, @@ -57,7 +54,6 @@ RSpec.describe Gitlab::Cache::Metrics do .to receive(:increment) .with( { - caller_id: caller_id, cache_identifier: cache_identifier, feature_category: feature_category, backing_resource: backing_resource, @@ -86,7 +82,6 @@ RSpec.describe Gitlab::Cache::Metrics do :redis_cache_generation_duration_seconds, 'Duration of Redis cache generation', { - caller_id: caller_id, cache_identifier: cache_identifier, feature_category: feature_category, backing_resource: backing_resource diff --git a/spec/lib/gitlab/changes_list_spec.rb b/spec/lib/gitlab/changes_list_spec.rb index 762a121340e..77deffe4b37 100644 --- a/spec/lib/gitlab/changes_list_spec.rb +++ b/spec/lib/gitlab/changes_list_spec.rb @@ -2,7 +2,7 @@ require 'fast_spec_helper' -RSpec.describe Gitlab::ChangesList do +RSpec.describe Gitlab::ChangesList, feature_category: :source_code_management do let(:valid_changes_string) { "\n000000 570e7b2 refs/heads/my_branch\nd14d6c 6fd24d refs/heads/master" } let(:invalid_changes) { 1 } diff --git a/spec/lib/gitlab/chat/responder_spec.rb b/spec/lib/gitlab/chat/responder_spec.rb index a9d290cb87c..15ca3427ae8 100644 --- a/spec/lib/gitlab/chat/responder_spec.rb +++ b/spec/lib/gitlab/chat/responder_spec.rb @@ -4,68 +4,32 @@ require 'spec_helper' RSpec.describe Gitlab::Chat::Responder, feature_category: :integrations do describe '.responder_for' do - context 'when the feature flag is disabled' do - before do - stub_feature_flags(use_response_url_for_chat_responder: false) - end - - context 'using a regular build' do - it 'returns nil' do - build = create(:ci_build) + context 'using a regular build' do + it 'returns nil' do + build = create(:ci_build) - expect(described_class.responder_for(build)).to be_nil - end - end - - context 'using a chat build' do - it 'returns the responder for the build' do - pipeline = create(:ci_pipeline) - build = create(:ci_build, pipeline: pipeline) - integration = double(:integration, chat_responder: Gitlab::Chat::Responder::Slack) - chat_name = double(:chat_name, integration: integration) - chat_data = double(:chat_data, chat_name: chat_name) - - allow(pipeline) - .to receive(:chat_data) - .and_return(chat_data) - - expect(described_class.responder_for(build)) - .to be_an_instance_of(Gitlab::Chat::Responder::Slack) - end + expect(described_class.responder_for(build)).to be_nil end end - context 'when the feature flag is enabled' do - before do - stub_feature_flags(use_response_url_for_chat_responder: true) - end - - context 'using a regular build' do - it 'returns nil' do - build = create(:ci_build) + context 'using a chat build' do + let_it_be(:pipeline) { create(:ci_pipeline) } + let_it_be(:build) { create(:ci_build, pipeline: pipeline) } - expect(described_class.responder_for(build)).to be_nil + context "when response_url starts with 'https://hooks.slack.com/'" do + before do + pipeline.build_chat_data(response_url: 'https://hooks.slack.com/services/12345', chat_name_id: 'U123') end + + it { expect(described_class.responder_for(build)).to be_an_instance_of(Gitlab::Chat::Responder::Slack) } end - context 'using a chat build' do - let(:chat_name) { create(:chat_name, chat_id: 'U123') } - let(:pipeline) do - pipeline = create(:ci_pipeline) - pipeline.create_chat_data!( - response_url: 'https://hooks.slack.com/services/12345', - chat_name_id: chat_name.id - ) - pipeline + context "when response_url does not start with 'https://hooks.slack.com/'" do + before do + pipeline.build_chat_data(response_url: 'https://mattermost.example.com/services/12345', chat_name_id: 'U123') end - let(:build) { create(:ci_build, pipeline: pipeline) } - let(:responder) { described_class.new(build) } - - it 'returns the responder for the build' do - expect(described_class.responder_for(build)) - .to be_an_instance_of(Gitlab::Chat::Responder::Slack) - end + it { expect(described_class.responder_for(build)).to be_an_instance_of(Gitlab::Chat::Responder::Mattermost) } end end end diff --git a/spec/lib/gitlab/checks/changes_access_spec.rb b/spec/lib/gitlab/checks/changes_access_spec.rb index 60118823b5a..552afcdb180 100644 --- a/spec/lib/gitlab/checks/changes_access_spec.rb +++ b/spec/lib/gitlab/checks/changes_access_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Checks::ChangesAccess do +RSpec.describe Gitlab::Checks::ChangesAccess, feature_category: :source_code_management do include_context 'changes access checks context' subject { changes_access } @@ -47,6 +47,16 @@ RSpec.describe Gitlab::Checks::ChangesAccess do expect(subject.commits).to match_array([]) end + context 'when change is for notes ref' do + let(:changes) do + [{ oldrev: oldrev, newrev: newrev, ref: 'refs/notes/commit' }] + end + + it 'does not return any commits' do + expect(subject.commits).to match_array([]) + end + end + context 'when changes contain empty revisions' do let(:expected_commit) { instance_double(Commit) } diff --git a/spec/lib/gitlab/checks/diff_check_spec.rb b/spec/lib/gitlab/checks/diff_check_spec.rb index 6b45b8d4628..0845c746545 100644 --- a/spec/lib/gitlab/checks/diff_check_spec.rb +++ b/spec/lib/gitlab/checks/diff_check_spec.rb @@ -2,10 +2,20 @@ require 'spec_helper' -RSpec.describe Gitlab::Checks::DiffCheck do +RSpec.describe Gitlab::Checks::DiffCheck, feature_category: :source_code_management do include_context 'change access checks context' describe '#validate!' do + context 'when ref is not tag or branch ref' do + let(:ref) { 'refs/notes/commit' } + + it 'does not call find_changed_paths' do + expect(project.repository).not_to receive(:find_changed_paths) + + subject.validate! + end + end + context 'when commits is empty' do it 'does not call find_changed_paths' do expect(project.repository).not_to receive(:find_changed_paths) diff --git a/spec/lib/gitlab/ci/ansi2json/state_spec.rb b/spec/lib/gitlab/ci/ansi2json/state_spec.rb new file mode 100644 index 00000000000..8dd4092f3d8 --- /dev/null +++ b/spec/lib/gitlab/ci/ansi2json/state_spec.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Ansi2json::State, feature_category: :continuous_integration do + def build_state + described_class.new('', 1000).tap do |state| + state.offset = 1 + state.new_line!(style: { fg: 'some-fg', bg: 'some-bg', mask: 1234 }) + state.set_last_line_offset + state.open_section('hello', 111, {}) + end + end + + let(:state) { build_state } + + describe '#initialize' do + it 'restores valid prior state', :aggregate_failures do + new_state = described_class.new(state.encode, 1000) + + expect(new_state.offset).to eq(1) + expect(new_state.inherited_style).to eq({ + bg: 'some-bg', + fg: 'some-fg', + mask: 1234 + }) + expect(new_state.open_sections).to eq({ 'hello' => 111 }) + end + + it 'ignores unsigned prior state', :aggregate_failures do + unsigned, _ = build_state.encode.split('--') + + expect(::Gitlab::AppLogger).to( + receive(:warn).with( + message: a_string_matching(/signature missing or invalid/), + invalid_state: unsigned + ) + ) + + new_state = described_class.new(unsigned, 0) + + expect(new_state.offset).to eq(0) + expect(new_state.inherited_style).to eq({}) + expect(new_state.open_sections).to eq({}) + end + + it 'ignores bad input', :aggregate_failures do + expect(::Gitlab::AppLogger).to( + receive(:warn).with( + message: a_string_matching(/signature missing or invalid/), + invalid_state: 'abcd' + ) + ) + + new_state = described_class.new('abcd', 0) + + expect(new_state.offset).to eq(0) + expect(new_state.inherited_style).to eq({}) + expect(new_state.open_sections).to eq({}) + end + end + + describe '#encode' do + it 'deterministically signs the state' do + expect(state.encode).to eq state.encode + end + end +end diff --git a/spec/lib/gitlab/ci/ansi2json_spec.rb b/spec/lib/gitlab/ci/ansi2json_spec.rb index 0f8f3759834..98fca40e8ea 100644 --- a/spec/lib/gitlab/ci/ansi2json_spec.rb +++ b/spec/lib/gitlab/ci/ansi2json_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Ansi2json do +RSpec.describe Gitlab::Ci::Ansi2json, feature_category: :continuous_integration do subject { described_class } describe 'lines' do diff --git a/spec/lib/gitlab/ci/badge/release/template_spec.rb b/spec/lib/gitlab/ci/badge/release/template_spec.rb index 2b66c296a94..6be0dcaae99 100644 --- a/spec/lib/gitlab/ci/badge/release/template_spec.rb +++ b/spec/lib/gitlab/ci/badge/release/template_spec.rb @@ -59,9 +59,30 @@ RSpec.describe Gitlab::Ci::Badge::Release::Template do end describe '#value_width' do - it 'has a fixed value width' do + it 'returns the default value width' do expect(template.value_width).to eq 54 end + + it 'returns custom value width' do + value_width = 100 + badge = Gitlab::Ci::Badge::Release::LatestRelease.new(project, user, opts: { value_width: value_width }) + + expect(described_class.new(badge).value_width).to eq value_width + end + + it 'returns VALUE_WIDTH_DEFAULT if the custom value_width supplied is greater than permissible limit' do + value_width = 250 + badge = Gitlab::Ci::Badge::Release::LatestRelease.new(project, user, opts: { value_width: value_width }) + + expect(described_class.new(badge).value_width).to eq 54 + end + + it 'returns VALUE_WIDTH_DEFAULT if value_width is not a number' do + value_width = "string" + badge = Gitlab::Ci::Badge::Release::LatestRelease.new(project, user, opts: { value_width: value_width }) + + expect(described_class.new(badge).value_width).to eq 54 + end end describe '#key_color' do diff --git a/spec/lib/gitlab/ci/build/auto_retry_spec.rb b/spec/lib/gitlab/ci/build/auto_retry_spec.rb index 314714c543b..0b275e7d564 100644 --- a/spec/lib/gitlab/ci/build/auto_retry_spec.rb +++ b/spec/lib/gitlab/ci/build/auto_retry_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Build::AutoRetry, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Build::AutoRetry, feature_category: :pipeline_composition do let(:auto_retry) { described_class.new(build) } describe '#allowed?' do diff --git a/spec/lib/gitlab/ci/build/cache_spec.rb b/spec/lib/gitlab/ci/build/cache_spec.rb index a8fa14b4b4c..68d6a7978d7 100644 --- a/spec/lib/gitlab/ci/build/cache_spec.rb +++ b/spec/lib/gitlab/ci/build/cache_spec.rb @@ -3,16 +3,21 @@ require 'spec_helper' RSpec.describe Gitlab::Ci::Build::Cache do + let(:cache_config) { [] } + let(:pipeline) { double(::Ci::Pipeline) } + let(:cache_seed_a) { double(Gitlab::Ci::Pipeline::Seed::Build::Cache) } + let(:cache_seed_b) { double(Gitlab::Ci::Pipeline::Seed::Build::Cache) } + + subject(:cache) { described_class.new(cache_config, pipeline) } + describe '.initialize' do context 'when the cache is an array' do + let(:cache_config) { [{ key: 'key-a' }, { key: 'key-b' }] } + it 'instantiates an array of cache seeds' do - cache_config = [{ key: 'key-a' }, { key: 'key-b' }] - pipeline = double(::Ci::Pipeline) - cache_seed_a = double(Gitlab::Ci::Pipeline::Seed::Build::Cache) - cache_seed_b = double(Gitlab::Ci::Pipeline::Seed::Build::Cache) allow(Gitlab::Ci::Pipeline::Seed::Build::Cache).to receive(:new).and_return(cache_seed_a, cache_seed_b) - cache = described_class.new(cache_config, pipeline) + cache expect(Gitlab::Ci::Pipeline::Seed::Build::Cache).to have_received(:new).with(pipeline, { key: 'key-a' }, 0) expect(Gitlab::Ci::Pipeline::Seed::Build::Cache).to have_received(:new).with(pipeline, { key: 'key-b' }, 1) @@ -21,16 +26,31 @@ RSpec.describe Gitlab::Ci::Build::Cache do end context 'when the cache is a hash' do + let(:cache_config) { { key: 'key-a' } } + it 'instantiates a cache seed' do - cache_config = { key: 'key-a' } - pipeline = double(::Ci::Pipeline) - cache_seed = double(Gitlab::Ci::Pipeline::Seed::Build::Cache) - allow(Gitlab::Ci::Pipeline::Seed::Build::Cache).to receive(:new).and_return(cache_seed) + allow(Gitlab::Ci::Pipeline::Seed::Build::Cache).to receive(:new).and_return(cache_seed_a) - cache = described_class.new(cache_config, pipeline) + cache expect(Gitlab::Ci::Pipeline::Seed::Build::Cache).to have_received(:new).with(pipeline, cache_config, 0) - expect(cache.instance_variable_get(:@cache)).to eq([cache_seed]) + expect(cache.instance_variable_get(:@cache)).to eq([cache_seed_a]) + end + end + + context 'when the cache is an array with files inside hashes' do + let(:cache_config) { [{ key: { files: ['file1.json'] } }, { key: { files: ['file1.json', 'file2.json'] } }] } + + it 'instantiates a cache seed' do + allow(Gitlab::Ci::Pipeline::Seed::Build::Cache).to receive(:new).and_return(cache_seed_a, cache_seed_b) + + cache + + expect(Gitlab::Ci::Pipeline::Seed::Build::Cache).to have_received(:new) + .with(pipeline, cache_config.first, '0_file1') + expect(Gitlab::Ci::Pipeline::Seed::Build::Cache).to have_received(:new) + .with(pipeline, cache_config.second, '1_file1_file2') + expect(cache.instance_variable_get(:@cache)).to match_array([cache_seed_a, cache_seed_b]) end end end @@ -38,10 +58,6 @@ RSpec.describe Gitlab::Ci::Build::Cache do describe '#cache_attributes' do context 'when there are no caches' do it 'returns an empty hash' do - cache_config = [] - pipeline = double(::Ci::Pipeline) - cache = described_class.new(cache_config, pipeline) - attributes = cache.cache_attributes expect(attributes).to eq({}) @@ -51,7 +67,6 @@ RSpec.describe Gitlab::Ci::Build::Cache do context 'when there are caches' do it 'returns the structured attributes for the caches' do cache_config = [{ key: 'key-a' }, { key: 'key-b' }] - pipeline = double(::Ci::Pipeline) cache = described_class.new(cache_config, pipeline) attributes = cache.cache_attributes diff --git a/spec/lib/gitlab/ci/build/context/build_spec.rb b/spec/lib/gitlab/ci/build/context/build_spec.rb index 74739a67be0..d4a2af0015f 100644 --- a/spec/lib/gitlab/ci/build/context/build_spec.rb +++ b/spec/lib/gitlab/ci/build/context/build_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Build::Context::Build, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Build::Context::Build, feature_category: :pipeline_composition do let(:pipeline) { create(:ci_pipeline) } let(:seed_attributes) { { 'name' => 'some-job' } } @@ -13,14 +13,29 @@ RSpec.describe Gitlab::Ci::Build::Context::Build, feature_category: :pipeline_au it { is_expected.to include('CI_PIPELINE_IID' => pipeline.iid.to_s) } it { is_expected.to include('CI_PROJECT_PATH' => pipeline.project.full_path) } it { is_expected.to include('CI_JOB_NAME' => 'some-job') } - it { is_expected.to include('CI_BUILD_REF_NAME' => 'master') } + + context 'when FF `ci_remove_legacy_predefined_variables` is disabled' do + before do + stub_feature_flags(ci_remove_legacy_predefined_variables: false) + end + + it { is_expected.to include('CI_BUILD_REF_NAME' => 'master') } + end context 'without passed build-specific attributes' do let(:context) { described_class.new(pipeline) } - it { is_expected.to include('CI_JOB_NAME' => nil) } - it { is_expected.to include('CI_BUILD_REF_NAME' => 'master') } - it { is_expected.to include('CI_PROJECT_PATH' => pipeline.project.full_path) } + it { is_expected.to include('CI_JOB_NAME' => nil) } + it { is_expected.to include('CI_COMMIT_REF_NAME' => 'master') } + it { is_expected.to include('CI_PROJECT_PATH' => pipeline.project.full_path) } + + context 'when FF `ci_remove_legacy_predefined_variables` is disabled' do + before do + stub_feature_flags(ci_remove_legacy_predefined_variables: false) + end + + it { is_expected.to include('CI_BUILD_REF_NAME' => 'master') } + end end context 'when environment:name is provided' do diff --git a/spec/lib/gitlab/ci/build/context/global_spec.rb b/spec/lib/gitlab/ci/build/context/global_spec.rb index d4141eb8389..328b5eb62fa 100644 --- a/spec/lib/gitlab/ci/build/context/global_spec.rb +++ b/spec/lib/gitlab/ci/build/context/global_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Build::Context::Global do +RSpec.describe Gitlab::Ci::Build::Context::Global, feature_category: :pipeline_composition do let(:pipeline) { create(:ci_pipeline) } let(:yaml_variables) { {} } @@ -14,7 +14,14 @@ RSpec.describe Gitlab::Ci::Build::Context::Global do it { is_expected.to include('CI_PROJECT_PATH' => pipeline.project.full_path) } it { is_expected.not_to have_key('CI_JOB_NAME') } - it { is_expected.not_to have_key('CI_BUILD_REF_NAME') } + + context 'when FF `ci_remove_legacy_predefined_variables` is disabled' do + before do + stub_feature_flags(ci_remove_legacy_predefined_variables: false) + end + + it { is_expected.not_to have_key('CI_BUILD_REF_NAME') } + end context 'with passed yaml variables' do let(:yaml_variables) { [{ key: 'SUPPORTED', value: 'parsed', public: true }] } diff --git a/spec/lib/gitlab/ci/build/hook_spec.rb b/spec/lib/gitlab/ci/build/hook_spec.rb index 6ed40a44c97..6c9175b4260 100644 --- a/spec/lib/gitlab/ci/build/hook_spec.rb +++ b/spec/lib/gitlab/ci/build/hook_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Build::Hook, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Build::Hook, feature_category: :pipeline_composition do let_it_be(:build1) do FactoryBot.build(:ci_build, options: { hooks: { pre_get_sources_script: ["echo 'hello pre_get_sources_script'"] } }) diff --git a/spec/lib/gitlab/ci/build/rules_spec.rb b/spec/lib/gitlab/ci/build/rules_spec.rb index e82dcd0254d..1ece0f6b7b9 100644 --- a/spec/lib/gitlab/ci/build/rules_spec.rb +++ b/spec/lib/gitlab/ci/build/rules_spec.rb @@ -181,6 +181,108 @@ RSpec.describe Gitlab::Ci::Build::Rules do end end + context 'with needs' do + context 'when single needs is specified' do + let(:rule_list) do + [{ if: '$VAR == null', needs: [{ name: 'test', artifacts: true, optional: false }] }] + end + + it { + is_expected.to eq(described_class::Result.new('on_success', nil, nil, nil, + [{ name: 'test', artifacts: true, optional: false }], nil)) + } + end + + context 'when multiple needs are specified' do + let(:rule_list) do + [{ if: '$VAR == null', + needs: [{ name: 'test', artifacts: true, optional: false }, + { name: 'rspec', artifacts: true, optional: false }] }] + end + + it { + is_expected.to eq(described_class::Result.new('on_success', nil, nil, nil, + [{ name: 'test', artifacts: true, optional: false }, + { name: 'rspec', artifacts: true, optional: false }], nil)) + } + end + + context 'when there are no needs specified' do + let(:rule_list) { [{ if: '$VAR == null' }] } + + it { is_expected.to eq(described_class::Result.new('on_success', nil, nil, nil, nil, nil)) } + end + + context 'when need is specified with additional attibutes' do + let(:rule_list) do + [{ if: '$VAR == null', needs: [{ + artifacts: true, + name: 'test', + optional: false, + when: 'never' + }] }] + end + + it { + is_expected.to eq( + described_class::Result.new('on_success', nil, nil, nil, + [{ artifacts: true, name: 'test', optional: false, when: 'never' }], nil)) + } + end + + context 'when feature flag is disabled' do + before do + stub_feature_flags(introduce_rules_with_needs: false) + end + + context 'with needs' do + context 'when single needs is specified' do + let(:rule_list) do + [{ if: '$VAR == null', needs: [{ name: 'test', artifacts: true, optional: false }] }] + end + + it { + is_expected.to eq(described_class::Result.new('on_success', nil, nil, nil, nil, nil)) + } + end + + context 'when multiple needs are specified' do + let(:rule_list) do + [{ if: '$VAR == null', + needs: [{ name: 'test', artifacts: true, optional: false }, + { name: 'rspec', artifacts: true, optional: false }] }] + end + + it { + is_expected.to eq(described_class::Result.new('on_success', nil, nil, nil, nil, nil)) + } + end + + context 'when there are no needs specified' do + let(:rule_list) { [{ if: '$VAR == null' }] } + + it { is_expected.to eq(described_class::Result.new('on_success', nil, nil, nil, nil, nil)) } + end + + context 'when need is specified with additional attibutes' do + let(:rule_list) do + [{ if: '$VAR == null', needs: [{ + artifacts: true, + name: 'test', + optional: false, + when: 'never' + }] }] + end + + it { + is_expected.to eq( + described_class::Result.new('on_success', nil, nil, nil, nil, nil)) + } + end + end + end + end + context 'with variables' do context 'with matching rule' do let(:rule_list) { [{ if: '$VAR == null', variables: { MY_VAR: 'my var' } }] } @@ -208,9 +310,10 @@ RSpec.describe Gitlab::Ci::Build::Rules do let(:start_in) { nil } let(:allow_failure) { nil } let(:variables) { nil } + let(:needs) { nil } subject(:result) do - Gitlab::Ci::Build::Rules::Result.new(when_value, start_in, allow_failure, variables) + Gitlab::Ci::Build::Rules::Result.new(when_value, start_in, allow_failure, variables, needs) end describe '#build_attributes' do @@ -221,6 +324,45 @@ RSpec.describe Gitlab::Ci::Build::Rules do it 'compacts nil values' do is_expected.to eq(options: {}, when: 'on_success') end + + context 'scheduling_type' do + context 'when rules have needs' do + context 'single need' do + let(:needs) do + { job: [{ name: 'test' }] } + end + + it 'saves needs' do + expect(subject[:needs_attributes]).to eq([{ name: "test" }]) + end + + it 'adds schedule type to the build_attributes' do + expect(subject[:scheduling_type]).to eq(:dag) + end + end + + context 'multiple needs' do + let(:needs) do + { job: [{ name: 'test' }, { name: 'test_2', artifacts: true, optional: false }] } + end + + it 'saves needs' do + expect(subject[:needs_attributes]).to match_array([{ name: "test" }, + { name: 'test_2', artifacts: true, optional: false }]) + end + + it 'adds schedule type to the build_attributes' do + expect(subject[:scheduling_type]).to eq(:dag) + end + end + end + + context 'when rules do not have needs' do + it 'does not add schedule type to the build_attributes' do + expect(subject.key?(:scheduling_type)).to be_falsy + end + end + end end describe '#pass?' do diff --git a/spec/lib/gitlab/ci/components/instance_path_spec.rb b/spec/lib/gitlab/ci/components/instance_path_spec.rb index d9beae0555c..b80422d03e5 100644 --- a/spec/lib/gitlab/ci/components/instance_path_spec.rb +++ b/spec/lib/gitlab/ci/components/instance_path_spec.rb @@ -2,11 +2,11 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Components::InstancePath, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Components::InstancePath, feature_category: :pipeline_composition do let_it_be(:user) { create(:user) } let(:path) { described_class.new(address: address, content_filename: 'template.yml') } - let(:settings) { Settingslogic.new({ 'component_fqdn' => current_host }) } + let(:settings) { GitlabSettings::Options.build({ 'component_fqdn' => current_host }) } let(:current_host) { 'acme.com/' } before do @@ -98,6 +98,37 @@ RSpec.describe Gitlab::Ci::Components::InstancePath, feature_category: :pipeline end end + context 'when version is `~latest`' do + let(:version) { '~latest' } + + context 'when project is a catalog resource' do + before do + create(:catalog_resource, project: existing_project) + end + + context 'when project has releases' do + let_it_be(:releases) do + [ + create(:release, project: existing_project, sha: 'sha-1', released_at: Time.zone.now - 1.day), + create(:release, project: existing_project, sha: 'sha-2', released_at: Time.zone.now) + ] + end + + it 'returns the sha of the latest release' do + expect(path.sha).to eq(releases.last.sha) + end + end + + context 'when project does not have releases' do + it { expect(path.sha).to be_nil } + end + end + + context 'when project is not a catalog resource' do + it { expect(path.sha).to be_nil } + end + end + context 'when project does not exist' do let(:project_path) { 'non-existent/project' } diff --git a/spec/lib/gitlab/ci/config/entry/cache_spec.rb b/spec/lib/gitlab/ci/config/entry/cache_spec.rb index 67252eed938..82db116fa0d 100644 --- a/spec/lib/gitlab/ci/config/entry/cache_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/cache_spec.rb @@ -17,6 +17,7 @@ RSpec.describe Gitlab::Ci::Config::Entry::Cache do let(:key) { 'some key' } let(:when_config) { nil } let(:unprotect) { false } + let(:fallback_keys) { [] } let(:config) do { @@ -27,13 +28,22 @@ RSpec.describe Gitlab::Ci::Config::Entry::Cache do }.tap do |config| config[:policy] = policy if policy config[:when] = when_config if when_config + config[:fallback_keys] = fallback_keys if fallback_keys end end describe '#value' do shared_examples 'hash key value' do it 'returns hash value' do - expect(entry.value).to eq(key: key, untracked: true, paths: ['some/path/'], policy: 'pull-push', when: 'on_success', unprotect: false) + expect(entry.value).to eq( + key: key, + untracked: true, + paths: ['some/path/'], + policy: 'pull-push', + when: 'on_success', + unprotect: false, + fallback_keys: [] + ) end end @@ -104,6 +114,20 @@ RSpec.describe Gitlab::Ci::Config::Entry::Cache do expect(entry.value).to include(when: 'on_success') end end + + context 'with `fallback_keys`' do + let(:fallback_keys) { %w[key-1 key-2] } + + it 'matches the list of fallback keys' do + expect(entry.value).to match(a_hash_including(fallback_keys: %w[key-1 key-2])) + end + end + + context 'without `fallback_keys`' do + it 'assigns an empty list' do + expect(entry.value).to match(a_hash_including(fallback_keys: [])) + end + end end describe '#valid?' do diff --git a/spec/lib/gitlab/ci/config/entry/job_spec.rb b/spec/lib/gitlab/ci/config/entry/job_spec.rb index c1b9bd58d98..4be7c11fab0 100644 --- a/spec/lib/gitlab/ci/config/entry/job_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/job_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::Entry::Job, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::Entry::Job, feature_category: :pipeline_composition do let(:entry) { described_class.new(config, name: :rspec) } it_behaves_like 'with inheritable CI config' do @@ -261,13 +261,13 @@ RSpec.describe Gitlab::Ci::Config::Entry::Job, feature_category: :pipeline_autho end end - context 'when it is lower than two' do - let(:config) { { script: 'echo', parallel: 1 } } + context 'when it is lower than one' do + let(:config) { { script: 'echo', parallel: 0 } } it 'returns error about value too low' do expect(entry).not_to be_valid expect(entry.errors) - .to include 'parallel config must be greater than or equal to 2' + .to include 'parallel config must be greater than or equal to 1' end end @@ -595,6 +595,39 @@ RSpec.describe Gitlab::Ci::Config::Entry::Job, feature_category: :pipeline_autho end end end + + context 'when job is not a pages job' do + let(:name) { :rspec } + + context 'if the config contains a publish entry' do + let(:entry) { described_class.new({ script: 'echo', publish: 'foo' }, name: name) } + + it 'is invalid' do + expect(entry).not_to be_valid + expect(entry.errors).to include /job publish can only be used within a `pages` job/ + end + end + end + + context 'when job is a pages job' do + let(:name) { :pages } + + context 'when it does not have a publish entry' do + let(:entry) { described_class.new({ script: 'echo' }, name: name) } + + it 'is valid' do + expect(entry).to be_valid + end + end + + context 'when it has a publish entry' do + let(:entry) { described_class.new({ script: 'echo', publish: 'foo' }, name: name) } + + it 'is valid' do + expect(entry).to be_valid + end + end + end end describe '#relevant?' do @@ -631,7 +664,13 @@ RSpec.describe Gitlab::Ci::Config::Entry::Job, feature_category: :pipeline_autho it 'overrides default config' do expect(entry[:image].value).to eq(name: 'some_image') - expect(entry[:cache].value).to eq([key: 'test', policy: 'pull-push', when: 'on_success', unprotect: false]) + expect(entry[:cache].value).to match_array([ + key: 'test', + policy: 'pull-push', + when: 'on_success', + unprotect: false, + fallback_keys: [] + ]) end end @@ -646,7 +685,13 @@ RSpec.describe Gitlab::Ci::Config::Entry::Job, feature_category: :pipeline_autho it 'uses config from default entry' do expect(entry[:image].value).to eq 'specified' - expect(entry[:cache].value).to eq([key: 'test', policy: 'pull-push', when: 'on_success', unprotect: false]) + expect(entry[:cache].value).to match_array([ + key: 'test', + policy: 'pull-push', + when: 'on_success', + unprotect: false, + fallback_keys: [] + ]) end end @@ -728,27 +773,6 @@ RSpec.describe Gitlab::Ci::Config::Entry::Job, feature_category: :pipeline_autho scheduling_type: :stage, id_tokens: { TEST_ID_TOKEN: { aud: 'https://gitlab.com' } }) end - - context 'when the FF ci_hooks_pre_get_sources_script is disabled' do - before do - stub_feature_flags(ci_hooks_pre_get_sources_script: false) - end - - it 'returns correct value' do - expect(entry.value) - .to eq(name: :rspec, - before_script: %w[ls pwd], - script: %w[rspec], - stage: 'test', - ignore: false, - after_script: %w[cleanup], - only: { refs: %w[branches tags] }, - job_variables: {}, - root_variables_inheritance: true, - scheduling_type: :stage, - id_tokens: { TEST_ID_TOKEN: { aud: 'https://gitlab.com' } }) - end - end end end diff --git a/spec/lib/gitlab/ci/config/entry/policy_spec.rb b/spec/lib/gitlab/ci/config/entry/policy_spec.rb index 378c0947e8a..7093a0a6edf 100644 --- a/spec/lib/gitlab/ci/config/entry/policy_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/policy_spec.rb @@ -1,8 +1,8 @@ # frozen_string_literal: true -require 'fast_spec_helper' +require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::Entry::Policy do +RSpec.describe Gitlab::Ci::Config::Entry::Policy, feature_category: :continuous_integration do let(:entry) { described_class.new(config) } context 'when using simplified policy' do diff --git a/spec/lib/gitlab/ci/config/entry/processable_spec.rb b/spec/lib/gitlab/ci/config/entry/processable_spec.rb index b28562ba2ea..4f13940d7e2 100644 --- a/spec/lib/gitlab/ci/config/entry/processable_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/processable_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::Entry::Processable, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::Entry::Processable, feature_category: :pipeline_composition do let(:node_class) do Class.new(::Gitlab::Config::Entry::Node) do include Gitlab::Ci::Config::Entry::Processable diff --git a/spec/lib/gitlab/ci/config/entry/product/parallel_spec.rb b/spec/lib/gitlab/ci/config/entry/product/parallel_spec.rb index ec21519a8f6..1025c41477d 100644 --- a/spec/lib/gitlab/ci/config/entry/product/parallel_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/product/parallel_spec.rb @@ -27,10 +27,10 @@ RSpec.describe ::Gitlab::Ci::Config::Entry::Product::Parallel do it_behaves_like 'invalid config', /should be an integer or a hash/ end - context 'when it is lower than two' do - let(:config) { 1 } + context 'when it is lower than one' do + let(:config) { 0 } - it_behaves_like 'invalid config', /must be greater than or equal to 2/ + it_behaves_like 'invalid config', /must be greater than or equal to 1/ end context 'when it is bigger than 200' do diff --git a/spec/lib/gitlab/ci/config/entry/publish_spec.rb b/spec/lib/gitlab/ci/config/entry/publish_spec.rb new file mode 100644 index 00000000000..53ad868a05e --- /dev/null +++ b/spec/lib/gitlab/ci/config/entry/publish_spec.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Config::Entry::Publish, feature_category: :pages do + let(:publish) { described_class.new(config) } + + describe 'validations' do + context 'when publish config value is correct' do + let(:config) { 'dist/static' } + + describe '#config' do + it 'returns the publish directory' do + expect(publish.config).to eq config + end + end + + describe '#valid?' do + it 'is valid' do + expect(publish).to be_valid + end + end + end + + context 'when the value has a wrong type' do + let(:config) { { test: true } } + + it 'reports an error' do + expect(publish.errors) + .to include 'publish config should be a string' + end + end + end + + describe '.default' do + it 'returns the default value' do + expect(described_class.default).to eq 'public' + end + end +end diff --git a/spec/lib/gitlab/ci/config/entry/pull_policy_spec.rb b/spec/lib/gitlab/ci/config/entry/pull_policy_spec.rb index c35355b10c6..40507a66c2d 100644 --- a/spec/lib/gitlab/ci/config/entry/pull_policy_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/pull_policy_spec.rb @@ -1,8 +1,8 @@ # frozen_string_literal: true -require 'fast_spec_helper' +require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::Entry::PullPolicy do +RSpec.describe Gitlab::Ci::Config::Entry::PullPolicy, feature_category: :continuous_integration do let(:entry) { described_class.new(config) } describe '#value' do diff --git a/spec/lib/gitlab/ci/config/entry/reports/coverage_report_spec.rb b/spec/lib/gitlab/ci/config/entry/reports/coverage_report_spec.rb index ccd6f6ab427..6f37dd72083 100644 --- a/spec/lib/gitlab/ci/config/entry/reports/coverage_report_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/reports/coverage_report_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::Entry::Reports::CoverageReport, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::Entry::Reports::CoverageReport, feature_category: :pipeline_composition do let(:entry) { described_class.new(config) } describe 'validations' do diff --git a/spec/lib/gitlab/ci/config/entry/reports_spec.rb b/spec/lib/gitlab/ci/config/entry/reports_spec.rb index 715cb18fb92..73bf2d422b7 100644 --- a/spec/lib/gitlab/ci/config/entry/reports_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/reports_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::Entry::Reports, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::Entry::Reports, feature_category: :pipeline_composition do let(:entry) { described_class.new(config) } describe 'validates ALLOWED_KEYS' do diff --git a/spec/lib/gitlab/ci/config/entry/root_spec.rb b/spec/lib/gitlab/ci/config/entry/root_spec.rb index 9722609aef6..5fac5298e8e 100644 --- a/spec/lib/gitlab/ci/config/entry/root_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/root_spec.rb @@ -128,7 +128,7 @@ RSpec.describe Gitlab::Ci::Config::Entry::Root do services: [{ name: 'postgres:9.1' }, { name: 'mysql:5.5' }], stage: 'test', cache: [{ key: 'k', untracked: true, paths: ['public/'], policy: 'pull-push', when: 'on_success', - unprotect: false }], + unprotect: false, fallback_keys: [] }], job_variables: {}, root_variables_inheritance: true, ignore: false, @@ -144,7 +144,7 @@ RSpec.describe Gitlab::Ci::Config::Entry::Root do services: [{ name: 'postgres:9.1' }, { name: 'mysql:5.5' }], stage: 'test', cache: [{ key: 'k', untracked: true, paths: ['public/'], policy: 'pull-push', when: 'on_success', - unprotect: false }], + unprotect: false, fallback_keys: [] }], job_variables: {}, root_variables_inheritance: true, ignore: false, @@ -161,7 +161,7 @@ RSpec.describe Gitlab::Ci::Config::Entry::Root do image: { name: "image:1.0" }, services: [{ name: "postgres:9.1" }, { name: "mysql:5.5" }], cache: [{ key: "k", untracked: true, paths: ["public/"], policy: "pull-push", when: 'on_success', - unprotect: false }], + unprotect: false, fallback_keys: [] }], only: { refs: %w(branches tags) }, job_variables: { 'VAR' => { value: 'job' } }, root_variables_inheritance: true, @@ -209,7 +209,7 @@ RSpec.describe Gitlab::Ci::Config::Entry::Root do image: { name: 'image:1.0' }, services: [{ name: 'postgres:9.1' }, { name: 'mysql:5.5' }], stage: 'test', - cache: [{ key: 'k', untracked: true, paths: ['public/'], policy: 'pull-push', when: 'on_success', unprotect: false }], + cache: [{ key: 'k', untracked: true, paths: ['public/'], policy: 'pull-push', when: 'on_success', unprotect: false, fallback_keys: [] }], job_variables: {}, root_variables_inheritance: true, ignore: false, @@ -222,7 +222,7 @@ RSpec.describe Gitlab::Ci::Config::Entry::Root do image: { name: 'image:1.0' }, services: [{ name: 'postgres:9.1' }, { name: 'mysql:5.5' }], stage: 'test', - cache: [{ key: 'k', untracked: true, paths: ['public/'], policy: 'pull-push', when: 'on_success', unprotect: false }], + cache: [{ key: 'k', untracked: true, paths: ['public/'], policy: 'pull-push', when: 'on_success', unprotect: false, fallback_keys: [] }], job_variables: { 'VAR' => { value: 'job' } }, root_variables_inheritance: true, ignore: false, @@ -277,7 +277,13 @@ RSpec.describe Gitlab::Ci::Config::Entry::Root do describe '#cache_value' do it 'returns correct cache definition' do - expect(root.cache_value).to eq([key: 'a', policy: 'pull-push', when: 'on_success', unprotect: false]) + expect(root.cache_value).to match_array([ + key: 'a', + policy: 'pull-push', + when: 'on_success', + unprotect: false, + fallback_keys: [] + ]) end end end diff --git a/spec/lib/gitlab/ci/config/entry/trigger_spec.rb b/spec/lib/gitlab/ci/config/entry/trigger_spec.rb index f47923af45a..fdd598c2ab2 100644 --- a/spec/lib/gitlab/ci/config/entry/trigger_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/trigger_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::Entry::Trigger, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::Entry::Trigger, feature_category: :pipeline_composition do subject { described_class.new(config) } context 'when trigger config is a non-empty string' do diff --git a/spec/lib/gitlab/ci/config/external/context_spec.rb b/spec/lib/gitlab/ci/config/external/context_spec.rb index 1fd3cf3c99f..d917924f257 100644 --- a/spec/lib/gitlab/ci/config/external/context_spec.rb +++ b/spec/lib/gitlab/ci/config/external/context_spec.rb @@ -2,12 +2,21 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::External::Context, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::External::Context, feature_category: :pipeline_composition do let(:project) { build(:project) } let(:user) { double('User') } let(:sha) { '12345' } let(:variables) { Gitlab::Ci::Variables::Collection.new([{ 'key' => 'a', 'value' => 'b' }]) } - let(:attributes) { { project: project, user: user, sha: sha, variables: variables } } + let(:pipeline_config) { instance_double(Gitlab::Ci::ProjectConfig) } + let(:attributes) do + { + project: project, + user: user, + sha: sha, + variables: variables, + pipeline_config: pipeline_config + } + end subject(:subject) { described_class.new(**attributes) } @@ -15,11 +24,11 @@ RSpec.describe Gitlab::Ci::Config::External::Context, feature_category: :pipelin context 'with values' do it { is_expected.to have_attributes(**attributes) } it { expect(subject.expandset).to eq([]) } - it { expect(subject.max_includes).to eq(Gitlab::Ci::Config::External::Context::NEW_MAX_INCLUDES) } it { expect(subject.execution_deadline).to eq(0) } it { expect(subject.variables).to be_instance_of(Gitlab::Ci::Variables::Collection) } it { expect(subject.variables_hash).to be_instance_of(ActiveSupport::HashWithIndifferentAccess) } it { expect(subject.variables_hash).to include('a' => 'b') } + it { expect(subject.pipeline_config).to eq(pipeline_config) } end context 'without values' do @@ -27,36 +36,25 @@ RSpec.describe Gitlab::Ci::Config::External::Context, feature_category: :pipelin it { is_expected.to have_attributes(**attributes) } it { expect(subject.expandset).to eq([]) } - it { expect(subject.max_includes).to eq(Gitlab::Ci::Config::External::Context::NEW_MAX_INCLUDES) } it { expect(subject.execution_deadline).to eq(0) } it { expect(subject.variables).to be_instance_of(Gitlab::Ci::Variables::Collection) } it { expect(subject.variables_hash).to be_instance_of(ActiveSupport::HashWithIndifferentAccess) } + it { expect(subject.pipeline_config).to be_nil } end - context 'when FF ci_includes_count_duplicates is disabled' do - before do - stub_feature_flags(ci_includes_count_duplicates: false) - end - - context 'with values' do - it { is_expected.to have_attributes(**attributes) } - it { expect(subject.expandset).to eq(Set.new) } - it { expect(subject.max_includes).to eq(Gitlab::Ci::Config::External::Context::MAX_INCLUDES) } - it { expect(subject.execution_deadline).to eq(0) } - it { expect(subject.variables).to be_instance_of(Gitlab::Ci::Variables::Collection) } - it { expect(subject.variables_hash).to be_instance_of(ActiveSupport::HashWithIndifferentAccess) } - it { expect(subject.variables_hash).to include('a' => 'b') } + describe 'max_includes' do + it 'returns the default value of application setting `ci_max_includes`' do + expect(subject.max_includes).to eq(150) end - context 'without values' do - let(:attributes) { { project: nil, user: nil, sha: nil } } + context 'when application setting `ci_max_includes` is changed' do + before do + stub_application_setting(ci_max_includes: 200) + end - it { is_expected.to have_attributes(**attributes) } - it { expect(subject.expandset).to eq(Set.new) } - it { expect(subject.max_includes).to eq(Gitlab::Ci::Config::External::Context::MAX_INCLUDES) } - it { expect(subject.execution_deadline).to eq(0) } - it { expect(subject.variables).to be_instance_of(Gitlab::Ci::Variables::Collection) } - it { expect(subject.variables_hash).to be_instance_of(ActiveSupport::HashWithIndifferentAccess) } + it 'returns the new value of application setting `ci_max_includes`' do + expect(subject.max_includes).to eq(200) + end end end end @@ -170,4 +168,26 @@ RSpec.describe Gitlab::Ci::Config::External::Context, feature_category: :pipelin describe '#sentry_payload' do it { expect(subject.sentry_payload).to match(a_hash_including(:project, :user)) } end + + describe '#internal_include?' do + context 'when pipeline_config is provided' do + where(:value) { [true, false] } + + with_them do + it 'returns the value of .internal_include_prepended?' do + allow(pipeline_config).to receive(:internal_include_prepended?).and_return(value) + + expect(subject.internal_include?).to eq(value) + end + end + end + + context 'when pipeline_config is not provided' do + let(:pipeline_config) { nil } + + it 'returns false' do + expect(subject.internal_include?).to eq(false) + end + end + end end diff --git a/spec/lib/gitlab/ci/config/external/file/artifact_spec.rb b/spec/lib/gitlab/ci/config/external/file/artifact_spec.rb index 45a15fb5f36..087dacd5ef0 100644 --- a/spec/lib/gitlab/ci/config/external/file/artifact_spec.rb +++ b/spec/lib/gitlab/ci/config/external/file/artifact_spec.rb @@ -2,11 +2,13 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::External::File::Artifact, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::External::File::Artifact, feature_category: :pipeline_composition do let(:parent_pipeline) { create(:ci_pipeline) } + let(:project) { parent_pipeline.project } let(:variables) {} let(:context) do - Gitlab::Ci::Config::External::Context.new(variables: variables, parent_pipeline: parent_pipeline) + Gitlab::Ci::Config::External::Context + .new(variables: variables, parent_pipeline: parent_pipeline, project: project) end let(:external_file) { described_class.new(params, context) } @@ -43,7 +45,7 @@ RSpec.describe Gitlab::Ci::Config::External::File::Artifact, feature_category: : end describe 'when used in non child pipeline context' do - let(:parent_pipeline) { nil } + let(:context) { Gitlab::Ci::Config::External::Context.new } let(:params) { { artifact: 'generated.yml' } } let(:expected_error) do @@ -201,7 +203,7 @@ RSpec.describe Gitlab::Ci::Config::External::File::Artifact, feature_category: : it { is_expected.to eq( - context_project: nil, + context_project: project.full_path, context_sha: nil, type: :artifact, location: 'generated.yml', @@ -218,7 +220,7 @@ RSpec.describe Gitlab::Ci::Config::External::File::Artifact, feature_category: : it { is_expected.to eq( - context_project: nil, + context_project: project.full_path, context_sha: nil, type: :artifact, location: 'generated.yml', @@ -227,4 +229,35 @@ RSpec.describe Gitlab::Ci::Config::External::File::Artifact, feature_category: : } end end + + describe '#to_hash' do + context 'when interpolation is being used' do + let!(:job) { create(:ci_build, name: 'generator', pipeline: parent_pipeline) } + let!(:artifacts) { create(:ci_job_artifact, :archive, job: job) } + let!(:metadata) { create(:ci_job_artifact, :metadata, job: job) } + + before do + allow_next_instance_of(Gitlab::Ci::ArtifactFileReader) do |reader| + allow(reader).to receive(:read).and_return(template) + end + end + + let(:template) do + <<~YAML + spec: + inputs: + env: + --- + deploy: + script: deploy $[[ inputs.env ]] + YAML + end + + let(:params) { { artifact: 'generated.yml', job: 'generator', inputs: { env: 'production' } } } + + it 'correctly interpolates content' do + expect(external_file.to_hash).to eq({ deploy: { script: 'deploy production' } }) + end + end + end end diff --git a/spec/lib/gitlab/ci/config/external/file/base_spec.rb b/spec/lib/gitlab/ci/config/external/file/base_spec.rb index 55d95d0c1f8..1c5918f77ca 100644 --- a/spec/lib/gitlab/ci/config/external/file/base_spec.rb +++ b/spec/lib/gitlab/ci/config/external/file/base_spec.rb @@ -2,15 +2,16 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::External::File::Base, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::External::File::Base, feature_category: :pipeline_composition do + let_it_be(:project) { create(:project) } let(:variables) {} - let(:context_params) { { sha: 'HEAD', variables: variables } } - let(:context) { Gitlab::Ci::Config::External::Context.new(**context_params) } + let(:context_params) { { sha: 'HEAD', variables: variables, project: project } } + let(:ctx) { Gitlab::Ci::Config::External::Context.new(**context_params) } let(:test_class) do Class.new(described_class) do - def initialize(params, context) - @location = params + def initialize(params, ctx) + @location = params[:location] super end @@ -18,15 +19,18 @@ RSpec.describe Gitlab::Ci::Config::External::File::Base, feature_category: :pipe def validate_context! # no-op end + + def content + params[:content] + end end end - subject(:file) { test_class.new(location, context) } + let(:content) { 'key: value' } - before do - allow_any_instance_of(test_class) - .to receive(:content).and_return('key: value') + subject(:file) { test_class.new({ location: location, content: content }, ctx) } + before do allow_any_instance_of(Gitlab::Ci::Config::External::Context) .to receive(:check_execution_time!) end @@ -51,7 +55,7 @@ RSpec.describe Gitlab::Ci::Config::External::File::Base, feature_category: :pipe describe '#valid?' do subject(:valid?) do - Gitlab::Ci::Config::External::Mapper::Verifier.new(context).process([file]) + Gitlab::Ci::Config::External::Mapper::Verifier.new(ctx).process([file]) file.valid? end @@ -87,7 +91,12 @@ RSpec.describe Gitlab::Ci::Config::External::File::Base, feature_category: :pipe context 'when there are YAML syntax errors' do let(:location) { 'some/file/secret_file_name.yml' } - let(:variables) { Gitlab::Ci::Variables::Collection.new([{ 'key' => 'GITLAB_TOKEN', 'value' => 'secret_file_name', 'masked' => true }]) } + + let(:variables) do + Gitlab::Ci::Variables::Collection.new( + [{ 'key' => 'GITLAB_TOKEN', 'value' => 'secret_file_name', 'masked' => true }] + ) + end before do allow_any_instance_of(test_class) @@ -96,15 +105,16 @@ RSpec.describe Gitlab::Ci::Config::External::File::Base, feature_category: :pipe it 'is not a valid file' do expect(valid?).to be_falsy - expect(file.error_message).to eq('Included file `some/file/xxxxxxxxxxxxxxxx.yml` does not have valid YAML syntax!') + expect(file.error_message) + .to eq('`some/file/xxxxxxxxxxxxxxxx.yml`: content does not have a valid YAML syntax') end end context 'when the class has no validate_context!' do let(:test_class) do Class.new(described_class) do - def initialize(params, context) - @location = params + def initialize(params, ctx) + @location = params[:location] super end @@ -117,6 +127,88 @@ RSpec.describe Gitlab::Ci::Config::External::File::Base, feature_category: :pipe expect { valid? }.to raise_error(NotImplementedError) end end + + context 'when interpolation is disabled but there is a spec header' do + before do + stub_feature_flags(ci_includable_files_interpolation: false) + end + + let(:location) { 'some-location.yml' } + + let(:content) do + <<~YAML + spec: + include: + website: + --- + run: + script: deploy $[[ inputs.website ]] + YAML + end + + it 'returns an error saying that interpolation is disabled' do + expect(valid?).to be_falsy + expect(file.errors) + .to include('`some-location.yml`: can not evaluate included file because interpolation is disabled') + end + end + + context 'when interpolation was unsuccessful' do + let(:location) { 'some-location.yml' } + + context 'when context key is missing' do + let(:content) do + <<~YAML + spec: + inputs: + --- + run: + script: deploy $[[ inputs.abcd ]] + YAML + end + + it 'surfaces interpolation errors' do + expect(valid?).to be_falsy + expect(file.errors) + .to include('`some-location.yml`: interpolation interrupted by errors, unknown interpolation key: `abcd`') + end + end + + context 'when header is invalid' do + let(:content) do + <<~YAML + spec: + a: abc + --- + run: + script: deploy $[[ inputs.abcd ]] + YAML + end + + it 'surfaces header errors' do + expect(valid?).to be_falsy + expect(file.errors) + .to include('`some-location.yml`: header:spec config contains unknown keys: a') + end + end + + context 'when header is not a hash' do + let(:content) do + <<~YAML + spec: abcd + --- + run: + script: deploy $[[ inputs.abcd ]] + YAML + end + + it 'surfaces header errors' do + expect(valid?).to be_falsy + expect(file.errors) + .to contain_exactly('`some-location.yml`: header:spec config should be a hash') + end + end + end end describe '#to_hash' do @@ -142,7 +234,7 @@ RSpec.describe Gitlab::Ci::Config::External::File::Base, feature_category: :pipe it { is_expected.to eq( - context_project: nil, + context_project: project.full_path, context_sha: 'HEAD' ) } @@ -154,13 +246,13 @@ RSpec.describe Gitlab::Ci::Config::External::File::Base, feature_category: :pipe subject(:eql) { file.eql?(other_file) } context 'when the other file has the same params' do - let(:other_file) { test_class.new(location, context) } + let(:other_file) { test_class.new({ location: location, content: content }, ctx) } it { is_expected.to eq(true) } end context 'when the other file has not the same params' do - let(:other_file) { test_class.new('some/other/file', context) } + let(:other_file) { test_class.new({ location: 'some/other/file', content: content }, ctx) } it { is_expected.to eq(false) } end @@ -172,14 +264,15 @@ RSpec.describe Gitlab::Ci::Config::External::File::Base, feature_category: :pipe subject(:filehash) { file.hash } context 'with a project' do - let(:project) { create(:project) } let(:context_params) { { project: project, sha: 'HEAD', variables: variables } } - it { is_expected.to eq([location, project.full_path, 'HEAD'].hash) } + it { is_expected.to eq([{ location: location, content: content }, project.full_path, 'HEAD'].hash) } end context 'without a project' do - it { is_expected.to eq([location, nil, 'HEAD'].hash) } + let(:context_params) { { sha: 'HEAD', variables: variables } } + + it { is_expected.to eq([{ location: location, content: content }, nil, 'HEAD'].hash) } end end end diff --git a/spec/lib/gitlab/ci/config/external/file/component_spec.rb b/spec/lib/gitlab/ci/config/external/file/component_spec.rb index a162a1a8abf..fe811bce9fe 100644 --- a/spec/lib/gitlab/ci/config/external/file/component_spec.rb +++ b/spec/lib/gitlab/ci/config/external/file/component_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::External::File::Component, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::External::File::Component, feature_category: :pipeline_composition do let_it_be(:context_project) { create(:project, :repository) } let_it_be(:project) { create(:project, :repository) } let_it_be(:user) { create(:user) } @@ -121,7 +121,7 @@ RSpec.describe Gitlab::Ci::Config::External::File::Component, feature_category: it 'is invalid' do expect(subject).to be_falsy - expect(external_resource.error_message).to match(/does not have valid YAML syntax/) + expect(external_resource.error_message).to match(/does not have a valid YAML syntax/) end end end @@ -176,4 +176,35 @@ RSpec.describe Gitlab::Ci::Config::External::File::Component, feature_category: variables: context.variables) end end + + describe '#to_hash' do + context 'when interpolation is being used' do + let(:response) do + ServiceResponse.success(payload: { content: content, path: path }) + end + + let(:path) do + instance_double(::Gitlab::Ci::Components::InstancePath, project: project, sha: '12345') + end + + let(:content) do + <<~YAML + spec: + inputs: + env: + --- + deploy: + script: deploy $[[ inputs.env ]] + YAML + end + + let(:params) do + { component: 'gitlab.com/acme/components/my-component@1.0', with: { env: 'production' } } + end + + it 'correctly interpolates the content' do + expect(external_resource.to_hash).to eq({ deploy: { script: 'deploy production' } }) + end + end + end end diff --git a/spec/lib/gitlab/ci/config/external/file/local_spec.rb b/spec/lib/gitlab/ci/config/external/file/local_spec.rb index b5895b4bc81..0643bf0c046 100644 --- a/spec/lib/gitlab/ci/config/external/file/local_spec.rb +++ b/spec/lib/gitlab/ci/config/external/file/local_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::External::File::Local, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::External::File::Local, feature_category: :pipeline_composition do include RepoHelpers let_it_be(:project) { create(:project, :repository) } @@ -228,6 +228,34 @@ RSpec.describe Gitlab::Ci::Config::External::File::Local, feature_category: :pip expect(local_file.to_hash).to include(:rspec) end end + + context 'when interpolaton is being used' do + let(:local_file_content) do + <<~YAML + spec: + inputs: + website: + --- + test: + script: cap deploy $[[ inputs.website ]] + YAML + end + + let(:location) { '/lib/gitlab/ci/templates/existent-file.yml' } + let(:params) { { local: location, inputs: { website: 'gitlab.com' } } } + + before do + allow_any_instance_of(described_class) + .to receive(:fetch_local_content) + .and_return(local_file_content) + end + + it 'correctly interpolates the local template' do + expect(local_file).to be_valid + expect(local_file.to_hash) + .to eq({ test: { script: 'cap deploy gitlab.com' } }) + end + end end describe '#metadata' do diff --git a/spec/lib/gitlab/ci/config/external/file/project_spec.rb b/spec/lib/gitlab/ci/config/external/file/project_spec.rb index abe38cdbc3e..636241ed763 100644 --- a/spec/lib/gitlab/ci/config/external/file/project_spec.rb +++ b/spec/lib/gitlab/ci/config/external/file/project_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::External::File::Project, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::External::File::Project, feature_category: :pipeline_composition do include RepoHelpers let_it_be(:context_project) { create(:project) } @@ -97,6 +97,36 @@ RSpec.describe Gitlab::Ci::Config::External::File::Project, feature_category: :p end end + context 'when a valid path is used in uppercase' do + let(:params) do + { project: project.full_path.upcase, file: '/file.yml' } + end + + around do |example| + create_and_delete_files(project, { '/file.yml' => 'image: image:1.0' }) do + example.run + end + end + + it { is_expected.to be_truthy } + end + + context 'when a valid different case path is used' do + let_it_be(:project) { create(:project, :repository, path: 'mY-teSt-proJect', name: 'My Test Project') } + + let(:params) do + { project: "#{project.namespace.full_path}/my-test-projecT", file: '/file.yml' } + end + + around do |example| + create_and_delete_files(project, { '/file.yml' => 'image: image:1.0' }) do + example.run + end + end + + it { is_expected.to be_truthy } + end + context 'when a valid path with custom ref is used' do let(:params) do { project: project.full_path, ref: 'master', file: '/file.yml' } @@ -230,16 +260,16 @@ RSpec.describe Gitlab::Ci::Config::External::File::Project, feature_category: :p } context 'when project name and ref include masked variables' do - let(:project_name) { 'my_project_name' } + let_it_be(:project) { create(:project, :repository, path: 'my_project_path') } + let(:branch_name) { 'merge-commit-analyze-after' } - let(:project) { create(:project, :repository, name: project_name) } let(:namespace_path) { project.namespace.full_path } let(:included_project_sha) { project.commit(branch_name).sha } let(:variables) do Gitlab::Ci::Variables::Collection.new( [ - { key: 'VAR1', value: project_name, masked: true }, + { key: 'VAR1', value: 'my_project_path', masked: true }, { key: 'VAR2', value: branch_name, masked: true } ]) end @@ -259,4 +289,37 @@ RSpec.describe Gitlab::Ci::Config::External::File::Project, feature_category: :p } end end + + describe '#to_hash' do + context 'when interpolation is being used' do + before do + project.repository.create_file( + user, + 'template-file.yml', + template, + message: 'Add template', + branch_name: 'master' + ) + end + + let(:template) do + <<~YAML + spec: + inputs: + name: + --- + rspec: + script: rspec --suite $[[ inputs.name ]] + YAML + end + + let(:params) do + { file: 'template-file.yml', ref: 'master', project: project.full_path, inputs: { name: 'abc' } } + end + + it 'correctly interpolates the content' do + expect(project_file.to_hash).to eq({ rspec: { script: 'rspec --suite abc' } }) + end + end + end end diff --git a/spec/lib/gitlab/ci/config/external/file/remote_spec.rb b/spec/lib/gitlab/ci/config/external/file/remote_spec.rb index 27f401db76e..f8d3d1019f5 100644 --- a/spec/lib/gitlab/ci/config/external/file/remote_spec.rb +++ b/spec/lib/gitlab/ci/config/external/file/remote_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::External::File::Remote, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::External::File::Remote, feature_category: :pipeline_composition do include StubRequests let(:variables) { Gitlab::Ci::Variables::Collection.new([{ 'key' => 'GITLAB_TOKEN', 'value' => 'secret_file', 'masked' => true }]) } @@ -234,15 +234,13 @@ RSpec.describe Gitlab::Ci::Config::External::File::Remote, feature_category: :pi end describe '#to_hash' do - subject(:to_hash) { remote_file.to_hash } - before do stub_full_request(location).to_return(body: remote_file_content) end context 'with a valid remote file' do it 'returns the content as a hash' do - expect(to_hash).to eql( + expect(remote_file.to_hash).to eql( before_script: ["apt-get update -qq && apt-get install -y -qq sqlite3 libsqlite3-dev nodejs", "ruby -v", "which ruby", @@ -262,7 +260,7 @@ RSpec.describe Gitlab::Ci::Config::External::File::Remote, feature_category: :pi end it 'returns the content as a hash' do - expect(to_hash).to eql( + expect(remote_file.to_hash).to eql( include: [ { local: 'another-file.yml', rules: [{ exists: ['Dockerfile'] }] } @@ -270,5 +268,38 @@ RSpec.describe Gitlab::Ci::Config::External::File::Remote, feature_category: :pi ) end end + + context 'when interpolation has been used' do + let_it_be(:project) { create(:project) } + + let(:remote_file_content) do + <<~YAML + spec: + inputs: + include: + --- + include: + - local: $[[ inputs.include ]] + rules: + - exists: [Dockerfile] + YAML + end + + let(:params) { { remote: location, inputs: { include: 'some-file.yml' } } } + + let(:context_params) do + { sha: '12345', variables: variables, project: project, user: build(:user) } + end + + it 'returns the content as a hash' do + expect(remote_file).to be_valid + expect(remote_file.to_hash).to eql( + include: [ + { local: 'some-file.yml', + rules: [{ exists: ['Dockerfile'] }] } + ] + ) + end + end end end diff --git a/spec/lib/gitlab/ci/config/external/file/template_spec.rb b/spec/lib/gitlab/ci/config/external/file/template_spec.rb index 83e98874118..078b8831dc3 100644 --- a/spec/lib/gitlab/ci/config/external/file/template_spec.rb +++ b/spec/lib/gitlab/ci/config/external/file/template_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::External::File::Template, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::External::File::Template, feature_category: :pipeline_composition do let_it_be(:project) { create(:project) } let_it_be(:user) { create(:user) } @@ -130,4 +130,37 @@ RSpec.describe Gitlab::Ci::Config::External::File::Template, feature_category: : ) } end + + describe '#to_hash' do + context 'when interpolation is being used' do + before do + allow(Gitlab::Template::GitlabCiYmlTemplate) + .to receive(:find) + .and_return(template_double) + end + + let(:template_double) do + instance_double(Gitlab::Template::GitlabCiYmlTemplate, content: template_content) + end + + let(:template_content) do + <<~YAML + spec: + inputs: + env: + --- + deploy: + script: deploy $[[ inputs.env ]] + YAML + end + + let(:params) do + { template: template, inputs: { env: 'production' } } + end + + it 'correctly interpolates the content' do + expect(template_file.to_hash).to eq({ deploy: { script: 'deploy production' } }) + end + end + end end diff --git a/spec/lib/gitlab/ci/config/external/interpolator_spec.rb b/spec/lib/gitlab/ci/config/external/interpolator_spec.rb new file mode 100644 index 00000000000..fe6f97a66a5 --- /dev/null +++ b/spec/lib/gitlab/ci/config/external/interpolator_spec.rb @@ -0,0 +1,319 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Config::External::Interpolator, feature_category: :pipeline_composition do + let_it_be(:project) { create(:project) } + + let(:ctx) { instance_double(Gitlab::Ci::Config::External::Context, project: project, user: build(:user, id: 1234)) } + let(:result) { ::Gitlab::Ci::Config::Yaml::Result.new(config: [header, content]) } + + subject { described_class.new(result, arguments, ctx) } + + context 'when input data is valid' do + let(:header) do + { spec: { inputs: { website: nil } } } + end + + let(:content) do + { test: 'deploy $[[ inputs.website ]]' } + end + + let(:arguments) do + { website: 'gitlab.com' } + end + + it 'correctly interpolates the config' do + subject.interpolate! + + expect(subject).to be_valid + expect(subject.to_hash).to eq({ test: 'deploy gitlab.com' }) + end + + it 'tracks the event' do + expect(::Gitlab::UsageDataCounters::HLLRedisCounter).to receive(:track_event) + .with('ci_interpolation_users', { values: 1234 }) + + subject.interpolate! + end + end + + context 'when config has a syntax error' do + let(:result) { ::Gitlab::Ci::Config::Yaml::Result.new(error: ArgumentError.new) } + + let(:arguments) do + { website: 'gitlab.com' } + end + + it 'surfaces an error about invalid config' do + subject.interpolate! + + expect(subject).not_to be_valid + expect(subject.error_message).to eq subject.errors.first + expect(subject.errors).to include 'content does not have a valid YAML syntax' + end + end + + context 'when spec header is invalid' do + let(:header) do + { spec: { arguments: { website: nil } } } + end + + let(:content) do + { test: 'deploy $[[ inputs.website ]]' } + end + + let(:arguments) do + { website: 'gitlab.com' } + end + + it 'surfaces an error about invalid header' do + subject.interpolate! + + expect(subject).not_to be_valid + expect(subject.error_message).to eq subject.errors.first + expect(subject.errors).to include('header:spec config contains unknown keys: arguments') + end + end + + context 'when interpolation block is invalid' do + let(:header) do + { spec: { inputs: { website: nil } } } + end + + let(:content) do + { test: 'deploy $[[ inputs.abc ]]' } + end + + let(:arguments) do + { website: 'gitlab.com' } + end + + it 'correctly interpolates the config' do + subject.interpolate! + + expect(subject).not_to be_valid + expect(subject.errors).to include 'unknown interpolation key: `abc`' + expect(subject.error_message).to eq 'interpolation interrupted by errors, unknown interpolation key: `abc`' + end + end + + context 'when provided interpolation argument is invalid' do + let(:header) do + { spec: { inputs: { website: nil } } } + end + + let(:content) do + { test: 'deploy $[[ inputs.website ]]' } + end + + let(:arguments) do + { website: ['gitlab.com'] } + end + + it 'correctly interpolates the config' do + subject.interpolate! + + expect(subject).not_to be_valid + expect(subject.error_message).to eq subject.errors.first + expect(subject.errors).to include 'unsupported value in input argument `website`' + end + end + + context 'when multiple interpolation blocks are invalid' do + let(:header) do + { spec: { inputs: { website: nil } } } + end + + let(:content) do + { test: 'deploy $[[ inputs.something.abc ]] $[[ inputs.cde ]] $[[ efg ]]' } + end + + let(:arguments) do + { website: 'gitlab.com' } + end + + it 'correctly interpolates the config' do + subject.interpolate! + + expect(subject).not_to be_valid + expect(subject.error_message).to eq 'interpolation interrupted by errors, unknown interpolation key: `something`' + end + end + + describe '#to_hash' do + context 'when interpolation is disabled' do + before do + stub_feature_flags(ci_includable_files_interpolation: false) + end + + let(:header) do + { spec: { inputs: { website: nil } } } + end + + let(:content) do + { test: 'deploy $[[ inputs.website ]]' } + end + + let(:arguments) { {} } + + it 'returns an empty hash' do + subject.interpolate! + + expect(subject.to_hash).to be_empty + end + end + + context 'when interpolation is not used' do + let(:result) do + ::Gitlab::Ci::Config::Yaml::Result.new(config: content) + end + + let(:content) do + { test: 'deploy production' } + end + + let(:arguments) { nil } + + it 'returns original content' do + subject.interpolate! + + expect(subject.to_hash).to eq(content) + end + end + + context 'when interpolation is available' do + let(:header) do + { spec: { inputs: { website: nil } } } + end + + let(:content) do + { test: 'deploy $[[ inputs.website ]]' } + end + + let(:arguments) do + { website: 'gitlab.com' } + end + + it 'correctly interpolates content' do + subject.interpolate! + + expect(subject.to_hash).to eq({ test: 'deploy gitlab.com' }) + end + end + end + + describe '#ready?' do + let(:header) do + { spec: { inputs: { website: nil } } } + end + + let(:content) do + { test: 'deploy $[[ inputs.website ]]' } + end + + let(:arguments) do + { website: 'gitlab.com' } + end + + it 'returns false if interpolation has not been done yet' do + expect(subject).not_to be_ready + end + + it 'returns true if interpolation has been performed' do + subject.interpolate! + + expect(subject).to be_ready + end + + context 'when interpolation can not be performed' do + let(:result) do + ::Gitlab::Ci::Config::Yaml::Result.new(error: ArgumentError.new) + end + + it 'returns true if interpolator has preliminary errors' do + expect(subject).to be_ready + end + + it 'returns true if interpolation has been attempted' do + subject.interpolate! + + expect(subject).to be_ready + end + end + end + + describe '#interpolate?' do + let(:header) do + { spec: { inputs: { website: nil } } } + end + + let(:content) do + { test: 'deploy $[[ inputs.something.abc ]] $[[ inputs.cde ]] $[[ efg ]]' } + end + + let(:arguments) do + { website: 'gitlab.com' } + end + + context 'when interpolation can be performed' do + it 'will perform interpolation' do + expect(subject.interpolate?).to eq true + end + end + + context 'when interpolation is disabled' do + before do + stub_feature_flags(ci_includable_files_interpolation: false) + end + + it 'will not perform interpolation' do + expect(subject.interpolate?).to eq false + end + end + + context 'when an interpolation header is missing' do + let(:header) { nil } + + it 'will not perform interpolation' do + expect(subject.interpolate?).to eq false + end + end + + context 'when interpolator has preliminary errors' do + let(:result) do + ::Gitlab::Ci::Config::Yaml::Result.new(error: ArgumentError.new) + end + + it 'will not perform interpolation' do + expect(subject.interpolate?).to eq false + end + end + end + + describe '#has_header?' do + let(:content) do + { test: 'deploy $[[ inputs.something.abc ]] $[[ inputs.cde ]] $[[ efg ]]' } + end + + let(:arguments) do + { website: 'gitlab.com' } + end + + context 'when header is an empty hash' do + let(:header) { {} } + + it 'does not have a header available' do + expect(subject).not_to have_header + end + end + + context 'when header is not specified' do + let(:header) { nil } + + it 'does not have a header available' do + expect(subject).not_to have_header + end + end + end +end diff --git a/spec/lib/gitlab/ci/config/external/mapper/base_spec.rb b/spec/lib/gitlab/ci/config/external/mapper/base_spec.rb index 0fdcc5e8ff7..ce8f3756cbc 100644 --- a/spec/lib/gitlab/ci/config/external/mapper/base_spec.rb +++ b/spec/lib/gitlab/ci/config/external/mapper/base_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::External::Mapper::Base, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::External::Mapper::Base, feature_category: :pipeline_composition do let(:test_class) do Class.new(described_class) do def self.name diff --git a/spec/lib/gitlab/ci/config/external/mapper/filter_spec.rb b/spec/lib/gitlab/ci/config/external/mapper/filter_spec.rb index df2a2f0fd01..5195567ebb4 100644 --- a/spec/lib/gitlab/ci/config/external/mapper/filter_spec.rb +++ b/spec/lib/gitlab/ci/config/external/mapper/filter_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::External::Mapper::Filter, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::External::Mapper::Filter, feature_category: :pipeline_composition do let_it_be(:variables) do Gitlab::Ci::Variables::Collection.new.tap do |variables| variables.append(key: 'VARIABLE1', value: 'hello') diff --git a/spec/lib/gitlab/ci/config/external/mapper/location_expander_spec.rb b/spec/lib/gitlab/ci/config/external/mapper/location_expander_spec.rb index b14b6b0ca29..1e490bf1d16 100644 --- a/spec/lib/gitlab/ci/config/external/mapper/location_expander_spec.rb +++ b/spec/lib/gitlab/ci/config/external/mapper/location_expander_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::External::Mapper::LocationExpander, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::External::Mapper::LocationExpander, feature_category: :pipeline_composition do include RepoHelpers let_it_be(:project) { create(:project, :repository) } diff --git a/spec/lib/gitlab/ci/config/external/mapper/matcher_spec.rb b/spec/lib/gitlab/ci/config/external/mapper/matcher_spec.rb index 11c79e19cff..719c75dca80 100644 --- a/spec/lib/gitlab/ci/config/external/mapper/matcher_spec.rb +++ b/spec/lib/gitlab/ci/config/external/mapper/matcher_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::External::Mapper::Matcher, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::External::Mapper::Matcher, feature_category: :pipeline_composition do let_it_be(:variables) do Gitlab::Ci::Variables::Collection.new.tap do |variables| variables.append(key: 'A_MASKED_VAR', value: 'this-is-secret', masked: true) @@ -16,28 +16,56 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper::Matcher, feature_category: subject(:matcher) { described_class.new(context) } describe '#process' do - let(:locations) do - [ - { local: 'file.yml' }, - { file: 'file.yml', project: 'namespace/project' }, - { component: 'gitlab.com/org/component@1.0' }, - { remote: 'https://example.com/.gitlab-ci.yml' }, - { template: 'file.yml' }, - { artifact: 'generated.yml', job: 'test' } - ] + subject(:process) { matcher.process(locations) } + + context 'with ci_include_components FF disabled' do + before do + stub_feature_flags(ci_include_components: false) + end + + let(:locations) do + [ + { local: 'file.yml' }, + { file: 'file.yml', project: 'namespace/project' }, + { remote: 'https://example.com/.gitlab-ci.yml' }, + { template: 'file.yml' }, + { artifact: 'generated.yml', job: 'test' } + ] + end + + it 'returns an array of file objects' do + is_expected.to contain_exactly( + an_instance_of(Gitlab::Ci::Config::External::File::Local), + an_instance_of(Gitlab::Ci::Config::External::File::Project), + an_instance_of(Gitlab::Ci::Config::External::File::Remote), + an_instance_of(Gitlab::Ci::Config::External::File::Template), + an_instance_of(Gitlab::Ci::Config::External::File::Artifact) + ) + end end - subject(:process) { matcher.process(locations) } + context 'with ci_include_components FF enabled' do + let(:locations) do + [ + { local: 'file.yml' }, + { file: 'file.yml', project: 'namespace/project' }, + { component: 'gitlab.com/org/component@1.0' }, + { remote: 'https://example.com/.gitlab-ci.yml' }, + { template: 'file.yml' }, + { artifact: 'generated.yml', job: 'test' } + ] + end - it 'returns an array of file objects' do - is_expected.to contain_exactly( - an_instance_of(Gitlab::Ci::Config::External::File::Local), - an_instance_of(Gitlab::Ci::Config::External::File::Project), - an_instance_of(Gitlab::Ci::Config::External::File::Component), - an_instance_of(Gitlab::Ci::Config::External::File::Remote), - an_instance_of(Gitlab::Ci::Config::External::File::Template), - an_instance_of(Gitlab::Ci::Config::External::File::Artifact) - ) + it 'returns an array of file objects' do + is_expected.to contain_exactly( + an_instance_of(Gitlab::Ci::Config::External::File::Local), + an_instance_of(Gitlab::Ci::Config::External::File::Project), + an_instance_of(Gitlab::Ci::Config::External::File::Component), + an_instance_of(Gitlab::Ci::Config::External::File::Remote), + an_instance_of(Gitlab::Ci::Config::External::File::Template), + an_instance_of(Gitlab::Ci::Config::External::File::Artifact) + ) + end end context 'when a location is not valid' do diff --git a/spec/lib/gitlab/ci/config/external/mapper/normalizer_spec.rb b/spec/lib/gitlab/ci/config/external/mapper/normalizer_spec.rb index 709c234253b..09212833d84 100644 --- a/spec/lib/gitlab/ci/config/external/mapper/normalizer_spec.rb +++ b/spec/lib/gitlab/ci/config/external/mapper/normalizer_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::External::Mapper::Normalizer, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::External::Mapper::Normalizer, feature_category: :pipeline_composition do let_it_be(:variables) do Gitlab::Ci::Variables::Collection.new.tap do |variables| variables.append(key: 'VARIABLE1', value: 'config') diff --git a/spec/lib/gitlab/ci/config/external/mapper/variables_expander_spec.rb b/spec/lib/gitlab/ci/config/external/mapper/variables_expander_spec.rb index f7454dcd4be..5def516bb1e 100644 --- a/spec/lib/gitlab/ci/config/external/mapper/variables_expander_spec.rb +++ b/spec/lib/gitlab/ci/config/external/mapper/variables_expander_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::External::Mapper::VariablesExpander, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::External::Mapper::VariablesExpander, feature_category: :secrets_management do let_it_be(:variables) do Gitlab::Ci::Variables::Collection.new.tap do |variables| variables.append(key: 'VARIABLE1', value: 'hello') diff --git a/spec/lib/gitlab/ci/config/external/mapper/verifier_spec.rb b/spec/lib/gitlab/ci/config/external/mapper/verifier_spec.rb index a219666f24e..1ee46daa196 100644 --- a/spec/lib/gitlab/ci/config/external/mapper/verifier_spec.rb +++ b/spec/lib/gitlab/ci/config/external/mapper/verifier_spec.rb @@ -2,11 +2,11 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::External::Mapper::Verifier, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::External::Mapper::Verifier, feature_category: :pipeline_composition do include RepoHelpers include StubRequests - let_it_be(:project) { create(:project, :repository) } + let_it_be(:project) { create(:project, :small_repo) } let_it_be(:user) { project.owner } let(:context) do @@ -38,7 +38,7 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper::Verifier, feature_category: } end - around(:all) do |example| + around do |example| create_and_delete_files(project, project_files) do example.run end @@ -84,42 +84,140 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper::Verifier, feature_category: end context 'when files are project files' do - let_it_be(:included_project) { create(:project, :repository, namespace: project.namespace, creator: user) } + let_it_be(:included_project1) { create(:project, :small_repo, namespace: project.namespace, creator: user) } + let_it_be(:included_project2) { create(:project, :small_repo, namespace: project.namespace, creator: user) } let(:files) do [ Gitlab::Ci::Config::External::File::Project.new( - { file: 'myfolder/file1.yml', project: included_project.full_path }, context + { file: 'myfolder/file1.yml', project: included_project1.full_path }, context ), Gitlab::Ci::Config::External::File::Project.new( - { file: 'myfolder/file2.yml', project: included_project.full_path }, context + { file: 'myfolder/file2.yml', project: included_project1.full_path }, context ), Gitlab::Ci::Config::External::File::Project.new( - { file: 'myfolder/file3.yml', project: included_project.full_path }, context + { file: 'myfolder/file3.yml', project: included_project1.full_path, ref: 'master' }, context + ), + Gitlab::Ci::Config::External::File::Project.new( + { file: 'myfolder/file1.yml', project: included_project2.full_path }, context + ), + Gitlab::Ci::Config::External::File::Project.new( + { file: 'myfolder/file2.yml', project: included_project2.full_path }, context ) ] end - around(:all) do |example| - create_and_delete_files(included_project, project_files) do - example.run + around do |example| + create_and_delete_files(included_project1, project_files) do + create_and_delete_files(included_project2, project_files) do + example.run + end end end - it 'returns an array of file objects' do + it 'returns an array of valid file objects' do expect(process.map(&:location)).to contain_exactly( - 'myfolder/file1.yml', 'myfolder/file2.yml', 'myfolder/file3.yml' + 'myfolder/file1.yml', 'myfolder/file2.yml', 'myfolder/file3.yml', 'myfolder/file1.yml', 'myfolder/file2.yml' ) + + expect(process.all?(&:valid?)).to be_truthy end it 'adds files to the expandset' do - expect { process }.to change { context.expandset.count }.by(3) + expect { process }.to change { context.expandset.count }.by(5) end it 'calls Gitaly only once for all files', :request_store do - # 1 for project.commit.id, 3 for the sha check, 1 for the files + files # calling this to load project creations and the `project.commit.id` call + + # 3 for the sha check, 2 for the files in batch expect { process }.to change { Gitlab::GitalyClient.get_request_count }.by(5) end + + it 'queries with batch', :use_sql_query_cache do + files # calling this to load project creations and the `project.commit.id` call + + queries = ActiveRecord::QueryRecorder.new(skip_cached: false) { process } + projects_queries = queries.occurrences_starting_with('SELECT "projects"') + access_check_queries = queries.occurrences_starting_with('SELECT MAX("project_authorizations"."access_level")') + + # We could not reduce the number of projects queries because we need to call project for + # the `can_access_local_content?` and `sha` BatchLoaders. + expect(projects_queries.values.sum).to eq(2) + expect(access_check_queries.values.sum).to eq(2) + end + + context 'when the FF ci_batch_project_includes_context is disabled' do + before do + stub_feature_flags(ci_batch_project_includes_context: false) + end + + it 'returns an array of file objects' do + expect(process.map(&:location)).to contain_exactly( + 'myfolder/file1.yml', 'myfolder/file2.yml', 'myfolder/file3.yml', + 'myfolder/file1.yml', 'myfolder/file2.yml' + ) + end + + it 'adds files to the expandset' do + expect { process }.to change { context.expandset.count }.by(5) + end + + it 'calls Gitaly for all files', :request_store do + files # calling this to load project creations and the `project.commit.id` call + + # 5 for the sha check, 2 for the files in batch + expect { process }.to change { Gitlab::GitalyClient.get_request_count }.by(7) + end + + it 'queries without batch', :use_sql_query_cache do + files # calling this to load project creations and the `project.commit.id` call + + queries = ActiveRecord::QueryRecorder.new(skip_cached: false) { process } + projects_queries = queries.occurrences_starting_with('SELECT "projects"') + access_check_queries = queries.occurrences_starting_with( + 'SELECT MAX("project_authorizations"."access_level")' + ) + + expect(projects_queries.values.sum).to eq(5) + expect(access_check_queries.values.sum).to eq(5) + end + end + + context 'when a project is missing' do + let(:files) do + [ + Gitlab::Ci::Config::External::File::Project.new( + { file: 'myfolder/file1.yml', project: included_project1.full_path }, context + ), + Gitlab::Ci::Config::External::File::Project.new( + { file: 'myfolder/file2.yml', project: 'invalid-project' }, context + ) + ] + end + + it 'returns an array of file objects' do + expect(process.map(&:location)).to contain_exactly( + 'myfolder/file1.yml', 'myfolder/file2.yml' + ) + + expect(process.all?(&:valid?)).to be_falsey + end + + context 'when the FF ci_batch_project_includes_context is disabled' do + before do + stub_feature_flags(ci_batch_project_includes_context: false) + end + + it 'returns an array of file objects' do + expect(process.map(&:location)).to contain_exactly( + 'myfolder/file1.yml', 'myfolder/file2.yml' + ) + + expect(process.all?(&:valid?)).to be_falsey + end + end + end end context 'when a file includes other files' do @@ -150,7 +248,30 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper::Verifier, feature_category: end end - context 'when total file count exceeds max_includes' do + describe 'max includes detection' do + shared_examples 'verifies max includes' do + context 'when total file count is equal to max_includes' do + before do + allow(context).to receive(:max_includes).and_return(expected_total_file_count) + end + + it 'adds the expected number of files to expandset' do + expect { process }.not_to raise_error + expect(context.expandset.count).to eq(expected_total_file_count) + end + end + + context 'when total file count exceeds max_includes' do + before do + allow(context).to receive(:max_includes).and_return(expected_total_file_count - 1) + end + + it 'raises error' do + expect { process }.to raise_error(expected_error_class) + end + end + end + context 'when files are nested' do let(:files) do [ @@ -158,9 +279,21 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper::Verifier, feature_category: ] end - it 'raises Processor::IncludeError' do - allow(context).to receive(:max_includes).and_return(1) - expect { process }.to raise_error(Gitlab::Ci::Config::External::Processor::IncludeError) + let(:expected_total_file_count) { 4 } # Includes nested_configs.yml + 3 nested files + let(:expected_error_class) { Gitlab::Ci::Config::External::Processor::IncludeError } + + it_behaves_like 'verifies max includes' + + context 'when duplicate files are included' do + let(:expected_total_file_count) { 8 } # 2 x (Includes nested_configs.yml + 3 nested files) + let(:files) do + [ + Gitlab::Ci::Config::External::File::Local.new({ local: 'nested_configs.yml' }, context), + Gitlab::Ci::Config::External::File::Local.new({ local: 'nested_configs.yml' }, context) + ] + end + + it_behaves_like 'verifies max includes' end end @@ -172,34 +305,112 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper::Verifier, feature_category: ] end - it 'raises Mapper::TooManyIncludesError' do - allow(context).to receive(:max_includes).and_return(1) - expect { process }.to raise_error(Gitlab::Ci::Config::External::Mapper::TooManyIncludesError) + let(:expected_total_file_count) { files.count } + let(:expected_error_class) { Gitlab::Ci::Config::External::Mapper::TooManyIncludesError } + + it_behaves_like 'verifies max includes' + + context 'when duplicate files are included' do + let(:files) do + [ + Gitlab::Ci::Config::External::File::Local.new({ local: 'myfolder/file1.yml' }, context), + Gitlab::Ci::Config::External::File::Local.new({ local: 'myfolder/file2.yml' }, context), + Gitlab::Ci::Config::External::File::Local.new({ local: 'myfolder/file2.yml' }, context) + ] + end + + let(:expected_total_file_count) { files.count } + + it_behaves_like 'verifies max includes' end end - context 'when files are duplicates' do + context 'when there is a circular include' do + let(:project_files) do + { + 'myfolder/file1.yml' => <<~YAML + include: myfolder/file1.yml + YAML + } + end + let(:files) do [ - Gitlab::Ci::Config::External::File::Local.new({ local: 'myfolder/file1.yml' }, context), - Gitlab::Ci::Config::External::File::Local.new({ local: 'myfolder/file1.yml' }, context), Gitlab::Ci::Config::External::File::Local.new({ local: 'myfolder/file1.yml' }, context) ] end + before do + allow(context).to receive(:max_includes).and_return(10) + end + it 'raises error' do - allow(context).to receive(:max_includes).and_return(2) - expect { process }.to raise_error(Gitlab::Ci::Config::External::Mapper::TooManyIncludesError) + expect { process }.to raise_error(Gitlab::Ci::Config::External::Processor::IncludeError) end + end - context 'when FF ci_includes_count_duplicates is disabled' do - before do - stub_feature_flags(ci_includes_count_duplicates: false) - end + context 'when a file is an internal include' do + let(:project_files) do + { + 'myfolder/file1.yml' => <<~YAML, + my_build: + script: echo Hello World + YAML + '.internal-include.yml' => <<~YAML + include: + - local: myfolder/file1.yml + YAML + } + end + + let(:files) do + [ + Gitlab::Ci::Config::External::File::Local.new({ local: '.internal-include.yml' }, context) + ] + end - it 'does not raise error' do - allow(context).to receive(:max_includes).and_return(2) + let(:total_file_count) { 2 } # Includes .internal-include.yml + myfolder/file1.yml + let(:pipeline_config) { instance_double(Gitlab::Ci::ProjectConfig) } + + let(:context) do + Gitlab::Ci::Config::External::Context.new( + project: project, + user: user, + sha: project.commit.id, + pipeline_config: pipeline_config + ) + end + + before do + allow(pipeline_config).to receive(:internal_include_prepended?).and_return(true) + allow(context).to receive(:max_includes).and_return(1) + end + + context 'when total file count excluding internal include is equal to max_includes' do + it 'does not add the internal include to expandset' do expect { process }.not_to raise_error + expect(context.expandset.count).to eq(total_file_count - 1) + expect(context.expandset.first.location).to eq('myfolder/file1.yml') + end + end + + context 'when total file count excluding internal include exceeds max_includes' do + let(:project_files) do + { + 'myfolder/file1.yml' => <<~YAML, + my_build: + script: echo Hello World + YAML + '.internal-include.yml' => <<~YAML + include: + - local: myfolder/file1.yml + - local: myfolder/file1.yml + YAML + } + end + + it 'raises error' do + expect { process }.to raise_error(Gitlab::Ci::Config::External::Processor::IncludeError) end end end diff --git a/spec/lib/gitlab/ci/config/external/mapper_spec.rb b/spec/lib/gitlab/ci/config/external/mapper_spec.rb index b3115617084..56d1ddee4b8 100644 --- a/spec/lib/gitlab/ci/config/external/mapper_spec.rb +++ b/spec/lib/gitlab/ci/config/external/mapper_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::External::Mapper, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::External::Mapper, feature_category: :pipeline_composition do include StubRequests include RepoHelpers @@ -234,17 +234,6 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper, feature_category: :pipeline process expect(context.expandset.size).to eq(2) end - - context 'when FF ci_includes_count_duplicates is disabled' do - before do - stub_feature_flags(ci_includes_count_duplicates: false) - end - - it 'has expanset with one' do - process - expect(context.expandset.size).to eq(1) - end - end end context 'when passing max number of files' do diff --git a/spec/lib/gitlab/ci/config/external/processor_spec.rb b/spec/lib/gitlab/ci/config/external/processor_spec.rb index bb65c2ef10c..74afb3b1e97 100644 --- a/spec/lib/gitlab/ci/config/external/processor_spec.rb +++ b/spec/lib/gitlab/ci/config/external/processor_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::External::Processor, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::External::Processor, feature_category: :pipeline_composition do include StubRequests include RepoHelpers @@ -221,7 +221,7 @@ RSpec.describe Gitlab::Ci::Config::External::Processor, feature_category: :pipel it 'raises an error' do expect { processor.perform }.to raise_error( described_class::IncludeError, - "Included file `lib/gitlab/ci/templates/template.yml` does not have valid YAML syntax!" + '`lib/gitlab/ci/templates/template.yml`: content does not have a valid YAML syntax' ) end end diff --git a/spec/lib/gitlab/ci/config/external/rules_spec.rb b/spec/lib/gitlab/ci/config/external/rules_spec.rb index 227b62d8ce8..cc73338b5a8 100644 --- a/spec/lib/gitlab/ci/config/external/rules_spec.rb +++ b/spec/lib/gitlab/ci/config/external/rules_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::External::Rules, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::External::Rules, feature_category: :pipeline_composition do let(:rule_hashes) {} subject(:rules) { described_class.new(rule_hashes) } diff --git a/spec/lib/gitlab/ci/config/header/input_spec.rb b/spec/lib/gitlab/ci/config/header/input_spec.rb new file mode 100644 index 00000000000..73b5b8f9497 --- /dev/null +++ b/spec/lib/gitlab/ci/config/header/input_spec.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Config::Header::Input, feature_category: :pipeline_composition do + let(:factory) do + Gitlab::Config::Entry::Factory + .new(described_class) + .value(input_hash) + .with(key: input_name) + end + + let(:input_name) { 'foo' } + + subject(:config) { factory.create!.tap(&:compose!) } + + shared_examples 'a valid input' do + let(:expected_hash) { input_hash } + + it 'passes validations' do + expect(config).to be_valid + expect(config.errors).to be_empty + end + + it 'returns the value' do + expect(config.value).to eq(expected_hash) + end + end + + shared_examples 'an invalid input' do + let(:expected_hash) { input_hash } + + it 'fails validations' do + expect(config).not_to be_valid + expect(config.errors).to eq(expected_errors) + end + + it 'returns the value' do + expect(config.value).to eq(expected_hash) + end + end + + context 'when has a default value' do + let(:input_hash) { { default: 'bar' } } + + it_behaves_like 'a valid input' + end + + context 'when is a required required input' do + let(:input_hash) { nil } + + it_behaves_like 'a valid input' + end + + context 'when contains unknown keywords' do + let(:input_hash) { { test: 123 } } + let(:expected_errors) { ['foo config contains unknown keys: test'] } + + it_behaves_like 'an invalid input' + end + + context 'when has invalid name' do + let(:input_name) { [123] } + let(:input_hash) { {} } + + let(:expected_errors) { ['123 key must be an alphanumeric string'] } + + it_behaves_like 'an invalid input' + end +end diff --git a/spec/lib/gitlab/ci/config/header/root_spec.rb b/spec/lib/gitlab/ci/config/header/root_spec.rb new file mode 100644 index 00000000000..55f77137619 --- /dev/null +++ b/spec/lib/gitlab/ci/config/header/root_spec.rb @@ -0,0 +1,133 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Config::Header::Root, feature_category: :pipeline_composition do + let(:factory) { Gitlab::Config::Entry::Factory.new(described_class).value(header_hash) } + + subject(:config) { factory.create!.tap(&:compose!) } + + shared_examples 'a valid header' do + let(:expected_hash) { header_hash } + + it 'passes validations' do + expect(config).to be_valid + expect(config.errors).to be_empty + end + + it 'returns the value' do + expect(config.value).to eq(expected_hash) + end + end + + shared_examples 'an invalid header' do + let(:expected_hash) { header_hash } + + it 'fails validations' do + expect(config).not_to be_valid + expect(config.errors).to eq(expected_errors) + end + + it 'returns the value' do + expect(config.value).to eq(expected_hash) + end + end + + context 'when header contains default and required values for inputs' do + let(:header_hash) do + { + spec: { + inputs: { + test: {}, + foo: { + default: 'bar' + } + } + } + } + end + + it_behaves_like 'a valid header' + end + + context 'when header contains minimal data' do + let(:header_hash) do + { + spec: { + inputs: nil + } + } + end + + it_behaves_like 'a valid header' do + let(:expected_hash) { { spec: {} } } + end + end + + context 'when header contains required inputs' do + let(:header_hash) do + { + spec: { + inputs: { foo: nil } + } + } + end + + it_behaves_like 'a valid header' do + let(:expected_hash) do + { + spec: { + inputs: { foo: {} } + } + } + end + end + end + + context 'when header contains unknown keywords' do + let(:header_hash) { { test: 123 } } + let(:expected_errors) { ['root config contains unknown keys: test'] } + + it_behaves_like 'an invalid header' + end + + context 'when header input entry has an unknown key' do + let(:header_hash) do + { + spec: { + inputs: { + foo: { + bad: 'value' + } + } + } + } + end + + let(:expected_errors) { ['spec:inputs:foo config contains unknown keys: bad'] } + + it_behaves_like 'an invalid header' + end + + describe '#inputs_value' do + let(:header_hash) do + { + spec: { + inputs: { + foo: nil, + bar: { + default: 'baz' + } + } + } + } + end + + it 'returns the inputs' do + expect(config.inputs_value).to eq({ + foo: {}, + bar: { default: 'baz' } + }) + end + end +end diff --git a/spec/lib/gitlab/ci/config/header/spec_spec.rb b/spec/lib/gitlab/ci/config/header/spec_spec.rb new file mode 100644 index 00000000000..74cfb39dfd5 --- /dev/null +++ b/spec/lib/gitlab/ci/config/header/spec_spec.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Config::Header::Spec, feature_category: :pipeline_composition do + let(:factory) { Gitlab::Config::Entry::Factory.new(described_class).value(spec_hash) } + + subject(:config) { factory.create!.tap(&:compose!) } + + context 'when spec contains default values for inputs' do + let(:spec_hash) do + { + inputs: { + foo: { + default: 'bar' + } + } + } + end + + it 'passes validations' do + expect(config).to be_valid + expect(config.errors).to be_empty + end + + it 'returns the value' do + expect(config.value).to eq(spec_hash) + end + end + + context 'when spec contains a required value' do + let(:spec_hash) do + { inputs: { foo: nil } } + end + + it 'parses the config correctly' do + expect(config).to be_valid + expect(config.errors).to be_empty + expect(config.value).to eq({ inputs: { foo: {} } }) + end + end + + context 'when spec contains unknown keywords' do + let(:spec_hash) { { test: 123 } } + let(:expected_errors) { ['spec config contains unknown keys: test'] } + + it 'fails validations' do + expect(config).not_to be_valid + expect(config.errors).to eq(expected_errors) + end + + it 'returns the value' do + expect(config.value).to eq(spec_hash) + end + end +end diff --git a/spec/lib/gitlab/ci/config/normalizer/number_strategy_spec.rb b/spec/lib/gitlab/ci/config/normalizer/number_strategy_spec.rb index 06f47fe11c6..965963d40cd 100644 --- a/spec/lib/gitlab/ci/config/normalizer/number_strategy_spec.rb +++ b/spec/lib/gitlab/ci/config/normalizer/number_strategy_spec.rb @@ -53,6 +53,22 @@ RSpec.describe Gitlab::Ci::Config::Normalizer::NumberStrategy do end end + shared_examples 'single parallelized job' do + it { expect(subject.size).to eq(1) } + + it 'has attributes' do + expect(subject.map(&:attributes)).to match_array( + [ + { name: 'test 1/1', instance: 1, parallel: { total: 1 } } + ] + ) + end + + it 'has parallelized name' do + expect(subject.map(&:name)).to match_array(['test 1/1']) + end + end + context 'with numbers' do let(:config) { 3 } @@ -64,5 +80,11 @@ RSpec.describe Gitlab::Ci::Config::Normalizer::NumberStrategy do it_behaves_like 'parallelized job' end + + context 'with one' do + let(:config) { 1 } + + it_behaves_like 'single parallelized job' + end end end diff --git a/spec/lib/gitlab/ci/config/yaml/result_spec.rb b/spec/lib/gitlab/ci/config/yaml/result_spec.rb new file mode 100644 index 00000000000..72d96349668 --- /dev/null +++ b/spec/lib/gitlab/ci/config/yaml/result_spec.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Config::Yaml::Result, feature_category: :pipeline_composition do + it 'does not have a header when config is a single hash' do + result = described_class.new(config: { a: 1, b: 2 }) + + expect(result).not_to have_header + end + + context 'when config is an array of hashes' do + context 'when first document matches the header schema' do + it 'has a header' do + result = described_class.new(config: [{ spec: { inputs: {} } }, { b: 2 }]) + + expect(result).to have_header + expect(result.header).to eq({ spec: { inputs: {} } }) + expect(result.content).to eq({ b: 2 }) + end + end + + context 'when first document does not match the header schema' do + it 'does not have header' do + result = described_class.new(config: [{ a: 1 }, { b: 2 }]) + + expect(result).not_to have_header + expect(result.content).to eq({ a: 1 }) + end + end + end + + context 'when the first document is undefined' do + it 'does not have header' do + result = described_class.new(config: [nil, { a: 1 }]) + + expect(result).not_to have_header + expect(result.content).to be_nil + end + end + + it 'raises an error when reading a header when there is none' do + result = described_class.new(config: { b: 2 }) + + expect { result.header }.to raise_error(ArgumentError) + end + + it 'stores an error / exception when initialized with it' do + result = described_class.new(error: ArgumentError.new('abc')) + + expect(result).not_to be_valid + expect(result.error).to be_a ArgumentError + end +end diff --git a/spec/lib/gitlab/ci/config/yaml_spec.rb b/spec/lib/gitlab/ci/config/yaml_spec.rb index 4b34553f55e..beb872071d2 100644 --- a/spec/lib/gitlab/ci/config/yaml_spec.rb +++ b/spec/lib/gitlab/ci/config/yaml_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config::Yaml, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config::Yaml, feature_category: :pipeline_composition do describe '.load!' do it 'loads a single-doc YAML file' do yaml = <<~YAML @@ -50,6 +50,15 @@ RSpec.describe Gitlab::Ci::Config::Yaml, feature_category: :pipeline_authoring d }) end + context 'when YAML is invalid' do + let(:yaml) { 'some: invalid: syntax' } + + it 'raises an error' do + expect { described_class.load!(yaml) } + .to raise_error ::Gitlab::Config::Loader::FormatError, /mapping values are not allowed in this context/ + end + end + context 'when ci_multi_doc_yaml is disabled' do before do stub_feature_flags(ci_multi_doc_yaml: false) @@ -102,4 +111,152 @@ RSpec.describe Gitlab::Ci::Config::Yaml, feature_category: :pipeline_authoring d end end end + + describe '.load_result!' do + let_it_be(:project) { create(:project) } + + subject(:result) { described_class.load_result!(yaml, project: project) } + + context 'when syntax is invalid' do + let(:yaml) { 'some: invalid: syntax' } + + it 'returns an invalid result object' do + expect(result).not_to be_valid + expect(result.error).to be_a ::Gitlab::Config::Loader::FormatError + end + end + + context 'when the first document is a header' do + context 'with explicit document start marker' do + let(:yaml) do + <<~YAML + --- + spec: + --- + b: 2 + YAML + end + + it 'considers the first document as header and the second as content' do + expect(result).to be_valid + expect(result.error).to be_nil + expect(result.header).to eq({ spec: nil }) + expect(result.content).to eq({ b: 2 }) + end + end + end + + context 'when first document is empty' do + let(:yaml) do + <<~YAML + --- + --- + b: 2 + YAML + end + + it 'considers the first document as header and the second as content' do + expect(result).not_to have_header + end + end + + context 'when first document is an empty hash' do + let(:yaml) do + <<~YAML + {} + --- + b: 2 + YAML + end + + it 'returns second document as a content' do + expect(result).not_to have_header + expect(result.content).to eq({ b: 2 }) + end + end + + context 'when first an array' do + let(:yaml) do + <<~YAML + --- + - a + - b + --- + b: 2 + YAML + end + + it 'considers the first document as header and the second as content' do + expect(result).not_to have_header + end + end + + context 'when the first document is not a header' do + let(:yaml) do + <<~YAML + a: 1 + --- + b: 2 + YAML + end + + it 'considers the first document as content for backwards compatibility' do + expect(result).to be_valid + expect(result.error).to be_nil + expect(result).not_to have_header + expect(result.content).to eq({ a: 1 }) + end + + context 'with explicit document start marker' do + let(:yaml) do + <<~YAML + --- + a: 1 + --- + b: 2 + YAML + end + + it 'considers the first document as content for backwards compatibility' do + expect(result).to be_valid + expect(result.error).to be_nil + expect(result).not_to have_header + expect(result.content).to eq({ a: 1 }) + end + end + end + + context 'when the first document is not a header and second document is empty' do + let(:yaml) do + <<~YAML + a: 1 + --- + YAML + end + + it 'considers the first document as content' do + expect(result).to be_valid + expect(result.error).to be_nil + expect(result).not_to have_header + expect(result.content).to eq({ a: 1 }) + end + + context 'with explicit document start marker' do + let(:yaml) do + <<~YAML + --- + a: 1 + --- + YAML + end + + it 'considers the first document as content' do + expect(result).to be_valid + expect(result.error).to be_nil + expect(result).not_to have_header + expect(result.content).to eq({ a: 1 }) + end + end + end + end end diff --git a/spec/lib/gitlab/ci/config_spec.rb b/spec/lib/gitlab/ci/config_spec.rb index 5cdc9c21561..fdf152b3584 100644 --- a/spec/lib/gitlab/ci/config_spec.rb +++ b/spec/lib/gitlab/ci/config_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Config, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Config, feature_category: :pipeline_composition do include StubRequests include RepoHelpers diff --git a/spec/lib/gitlab/ci/input/arguments/base_spec.rb b/spec/lib/gitlab/ci/input/arguments/base_spec.rb new file mode 100644 index 00000000000..ed8e99b7257 --- /dev/null +++ b/spec/lib/gitlab/ci/input/arguments/base_spec.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +RSpec.describe Gitlab::Ci::Input::Arguments::Base, feature_category: :pipeline_composition do + subject do + Class.new(described_class) do + def validate!; end + def to_value; end + end + end + + it 'fabricates an invalid input argument if unknown value is provided' do + argument = subject.new(:something, { spec: 123 }, [:a, :b]) + + expect(argument).not_to be_valid + expect(argument.errors.first).to eq 'unsupported value in input argument `something`' + end +end diff --git a/spec/lib/gitlab/ci/input/arguments/default_spec.rb b/spec/lib/gitlab/ci/input/arguments/default_spec.rb new file mode 100644 index 00000000000..bc0cee6ac4e --- /dev/null +++ b/spec/lib/gitlab/ci/input/arguments/default_spec.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +RSpec.describe Gitlab::Ci::Input::Arguments::Default, feature_category: :pipeline_composition do + it 'returns a user-provided value if it is present' do + argument = described_class.new(:website, { default: 'https://gitlab.com' }, 'https://example.gitlab.com') + + expect(argument).to be_valid + expect(argument.to_value).to eq 'https://example.gitlab.com' + expect(argument.to_hash).to eq({ website: 'https://example.gitlab.com' }) + end + + it 'returns an empty value if user-provider input is empty' do + argument = described_class.new(:website, { default: 'https://gitlab.com' }, '') + + expect(argument).to be_valid + expect(argument.to_value).to eq '' + expect(argument.to_hash).to eq({ website: '' }) + end + + it 'returns a default value if user-provider one is unknown' do + argument = described_class.new(:website, { default: 'https://gitlab.com' }, nil) + + expect(argument).to be_valid + expect(argument.to_value).to eq 'https://gitlab.com' + expect(argument.to_hash).to eq({ website: 'https://gitlab.com' }) + end + + it 'returns an error if the default argument has not been recognized' do + argument = described_class.new(:website, { default: ['gitlab.com'] }, 'abc') + + expect(argument).not_to be_valid + end + + it 'returns an error if the argument has not been fabricated correctly' do + argument = described_class.new(:website, { required: 'https://gitlab.com' }, 'https://example.gitlab.com') + + expect(argument).not_to be_valid + end + + describe '.matches?' do + it 'matches specs with default configuration' do + expect(described_class.matches?({ default: 'abc' })).to be true + end + + it 'does not match specs different configuration keyword' do + expect(described_class.matches?({ options: %w[a b] })).to be false + expect(described_class.matches?('a b c')).to be false + expect(described_class.matches?(%w[default a])).to be false + end + end +end diff --git a/spec/lib/gitlab/ci/input/arguments/options_spec.rb b/spec/lib/gitlab/ci/input/arguments/options_spec.rb new file mode 100644 index 00000000000..17e3469b294 --- /dev/null +++ b/spec/lib/gitlab/ci/input/arguments/options_spec.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +RSpec.describe Gitlab::Ci::Input::Arguments::Options, feature_category: :pipeline_composition do + it 'returns a user-provided value if it is an allowed one' do + argument = described_class.new(:run, { options: %w[opt1 opt2] }, 'opt1') + + expect(argument).to be_valid + expect(argument.to_value).to eq 'opt1' + expect(argument.to_hash).to eq({ run: 'opt1' }) + end + + it 'returns an error if user-provided value is not allowlisted' do + argument = described_class.new(:run, { options: %w[opt1 opt2] }, 'opt3') + + expect(argument).not_to be_valid + expect(argument.errors.first).to eq '`run` input: argument value opt3 not allowlisted' + end + + it 'returns an error if specification is not correct' do + argument = described_class.new(:website, { options: nil }, 'opt1') + + expect(argument).not_to be_valid + expect(argument.errors.first).to eq '`website` input: argument specification invalid' + end + + it 'returns an error if specification is using a hash' do + argument = described_class.new(:website, { options: { a: 1 } }, 'opt1') + + expect(argument).not_to be_valid + expect(argument.errors.first).to eq '`website` input: argument specification invalid' + end + + it 'returns an empty value if it is allowlisted' do + argument = described_class.new(:run, { options: ['opt1', ''] }, '') + + expect(argument).to be_valid + expect(argument.to_value).to be_empty + expect(argument.to_hash).to eq({ run: '' }) + end + + describe '.matches?' do + it 'matches specs with options configuration' do + expect(described_class.matches?({ options: %w[a b] })).to be true + end + + it 'does not match specs different configuration keyword' do + expect(described_class.matches?({ default: 'abc' })).to be false + expect(described_class.matches?(['options'])).to be false + expect(described_class.matches?('options')).to be false + end + end +end diff --git a/spec/lib/gitlab/ci/input/arguments/required_spec.rb b/spec/lib/gitlab/ci/input/arguments/required_spec.rb new file mode 100644 index 00000000000..847272998c2 --- /dev/null +++ b/spec/lib/gitlab/ci/input/arguments/required_spec.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +RSpec.describe Gitlab::Ci::Input::Arguments::Required, feature_category: :pipeline_composition do + it 'returns a user-provided value if it is present' do + argument = described_class.new(:website, nil, 'https://example.gitlab.com') + + expect(argument).to be_valid + expect(argument.to_value).to eq 'https://example.gitlab.com' + expect(argument.to_hash).to eq({ website: 'https://example.gitlab.com' }) + end + + it 'returns an empty value if user-provider value is empty' do + argument = described_class.new(:website, nil, '') + + expect(argument).to be_valid + expect(argument.to_hash).to eq(website: '') + end + + it 'returns an error if user-provided value is unspecified' do + argument = described_class.new(:website, nil, nil) + + expect(argument).not_to be_valid + expect(argument.errors.first).to eq '`website` input: required value has not been provided' + end + + describe '.matches?' do + it 'matches specs without configuration' do + expect(described_class.matches?(nil)).to be true + end + + it 'matches specs with empty configuration' do + expect(described_class.matches?('')).to be true + end + + it 'matches specs with an empty hash configuration' do + expect(described_class.matches?({})).to be true + end + + it 'does not match specs with configuration' do + expect(described_class.matches?({ options: %w[a b] })).to be false + end + end +end diff --git a/spec/lib/gitlab/ci/input/arguments/unknown_spec.rb b/spec/lib/gitlab/ci/input/arguments/unknown_spec.rb new file mode 100644 index 00000000000..1270423ac72 --- /dev/null +++ b/spec/lib/gitlab/ci/input/arguments/unknown_spec.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +RSpec.describe Gitlab::Ci::Input::Arguments::Unknown, feature_category: :pipeline_composition do + it 'raises an error when someone tries to evaluate the value' do + argument = described_class.new(:website, nil, 'https://example.gitlab.com') + + expect(argument).not_to be_valid + expect { argument.to_value }.to raise_error ArgumentError + end + + describe '.matches?' do + it 'always matches' do + expect(described_class.matches?('abc')).to be true + end + end +end diff --git a/spec/lib/gitlab/ci/input/inputs_spec.rb b/spec/lib/gitlab/ci/input/inputs_spec.rb new file mode 100644 index 00000000000..5d2d5192299 --- /dev/null +++ b/spec/lib/gitlab/ci/input/inputs_spec.rb @@ -0,0 +1,126 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +RSpec.describe Gitlab::Ci::Input::Inputs, feature_category: :pipeline_composition do + describe '#valid?' do + let(:spec) { { website: nil } } + + it 'describes user-provided inputs' do + inputs = described_class.new(spec, { website: 'http://example.gitlab.com' }) + + expect(inputs).to be_valid + end + end + + context 'when proper specification has been provided' do + let(:spec) do + { + website: nil, + env: { default: 'development' }, + run: { options: %w[tests spec e2e] } + } + end + + let(:args) { { website: 'https://gitlab.com', run: 'tests' } } + + it 'fabricates desired input arguments' do + inputs = described_class.new(spec, args) + + expect(inputs).to be_valid + expect(inputs.count).to eq 3 + expect(inputs.to_hash).to eq(args.merge(env: 'development')) + end + end + + context 'when inputs and args are empty' do + it 'is a valid use-case' do + inputs = described_class.new({}, {}) + + expect(inputs).to be_valid + expect(inputs.to_hash).to be_empty + end + end + + context 'when there are arguments recoincilation errors present' do + context 'when required argument is missing' do + let(:spec) { { website: nil } } + + it 'returns an error' do + inputs = described_class.new(spec, {}) + + expect(inputs).not_to be_valid + expect(inputs.errors.first).to eq '`website` input: required value has not been provided' + end + end + + context 'when argument is not present but configured as allowlist' do + let(:spec) do + { run: { options: %w[opt1 opt2] } } + end + + it 'returns an error' do + inputs = described_class.new(spec, {}) + + expect(inputs).not_to be_valid + expect(inputs.errors.first).to eq '`run` input: argument not provided' + end + end + end + + context 'when unknown specification argument has been used' do + let(:spec) do + { + website: nil, + env: { default: 'development' }, + run: { options: %w[tests spec e2e] }, + test: { unknown: 'something' } + } + end + + let(:args) { { website: 'https://gitlab.com', run: 'tests' } } + + it 'fabricates an unknown argument entry and returns an error' do + inputs = described_class.new(spec, args) + + expect(inputs).not_to be_valid + expect(inputs.count).to eq 4 + expect(inputs.errors.first).to eq '`test` input: unrecognized input argument specification: `unknown`' + end + end + + context 'when unknown arguments are being passed by a user' do + let(:spec) do + { env: { default: 'development' } } + end + + let(:args) { { website: 'https://gitlab.com', run: 'tests' } } + + it 'returns an error with a list of unknown arguments' do + inputs = described_class.new(spec, args) + + expect(inputs).not_to be_valid + expect(inputs.errors.first).to eq 'unknown input arguments: [:website, :run]' + end + end + + context 'when composite specification is being used' do + let(:spec) do + { + env: { + default: 'dev', + options: %w[test dev prod] + } + } + end + + let(:args) { { env: 'dev' } } + + it 'returns an error describing an unknown specification' do + inputs = described_class.new(spec, args) + + expect(inputs).not_to be_valid + expect(inputs.errors.first).to eq '`env` input: unrecognized input argument definition' + end + end +end diff --git a/spec/lib/gitlab/ci/interpolation/access_spec.rb b/spec/lib/gitlab/ci/interpolation/access_spec.rb index 9f6108a328d..f327377b7e3 100644 --- a/spec/lib/gitlab/ci/interpolation/access_spec.rb +++ b/spec/lib/gitlab/ci/interpolation/access_spec.rb @@ -2,7 +2,7 @@ require 'fast_spec_helper' -RSpec.describe Gitlab::Ci::Interpolation::Access, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Interpolation::Access, feature_category: :pipeline_composition do subject { described_class.new(access, ctx) } let(:access) do diff --git a/spec/lib/gitlab/ci/interpolation/block_spec.rb b/spec/lib/gitlab/ci/interpolation/block_spec.rb index 7f2be505d17..4a8709df3dc 100644 --- a/spec/lib/gitlab/ci/interpolation/block_spec.rb +++ b/spec/lib/gitlab/ci/interpolation/block_spec.rb @@ -2,7 +2,7 @@ require 'fast_spec_helper' -RSpec.describe Gitlab::Ci::Interpolation::Block, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Interpolation::Block, feature_category: :pipeline_composition do subject { described_class.new(block, data, ctx) } let(:data) do diff --git a/spec/lib/gitlab/ci/interpolation/config_spec.rb b/spec/lib/gitlab/ci/interpolation/config_spec.rb index e5987776e00..e745269d8c0 100644 --- a/spec/lib/gitlab/ci/interpolation/config_spec.rb +++ b/spec/lib/gitlab/ci/interpolation/config_spec.rb @@ -2,7 +2,7 @@ require 'fast_spec_helper' -RSpec.describe Gitlab::Ci::Interpolation::Config, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Interpolation::Config, feature_category: :pipeline_composition do subject { described_class.new(YAML.safe_load(config)) } let(:config) do diff --git a/spec/lib/gitlab/ci/interpolation/context_spec.rb b/spec/lib/gitlab/ci/interpolation/context_spec.rb index ada896f4980..2b126f4a8b3 100644 --- a/spec/lib/gitlab/ci/interpolation/context_spec.rb +++ b/spec/lib/gitlab/ci/interpolation/context_spec.rb @@ -2,7 +2,7 @@ require 'fast_spec_helper' -RSpec.describe Gitlab::Ci::Interpolation::Context, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Interpolation::Context, feature_category: :pipeline_composition do subject { described_class.new(ctx) } let(:ctx) do diff --git a/spec/lib/gitlab/ci/interpolation/template_spec.rb b/spec/lib/gitlab/ci/interpolation/template_spec.rb index 8a243b4db05..a3ef1bb4445 100644 --- a/spec/lib/gitlab/ci/interpolation/template_spec.rb +++ b/spec/lib/gitlab/ci/interpolation/template_spec.rb @@ -2,7 +2,7 @@ require 'fast_spec_helper' -RSpec.describe Gitlab::Ci::Interpolation::Template, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Interpolation::Template, feature_category: :pipeline_composition do subject { described_class.new(YAML.safe_load(config), ctx) } let(:config) do diff --git a/spec/lib/gitlab/ci/jwt_spec.rb b/spec/lib/gitlab/ci/jwt_spec.rb index 147801b6217..a6de5b9879c 100644 --- a/spec/lib/gitlab/ci/jwt_spec.rb +++ b/spec/lib/gitlab/ci/jwt_spec.rb @@ -58,26 +58,31 @@ RSpec.describe Gitlab::Ci::Jwt do expect { payload }.not_to raise_error end - describe 'ref type' do - context 'branches' do + describe 'references' do + context 'with a branch pipepline' do it 'is "branch"' do expect(payload[:ref_type]).to eq('branch') + expect(payload[:ref_path]).to eq('refs/heads/auto-deploy-2020-03-19') end end - context 'tags' do - let(:build) { build_stubbed(:ci_build, :on_tag, project: project) } + context 'with a tag pipeline' do + let(:pipeline) { build_stubbed(:ci_pipeline, ref: 'auto-deploy-2020-03-19', tag: true) } + let(:build) { build_stubbed(:ci_build, :on_tag, project: project, pipeline: pipeline) } it 'is "tag"' do expect(payload[:ref_type]).to eq('tag') + expect(payload[:ref_path]).to eq('refs/tags/auto-deploy-2020-03-19') end end - context 'merge requests' do - let(:pipeline) { build_stubbed(:ci_pipeline, :detached_merge_request_pipeline) } + context 'with a merge request pipeline' do + let(:merge_request) { build_stubbed(:merge_request, source_branch: 'feature-branch') } + let(:pipeline) { build_stubbed(:ci_pipeline, :detached_merge_request_pipeline, merge_request: merge_request) } it 'is "branch"' do expect(payload[:ref_type]).to eq('branch') + expect(payload[:ref_path]).to eq('refs/heads/feature-branch') end end end diff --git a/spec/lib/gitlab/ci/jwt_v2_spec.rb b/spec/lib/gitlab/ci/jwt_v2_spec.rb index 5eeab658a8e..528be4b5da7 100644 --- a/spec/lib/gitlab/ci/jwt_v2_spec.rb +++ b/spec/lib/gitlab/ci/jwt_v2_spec.rb @@ -2,11 +2,18 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::JwtV2 do +RSpec.describe Gitlab::Ci::JwtV2, feature_category: :continuous_integration do let(:namespace) { build_stubbed(:namespace) } let(:project) { build_stubbed(:project, namespace: namespace) } - let(:user) { build_stubbed(:user) } + let(:user) do + build_stubbed( + :user, + identities: [build_stubbed(:identity, extern_uid: '1', provider: 'github')] + ) + end + let(:pipeline) { build_stubbed(:ci_pipeline, ref: 'auto-deploy-2020-03-19') } + let(:runner) { build_stubbed(:ci_runner) } let(:aud) { described_class::DEFAULT_AUD } let(:build) do @@ -14,7 +21,8 @@ RSpec.describe Gitlab::Ci::JwtV2 do :ci_build, project: project, user: user, - pipeline: pipeline + pipeline: pipeline, + runner: runner ) end @@ -33,6 +41,18 @@ RSpec.describe Gitlab::Ci::JwtV2 do end end + it 'includes user identities when enabled' do + expect(user).to receive(:pass_user_identities_to_ci_jwt).and_return(true) + identities = payload[:user_identities].map { |identity| identity.slice(:extern_uid, :provider) } + expect(identities).to eq([{ extern_uid: '1', provider: 'github' }]) + end + + it 'does not include user identities when disabled' do + expect(user).to receive(:pass_user_identities_to_ci_jwt).and_return(false) + + expect(payload).not_to include(:user_identities) + end + context 'when given an aud' do let(:aud) { 'AWS' } @@ -40,5 +60,57 @@ RSpec.describe Gitlab::Ci::JwtV2 do expect(payload[:aud]).to eq('AWS') end end + + describe 'custom claims' do + describe 'runner_id' do + it 'is the ID of the runner executing the job' do + expect(payload[:runner_id]).to eq(runner.id) + end + + context 'when build is not associated with a runner' do + let(:runner) { nil } + + it 'is nil' do + expect(payload[:runner_id]).to be_nil + end + end + end + + describe 'runner_environment' do + context 'when runner is gitlab-hosted' do + before do + allow(runner).to receive(:gitlab_hosted?).and_return(true) + end + + it "is #{described_class::GITLAB_HOSTED_RUNNER}" do + expect(payload[:runner_environment]).to eq(described_class::GITLAB_HOSTED_RUNNER) + end + end + + context 'when runner is self-hosted' do + before do + allow(runner).to receive(:gitlab_hosted?).and_return(false) + end + + it "is #{described_class::SELF_HOSTED_RUNNER}" do + expect(payload[:runner_environment]).to eq(described_class::SELF_HOSTED_RUNNER) + end + end + + context 'when build is not associated with a runner' do + let(:runner) { nil } + + it 'is nil' do + expect(payload[:runner_environment]).to be_nil + end + end + end + + describe 'sha' do + it 'is the commit revision the project is built for' do + expect(payload[:sha]).to eq(pipeline.sha) + end + end + end end end diff --git a/spec/lib/gitlab/ci/lint_spec.rb b/spec/lib/gitlab/ci/lint_spec.rb index b836ca395fa..b238e9161eb 100644 --- a/spec/lib/gitlab/ci/lint_spec.rb +++ b/spec/lib/gitlab/ci/lint_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Lint, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Lint, feature_category: :pipeline_composition do let_it_be(:project) { create(:project, :repository) } let_it_be(:user) { create(:user) } @@ -100,8 +100,8 @@ RSpec.describe Gitlab::Ci::Lint, feature_category: :pipeline_authoring do end it 'sets merged_config' do - root_config = YAML.safe_load(content, [Symbol]) - included_config = YAML.safe_load(included_content, [Symbol]) + root_config = YAML.safe_load(content, permitted_classes: [Symbol]) + included_config = YAML.safe_load(included_content, permitted_classes: [Symbol]) expected_config = included_config.merge(root_config).except(:include).deep_stringify_keys expect(subject.merged_yaml).to eq(expected_config.to_yaml) diff --git a/spec/lib/gitlab/ci/parsers/security/common_spec.rb b/spec/lib/gitlab/ci/parsers/security/common_spec.rb index 5d2d22c04fc..421aa29f860 100644 --- a/spec/lib/gitlab/ci/parsers/security/common_spec.rb +++ b/spec/lib/gitlab/ci/parsers/security/common_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Parsers::Security::Common do +RSpec.describe Gitlab::Ci::Parsers::Security::Common, feature_category: :vulnerability_management do describe '#parse!' do let_it_be(:scanner_data) do { @@ -410,6 +410,12 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Common do end end + describe 'setting the `found_by_pipeline` attribute' do + subject { report.findings.map(&:found_by_pipeline).uniq } + + it { is_expected.to eq([pipeline]) } + end + describe 'parsing tracking' do let(:finding) { report.findings.first } diff --git a/spec/lib/gitlab/ci/parsers/security/sast_spec.rb b/spec/lib/gitlab/ci/parsers/security/sast_spec.rb index f6113308201..d1ce6808d23 100644 --- a/spec/lib/gitlab/ci/parsers/security/sast_spec.rb +++ b/spec/lib/gitlab/ci/parsers/security/sast_spec.rb @@ -13,8 +13,8 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Sast do context "when passing valid report" do # rubocop: disable Layout/LineLength where(:report_format, :report_version, :scanner_length, :finding_length, :identifier_length, :file_path, :start_line, :end_line, :primary_identifiers_length) do - :sast | '14.0.0' | 1 | 5 | 6 | 'groovy/src/main/java/com/gitlab/security_products/tests/App.groovy' | 47 | 47 | nil - :sast_semgrep_for_multiple_findings | '14.0.4' | 1 | 2 | 6 | 'app/app.py' | 39 | nil | 2 + :sast | '15.0.0' | 1 | 5 | 6 | 'groovy/src/main/java/com/gitlab/security_products/tests/App.groovy' | 47 | 47 | nil + :sast_semgrep_for_multiple_findings | '15.0.4' | 1 | 2 | 6 | 'app/app.py' | 39 | nil | 2 end # rubocop: enable Layout/LineLength diff --git a/spec/lib/gitlab/ci/parsers/security/secret_detection_spec.rb b/spec/lib/gitlab/ci/parsers/security/secret_detection_spec.rb index e8f1d617cb7..13999b2a9e5 100644 --- a/spec/lib/gitlab/ci/parsers/security/secret_detection_spec.rb +++ b/spec/lib/gitlab/ci/parsers/security/secret_detection_spec.rb @@ -39,7 +39,7 @@ RSpec.describe Gitlab::Ci::Parsers::Security::SecretDetection do end it "generates expected metadata_version" do - expect(report.findings.first.metadata_version).to eq('14.1.2') + expect(report.findings.first.metadata_version).to eq('15.0.0') end end end diff --git a/spec/lib/gitlab/ci/parsers/security/validators/schema_validator_spec.rb b/spec/lib/gitlab/ci/parsers/security/validators/schema_validator_spec.rb index 5fbaae58a73..2064a592246 100644 --- a/spec/lib/gitlab/ci/parsers/security/validators/schema_validator_spec.rb +++ b/spec/lib/gitlab/ci/parsers/security/validators/schema_validator_spec.rb @@ -5,55 +5,42 @@ require 'spec_helper' RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator, feature_category: :vulnerability_management do let_it_be(:project) { create(:project) } - let(:current_dast_versions) { described_class::CURRENT_VERSIONS[:dast].join(', ') } let(:supported_dast_versions) { described_class::SUPPORTED_VERSIONS[:dast].join(', ') } - let(:deprecated_schema_version_message) {} - let(:missing_schema_version_message) do - "Report version not provided, dast report type supports versions: #{supported_dast_versions}" - end let(:scanner) do { - 'id' => 'gemnasium', - 'name' => 'Gemnasium', - 'version' => '2.1.0' + 'id' => 'my-dast-scanner', + 'name' => 'My DAST scanner', + 'version' => '0.2.0', + 'vendor' => { 'name' => 'A DAST scanner' } } end - let(:analyzer_vendor) do - { 'name' => 'A DAST analyzer' } - end - - let(:scanner_vendor) do - { 'name' => 'A DAST scanner' } - end + let(:report_type) { :dast } - let(:report_data) do + let(:valid_data) do { 'scan' => { 'analyzer' => { 'id' => 'my-dast-analyzer', 'name' => 'My DAST analyzer', 'version' => '0.1.0', - 'vendor' => analyzer_vendor + 'vendor' => { 'name' => 'A DAST analyzer' } }, 'end_time' => '2020-01-28T03:26:02', 'scanned_resources' => [], - 'scanner' => { - 'id' => 'my-dast-scanner', - 'name' => 'My DAST scanner', - 'version' => '0.2.0', - 'vendor' => scanner_vendor - }, + 'scanner' => scanner, 'start_time' => '2020-01-28T03:26:01', 'status' => 'success', - 'type' => 'dast' + 'type' => report_type.to_s }, 'version' => report_version, 'vulnerabilities' => [] } end + let(:report_data) { valid_data } + let(:validator) { described_class.new(report_type, report_data, report_version, project: project, scanner: scanner) } shared_examples 'report is valid' do @@ -70,8 +57,8 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator, featu security_report_version: report_version, project_id: project.id, security_report_failure: security_report_failure, - security_report_scanner_id: 'gemnasium', - security_report_scanner_version: '2.1.0' + security_report_scanner_id: scanner['id'], + security_report_scanner_version: scanner['version'] ) subject @@ -142,7 +129,6 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator, featu subject { validator.valid? } context 'when given a supported MAJOR.MINOR schema version' do - let(:report_type) { :dast } let(:report_version) do latest_vendored_version = described_class::SUPPORTED_VERSIONS[report_type].last.split(".") (latest_vendored_version[0...2] << "34").join(".") @@ -153,7 +139,6 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator, featu end context 'when given a supported schema version' do - let(:report_type) { :dast } let(:report_version) { described_class::SUPPORTED_VERSIONS[report_type].last } it_behaves_like 'report is valid' @@ -161,7 +146,6 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator, featu end context 'when given a deprecated schema version' do - let(:report_type) { :dast } let(:deprecations_hash) do { dast: %w[10.0.0] @@ -175,13 +159,6 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator, featu end context 'and the report passes schema validation' do - let(:report_data) do - { - 'version' => '10.0.0', - 'vulnerabilities' => [] - } - end - let(:security_report_failure) { 'using_deprecated_schema_version' } it { is_expected.to be_truthy } @@ -191,9 +168,8 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator, featu context 'and the report does not pass schema validation' do let(:report_data) do - { - 'version' => 'V2.7.0' - } + valid_data.delete('vulnerabilities') + valid_data end it { is_expected.to be_falsey } @@ -201,17 +177,9 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator, featu end context 'when given an unsupported schema version' do - let(:report_type) { :dast } let(:report_version) { "12.37.0" } context 'and the report is valid' do - let(:report_data) do - { - 'version' => report_version, - 'vulnerabilities' => [] - } - end - let(:security_report_failure) { 'using_unsupported_schema_version' } it { is_expected.to be_falsey } @@ -259,8 +227,8 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator, featu end context 'when not given a schema version' do - let(:report_type) { :dast } let(:report_version) { nil } + let(:report_data) do { 'vulnerabilities' => [] @@ -285,21 +253,19 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator, featu subject { validator.errors } context 'when given a supported schema version' do - let(:report_type) { :dast } let(:report_version) { described_class::SUPPORTED_VERSIONS[report_type].last } it_behaves_like 'report is valid with no error' context 'and the report is invalid' do let(:report_data) do - { - 'version' => report_version - } + valid_data.delete('vulnerabilities') + valid_data end let(:expected_errors) do [ - 'root is missing required keys: scan, vulnerabilities' + 'root is missing required keys: vulnerabilities' ] end @@ -308,7 +274,6 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator, featu end context 'when given a deprecated schema version' do - let(:report_type) { :dast } let(:deprecations_hash) do { dast: %w[10.0.0] @@ -325,9 +290,9 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator, featu context 'and the report does not pass schema validation' do let(:report_data) do - { - 'version' => 'V2.7.0' - } + valid_data['version'] = "V2.7.0" + valid_data.delete('vulnerabilities') + valid_data end let(:expected_errors) do @@ -342,7 +307,6 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator, featu end context 'when given an unsupported schema version' do - let(:report_type) { :dast } let(:report_version) { "12.37.0" } let(:expected_unsupported_message) do "Version #{report_version} for report type #{report_type} is unsupported, supported versions for this report type are: "\ @@ -351,13 +315,6 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator, featu end context 'and the report is valid' do - let(:report_data) do - { - 'version' => report_version, - 'vulnerabilities' => [] - } - end - let(:expected_errors) do [ expected_unsupported_message @@ -369,9 +326,8 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator, featu context 'and the report is invalid' do let(:report_data) do - { - 'version' => report_version - } + valid_data.delete('vulnerabilities') + valid_data end let(:expected_errors) do @@ -386,7 +342,6 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator, featu end context 'when not given a schema version' do - let(:report_type) { :dast } let(:report_version) { nil } let(:expected_missing_version_message) do "Report version not provided, #{report_type} report type supports versions: #{supported_dast_versions}. GitLab "\ @@ -395,9 +350,8 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator, featu end let(:report_data) do - { - 'vulnerabilities' => [] - } + valid_data.delete('version') + valid_data end let(:expected_errors) do @@ -413,13 +367,6 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator, featu shared_examples 'report is valid with no warning' do context 'and the report is valid' do - let(:report_data) do - { - 'version' => report_version, - 'vulnerabilities' => [] - } - end - it { is_expected.to be_empty } end end @@ -432,25 +379,16 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator, featu subject { validator.deprecation_warnings } context 'when given a supported schema version' do - let(:report_type) { :dast } let(:report_version) { described_class::SUPPORTED_VERSIONS[report_type].last } context 'and the report is valid' do - let(:report_data) do - { - 'version' => report_version, - 'vulnerabilities' => [] - } - end - it { is_expected.to be_empty } end context 'and the report is invalid' do let(:report_data) do - { - 'version' => report_version - } + valid_data.delete('vulnerabilities') + valid_data end it { is_expected.to be_empty } @@ -458,7 +396,6 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator, featu end context 'when given a deprecated schema version' do - let(:report_type) { :dast } let(:deprecations_hash) do { dast: %w[V2.7.0] @@ -466,6 +403,7 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator, featu end let(:report_version) { described_class::DEPRECATED_VERSIONS[report_type].last } + let(:current_dast_versions) { described_class::CURRENT_VERSIONS[:dast].join(', ') } let(:expected_deprecation_message) do "version #{report_version} for report type #{report_type} is deprecated. "\ "However, GitLab will still attempt to parse and ingest this report. "\ @@ -483,53 +421,23 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator, featu end context 'and the report passes schema validation' do - let(:report_data) do - { - 'version' => report_version, - 'vulnerabilities' => [] - } - end - it_behaves_like 'report with expected warnings' end context 'and the report does not pass schema validation' do let(:report_data) do - { - 'version' => 'V2.7.0' - } + valid_data['version'] = "V2.7.0" + valid_data.delete('vulnerabilities') + valid_data end it_behaves_like 'report with expected warnings' end - - context 'and the report passes schema validation as a GitLab-vendored analyzer' do - let(:analyzer_vendor) do - { 'name' => 'GitLab' } - end - - it { is_expected.to be_empty } - end - - context 'and the report passes schema validation as a GitLab-vendored scanner' do - let(:scanner_vendor) do - { 'name' => 'GitLab' } - end - - it { is_expected.to be_empty } - end end context 'when given an unsupported schema version' do - let(:report_type) { :dast } let(:report_version) { "21.37.0" } let(:expected_deprecation_warnings) { [] } - let(:report_data) do - { - 'version' => report_version, - 'vulnerabilities' => [] - } - end it_behaves_like 'report with expected warnings' end @@ -539,7 +447,6 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator, featu subject { validator.warnings } context 'when given a supported MAJOR.MINOR schema version' do - let(:report_type) { :dast } let(:report_version) do latest_vendored_version = described_class::SUPPORTED_VERSIONS[report_type].last.split(".") (latest_vendored_version[0...2] << "34").join(".") @@ -559,13 +466,6 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator, featu end context 'and the report is valid' do - let(:report_data) do - { - 'version' => report_version, - 'vulnerabilities' => [] - } - end - it { is_expected.to match_array([message]) } context 'without license', unless: Gitlab.ee? do @@ -607,7 +507,6 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator, featu end context 'when given a supported schema version' do - let(:report_type) { :dast } let(:report_version) { described_class::SUPPORTED_VERSIONS[report_type].last } it_behaves_like 'report is valid with no warning' @@ -624,34 +523,26 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator, featu end context 'when given a deprecated schema version' do - let(:report_type) { :dast } + let(:deprecated_version) { '14.1.3' } + let(:report_version) { deprecated_version } let(:deprecations_hash) do { - dast: %w[V2.7.0] + dast: %w[deprecated_version] } end - let(:report_version) { described_class::DEPRECATED_VERSIONS[report_type].last } - before do stub_const("#{described_class}::DEPRECATED_VERSIONS", deprecations_hash) end context 'and the report passes schema validation' do - let(:report_data) do - { - 'vulnerabilities' => [] - } - end - it { is_expected.to be_empty } end context 'and the report does not pass schema validation' do let(:report_data) do - { - 'version' => 'V2.7.0' - } + valid_data.delete('vulnerabilities') + valid_data end it { is_expected.to be_empty } @@ -659,7 +550,6 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator, featu end context 'when given an unsupported schema version' do - let(:report_type) { :dast } let(:report_version) { "12.37.0" } it_behaves_like 'report is valid with no warning' @@ -676,13 +566,7 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator, featu end context 'when not given a schema version' do - let(:report_type) { :dast } let(:report_version) { nil } - let(:report_data) do - { - 'vulnerabilities' => [] - } - end it { is_expected.to be_empty } end diff --git a/spec/lib/gitlab/ci/pipeline/chain/config/content_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/config/content_spec.rb index e0d656f456e..a9a52972294 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/config/content_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/config/content_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Pipeline::Chain::Config::Content do +RSpec.describe Gitlab::Ci::Pipeline::Chain::Config::Content, feature_category: :continuous_integration do let(:project) { create(:project, ci_config_path: ci_config_path) } let(:pipeline) { build(:ci_pipeline, project: project) } let(:content) { nil } @@ -26,6 +26,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Config::Content do expect(pipeline.config_source).to eq 'bridge_source' expect(command.config_content).to eq 'the-yaml' + expect(command.pipeline_config.internal_include_prepended?).to eq(false) end end @@ -52,6 +53,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Config::Content do expect(pipeline.config_source).to eq 'repository_source' expect(pipeline.pipeline_config.content).to eq(config_content_result) expect(command.config_content).to eq(config_content_result) + expect(command.pipeline_config.internal_include_prepended?).to eq(true) end end @@ -71,6 +73,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Config::Content do expect(pipeline.config_source).to eq 'remote_source' expect(pipeline.pipeline_config.content).to eq(config_content_result) expect(command.config_content).to eq(config_content_result) + expect(command.pipeline_config.internal_include_prepended?).to eq(true) end end @@ -91,6 +94,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Config::Content do expect(pipeline.config_source).to eq 'external_project_source' expect(pipeline.pipeline_config.content).to eq(config_content_result) expect(command.config_content).to eq(config_content_result) + expect(command.pipeline_config.internal_include_prepended?).to eq(true) end context 'when path specifies a refname' do @@ -111,6 +115,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Config::Content do expect(pipeline.config_source).to eq 'external_project_source' expect(pipeline.pipeline_config.content).to eq(config_content_result) expect(command.config_content).to eq(config_content_result) + expect(command.pipeline_config.internal_include_prepended?).to eq(true) end end end @@ -138,6 +143,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Config::Content do expect(pipeline.config_source).to eq 'repository_source' expect(pipeline.pipeline_config.content).to eq(config_content_result) expect(command.config_content).to eq(config_content_result) + expect(command.pipeline_config.internal_include_prepended?).to eq(true) end end @@ -161,6 +167,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Config::Content do expect(pipeline.config_source).to eq 'auto_devops_source' expect(pipeline.pipeline_config.content).to eq(config_content_result) expect(command.config_content).to eq(config_content_result) + expect(command.pipeline_config.internal_include_prepended?).to eq(true) end end @@ -181,6 +188,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Config::Content do expect(pipeline.config_source).to eq 'parameter_source' expect(pipeline.pipeline_config.content).to eq(content) expect(command.config_content).to eq(content) + expect(command.pipeline_config.internal_include_prepended?).to eq(false) end end @@ -197,6 +205,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Config::Content do expect(pipeline.config_source).to eq('unknown_source') expect(pipeline.pipeline_config).to be_nil expect(command.config_content).to be_nil + expect(command.pipeline_config).to be_nil expect(pipeline.errors.full_messages).to include('Missing CI config file') end end diff --git a/spec/lib/gitlab/ci/pipeline/duration_spec.rb b/spec/lib/gitlab/ci/pipeline/duration_spec.rb index 36714413da6..89c0ce46237 100644 --- a/spec/lib/gitlab/ci/pipeline/duration_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/duration_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Pipeline::Duration do +RSpec.describe Gitlab::Ci::Pipeline::Duration, feature_category: :continuous_integration do describe '.from_periods' do let(:calculated_duration) { calculate(data) } @@ -113,16 +113,17 @@ RSpec.describe Gitlab::Ci::Pipeline::Duration do described_class::Period.new(first, last) end - described_class.from_periods(periods.sort_by(&:first)) + described_class.send(:from_periods, periods.sort_by(&:first)) end end describe '.from_pipeline' do + let_it_be_with_reload(:pipeline) { create(:ci_pipeline) } + let_it_be(:start_time) { Time.current.change(usec: 0) } let_it_be(:current) { start_time + 1000 } - let_it_be(:pipeline) { create(:ci_pipeline) } - let_it_be(:success_build) { create_build(:success, started_at: start_time, finished_at: start_time + 60) } - let_it_be(:failed_build) { create_build(:failed, started_at: start_time + 60, finished_at: start_time + 120) } + let_it_be(:success_build) { create_build(:success, started_at: start_time, finished_at: start_time + 50) } + let_it_be(:failed_build) { create_build(:failed, started_at: start_time + 60, finished_at: start_time + 110) } let_it_be(:canceled_build) { create_build(:canceled, started_at: start_time + 120, finished_at: start_time + 180) } let_it_be(:skipped_build) { create_build(:skipped, started_at: start_time) } let_it_be(:pending_build) { create_build(:pending) } @@ -141,21 +142,55 @@ RSpec.describe Gitlab::Ci::Pipeline::Duration do end context 'when there is no running build' do - let(:running_build) { nil } + let!(:running_build) { nil } it 'returns the duration for all the builds' do travel_to(current) do - expect(described_class.from_pipeline(pipeline)).to eq 180.seconds + # 160 = success (50) + failed (50) + canceled (60) + expect(described_class.from_pipeline(pipeline)).to eq 160.seconds end end end - context 'when there are bridge jobs' do - let!(:success_bridge) { create_bridge(:success, started_at: start_time + 220, finished_at: start_time + 280) } - let!(:failed_bridge) { create_bridge(:failed, started_at: start_time + 180, finished_at: start_time + 240) } - let!(:skipped_bridge) { create_bridge(:skipped, started_at: start_time) } - let!(:created_bridge) { create_bridge(:created) } - let!(:manual_bridge) { create_bridge(:manual) } + context 'when there are direct bridge jobs' do + let_it_be(:success_bridge) do + create_bridge(:success, started_at: start_time + 220, finished_at: start_time + 280) + end + + let_it_be(:failed_bridge) { create_bridge(:failed, started_at: start_time + 180, finished_at: start_time + 240) } + # NOTE: bridge won't be `canceled` as it will be marked as failed when downstream pipeline is canceled + # @see Ci::Bridge#inherit_status_from_downstream + let_it_be(:canceled_bridge) do + create_bridge(:failed, started_at: start_time + 180, finished_at: start_time + 210) + end + + let_it_be(:skipped_bridge) { create_bridge(:skipped, started_at: start_time) } + let_it_be(:created_bridge) { create_bridge(:created) } + let_it_be(:manual_bridge) { create_bridge(:manual) } + + let_it_be(:success_bridge_pipeline) do + create(:ci_pipeline, :success, started_at: start_time + 230, finished_at: start_time + 280).tap do |p| + create(:ci_sources_pipeline, source_job: success_bridge, pipeline: p) + create_build(:success, pipeline: p, started_at: start_time + 235, finished_at: start_time + 280) + create_bridge(:success, pipeline: p, started_at: start_time + 240, finished_at: start_time + 280) + end + end + + let_it_be(:failed_bridge_pipeline) do + create(:ci_pipeline, :failed, started_at: start_time + 225, finished_at: start_time + 240).tap do |p| + create(:ci_sources_pipeline, source_job: failed_bridge, pipeline: p) + create_build(:failed, pipeline: p, started_at: start_time + 230, finished_at: start_time + 240) + create_bridge(:success, pipeline: p, started_at: start_time + 235, finished_at: start_time + 240) + end + end + + let_it_be(:canceled_bridge_pipeline) do + create(:ci_pipeline, :canceled, started_at: start_time + 190, finished_at: start_time + 210).tap do |p| + create(:ci_sources_pipeline, source_job: canceled_bridge, pipeline: p) + create_build(:canceled, pipeline: p, started_at: start_time + 200, finished_at: start_time + 210) + create_bridge(:success, pipeline: p, started_at: start_time + 205, finished_at: start_time + 210) + end + end it 'returns the duration of the running build' do travel_to(current) do @@ -166,12 +201,99 @@ RSpec.describe Gitlab::Ci::Pipeline::Duration do context 'when there is no running build' do let!(:running_build) { nil } - it 'returns the duration for all the builds and bridge jobs' do + it 'returns the duration for all the builds (including self and downstreams)' do travel_to(current) do - expect(described_class.from_pipeline(pipeline)).to eq 280.seconds + # 220 = 160 (see above) + # + success build (45) + failed (10) + canceled (10) - overlapping (success & failed) (5) + expect(described_class.from_pipeline(pipeline)).to eq 220.seconds end end end + + # rubocop:disable RSpec/MultipleMemoizedHelpers + context 'when there are downstream bridge jobs' do + let_it_be(:success_direct_bridge) do + create_bridge(:success, started_at: start_time + 280, finished_at: start_time + 400) + end + + let_it_be(:success_downstream_pipeline) do + create(:ci_pipeline, :success, started_at: start_time + 285, finished_at: start_time + 300).tap do |p| + create(:ci_sources_pipeline, source_job: success_direct_bridge, pipeline: p) + create_build(:success, pipeline: p, started_at: start_time + 290, finished_at: start_time + 296) + create_bridge(:success, pipeline: p, started_at: start_time + 285, finished_at: start_time + 288) + end + end + + let_it_be(:failed_downstream_pipeline) do + create(:ci_pipeline, :failed, started_at: start_time + 305, finished_at: start_time + 350).tap do |p| + create(:ci_sources_pipeline, source_job: success_direct_bridge, pipeline: p) + create_build(:failed, pipeline: p, started_at: start_time + 320, finished_at: start_time + 327) + create_bridge(:success, pipeline: p, started_at: start_time + 305, finished_at: start_time + 350) + end + end + + let_it_be(:canceled_downstream_pipeline) do + create(:ci_pipeline, :canceled, started_at: start_time + 360, finished_at: start_time + 400).tap do |p| + create(:ci_sources_pipeline, source_job: success_direct_bridge, pipeline: p) + create_build(:canceled, pipeline: p, started_at: start_time + 390, finished_at: start_time + 398) + create_bridge(:success, pipeline: p, started_at: start_time + 360, finished_at: start_time + 378) + end + end + + it 'returns the duration of the running build' do + travel_to(current) do + expect(described_class.from_pipeline(pipeline)).to eq 1000.seconds + end + end + + context 'when there is no running build' do + let!(:running_build) { nil } + + it 'returns the duration for all the builds (including self and downstreams)' do + travel_to(current) do + # 241 = 220 (see above) + # + success downstream build (6) + failed (7) + canceled (8) + expect(described_class.from_pipeline(pipeline)).to eq 241.seconds + end + end + end + end + # rubocop:enable RSpec/MultipleMemoizedHelpers + end + + it 'does not generate N+1 queries if more builds are added' do + travel_to(current) do + expect do + described_class.from_pipeline(pipeline) + end.not_to exceed_query_limit(1) + + create_list(:ci_build, 2, :success, pipeline: pipeline, started_at: start_time, finished_at: start_time + 50) + + expect do + described_class.from_pipeline(pipeline) + end.not_to exceed_query_limit(1) + end + end + + it 'does not generate N+1 queries if more bridges and their pipeline builds are added' do + travel_to(current) do + expect do + described_class.from_pipeline(pipeline) + end.not_to exceed_query_limit(1) + + create_list( + :ci_bridge, 2, :success, + pipeline: pipeline, started_at: start_time + 220, finished_at: start_time + 280).each do |bridge| + create(:ci_pipeline, :success, started_at: start_time + 235, finished_at: start_time + 280).tap do |p| + create(:ci_sources_pipeline, source_job: bridge, pipeline: p) + create_builds(3, :success) + end + end + + expect do + described_class.from_pipeline(pipeline) + end.not_to exceed_query_limit(1) + end end private @@ -180,6 +302,10 @@ RSpec.describe Gitlab::Ci::Pipeline::Duration do create(:ci_build, trait, pipeline: pipeline, **opts) end + def create_builds(counts, trait, **opts) + create_list(:ci_build, counts, trait, pipeline: pipeline, **opts) + end + def create_bridge(trait, **opts) create(:ci_bridge, trait, pipeline: pipeline, **opts) end diff --git a/spec/lib/gitlab/ci/pipeline/seed/build/cache_spec.rb b/spec/lib/gitlab/ci/pipeline/seed/build/cache_spec.rb index c264ea3bece..07e2d6960bf 100644 --- a/spec/lib/gitlab/ci/pipeline/seed/build/cache_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/seed/build/cache_spec.rb @@ -7,8 +7,9 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build::Cache do let_it_be(:head_sha) { project.repository.head_commit.id } let_it_be(:pipeline) { create(:ci_pipeline, project: project, sha: head_sha) } let(:index) { 1 } + let(:cache_prefix) { index } - let(:processor) { described_class.new(pipeline, config, index) } + let(:processor) { described_class.new(pipeline, config, cache_prefix) } describe '#attributes' do subject { processor.attributes } @@ -32,7 +33,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build::Cache do } end - it { is_expected.to include(config.merge(key: "a_key")) } + it { is_expected.to include(config.merge(key: 'a_key')) } end context 'with cache:key:files' do @@ -42,8 +43,8 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build::Cache do end context 'without a prefix' do - it 'uses default key with an index as a prefix' do - expected = { key: '1-default' } + it 'uses default key with an index and file names as a prefix' do + expected = { key: "#{cache_prefix}-default" } is_expected.to include(expected) end @@ -61,9 +62,9 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build::Cache do end context 'without a prefix' do - it 'builds a string key with an index as a prefix' do + it 'builds a string key with an index and file names as a prefix' do expected = { - key: '1-703ecc8fef1635427a1f86a8a1a308831c122392', + key: "#{cache_prefix}-703ecc8fef1635427a1f86a8a1a308831c122392", paths: ['vendor/ruby'] } @@ -74,30 +75,41 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build::Cache do context 'with existing files' do let(:files) { ['VERSION', 'Gemfile.zip'] } + let(:cache_prefix) { '1_VERSION_Gemfile' } it_behaves_like 'version and gemfile files' end context 'with files starting with ./' do let(:files) { ['Gemfile.zip', './VERSION'] } + let(:cache_prefix) { '1_Gemfile_' } it_behaves_like 'version and gemfile files' end + context 'with no files' do + let(:files) { [] } + + it_behaves_like 'default key' + end + context 'with files ending with /' do let(:files) { ['Gemfile.zip/'] } + let(:cache_prefix) { '1_Gemfile' } it_behaves_like 'default key' end context 'with new line in filenames' do - let(:files) { ["Gemfile.zip\nVERSION"] } + let(:files) { ['Gemfile.zip\nVERSION'] } + let(:cache_prefix) { '1_Gemfile' } it_behaves_like 'default key' end context 'with missing files' do let(:files) { ['project-gemfile.lock', ''] } + let(:cache_prefix) { '1_project-gemfile_' } it_behaves_like 'default key' end @@ -113,8 +125,8 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build::Cache do end context 'without a prefix' do - it 'builds a string key with an index as a prefix' do - expected = { key: '1-74bf43fb1090f161bdd4e265802775dbda2f03d1' } + it 'builds a string key with an index and file names as a prefix' do + expected = { key: "#{cache_prefix}-74bf43fb1090f161bdd4e265802775dbda2f03d1" } is_expected.to include(expected) end @@ -123,18 +135,21 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build::Cache do context 'with directory' do let(:files) { ['foo/bar'] } + let(:cache_prefix) { '1_foo/bar' } it_behaves_like 'foo/bar directory key' end context 'with directory ending in slash' do let(:files) { ['foo/bar/'] } + let(:cache_prefix) { '1_foo/bar/' } it_behaves_like 'foo/bar directory key' end context 'with directories ending in slash star' do let(:files) { ['foo/bar/*'] } + let(:cache_prefix) { '1_foo/bar/*' } it_behaves_like 'foo/bar directory key' end @@ -205,6 +220,18 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build::Cache do end end + context 'with cache:fallback_keys' do + let(:config) do + { + key: 'ruby-branch-key', + paths: ['vendor/ruby'], + fallback_keys: ['ruby-default'] + } + end + + it { is_expected.to include(config) } + end + context 'with all cache option keys' do let(:config) do { @@ -213,7 +240,8 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build::Cache do untracked: true, policy: 'push', unprotect: true, - when: 'on_success' + when: 'on_success', + fallback_keys: ['default-ruby'] } end diff --git a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb index 3043d7f5381..9d5a9bc8058 100644 --- a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Pipeline::Seed::Build, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Pipeline::Seed::Build, feature_category: :pipeline_composition do let_it_be_with_reload(:project) { create(:project, :repository) } let_it_be(:head_sha) { project.repository.head_commit.id } @@ -109,6 +109,104 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build, feature_category: :pipeline_au end end + context 'with job:rules:[needs:]' do + context 'with a single rule' do + let(:job_needs_attributes) { [{ name: 'rspec' }] } + + context 'when job has needs set' do + context 'when rule evaluates to true' do + let(:attributes) do + { name: 'rspec', + ref: 'master', + needs_attributes: job_needs_attributes, + rules: [{ if: '$VAR == null', needs: { job: [{ name: 'build-job' }] } }] } + end + + it 'overrides the job needs' do + expect(subject).to include(needs_attributes: [{ name: 'build-job' }]) + end + end + + context 'when rule evaluates to false' do + let(:attributes) do + { name: 'rspec', + ref: 'master', + needs_attributes: job_needs_attributes, + rules: [{ if: '$VAR == true', needs: { job: [{ name: 'build-job' }] } }] } + end + + it 'keeps the job needs' do + expect(subject).to include(needs_attributes: job_needs_attributes) + end + end + + context 'with subkeys: artifacts, optional' do + let(:attributes) do + { name: 'rspec', + ref: 'master', + rules: + [ + { if: '$VAR == null', + needs: { + job: [{ + name: 'build-job', + optional: false, + artifacts: true + }] + } } + ] } + end + + context 'when rule evaluates to true' do + it 'sets the job needs as well as the job subkeys' do + expect(subject[:needs_attributes]).to match_array([{ name: 'build-job', optional: false, artifacts: true }]) + end + + it 'sets the scheduling type to dag' do + expect(subject[:scheduling_type]).to eq(:dag) + end + end + end + end + + context 'with multiple rules' do + context 'when a rule evaluates to true' do + let(:attributes) do + { name: 'rspec', + ref: 'master', + needs_attributes: job_needs_attributes, + rules: [ + { if: '$VAR == true', needs: { job: [{ name: 'rspec-1' }] } }, + { if: '$VAR2 == true', needs: { job: [{ name: 'rspec-2' }] } }, + { if: '$VAR3 == null', needs: { job: [{ name: 'rspec' }, { name: 'lint' }] } } + ] } + end + + it 'overrides the job needs' do + expect(subject).to include(needs_attributes: [{ name: 'rspec' }, { name: 'lint' }]) + end + end + + context 'when all rules evaluates to false' do + let(:attributes) do + { name: 'rspec', + ref: 'master', + needs_attributes: job_needs_attributes, + rules: [ + { if: '$VAR == true', needs: { job: [{ name: 'rspec-1' }] } }, + { if: '$VAR2 == true', needs: { job: [{ name: 'rspec-2' }] } }, + { if: '$VAR3 == true', needs: { job: [{ name: 'rspec-3' }] } } + ] } + end + + it 'keeps the job needs' do + expect(subject).to include(needs_attributes: job_needs_attributes) + end + end + end + end + end + context 'with job:tags' do let(:attributes) do { @@ -152,7 +250,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build, feature_category: :pipeline_au it 'includes cache options' do cache_options = { options: { - cache: [a_hash_including(key: '0-f155568ad0933d8358f66b846133614f76dd0ca4')] + cache: [a_hash_including(key: '0_VERSION-f155568ad0933d8358f66b846133614f76dd0ca4')] } } @@ -798,7 +896,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build, feature_category: :pipeline_au [ [[{ if: '$CI_JOB_NAME == "rspec" && $VAR == null', when: 'on_failure' }]], [[{ if: '$VARIABLE != null', when: 'delayed', start_in: '1 day' }, { if: '$CI_JOB_NAME == "rspec"', when: 'on_failure' }]], - [[{ if: '$VARIABLE == "the wrong value"', when: 'delayed', start_in: '1 day' }, { if: '$CI_BUILD_NAME == "rspec"', when: 'on_failure' }]] + [[{ if: '$VARIABLE == "the wrong value"', when: 'delayed', start_in: '1 day' }, { if: '$CI_JOB_NAME == "rspec"', when: 'on_failure' }]] ] end @@ -811,6 +909,30 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build, feature_category: :pipeline_au end end + context 'when FF `ci_remove_legacy_predefined_variables` is disabled' do + before do + stub_feature_flags(ci_remove_legacy_predefined_variables: false) + end + + context 'with an explicit `when: on_failure`' do + where(:rule_set) do + [ + [[{ if: '$CI_JOB_NAME == "rspec" && $VAR == null', when: 'on_failure' }]], + [[{ if: '$VARIABLE != null', when: 'delayed', start_in: '1 day' }, { if: '$CI_JOB_NAME == "rspec"', when: 'on_failure' }]], + [[{ if: '$VARIABLE == "the wrong value"', when: 'delayed', start_in: '1 day' }, { if: '$CI_BUILD_NAME == "rspec"', when: 'on_failure' }]] + ] + end + + with_them do + it { is_expected.to be_included } + + it 'correctly populates when:' do + expect(seed_build.attributes).to include(when: 'on_failure') + end + end + end + end + context 'with an explicit `when: delayed`' do where(:rule_set) do [ diff --git a/spec/lib/gitlab/ci/pipeline/seed/stage_spec.rb b/spec/lib/gitlab/ci/pipeline/seed/stage_spec.rb index 288ac3f3854..ae40626510f 100644 --- a/spec/lib/gitlab/ci/pipeline/seed/stage_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/seed/stage_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Pipeline::Seed::Stage, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Pipeline::Seed::Stage, feature_category: :pipeline_composition do let(:project) { create(:project, :repository) } let(:pipeline) { create(:ci_empty_pipeline, project: project) } let(:previous_stages) { [] } diff --git a/spec/lib/gitlab/ci/project_config/repository_spec.rb b/spec/lib/gitlab/ci/project_config/repository_spec.rb index 2105b691d9e..e8a997a7e43 100644 --- a/spec/lib/gitlab/ci/project_config/repository_spec.rb +++ b/spec/lib/gitlab/ci/project_config/repository_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::ProjectConfig::Repository do +RSpec.describe Gitlab::Ci::ProjectConfig::Repository, feature_category: :continuous_integration do let(:project) { create(:project, :custom_repo, files: files) } let(:sha) { project.repository.head_commit.sha } let(:files) { { 'README.md' => 'hello' } } @@ -44,4 +44,10 @@ RSpec.describe Gitlab::Ci::ProjectConfig::Repository do it { is_expected.to eq(:repository_source) } end + + describe '#internal_include_prepended?' do + subject { config.internal_include_prepended? } + + it { is_expected.to eq(true) } + end end diff --git a/spec/lib/gitlab/ci/project_config/source_spec.rb b/spec/lib/gitlab/ci/project_config/source_spec.rb index dda5c7cdce8..eefabe1babb 100644 --- a/spec/lib/gitlab/ci/project_config/source_spec.rb +++ b/spec/lib/gitlab/ci/project_config/source_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::ProjectConfig::Source do +RSpec.describe Gitlab::Ci::ProjectConfig::Source, feature_category: :continuous_integration do let_it_be(:custom_config_class) { Class.new(described_class) } let_it_be(:project) { build_stubbed(:project) } let_it_be(:sha) { '123456' } @@ -20,4 +20,10 @@ RSpec.describe Gitlab::Ci::ProjectConfig::Source do it { expect { source }.to raise_error(NotImplementedError) } end + + describe '#internal_include_prepended?' do + subject(:internal_include_prepended) { custom_config.internal_include_prepended? } + + it { expect(internal_include_prepended).to eq(false) } + end end diff --git a/spec/lib/gitlab/ci/reports/codequality_mr_diff_spec.rb b/spec/lib/gitlab/ci/reports/codequality_mr_diff_spec.rb index 73b916da2e9..79fa1c3ec75 100644 --- a/spec/lib/gitlab/ci/reports/codequality_mr_diff_spec.rb +++ b/spec/lib/gitlab/ci/reports/codequality_mr_diff_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Reports::CodequalityMrDiff do +RSpec.describe Gitlab::Ci::Reports::CodequalityMrDiff, feature_category: :code_quality do let(:codequality_report) { Gitlab::Ci::Reports::CodequalityReports.new } let(:degradation_1) { build(:codequality_degradation_1) } let(:degradation_2) { build(:codequality_degradation_2) } diff --git a/spec/lib/gitlab/ci/reports/security/scanner_spec.rb b/spec/lib/gitlab/ci/reports/security/scanner_spec.rb index d7ac82e3b53..79c59fb0da8 100644 --- a/spec/lib/gitlab/ci/reports/security/scanner_spec.rb +++ b/spec/lib/gitlab/ci/reports/security/scanner_spec.rb @@ -131,7 +131,7 @@ RSpec.describe Gitlab::Ci::Reports::Security::Scanner do context 'when the `name` of the scanners are equal' do where(:scanner_1_attributes, :scanner_2_attributes, :expected_comparison_result) do - { external_id: 'gemnasium', name: 'foo', vendor: 'a' } | { external_id: 'gemnasium', name: 'foo', vendor: 'a' } | 0 # rubocop:disable Lint/BinaryOperatorWithIdenticalOperands + { external_id: 'gemnasium', name: 'foo', vendor: 'a' } | { external_id: 'gemnasium', name: 'foo', vendor: 'a' } | 0 { external_id: 'gemnasium', name: 'foo', vendor: 'a' } | { external_id: 'gemnasium', name: 'foo', vendor: 'b' } | -1 { external_id: 'gemnasium', name: 'foo', vendor: 'b' } | { external_id: 'gemnasium', name: 'foo', vendor: 'a' } | 1 end diff --git a/spec/lib/gitlab/ci/reports/security/vulnerability_reports_comparer_spec.rb b/spec/lib/gitlab/ci/reports/security/vulnerability_reports_comparer_spec.rb deleted file mode 100644 index 6f75e2c55e8..00000000000 --- a/spec/lib/gitlab/ci/reports/security/vulnerability_reports_comparer_spec.rb +++ /dev/null @@ -1,163 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Ci::Reports::Security::VulnerabilityReportsComparer do - let(:identifier) { build(:ci_reports_security_identifier) } - - let_it_be(:project) { create(:project, :repository) } - - let(:location_param) { build(:ci_reports_security_locations_sast, :dynamic) } - let(:vulnerability_params) { vuln_params(project.id, [identifier], confidence: :low, severity: :critical) } - let(:base_vulnerability) { build(:ci_reports_security_finding, location: location_param, **vulnerability_params) } - let(:base_report) { build(:ci_reports_security_aggregated_reports, findings: [base_vulnerability]) } - - let(:head_vulnerability) { build(:ci_reports_security_finding, location: location_param, uuid: base_vulnerability.uuid, **vulnerability_params) } - let(:head_report) { build(:ci_reports_security_aggregated_reports, findings: [head_vulnerability]) } - - shared_context 'comparing reports' do - let(:vul_params) { vuln_params(project.id, [identifier]) } - let(:base_vulnerability) { build(:ci_reports_security_finding, :dynamic, **vul_params) } - let(:head_vulnerability) { build(:ci_reports_security_finding, :dynamic, **vul_params) } - let(:head_vul_findings) { [head_vulnerability, vuln] } - end - - subject { described_class.new(project, base_report, head_report) } - - where(vulnerability_finding_signatures: [true, false]) - - with_them do - before do - stub_licensed_features(vulnerability_finding_signatures: vulnerability_finding_signatures) - end - - describe '#base_report_out_of_date' do - context 'no base report' do - let(:base_report) { build(:ci_reports_security_aggregated_reports, reports: [], findings: []) } - - it 'is not out of date' do - expect(subject.base_report_out_of_date).to be false - end - end - - context 'base report older than one week' do - let(:report) { build(:ci_reports_security_report, created_at: 1.week.ago - 60.seconds) } - let(:base_report) { build(:ci_reports_security_aggregated_reports, reports: [report]) } - - it 'is not out of date' do - expect(subject.base_report_out_of_date).to be true - end - end - - context 'base report less than one week old' do - let(:report) { build(:ci_reports_security_report, created_at: 1.week.ago + 60.seconds) } - let(:base_report) { build(:ci_reports_security_aggregated_reports, reports: [report]) } - - it 'is not out of date' do - expect(subject.base_report_out_of_date).to be false - end - end - end - - describe '#added' do - let(:new_location) { build(:ci_reports_security_locations_sast, :dynamic) } - let(:vul_params) { vuln_params(project.id, [identifier], confidence: :high) } - let(:vuln) { build(:ci_reports_security_finding, severity: Enums::Vulnerability.severity_levels[:critical], location: new_location, **vul_params) } - let(:low_vuln) { build(:ci_reports_security_finding, severity: Enums::Vulnerability.severity_levels[:low], location: new_location, **vul_params) } - - context 'with new vulnerability' do - let(:head_report) { build(:ci_reports_security_aggregated_reports, findings: [head_vulnerability, vuln]) } - - it 'points to source tree' do - expect(subject.added).to eq([vuln]) - end - end - - context 'when comparing reports with different fingerprints' do - include_context 'comparing reports' - - let(:head_report) { build(:ci_reports_security_aggregated_reports, findings: head_vul_findings) } - - it 'does not find any overlap' do - expect(subject.added).to eq(head_vul_findings) - end - end - - context 'order' do - let(:head_report) { build(:ci_reports_security_aggregated_reports, findings: [head_vulnerability, vuln, low_vuln]) } - - it 'does not change' do - expect(subject.added).to eq([vuln, low_vuln]) - end - end - end - - describe '#fixed' do - let(:vul_params) { vuln_params(project.id, [identifier]) } - let(:vuln) { build(:ci_reports_security_finding, :dynamic, **vul_params ) } - let(:medium_vuln) { build(:ci_reports_security_finding, confidence: ::Enums::Vulnerability.confidence_levels[:high], severity: Enums::Vulnerability.severity_levels[:medium], uuid: vuln.uuid, **vul_params) } - - context 'with fixed vulnerability' do - let(:base_report) { build(:ci_reports_security_aggregated_reports, findings: [base_vulnerability, vuln]) } - - it 'points to base tree' do - expect(subject.fixed).to eq([vuln]) - end - end - - context 'when comparing reports with different fingerprints' do - include_context 'comparing reports' - - let(:base_report) { build(:ci_reports_security_aggregated_reports, findings: [base_vulnerability, vuln]) } - - it 'does not find any overlap' do - expect(subject.fixed).to eq([base_vulnerability, vuln]) - end - end - - context 'order' do - let(:vul_findings) { [vuln, medium_vuln] } - let(:base_report) { build(:ci_reports_security_aggregated_reports, findings: [*vul_findings, base_vulnerability]) } - - it 'does not change' do - expect(subject.fixed).to eq(vul_findings) - end - end - end - - describe 'with empty vulnerabilities' do - let(:empty_report) { build(:ci_reports_security_aggregated_reports, reports: [], findings: []) } - - it 'returns empty array when reports are not present' do - comparer = described_class.new(project, empty_report, empty_report) - - expect(comparer.fixed).to eq([]) - expect(comparer.added).to eq([]) - end - - it 'returns added vulnerability when base is empty and head is not empty' do - comparer = described_class.new(project, empty_report, head_report) - - expect(comparer.fixed).to eq([]) - expect(comparer.added).to eq([head_vulnerability]) - end - - it 'returns fixed vulnerability when head is empty and base is not empty' do - comparer = described_class.new(project, base_report, empty_report) - - expect(comparer.fixed).to eq([base_vulnerability]) - expect(comparer.added).to eq([]) - end - end - end - - def vuln_params(project_id, identifiers, confidence: :high, severity: :critical) - { - project_id: project_id, - report_type: :sast, - identifiers: identifiers, - confidence: ::Enums::Vulnerability.confidence_levels[confidence], - severity: ::Enums::Vulnerability.severity_levels[severity] - } - end -end diff --git a/spec/lib/gitlab/ci/runner_releases_spec.rb b/spec/lib/gitlab/ci/runner_releases_spec.rb index 14f3c95ec79..9e211327dee 100644 --- a/spec/lib/gitlab/ci/runner_releases_spec.rb +++ b/spec/lib/gitlab/ci/runner_releases_spec.rb @@ -177,6 +177,16 @@ RSpec.describe Gitlab::Ci::RunnerReleases, feature_category: :runner_fleet do it 'returns parsed and sorted Gitlab::VersionInfo objects' do expect(releases).to eq(expected_result) end + + context 'when fetching runner releases is disabled' do + before do + stub_application_setting(update_runner_versions_enabled: false) + end + + it 'returns nil' do + expect(releases).to be_nil + end + end end context 'when response contains unexpected input type' do @@ -218,6 +228,16 @@ RSpec.describe Gitlab::Ci::RunnerReleases, feature_category: :runner_fleet do it 'returns parsed and grouped Gitlab::VersionInfo objects' do expect(releases_by_minor).to eq(expected_result) end + + context 'when fetching runner releases is disabled' do + before do + stub_application_setting(update_runner_versions_enabled: false) + end + + it 'returns nil' do + expect(releases_by_minor).to be_nil + end + end end context 'when response contains unexpected input type' do @@ -233,6 +253,18 @@ RSpec.describe Gitlab::Ci::RunnerReleases, feature_category: :runner_fleet do end end + describe '#enabled?' do + it { is_expected.to be_enabled } + + context 'when fetching runner releases is disabled' do + before do + stub_application_setting(update_runner_versions_enabled: false) + end + + it { is_expected.not_to be_enabled } + end + end + def mock_http_response(response) http_response = instance_double(HTTParty::Response) diff --git a/spec/lib/gitlab/ci/secure_files/cer_spec.rb b/spec/lib/gitlab/ci/secure_files/cer_spec.rb index 6b9cd0e3bfc..76ce1785368 100644 --- a/spec/lib/gitlab/ci/secure_files/cer_spec.rb +++ b/spec/lib/gitlab/ci/secure_files/cer_spec.rb @@ -9,7 +9,7 @@ RSpec.describe Gitlab::Ci::SecureFiles::Cer do describe '#certificate_data' do it 'assigns the error message and returns nil' do expect(invalid_certificate.certificate_data).to be nil - expect(invalid_certificate.error).to eq('not enough data') + expect(invalid_certificate.error).to eq('PEM_read_bio_X509: no start line') end end @@ -50,7 +50,7 @@ RSpec.describe Gitlab::Ci::SecureFiles::Cer do describe '#expires_at' do it 'returns the certificate expiration timestamp' do - expect(subject.metadata[:expires_at]).to eq('2022-04-26 19:20:40 UTC') + expect(subject.metadata[:expires_at]).to eq('2023-04-26 19:20:39 UTC') end end diff --git a/spec/lib/gitlab/ci/secure_files/mobile_provision_spec.rb b/spec/lib/gitlab/ci/secure_files/mobile_provision_spec.rb index fb382174c64..1812b90df8b 100644 --- a/spec/lib/gitlab/ci/secure_files/mobile_provision_spec.rb +++ b/spec/lib/gitlab/ci/secure_files/mobile_provision_spec.rb @@ -10,7 +10,7 @@ RSpec.describe Gitlab::Ci::SecureFiles::MobileProvision do describe '#decoded_plist' do it 'assigns the error message and returns nil' do expect(invalid_profile.decoded_plist).to be nil - expect(invalid_profile.error).to eq('Could not parse the PKCS7: not enough data') + expect(invalid_profile.error).to eq('Could not parse the PKCS7: no start line') end end diff --git a/spec/lib/gitlab/ci/secure_files/p12_spec.rb b/spec/lib/gitlab/ci/secure_files/p12_spec.rb index beabf4b4856..7a855868ce8 100644 --- a/spec/lib/gitlab/ci/secure_files/p12_spec.rb +++ b/spec/lib/gitlab/ci/secure_files/p12_spec.rb @@ -62,7 +62,7 @@ RSpec.describe Gitlab::Ci::SecureFiles::P12 do describe '#expires_at' do it 'returns the certificate expiration timestamp' do - expect(subject.metadata[:expires_at]).to eq('2022-09-21 14:56:00 UTC') + expect(subject.metadata[:expires_at]).to eq('2023-09-21 14:55:59 UTC') end end diff --git a/spec/lib/gitlab/ci/status/composite_spec.rb b/spec/lib/gitlab/ci/status/composite_spec.rb index cceabc35e85..cbf0976c976 100644 --- a/spec/lib/gitlab/ci/status/composite_spec.rb +++ b/spec/lib/gitlab/ci/status/composite_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Status::Composite do +RSpec.describe Gitlab::Ci::Status::Composite, feature_category: :continuous_integration do let_it_be(:pipeline) { create(:ci_pipeline) } before_all do @@ -15,6 +15,18 @@ RSpec.describe Gitlab::Ci::Status::Composite do end end + describe '.initialize' do + subject(:composite_status) { described_class.new(all_statuses) } + + context 'when passing a single status' do + let(:all_statuses) { @statuses[:success] } + + it 'raises ArgumentError' do + expect { composite_status }.to raise_error(ArgumentError, 'all_jobs needs to respond to `.pluck`') + end + end + end + describe '#status' do using RSpec::Parameterized::TableSyntax @@ -51,8 +63,8 @@ RSpec.describe Gitlab::Ci::Status::Composite do %i(created success pending) | false | 'running' | false %i(skipped success failed) | false | 'failed' | false %i(skipped success failed) | true | 'skipped' | false - %i(success manual) | true | 'pending' | false - %i(success failed created) | true | 'pending' | false + %i(success manual) | true | 'manual' | false + %i(success failed created) | true | 'running' | false end with_them do diff --git a/spec/lib/gitlab/ci/status/processable/waiting_for_resource_spec.rb b/spec/lib/gitlab/ci/status/processable/waiting_for_resource_spec.rb index 26087fd771c..e1baa1097e4 100644 --- a/spec/lib/gitlab/ci/status/processable/waiting_for_resource_spec.rb +++ b/spec/lib/gitlab/ci/status/processable/waiting_for_resource_spec.rb @@ -2,12 +2,25 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Status::Processable::WaitingForResource do +RSpec.describe Gitlab::Ci::Status::Processable::WaitingForResource, feature_category: :continuous_integration do let(:user) { create(:user) } + let(:processable) { create(:ci_build, :waiting_for_resource, :resource_group) } - subject do - processable = create(:ci_build, :waiting_for_resource, :resource_group) - described_class.new(Gitlab::Ci::Status::Core.new(processable, user)) + subject { described_class.new(Gitlab::Ci::Status::Core.new(processable, user)) } + + it 'fabricates status with correct details' do + expect(subject.has_action?).to eq false + end + + context 'when resource is retained by a build' do + before do + processable.resource_group.assign_resource_to(create(:ci_build)) + end + + it 'fabricates status with correct details' do + expect(subject.has_action?).to eq true + expect(subject.action_path).to include 'jobs' + end end describe '#illustration' do diff --git a/spec/lib/gitlab/ci/templates/Jobs/build_gitlab_ci_yaml_spec.rb b/spec/lib/gitlab/ci/templates/Jobs/build_gitlab_ci_yaml_spec.rb index 07cfa939623..995922b6922 100644 --- a/spec/lib/gitlab/ci/templates/Jobs/build_gitlab_ci_yaml_spec.rb +++ b/spec/lib/gitlab/ci/templates/Jobs/build_gitlab_ci_yaml_spec.rb @@ -10,7 +10,7 @@ RSpec.describe 'Jobs/Build.gitlab-ci.yml' do describe 'AUTO_BUILD_IMAGE_VERSION' do it 'corresponds to a published image in the registry' do registry = "https://#{template_registry_host}" - repository = "gitlab-org/cluster-integration/auto-build-image" + repository = auto_build_image_repository reference = YAML.safe_load(template.content).dig('variables', 'AUTO_BUILD_IMAGE_VERSION') expect(public_image_exist?(registry, repository, reference)).to be true diff --git a/spec/lib/gitlab/ci/templates/Jobs/sast_iac_latest_gitlab_ci_yaml_spec.rb b/spec/lib/gitlab/ci/templates/Jobs/sast_iac_latest_gitlab_ci_yaml_spec.rb index 039a6a739dd..2b9213ea921 100644 --- a/spec/lib/gitlab/ci/templates/Jobs/sast_iac_latest_gitlab_ci_yaml_spec.rb +++ b/spec/lib/gitlab/ci/templates/Jobs/sast_iac_latest_gitlab_ci_yaml_spec.rb @@ -23,27 +23,33 @@ RSpec.describe 'Jobs/SAST-IaC.latest.gitlab-ci.yml', feature_category: :continuo allow(project).to receive(:default_branch).and_return(default_branch) end - context 'on feature branch' do - let(:pipeline_ref) { 'feature' } + context 'when SAST_DISABLED="false"' do + before do + create(:ci_variable, key: 'SAST_DISABLED', value: 'false', project: project) + end + + context 'on feature branch' do + let(:pipeline_ref) { 'feature' } - it 'creates the kics-iac-sast job' do - expect(build_names).to contain_exactly('kics-iac-sast') + it 'creates the kics-iac-sast job' do + expect(build_names).to contain_exactly('kics-iac-sast') + end end - end - context 'on merge request' do - let(:service) { MergeRequests::CreatePipelineService.new(project: project, current_user: user) } - let(:merge_request) { create(:merge_request, :simple, source_project: project) } - let(:pipeline) { service.execute(merge_request).payload } + context 'on merge request' do + let(:service) { MergeRequests::CreatePipelineService.new(project: project, current_user: user) } + let(:merge_request) { create(:merge_request, :simple, source_project: project) } + let(:pipeline) { service.execute(merge_request).payload } - it 'creates a pipeline with the expected jobs' do - expect(pipeline).to be_merge_request_event - expect(pipeline.errors.full_messages).to be_empty - expect(build_names).to match_array(%w(kics-iac-sast)) + it 'creates a pipeline with the expected jobs' do + expect(pipeline).to be_merge_request_event + expect(pipeline.errors.full_messages).to be_empty + expect(build_names).to match_array(%w(kics-iac-sast)) + end end end - context 'SAST_DISABLED is set' do + context 'when SAST_DISABLED="true"' do before do create(:ci_variable, key: 'SAST_DISABLED', value: 'true', project: project) end diff --git a/spec/lib/gitlab/ci/trace/chunked_io_spec.rb b/spec/lib/gitlab/ci/trace/chunked_io_spec.rb index 63625244fe8..7a926a06f16 100644 --- a/spec/lib/gitlab/ci/trace/chunked_io_spec.rb +++ b/spec/lib/gitlab/ci/trace/chunked_io_spec.rb @@ -446,15 +446,5 @@ RSpec.describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do expect(Ci::BuildTraceChunk.where(build: build).count).to eq(0) end - - context 'when the job does not have archived trace' do - it 'leaves a message in sidekiq log' do - expect(Sidekiq.logger).to receive(:warn).with( - message: 'The job does not have archived trace but going to be destroyed.', - job_id: build.id).and_call_original - - subject - end - end end end diff --git a/spec/lib/gitlab/ci/variables/builder/pipeline_spec.rb b/spec/lib/gitlab/ci/variables/builder/pipeline_spec.rb index a5365ae53b8..0a079a69682 100644 --- a/spec/lib/gitlab/ci/variables/builder/pipeline_spec.rb +++ b/spec/lib/gitlab/ci/variables/builder/pipeline_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Variables::Builder::Pipeline, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Variables::Builder::Pipeline, feature_category: :secrets_management do let_it_be(:project) { create_default(:project, :repository, create_tag: 'test').freeze } let_it_be(:user) { create(:user) } @@ -30,15 +30,13 @@ RSpec.describe Gitlab::Ci::Variables::Builder::Pipeline, feature_category: :pipe CI_COMMIT_REF_PROTECTED CI_COMMIT_TIMESTAMP CI_COMMIT_AUTHOR - CI_BUILD_REF - CI_BUILD_BEFORE_SHA - CI_BUILD_REF_NAME - CI_BUILD_REF_SLUG ]) end - context 'when the pipeline is running for a tag' do - let(:pipeline) { build(:ci_empty_pipeline, :created, project: project, ref: 'test', tag: true) } + context 'when FF `ci_remove_legacy_predefined_variables` is disabled' do + before do + stub_feature_flags(ci_remove_legacy_predefined_variables: false) + end it 'includes all predefined variables in a valid order' do keys = subject.pluck(:key) @@ -52,6 +50,7 @@ RSpec.describe Gitlab::Ci::Variables::Builder::Pipeline, feature_category: :pipe CI_COMMIT_BEFORE_SHA CI_COMMIT_REF_NAME CI_COMMIT_REF_SLUG + CI_COMMIT_BRANCH CI_COMMIT_MESSAGE CI_COMMIT_TITLE CI_COMMIT_DESCRIPTION @@ -62,11 +61,69 @@ RSpec.describe Gitlab::Ci::Variables::Builder::Pipeline, feature_category: :pipe CI_BUILD_BEFORE_SHA CI_BUILD_REF_NAME CI_BUILD_REF_SLUG + ]) + end + end + + context 'when the pipeline is running for a tag' do + let(:pipeline) { build(:ci_empty_pipeline, :created, project: project, ref: 'test', tag: true) } + + it 'includes all predefined variables in a valid order' do + keys = subject.pluck(:key) + + expect(keys).to contain_exactly(*%w[ + CI_PIPELINE_IID + CI_PIPELINE_SOURCE + CI_PIPELINE_CREATED_AT + CI_COMMIT_SHA + CI_COMMIT_SHORT_SHA + CI_COMMIT_BEFORE_SHA + CI_COMMIT_REF_NAME + CI_COMMIT_REF_SLUG + CI_COMMIT_MESSAGE + CI_COMMIT_TITLE + CI_COMMIT_DESCRIPTION + CI_COMMIT_REF_PROTECTED + CI_COMMIT_TIMESTAMP + CI_COMMIT_AUTHOR CI_COMMIT_TAG CI_COMMIT_TAG_MESSAGE - CI_BUILD_TAG ]) end + + context 'when FF `ci_remove_legacy_predefined_variables` is disabled' do + before do + stub_feature_flags(ci_remove_legacy_predefined_variables: false) + end + + it 'includes all predefined variables in a valid order' do + keys = subject.pluck(:key) + + expect(keys).to contain_exactly(*%w[ + CI_PIPELINE_IID + CI_PIPELINE_SOURCE + CI_PIPELINE_CREATED_AT + CI_COMMIT_SHA + CI_COMMIT_SHORT_SHA + CI_COMMIT_BEFORE_SHA + CI_COMMIT_REF_NAME + CI_COMMIT_REF_SLUG + CI_COMMIT_MESSAGE + CI_COMMIT_TITLE + CI_COMMIT_DESCRIPTION + CI_COMMIT_REF_PROTECTED + CI_COMMIT_TIMESTAMP + CI_COMMIT_AUTHOR + CI_BUILD_REF + CI_BUILD_BEFORE_SHA + CI_BUILD_REF_NAME + CI_BUILD_REF_SLUG + CI_COMMIT_TAG + CI_COMMIT_TAG_MESSAGE + CI_BUILD_TAG + ]) + end + end end context 'when merge request is present' do @@ -305,10 +362,24 @@ RSpec.describe Gitlab::Ci::Variables::Builder::Pipeline, feature_category: :pipe expect(subject.to_hash.keys) .not_to include( 'CI_COMMIT_TAG', - 'CI_COMMIT_TAG_MESSAGE', - 'CI_BUILD_TAG' + 'CI_COMMIT_TAG_MESSAGE' ) end + + context 'when FF `ci_remove_legacy_predefined_variables` is disabled' do + before do + stub_feature_flags(ci_remove_legacy_predefined_variables: false) + end + + it 'does not expose tag variables' do + expect(subject.to_hash.keys) + .not_to include( + 'CI_COMMIT_TAG', + 'CI_COMMIT_TAG_MESSAGE', + 'CI_BUILD_TAG' + ) + end + end end context 'without a commit' do diff --git a/spec/lib/gitlab/ci/variables/builder_spec.rb b/spec/lib/gitlab/ci/variables/builder_spec.rb index bbd3dc54e6a..10974993fa4 100644 --- a/spec/lib/gitlab/ci/variables/builder_spec.rb +++ b/spec/lib/gitlab/ci/variables/builder_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Variables::Builder, :clean_gitlab_redis_cache, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Variables::Builder, :clean_gitlab_redis_cache, feature_category: :secrets_management do include Ci::TemplateHelpers let_it_be(:group) { create(:group) } let_it_be(:project) { create(:project, :repository, namespace: group) } @@ -35,10 +35,6 @@ RSpec.describe Gitlab::Ci::Variables::Builder, :clean_gitlab_redis_cache, featur value: '1' }, { key: 'CI_ENVIRONMENT_NAME', value: 'test' }, - { key: 'CI_BUILD_NAME', - value: 'rspec:test 1' }, - { key: 'CI_BUILD_STAGE', - value: job.stage_name }, { key: 'CI', value: 'true' }, { key: 'GITLAB_CI', @@ -51,6 +47,10 @@ RSpec.describe Gitlab::Ci::Variables::Builder, :clean_gitlab_redis_cache, featur value: Gitlab.config.gitlab.port.to_s }, { key: 'CI_SERVER_PROTOCOL', value: Gitlab.config.gitlab.protocol }, + { key: 'CI_SERVER_SHELL_SSH_HOST', + value: Gitlab.config.gitlab_shell.ssh_host.to_s }, + { key: 'CI_SERVER_SHELL_SSH_PORT', + value: Gitlab.config.gitlab_shell.ssh_port.to_s }, { key: 'CI_SERVER_NAME', value: 'GitLab' }, { key: 'CI_SERVER_VERSION', @@ -101,6 +101,8 @@ RSpec.describe Gitlab::Ci::Variables::Builder, :clean_gitlab_redis_cache, featur value: project.pages_url }, { key: 'CI_API_V4_URL', value: API::Helpers::Version.new('v4').root_url }, + { key: 'CI_API_GRAPHQL_URL', + value: Gitlab::Routing.url_helpers.api_graphql_url }, { key: 'CI_TEMPLATE_REGISTRY_HOST', value: template_registry_host }, { key: 'CI_PIPELINE_IID', @@ -133,14 +135,6 @@ RSpec.describe Gitlab::Ci::Variables::Builder, :clean_gitlab_redis_cache, featur value: pipeline.git_commit_timestamp }, { key: 'CI_COMMIT_AUTHOR', value: pipeline.git_author_full_text }, - { key: 'CI_BUILD_REF', - value: job.sha }, - { key: 'CI_BUILD_BEFORE_SHA', - value: job.before_sha }, - { key: 'CI_BUILD_REF_NAME', - value: job.ref }, - { key: 'CI_BUILD_REF_SLUG', - value: job.ref_slug }, { key: 'YAML_VARIABLE', value: 'value' }, { key: 'GITLAB_USER_ID', @@ -160,6 +154,151 @@ RSpec.describe Gitlab::Ci::Variables::Builder, :clean_gitlab_redis_cache, featur it { expect(subject.to_runner_variables).to eq(predefined_variables) } + context 'when FF `ci_remove_legacy_predefined_variables` is disabled' do + before do + stub_feature_flags(ci_remove_legacy_predefined_variables: false) + end + + let(:predefined_variables) do + [ + { key: 'CI_JOB_NAME', + value: 'rspec:test 1' }, + { key: 'CI_JOB_NAME_SLUG', + value: 'rspec-test-1' }, + { key: 'CI_JOB_STAGE', + value: job.stage_name }, + { key: 'CI_NODE_TOTAL', + value: '1' }, + { key: 'CI_ENVIRONMENT_NAME', + value: 'test' }, + { key: 'CI_BUILD_NAME', + value: 'rspec:test 1' }, + { key: 'CI_BUILD_STAGE', + value: job.stage_name }, + { key: 'CI', + value: 'true' }, + { key: 'GITLAB_CI', + value: 'true' }, + { key: 'CI_SERVER_URL', + value: Gitlab.config.gitlab.url }, + { key: 'CI_SERVER_HOST', + value: Gitlab.config.gitlab.host }, + { key: 'CI_SERVER_PORT', + value: Gitlab.config.gitlab.port.to_s }, + { key: 'CI_SERVER_PROTOCOL', + value: Gitlab.config.gitlab.protocol }, + { key: 'CI_SERVER_SHELL_SSH_HOST', + value: Gitlab.config.gitlab_shell.ssh_host.to_s }, + { key: 'CI_SERVER_SHELL_SSH_PORT', + value: Gitlab.config.gitlab_shell.ssh_port.to_s }, + { key: 'CI_SERVER_NAME', + value: 'GitLab' }, + { key: 'CI_SERVER_VERSION', + value: Gitlab::VERSION }, + { key: 'CI_SERVER_VERSION_MAJOR', + value: Gitlab.version_info.major.to_s }, + { key: 'CI_SERVER_VERSION_MINOR', + value: Gitlab.version_info.minor.to_s }, + { key: 'CI_SERVER_VERSION_PATCH', + value: Gitlab.version_info.patch.to_s }, + { key: 'CI_SERVER_REVISION', + value: Gitlab.revision }, + { key: 'GITLAB_FEATURES', + value: project.licensed_features.join(',') }, + { key: 'CI_PROJECT_ID', + value: project.id.to_s }, + { key: 'CI_PROJECT_NAME', + value: project.path }, + { key: 'CI_PROJECT_TITLE', + value: project.title }, + { key: 'CI_PROJECT_DESCRIPTION', + value: project.description }, + { key: 'CI_PROJECT_PATH', + value: project.full_path }, + { key: 'CI_PROJECT_PATH_SLUG', + value: project.full_path_slug }, + { key: 'CI_PROJECT_NAMESPACE', + value: project.namespace.full_path }, + { key: 'CI_PROJECT_NAMESPACE_ID', + value: project.namespace.id.to_s }, + { key: 'CI_PROJECT_ROOT_NAMESPACE', + value: project.namespace.root_ancestor.path }, + { key: 'CI_PROJECT_URL', + value: project.web_url }, + { key: 'CI_PROJECT_VISIBILITY', + value: "private" }, + { key: 'CI_PROJECT_REPOSITORY_LANGUAGES', + value: project.repository_languages.map(&:name).join(',').downcase }, + { key: 'CI_PROJECT_CLASSIFICATION_LABEL', + value: project.external_authorization_classification_label }, + { key: 'CI_DEFAULT_BRANCH', + value: project.default_branch }, + { key: 'CI_CONFIG_PATH', + value: project.ci_config_path_or_default }, + { key: 'CI_PAGES_DOMAIN', + value: Gitlab.config.pages.host }, + { key: 'CI_PAGES_URL', + value: project.pages_url }, + { key: 'CI_API_V4_URL', + value: API::Helpers::Version.new('v4').root_url }, + { key: 'CI_API_GRAPHQL_URL', + value: Gitlab::Routing.url_helpers.api_graphql_url }, + { key: 'CI_TEMPLATE_REGISTRY_HOST', + value: template_registry_host }, + { key: 'CI_PIPELINE_IID', + value: pipeline.iid.to_s }, + { key: 'CI_PIPELINE_SOURCE', + value: pipeline.source }, + { key: 'CI_PIPELINE_CREATED_AT', + value: pipeline.created_at.iso8601 }, + { key: 'CI_COMMIT_SHA', + value: job.sha }, + { key: 'CI_COMMIT_SHORT_SHA', + value: job.short_sha }, + { key: 'CI_COMMIT_BEFORE_SHA', + value: job.before_sha }, + { key: 'CI_COMMIT_REF_NAME', + value: job.ref }, + { key: 'CI_COMMIT_REF_SLUG', + value: job.ref_slug }, + { key: 'CI_COMMIT_BRANCH', + value: job.ref }, + { key: 'CI_COMMIT_MESSAGE', + value: pipeline.git_commit_message }, + { key: 'CI_COMMIT_TITLE', + value: pipeline.git_commit_title }, + { key: 'CI_COMMIT_DESCRIPTION', + value: pipeline.git_commit_description }, + { key: 'CI_COMMIT_REF_PROTECTED', + value: (!!pipeline.protected_ref?).to_s }, + { key: 'CI_COMMIT_TIMESTAMP', + value: pipeline.git_commit_timestamp }, + { key: 'CI_COMMIT_AUTHOR', + value: pipeline.git_author_full_text }, + { key: 'CI_BUILD_REF', + value: job.sha }, + { key: 'CI_BUILD_BEFORE_SHA', + value: job.before_sha }, + { key: 'CI_BUILD_REF_NAME', + value: job.ref }, + { key: 'CI_BUILD_REF_SLUG', + value: job.ref_slug }, + { key: 'YAML_VARIABLE', + value: 'value' }, + { key: 'GITLAB_USER_ID', + value: user.id.to_s }, + { key: 'GITLAB_USER_EMAIL', + value: user.email }, + { key: 'GITLAB_USER_LOGIN', + value: user.username }, + { key: 'GITLAB_USER_NAME', + value: user.name } + ].map { |var| var.merge(public: true, masked: false) } + end + + it { expect(subject.to_runner_variables).to eq(predefined_variables) } + end + context 'variables ordering' do def var(name, value) { key: name, value: value.to_s, public: true, masked: false } diff --git a/spec/lib/gitlab/ci/variables/collection_spec.rb b/spec/lib/gitlab/ci/variables/collection_spec.rb index 4ee122cc607..181e37de9b9 100644 --- a/spec/lib/gitlab/ci/variables/collection_spec.rb +++ b/spec/lib/gitlab/ci/variables/collection_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Variables::Collection, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Ci::Variables::Collection, feature_category: :secrets_management do describe '.new' do it 'can be initialized with an array' do variable = { key: 'VAR', value: 'value', public: true, masked: false } diff --git a/spec/lib/gitlab/ci/yaml_processor/result_spec.rb b/spec/lib/gitlab/ci/yaml_processor/result_spec.rb index 5c9f156e054..36ada9050b2 100644 --- a/spec/lib/gitlab/ci/yaml_processor/result_spec.rb +++ b/spec/lib/gitlab/ci/yaml_processor/result_spec.rb @@ -47,8 +47,8 @@ module Gitlab end it 'returns expanded yaml config' do - expanded_config = YAML.safe_load(config_metadata[:merged_yaml], [Symbol]) - included_config = YAML.safe_load(included_yml, [Symbol]) + expanded_config = YAML.safe_load(config_metadata[:merged_yaml], permitted_classes: [Symbol]) + included_config = YAML.safe_load(included_yml, permitted_classes: [Symbol]) expect(expanded_config).to include(*included_config.keys) end diff --git a/spec/lib/gitlab/ci/yaml_processor_spec.rb b/spec/lib/gitlab/ci/yaml_processor_spec.rb index 360686ce65c..2c020e76cb6 100644 --- a/spec/lib/gitlab/ci/yaml_processor_spec.rb +++ b/spec/lib/gitlab/ci/yaml_processor_spec.rb @@ -4,7 +4,7 @@ require 'spec_helper' module Gitlab module Ci - RSpec.describe YamlProcessor, feature_category: :pipeline_authoring do + RSpec.describe YamlProcessor, feature_category: :pipeline_composition do include StubRequests include RepoHelpers @@ -659,6 +659,191 @@ module Gitlab it_behaves_like 'has warnings and expected error', /build job: need test is not defined in current or prior stages/ end + + describe '#validate_job_needs!' do + context "when all validations pass" do + let(:config) do + <<-EOYML + stages: + - lint + lint_job: + needs: [lint_job_2] + stage: lint + script: 'echo lint_job' + rules: + - if: $var == null + needs: + - lint_job_2 + - job: lint_job_3 + optional: true + lint_job_2: + stage: lint + script: 'echo job' + rules: + - if: $var == null + lint_job_3: + stage: lint + script: 'echo job' + rules: + - if: $var == null + EOYML + end + + it 'returns a valid response' do + expect(subject).to be_valid + expect(subject).to be_instance_of(Gitlab::Ci::YamlProcessor::Result) + end + end + + context 'needs as array' do + context 'single need in following stage' do + let(:config) do + <<-EOYML + stages: + - lint + - test + lint_job: + stage: lint + script: 'echo lint_job' + rules: + - if: $var == null + needs: [test_job] + test_job: + stage: test + script: 'echo job' + rules: + - if: $var == null + EOYML + end + + it_behaves_like 'returns errors', 'lint_job job: need test_job is not defined in current or prior stages' + end + + context 'multiple needs in the following stage' do + let(:config) do + <<-EOYML + stages: + - lint + - test + lint_job: + stage: lint + script: 'echo lint_job' + rules: + - if: $var == null + needs: [test_job, test_job_2] + test_job: + stage: test + script: 'echo job' + rules: + - if: $var == null + test_job_2: + stage: test + script: 'echo job' + rules: + - if: $var == null + EOYML + end + + it_behaves_like 'returns errors', 'lint_job job: need test_job is not defined in current or prior stages' + end + + context 'single need in following state - hyphen need' do + let(:config) do + <<-EOYML + stages: + - lint + - test + lint_job: + stage: lint + script: 'echo lint_job' + rules: + - if: $var == null + needs: + - test_job + test_job: + stage: test + script: 'echo job' + rules: + - if: $var == null + EOYML + end + + it_behaves_like 'returns errors', 'lint_job job: need test_job is not defined in current or prior stages' + end + + context 'when there are duplicate needs (string and hash)' do + let(:config) do + <<-EOYML + stages: + - test + test_job_1: + stage: test + script: 'echo lint_job' + rules: + - if: $var == null + needs: + - test_job_2 + - job: test_job_2 + test_job_2: + stage: test + script: 'echo job' + rules: + - if: $var == null + EOYML + end + + it_behaves_like 'returns errors', 'test_job_1 has the following needs duplicated: test_job_2.' + end + end + + context 'rule needs as hash' do + context 'single hash need in following stage' do + let(:config) do + <<-EOYML + stages: + - lint + - test + lint_job: + stage: lint + script: 'echo lint_job' + rules: + - if: $var == null + needs: + - job: test_job + artifacts: false + optional: false + test_job: + stage: test + script: 'echo job' + rules: + - if: $var == null + EOYML + end + + it_behaves_like 'returns errors', 'lint_job job: need test_job is not defined in current or prior stages' + end + end + + context 'job rule need does not exist' do + let(:config) do + <<-EOYML + build: + stage: build + script: echo + rules: + - when: always + test: + stage: test + script: echo + rules: + - if: $var == null + needs: [unknown_job] + EOYML + end + + it_behaves_like 'has warnings and expected error', /test job: undefined need: unknown_job/ + end + end end end @@ -1685,7 +1870,8 @@ module Gitlab key: 'key', policy: 'pull-push', when: 'on_success', - unprotect: false + unprotect: false, + fallback_keys: [] ]) end @@ -1710,7 +1896,8 @@ module Gitlab key: { files: ['file'] }, policy: 'pull-push', when: 'on_success', - unprotect: false + unprotect: false, + fallback_keys: [] ]) end @@ -1737,7 +1924,8 @@ module Gitlab key: 'keya', policy: 'pull-push', when: 'on_success', - unprotect: false + unprotect: false, + fallback_keys: [] }, { paths: ['logs/', 'binaries/'], @@ -1745,7 +1933,8 @@ module Gitlab key: 'key', policy: 'pull-push', when: 'on_success', - unprotect: false + unprotect: false, + fallback_keys: [] } ] ) @@ -1773,7 +1962,8 @@ module Gitlab key: { files: ['file'] }, policy: 'pull-push', when: 'on_success', - unprotect: false + unprotect: false, + fallback_keys: [] ]) end @@ -1799,7 +1989,8 @@ module Gitlab key: { files: ['file'], prefix: 'prefix' }, policy: 'pull-push', when: 'on_success', - unprotect: false + unprotect: false, + fallback_keys: [] ]) end @@ -1823,7 +2014,8 @@ module Gitlab key: 'local', policy: 'pull-push', when: 'on_success', - unprotect: false + unprotect: false, + fallback_keys: [] ]) end end @@ -2395,10 +2587,16 @@ module Gitlab end end - context 'undefined need' do + context 'when need is an undefined job' do let(:needs) { ['undefined'] } it_behaves_like 'returns errors', 'test1 job: undefined need: undefined' + + context 'when need is optional' do + let(:needs) { [{ job: 'undefined', optional: true }] } + + it { is_expected.to be_valid } + end end context 'needs to deploy' do @@ -2408,9 +2606,33 @@ module Gitlab end context 'duplicate needs' do - let(:needs) { %w(build1 build1) } + context 'when needs are specified in an array' do + let(:needs) { %w(build1 build1) } + + it_behaves_like 'returns errors', 'test1 has the following needs duplicated: build1.' + end + + context 'when a job is specified multiple times' do + let(:needs) do + [ + { job: "build2", artifacts: true, optional: false }, + { job: "build2", artifacts: true, optional: false } + ] + end - it_behaves_like 'returns errors', 'test1 has duplicate entries in the needs section.' + it_behaves_like 'returns errors', 'test1 has the following needs duplicated: build2.' + end + + context 'when job is specified multiple times with different attributes' do + let(:needs) do + [ + { job: "build2", artifacts: false, optional: true }, + { job: "build2", artifacts: true, optional: false } + ] + end + + it_behaves_like 'returns errors', 'test1 has the following needs duplicated: build2.' + end end context 'needs and dependencies that are mismatching' do diff --git a/spec/lib/gitlab/color_schemes_spec.rb b/spec/lib/gitlab/color_schemes_spec.rb index feb5648ff2d..bc69c8beeda 100644 --- a/spec/lib/gitlab/color_schemes_spec.rb +++ b/spec/lib/gitlab/color_schemes_spec.rb @@ -21,8 +21,9 @@ RSpec.describe Gitlab::ColorSchemes do end describe '.default' do - it 'returns the default scheme' do - expect(described_class.default.id).to eq 1 + it 'use config default' do + stub_application_setting(default_syntax_highlighting_theme: 2) + expect(described_class.default.id).to eq 2 end end @@ -36,7 +37,8 @@ RSpec.describe Gitlab::ColorSchemes do describe '.for_user' do it 'returns default when user is nil' do - expect(described_class.for_user(nil).id).to eq 1 + stub_application_setting(default_syntax_highlighting_theme: 2) + expect(described_class.for_user(nil).id).to eq 2 end it "returns user's preferred color scheme" do diff --git a/spec/lib/gitlab/color_spec.rb b/spec/lib/gitlab/color_spec.rb index 28719aa6199..45815bb6e53 100644 --- a/spec/lib/gitlab/color_spec.rb +++ b/spec/lib/gitlab/color_spec.rb @@ -125,12 +125,12 @@ RSpec.describe Gitlab::Color do expect(described_class.new('#fff')).to be_light end - specify '#a7a7a7 is light' do - expect(described_class.new('#a7a7a7')).to be_light + specify '#c2c2c2 is light' do + expect(described_class.new('#c2c2c2')).to be_light end - specify '#a6a7a7 is dark' do - expect(described_class.new('#a6a7a7')).not_to be_light + specify '#868686 is dark' do + expect(described_class.new('#868686')).not_to be_light end specify '#000 is dark' do @@ -145,7 +145,7 @@ RSpec.describe Gitlab::Color do describe '#contrast' do context 'with light colors' do it 'is dark' do - %w[#fff #fefefe #a7a7a7].each do |hex| + %w[#fff #fefefe #c2c2c2].each do |hex| expect(described_class.new(hex)).to have_attributes( contrast: described_class::Constants::DARK, luminosity: :light diff --git a/spec/lib/gitlab/config/entry/validators_spec.rb b/spec/lib/gitlab/config/entry/validators_spec.rb index 54a2adbefd2..abf3dbacb3d 100644 --- a/spec/lib/gitlab/config/entry/validators_spec.rb +++ b/spec/lib/gitlab/config/entry/validators_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Config::Entry::Validators, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Config::Entry::Validators, feature_category: :pipeline_composition do let(:klass) do Class.new do include ActiveModel::Validations diff --git a/spec/lib/gitlab/config/loader/multi_doc_yaml_spec.rb b/spec/lib/gitlab/config/loader/multi_doc_yaml_spec.rb index bae98f9bc35..438f3e5b17a 100644 --- a/spec/lib/gitlab/config/loader/multi_doc_yaml_spec.rb +++ b/spec/lib/gitlab/config/loader/multi_doc_yaml_spec.rb @@ -2,26 +2,122 @@ require 'spec_helper' -RSpec.describe Gitlab::Config::Loader::MultiDocYaml, feature_category: :pipeline_authoring do - let(:loader) { described_class.new(yml, max_documents: 2) } +RSpec.describe Gitlab::Config::Loader::MultiDocYaml, feature_category: :pipeline_composition do + let(:loader) { described_class.new(yml, max_documents: 2, reject_empty: reject_empty) } + let(:reject_empty) { false } describe '#load!' do - let(:yml) do - <<~YAML - spec: - inputs: - test_input: - --- - test_job: - script: echo "$[[ inputs.test_input ]]" - YAML + context 'when a simple single delimiter is being used' do + let(:yml) do + <<~YAML + spec: + inputs: + env: + --- + test: + script: echo "$[[ inputs.env ]]" + YAML + end + + it 'returns the loaded YAML with all keys as symbols' do + expect(loader.load!).to contain_exactly( + { spec: { inputs: { env: nil } } }, + { test: { script: 'echo "$[[ inputs.env ]]"' } } + ) + end + end + + context 'when the delimiter has a trailing configuration' do + let(:yml) do + <<~YAML + spec: + inputs: + test_input: + --- !test/content + test_job: + script: echo "$[[ inputs.test_input ]]" + YAML + end + + it 'returns the loaded YAML with all keys as symbols' do + expect(loader.load!).to contain_exactly( + { spec: { inputs: { test_input: nil } } }, + { test_job: { script: 'echo "$[[ inputs.test_input ]]"' } } + ) + end + end + + context 'when the YAML file has a leading delimiter' do + let(:yml) do + <<~YAML + --- + spec: + inputs: + test_input: + --- !test/content + test_job: + script: echo "$[[ inputs.test_input ]]" + YAML + end + + it 'returns the loaded YAML with all keys as symbols' do + expect(loader.load!).to contain_exactly( + { spec: { inputs: { test_input: nil } } }, + { test_job: { script: 'echo "$[[ inputs.test_input ]]"' } } + ) + end + end + + context 'when the delimiter is followed by content on the same line' do + let(:yml) do + <<~YAML + --- a: 1 + --- b: 2 + YAML + end + + it 'loads the content as part of the document' do + expect(loader.load!).to contain_exactly({ a: 1 }, { b: 2 }) + end end - it 'returns the loaded YAML with all keys as symbols' do - expect(loader.load!).to eq([ - { spec: { inputs: { test_input: nil } } }, - { test_job: { script: 'echo "$[[ inputs.test_input ]]"' } } - ]) + context 'when the delimiter does not have trailing whitespace' do + let(:yml) do + <<~YAML + --- a: 1 + ---b: 2 + YAML + end + + it 'is not a valid delimiter' do + expect(loader.load!).to contain_exactly({ :'---b' => 2, a: 1 }) # rubocop:disable Style/HashSyntax + end + end + + context 'when the YAML file has whitespace preceding the content' do + let(:yml) do + <<-EOYML + variables: + SUPPORTED: "parsed" + + workflow: + rules: + - if: $VAR == "value" + + hello: + script: echo world + EOYML + end + + it 'loads everything correctly' do + expect(loader.load!).to contain_exactly( + { + variables: { SUPPORTED: 'parsed' }, + workflow: { rules: [{ if: '$VAR == "value"' }] }, + hello: { script: 'echo world' } + } + ) + end end context 'when the YAML file is empty' do @@ -32,67 +128,89 @@ RSpec.describe Gitlab::Config::Loader::MultiDocYaml, feature_category: :pipeline end end - context 'when the parsed YAML is too big' do + context 'when there are more than the maximum number of documents' do + let(:yml) do + <<~YAML + --- a: 1 + --- b: 2 + --- c: 3 + --- d: 4 + YAML + end + + it 'stops splitting documents after the maximum number' do + expect(loader.load!).to contain_exactly({ a: 1 }, { b: 2 }) + end + end + + context 'when the YAML contains empty documents' do let(:yml) do <<~YAML - a: &a ["lol","lol","lol","lol","lol","lol","lol","lol","lol"] - b: &b [*a,*a,*a,*a,*a,*a,*a,*a,*a] - c: &c [*b,*b,*b,*b,*b,*b,*b,*b,*b] - d: &d [*c,*c,*c,*c,*c,*c,*c,*c,*c] - e: &e [*d,*d,*d,*d,*d,*d,*d,*d,*d] - f: &f [*e,*e,*e,*e,*e,*e,*e,*e,*e] - g: &g [*f,*f,*f,*f,*f,*f,*f,*f,*f] - h: &h [*g,*g,*g,*g,*g,*g,*g,*g,*g] - i: &i [*h,*h,*h,*h,*h,*h,*h,*h,*h] + a: 1 --- - a: &a ["lol","lol","lol","lol","lol","lol","lol","lol","lol"] - b: &b [*a,*a,*a,*a,*a,*a,*a,*a,*a] - c: &c [*b,*b,*b,*b,*b,*b,*b,*b,*b] - d: &d [*c,*c,*c,*c,*c,*c,*c,*c,*c] - e: &e [*d,*d,*d,*d,*d,*d,*d,*d,*d] - f: &f [*e,*e,*e,*e,*e,*e,*e,*e,*e] - g: &g [*f,*f,*f,*f,*f,*f,*f,*f,*f] - h: &h [*g,*g,*g,*g,*g,*g,*g,*g,*g] - i: &i [*h,*h,*h,*h,*h,*h,*h,*h,*h] YAML end - it 'raises a DataTooLargeError' do - expect { loader.load! }.to raise_error(described_class::DataTooLargeError, 'The parsed YAML is too big') + it 'raises an error' do + expect { loader.load! }.to raise_error(::Gitlab::Config::Loader::Yaml::NotHashError) + end + + context 'when reject_empty: true' do + let(:reject_empty) { true } + + it 'loads only non empty documents' do + expect(loader.load!).to contain_exactly({ a: 1 }) + end end end + end - context 'when a document is not a hash' do + describe '#load_raw!' do + let(:yml) do + <<~YAML + spec: + inputs: + test_input: + --- !test/content + test_job: + script: echo "$[[ inputs.test_input ]]" + YAML + end + + it 'returns the loaded YAML with all keys as strings' do + expect(loader.load_raw!).to contain_exactly( + { 'spec' => { 'inputs' => { 'test_input' => nil } } }, + { 'test_job' => { 'script' => 'echo "$[[ inputs.test_input ]]"' } } + ) + end + end + + describe '#valid?' do + context 'when a document is invalid' do let(:yml) do <<~YAML - not_a_hash + a: b --- - test_job: - script: echo "$[[ inputs.test_input ]]" + c YAML end - it 'raises a NotHashError' do - expect { loader.load! }.to raise_error(described_class::NotHashError, 'Invalid configuration format') + it 'returns false' do + expect(loader).not_to be_valid end end - context 'when there are too many documents' do + context 'when the number of documents is below the maximum and all documents are valid' do let(:yml) do <<~YAML a: b --- c: d - --- - e: f YAML end - it 'raises a TooManyDocumentsError' do - expect { loader.load! }.to raise_error( - described_class::TooManyDocumentsError, - 'The parsed YAML has too many documents' - ) + it 'returns true' do + expect(loader).to be_valid end end end diff --git a/spec/lib/gitlab/config/loader/yaml_spec.rb b/spec/lib/gitlab/config/loader/yaml_spec.rb index 346424d1681..bba66f33718 100644 --- a/spec/lib/gitlab/config/loader/yaml_spec.rb +++ b/spec/lib/gitlab/config/loader/yaml_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Config::Loader::Yaml, feature_category: :pipeline_authoring do +RSpec.describe Gitlab::Config::Loader::Yaml, feature_category: :pipeline_composition do let(:loader) { described_class.new(yml) } let(:yml) do @@ -182,4 +182,30 @@ RSpec.describe Gitlab::Config::Loader::Yaml, feature_category: :pipeline_authori ) end end + + describe '#blank?' do + context 'when the loaded YAML is empty' do + let(:yml) do + <<~YAML + # only comments here + YAML + end + + it 'returns true' do + expect(loader).to be_blank + end + end + + context 'when the loaded YAML has content' do + let(:yml) do + <<~YAML + test: value + YAML + end + + it 'returns false' do + expect(loader).not_to be_blank + end + end + end end diff --git a/spec/lib/gitlab/config_checker/external_database_checker_spec.rb b/spec/lib/gitlab/config_checker/external_database_checker_spec.rb index c962b9ad393..6379a5edb90 100644 --- a/spec/lib/gitlab/config_checker/external_database_checker_spec.rb +++ b/spec/lib/gitlab/config_checker/external_database_checker_spec.rb @@ -89,9 +89,9 @@ RSpec.describe Gitlab::ConfigChecker::ExternalDatabaseChecker do { type: 'warning', message: _('Database \'%{database_name}\' is using PostgreSQL %{pg_version_current}, ' \ - 'but PostgreSQL %{pg_version_minimum} is required for this version of GitLab. ' \ - 'Please upgrade your environment to a supported PostgreSQL version, ' \ - 'see %{pg_requirements_url} for details.') % \ + 'but this version of GitLab requires PostgreSQL %{pg_version_minimum}. ' \ + 'Please upgrade your environment to a supported PostgreSQL version. ' \ + 'See %{pg_requirements_url} for details.') % \ { database_name: database_name, pg_version_current: database_version, diff --git a/spec/lib/gitlab/console_spec.rb b/spec/lib/gitlab/console_spec.rb index f043433b4c5..5723a4421f6 100644 --- a/spec/lib/gitlab/console_spec.rb +++ b/spec/lib/gitlab/console_spec.rb @@ -1,8 +1,8 @@ # frozen_string_literal: true -require 'fast_spec_helper' +require 'spec_helper' -RSpec.describe Gitlab::Console do +RSpec.describe Gitlab::Console, feature_category: :application_instrumentation do describe '.welcome!' do context 'when running in the Rails console' do before do diff --git a/spec/lib/gitlab/consul/internal_spec.rb b/spec/lib/gitlab/consul/internal_spec.rb index 28dcaac9ff2..cd3436b3fa4 100644 --- a/spec/lib/gitlab/consul/internal_spec.rb +++ b/spec/lib/gitlab/consul/internal_spec.rb @@ -22,7 +22,7 @@ RSpec.describe Gitlab::Consul::Internal do context 'when consul setting is not present in gitlab.yml' do before do - allow(Gitlab.config).to receive(:consul).and_raise(Settingslogic::MissingSetting) + allow(Gitlab.config).to receive(:consul).and_raise(GitlabSettings::MissingSetting) end it 'does not fail' do @@ -77,9 +77,11 @@ RSpec.describe Gitlab::Consul::Internal do shared_examples 'returns nil given blank value of' do |input_symbol| [nil, ''].each do |value| - let(input_symbol) { value } + context "with #{value}" do + let(input_symbol) { value } - it { is_expected.to be_nil } + it { is_expected.to be_nil } + end end end diff --git a/spec/lib/gitlab/content_security_policy/config_loader_spec.rb b/spec/lib/gitlab/content_security_policy/config_loader_spec.rb index f298890623f..b40829d72a0 100644 --- a/spec/lib/gitlab/content_security_policy/config_loader_spec.rb +++ b/spec/lib/gitlab/content_security_policy/config_loader_spec.rb @@ -102,11 +102,7 @@ RSpec.describe Gitlab::ContentSecurityPolicy::ConfigLoader do end describe 'Zuora directives' do - context 'when is Gitlab.com?' do - before do - allow(::Gitlab).to receive(:com?).and_return(true) - end - + context 'when on SaaS', :saas do it 'adds Zuora host to CSP' do expect(directives['frame_src']).to include('https://*.zuora.com/apps/PublicHostedPageLite.do') end diff --git a/spec/lib/gitlab/data_builder/deployment_spec.rb b/spec/lib/gitlab/data_builder/deployment_spec.rb index bf08e782035..82ec3e791a4 100644 --- a/spec/lib/gitlab/data_builder/deployment_spec.rb +++ b/spec/lib/gitlab/data_builder/deployment_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::DataBuilder::Deployment do +RSpec.describe Gitlab::DataBuilder::Deployment, feature_category: :continuous_delivery do describe '.build' do it 'returns the object kind for a deployment' do deployment = build(:deployment, deployable: nil, environment: create(:environment)) @@ -40,6 +40,7 @@ RSpec.describe Gitlab::DataBuilder::Deployment do expect(data[:commit_url]).to eq(expected_commit_url) expect(data[:commit_title]).to eq(commit.title) expect(data[:ref]).to eq(deployment.ref) + expect(data[:environment_tier]).to eq('other') end it 'does not include the deployable URL when there is no deployable' do diff --git a/spec/lib/gitlab/database/async_constraints/migration_helpers_spec.rb b/spec/lib/gitlab/database/async_constraints/migration_helpers_spec.rb new file mode 100644 index 00000000000..4dd510499ab --- /dev/null +++ b/spec/lib/gitlab/database/async_constraints/migration_helpers_spec.rb @@ -0,0 +1,288 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::AsyncConstraints::MigrationHelpers, feature_category: :database do + let(:migration) { Gitlab::Database::Migration[2.1].new } + let(:connection) { ApplicationRecord.connection } + let(:constraint_model) { Gitlab::Database::AsyncConstraints::PostgresAsyncConstraintValidation } + let(:table_name) { '_test_async_fks' } + let(:column_name) { 'parent_id' } + let(:fk_name) { nil } + + context 'with async FK validation on regular tables' do + before do + allow(migration).to receive(:puts) + allow(migration.connection).to receive(:transaction_open?).and_return(false) + + connection.create_table(table_name) do |t| + t.integer column_name + end + + migration.add_concurrent_foreign_key( + table_name, table_name, + column: column_name, validate: false, name: fk_name) + end + + describe '#prepare_async_foreign_key_validation' do + it 'creates the record for the async FK validation' do + expect do + migration.prepare_async_foreign_key_validation(table_name, column_name) + end.to change { constraint_model.where(table_name: table_name).count }.by(1) + + record = constraint_model.find_by(table_name: table_name) + + expect(record.name).to start_with('fk_') + expect(record).to be_foreign_key + end + + context 'when an explicit name is given' do + let(:fk_name) { 'my_fk_name' } + + it 'creates the record with the given name' do + expect do + migration.prepare_async_foreign_key_validation(table_name, name: fk_name) + end.to change { constraint_model.where(name: fk_name).count }.by(1) + + record = constraint_model.find_by(name: fk_name) + + expect(record.table_name).to eq(table_name) + expect(record).to be_foreign_key + end + end + + context 'when the FK does not exist' do + it 'returns an error' do + expect do + migration.prepare_async_foreign_key_validation(table_name, name: 'no_fk') + end.to raise_error RuntimeError, /Could not find foreign key "no_fk" on table "_test_async_fks"/ + end + end + + context 'when the record already exists' do + let(:fk_name) { 'my_fk_name' } + + it 'does attempt to create the record' do + create(:postgres_async_constraint_validation, table_name: table_name, name: fk_name) + + expect do + migration.prepare_async_foreign_key_validation(table_name, name: fk_name) + end.not_to change { constraint_model.where(name: fk_name).count } + end + end + + context 'when the async FK validation table does not exist' do + it 'does not raise an error' do + connection.drop_table(constraint_model.table_name) + + expect(constraint_model).not_to receive(:safe_find_or_create_by!) + + expect { migration.prepare_async_foreign_key_validation(table_name, column_name) }.not_to raise_error + end + end + end + + describe '#unprepare_async_foreign_key_validation' do + context 'with foreign keys' do + before do + migration.prepare_async_foreign_key_validation(table_name, column_name, name: fk_name) + end + + it 'destroys the record' do + expect do + migration.unprepare_async_foreign_key_validation(table_name, column_name) + end.to change { constraint_model.where(table_name: table_name).count }.by(-1) + end + + context 'when an explicit name is given' do + let(:fk_name) { 'my_test_async_fk' } + + it 'destroys the record' do + expect do + migration.unprepare_async_foreign_key_validation(table_name, name: fk_name) + end.to change { constraint_model.where(name: fk_name).count }.by(-1) + end + end + + context 'when the async fk validation table does not exist' do + it 'does not raise an error' do + connection.drop_table(constraint_model.table_name) + + expect(constraint_model).not_to receive(:find_by) + + expect { migration.unprepare_async_foreign_key_validation(table_name, column_name) }.not_to raise_error + end + end + end + + context 'with other types of constraints' do + let(:name) { 'my_test_async_constraint' } + let(:constraint) { create(:postgres_async_constraint_validation, table_name: table_name, name: name) } + + it 'does not destroy the record' do + constraint.update_column(:constraint_type, 99) + + expect do + migration.unprepare_async_foreign_key_validation(table_name, name: name) + end.not_to change { constraint_model.where(name: name).count } + + expect(constraint).to be_present + end + end + end + end + + context 'with partitioned tables' do + let(:partition_schema) { 'gitlab_partitions_dynamic' } + let(:partition1_name) { "#{partition_schema}.#{table_name}_202001" } + let(:partition2_name) { "#{partition_schema}.#{table_name}_202002" } + let(:fk_name) { 'my_partitioned_fk_name' } + + before do + connection.execute(<<~SQL) + CREATE TABLE #{table_name} ( + id serial NOT NULL, + #{column_name} int NOT NULL, + created_at timestamptz NOT NULL, + PRIMARY KEY (id, created_at) + ) PARTITION BY RANGE (created_at); + + CREATE TABLE #{partition1_name} PARTITION OF #{table_name} + FOR VALUES FROM ('2020-01-01') TO ('2020-02-01'); + + CREATE TABLE #{partition2_name} PARTITION OF #{table_name} + FOR VALUES FROM ('2020-02-01') TO ('2020-03-01'); + SQL + end + + describe '#prepare_partitioned_async_foreign_key_validation' do + it 'delegates to prepare_async_foreign_key_validation for each partition' do + expect(migration) + .to receive(:prepare_async_foreign_key_validation) + .with(partition1_name, column_name, name: fk_name) + + expect(migration) + .to receive(:prepare_async_foreign_key_validation) + .with(partition2_name, column_name, name: fk_name) + + migration.prepare_partitioned_async_foreign_key_validation(table_name, column_name, name: fk_name) + end + end + + describe '#unprepare_partitioned_async_foreign_key_validation' do + it 'delegates to unprepare_async_foreign_key_validation for each partition' do + expect(migration) + .to receive(:unprepare_async_foreign_key_validation) + .with(partition1_name, column_name, name: fk_name) + + expect(migration) + .to receive(:unprepare_async_foreign_key_validation) + .with(partition2_name, column_name, name: fk_name) + + migration.unprepare_partitioned_async_foreign_key_validation(table_name, column_name, name: fk_name) + end + end + end + + context 'with async check constraint validations' do + let(:table_name) { '_test_async_check_constraints' } + let(:check_name) { 'partitioning_constraint' } + + before do + allow(migration).to receive(:puts) + allow(migration.connection).to receive(:transaction_open?).and_return(false) + + connection.create_table(table_name) do |t| + t.integer column_name + end + + migration.add_check_constraint( + table_name, "#{column_name} = 1", + check_name, validate: false) + end + + describe '#prepare_async_check_constraint_validation' do + it 'creates the record for async validation' do + expect do + migration.prepare_async_check_constraint_validation(table_name, name: check_name) + end.to change { constraint_model.where(name: check_name).count }.by(1) + + record = constraint_model.find_by(name: check_name) + + expect(record.table_name).to eq(table_name) + expect(record).to be_check_constraint + end + + context 'when the check constraint does not exist' do + it 'returns an error' do + expect do + migration.prepare_async_check_constraint_validation(table_name, name: 'missing') + end.to raise_error RuntimeError, /Could not find check constraint "missing" on table "#{table_name}"/ + end + end + + context 'when the record already exists' do + it 'does attempt to create the record' do + create(:postgres_async_constraint_validation, + table_name: table_name, + name: check_name, + constraint_type: :check_constraint) + + expect do + migration.prepare_async_check_constraint_validation(table_name, name: check_name) + end.not_to change { constraint_model.where(name: check_name).count } + end + end + + context 'when the async validation table does not exist' do + it 'does not raise an error' do + connection.drop_table(constraint_model.table_name) + + expect(constraint_model).not_to receive(:safe_find_or_create_by!) + + expect { migration.prepare_async_check_constraint_validation(table_name, name: check_name) } + .not_to raise_error + end + end + end + + describe '#unprepare_async_check_constraint_validation' do + context 'with check constraints' do + before do + migration.prepare_async_check_constraint_validation(table_name, name: check_name) + end + + it 'destroys the record' do + expect do + migration.unprepare_async_check_constraint_validation(table_name, name: check_name) + end.to change { constraint_model.where(name: check_name).count }.by(-1) + end + + context 'when the async validation table does not exist' do + it 'does not raise an error' do + connection.drop_table(constraint_model.table_name) + + expect(constraint_model).not_to receive(:find_by) + + expect { migration.unprepare_async_check_constraint_validation(table_name, name: check_name) } + .not_to raise_error + end + end + end + + context 'with other types of constraints' do + let(:constraint) { create(:postgres_async_constraint_validation, table_name: table_name, name: check_name) } + + it 'does not destroy the record' do + constraint.update_column(:constraint_type, 99) + + expect do + migration.unprepare_async_check_constraint_validation(table_name, name: check_name) + end.not_to change { constraint_model.where(name: check_name).count } + + expect(constraint).to be_present + end + end + end + end +end diff --git a/spec/lib/gitlab/database/async_constraints/postgres_async_constraint_validation_spec.rb b/spec/lib/gitlab/database/async_constraints/postgres_async_constraint_validation_spec.rb new file mode 100644 index 00000000000..52fbf6d2f9b --- /dev/null +++ b/spec/lib/gitlab/database/async_constraints/postgres_async_constraint_validation_spec.rb @@ -0,0 +1,109 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::AsyncConstraints::PostgresAsyncConstraintValidation, type: :model, + feature_category: :database do + it { is_expected.to be_a Gitlab::Database::SharedModel } + + describe 'validations' do + let_it_be(:constraint_validation) { create(:postgres_async_constraint_validation) } + let(:identifier_limit) { described_class::MAX_IDENTIFIER_LENGTH } + let(:last_error_limit) { described_class::MAX_LAST_ERROR_LENGTH } + + subject { constraint_validation } + + it { is_expected.to validate_presence_of(:name) } + it { is_expected.to validate_uniqueness_of(:name).scoped_to(:table_name) } + it { is_expected.to validate_length_of(:name).is_at_most(identifier_limit) } + it { is_expected.to validate_presence_of(:table_name) } + it { is_expected.to validate_length_of(:table_name).is_at_most(identifier_limit) } + it { is_expected.to validate_length_of(:last_error).is_at_most(last_error_limit) } + end + + describe 'scopes' do + let!(:failed_validation) { create(:postgres_async_constraint_validation, attempts: 1) } + let!(:new_validation) { create(:postgres_async_constraint_validation) } + + describe '.ordered' do + subject { described_class.ordered } + + it { is_expected.to eq([new_validation, failed_validation]) } + end + + describe '.foreign_key_type' do + before do + new_validation.update_column(:constraint_type, 99) + end + + subject { described_class.foreign_key_type } + + it { is_expected.to eq([failed_validation]) } + + it 'does not apply the filter if the column is not present' do + expect(described_class) + .to receive(:constraint_type_exists?) + .and_return(false) + + is_expected.to match_array([failed_validation, new_validation]) + end + end + + describe '.check_constraint_type' do + before do + new_validation.update!(constraint_type: :check_constraint) + end + + subject { described_class.check_constraint_type } + + it { is_expected.to eq([new_validation]) } + end + end + + describe '.table_available?' do + subject { described_class.table_available? } + + it { is_expected.to be_truthy } + + context 'when the table does not exist' do + before do + described_class + .connection + .drop_table(described_class.table_name) + end + + it { is_expected.to be_falsy } + end + end + + describe '.constraint_type_exists?' do + it { expect(described_class.constraint_type_exists?).to be_truthy } + + it 'always asks the database' do + control = ActiveRecord::QueryRecorder.new(skip_schema_queries: false) do + described_class.constraint_type_exists? + end + + expect(control.count).to be >= 1 + expect { described_class.constraint_type_exists? }.to issue_same_number_of_queries_as(control) + end + end + + describe '#handle_exception!' do + let_it_be_with_reload(:constraint_validation) { create(:postgres_async_constraint_validation) } + + let(:error) { instance_double(StandardError, message: 'Oups', backtrace: %w[this that]) } + + subject { constraint_validation.handle_exception!(error) } + + it 'increases the attempts number' do + expect { subject }.to change { constraint_validation.reload.attempts }.by(1) + end + + it 'saves error details' do + subject + + expect(constraint_validation.reload.last_error).to eq("Oups\nthis\nthat") + end + end +end diff --git a/spec/lib/gitlab/database/async_constraints/validators/check_constraint_spec.rb b/spec/lib/gitlab/database/async_constraints/validators/check_constraint_spec.rb new file mode 100644 index 00000000000..7622b39feb1 --- /dev/null +++ b/spec/lib/gitlab/database/async_constraints/validators/check_constraint_spec.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::AsyncConstraints::Validators::CheckConstraint, feature_category: :database do + it_behaves_like 'async constraints validation' do + let(:constraint_type) { :check_constraint } + + before do + connection.create_table(table_name) do |t| + t.integer :parent_id + end + + connection.execute(<<~SQL.squish) + ALTER TABLE #{table_name} ADD CONSTRAINT #{constraint_name} + CHECK ( parent_id = 101 ) NOT VALID; + SQL + end + end +end diff --git a/spec/lib/gitlab/database/async_constraints/validators/foreign_key_spec.rb b/spec/lib/gitlab/database/async_constraints/validators/foreign_key_spec.rb new file mode 100644 index 00000000000..0e345e0e9ae --- /dev/null +++ b/spec/lib/gitlab/database/async_constraints/validators/foreign_key_spec.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::AsyncConstraints::Validators::ForeignKey, feature_category: :database do + it_behaves_like 'async constraints validation' do + let(:constraint_type) { :foreign_key } + + before do + connection.create_table(table_name) do |t| + t.references :parent, foreign_key: { to_table: table_name, validate: false, name: constraint_name } + end + end + + context 'with fully qualified table names' do + let(:validation) do + create(:postgres_async_constraint_validation, + table_name: "public.#{table_name}", + name: constraint_name, + constraint_type: constraint_type + ) + end + + it 'validates the constraint' do + allow(connection).to receive(:execute).and_call_original + + expect(connection).to receive(:execute) + .with(/ALTER TABLE "public"."#{table_name}" VALIDATE CONSTRAINT "#{constraint_name}";/) + .ordered.and_call_original + + subject.perform + end + end + end +end diff --git a/spec/lib/gitlab/database/async_constraints/validators_spec.rb b/spec/lib/gitlab/database/async_constraints/validators_spec.rb new file mode 100644 index 00000000000..e903b79dd1b --- /dev/null +++ b/spec/lib/gitlab/database/async_constraints/validators_spec.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::AsyncConstraints::Validators, feature_category: :database do + describe '.for' do + subject { described_class.for(record) } + + context 'with foreign keys validations' do + let(:record) { build(:postgres_async_constraint_validation, :foreign_key) } + + it { is_expected.to be_a(described_class::ForeignKey) } + end + + context 'with check constraint validations' do + let(:record) { build(:postgres_async_constraint_validation, :check_constraint) } + + it { is_expected.to be_a(described_class::CheckConstraint) } + end + end +end diff --git a/spec/lib/gitlab/database/async_constraints_spec.rb b/spec/lib/gitlab/database/async_constraints_spec.rb new file mode 100644 index 00000000000..e5cf782485f --- /dev/null +++ b/spec/lib/gitlab/database/async_constraints_spec.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::AsyncConstraints, feature_category: :database do + describe '.validate_pending_entries!' do + subject { described_class.validate_pending_entries! } + + let!(:fk_validation) do + create(:postgres_async_constraint_validation, :foreign_key, attempts: 2) + end + + let(:check_validation) do + create(:postgres_async_constraint_validation, :check_constraint, attempts: 1) + end + + it 'executes pending validations' do + expect_next_instance_of(described_class::Validators::ForeignKey, fk_validation) do |validator| + expect(validator).to receive(:perform) + end + + expect_next_instance_of(described_class::Validators::CheckConstraint, check_validation) do |validator| + expect(validator).to receive(:perform) + end + + subject + end + end +end diff --git a/spec/lib/gitlab/database/async_foreign_keys/foreign_key_validator_spec.rb b/spec/lib/gitlab/database/async_foreign_keys/foreign_key_validator_spec.rb deleted file mode 100644 index 90137e259f5..00000000000 --- a/spec/lib/gitlab/database/async_foreign_keys/foreign_key_validator_spec.rb +++ /dev/null @@ -1,152 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Database::AsyncForeignKeys::ForeignKeyValidator, feature_category: :database do - include ExclusiveLeaseHelpers - - describe '#perform' do - let!(:lease) { stub_exclusive_lease(lease_key, :uuid, timeout: lease_timeout) } - let(:lease_key) { "gitlab/database/asyncddl/actions/#{Gitlab::Database::PRIMARY_DATABASE_NAME}" } - let(:lease_timeout) { described_class::TIMEOUT_PER_ACTION } - - let(:fk_model) { Gitlab::Database::AsyncForeignKeys::PostgresAsyncForeignKeyValidation } - let(:table_name) { '_test_async_fks' } - let(:fk_name) { 'fk_parent_id' } - let(:validation) { create(:postgres_async_foreign_key_validation, table_name: table_name, name: fk_name) } - let(:connection) { validation.connection } - - subject { described_class.new(validation) } - - before do - connection.create_table(table_name) do |t| - t.references :parent, foreign_key: { to_table: table_name, validate: false, name: fk_name } - end - end - - it 'validates the FK while controlling statement timeout' do - allow(connection).to receive(:execute).and_call_original - expect(connection).to receive(:execute) - .with("SET statement_timeout TO '43200s'").ordered.and_call_original - expect(connection).to receive(:execute) - .with('ALTER TABLE "_test_async_fks" VALIDATE CONSTRAINT "fk_parent_id";').ordered.and_call_original - expect(connection).to receive(:execute) - .with("RESET statement_timeout").ordered.and_call_original - - subject.perform - end - - context 'with fully qualified table names' do - let(:validation) do - create(:postgres_async_foreign_key_validation, - table_name: "public.#{table_name}", - name: fk_name - ) - end - - it 'validates the FK' do - allow(connection).to receive(:execute).and_call_original - - expect(connection).to receive(:execute) - .with('ALTER TABLE "public"."_test_async_fks" VALIDATE CONSTRAINT "fk_parent_id";').ordered.and_call_original - - subject.perform - end - end - - it 'removes the FK validation record from table' do - expect(validation).to receive(:destroy!).and_call_original - - expect { subject.perform }.to change { fk_model.count }.by(-1) - end - - it 'skips logic if not able to acquire exclusive lease' do - expect(lease).to receive(:try_obtain).ordered.and_return(false) - expect(connection).not_to receive(:execute).with(/ALTER TABLE/) - expect(validation).not_to receive(:destroy!) - - expect { subject.perform }.not_to change { fk_model.count } - end - - it 'logs messages around execution' do - allow(Gitlab::AppLogger).to receive(:info).and_call_original - - subject.perform - - expect(Gitlab::AppLogger) - .to have_received(:info) - .with(a_hash_including(message: 'Starting to validate foreign key')) - - expect(Gitlab::AppLogger) - .to have_received(:info) - .with(a_hash_including(message: 'Finished validating foreign key')) - end - - context 'when the FK does not exist' do - before do - connection.create_table(table_name, force: true) - end - - it 'skips validation and removes the record' do - expect(connection).not_to receive(:execute).with(/ALTER TABLE/) - - expect { subject.perform }.to change { fk_model.count }.by(-1) - end - - it 'logs an appropriate message' do - expected_message = "Skipping #{fk_name} validation since it does not exist. The queuing entry will be deleted" - - allow(Gitlab::AppLogger).to receive(:info).and_call_original - - subject.perform - - expect(Gitlab::AppLogger) - .to have_received(:info) - .with(a_hash_including(message: expected_message)) - end - end - - context 'with error handling' do - before do - allow(connection).to receive(:execute).and_call_original - - allow(connection).to receive(:execute) - .with('ALTER TABLE "_test_async_fks" VALIDATE CONSTRAINT "fk_parent_id";') - .and_raise(ActiveRecord::StatementInvalid) - end - - context 'on production' do - before do - allow(Gitlab::ErrorTracking).to receive(:should_raise_for_dev?).and_return(false) - end - - it 'increases execution attempts' do - expect { subject.perform }.to change { validation.attempts }.by(1) - - expect(validation.last_error).to be_present - expect(validation).not_to be_destroyed - end - - it 'logs an error message including the fk_name' do - expect(Gitlab::AppLogger) - .to receive(:error) - .with(a_hash_including(:message, :fk_name)) - .and_call_original - - subject.perform - end - end - - context 'on development' do - it 'also raises errors' do - expect { subject.perform } - .to raise_error(ActiveRecord::StatementInvalid) - .and change { validation.attempts }.by(1) - - expect(validation.last_error).to be_present - expect(validation).not_to be_destroyed - end - end - end - end -end diff --git a/spec/lib/gitlab/database/async_foreign_keys/migration_helpers_spec.rb b/spec/lib/gitlab/database/async_foreign_keys/migration_helpers_spec.rb deleted file mode 100644 index 0bd0e8045ff..00000000000 --- a/spec/lib/gitlab/database/async_foreign_keys/migration_helpers_spec.rb +++ /dev/null @@ -1,167 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Database::AsyncForeignKeys::MigrationHelpers, feature_category: :database do - let(:migration) { Gitlab::Database::Migration[2.1].new } - let(:connection) { ApplicationRecord.connection } - let(:fk_model) { Gitlab::Database::AsyncForeignKeys::PostgresAsyncForeignKeyValidation } - let(:table_name) { '_test_async_fks' } - let(:column_name) { 'parent_id' } - let(:fk_name) { nil } - - context 'with regular tables' do - before do - allow(migration).to receive(:puts) - allow(migration.connection).to receive(:transaction_open?).and_return(false) - - connection.create_table(table_name) do |t| - t.integer column_name - end - - migration.add_concurrent_foreign_key( - table_name, table_name, - column: column_name, validate: false, name: fk_name) - end - - describe '#prepare_async_foreign_key_validation' do - it 'creates the record for the async FK validation' do - expect do - migration.prepare_async_foreign_key_validation(table_name, column_name) - end.to change { fk_model.where(table_name: table_name).count }.by(1) - - record = fk_model.find_by(table_name: table_name) - - expect(record.name).to start_with('fk_') - end - - context 'when an explicit name is given' do - let(:fk_name) { 'my_fk_name' } - - it 'creates the record with the given name' do - expect do - migration.prepare_async_foreign_key_validation(table_name, name: fk_name) - end.to change { fk_model.where(name: fk_name).count }.by(1) - - record = fk_model.find_by(name: fk_name) - - expect(record.table_name).to eq(table_name) - end - end - - context 'when the FK does not exist' do - it 'returns an error' do - expect do - migration.prepare_async_foreign_key_validation(table_name, name: 'no_fk') - end.to raise_error RuntimeError, /Could not find foreign key "no_fk" on table "_test_async_fks"/ - end - end - - context 'when the record already exists' do - let(:fk_name) { 'my_fk_name' } - - it 'does attempt to create the record' do - create(:postgres_async_foreign_key_validation, table_name: table_name, name: fk_name) - - expect do - migration.prepare_async_foreign_key_validation(table_name, name: fk_name) - end.not_to change { fk_model.where(name: fk_name).count } - end - end - - context 'when the async FK validation table does not exist' do - it 'does not raise an error' do - connection.drop_table(:postgres_async_foreign_key_validations) - - expect(fk_model).not_to receive(:safe_find_or_create_by!) - - expect { migration.prepare_async_foreign_key_validation(table_name, column_name) }.not_to raise_error - end - end - end - - describe '#unprepare_async_foreign_key_validation' do - before do - migration.prepare_async_foreign_key_validation(table_name, column_name, name: fk_name) - end - - it 'destroys the record' do - expect do - migration.unprepare_async_foreign_key_validation(table_name, column_name) - end.to change { fk_model.where(table_name: table_name).count }.by(-1) - end - - context 'when an explicit name is given' do - let(:fk_name) { 'my_test_async_fk' } - - it 'destroys the record' do - expect do - migration.unprepare_async_foreign_key_validation(table_name, name: fk_name) - end.to change { fk_model.where(name: fk_name).count }.by(-1) - end - end - - context 'when the async fk validation table does not exist' do - it 'does not raise an error' do - connection.drop_table(:postgres_async_foreign_key_validations) - - expect(fk_model).not_to receive(:find_by) - - expect { migration.unprepare_async_foreign_key_validation(table_name, column_name) }.not_to raise_error - end - end - end - end - - context 'with partitioned tables' do - let(:partition_schema) { 'gitlab_partitions_dynamic' } - let(:partition1_name) { "#{partition_schema}.#{table_name}_202001" } - let(:partition2_name) { "#{partition_schema}.#{table_name}_202002" } - let(:fk_name) { 'my_partitioned_fk_name' } - - before do - connection.execute(<<~SQL) - CREATE TABLE #{table_name} ( - id serial NOT NULL, - #{column_name} int NOT NULL, - created_at timestamptz NOT NULL, - PRIMARY KEY (id, created_at) - ) PARTITION BY RANGE (created_at); - - CREATE TABLE #{partition1_name} PARTITION OF #{table_name} - FOR VALUES FROM ('2020-01-01') TO ('2020-02-01'); - - CREATE TABLE #{partition2_name} PARTITION OF #{table_name} - FOR VALUES FROM ('2020-02-01') TO ('2020-03-01'); - SQL - end - - describe '#prepare_partitioned_async_foreign_key_validation' do - it 'delegates to prepare_async_foreign_key_validation for each partition' do - expect(migration) - .to receive(:prepare_async_foreign_key_validation) - .with(partition1_name, column_name, name: fk_name) - - expect(migration) - .to receive(:prepare_async_foreign_key_validation) - .with(partition2_name, column_name, name: fk_name) - - migration.prepare_partitioned_async_foreign_key_validation(table_name, column_name, name: fk_name) - end - end - - describe '#unprepare_partitioned_async_foreign_key_validation' do - it 'delegates to unprepare_async_foreign_key_validation for each partition' do - expect(migration) - .to receive(:unprepare_async_foreign_key_validation) - .with(partition1_name, column_name, name: fk_name) - - expect(migration) - .to receive(:unprepare_async_foreign_key_validation) - .with(partition2_name, column_name, name: fk_name) - - migration.unprepare_partitioned_async_foreign_key_validation(table_name, column_name, name: fk_name) - end - end - end -end diff --git a/spec/lib/gitlab/database/async_foreign_keys/postgres_async_foreign_key_validation_spec.rb b/spec/lib/gitlab/database/async_foreign_keys/postgres_async_foreign_key_validation_spec.rb deleted file mode 100644 index ba201d93f52..00000000000 --- a/spec/lib/gitlab/database/async_foreign_keys/postgres_async_foreign_key_validation_spec.rb +++ /dev/null @@ -1,52 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Database::AsyncForeignKeys::PostgresAsyncForeignKeyValidation, type: :model, - feature_category: :database do - it { is_expected.to be_a Gitlab::Database::SharedModel } - - describe 'validations' do - let_it_be(:fk_validation) { create(:postgres_async_foreign_key_validation) } - let(:identifier_limit) { described_class::MAX_IDENTIFIER_LENGTH } - let(:last_error_limit) { described_class::MAX_LAST_ERROR_LENGTH } - - subject { fk_validation } - - it { is_expected.to validate_presence_of(:name) } - it { is_expected.to validate_uniqueness_of(:name) } - it { is_expected.to validate_length_of(:name).is_at_most(identifier_limit) } - it { is_expected.to validate_presence_of(:table_name) } - it { is_expected.to validate_length_of(:table_name).is_at_most(identifier_limit) } - it { is_expected.to validate_length_of(:last_error).is_at_most(last_error_limit) } - end - - describe 'scopes' do - let!(:failed_validation) { create(:postgres_async_foreign_key_validation, attempts: 1) } - let!(:new_validation) { create(:postgres_async_foreign_key_validation) } - - describe '.ordered' do - subject { described_class.ordered } - - it { is_expected.to eq([new_validation, failed_validation]) } - end - end - - describe '#handle_exception!' do - let_it_be_with_reload(:fk_validation) { create(:postgres_async_foreign_key_validation) } - - let(:error) { instance_double(StandardError, message: 'Oups', backtrace: %w[this that]) } - - subject { fk_validation.handle_exception!(error) } - - it 'increases the attempts number' do - expect { subject }.to change { fk_validation.reload.attempts }.by(1) - end - - it 'saves error details' do - subject - - expect(fk_validation.reload.last_error).to eq("Oups\nthis\nthat") - end - end -end diff --git a/spec/lib/gitlab/database/async_foreign_keys_spec.rb b/spec/lib/gitlab/database/async_foreign_keys_spec.rb deleted file mode 100644 index f15eb364929..00000000000 --- a/spec/lib/gitlab/database/async_foreign_keys_spec.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Database::AsyncForeignKeys, feature_category: :database do - describe '.validate_pending_entries!' do - subject { described_class.validate_pending_entries! } - - before do - create_list(:postgres_async_foreign_key_validation, 3) - end - - it 'takes 2 pending FK validations and executes them' do - validations = described_class::PostgresAsyncForeignKeyValidation.ordered.limit(2).to_a - - expect_next_instances_of(described_class::ForeignKeyValidator, 2, validations) do |validator| - expect(validator).to receive(:perform) - end - - subject - end - end -end diff --git a/spec/lib/gitlab/database/async_indexes/migration_helpers_spec.rb b/spec/lib/gitlab/database/async_indexes/migration_helpers_spec.rb index 7c5c368fcb5..b2ba1a60fbb 100644 --- a/spec/lib/gitlab/database/async_indexes/migration_helpers_spec.rb +++ b/spec/lib/gitlab/database/async_indexes/migration_helpers_spec.rb @@ -143,6 +143,92 @@ RSpec.describe Gitlab::Database::AsyncIndexes::MigrationHelpers, feature_categor end end + describe '#prepare_async_index_from_sql' do + let(:index_definition) { "CREATE INDEX CONCURRENTLY #{index_name} ON #{table_name} USING btree(id)" } + + subject(:prepare_async_index_from_sql) do + migration.prepare_async_index_from_sql(index_definition) + end + + before do + connection.create_table(table_name) + + allow(Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas).to receive(:require_ddl_mode!).and_call_original + end + + it 'requires ddl mode' do + prepare_async_index_from_sql + + expect(Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas).to have_received(:require_ddl_mode!) + end + + context 'when the given index is invalid' do + let(:index_definition) { "SELECT FROM users" } + + it 'raises a RuntimeError' do + expect { prepare_async_index_from_sql }.to raise_error(RuntimeError, 'Index statement not found!') + end + end + + context 'when the given index is valid' do + context 'when the index algorithm is not concurrent' do + let(:index_definition) { "CREATE INDEX #{index_name} ON #{table_name} USING btree(id)" } + + it 'raises a RuntimeError' do + expect { prepare_async_index_from_sql }.to raise_error(RuntimeError, 'Index must be created concurrently!') + end + end + + context 'when the index algorithm is concurrent' do + context 'when the statement tries to create an index for non-existing table' do + let(:index_definition) { "CREATE INDEX CONCURRENTLY #{index_name} ON foo_table USING btree(id)" } + + it 'raises a RuntimeError' do + expect { prepare_async_index_from_sql }.to raise_error(RuntimeError, 'Table does not exist!') + end + end + + context 'when the statement tries to create an index for an existing table' do + context 'when the async index creation is not available' do + before do + connection.drop_table(:postgres_async_indexes) + end + + it 'does not raise an error' do + expect { prepare_async_index_from_sql }.not_to raise_error + end + end + + context 'when the async index creation is available' do + context 'when there is already an index with the given name' do + before do + connection.add_index(table_name, 'id', name: index_name) + end + + it 'does not create the async index record' do + expect { prepare_async_index_from_sql }.not_to change { index_model.where(name: index_name).count } + end + end + + context 'when there is no index with the given name' do + let(:async_index) { index_model.find_by(name: index_name) } + + it 'creates the async index record' do + expect { prepare_async_index_from_sql }.to change { index_model.where(name: index_name).count }.by(1) + end + + it 'sets the async index attributes correctly' do + prepare_async_index_from_sql + + expect(async_index).to have_attributes(table_name: table_name, definition: index_definition) + end + end + end + end + end + end + end + describe '#prepare_async_index_removal' do before do connection.create_table(table_name) diff --git a/spec/lib/gitlab/database/background_migration/batch_optimizer_spec.rb b/spec/lib/gitlab/database/background_migration/batch_optimizer_spec.rb index c367f4a4493..fb9b16d46d6 100644 --- a/spec/lib/gitlab/database/background_migration/batch_optimizer_spec.rb +++ b/spec/lib/gitlab/database/background_migration/batch_optimizer_spec.rb @@ -113,5 +113,14 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchOptimizer do expect { subject }.to change { migration.reload.batch_size }.to(1_000) end end + + context 'when migration max_batch_size is less than MIN_BATCH_SIZE' do + let(:migration_params) { { max_batch_size: 900 } } + + it 'does not raise an error' do + mock_efficiency(0.7) + expect { subject }.not_to raise_error + end + end end end diff --git a/spec/lib/gitlab/database/background_migration/batched_job_spec.rb b/spec/lib/gitlab/database/background_migration/batched_job_spec.rb index cc9f3d5b7f1..d9b81a2be30 100644 --- a/spec/lib/gitlab/database/background_migration/batched_job_spec.rb +++ b/spec/lib/gitlab/database/background_migration/batched_job_spec.rb @@ -184,6 +184,35 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedJob, type: :model d expect(transition_log.exception_message).to eq('RuntimeError') end end + + context 'when job fails during sub batch processing' do + let(:args) { { error: ActiveRecord::StatementTimeout.new, from_sub_batch: true } } + let(:attempts) { 0 } + let(:failure) { job.failure!(**args) } + let(:job) do + create(:batched_background_migration_job, :running, batch_size: 20, sub_batch_size: 10, attempts: attempts) + end + + context 'when sub batch size can be reduced in 25%' do + it { expect { failure }.to change { job.sub_batch_size }.to 7 } + end + + context 'when retries exceeds 2 attempts' do + let(:attempts) { 3 } + + before do + allow(job).to receive(:split_and_retry!) + end + + it 'calls split_and_retry! once sub_batch_size cannot be decreased anymore' do + failure + + expect(job).to have_received(:split_and_retry!).once + end + + it { expect { failure }.not_to change { job.sub_batch_size } } + end + end end describe 'scopes' do @@ -271,6 +300,24 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedJob, type: :model d end end + describe '.extract_transition_options' do + let(:perform) { subject.class.extract_transition_options(args) } + + where(:args, :expected_result) do + [ + [[], []], + [[{ error: StandardError }], [StandardError, nil]], + [[{ error: StandardError, from_sub_batch: true }], [StandardError, true]] + ] + end + + with_them do + it 'matches expected keys and result' do + expect(perform).to match_array(expected_result) + end + end + end + describe '#can_split?' do subject { job.can_split?(exception) } @@ -327,6 +374,34 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedJob, type: :model d end end + describe '#can_reduce_sub_batch_size?' do + let(:attempts) { 0 } + let(:batch_size) { 10 } + let(:sub_batch_size) { 6 } + let(:job) do + create(:batched_background_migration_job, attempts: attempts, + batch_size: batch_size, sub_batch_size: sub_batch_size) + end + + context 'when the number of attempts is lower than the limit and batch size are within boundaries' do + let(:attempts) { 1 } + + it { expect(job.can_reduce_sub_batch_size?).to be(true) } + end + + context 'when the number of attempts is lower than the limit and batch size are outside boundaries' do + let(:batch_size) { 1 } + + it { expect(job.can_reduce_sub_batch_size?).to be(false) } + end + + context 'when the number of attempts is greater than the limit and batch size are within boundaries' do + let(:attempts) { 3 } + + it { expect(job.can_reduce_sub_batch_size?).to be(false) } + end + end + describe '#time_efficiency' do subject { job.time_efficiency } @@ -465,4 +540,80 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedJob, type: :model d end end end + + describe '#reduce_sub_batch_size!' do + let(:migration_batch_size) { 20 } + let(:migration_sub_batch_size) { 10 } + let(:job_batch_size) { 20 } + let(:job_sub_batch_size) { 10 } + let(:status) { :failed } + + let(:migration) do + create(:batched_background_migration, :active, batch_size: migration_batch_size, + sub_batch_size: migration_sub_batch_size) + end + + let(:job) do + create(:batched_background_migration_job, status, sub_batch_size: job_sub_batch_size, + batch_size: job_batch_size, batched_migration: migration) + end + + context 'when the job sub batch size can be reduced' do + let(:expected_sub_batch_size) { 7 } + + it 'reduces sub batch size in 25%' do + expect { job.reduce_sub_batch_size! }.to change { job.sub_batch_size }.to(expected_sub_batch_size) + end + + it 'log the changes' do + expect(Gitlab::AppLogger).to receive(:warn).with( + message: 'Sub batch size reduced due to timeout', + batched_job_id: job.id, + sub_batch_size: job_sub_batch_size, + reduced_sub_batch_size: expected_sub_batch_size, + attempts: job.attempts, + batched_migration_id: migration.id, + job_class_name: job.migration_job_class_name, + job_arguments: job.migration_job_arguments + ) + + job.reduce_sub_batch_size! + end + end + + context 'when reduced sub_batch_size is greater than sub_batch' do + let(:job_batch_size) { 5 } + + it "doesn't allow sub_batch_size to greater than sub_batch" do + expect { job.reduce_sub_batch_size! }.to change { job.sub_batch_size }.to 5 + end + end + + context 'when sub_batch_size is already 1' do + let(:job_sub_batch_size) { 1 } + + it "updates sub_batch_size to it's minimum value" do + expect { job.reduce_sub_batch_size! }.not_to change { job.sub_batch_size } + end + end + + context 'when job has not failed' do + let(:status) { :succeeded } + let(:error) { Gitlab::Database::BackgroundMigration::ReduceSubBatchSizeError } + + it 'raises an exception' do + expect { job.reduce_sub_batch_size! }.to raise_error(error) + end + end + + context 'when the amount to be reduced exceeds the threshold' do + let(:migration_batch_size) { 150 } + let(:migration_sub_batch_size) { 100 } + let(:job_sub_batch_size) { 30 } + + it 'prevents sub batch size to be reduced' do + expect { job.reduce_sub_batch_size! }.not_to change { job.sub_batch_size } + end + end + end end diff --git a/spec/lib/gitlab/database/background_migration/batched_migration_spec.rb b/spec/lib/gitlab/database/background_migration/batched_migration_spec.rb index d132559acea..546f9353808 100644 --- a/spec/lib/gitlab/database/background_migration/batched_migration_spec.rb +++ b/spec/lib/gitlab/database/background_migration/batched_migration_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigration, type: :model do +RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigration, type: :model, feature_category: :database do it_behaves_like 'having unique enum values' it { is_expected.to be_a Gitlab::Database::SharedModel } @@ -328,6 +328,17 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigration, type: :m end end + describe '.finalizing' do + let!(:migration1) { create(:batched_background_migration, :active) } + let!(:migration2) { create(:batched_background_migration, :paused) } + let!(:migration3) { create(:batched_background_migration, :finalizing) } + let!(:migration4) { create(:batched_background_migration, :finished) } + + it 'returns only finalizing migrations' do + expect(described_class.finalizing).to contain_exactly(migration3) + end + end + describe '.successful_rows_counts' do let!(:migration1) { create(:batched_background_migration) } let!(:migration2) { create(:batched_background_migration) } diff --git a/spec/lib/gitlab/database/background_migration/batched_migration_wrapper_spec.rb b/spec/lib/gitlab/database/background_migration/batched_migration_wrapper_spec.rb index f3a292abbae..8d74d16f4e5 100644 --- a/spec/lib/gitlab/database/background_migration/batched_migration_wrapper_spec.rb +++ b/spec/lib/gitlab/database/background_migration/batched_migration_wrapper_spec.rb @@ -8,6 +8,7 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper, ' let(:connection) { Gitlab::Database.database_base_models[:main].connection } let(:metrics_tracker) { instance_double('::Gitlab::Database::BackgroundMigration::PrometheusMetrics', track: nil) } let(:job_class) { Class.new(Gitlab::BackgroundMigration::BatchedMigrationJob) } + let(:sub_batch_exception) { Gitlab::Database::BackgroundMigration::SubBatchTimeoutError } let_it_be(:pause_ms) { 250 } let_it_be(:active_migration) { create(:batched_background_migration, :active, job_arguments: [:id, :other_id]) } @@ -39,7 +40,8 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper, ' sub_batch_size: 1, pause_ms: pause_ms, job_arguments: active_migration.job_arguments, - connection: connection) + connection: connection, + sub_batch_exception: sub_batch_exception) .and_return(job_instance) expect(job_instance).to receive(:perform).with(no_args) @@ -119,12 +121,14 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper, ' end context 'when the migration job raises an error' do - shared_examples 'an error is raised' do |error_class| + shared_examples 'an error is raised' do |error_class, cause| + let(:expected_to_raise) { cause || error_class } + it 'marks the tracking record as failed' do expect(job_instance).to receive(:perform).with(no_args).and_raise(error_class) freeze_time do - expect { perform }.to raise_error(error_class) + expect { perform }.to raise_error(expected_to_raise) reloaded_job_record = job_record.reload @@ -137,13 +141,16 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper, ' expect(job_instance).to receive(:perform).with(no_args).and_raise(error_class) expect(metrics_tracker).to receive(:track).with(job_record) - expect { perform }.to raise_error(error_class) + expect { perform }.to raise_error(expected_to_raise) end end it_behaves_like 'an error is raised', RuntimeError.new('Something broke!') it_behaves_like 'an error is raised', SignalException.new('SIGTERM') it_behaves_like 'an error is raised', ActiveRecord::StatementTimeout.new('Timeout!') + + error = StandardError.new + it_behaves_like('an error is raised', Gitlab::Database::BackgroundMigration::SubBatchTimeoutError.new(error), error) end context 'when the batched background migration does not inherit from BatchedMigrationJob' do diff --git a/spec/lib/gitlab/database/background_migration/health_status/indicators/patroni_apdex_spec.rb b/spec/lib/gitlab/database/background_migration/health_status/indicators/patroni_apdex_spec.rb new file mode 100644 index 00000000000..d3102a105ea --- /dev/null +++ b/spec/lib/gitlab/database/background_migration/health_status/indicators/patroni_apdex_spec.rb @@ -0,0 +1,148 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::BackgroundMigration::HealthStatus::Indicators::PatroniApdex, :aggregate_failures, feature_category: :database do # rubocop:disable Layout/LineLength + let(:schema) { :main } + let(:connection) { Gitlab::Database.database_base_models[schema].connection } + + around do |example| + Gitlab::Database::SharedModel.using_connection(connection) do + example.run + end + end + + describe '#evaluate' do + let(:prometheus_url) { 'http://thanos:9090' } + let(:prometheus_config) { [prometheus_url, { allow_local_requests: true, verify: true }] } + + let(:prometheus_client) { instance_double(Gitlab::PrometheusClient) } + + let(:context) do + Gitlab::Database::BackgroundMigration::HealthStatus::Context + .new(connection, ['users'], gitlab_schema) + end + + let(:gitlab_schema) { "gitlab_#{schema}" } + let(:client_ready) { true } + let(:database_apdex_sli_query_main) { 'Apdex query for main' } + let(:database_apdex_sli_query_ci) { 'Apdex query for ci' } + let(:database_apdex_slo_main) { 0.99 } + let(:database_apdex_slo_ci) { 0.95 } + let(:database_apdex_settings) do + { + prometheus_api_url: prometheus_url, + apdex_sli_query: { + main: database_apdex_sli_query_main, + ci: database_apdex_sli_query_ci + }, + apdex_slo: { + main: database_apdex_slo_main, + ci: database_apdex_slo_ci + } + } + end + + subject(:evaluate) { described_class.new(context).evaluate } + + before do + stub_application_setting(database_apdex_settings: database_apdex_settings) + + allow(Gitlab::PrometheusClient).to receive(:new).with(*prometheus_config).and_return(prometheus_client) + allow(prometheus_client).to receive(:ready?).and_return(client_ready) + end + + shared_examples 'Patroni Apdex Evaluator' do |schema| + context "with #{schema} schema" do + let(:schema) { schema } + let(:apdex_slo_above_sli) { { main: 0.991, ci: 0.951 } } + let(:apdex_slo_below_sli) { { main: 0.989, ci: 0.949 } } + + it 'returns NoSignal signal in case the feature flag is disabled' do + stub_feature_flags(batched_migrations_health_status_patroni_apdex: false) + + expect(evaluate).to be_a(Gitlab::Database::BackgroundMigration::HealthStatus::Signals::NotAvailable) + expect(evaluate.reason).to include('indicator disabled') + end + + context 'without database_apdex_settings' do + let(:database_apdex_settings) { nil } + + it 'returns Unknown signal' do + expect(evaluate).to be_a(Gitlab::Database::BackgroundMigration::HealthStatus::Signals::Unknown) + expect(evaluate.reason).to include('Patroni Apdex Settings not configured') + end + end + + context 'when Prometheus client is not ready' do + let(:client_ready) { false } + + it 'returns Unknown signal' do + expect(evaluate).to be_a(Gitlab::Database::BackgroundMigration::HealthStatus::Signals::Unknown) + expect(evaluate.reason).to include('Prometheus client is not ready') + end + end + + context 'when apdex SLI query is not configured' do + let(:"database_apdex_sli_query_#{schema}") { nil } + + it 'returns Unknown signal' do + expect(evaluate).to be_a(Gitlab::Database::BackgroundMigration::HealthStatus::Signals::Unknown) + expect(evaluate.reason).to include('Apdex SLI query is not configured') + end + end + + context 'when slo is not configured' do + let(:"database_apdex_slo_#{schema}") { nil } + + it 'returns Unknown signal' do + expect(evaluate).to be_a(Gitlab::Database::BackgroundMigration::HealthStatus::Signals::Unknown) + expect(evaluate.reason).to include('Apdex SLO is not configured') + end + end + + it 'returns Normal signal when Patroni apdex SLI is above SLO' do + expect(prometheus_client).to receive(:query) + .with(send("database_apdex_sli_query_#{schema}")) + .and_return([{ "value" => [1662423310.878, apdex_slo_above_sli[schema]] }]) + expect(evaluate).to be_a(Gitlab::Database::BackgroundMigration::HealthStatus::Signals::Normal) + expect(evaluate.reason).to include('Patroni service apdex is above SLO') + end + + it 'returns Stop signal when Patroni apdex is below SLO' do + expect(prometheus_client).to receive(:query) + .with(send("database_apdex_sli_query_#{schema}")) + .and_return([{ "value" => [1662423310.878, apdex_slo_below_sli[schema]] }]) + expect(evaluate).to be_a(Gitlab::Database::BackgroundMigration::HealthStatus::Signals::Stop) + expect(evaluate.reason).to include('Patroni service apdex is below SLO') + end + + context 'when Patroni apdex can not be calculated' do + where(:result) do + [ + nil, + [], + [{}], + [{ 'value' => 1 }], + [{ 'value' => [1] }] + ] + end + + with_them do + it 'returns Unknown signal' do + expect(prometheus_client).to receive(:query).and_return(result) + expect(evaluate).to be_a(Gitlab::Database::BackgroundMigration::HealthStatus::Signals::Unknown) + expect(evaluate.reason).to include('Patroni service apdex can not be calculated') + end + end + end + end + end + + Gitlab::Database.database_base_models.each do |database_base_model, connection| + next unless connection.present? + + it_behaves_like 'Patroni Apdex Evaluator', database_base_model.to_sym + end + end +end diff --git a/spec/lib/gitlab/database/background_migration/health_status_spec.rb b/spec/lib/gitlab/database/background_migration/health_status_spec.rb index 8bc04d80fa1..4d6c729f080 100644 --- a/spec/lib/gitlab/database/background_migration/health_status_spec.rb +++ b/spec/lib/gitlab/database/background_migration/health_status_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::BackgroundMigration::HealthStatus do +RSpec.describe Gitlab::Database::BackgroundMigration::HealthStatus, feature_category: :database do let(:connection) { Gitlab::Database.database_base_models[:main].connection } around do |example| @@ -19,8 +19,10 @@ RSpec.describe Gitlab::Database::BackgroundMigration::HealthStatus do let(:health_status) { Gitlab::Database::BackgroundMigration::HealthStatus } let(:autovacuum_indicator_class) { health_status::Indicators::AutovacuumActiveOnTable } let(:wal_indicator_class) { health_status::Indicators::WriteAheadLog } + let(:patroni_apdex_indicator_class) { health_status::Indicators::PatroniApdex } let(:autovacuum_indicator) { instance_double(autovacuum_indicator_class) } let(:wal_indicator) { instance_double(wal_indicator_class) } + let(:patroni_apdex_indicator) { instance_double(patroni_apdex_indicator_class) } before do allow(autovacuum_indicator_class).to receive(:new).with(migration.health_context).and_return(autovacuum_indicator) @@ -36,8 +38,11 @@ RSpec.describe Gitlab::Database::BackgroundMigration::HealthStatus do expect(autovacuum_indicator).to receive(:evaluate).and_return(normal_signal) expect(wal_indicator_class).to receive(:new).with(migration.health_context).and_return(wal_indicator) expect(wal_indicator).to receive(:evaluate).and_return(not_available_signal) + expect(patroni_apdex_indicator_class).to receive(:new).with(migration.health_context) + .and_return(patroni_apdex_indicator) + expect(patroni_apdex_indicator).to receive(:evaluate).and_return(not_available_signal) - expect(evaluate).to contain_exactly(normal_signal, not_available_signal) + expect(evaluate).to contain_exactly(normal_signal, not_available_signal, not_available_signal) end end @@ -50,10 +55,23 @@ RSpec.describe Gitlab::Database::BackgroundMigration::HealthStatus do end it 'logs interesting signals' do - signal = instance_double("#{health_status}::Signals::Stop", log_info?: true) + signal = instance_double( + "#{health_status}::Signals::Stop", + log_info?: true, + indicator_class: autovacuum_indicator_class, + short_name: 'Stop', + reason: 'Test Exception' + ) expect(autovacuum_indicator).to receive(:evaluate).and_return(signal) - expect(described_class).to receive(:log_signal).with(signal, migration) + + expect(Gitlab::BackgroundMigration::Logger).to receive(:info).with( + migration_id: migration.id, + health_status_indicator: autovacuum_indicator_class.to_s, + indicator_signal: 'Stop', + signal_reason: 'Test Exception', + message: "#{migration} signaled: #{signal}" + ) evaluate end diff --git a/spec/lib/gitlab/database/background_migration_job_spec.rb b/spec/lib/gitlab/database/background_migration_job_spec.rb index 1117c17c84a..6a1bedd800b 100644 --- a/spec/lib/gitlab/database/background_migration_job_spec.rb +++ b/spec/lib/gitlab/database/background_migration_job_spec.rb @@ -27,26 +27,6 @@ RSpec.describe Gitlab::Database::BackgroundMigrationJob do end end - describe '.for_partitioning_migration' do - let!(:job1) { create(:background_migration_job, arguments: [1, 100, 'other_table']) } - let!(:job2) { create(:background_migration_job, arguments: [1, 100, 'audit_events']) } - let!(:job3) { create(:background_migration_job, class_name: 'OtherJob', arguments: [1, 100, 'audit_events']) } - - it 'returns jobs matching class_name and the table_name job argument' do - relation = described_class.for_partitioning_migration('TestJob', 'audit_events') - - expect(relation.count).to eq(1) - expect(relation.first).to have_attributes(class_name: 'TestJob', arguments: [1, 100, 'audit_events']) - end - - it 'normalizes class names by removing leading ::' do - relation = described_class.for_partitioning_migration('::TestJob', 'audit_events') - - expect(relation.count).to eq(1) - expect(relation.first).to have_attributes(class_name: 'TestJob', arguments: [1, 100, 'audit_events']) - end - end - describe '.mark_all_as_succeeded' do let!(:job1) { create(:background_migration_job, arguments: [1, 100]) } let!(:job2) { create(:background_migration_job, arguments: [1, 100]) } diff --git a/spec/lib/gitlab/database/consistency_checker_spec.rb b/spec/lib/gitlab/database/consistency_checker_spec.rb index c0f0c349ddd..be03bd00619 100644 --- a/spec/lib/gitlab/database/consistency_checker_spec.rb +++ b/spec/lib/gitlab/database/consistency_checker_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::ConsistencyChecker, feature_category: :pods do +RSpec.describe Gitlab::Database::ConsistencyChecker, feature_category: :cell do let(:batch_size) { 10 } let(:max_batches) { 4 } let(:max_runtime) { described_class::MAX_RUNTIME } diff --git a/spec/lib/gitlab/database/dynamic_model_helpers_spec.rb b/spec/lib/gitlab/database/dynamic_model_helpers_spec.rb index 31486240bfa..fe423b3639b 100644 --- a/spec/lib/gitlab/database/dynamic_model_helpers_spec.rb +++ b/spec/lib/gitlab/database/dynamic_model_helpers_spec.rb @@ -49,6 +49,21 @@ RSpec.describe Gitlab::Database::DynamicModelHelpers do expect { |b| each_batch_size.call(&b) } .to yield_successive_args(1, 1) end + + context 'when a column to be batched over is specified' do + let(:projects) { Project.order(project_namespace_id: :asc) } + + it 'iterates table in batches using the given column' do + each_batch_ids = ->(&block) do + subject.each_batch(table_name, connection: connection, of: 1, column: :project_namespace_id) do |batch| + block.call(batch.pluck(:project_namespace_id)) + end + end + + expect { |b| each_batch_ids.call(&b) } + .to yield_successive_args([projects.first.project_namespace_id], [projects.last.project_namespace_id]) + end + end end context 'when transaction is open' do @@ -95,6 +110,35 @@ RSpec.describe Gitlab::Database::DynamicModelHelpers do expect { |b| each_batch_limited.call(&b) } .to yield_successive_args([first_project.id, first_project.id]) end + + context 'when primary key is not named id' do + let(:namespace_settings1) { create(:namespace_settings) } + let(:namespace_settings2) { create(:namespace_settings) } + let(:table_name) { NamespaceSetting.table_name } + let(:connection) { NamespaceSetting.connection } + let(:primary_key) { subject.define_batchable_model(table_name, connection: connection).primary_key } + + it 'iterates table in batch ranges using the correct primary key' do + expect(primary_key).to eq("namespace_id") # Sanity check the primary key is not id + expect { |b| subject.each_batch_range(table_name, connection: connection, of: 1, &b) } + .to yield_successive_args( + [namespace_settings1.namespace_id, namespace_settings1.namespace_id], + [namespace_settings2.namespace_id, namespace_settings2.namespace_id] + ) + end + end + + context 'when a column to be batched over is specified' do + it 'iterates table in batch ranges using the given column' do + expect do |b| + subject.each_batch_range(table_name, connection: connection, of: 1, column: :project_namespace_id, &b) + end + .to yield_successive_args( + [first_project.project_namespace_id, first_project.project_namespace_id], + [second_project.project_namespace_id, second_project.project_namespace_id] + ) + end + end end context 'when transaction is open' do diff --git a/spec/lib/gitlab/database/gitlab_schema_spec.rb b/spec/lib/gitlab/database/gitlab_schema_spec.rb index 28a087d5401..5d3260a77c9 100644 --- a/spec/lib/gitlab/database/gitlab_schema_spec.rb +++ b/spec/lib/gitlab/database/gitlab_schema_spec.rb @@ -16,19 +16,28 @@ RSpec.shared_examples 'validate schema data' do |tables_and_views| end end -RSpec.describe Gitlab::Database::GitlabSchema do +RSpec.describe Gitlab::Database::GitlabSchema, feature_category: :database do shared_examples 'maps table name to table schema' do using RSpec::Parameterized::TableSyntax + before do + ApplicationRecord.connection.execute(<<~SQL) + CREATE INDEX index_name_on_table_belonging_to_gitlab_main ON public.projects (name); + SQL + end + where(:name, :classification) do - 'ci_builds' | :gitlab_ci - 'my_schema.ci_builds' | :gitlab_ci - 'information_schema.columns' | :gitlab_internal - 'audit_events_part_5fc467ac26' | :gitlab_main - '_test_gitlab_main_table' | :gitlab_main - '_test_gitlab_ci_table' | :gitlab_ci - '_test_my_table' | :gitlab_shared - 'pg_attribute' | :gitlab_internal + 'ci_builds' | :gitlab_ci + 'my_schema.ci_builds' | :gitlab_ci + 'my_schema.ci_runner_machine_builds_100' | :gitlab_ci + 'my_schema._test_gitlab_main_table' | :gitlab_main + 'information_schema.columns' | :gitlab_internal + 'audit_events_part_5fc467ac26' | :gitlab_main + '_test_gitlab_main_table' | :gitlab_main + '_test_gitlab_ci_table' | :gitlab_ci + '_test_my_table' | :gitlab_shared + 'pg_attribute' | :gitlab_internal + 'index_name_on_table_belonging_to_gitlab_main' | :gitlab_main end with_them do @@ -49,8 +58,10 @@ RSpec.describe Gitlab::Database::GitlabSchema do context "for #{db_config_name} using #{db_class}" do let(:db_data_sources) { db_class.connection.data_sources } - # The Geo database does not share the same structure as all decomposed databases - subject { described_class.views_and_tables_to_schema.select { |_, v| v != :gitlab_geo } } + # The embedding and Geo databases do not share the same structure as all decomposed databases + subject do + described_class.views_and_tables_to_schema.reject { |_, v| v == :gitlab_embedding || v == :gitlab_geo } + end it 'new data sources are added' do missing_data_sources = db_data_sources.to_set - subject.keys @@ -116,10 +127,10 @@ RSpec.describe Gitlab::Database::GitlabSchema do end end - describe '.table_schemas' do + describe '.table_schemas!' do let(:tables) { %w[users projects ci_builds] } - subject { described_class.table_schemas(tables) } + subject { described_class.table_schemas!(tables) } it 'returns the matched schemas' do expect(subject).to match_array %i[gitlab_main gitlab_ci].to_set @@ -128,26 +139,8 @@ RSpec.describe Gitlab::Database::GitlabSchema do context 'when one of the tables does not have a matching table schema' do let(:tables) { %w[users projects unknown ci_builds] } - context 'and undefined parameter is false' do - subject { described_class.table_schemas(tables, undefined: false) } - - it 'includes a nil value' do - is_expected.to match_array [:gitlab_main, nil, :gitlab_ci].to_set - end - end - - context 'and undefined parameter is true' do - subject { described_class.table_schemas(tables, undefined: true) } - - it 'includes "undefined_<table_name>"' do - is_expected.to match_array [:gitlab_main, :undefined_unknown, :gitlab_ci].to_set - end - end - - context 'and undefined parameter is not specified' do - it 'includes a nil value' do - is_expected.to match_array [:gitlab_main, :undefined_unknown, :gitlab_ci].to_set - end + it 'raises error' do + expect { subject }.to raise_error(/Could not find gitlab schema for table unknown/) end end end @@ -160,23 +153,7 @@ RSpec.describe Gitlab::Database::GitlabSchema do context 'when mapping fails' do let(:name) { 'unknown_table' } - context "and parameter 'undefined' is set to true" do - subject { described_class.table_schema(name, undefined: true) } - - it { is_expected.to eq(:undefined_unknown_table) } - end - - context "and parameter 'undefined' is set to false" do - subject { described_class.table_schema(name, undefined: false) } - - it { is_expected.to be_nil } - end - - context "and parameter 'undefined' is not set" do - subject { described_class.table_schema(name) } - - it { is_expected.to eq(:undefined_unknown_table) } - end + it { is_expected.to be_nil } end end @@ -192,7 +169,8 @@ RSpec.describe Gitlab::Database::GitlabSchema do expect { subject }.to raise_error( Gitlab::Database::GitlabSchema::UnknownSchemaError, "Could not find gitlab schema for table #{name}: " \ - "Any new tables must be added to the database dictionary" + "Any new or deleted tables must be added to the database dictionary " \ + "See https://docs.gitlab.com/ee/development/database/database_dictionary.html" ) end end diff --git a/spec/lib/gitlab/database/load_balancing/action_cable_callbacks_spec.rb b/spec/lib/gitlab/database/load_balancing/action_cable_callbacks_spec.rb index 768855464c1..a57f02b22df 100644 --- a/spec/lib/gitlab/database/load_balancing/action_cable_callbacks_spec.rb +++ b/spec/lib/gitlab/database/load_balancing/action_cable_callbacks_spec.rb @@ -2,18 +2,13 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::LoadBalancing::ActionCableCallbacks, :request_store do +RSpec.describe Gitlab::Database::LoadBalancing::ActionCableCallbacks, :request_store, feature_category: :shared do describe '.wrapper' do - it 'uses primary and then releases the connection and clears the session' do + it 'releases the connection and clears the session' do expect(Gitlab::Database::LoadBalancing).to receive(:release_hosts) expect(Gitlab::Database::LoadBalancing::Session).to receive(:clear_session) - described_class.wrapper.call( - nil, - lambda do - expect(Gitlab::Database::LoadBalancing::Session.current.use_primary?).to eq(true) - end - ) + described_class.wrapper.call(nil, lambda {}) end context 'with an exception' do diff --git a/spec/lib/gitlab/database/load_balancing/logger_spec.rb b/spec/lib/gitlab/database/load_balancing/logger_spec.rb new file mode 100644 index 00000000000..81883fa6f1a --- /dev/null +++ b/spec/lib/gitlab/database/load_balancing/logger_spec.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::LoadBalancing::Logger, feature_category: :database do + subject { described_class.new('/dev/null') } + + it_behaves_like 'a json logger', {} + + it 'excludes context' do + expect(described_class.exclude_context?).to be(true) + end +end diff --git a/spec/lib/gitlab/database/load_balancing/service_discovery_spec.rb b/spec/lib/gitlab/database/load_balancing/service_discovery_spec.rb index bfd9c644ffa..9a559c7ccb4 100644 --- a/spec/lib/gitlab/database/load_balancing/service_discovery_spec.rb +++ b/spec/lib/gitlab/database/load_balancing/service_discovery_spec.rb @@ -90,7 +90,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::ServiceDiscovery, feature_catego end end - context 'with failures' do + context 'with StandardError' do before do allow(Gitlab::ErrorTracking).to receive(:track_exception) allow(service).to receive(:sleep) @@ -142,6 +142,21 @@ RSpec.describe Gitlab::Database::LoadBalancing::ServiceDiscovery, feature_catego service.perform_service_discovery end end + + context 'with Exception' do + it 'logs error and re-raises the exception' do + error = Exception.new('uncaught-test-error') + + expect(service).to receive(:refresh_if_necessary).and_raise(error) + + expect(Gitlab::Database::LoadBalancing::Logger).to receive(:error).with( + event: :service_discovery_unexpected_exception, + message: "Service discovery encountered an uncaught error: uncaught-test-error" + ) + + expect { service.perform_service_discovery }.to raise_error(Exception, error.message) + end + end end describe '#refresh_if_necessary' do diff --git a/spec/lib/gitlab/database/load_balancing/sidekiq_client_middleware_spec.rb b/spec/lib/gitlab/database/load_balancing/sidekiq_client_middleware_spec.rb index 7eb20f77417..5a52394742f 100644 --- a/spec/lib/gitlab/database/load_balancing/sidekiq_client_middleware_spec.rb +++ b/spec/lib/gitlab/database/load_balancing/sidekiq_client_middleware_spec.rb @@ -62,59 +62,40 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqClientMiddleware, feature include_examples 'job data consistency' end - shared_examples_for 'mark data consistency location' do |data_consistency| - include_context 'data consistency worker class', data_consistency, :load_balancing_for_test_data_consistency_worker - + shared_examples_for 'mark data consistency location' do |data_consistency, worker_klass| let(:location) { '0/D525E3A8' } + include_context 'when tracking WAL location reference' - context 'when feature flag is disabled' do - let(:expected_consistency) { :always } - - before do - stub_feature_flags(load_balancing_for_test_data_consistency_worker: false) - end - - include_examples 'does not pass database locations' + if worker_klass + let(:worker_class) { worker_klass } + let(:expected_consistency) { data_consistency } + else + include_context 'data consistency worker class', data_consistency, :load_balancing_for_test_data_consistency_worker end context 'when write was not performed' do before do - allow(Gitlab::Database::LoadBalancing::Session.current).to receive(:use_primary?).and_return(false) + stub_no_writes_performed! end context 'when replica hosts are available' do it 'passes database_replica_location' do - expected_location = {} - - Gitlab::Database::LoadBalancing.each_load_balancer do |lb| - expect(lb.host) - .to receive(:database_replica_location) - .and_return(location) - - expected_location[lb.name] = location - end + expected_locations = expect_tracked_locations_when_replicas_available run_middleware - expect(job['wal_locations']).to eq(expected_location) + expect(job['wal_locations']).to eq(expected_locations) expect(job['wal_location_source']).to eq(:replica) end end context 'when no replica hosts are available' do it 'passes primary_write_location' do - expected_location = {} - - Gitlab::Database::LoadBalancing.each_load_balancer do |lb| - expect(lb).to receive(:host).and_return(nil) - expect(lb).to receive(:primary_write_location).and_return(location) - - expected_location[lb.name] = location - end + expected_locations = expect_tracked_locations_when_no_replicas_available run_middleware - expect(job['wal_locations']).to eq(expected_location) + expect(job['wal_locations']).to eq(expected_locations) expect(job['wal_location_source']).to eq(:replica) end end @@ -124,23 +105,15 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqClientMiddleware, feature context 'when write was performed' do before do - allow(Gitlab::Database::LoadBalancing::Session.current).to receive(:use_primary?).and_return(true) + stub_write_performed! end it 'passes primary write location', :aggregate_failures do - expected_location = {} - - Gitlab::Database::LoadBalancing.each_load_balancer do |lb| - expect(lb) - .to receive(:primary_write_location) - .and_return(location) - - expected_location[lb.name] = location - end + expected_locations = expect_tracked_locations_from_primary_only run_middleware - expect(job['wal_locations']).to eq(expected_location) + expect(job['wal_locations']).to eq(expected_locations) expect(job['wal_location_source']).to eq(:primary) end @@ -149,19 +122,36 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqClientMiddleware, feature end context 'when worker cannot be constantized' do - let(:worker_class) { 'ActionMailer::MailDeliveryJob' } + let(:worker_class) { 'InvalidWorker' } let(:expected_consistency) { :always } include_examples 'does not pass database locations' end context 'when worker class does not include ApplicationWorker' do - let(:worker_class) { ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper } + let(:worker_class) { Gitlab::SidekiqConfig::DummyWorker } let(:expected_consistency) { :always } include_examples 'does not pass database locations' end + context 'when job contains wrapped worker' do + let(:worker_class) { ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper } + + context 'when wrapped worker does not include WorkerAttributes' do + let(:job) { { "job_id" => "a180b47c-3fd6-41b8-81e9-34da61c3400e", "wrapped" => Gitlab::SidekiqConfig::DummyWorker } } + let(:expected_consistency) { :always } + + include_examples 'does not pass database locations' + end + + context 'when wrapped worker includes WorkerAttributes' do + let(:job) { { "job_id" => "a180b47c-3fd6-41b8-81e9-34da61c3400e", "wrapped" => ActionMailer::MailDeliveryJob } } + + include_examples 'mark data consistency location', :delayed, ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper + end + end + context 'database wal location was already provided' do let(:old_location) { '0/D525E3A8' } let(:new_location) { 'AB/12345' } diff --git a/spec/lib/gitlab/database/load_balancing/sidekiq_server_middleware_spec.rb b/spec/lib/gitlab/database/load_balancing/sidekiq_server_middleware_spec.rb index abf10456d0a..7703b5680c2 100644 --- a/spec/lib/gitlab/database/load_balancing/sidekiq_server_middleware_spec.rb +++ b/spec/lib/gitlab/database/load_balancing/sidekiq_server_middleware_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware, :clean_gitlab_redis_queues do +RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware, :clean_gitlab_redis_queues, feature_category: :scalability do let(:middleware) { described_class.new } let(:worker) { worker_class.new } let(:location) { '0/D525E3A8' } @@ -15,6 +15,8 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware, :clean_ replication_lag!(false) Gitlab::Database::LoadBalancing::Session.clear_session + + stub_const("#{described_class.name}::REPLICA_WAIT_SLEEP_SECONDS", 0.0) end after do @@ -76,14 +78,6 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware, :clean_ end shared_examples_for 'sticks based on data consistency' do - context 'when load_balancing_for_test_data_consistency_worker is disabled' do - before do - stub_feature_flags(load_balancing_for_test_data_consistency_worker: false) - end - - include_examples 'stick to the primary', 'primary' - end - context 'when database wal location is set' do let(:job) { { 'job_id' => 'a180b47c-3fd6-41b8-81e9-34da61c3400e', 'wal_locations' => wal_locations } } @@ -119,9 +113,9 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware, :clean_ end end - shared_examples_for 'sleeps when necessary' do + shared_examples_for 'essential sleep' do context 'when WAL locations are blank', :freeze_time do - let(:job) { { "retry" => 3, "job_id" => "a180b47c-3fd6-41b8-81e9-34da61c3400e", "wal_locations" => {}, "created_at" => Time.current.to_f - (described_class::MINIMUM_DELAY_INTERVAL_SECONDS - 0.3) } } + let(:job) { { "retry" => 3, "job_id" => "a180b47c-3fd6-41b8-81e9-34da61c3400e", "wal_locations" => {}, "created_at" => Time.current.to_f - (described_class::REPLICA_WAIT_SLEEP_SECONDS + 0.2) } } it 'does not sleep' do expect(middleware).not_to receive(:sleep) @@ -134,7 +128,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware, :clean_ let(:job) { { "retry" => 3, "job_id" => "a180b47c-3fd6-41b8-81e9-34da61c3400e", 'wal_locations' => wal_locations, "created_at" => Time.current.to_f - elapsed_time } } context 'when delay interval has not elapsed' do - let(:elapsed_time) { described_class::MINIMUM_DELAY_INTERVAL_SECONDS - 0.3 } + let(:elapsed_time) { described_class::REPLICA_WAIT_SLEEP_SECONDS + 0.2 } context 'when replica is up to date' do before do @@ -158,41 +152,46 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware, :clean_ end it 'sleeps until the minimum delay is reached' do - expect(middleware).to receive(:sleep).with(be_within(0.01).of(described_class::MINIMUM_DELAY_INTERVAL_SECONDS - elapsed_time)) + expect(middleware).to receive(:sleep).with(described_class::REPLICA_WAIT_SLEEP_SECONDS) run_middleware end end - end - - context 'when delay interval has elapsed' do - let(:elapsed_time) { described_class::MINIMUM_DELAY_INTERVAL_SECONDS + 0.3 } - - it 'does not sleep' do - expect(middleware).not_to receive(:sleep) - - run_middleware - end - end - context 'when created_at is in the future' do - let(:elapsed_time) { -5 } + context 'when replica is never not up to date' do + before do + Gitlab::Database::LoadBalancing.each_load_balancer do |lb| + allow(lb).to receive(:select_up_to_date_host).and_return(false, false) + end + end - it 'does not sleep' do - expect(middleware).not_to receive(:sleep) + it 'sleeps until the maximum delay is reached' do + expect(middleware).to receive(:sleep).exactly(3).times.with(described_class::REPLICA_WAIT_SLEEP_SECONDS) - run_middleware + run_middleware + end end end end end - context 'when worker class does not include ApplicationWorker' do + context 'when worker class does not include WorkerAttributes' do let(:worker) { ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper.new } include_examples 'stick to the primary', 'primary' end + context 'when job contains wrapped worker class' do + let(:worker) { ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper.new } + let(:job) { { "retry" => 3, "job_id" => "a180b47c-3fd6-41b8-81e9-34da61c3400e", 'wal_locations' => wal_locations, 'wrapped' => 'ActionMailer::MailDeliveryJob' } } + + it 'uses wrapped job if available' do + expect(middleware).to receive(:select_load_balancing_strategy).with(ActionMailer::MailDeliveryJob, job).and_call_original + + run_middleware + end + end + context 'when worker data consistency is :always' do include_context 'data consistency worker class', :always, :load_balancing_for_test_data_consistency_worker @@ -200,7 +199,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware, :clean_ context 'when delay interval has not elapsed', :freeze_time do let(:job) { { "retry" => 3, "job_id" => "a180b47c-3fd6-41b8-81e9-34da61c3400e", 'wal_locations' => wal_locations, "created_at" => Time.current.to_f - elapsed_time } } - let(:elapsed_time) { described_class::MINIMUM_DELAY_INTERVAL_SECONDS - 0.3 } + let(:elapsed_time) { described_class::REPLICA_WAIT_SLEEP_SECONDS + 0.2 } it 'does not sleep' do expect(middleware).not_to receive(:sleep) @@ -214,7 +213,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware, :clean_ include_context 'data consistency worker class', :delayed, :load_balancing_for_test_data_consistency_worker include_examples 'sticks based on data consistency' - include_examples 'sleeps when necessary' + include_examples 'essential sleep' context 'when replica is not up to date' do before do @@ -263,7 +262,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::SidekiqServerMiddleware, :clean_ include_context 'data consistency worker class', :sticky, :load_balancing_for_test_data_consistency_worker include_examples 'sticks based on data consistency' - include_examples 'sleeps when necessary' + include_examples 'essential sleep' context 'when replica is not up to date' do before do diff --git a/spec/lib/gitlab/database/load_balancing_spec.rb b/spec/lib/gitlab/database/load_balancing_spec.rb index 59e16e6ca8b..f65c27688b8 100644 --- a/spec/lib/gitlab/database/load_balancing_spec.rb +++ b/spec/lib/gitlab/database/load_balancing_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::LoadBalancing, :suppress_gitlab_schemas_validate_connection, feature_category: :pods do +RSpec.describe Gitlab::Database::LoadBalancing, :suppress_gitlab_schemas_validate_connection, feature_category: :cell do describe '.base_models' do it 'returns the models to apply load balancing to' do models = described_class.base_models @@ -497,14 +497,15 @@ RSpec.describe Gitlab::Database::LoadBalancing, :suppress_gitlab_schemas_validat where(:queries, :expected_role) do [ # Reload cache. The schema loading queries should be handled by - # primary. + # replica even when the current session is stuck to the primary. [ -> { + ::Gitlab::Database::LoadBalancing::Session.current.use_primary! model.connection.clear_cache! model.connection.schema_cache.add('users') model.connection.pool.release_connection }, - :primary + :replica ], # Call model's connection method diff --git a/spec/lib/gitlab/database/lock_writes_manager_spec.rb b/spec/lib/gitlab/database/lock_writes_manager_spec.rb index c06c463d918..2aa95372338 100644 --- a/spec/lib/gitlab/database/lock_writes_manager_spec.rb +++ b/spec/lib/gitlab/database/lock_writes_manager_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::LockWritesManager, :delete, feature_category: :pods do +RSpec.describe Gitlab::Database::LockWritesManager, :delete, feature_category: :cell do let(:connection) { ApplicationRecord.connection } let(:test_table) { '_test_table' } let(:logger) { instance_double(Logger) } @@ -122,6 +122,13 @@ RSpec.describe Gitlab::Database::LockWritesManager, :delete, feature_category: : } end + it 'returns result hash with action skipped' do + subject.lock_writes + + expect(subject.lock_writes).to eq({ action: "skipped", database: "main", dry_run: false, +table: test_table }) + end + context 'when running in dry_run mode' do let(:dry_run) { true } @@ -146,6 +153,11 @@ RSpec.describe Gitlab::Database::LockWritesManager, :delete, feature_category: : connection.execute("truncate #{test_table}") end.not_to raise_error end + + it 'returns result hash with action locked' do + expect(subject.lock_writes).to eq({ action: "locked", database: "main", dry_run: dry_run, +table: test_table }) + end end end @@ -186,6 +198,11 @@ RSpec.describe Gitlab::Database::LockWritesManager, :delete, feature_category: : subject.unlock_writes end + it 'returns result hash with action unlocked' do + expect(subject.unlock_writes).to eq({ action: "unlocked", database: "main", dry_run: dry_run, +table: test_table }) + end + context 'when running in dry_run mode' do let(:dry_run) { true } @@ -206,6 +223,11 @@ RSpec.describe Gitlab::Database::LockWritesManager, :delete, feature_category: : connection.execute("delete from #{test_table}") end.to raise_error(ActiveRecord::StatementInvalid, /Table: "#{test_table}" is write protected/) end + + it 'returns result hash with dry_run true' do + expect(subject.unlock_writes).to eq({ action: "unlocked", database: "main", dry_run: dry_run, +table: test_table }) + end end end diff --git a/spec/lib/gitlab/database/loose_foreign_keys_spec.rb b/spec/lib/gitlab/database/loose_foreign_keys_spec.rb index 3c2d9ca82f2..552df64096a 100644 --- a/spec/lib/gitlab/database/loose_foreign_keys_spec.rb +++ b/spec/lib/gitlab/database/loose_foreign_keys_spec.rb @@ -85,31 +85,40 @@ RSpec.describe Gitlab::Database::LooseForeignKeys do end end - describe '.definitions' do - subject(:definitions) { described_class.definitions } - - it 'contains at least all parent tables that have triggers' do - all_definition_parent_tables = definitions.map { |d| d.to_table }.to_set + context 'all tables have correct triggers installed' do + let(:all_tables_from_yaml) { described_class.definitions.pluck(:to_table).uniq } + let(:all_tables_with_triggers) do triggers_query = <<~SQL - SELECT event_object_table, trigger_name - FROM information_schema.triggers + SELECT event_object_table FROM information_schema.triggers WHERE trigger_name LIKE '%_loose_fk_trigger' - GROUP BY event_object_table, trigger_name SQL - all_triggers = ApplicationRecord.connection.execute(triggers_query) - - all_triggers.each do |trigger| - table = trigger['event_object_table'] - trigger_name = trigger['trigger_name'] - error_message = <<~END - Missing a loose foreign key definition for parent table: #{table} with trigger: #{trigger_name}. - Loose foreign key definitions must be added before triggers are added and triggers must be removed before removing the loose foreign key definition. - Read more at https://docs.gitlab.com/ee/development/database/loose_foreign_keys.html ." - END - expect(all_definition_parent_tables).to include(table), error_message - end + ApplicationRecord.connection.execute(triggers_query) + .pluck('event_object_table').uniq + end + + it 'all YAML tables do have `track_record_deletions` installed' do + missing_trigger_tables = all_tables_from_yaml - all_tables_with_triggers + + expect(missing_trigger_tables).to be_empty, <<~END + The loose foreign keys definitions require using `track_record_deletions` + for the following tables: #{missing_trigger_tables}. + Read more at https://docs.gitlab.com/ee/development/database/loose_foreign_keys.html." + END + end + + it 'no extra tables have `track_record_deletions` installed' do + extra_trigger_tables = all_tables_with_triggers - all_tables_from_yaml + + pending 'This result of this test is informatory, and not critical' if extra_trigger_tables.any? + + expect(extra_trigger_tables).to be_empty, <<~END + The following tables have unused `track_record_deletions` triggers installed, + but they are not referenced by any of the loose foreign key definitions: #{extra_trigger_tables}. + You can remove them in one of the future releases as part of `db/post_migrate`. + Read more at https://docs.gitlab.com/ee/development/database/loose_foreign_keys.html." + END end end diff --git a/spec/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables_spec.rb b/spec/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables_spec.rb index be9346e3829..e4241348b54 100644 --- a/spec/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables_spec.rb +++ b/spec/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::Database::MigrationHelpers::AutomaticLockWritesOnTables, - :reestablished_active_record_base, :delete, query_analyzers: false, feature_category: :pods do + :reestablished_active_record_base, :delete, query_analyzers: false, feature_category: :cell do using RSpec::Parameterized::TableSyntax let(:schema_class) { Class.new(Gitlab::Database::Migration[2.1]) } @@ -86,7 +86,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers::AutomaticLockWritesOnTables, let(:create_gitlab_shared_table_migration_class) { create_table_migration(gitlab_shared_table_name) } before do - skip_if_multiple_databases_are_setup(:ci) + skip_if_database_exists(:ci) end it 'does not lock any newly created tables' do @@ -106,7 +106,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers::AutomaticLockWritesOnTables, context 'when multiple databases' do before do - skip_if_multiple_databases_not_setup(:ci) + skip_if_shared_database(:ci) end let(:migration_class) { create_table_migration(table_name, skip_automatic_lock_on_writes) } @@ -224,13 +224,12 @@ RSpec.describe Gitlab::Database::MigrationHelpers::AutomaticLockWritesOnTables, let(:config_model) { Gitlab::Database.database_base_models[:main] } it "raises an error about undefined gitlab_schema" do - expected_error_message = <<~ERROR - No gitlab_schema is defined for the table #{table_name}. Please consider - adding it to the database dictionary. - More info: https://docs.gitlab.com/ee/development/database/database_dictionary.html - ERROR - - expect { run_migration }.to raise_error(expected_error_message) + expect { run_migration }.to raise_error( + Gitlab::Database::GitlabSchema::UnknownSchemaError, + "Could not find gitlab schema for table foobar: " \ + "Any new or deleted tables must be added to the database dictionary " \ + "See https://docs.gitlab.com/ee/development/database/database_dictionary.html" + ) end end end @@ -238,7 +237,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers::AutomaticLockWritesOnTables, context 'when renaming a table' do before do - skip_if_multiple_databases_not_setup(:ci) + skip_if_shared_database(:ci) create_table_migration(old_table_name).migrate(:up) # create the table first before renaming it end @@ -277,7 +276,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers::AutomaticLockWritesOnTables, let(:config_model) { Gitlab::Database.database_base_models[:main] } before do - skip_if_multiple_databases_are_setup(:ci) + skip_if_database_exists(:ci) end it 'does not lock any newly created tables' do @@ -305,7 +304,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers::AutomaticLockWritesOnTables, context 'when multiple databases' do before do - skip_if_multiple_databases_not_setup(:ci) + skip_if_shared_database(:ci) migration_class.connection.execute("CREATE TABLE #{table_name}()") migration_class.migrate(:up) end diff --git a/spec/lib/gitlab/database/migration_helpers/convert_to_bigint_spec.rb b/spec/lib/gitlab/database/migration_helpers/convert_to_bigint_spec.rb new file mode 100644 index 00000000000..cee5f54bd6a --- /dev/null +++ b/spec/lib/gitlab/database/migration_helpers/convert_to_bigint_spec.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::MigrationHelpers::ConvertToBigint, feature_category: :database do + describe 'com_or_dev_or_test_but_not_jh?' do + using RSpec::Parameterized::TableSyntax + + where(:dot_com, :dev_or_test, :jh, :expectation) do + true | true | true | true + true | false | true | false + false | true | true | true + false | false | true | false + true | true | false | true + true | false | false | true + false | true | false | true + false | false | false | false + end + + with_them do + it 'returns true for GitLab.com (but not JH), dev, or test' do + allow(Gitlab).to receive(:com?).and_return(dot_com) + allow(Gitlab).to receive(:dev_or_test_env?).and_return(dev_or_test) + allow(Gitlab).to receive(:jh?).and_return(jh) + + migration = Class + .new + .include(Gitlab::Database::MigrationHelpers::ConvertToBigint) + .new + + expect(migration.com_or_dev_or_test_but_not_jh?).to eq(expectation) + end + end + end +end diff --git a/spec/lib/gitlab/database/migration_helpers/loose_foreign_key_helpers_spec.rb b/spec/lib/gitlab/database/migration_helpers/loose_foreign_key_helpers_spec.rb index 25fc676d09e..2b58cdff931 100644 --- a/spec/lib/gitlab/database/migration_helpers/loose_foreign_key_helpers_spec.rb +++ b/spec/lib/gitlab/database/migration_helpers/loose_foreign_key_helpers_spec.rb @@ -7,20 +7,22 @@ RSpec.describe Gitlab::Database::MigrationHelpers::LooseForeignKeyHelpers do ActiveRecord::Migration.new.extend(described_class) end + let_it_be(:table_name) { :_test_loose_fk_test_table } + let(:model) do Class.new(ApplicationRecord) do - self.table_name = '_test_loose_fk_test_table' + self.table_name = :_test_loose_fk_test_table end end before(:all) do - migration.create_table :_test_loose_fk_test_table do |t| + migration.create_table table_name do |t| t.timestamps end end after(:all) do - migration.drop_table :_test_loose_fk_test_table + migration.drop_table table_name end before do @@ -33,11 +35,13 @@ RSpec.describe Gitlab::Database::MigrationHelpers::LooseForeignKeyHelpers do expect(LooseForeignKeys::DeletedRecord.count).to eq(0) end + + it { expect(migration.has_loose_foreign_key?(table_name)).to be_falsy } end context 'when the record deletion tracker trigger is installed' do before do - migration.track_record_deletions(:_test_loose_fk_test_table) + migration.track_record_deletions(table_name) end it 'stores the record deletion' do @@ -55,7 +59,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers::LooseForeignKeyHelpers do .first expect(deleted_record.primary_key_value).to eq(record_to_be_deleted.id) - expect(deleted_record.fully_qualified_table_name).to eq('public._test_loose_fk_test_table') + expect(deleted_record.fully_qualified_table_name).to eq("public.#{table_name}") expect(deleted_record.partition_number).to eq(1) end @@ -64,5 +68,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers::LooseForeignKeyHelpers do expect(LooseForeignKeys::DeletedRecord.count).to eq(3) end + + it { expect(migration.has_loose_foreign_key?(table_name)).to be_truthy } end end diff --git a/spec/lib/gitlab/database/migration_helpers/restrict_gitlab_schema_spec.rb b/spec/lib/gitlab/database/migration_helpers/restrict_gitlab_schema_spec.rb index 714fbab5aff..faf0447c054 100644 --- a/spec/lib/gitlab/database/migration_helpers/restrict_gitlab_schema_spec.rb +++ b/spec/lib/gitlab/database/migration_helpers/restrict_gitlab_schema_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::Database::MigrationHelpers::RestrictGitlabSchema, query_analyzers: false, - stub_feature_flags: false, feature_category: :pods do + stub_feature_flags: false, feature_category: :cell do let(:schema_class) { Class.new(Gitlab::Database::Migration[1.0]).include(described_class) } # We keep only the GitlabSchemasValidateConnection analyzer running @@ -506,7 +506,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers::RestrictGitlabSchema, query_a def down; end end, query_matcher: /FROM ci_builds/, - setup: -> (_) { skip_if_multiple_databases_not_setup }, + setup: -> (_) { skip_if_shared_database(:ci) }, expected: { no_gitlab_schema: { main: :cross_schema_error, diff --git a/spec/lib/gitlab/database/migration_helpers/v2_spec.rb b/spec/lib/gitlab/database/migration_helpers/v2_spec.rb index 0d75094a2fd..8b653e2d89d 100644 --- a/spec/lib/gitlab/database/migration_helpers/v2_spec.rb +++ b/spec/lib/gitlab/database/migration_helpers/v2_spec.rb @@ -416,4 +416,83 @@ RSpec.describe Gitlab::Database::MigrationHelpers::V2 do end end end + + describe '#truncate_tables!' do + before do + ApplicationRecord.connection.execute(<<~SQL) + CREATE TABLE _test_gitlab_main_table (id serial primary key); + CREATE TABLE _test_gitlab_main_table2 (id serial primary key); + + INSERT INTO _test_gitlab_main_table DEFAULT VALUES; + INSERT INTO _test_gitlab_main_table2 DEFAULT VALUES; + SQL + + Ci::ApplicationRecord.connection.execute(<<~SQL) + CREATE TABLE _test_gitlab_ci_table (id serial primary key); + SQL + end + + it 'truncates the table' do + expect(migration).to receive(:execute).with('TRUNCATE TABLE "_test_gitlab_main_table"').and_call_original + + expect { migration.truncate_tables!('_test_gitlab_main_table') } + .to change { ApplicationRecord.connection.select_value('SELECT count(1) from _test_gitlab_main_table') }.to(0) + end + + it 'truncates multiple tables' do + expect(migration).to receive(:execute).with('TRUNCATE TABLE "_test_gitlab_main_table", "_test_gitlab_main_table2"').and_call_original + + expect { migration.truncate_tables!('_test_gitlab_main_table', '_test_gitlab_main_table2') } + .to change { ApplicationRecord.connection.select_value('SELECT count(1) from _test_gitlab_main_table') }.to(0) + .and change { ApplicationRecord.connection.select_value('SELECT count(1) from _test_gitlab_main_table2') }.to(0) + end + + it 'raises an ArgumentError if truncating multiple gitlab_schema' do + expect do + migration.truncate_tables!('_test_gitlab_main_table', '_test_gitlab_ci_table') + end.to raise_error(ArgumentError, /one `gitlab_schema`/) + end + + context 'with multiple databases' do + before do + skip_if_shared_database(:ci) + end + + context 'for ci database' do + before do + migration.instance_variable_set :@connection, Ci::ApplicationRecord.connection + end + + it 'skips the TRUNCATE statement tables not in schema for connection' do + expect(migration).not_to receive(:execute) + + migration.truncate_tables!('_test_gitlab_main_table') + end + end + + context 'for main database' do + before do + migration.instance_variable_set :@connection, ApplicationRecord.connection + end + + it 'executes a TRUNCATE statement' do + expect(migration).to receive(:execute).with('TRUNCATE TABLE "_test_gitlab_main_table"') + + migration.truncate_tables!('_test_gitlab_main_table') + end + end + end + + context 'with single database' do + before do + skip_if_database_exists(:ci) + end + + it 'executes a TRUNCATE statement' do + expect(migration).to receive(:execute).with('TRUNCATE TABLE "_test_gitlab_main_table"') + + migration.truncate_tables!('_test_gitlab_main_table') + end + end + end end diff --git a/spec/lib/gitlab/database/migration_helpers/wraparound_vacuum_helpers_spec.rb b/spec/lib/gitlab/database/migration_helpers/wraparound_vacuum_helpers_spec.rb new file mode 100644 index 00000000000..f7d11184ac7 --- /dev/null +++ b/spec/lib/gitlab/database/migration_helpers/wraparound_vacuum_helpers_spec.rb @@ -0,0 +1,97 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::MigrationHelpers::WraparoundVacuumHelpers, feature_category: :database do + include Database::DatabaseHelpers + + let(:table_name) { 'ci_builds' } + + describe '#check_if_wraparound_in_progress' do + let(:migration) do + ActiveRecord::Migration.new.extend(described_class) + end + + subject { migration.check_if_wraparound_in_progress(table_name) } + + it 'delegates to the wraparound class' do + expect(described_class::WraparoundCheck) + .to receive(:new) + .with(table_name, migration: migration) + .and_call_original + + expect { subject }.not_to raise_error + end + end + + describe described_class::WraparoundCheck do + let(:migration) do + ActiveRecord::Migration.new.extend(Gitlab::Database::MigrationHelpers::WraparoundVacuumHelpers) + end + + describe '#execute' do + subject do + described_class.new(table_name, migration: migration).execute + end + + context 'with wraparound vacuuum running' do + before do + swapout_view_for_table(:pg_stat_activity, connection: migration.connection, schema: 'pg_temp') + + migration.connection.execute(<<~SQL.squish) + INSERT INTO pg_stat_activity ( + datid, datname, pid, backend_start, xact_start, query_start, + state_change, wait_event_type, wait_event, state, backend_xmin, + query, backend_type) + VALUES ( + 16401, current_database(), 178, '2023-03-30 08:10:50.851322+00', + '2023-03-30 08:10:50.890485+00', now() - '150 minutes'::interval, + '2023-03-30 08:10:50.890485+00', 'IO', 'DataFileRead', 'active','3214790381'::xid, + 'autovacuum: VACUUM public.ci_builds (to prevent wraparound)', 'autovacuum worker') + SQL + end + + it 'outputs a message related to autovacuum' do + expect { subject } + .to output(/Autovacuum with wraparound prevention mode is running on `ci_builds`/).to_stdout + end + + it { expect { subject }.to output(/autovacuum: VACUUM public.ci_builds \(to prevent wraparound\)/).to_stdout } + it { expect { subject }.to output(/Current duration: 2 hours, 30 minutes/).to_stdout } + + context 'when GITLAB_MIGRATIONS_DISABLE_WRAPAROUND_CHECK is set' do + before do + stub_env('GITLAB_MIGRATIONS_DISABLE_WRAPAROUND_CHECK' => 'true') + end + + it { expect { subject }.not_to output(/autovacuum/i).to_stdout } + + it 'is disabled on .com' do + expect(Gitlab).to receive(:com?).and_return(true) + + expect { subject }.not_to raise_error + end + end + + context 'when executed by self-managed' do + before do + allow(Gitlab).to receive(:com?).and_return(false) + allow(Gitlab).to receive(:dev_or_test_env?).and_return(false) + end + + it { expect { subject }.not_to output(/autovacuum/i).to_stdout } + end + end + + context 'with wraparound vacuuum not running' do + it { expect { subject }.not_to output(/autovacuum/i).to_stdout } + end + + context 'when the table does not exist' do + let(:table_name) { :no_table } + + it { expect { subject }.to raise_error described_class::WraparoundError, /no_table/ } + end + end + end +end diff --git a/spec/lib/gitlab/database/migration_helpers_spec.rb b/spec/lib/gitlab/database/migration_helpers_spec.rb index 9df23776be8..b1e8301d69f 100644 --- a/spec/lib/gitlab/database/migration_helpers_spec.rb +++ b/spec/lib/gitlab/database/migration_helpers_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::MigrationHelpers do +RSpec.describe Gitlab::Database::MigrationHelpers, feature_category: :database do include Database::TableSchemaHelpers include Database::TriggerHelpers @@ -14,8 +14,10 @@ RSpec.describe Gitlab::Database::MigrationHelpers do allow(model).to receive(:puts) end + it { expect(model.singleton_class.ancestors).to include(described_class::WraparoundVacuumHelpers) } + describe 'overridden dynamic model helpers' do - let(:test_table) { '_test_batching_table' } + let(:test_table) { :_test_batching_table } before do model.connection.execute(<<~SQL) @@ -120,157 +122,6 @@ RSpec.describe Gitlab::Database::MigrationHelpers do end end - describe '#create_table_with_constraints' do - let(:table_name) { :test_table } - let(:column_attributes) do - [ - { name: 'id', sql_type: 'bigint', null: false, default: nil }, - { name: 'created_at', sql_type: 'timestamp with time zone', null: false, default: nil }, - { name: 'updated_at', sql_type: 'timestamp with time zone', null: false, default: nil }, - { name: 'some_id', sql_type: 'integer', null: false, default: nil }, - { name: 'active', sql_type: 'boolean', null: false, default: 'true' }, - { name: 'name', sql_type: 'text', null: true, default: nil } - ] - end - - before do - allow(model).to receive(:transaction_open?).and_return(true) - end - - context 'when no check constraints are defined' do - it 'creates the table as expected' do - model.create_table_with_constraints table_name do |t| - t.timestamps_with_timezone - t.integer :some_id, null: false - t.boolean :active, null: false, default: true - t.text :name - end - - expect_table_columns_to_match(column_attributes, table_name) - end - end - - context 'when check constraints are defined' do - context 'when the text_limit is explicity named' do - it 'creates the table as expected' do - model.create_table_with_constraints table_name do |t| - t.timestamps_with_timezone - t.integer :some_id, null: false - t.boolean :active, null: false, default: true - t.text :name - - t.text_limit :name, 255, name: 'check_name_length' - t.check_constraint :some_id_is_positive, 'some_id > 0' - end - - expect_table_columns_to_match(column_attributes, table_name) - - expect_check_constraint(table_name, 'check_name_length', 'char_length(name) <= 255') - expect_check_constraint(table_name, 'some_id_is_positive', 'some_id > 0') - end - end - - context 'when the text_limit is not named' do - it 'creates the table as expected, naming the text limit' do - model.create_table_with_constraints table_name do |t| - t.timestamps_with_timezone - t.integer :some_id, null: false - t.boolean :active, null: false, default: true - t.text :name - - t.text_limit :name, 255 - t.check_constraint :some_id_is_positive, 'some_id > 0' - end - - expect_table_columns_to_match(column_attributes, table_name) - - expect_check_constraint(table_name, 'check_cda6f69506', 'char_length(name) <= 255') - expect_check_constraint(table_name, 'some_id_is_positive', 'some_id > 0') - end - end - - it 'runs the change within a with_lock_retries' do - expect(model).to receive(:with_lock_retries).ordered.and_yield - expect(model).to receive(:create_table).ordered.and_call_original - expect(model).to receive(:execute).with(<<~SQL).ordered - ALTER TABLE "#{table_name}"\nADD CONSTRAINT "check_cda6f69506" CHECK (char_length("name") <= 255) - SQL - - model.create_table_with_constraints table_name do |t| - t.text :name - t.text_limit :name, 255 - end - end - - context 'when with_lock_retries re-runs the block' do - it 'only creates constraint for unique definitions' do - expected_sql = <<~SQL - ALTER TABLE "#{table_name}"\nADD CONSTRAINT "check_cda6f69506" CHECK (char_length("name") <= 255) - SQL - - expect(model).to receive(:create_table).twice.and_call_original - - expect(model).to receive(:execute).with(expected_sql).and_raise(ActiveRecord::LockWaitTimeout) - expect(model).to receive(:execute).with(expected_sql).and_call_original - - model.create_table_with_constraints table_name do |t| - t.timestamps_with_timezone - t.integer :some_id, null: false - t.boolean :active, null: false, default: true - t.text :name - - t.text_limit :name, 255 - end - - expect_table_columns_to_match(column_attributes, table_name) - - expect_check_constraint(table_name, 'check_cda6f69506', 'char_length(name) <= 255') - end - end - - context 'when constraints are given invalid names' do - let(:expected_max_length) { described_class::MAX_IDENTIFIER_NAME_LENGTH } - let(:expected_error_message) { "The maximum allowed constraint name is #{expected_max_length} characters" } - - context 'when the explicit text limit name is not valid' do - it 'raises an error' do - too_long_length = expected_max_length + 1 - - expect do - model.create_table_with_constraints table_name do |t| - t.timestamps_with_timezone - t.integer :some_id, null: false - t.boolean :active, null: false, default: true - t.text :name - - t.text_limit :name, 255, name: ('a' * too_long_length) - t.check_constraint :some_id_is_positive, 'some_id > 0' - end - end.to raise_error(expected_error_message) - end - end - - context 'when a check constraint name is not valid' do - it 'raises an error' do - too_long_length = expected_max_length + 1 - - expect do - model.create_table_with_constraints table_name do |t| - t.timestamps_with_timezone - t.integer :some_id, null: false - t.boolean :active, null: false, default: true - t.text :name - - t.text_limit :name, 255 - t.check_constraint ('a' * too_long_length), 'some_id > 0' - end - end.to raise_error(expected_error_message) - end - end - end - end - end - describe '#add_concurrent_index' do context 'outside a transaction' do before do @@ -392,7 +243,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers do context 'when targeting a partition table' do let(:schema) { 'public' } - let(:name) { '_test_partition_01' } + let(:name) { :_test_partition_01 } let(:identifier) { "#{schema}.#{name}" } before do @@ -471,10 +322,10 @@ RSpec.describe Gitlab::Database::MigrationHelpers do context 'when targeting a partition table' do let(:schema) { 'public' } - let(:partition_table_name) { '_test_partition_01' } + let(:partition_table_name) { :_test_partition_01 } let(:identifier) { "#{schema}.#{partition_table_name}" } - let(:index_name) { '_test_partitioned_index' } - let(:partition_index_name) { '_test_partition_01_partition_id_idx' } + let(:index_name) { :_test_partitioned_index } + let(:partition_index_name) { :_test_partition_01_partition_id_idx } let(:column_name) { 'partition_id' } before do @@ -544,10 +395,10 @@ RSpec.describe Gitlab::Database::MigrationHelpers do context 'when targeting a partition table' do let(:schema) { 'public' } - let(:partition_table_name) { '_test_partition_01' } + let(:partition_table_name) { :_test_partition_01 } let(:identifier) { "#{schema}.#{partition_table_name}" } - let(:index_name) { '_test_partitioned_index' } - let(:partition_index_name) { '_test_partition_01_partition_id_idx' } + let(:index_name) { :_test_partitioned_index } + let(:partition_index_name) { :_test_partition_01_partition_id_idx } before do model.execute(<<~SQL) @@ -928,13 +779,13 @@ RSpec.describe Gitlab::Database::MigrationHelpers do it 'references the custom target columns when provided', :aggregate_failures do expect(model).to receive(:with_lock_retries).and_yield expect(model).to receive(:execute).with( - "ALTER TABLE projects\n" \ - "ADD CONSTRAINT fk_multiple_columns\n" \ - "FOREIGN KEY \(partition_number, user_id\)\n" \ - "REFERENCES users \(partition_number, id\)\n" \ - "ON UPDATE CASCADE\n" \ - "ON DELETE CASCADE\n" \ - "NOT VALID;\n" + "ALTER TABLE projects " \ + "ADD CONSTRAINT fk_multiple_columns " \ + "FOREIGN KEY \(partition_number, user_id\) " \ + "REFERENCES users \(partition_number, id\) " \ + "ON UPDATE CASCADE " \ + "ON DELETE CASCADE " \ + "NOT VALID;" ) model.add_concurrent_foreign_key( @@ -979,6 +830,80 @@ RSpec.describe Gitlab::Database::MigrationHelpers do end end end + + context 'when creating foreign key on a partitioned table' do + let(:source) { :_test_source_partitioned_table } + let(:dest) { :_test_dest_partitioned_table } + let(:args) { [source, dest] } + let(:options) { { column: [:partition_id, :owner_id], target_column: [:partition_id, :id] } } + + before do + model.execute(<<~SQL) + CREATE TABLE public.#{source} ( + id serial NOT NULL, + partition_id serial NOT NULL, + owner_id bigint NOT NULL, + PRIMARY KEY (id, partition_id) + ) PARTITION BY LIST(partition_id); + + CREATE TABLE #{source}_1 + PARTITION OF public.#{source} + FOR VALUES IN (1); + + CREATE TABLE public.#{dest} ( + id serial NOT NULL, + partition_id serial NOT NULL, + PRIMARY KEY (id, partition_id) + ); + SQL + end + + it 'creates the FK without using NOT VALID', :aggregate_failures do + allow(model).to receive(:execute).and_call_original + + expect(model).to receive(:with_lock_retries).and_yield + + expect(model).to receive(:execute).with( + "ALTER TABLE #{source} " \ + "ADD CONSTRAINT fk_multiple_columns " \ + "FOREIGN KEY \(partition_id, owner_id\) " \ + "REFERENCES #{dest} \(partition_id, id\) " \ + "ON UPDATE CASCADE ON DELETE CASCADE ;" + ) + + model.add_concurrent_foreign_key( + *args, + name: :fk_multiple_columns, + on_update: :cascade, + allow_partitioned: true, + **options + ) + end + + it 'raises an error if allow_partitioned is not set' do + expect(model).not_to receive(:with_lock_retries).and_yield + expect(model).not_to receive(:execute).with(/FOREIGN KEY/) + + expect { model.add_concurrent_foreign_key(*args, **options) } + .to raise_error ArgumentError, /use add_concurrent_partitioned_foreign_key/ + end + + context 'when the reverse_lock_order flag is set' do + it 'explicitly locks the tables in target-source order', :aggregate_failures do + expect(model).to receive(:with_lock_retries).and_call_original + expect(model).to receive(:disable_statement_timeout).and_call_original + expect(model).to receive(:statement_timeout_disabled?).and_return(false) + expect(model).to receive(:execute).with(/SET statement_timeout TO/) + expect(model).to receive(:execute).ordered.with(/VALIDATE CONSTRAINT/) + expect(model).to receive(:execute).ordered.with(/RESET statement_timeout/) + + expect(model).to receive(:execute).with("LOCK TABLE #{dest}, #{source} IN ACCESS EXCLUSIVE MODE") + expect(model).to receive(:execute).with(/REFERENCES #{dest} \(partition_id, id\)/) + + model.add_concurrent_foreign_key(*args, reverse_lock_order: true, allow_partitioned: true, **options) + end + end + end end end @@ -1047,8 +972,10 @@ RSpec.describe Gitlab::Database::MigrationHelpers do end describe '#foreign_key_exists?' do - let(:referenced_table_name) { '_test_gitlab_main_referenced' } - let(:referencing_table_name) { '_test_gitlab_main_referencing' } + let(:referenced_table_name) { :_test_gitlab_main_referenced } + let(:referencing_table_name) { :_test_gitlab_main_referencing } + let(:schema) { 'public' } + let(:identifier) { "#{schema}.#{referencing_table_name}" } before do model.connection.execute(<<~SQL) @@ -1085,6 +1012,10 @@ RSpec.describe Gitlab::Database::MigrationHelpers do expect(model.foreign_key_exists?(referencing_table_name, target_table)).to be_truthy end + it 'finds existing foreign_keys by identifier' do + expect(model.foreign_key_exists?(identifier, target_table)).to be_truthy + end + it 'compares by column name if given' do expect(model.foreign_key_exists?(referencing_table_name, target_table, column: :user_id)).to be_falsey end @@ -1119,6 +1050,38 @@ RSpec.describe Gitlab::Database::MigrationHelpers do it_behaves_like 'foreign key checks' end + context 'if the schema cache does not include the constrained_columns column' do + let(:target_table) { nil } + + around do |ex| + model.transaction do + require_migration!('add_columns_to_postgres_foreign_keys') + AddColumnsToPostgresForeignKeys.new.down + Gitlab::Database::PostgresForeignKey.reset_column_information + Gitlab::Database::PostgresForeignKey.columns_hash # Force populate the column hash in the old schema + AddColumnsToPostgresForeignKeys.new.up + + # Rolling back reverts the schema cache information, so we need to run the example here before the rollback. + ex.run + + raise ActiveRecord::Rollback + end + + # make sure that we're resetting the schema cache here so that we don't leak the change to other tests. + Gitlab::Database::PostgresForeignKey.reset_column_information + # Double-check that the column information is back to normal + expect(Gitlab::Database::PostgresForeignKey.columns_hash.keys).to include('constrained_columns') + end + + # This test verifies that the situation we're trying to set up for the shared examples is actually being + # set up correctly + it 'correctly sets up the test without the column in the columns_hash' do + expect(Gitlab::Database::PostgresForeignKey.columns_hash.keys).not_to include('constrained_columns') + end + + it_behaves_like 'foreign key checks' + end + it 'compares by target table if no column given' do expect(model.foreign_key_exists?(:projects, :other_table)).to be_falsey end @@ -1129,8 +1092,8 @@ RSpec.describe Gitlab::Database::MigrationHelpers do end context 'with foreign key using multiple columns' do - let(:p_referenced_table_name) { '_test_gitlab_main_p_referenced' } - let(:p_referencing_table_name) { '_test_gitlab_main_p_referencing' } + let(:p_referenced_table_name) { :_test_gitlab_main_p_referenced } + let(:p_referencing_table_name) { :_test_gitlab_main_p_referencing } before do model.connection.execute(<<~SQL) @@ -1254,7 +1217,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers do end context 'when the table is write-locked' do - let(:test_table) { '_test_table' } + let(:test_table) { :_test_table } let(:lock_writes_manager) do Gitlab::Database::LockWritesManager.new( table_name: test_table, @@ -1436,7 +1399,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers do end context 'when the table in the other database is write-locked' do - let(:test_table) { '_test_table' } + let(:test_table) { :_test_table } let(:lock_writes_manager) do Gitlab::Database::LockWritesManager.new( table_name: test_table, @@ -2129,7 +2092,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers do end describe '#create_temporary_columns_and_triggers' do - let(:table) { :test_table } + let(:table) { :_test_table } let(:column) { :id } let(:mappings) do { @@ -2223,7 +2186,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers do end describe '#initialize_conversion_of_integer_to_bigint' do - let(:table) { :test_table } + let(:table) { :_test_table } let(:column) { :id } let(:tmp_column) { model.convert_to_bigint_column(column) } @@ -2308,7 +2271,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers do end describe '#restore_conversion_of_integer_to_bigint' do - let(:table) { :test_table } + let(:table) { :_test_table } let(:column) { :id } let(:tmp_column) { model.convert_to_bigint_column(column) } @@ -2363,7 +2326,7 @@ RSpec.describe Gitlab::Database::MigrationHelpers do end describe '#revert_initialize_conversion_of_integer_to_bigint' do - let(:table) { :test_table } + let(:table) { :_test_table } before do model.create_table table, id: false do |t| @@ -2794,39 +2757,39 @@ RSpec.describe Gitlab::Database::MigrationHelpers do describe '#add_primary_key_using_index' do it "executes the statement to add the primary key" do - expect(model).to receive(:execute).with /ALTER TABLE "test_table" ADD CONSTRAINT "old_name" PRIMARY KEY USING INDEX "new_name"/ + expect(model).to receive(:execute).with /ALTER TABLE "_test_table" ADD CONSTRAINT "old_name" PRIMARY KEY USING INDEX "new_name"/ - model.add_primary_key_using_index(:test_table, :old_name, :new_name) + model.add_primary_key_using_index(:_test_table, :old_name, :new_name) end end context 'when changing the primary key of a given table' do before do - model.create_table(:test_table, primary_key: :id) do |t| + model.create_table(:_test_table, primary_key: :id) do |t| t.integer :partition_number, default: 1 end - model.add_index(:test_table, :id, unique: true, name: :old_index_name) - model.add_index(:test_table, [:id, :partition_number], unique: true, name: :new_index_name) + model.add_index(:_test_table, :id, unique: true, name: :old_index_name) + model.add_index(:_test_table, [:id, :partition_number], unique: true, name: :new_index_name) end describe '#swap_primary_key' do it 'executes statements to swap primary key', :aggregate_failures do expect(model).to receive(:with_lock_retries).with(raise_on_exhaustion: true).ordered.and_yield - expect(model).to receive(:execute).with(/ALTER TABLE "test_table" DROP CONSTRAINT "test_table_pkey" CASCADE/).and_call_original - expect(model).to receive(:execute).with(/ALTER TABLE "test_table" ADD CONSTRAINT "test_table_pkey" PRIMARY KEY USING INDEX "new_index_name"/).and_call_original + expect(model).to receive(:execute).with(/ALTER TABLE "_test_table" DROP CONSTRAINT "_test_table_pkey" CASCADE/).and_call_original + expect(model).to receive(:execute).with(/ALTER TABLE "_test_table" ADD CONSTRAINT "_test_table_pkey" PRIMARY KEY USING INDEX "new_index_name"/).and_call_original - model.swap_primary_key(:test_table, :test_table_pkey, :new_index_name) + model.swap_primary_key(:_test_table, :_test_table_pkey, :new_index_name) end context 'when new index does not exist' do before do - model.remove_index(:test_table, column: [:id, :partition_number]) + model.remove_index(:_test_table, column: [:id, :partition_number]) end it 'raises ActiveRecord::StatementInvalid' do expect do - model.swap_primary_key(:test_table, :test_table_pkey, :new_index_name) + model.swap_primary_key(:_test_table, :_test_table_pkey, :new_index_name) end.to raise_error(ActiveRecord::StatementInvalid) end end @@ -2835,27 +2798,27 @@ RSpec.describe Gitlab::Database::MigrationHelpers do describe '#unswap_primary_key' do it 'executes statements to unswap primary key' do expect(model).to receive(:with_lock_retries).with(raise_on_exhaustion: true).ordered.and_yield - expect(model).to receive(:execute).with(/ALTER TABLE "test_table" DROP CONSTRAINT "test_table_pkey" CASCADE/).ordered.and_call_original - expect(model).to receive(:execute).with(/ALTER TABLE "test_table" ADD CONSTRAINT "test_table_pkey" PRIMARY KEY USING INDEX "old_index_name"/).ordered.and_call_original + expect(model).to receive(:execute).with(/ALTER TABLE "_test_table" DROP CONSTRAINT "_test_table_pkey" CASCADE/).ordered.and_call_original + expect(model).to receive(:execute).with(/ALTER TABLE "_test_table" ADD CONSTRAINT "_test_table_pkey" PRIMARY KEY USING INDEX "old_index_name"/).ordered.and_call_original - model.unswap_primary_key(:test_table, :test_table_pkey, :old_index_name) + model.unswap_primary_key(:_test_table, :_test_table_pkey, :old_index_name) end end end describe '#drop_sequence' do it "executes the statement to drop the sequence" do - expect(model).to receive(:execute).with /ALTER TABLE "test_table" ALTER COLUMN "test_column" DROP DEFAULT;\nDROP SEQUENCE IF EXISTS "test_table_id_seq"/ + expect(model).to receive(:execute).with /ALTER TABLE "_test_table" ALTER COLUMN "test_column" DROP DEFAULT;\nDROP SEQUENCE IF EXISTS "_test_table_id_seq"/ - model.drop_sequence(:test_table, :test_column, :test_table_id_seq) + model.drop_sequence(:_test_table, :test_column, :_test_table_id_seq) end end describe '#add_sequence' do it "executes the statement to add the sequence" do - expect(model).to receive(:execute).with "CREATE SEQUENCE \"test_table_id_seq\" START 1;\nALTER TABLE \"test_table\" ALTER COLUMN \"test_column\" SET DEFAULT nextval(\'test_table_id_seq\')\n" + expect(model).to receive(:execute).with "CREATE SEQUENCE \"_test_table_id_seq\" START 1;\nALTER TABLE \"_test_table\" ALTER COLUMN \"test_column\" SET DEFAULT nextval(\'_test_table_id_seq\')\n" - model.add_sequence(:test_table, :test_column, :test_table_id_seq, 1) + model.add_sequence(:_test_table, :test_column, :_test_table_id_seq, 1) end end @@ -2890,4 +2853,18 @@ RSpec.describe Gitlab::Database::MigrationHelpers do it { is_expected.to be_falsey } end end + + describe "#table_partitioned?" do + subject { model.table_partitioned?(table_name) } + + let(:table_name) { 'p_ci_builds_metadata' } + + it { is_expected.to be_truthy } + + context 'with a non-partitioned table' do + let(:table_name) { 'users' } + + it { is_expected.to be_falsey } + end + end end diff --git a/spec/lib/gitlab/database/migrations/batched_background_migration_helpers_spec.rb b/spec/lib/gitlab/database/migrations/batched_background_migration_helpers_spec.rb index 3e249b14f2e..f5ce207773f 100644 --- a/spec/lib/gitlab/database/migrations/batched_background_migration_helpers_spec.rb +++ b/spec/lib/gitlab/database/migrations/batched_background_migration_helpers_spec.rb @@ -482,16 +482,46 @@ RSpec.describe Gitlab::Database::Migrations::BatchedBackgroundMigrationHelpers d .not_to raise_error end - it 'logs a warning when migration does not exist' do - expect(Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas).to receive(:require_dml_mode!) + context 'when specified migration does not exist' do + let(:lab_key) { 'DBLAB_ENVIRONMENT' } - create(:batched_background_migration, :active, migration_attributes.merge(gitlab_schema: :gitlab_something_else)) + context 'when DBLAB_ENVIRONMENT is not set' do + it 'logs a warning' do + stub_env(lab_key, nil) + expect(Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas).to receive(:require_dml_mode!) - expect(Gitlab::AppLogger).to receive(:warn) - .with("Could not find batched background migration for the given configuration: #{configuration}") + create(:batched_background_migration, :active, migration_attributes.merge(gitlab_schema: :gitlab_something_else)) - expect { ensure_batched_background_migration_is_finished } - .not_to raise_error + expect(Gitlab::AppLogger).to receive(:warn) + .with("Could not find batched background migration for the given configuration: #{configuration}") + + expect { ensure_batched_background_migration_is_finished } + .not_to raise_error + end + end + + context 'when DBLAB_ENVIRONMENT is set' do + it 'raises an error' do + stub_env(lab_key, 'foo') + expect(Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas).to receive(:require_dml_mode!) + + create(:batched_background_migration, :active, migration_attributes.merge(gitlab_schema: :gitlab_something_else)) + + expect { ensure_batched_background_migration_is_finished } + .to raise_error(Gitlab::Database::Migrations::BatchedBackgroundMigrationHelpers::NonExistentMigrationError) + end + end + end + + context 'when within transaction' do + before do + allow(migration).to receive(:transaction_open?).and_return(true) + end + + it 'does raise an exception' do + expect { ensure_batched_background_migration_is_finished } + .to raise_error /`ensure_batched_background_migration_is_finished` cannot be run inside a transaction./ + end end it 'finalizes the migration' do diff --git a/spec/lib/gitlab/database/migrations/constraints_helpers_spec.rb b/spec/lib/gitlab/database/migrations/constraints_helpers_spec.rb index 6848fc85aa1..07d913cf5cc 100644 --- a/spec/lib/gitlab/database/migrations/constraints_helpers_spec.rb +++ b/spec/lib/gitlab/database/migrations/constraints_helpers_spec.rb @@ -23,43 +23,46 @@ RSpec.describe Gitlab::Database::Migrations::ConstraintsHelpers do end end - describe '#check_constraint_exists?' do + describe '#check_constraint_exists?', :aggregate_failures do before do - ActiveRecord::Migration.connection.execute( - 'ALTER TABLE projects ADD CONSTRAINT check_1 CHECK (char_length(path) <= 5) NOT VALID' - ) - - ActiveRecord::Migration.connection.execute( - 'CREATE SCHEMA new_test_schema' - ) - - ActiveRecord::Migration.connection.execute( - 'CREATE TABLE new_test_schema.projects (id integer, name character varying)' - ) - - ActiveRecord::Migration.connection.execute( - 'ALTER TABLE new_test_schema.projects ADD CONSTRAINT check_2 CHECK (char_length(name) <= 5)' - ) + ActiveRecord::Migration.connection.execute(<<~SQL) + ALTER TABLE projects ADD CONSTRAINT check_1 CHECK (char_length(path) <= 5) NOT VALID; + CREATE SCHEMA new_test_schema; + CREATE TABLE new_test_schema.projects (id integer, name character varying); + ALTER TABLE new_test_schema.projects ADD CONSTRAINT check_2 CHECK (char_length(name) <= 5); + SQL end it 'returns true if a constraint exists' do expect(model) .to be_check_constraint_exists(:projects, 'check_1') + + expect(described_class) + .to be_check_constraint_exists(:projects, 'check_1', connection: model.connection) end it 'returns false if a constraint does not exist' do expect(model) .not_to be_check_constraint_exists(:projects, 'this_does_not_exist') + + expect(described_class) + .not_to be_check_constraint_exists(:projects, 'this_does_not_exist', connection: model.connection) end it 'returns false if a constraint with the same name exists in another table' do expect(model) .not_to be_check_constraint_exists(:users, 'check_1') + + expect(described_class) + .not_to be_check_constraint_exists(:users, 'check_1', connection: model.connection) end it 'returns false if a constraint with the same name exists for the same table in another schema' do expect(model) .not_to be_check_constraint_exists(:projects, 'check_2') + + expect(described_class) + .not_to be_check_constraint_exists(:projects, 'check_2', connection: model.connection) end end diff --git a/spec/lib/gitlab/database/migrations/instrumentation_spec.rb b/spec/lib/gitlab/database/migrations/instrumentation_spec.rb index 4f347034c0b..0b25389c667 100644 --- a/spec/lib/gitlab/database/migrations/instrumentation_spec.rb +++ b/spec/lib/gitlab/database/migrations/instrumentation_spec.rb @@ -18,7 +18,9 @@ RSpec.describe Gitlab::Database::Migrations::Instrumentation do let(:migration_name) { 'test' } let(:migration_version) { '12345' } let(:migration_meta) { { 'max_batch_size' => 1, 'total_tuple_count' => 10, 'interval' => 60 } } - let(:expected_json_keys) { %w[version name walltime success total_database_size_change query_statistics] } + let(:expected_json_keys) do + %w[version name walltime success total_database_size_change query_statistics error_message] + end it 'executes the given block' do expect do |b| @@ -90,16 +92,14 @@ RSpec.describe Gitlab::Database::Migrations::Instrumentation do end context 'upon failure' do - where(exception: ['something went wrong', SystemStackError, Interrupt]) + where(:exception, :error_message) do + [[StandardError, 'something went wrong'], [ActiveRecord::StatementTimeout, 'timeout']] + end with_them do subject(:observe) do instrumentation.observe(version: migration_version, name: migration_name, - connection: connection, meta: migration_meta) { raise exception } - end - - it 'raises the exception' do - expect { observe }.to raise_error(exception) + connection: connection, meta: migration_meta) { raise exception, error_message } end context 'retrieving observations' do @@ -107,10 +107,6 @@ RSpec.describe Gitlab::Database::Migrations::Instrumentation do before do observe - # rubocop:disable Lint/RescueException - rescue Exception - # rubocop:enable Lint/RescueException - # ignore (we expect this exception) end it 'records a valid observation', :aggregate_failures do @@ -118,6 +114,7 @@ RSpec.describe Gitlab::Database::Migrations::Instrumentation do expect(subject['success']).to be_falsey expect(subject['version']).to eq(migration_version) expect(subject['name']).to eq(migration_name) + expect(subject['error_message']).to eq(error_message) end it 'transforms observation to expected json' do diff --git a/spec/lib/gitlab/database/migrations/pg_backend_pid_spec.rb b/spec/lib/gitlab/database/migrations/pg_backend_pid_spec.rb new file mode 100644 index 00000000000..33e83ea2575 --- /dev/null +++ b/spec/lib/gitlab/database/migrations/pg_backend_pid_spec.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::Migrations::PgBackendPid, feature_category: :database do + describe Gitlab::Database::Migrations::PgBackendPid::MigratorPgBackendPid do + let(:klass) do + Class.new do + def with_advisory_lock_connection + yield :conn + end + end + end + + it 're-yields with same arguments and wraps it with calls to .say' do + patched_instance = klass.prepend(described_class).new + expect(Gitlab::Database::Migrations::PgBackendPid).to receive(:say).twice + + expect { |b| patched_instance.with_advisory_lock_connection(&b) }.to yield_with_args(:conn) + end + end + + describe '.patch!' do + it 'patches ActiveRecord::Migrator' do + expect(ActiveRecord::Migrator).to receive(:prepend).with(described_class::MigratorPgBackendPid) + + described_class.patch! + end + end + + describe '.say' do + it 'outputs the connection information' do + conn = ActiveRecord::Base.connection + + expect(conn).to receive(:object_id).and_return(9876) + expect(conn).to receive(:select_value).with('SELECT pg_backend_pid()').and_return(12345) + expect(Gitlab::Database).to receive(:db_config_name).with(conn).and_return('main') + + expected_output = "main: == [advisory_lock_connection] object_id: 9876, pg_backend_pid: 12345\n" + + expect { described_class.say(conn) }.to output(expected_output).to_stdout + end + + it 'outputs nothing if ActiveRecord::Migration.verbose is false' do + conn = ActiveRecord::Base.connection + + allow(ActiveRecord::Migration).to receive(:verbose).and_return(false) + + expect { described_class.say(conn) }.not_to output.to_stdout + end + end +end diff --git a/spec/lib/gitlab/database/migrations/runner_backoff/active_record_mixin_spec.rb b/spec/lib/gitlab/database/migrations/runner_backoff/active_record_mixin_spec.rb new file mode 100644 index 00000000000..ddf11598d21 --- /dev/null +++ b/spec/lib/gitlab/database/migrations/runner_backoff/active_record_mixin_spec.rb @@ -0,0 +1,106 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::Migrations::RunnerBackoff::ActiveRecordMixin, feature_category: :database do + let(:migration_class) { Gitlab::Database::Migration[2.1] } + + describe described_class::ActiveRecordMigrationProxyRunnerBackoff do + let(:migration) { instance_double(migration_class) } + + let(:class_def) do + Class.new do + attr_reader :migration + + def initialize(migration) + @migration = migration + end + end.prepend(described_class) + end + + describe '#enable_runner_backoff?' do + subject { class_def.new(migration).enable_runner_backoff? } + + it 'delegates to #migration' do + expect(migration).to receive(:enable_runner_backoff?).and_return(true) + + expect(subject).to eq(true) + end + + it 'returns false if migration does not implement it' do + expect(migration).to receive(:respond_to?).with(:enable_runner_backoff?).and_return(false) + + expect(subject).to eq(false) + end + end + end + + describe described_class::ActiveRecordMigratorRunnerBackoff do + let(:class_def) do + Class.new do + attr_reader :receiver + + def initialize(receiver) + @receiver = receiver + end + + def execute_migration_in_transaction(migration) + receiver.execute_migration_in_transaction(migration) + end + end.prepend(described_class) + end + + let(:receiver) { instance_double(ActiveRecord::Migrator, 'receiver') } + + subject { class_def.new(receiver) } + + before do + allow(migration).to receive(:name).and_return('TestClass') + allow(receiver).to receive(:execute_migration_in_transaction) + end + + context 'with runner backoff disabled' do + let(:migration) { instance_double(migration_class, enable_runner_backoff?: false) } + + it 'calls super method' do + expect(receiver).to receive(:execute_migration_in_transaction).with(migration) + + subject.execute_migration_in_transaction(migration) + end + end + + context 'with runner backoff enabled' do + let(:migration) { instance_double(migration_class, enable_runner_backoff?: true) } + + it 'calls super method' do + expect(Gitlab::Database::Migrations::RunnerBackoff::Communicator) + .to receive(:execute_with_lock).with(migration).and_call_original + + expect(receiver).to receive(:execute_migration_in_transaction) + .with(migration) + + subject.execute_migration_in_transaction(migration) + end + end + end + + describe '.patch!' do + subject { described_class.patch! } + + it 'patches MigrationProxy' do + expect(ActiveRecord::MigrationProxy) + .to receive(:prepend) + .with(described_class::ActiveRecordMigrationProxyRunnerBackoff) + + subject + end + + it 'patches Migrator' do + expect(ActiveRecord::Migrator) + .to receive(:prepend) + .with(described_class::ActiveRecordMigratorRunnerBackoff) + + subject + end + end +end diff --git a/spec/lib/gitlab/database/migrations/runner_backoff/communicator_spec.rb b/spec/lib/gitlab/database/migrations/runner_backoff/communicator_spec.rb new file mode 100644 index 00000000000..cfc3fb398e2 --- /dev/null +++ b/spec/lib/gitlab/database/migrations/runner_backoff/communicator_spec.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::Migrations::RunnerBackoff::Communicator, :clean_gitlab_redis_shared_state, feature_category: :database do + let(:migration) { instance_double(Gitlab::Database::Migration[2.1], name: 'TestClass') } + + describe '.execute_with_lock' do + it 'delegates to a new instance object' do + expect_next_instance_of(described_class, migration) do |communicator| + expect(communicator).to receive(:execute_with_lock).and_call_original + end + + expect { |b| described_class.execute_with_lock(migration, &b) }.to yield_control + end + end + + describe '.backoff_runner?' do + subject { described_class.backoff_runner? } + + it { is_expected.to be_falsey } + + it 'is true when the lock is held' do + described_class.execute_with_lock(migration) do + is_expected.to be_truthy + end + end + + it 'reads from Redis' do + recorder = RedisCommands::Recorder.new { subject } + expect(recorder.log).to include([:exists, 'gitlab:exclusive_lease:gitlab/database/migration/runner/backoff']) + end + + context 'with runner_migrations_backoff disabled' do + before do + stub_feature_flags(runner_migrations_backoff: false) + end + + it 'is false when the lock is held' do + described_class.execute_with_lock(migration) do + is_expected.to be_falsey + end + end + end + end + + describe '#execute_with_lock' do + include ExclusiveLeaseHelpers + + let(:communicator) { described_class.new(migration) } + let!(:lease) { stub_exclusive_lease(described_class::KEY, :uuid, timeout: described_class::EXPIRY) } + + it { expect { |b| communicator.execute_with_lock(&b) }.to yield_control } + + it 'raises error if it can not set the key' do + expect(lease).to receive(:try_obtain).ordered.and_return(false) + + expect { communicator.execute_with_lock { 1 / 0 } }.to raise_error 'Could not set backoff key' + end + + it 'removes the lease after executing the migration' do + expect(lease).to receive(:try_obtain).ordered.and_return(true) + expect(lease).to receive(:cancel).ordered.and_return(true) + + expect { communicator.execute_with_lock }.not_to raise_error + end + + context 'with logger' do + let(:logger) { instance_double(Gitlab::AppLogger) } + let(:communicator) { described_class.new(migration, logger: logger) } + + it 'logs messages around execution' do + expect(logger).to receive(:info).ordered + .with({ class: 'TestClass', message: 'Executing migration with Runner backoff' }) + expect(logger).to receive(:info).ordered + .with({ class: 'TestClass', message: 'Runner backoff key is set' }) + expect(logger).to receive(:info).ordered + .with({ class: 'TestClass', message: 'Runner backoff key was removed' }) + + communicator.execute_with_lock + end + end + end +end diff --git a/spec/lib/gitlab/database/migrations/runner_backoff/migration_helpers_spec.rb b/spec/lib/gitlab/database/migrations/runner_backoff/migration_helpers_spec.rb new file mode 100644 index 00000000000..9eefc34a7cc --- /dev/null +++ b/spec/lib/gitlab/database/migrations/runner_backoff/migration_helpers_spec.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::Migrations::RunnerBackoff::MigrationHelpers, feature_category: :database do + let(:class_def) do + Class.new.prepend(described_class) + end + + describe '.enable_runner_backoff!' do + it 'sets the flag' do + expect { class_def.enable_runner_backoff! } + .to change { class_def.enable_runner_backoff? } + .from(false).to(true) + end + end + + describe '.enable_runner_backoff?' do + subject { class_def.enable_runner_backoff? } + + it { is_expected.to be_falsy } + + it 'returns true if the flag is set' do + class_def.enable_runner_backoff! + + is_expected.to be_truthy + end + end + + describe '#enable_runner_backoff?' do + subject { class_def.new.enable_runner_backoff? } + + it { is_expected.to be_falsy } + + it 'returns true if the flag is set' do + class_def.enable_runner_backoff! + + is_expected.to be_truthy + end + end +end diff --git a/spec/lib/gitlab/database/migrations/runner_spec.rb b/spec/lib/gitlab/database/migrations/runner_spec.rb index 66eb5a5de51..7c71076e8f3 100644 --- a/spec/lib/gitlab/database/migrations/runner_spec.rb +++ b/spec/lib/gitlab/database/migrations/runner_spec.rb @@ -65,7 +65,7 @@ RSpec.describe Gitlab::Database::Migrations::Runner, :reestablished_active_recor end before do - skip_if_multiple_databases_not_setup unless database == :main + skip_if_shared_database(database) stub_const('Gitlab::Database::Migrations::Runner::BASE_RESULT_DIR', base_result_dir) allow(ActiveRecord::Migrator).to receive(:new) do |dir, _all_migrations, _schema_migration_class, version_to_migrate| diff --git a/spec/lib/gitlab/database/migrations/sidekiq_helpers_spec.rb b/spec/lib/gitlab/database/migrations/sidekiq_helpers_spec.rb index fb1cb46171f..bf3a9e16548 100644 --- a/spec/lib/gitlab/database/migrations/sidekiq_helpers_spec.rb +++ b/spec/lib/gitlab/database/migrations/sidekiq_helpers_spec.rb @@ -78,158 +78,174 @@ RSpec.describe Gitlab::Database::Migrations::SidekiqHelpers do clear_queues end - context "when the constant is not defined" do - it "doesn't try to delete it" do - my_non_constant = +"SomeThingThatIsNotAConstant" + context 'when inside a transaction' do + it 'raises RuntimeError' do + expect(model).to receive(:transaction_open?).and_return(true) - expect(Sidekiq::Queue).not_to receive(:new).with(any_args) - model.sidekiq_remove_jobs(job_klasses: [my_non_constant]) + expect { model.sidekiq_remove_jobs(job_klasses: [worker.name]) } + .to raise_error(RuntimeError) end end - context "when the constant is defined" do - it "will use it find job instances to delete" do - my_constant = worker.name - expect(Sidekiq::Queue) - .to receive(:new) - .with(worker.queue) - .and_call_original - model.sidekiq_remove_jobs(job_klasses: [my_constant]) + context 'when outside a transaction' do + before do + allow(model).to receive(:transaction_open?).and_return(false) + allow(model).to receive(:disable_statement_timeout).and_call_original end - end - it "removes all related job instances from the job classes' queues" do - worker.perform_async - worker_two.perform_async - same_queue_different_worker.perform_async - unrelated_worker.perform_async - - worker_queue = Sidekiq::Queue.new(worker.queue) - worker_two_queue = Sidekiq::Queue.new(worker_two.queue) - unrelated_queue = Sidekiq::Queue.new(unrelated_worker.queue) - - expect(worker_queue.size).to eq(2) - expect(worker_two_queue.size).to eq(1) - expect(unrelated_queue.size).to eq(1) - - model.sidekiq_remove_jobs(job_klasses: [worker.name, worker_two.name]) - - expect(worker_queue.size).to eq(1) - expect(worker_two_queue.size).to eq(0) - expect(worker_queue.map(&:klass)).not_to include(worker.name) - expect(worker_queue.map(&:klass)).to include( - same_queue_different_worker.name - ) - expect(worker_two_queue.map(&:klass)).not_to include(worker_two.name) - expect(unrelated_queue.size).to eq(1) - end + context "when the constant is not defined" do + it "doesn't try to delete it" do + my_non_constant = +"SomeThingThatIsNotAConstant" - context "when job instances are in the scheduled set" do - it "removes all related job instances from the scheduled set" do - worker.perform_in(1.hour) - worker_two.perform_in(1.hour) - unrelated_worker.perform_in(1.hour) + expect(Sidekiq::Queue).not_to receive(:new).with(any_args) + model.sidekiq_remove_jobs(job_klasses: [my_non_constant]) + end + end - scheduled = Sidekiq::ScheduledSet.new + context "when the constant is defined" do + it "will use it find job instances to delete" do + my_constant = worker.name + expect(Sidekiq::Queue) + .to receive(:new) + .with(worker.queue) + .and_call_original + model.sidekiq_remove_jobs(job_klasses: [my_constant]) + end + end - expect(scheduled.size).to eq(3) - expect(scheduled.map(&:klass)).to include( - worker.name, - worker_two.name, - unrelated_worker.name - ) + it "removes all related job instances from the job classes' queues" do + worker.perform_async + worker_two.perform_async + same_queue_different_worker.perform_async + unrelated_worker.perform_async + + worker_queue = Sidekiq::Queue.new(worker.queue) + worker_two_queue = Sidekiq::Queue.new(worker_two.queue) + unrelated_queue = Sidekiq::Queue.new(unrelated_worker.queue) + + expect(worker_queue.size).to eq(2) + expect(worker_two_queue.size).to eq(1) + expect(unrelated_queue.size).to eq(1) model.sidekiq_remove_jobs(job_klasses: [worker.name, worker_two.name]) - expect(scheduled.size).to eq(1) - expect(scheduled.map(&:klass)).not_to include(worker.name) - expect(scheduled.map(&:klass)).not_to include(worker_two.name) - expect(scheduled.map(&:klass)).to include(unrelated_worker.name) + expect(worker_queue.size).to eq(1) + expect(worker_two_queue.size).to eq(0) + expect(worker_queue.map(&:klass)).not_to include(worker.name) + expect(worker_queue.map(&:klass)).to include( + same_queue_different_worker.name + ) + expect(worker_two_queue.map(&:klass)).not_to include(worker_two.name) + expect(unrelated_queue.size).to eq(1) end - end - - context "when job instances are in the retry set" do - include_context "when handling retried jobs" - it "removes all related job instances from the retry set" do - retry_in(worker, 1.hour) - retry_in(worker, 2.hours) - retry_in(worker, 3.hours) - retry_in(worker_two, 4.hours) - retry_in(unrelated_worker, 5.hours) + context "when job instances are in the scheduled set" do + it "removes all related job instances from the scheduled set" do + worker.perform_in(1.hour) + worker_two.perform_in(1.hour) + unrelated_worker.perform_in(1.hour) - retries = Sidekiq::RetrySet.new + scheduled = Sidekiq::ScheduledSet.new - expect(retries.size).to eq(5) - expect(retries.map(&:klass)).to include( - worker.name, - worker_two.name, - unrelated_worker.name - ) + expect(scheduled.size).to eq(3) + expect(scheduled.map(&:klass)).to include( + worker.name, + worker_two.name, + unrelated_worker.name + ) - model.sidekiq_remove_jobs(job_klasses: [worker.name, worker_two.name]) + model.sidekiq_remove_jobs(job_klasses: [worker.name, worker_two.name]) - expect(retries.size).to eq(1) - expect(retries.map(&:klass)).not_to include(worker.name) - expect(retries.map(&:klass)).not_to include(worker_two.name) - expect(retries.map(&:klass)).to include(unrelated_worker.name) + expect(scheduled.size).to eq(1) + expect(scheduled.map(&:klass)).not_to include(worker.name) + expect(scheduled.map(&:klass)).not_to include(worker_two.name) + expect(scheduled.map(&:klass)).to include(unrelated_worker.name) + end end - end - # Imitate job deletion returning zero and then non zero. - context "when job fails to be deleted" do - let(:job_double) do - instance_double( - "Sidekiq::JobRecord", - klass: worker.name - ) - end + context "when job instances are in the retry set" do + include_context "when handling retried jobs" - context "and does not work enough times in a row before max attempts" do - it "tries the max attempts without succeeding" do - worker.perform_async + it "removes all related job instances from the retry set" do + retry_in(worker, 1.hour) + retry_in(worker, 2.hours) + retry_in(worker, 3.hours) + retry_in(worker_two, 4.hours) + retry_in(unrelated_worker, 5.hours) - allow(job_double).to receive(:delete).and_return(true) + retries = Sidekiq::RetrySet.new - # Scheduled set runs last so only need to stub out its values. - allow(Sidekiq::ScheduledSet) - .to receive(:new) - .and_return([job_double]) - - expect(model.sidekiq_remove_jobs(job_klasses: [worker.name])) - .to eq( - { - attempts: 5, - success: false - } - ) + expect(retries.size).to eq(5) + expect(retries.map(&:klass)).to include( + worker.name, + worker_two.name, + unrelated_worker.name + ) + + model.sidekiq_remove_jobs(job_klasses: [worker.name, worker_two.name]) + + expect(retries.size).to eq(1) + expect(retries.map(&:klass)).not_to include(worker.name) + expect(retries.map(&:klass)).not_to include(worker_two.name) + expect(retries.map(&:klass)).to include(unrelated_worker.name) end end - context "and then it works enough times in a row before max attempts" do - it "succeeds" do - worker.perform_async - - # attempt 1: false will increment the streak once to 1 - # attempt 2: true resets it back to 0 - # attempt 3: false will increment the streak once to 1 - # attempt 4: false will increment the streak once to 2, loop breaks - allow(job_double).to receive(:delete).and_return(false, true, false) + # Imitate job deletion returning zero and then non zero. + context "when job fails to be deleted" do + let(:job_double) do + instance_double( + "Sidekiq::JobRecord", + klass: worker.name + ) + end - worker.perform_async + context "and does not work enough times in a row before max attempts" do + it "tries the max attempts without succeeding" do + worker.perform_async + + allow(job_double).to receive(:delete).and_return(true) + + # Scheduled set runs last so only need to stub out its values. + allow(Sidekiq::ScheduledSet) + .to receive(:new) + .and_return([job_double]) + + expect(model.sidekiq_remove_jobs(job_klasses: [worker.name])) + .to eq( + { + attempts: 5, + success: false + } + ) + end + end - # Scheduled set runs last so only need to stub out its values. - allow(Sidekiq::ScheduledSet) - .to receive(:new) - .and_return([job_double]) - - expect(model.sidekiq_remove_jobs(job_klasses: [worker.name])) - .to eq( - { - attempts: 4, - success: true - } - ) + context "and then it works enough times in a row before max attempts" do + it "succeeds" do + worker.perform_async + + # attempt 1: false will increment the streak once to 1 + # attempt 2: true resets it back to 0 + # attempt 3: false will increment the streak once to 1 + # attempt 4: false will increment the streak once to 2, loop breaks + allow(job_double).to receive(:delete).and_return(false, true, false) + + worker.perform_async + + # Scheduled set runs last so only need to stub out its values. + allow(Sidekiq::ScheduledSet) + .to receive(:new) + .and_return([job_double]) + + expect(model.sidekiq_remove_jobs(job_klasses: [worker.name])) + .to eq( + { + attempts: 4, + success: true + } + ) + end end end end diff --git a/spec/lib/gitlab/database/migrations/test_batched_background_runner_spec.rb b/spec/lib/gitlab/database/migrations/test_batched_background_runner_spec.rb index 57c5011590c..6bcefa455cf 100644 --- a/spec/lib/gitlab/database/migrations/test_batched_background_runner_spec.rb +++ b/spec/lib/gitlab/database/migrations/test_batched_background_runner_spec.rb @@ -48,6 +48,7 @@ RSpec.describe Gitlab::Database::Migrations::TestBatchedBackgroundRunner, :freez let(:result_dir) { Pathname.new(Dir.mktmpdir) } let(:connection) { base_model.connection } let(:table_name) { "_test_column_copying" } + let(:num_rows_in_table) { 1000 } let(:from_id) { 0 } after do @@ -61,7 +62,7 @@ RSpec.describe Gitlab::Database::Migrations::TestBatchedBackgroundRunner, :freez data bigint default 0 ); - insert into #{table_name} (id) select i from generate_series(1, 1000) g(i); + insert into #{table_name} (id) select i from generate_series(1, #{num_rows_in_table}) g(i); SQL end @@ -134,6 +135,24 @@ RSpec.describe Gitlab::Database::Migrations::TestBatchedBackgroundRunner, :freez expect(calls).not_to be_empty end + it 'samples 1 job with a batch size higher than the table size' do + calls = [] + define_background_migration(migration_name) do |*args| + travel 1.minute + calls << args + end + + queue_migration(migration_name, table_name, :id, + job_interval: 5.minutes, + batch_size: num_rows_in_table * 2, + sub_batch_size: num_rows_in_table * 2) + + described_class.new(result_dir: result_dir, connection: connection, + from_id: from_id).run_jobs(for_duration: 3.minutes) + + expect(calls.size).to eq(1) + end + context 'with multiple jobs to run' do let(:last_id) do Gitlab::Database::SharedModel.using_connection(connection) do diff --git a/spec/lib/gitlab/database/no_cross_db_foreign_keys_spec.rb b/spec/lib/gitlab/database/no_cross_db_foreign_keys_spec.rb index f94a40c93e1..e48937037fa 100644 --- a/spec/lib/gitlab/database/no_cross_db_foreign_keys_spec.rb +++ b/spec/lib/gitlab/database/no_cross_db_foreign_keys_spec.rb @@ -16,7 +16,7 @@ RSpec.describe 'cross-database foreign keys' do end def is_cross_db?(fk_record) - Gitlab::Database::GitlabSchema.table_schemas([fk_record.from_table, fk_record.to_table]).many? + Gitlab::Database::GitlabSchema.table_schemas!([fk_record.from_table, fk_record.to_table]).many? end it 'onlies have allowed list of cross-database foreign keys', :aggregate_failures do diff --git a/spec/lib/gitlab/database/obsolete_ignored_columns_spec.rb b/spec/lib/gitlab/database/obsolete_ignored_columns_spec.rb index b39b273bba9..fa7645d581c 100644 --- a/spec/lib/gitlab/database/obsolete_ignored_columns_spec.rb +++ b/spec/lib/gitlab/database/obsolete_ignored_columns_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::ObsoleteIgnoredColumns do +RSpec.describe Gitlab::Database::ObsoleteIgnoredColumns, feature_category: :database do before do stub_const('Testing', Module.new) stub_const('Testing::MyBase', Class.new(ActiveRecord::Base)) @@ -16,11 +16,10 @@ RSpec.describe Gitlab::Database::ObsoleteIgnoredColumns do Testing.module_eval do Testing::MyBase.class_eval do + include IgnorableColumns end SomeAbstract.class_eval do - include IgnorableColumns - self.abstract_class = true self.table_name = 'projects' @@ -29,8 +28,6 @@ RSpec.describe Gitlab::Database::ObsoleteIgnoredColumns do end Testing::B.class_eval do - include IgnorableColumns - self.table_name = 'issues' ignore_column :id, :other, remove_after: '2019-01-01', remove_with: '12.0' diff --git a/spec/lib/gitlab/database/partitioning/ci_sliding_list_strategy_spec.rb b/spec/lib/gitlab/database/partitioning/ci_sliding_list_strategy_spec.rb new file mode 100644 index 00000000000..f415e892818 --- /dev/null +++ b/spec/lib/gitlab/database/partitioning/ci_sliding_list_strategy_spec.rb @@ -0,0 +1,178 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::Partitioning::CiSlidingListStrategy, feature_category: :database do + let(:connection) { ActiveRecord::Base.connection } + let(:table_name) { :_test_gitlab_ci_partitioned_test } + let(:model) { class_double(ApplicationRecord, table_name: table_name, connection: connection) } + let(:next_partition_if) { nil } + let(:detach_partition_if) { nil } + + subject(:strategy) do + described_class.new(model, :partition, + next_partition_if: next_partition_if, + detach_partition_if: detach_partition_if) + end + + before do + next if table_name.to_s.starts_with?('p_') + + connection.execute(<<~SQL) + create table #{table_name} + ( + id serial not null, + partition_id bigint not null, + created_at timestamptz not null, + primary key (id, partition_id) + ) + partition by list(partition_id); + + create table #{table_name}_100 + partition of #{table_name} for values in (100); + + create table #{table_name}_101 + partition of #{table_name} for values in (101); + SQL + end + + describe '#current_partitions' do + it 'detects both partitions' do + expect(strategy.current_partitions).to eq( + [ + Gitlab::Database::Partitioning::SingleNumericListPartition.new( + table_name, 100, partition_name: "#{table_name}_100" + ), + Gitlab::Database::Partitioning::SingleNumericListPartition.new( + table_name, 101, partition_name: "#{table_name}_101" + ) + ]) + end + end + + describe '#validate_and_fix' do + it 'does not call change_column_default' do + expect(strategy.model.connection).not_to receive(:change_column_default) + + strategy.validate_and_fix + end + end + + describe '#active_partition' do + it 'is the partition with the largest value' do + expect(strategy.active_partition.value).to eq(101) + end + end + + describe '#missing_partitions' do + context 'when next_partition_if returns true' do + let(:next_partition_if) { proc { true } } + + it 'is a partition definition for the next partition in the series' do + extra = strategy.missing_partitions + + expect(extra.length).to eq(1) + expect(extra.first.value).to eq(102) + end + end + + context 'when next_partition_if returns false' do + let(:next_partition_if) { proc { false } } + + it 'is empty' do + expect(strategy.missing_partitions).to be_empty + end + end + + context 'when there are no partitions for the table' do + it 'returns a partition for value 1' do + connection.execute("drop table #{table_name}_100; drop table #{table_name}_101;") + + missing_partitions = strategy.missing_partitions + + expect(missing_partitions.size).to eq(1) + missing_partition = missing_partitions.first + + expect(missing_partition.value).to eq(100) + end + end + end + + describe '#extra_partitions' do + context 'when all partitions are true for detach_partition_if' do + let(:detach_partition_if) { ->(_p) { true } } + + it { expect(strategy.extra_partitions).to be_empty } + end + + context 'when all partitions are false for detach_partition_if' do + let(:detach_partition_if) { proc { false } } + + it { expect(strategy.extra_partitions).to be_empty } + end + end + + describe '#initial_partition' do + it 'starts with the value 100', :aggregate_failures do + initial_partition = strategy.initial_partition + expect(initial_partition.value).to eq(100) + expect(initial_partition.table).to eq(strategy.table_name) + expect(initial_partition.partition_name).to eq("#{strategy.table_name}_100") + end + + context 'with routing tables' do + let(:table_name) { :p_test_gitlab_ci_partitioned_test } + + it 'removes the prefix', :aggregate_failures do + initial_partition = strategy.initial_partition + + expect(initial_partition.value).to eq(100) + expect(initial_partition.table).to eq(strategy.table_name) + expect(initial_partition.partition_name).to eq('test_gitlab_ci_partitioned_test_100') + end + end + end + + describe '#next_partition' do + before do + allow(strategy) + .to receive(:active_partition) + .and_return(instance_double(Gitlab::Database::Partitioning::SingleNumericListPartition, value: 105)) + end + + it 'is one after the active partition', :aggregate_failures do + next_partition = strategy.next_partition + + expect(next_partition.value).to eq(106) + expect(next_partition.table).to eq(strategy.table_name) + expect(next_partition.partition_name).to eq("#{strategy.table_name}_106") + end + + context 'with routing tables' do + let(:table_name) { :p_test_gitlab_ci_partitioned_test } + + it 'removes the prefix', :aggregate_failures do + next_partition = strategy.next_partition + + expect(next_partition.value).to eq(106) + expect(next_partition.table).to eq(strategy.table_name) + expect(next_partition.partition_name).to eq('test_gitlab_ci_partitioned_test_106') + end + end + end + + describe '#ensure_partitioning_column_ignored_or_readonly!' do + it 'does not raise when the column is not ignored' do + expect do + Class.new(ApplicationRecord) do + include PartitionedTable + + partitioned_by :partition_id, + strategy: :ci_sliding_list, + next_partition_if: proc { false }, + detach_partition_if: proc { false } + end + end.not_to raise_error + end + end +end diff --git a/spec/lib/gitlab/database/partitioning/convert_table_to_first_list_partition_spec.rb b/spec/lib/gitlab/database/partitioning/convert_table_to_first_list_partition_spec.rb deleted file mode 100644 index cd3a94f5737..00000000000 --- a/spec/lib/gitlab/database/partitioning/convert_table_to_first_list_partition_spec.rb +++ /dev/null @@ -1,273 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Database::Partitioning::ConvertTableToFirstListPartition do - include Gitlab::Database::DynamicModelHelpers - include Database::TableSchemaHelpers - - let(:migration_context) { Gitlab::Database::Migration[2.0].new } - - let(:connection) { migration_context.connection } - let(:table_name) { '_test_table_to_partition' } - let(:table_identifier) { "#{connection.current_schema}.#{table_name}" } - let(:partitioning_column) { :partition_number } - let(:partitioning_default) { 1 } - let(:referenced_table_name) { '_test_referenced_table' } - let(:other_referenced_table_name) { '_test_other_referenced_table' } - let(:parent_table_name) { "#{table_name}_parent" } - let(:lock_tables) { [] } - - let(:model) { define_batchable_model(table_name, connection: connection) } - - let(:parent_model) { define_batchable_model(parent_table_name, connection: connection) } - - let(:converter) do - described_class.new( - migration_context: migration_context, - table_name: table_name, - partitioning_column: partitioning_column, - parent_table_name: parent_table_name, - zero_partition_value: partitioning_default, - lock_tables: lock_tables - ) - end - - before do - # Suppress printing migration progress - allow(migration_context).to receive(:puts) - allow(migration_context.connection).to receive(:transaction_open?).and_return(false) - - connection.execute(<<~SQL) - create table #{referenced_table_name} ( - id bigserial primary key not null - ) - SQL - - connection.execute(<<~SQL) - create table #{other_referenced_table_name} ( - id bigserial primary key not null - ) - SQL - - connection.execute(<<~SQL) - insert into #{referenced_table_name} default values; - insert into #{other_referenced_table_name} default values; - SQL - - connection.execute(<<~SQL) - create table #{table_name} ( - id bigserial not null, - #{partitioning_column} bigint not null default #{partitioning_default}, - referenced_id bigint not null references #{referenced_table_name} (id) on delete cascade, - other_referenced_id bigint not null references #{other_referenced_table_name} (id) on delete set null, - primary key (id, #{partitioning_column}) - ) - SQL - - connection.execute(<<~SQL) - insert into #{table_name} (referenced_id, other_referenced_id) - select #{referenced_table_name}.id, #{other_referenced_table_name}.id - from #{referenced_table_name}, #{other_referenced_table_name}; - SQL - end - - describe "#prepare_for_partitioning" do - subject(:prepare) { converter.prepare_for_partitioning } - - it 'adds a check constraint' do - expect { prepare }.to change { - Gitlab::Database::PostgresConstraint - .check_constraints - .by_table_identifier(table_identifier) - .count - }.from(0).to(1) - end - end - - describe '#revert_prepare_for_partitioning' do - before do - converter.prepare_for_partitioning - end - - subject(:revert_prepare) { converter.revert_preparation_for_partitioning } - - it 'removes a check constraint' do - expect { revert_prepare }.to change { - Gitlab::Database::PostgresConstraint - .check_constraints - .by_table_identifier("#{connection.current_schema}.#{table_name}") - .count - }.from(1).to(0) - end - end - - describe "#convert_to_zero_partition" do - subject(:partition) { converter.partition } - - before do - converter.prepare_for_partitioning - end - - context 'when the primary key is incorrect' do - before do - connection.execute(<<~SQL) - alter table #{table_name} drop constraint #{table_name}_pkey; - alter table #{table_name} add constraint #{table_name}_pkey PRIMARY KEY (id); - SQL - end - - it 'throws a reasonable error message' do - expect { partition }.to raise_error(described_class::UnableToPartition, /#{partitioning_column}/) - end - end - - context 'when there is not a supporting check constraint' do - before do - connection.execute(<<~SQL) - alter table #{table_name} drop constraint partitioning_constraint; - SQL - end - - it 'throws a reasonable error message' do - expect { partition }.to raise_error(described_class::UnableToPartition, /constraint /) - end - end - - it 'migrates the table to a partitioned table' do - fks_before = migration_context.foreign_keys(table_name) - - partition - - expect(Gitlab::Database::PostgresPartition.for_parent_table(parent_table_name).count).to eq(1) - expect(migration_context.foreign_keys(parent_table_name).map(&:options)).to match_array(fks_before.map(&:options)) - - connection.execute(<<~SQL) - insert into #{table_name} (referenced_id, other_referenced_id) select #{referenced_table_name}.id, #{other_referenced_table_name}.id from #{referenced_table_name}, #{other_referenced_table_name}; - SQL - - # Create a second partition - connection.execute(<<~SQL) - create table #{table_name}2 partition of #{parent_table_name} FOR VALUES IN (2) - SQL - - parent_model.create!(partitioning_column => 2, :referenced_id => 1, :other_referenced_id => 1) - expect(parent_model.pluck(:id)).to match_array([1, 2, 3]) - end - - context 'when the existing table is owned by a different user' do - before do - connection.execute(<<~SQL) - CREATE USER other_user SUPERUSER; - ALTER TABLE #{table_name} OWNER TO other_user; - SQL - end - - let(:current_user) { model.connection.select_value('select current_user') } - - it 'partitions without error' do - expect { partition }.not_to raise_error - end - end - - context 'with locking tables' do - let(:lock_tables) { [table_name] } - - it 'locks the table' do - recorder = ActiveRecord::QueryRecorder.new { partition } - - expect(recorder.log).to include(/LOCK "_test_table_to_partition" IN ACCESS EXCLUSIVE MODE/) - end - end - - context 'when an error occurs during the conversion' do - def fail_first_time - # We can't directly use a boolean here, as we need something that will be passed by-reference to the proc - fault_status = { faulted: false } - proc do |m, *args, **kwargs| - next m.call(*args, **kwargs) if fault_status[:faulted] - - fault_status[:faulted] = true - raise 'fault!' - end - end - - def fail_sql_matching(regex) - proc do - allow(migration_context.connection).to receive(:execute).and_call_original - allow(migration_context.connection).to receive(:execute).with(regex).and_wrap_original(&fail_first_time) - end - end - - def fail_adding_fk(from_table, to_table) - proc do - allow(migration_context.connection).to receive(:add_foreign_key).and_call_original - expect(migration_context.connection).to receive(:add_foreign_key).with(from_table, to_table, any_args) - .and_wrap_original(&fail_first_time) - end - end - - where(:case_name, :fault) do - [ - ["creating parent table", lazy { fail_sql_matching(/CREATE/i) }], - ["adding the first foreign key", lazy { fail_adding_fk(parent_table_name, referenced_table_name) }], - ["adding the second foreign key", lazy { fail_adding_fk(parent_table_name, other_referenced_table_name) }], - ["attaching table", lazy { fail_sql_matching(/ATTACH/i) }] - ] - end - - before do - # Set up the fault that we'd like to inject - fault.call - end - - with_them do - it 'recovers from a fault', :aggregate_failures do - expect { converter.partition }.to raise_error(/fault/) - expect(Gitlab::Database::PostgresPartition.for_parent_table(parent_table_name).count).to eq(0) - - expect { converter.partition }.not_to raise_error - expect(Gitlab::Database::PostgresPartition.for_parent_table(parent_table_name).count).to eq(1) - end - end - end - end - - describe '#revert_conversion_to_zero_partition' do - before do - converter.prepare_for_partitioning - converter.partition - end - - subject(:revert_conversion) { converter.revert_partitioning } - - it 'detaches the partition' do - expect { revert_conversion }.to change { - Gitlab::Database::PostgresPartition - .for_parent_table(parent_table_name).count - }.from(1).to(0) - end - - it 'does not drop the child partition' do - expect { revert_conversion }.not_to change { table_oid(table_name) } - end - - it 'removes the parent table' do - expect { revert_conversion }.to change { table_oid(parent_table_name).present? }.from(true).to(false) - end - - it 're-adds the check constraint' do - expect { revert_conversion }.to change { - Gitlab::Database::PostgresConstraint - .check_constraints - .by_table_identifier(table_identifier) - .count - }.by(1) - end - - it 'moves sequences back to the original table' do - expect { revert_conversion }.to change { converter.send(:sequences_owned_by, table_name).count }.from(0) - .and change { converter.send(:sequences_owned_by, parent_table_name).count }.to(0) - end - end -end diff --git a/spec/lib/gitlab/database/partitioning/detached_partition_dropper_spec.rb b/spec/lib/gitlab/database/partitioning/detached_partition_dropper_spec.rb index 646ae50fb44..04940028aee 100644 --- a/spec/lib/gitlab/database/partitioning/detached_partition_dropper_spec.rb +++ b/spec/lib/gitlab/database/partitioning/detached_partition_dropper_spec.rb @@ -25,23 +25,23 @@ RSpec.describe Gitlab::Database::Partitioning::DetachedPartitionDropper do before do connection.execute(<<~SQL) - CREATE TABLE referenced_table ( + CREATE TABLE _test_referenced_table ( id bigserial primary key not null ) SQL connection.execute(<<~SQL) - CREATE TABLE parent_table ( + CREATE TABLE _test_parent_table ( id bigserial not null, referenced_id bigint not null, created_at timestamptz not null, primary key (id, created_at), - constraint fk_referenced foreign key (referenced_id) references referenced_table(id) + constraint fk_referenced foreign key (referenced_id) references _test_referenced_table(id) ) PARTITION BY RANGE(created_at) SQL end - def create_partition(name:, from:, to:, attached:, drop_after:, table: 'parent_table') + def create_partition(name:, from:, to:, attached:, drop_after:, table: :_test_parent_table) from = from.beginning_of_month to = to.beginning_of_month full_name = "#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.#{name}" @@ -64,20 +64,20 @@ RSpec.describe Gitlab::Database::Partitioning::DetachedPartitionDropper do describe '#perform' do context 'when the partition should not be dropped yet' do it 'does not drop the partition' do - create_partition(name: 'test_partition', + create_partition(name: :_test_partition, from: 2.months.ago, to: 1.month.ago, attached: false, drop_after: 1.day.from_now) dropper.perform - expect_partition_present('test_partition') + expect_partition_present(:_test_partition) end end context 'with a partition to drop' do before do - create_partition(name: 'test_partition', + create_partition(name: :_test_partition, from: 2.months.ago, to: 1.month.ago.beginning_of_month, attached: false, @@ -87,45 +87,45 @@ RSpec.describe Gitlab::Database::Partitioning::DetachedPartitionDropper do it 'drops the partition' do dropper.perform - expect(table_oid('test_partition')).to be_nil + expect(table_oid(:_test_partition)).to be_nil end context 'removing foreign keys' do it 'removes foreign keys from the table before dropping it' do expect(dropper).to receive(:drop_detached_partition).and_wrap_original do |drop_method, partition| - expect(partition.table_name).to eq('test_partition') + expect(partition.table_name).to eq('_test_partition') expect(foreign_key_exists_by_name(partition.table_name, 'fk_referenced', schema: Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA)).to be_falsey drop_method.call(partition) end - expect(foreign_key_exists_by_name('test_partition', 'fk_referenced', schema: Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA)).to be_truthy + expect(foreign_key_exists_by_name(:_test_partition, 'fk_referenced', schema: Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA)).to be_truthy dropper.perform end it 'does not remove foreign keys from the parent table' do - expect { dropper.perform }.not_to change { foreign_key_exists_by_name('parent_table', 'fk_referenced') }.from(true) + expect { dropper.perform }.not_to change { foreign_key_exists_by_name('_test_parent_table', 'fk_referenced') }.from(true) end context 'when another process drops the foreign key' do it 'skips dropping that foreign key' do expect(dropper).to receive(:drop_foreign_key_if_present).and_wrap_original do |drop_meth, *args| - connection.execute('alter table gitlab_partitions_dynamic.test_partition drop constraint fk_referenced;') + connection.execute('alter table gitlab_partitions_dynamic._test_partition drop constraint fk_referenced;') drop_meth.call(*args) end dropper.perform - expect_partition_removed('test_partition') + expect_partition_removed(:_test_partition) end end context 'when another process drops the partition' do it 'skips dropping the foreign key' do expect(dropper).to receive(:drop_foreign_key_if_present).and_wrap_original do |drop_meth, *args| - connection.execute('drop table gitlab_partitions_dynamic.test_partition') - Postgresql::DetachedPartition.where(table_name: 'test_partition').delete_all + connection.execute('drop table gitlab_partitions_dynamic._test_partition') + Postgresql::DetachedPartition.where(table_name: :_test_partition).delete_all end expect(Gitlab::AppLogger).not_to receive(:error) @@ -159,7 +159,7 @@ RSpec.describe Gitlab::Database::Partitioning::DetachedPartitionDropper do context 'when the partition to drop is still attached to its table' do before do - create_partition(name: 'test_partition', + create_partition(name: :_test_partition, from: 2.months.ago, to: 1.month.ago.beginning_of_month, attached: true, @@ -169,8 +169,8 @@ RSpec.describe Gitlab::Database::Partitioning::DetachedPartitionDropper do it 'does not drop the partition, but does remove the DetachedPartition entry' do dropper.perform aggregate_failures do - expect(table_oid('test_partition')).not_to be_nil - expect(Postgresql::DetachedPartition.find_by(table_name: 'test_partition')).to be_nil + expect(table_oid(:_test_partition)).not_to be_nil + expect(Postgresql::DetachedPartition.find_by(table_name: :_test_partition)).to be_nil end end @@ -185,20 +185,20 @@ RSpec.describe Gitlab::Database::Partitioning::DetachedPartitionDropper do dropper.perform - expect(table_oid('test_partition')).not_to be_nil + expect(table_oid(:_test_partition)).not_to be_nil end end end context 'with multiple partitions to drop' do before do - create_partition(name: 'partition_1', + create_partition(name: :_test_partition_1, from: 3.months.ago, to: 2.months.ago, attached: false, drop_after: 1.second.ago) - create_partition(name: 'partition_2', + create_partition(name: :_test_partition_2, from: 2.months.ago, to: 1.month.ago, attached: false, @@ -208,8 +208,8 @@ RSpec.describe Gitlab::Database::Partitioning::DetachedPartitionDropper do it 'drops both partitions' do dropper.perform - expect_partition_removed('partition_1') - expect_partition_removed('partition_2') + expect_partition_removed(:_test_partition_1) + expect_partition_removed(:_test_partition_2) end context 'when the first drop returns an error' do @@ -223,7 +223,7 @@ RSpec.describe Gitlab::Database::Partitioning::DetachedPartitionDropper do expect(Postgresql::DetachedPartition.count).to eq(1) errored_partition_name = Postgresql::DetachedPartition.first!.table_name - dropped_partition_name = (%w[partition_1 partition_2] - [errored_partition_name]).first + dropped_partition_name = (%w[_test_partition_1 _test_partition_2] - [errored_partition_name]).first expect_partition_present(errored_partition_name) expect_partition_removed(dropped_partition_name) end diff --git a/spec/lib/gitlab/database/partitioning/list/convert_table_spec.rb b/spec/lib/gitlab/database/partitioning/list/convert_table_spec.rb new file mode 100644 index 00000000000..8e2a53ea76f --- /dev/null +++ b/spec/lib/gitlab/database/partitioning/list/convert_table_spec.rb @@ -0,0 +1,365 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::Partitioning::List::ConvertTable, feature_category: :database do + include Gitlab::Database::DynamicModelHelpers + include Database::TableSchemaHelpers + include Database::InjectFailureHelpers + + include_context 'with a table structure for converting a table to a list partition' + + let(:converter) do + described_class.new( + migration_context: migration_context, + table_name: table_name, + partitioning_column: partitioning_column, + parent_table_name: parent_table_name, + zero_partition_value: partitioning_default, + lock_tables: lock_tables + ) + end + + describe "#prepare_for_partitioning" do + subject(:prepare) { converter.prepare_for_partitioning(async: async) } + + let(:async) { false } + + it 'adds a check constraint' do + expect { prepare }.to change { + Gitlab::Database::PostgresConstraint + .check_constraints + .by_table_identifier(table_identifier) + .count + }.from(0).to(1) + end + + context 'when it fails to add constraint' do + before do + allow(migration_context).to receive(:add_check_constraint) + end + + it 'raises UnableToPartition error' do + expect { prepare } + .to raise_error(described_class::UnableToPartition) + .and change { + Gitlab::Database::PostgresConstraint + .check_constraints + .by_table_identifier(table_identifier) + .count + }.by(0) + end + end + + context 'when async' do + let(:async) { true } + + it 'adds a NOT VALID check constraint' do + expect { prepare }.to change { + Gitlab::Database::PostgresConstraint + .check_constraints + .by_table_identifier(table_identifier) + .count + }.from(0).to(1) + + constraint = + Gitlab::Database::PostgresConstraint + .check_constraints + .by_table_identifier(table_identifier) + .last + + expect(constraint.definition).to end_with('NOT VALID') + end + + it 'adds a PostgresAsyncConstraintValidation record' do + expect { prepare }.to change { + Gitlab::Database::AsyncConstraints::PostgresAsyncConstraintValidation.count + }.by(1) + + record = Gitlab::Database::AsyncConstraints::PostgresAsyncConstraintValidation + .where(table_name: table_name).last + + expect(record.name).to eq described_class::PARTITIONING_CONSTRAINT_NAME + expect(record).to be_check_constraint + end + + context 'when constraint exists but is not valid' do + before do + converter.prepare_for_partitioning(async: true) + end + + it 'validates the check constraint' do + expect { prepare }.to change { + Gitlab::Database::PostgresConstraint + .check_constraints + .by_table_identifier(table_identifier).first.constraint_valid? + }.from(false).to(true) + end + + context 'when it fails to validate constraint' do + before do + allow(migration_context).to receive(:validate_check_constraint) + end + + it 'raises UnableToPartition error' do + expect { prepare } + .to raise_error(described_class::UnableToPartition, + starting_with('Error validating partitioning constraint')) + .and change { + Gitlab::Database::PostgresConstraint + .check_constraints + .by_table_identifier(table_identifier) + .count + }.by(0) + end + end + end + + context 'when constraint exists and is valid' do + before do + converter.prepare_for_partitioning(async: false) + end + + it 'raises UnableToPartition error' do + expect(Gitlab::AppLogger).to receive(:info).with(starting_with('Nothing to do')) + prepare + end + end + end + end + + describe '#revert_preparation_for_partitioning' do + before do + converter.prepare_for_partitioning + end + + subject(:revert_prepare) { converter.revert_preparation_for_partitioning } + + it 'removes a check constraint' do + expect { revert_prepare }.to change { + Gitlab::Database::PostgresConstraint + .check_constraints + .by_table_identifier("#{connection.current_schema}.#{table_name}") + .count + }.from(1).to(0) + end + end + + describe "#partition" do + subject(:partition) { converter.partition } + + let(:async) { false } + + before do + converter.prepare_for_partitioning(async: async) + end + + context 'when the primary key is incorrect' do + before do + connection.execute(<<~SQL) + alter table #{referencing_table_name} drop constraint fk_referencing; -- this depends on the primary key + alter table #{other_referencing_table_name} drop constraint fk_referencing_other; -- this does too + alter table #{table_name} drop constraint #{table_name}_pkey; + alter table #{table_name} add constraint #{table_name}_pkey PRIMARY KEY (id); + SQL + end + + it 'throws a reasonable error message' do + expect { partition }.to raise_error(described_class::UnableToPartition, /#{partitioning_column}/) + end + end + + context 'when there is not a supporting check constraint' do + before do + connection.execute(<<~SQL) + alter table #{table_name} drop constraint partitioning_constraint; + SQL + end + + it 'throws a reasonable error message' do + expect { partition }.to raise_error(described_class::UnableToPartition, /is not ready for partitioning./) + end + end + + context 'when supporting check constraint is not valid' do + let(:async) { true } + + it 'throws a reasonable error message' do + expect { partition }.to raise_error(described_class::UnableToPartition, /is not ready for partitioning./) + end + end + + it 'migrates the table to a partitioned table' do + fks_before = migration_context.foreign_keys(table_name) + + partition + + expect(Gitlab::Database::PostgresPartition.for_parent_table(parent_table_name).count).to eq(1) + expect(migration_context.foreign_keys(parent_table_name).map(&:options)).to match_array(fks_before.map(&:options)) + + connection.execute(<<~SQL) + insert into #{table_name} (referenced_id, other_referenced_id) select #{referenced_table_name}.id, #{other_referenced_table_name}.id from #{referenced_table_name}, #{other_referenced_table_name}; + SQL + + # Create a second partition + connection.execute(<<~SQL) + create table #{table_name}2 partition of #{parent_table_name} FOR VALUES IN (2) + SQL + + parent_model.create!(partitioning_column => 2, :referenced_id => 1, :other_referenced_id => 1) + expect(parent_model.pluck(:id)).to match_array([1, 2, 3]) + + expect { referencing_model.create!(partitioning_column => 1, :ref_id => 1) }.not_to raise_error + end + + context 'when the existing table is owned by a different user' do + before do + connection.execute(<<~SQL) + CREATE USER other_user SUPERUSER; + ALTER TABLE #{table_name} OWNER TO other_user; + SQL + end + + let(:current_user) { model.connection.select_value('select current_user') } + + it 'partitions without error' do + expect { partition }.not_to raise_error + end + end + + context 'with locking tables' do + let(:lock_tables) { [table_name] } + + it 'locks the table' do + recorder = ActiveRecord::QueryRecorder.new { partition } + + expect(recorder.log).to include(/LOCK "_test_table_to_partition" IN ACCESS EXCLUSIVE MODE/) + end + end + + context 'when an error occurs during the conversion' do + before do + # Set up the fault that we'd like to inject + fault.call + end + + let(:old_fks) do + Gitlab::Database::PostgresForeignKey.by_referenced_table_identifier(table_identifier).not_inherited + end + + let(:new_fks) do + Gitlab::Database::PostgresForeignKey.by_referenced_table_identifier(parent_table_identifier).not_inherited + end + + context 'when partitioning fails the first time' do + where(:case_name, :fault) do + [ + ["creating parent table", lazy { fail_sql_matching(/CREATE/i) }], + ["adding the first foreign key", lazy { fail_adding_fk(parent_table_name, referenced_table_name) }], + ["adding the second foreign key", lazy { fail_adding_fk(parent_table_name, other_referenced_table_name) }], + ["attaching table", lazy { fail_sql_matching(/ATTACH/i) }] + ] + end + + with_them do + it 'recovers from a fault', :aggregate_failures do + expect { converter.partition }.to raise_error(/fault/) + expect(Gitlab::Database::PostgresPartition.for_parent_table(parent_table_name).count).to eq(0) + + expect { converter.partition }.not_to raise_error + expect(Gitlab::Database::PostgresPartition.for_parent_table(parent_table_name).count).to eq(1) + end + end + end + end + + context 'when table has LFK triggers' do + before do + migration_context.track_record_deletions(table_name) + end + + it 'moves the trigger on the parent table', :aggregate_failures do + expect(migration_context.has_loose_foreign_key?(table_name)).to be_truthy + + expect { partition }.not_to raise_error + + expect(migration_context.has_loose_foreign_key?(table_name)).to be_truthy + expect(migration_context.has_loose_foreign_key?(parent_table_name)).to be_truthy + end + + context 'with locking tables' do + let(:lock_tables) { [table_name] } + + it 'locks the table before dropping the triggers' do + recorder = ActiveRecord::QueryRecorder.new { partition } + + lock_index = recorder.log.find_index do |log| + log.start_with?('LOCK "_test_table_to_partition" IN ACCESS EXCLUSIVE MODE') + end + + trigger_index = recorder.log.find_index do |log| + log.start_with?('DROP TRIGGER IF EXISTS _test_table_to_partition_loose_fk_trigger') + end + + expect(lock_index).to be_present + expect(trigger_index).to be_present + expect(lock_index).to be < trigger_index + end + end + end + end + + describe '#revert_partitioning' do + before do + converter.prepare_for_partitioning + converter.partition + end + + subject(:revert_conversion) { converter.revert_partitioning } + + it 'detaches the partition' do + expect { revert_conversion }.to change { + Gitlab::Database::PostgresPartition + .for_parent_table(parent_table_name).count + }.from(1).to(0) + end + + it 'does not drop the child partition' do + expect { revert_conversion }.not_to change { table_oid(table_name) } + end + + it 'removes the parent table' do + expect { revert_conversion }.to change { table_oid(parent_table_name).present? }.from(true).to(false) + end + + it 're-adds the check constraint' do + expect { revert_conversion }.to change { + Gitlab::Database::PostgresConstraint + .check_constraints + .by_table_identifier(table_identifier) + .count + }.by(1) + end + + it 'moves sequences back to the original table' do + expect { revert_conversion }.to change { converter.send(:sequences_owned_by, table_name).count }.from(0) + .and change { converter.send(:sequences_owned_by, parent_table_name).count }.to(0) + end + + context 'when table has LFK triggers' do + before do + migration_context.track_record_deletions(parent_table_name) + migration_context.track_record_deletions(table_name) + end + + it 'restores the trigger on the partition', :aggregate_failures do + expect(migration_context.has_loose_foreign_key?(table_name)).to be_truthy + expect(migration_context.has_loose_foreign_key?(parent_table_name)).to be_truthy + + expect { revert_conversion }.not_to raise_error + + expect(migration_context.has_loose_foreign_key?(table_name)).to be_truthy + end + end + end +end diff --git a/spec/lib/gitlab/database/partitioning/list/locking_configuration_spec.rb b/spec/lib/gitlab/database/partitioning/list/locking_configuration_spec.rb new file mode 100644 index 00000000000..851add43e3c --- /dev/null +++ b/spec/lib/gitlab/database/partitioning/list/locking_configuration_spec.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::Partitioning::List::LockingConfiguration, feature_category: :database do + let(:migration_context) do + Gitlab::Database::Migration[2.1].new.tap do |migration| + migration.extend Gitlab::Database::PartitioningMigrationHelpers::TableManagementHelpers + migration.extend Gitlab::Database::PartitioningMigrationHelpers::ForeignKeyHelpers + end + end + + let(:locking_order) { %w[table_1 table_2 table_3] } + + subject(:locking_configuration) { described_class.new(migration_context, table_locking_order: locking_order) } + + describe '#locking_statement_for' do + it 'only includes locking information for tables in the locking specification' do + expect(subject.locking_statement_for(%w[table_1 table_other])).to eq(subject.locking_statement_for('table_1')) + end + + it 'is nil when none of the tables match the lock configuration' do + expect(subject.locking_statement_for('table_other')).to be_nil + end + + it 'is a lock tables statement' do + expect(subject.locking_statement_for(%w[table_3 table_2])).to eq(<<~SQL) + LOCK "table_2", "table_3" IN ACCESS EXCLUSIVE MODE + SQL + end + + it 'raises if a table name with schema is passed' do + expect { subject.locking_statement_for('public.test') }.to raise_error(ArgumentError) + end + end + + describe '#lock_ordering_for' do + it 'is the intersection with the locking specification, in the order of the specification' do + expect(subject.locking_order_for(%w[table_other table_3 table_1])).to eq(%w[table_1 table_3]) + end + + it 'raises if a table name with schema is passed' do + expect { subject.locking_order_for('public.test') }.to raise_error(ArgumentError) + end + end +end diff --git a/spec/lib/gitlab/database/partitioning/partition_manager_spec.rb b/spec/lib/gitlab/database/partitioning/partition_manager_spec.rb index 2212cb09888..eac4a162879 100644 --- a/spec/lib/gitlab/database/partitioning/partition_manager_spec.rb +++ b/spec/lib/gitlab/database/partitioning/partition_manager_spec.rb @@ -6,7 +6,7 @@ RSpec.describe Gitlab::Database::Partitioning::PartitionManager do include Database::PartitioningHelpers include ExclusiveLeaseHelpers - let(:partitioned_table_name) { "_test_gitlab_main_my_model_example_table" } + let(:partitioned_table_name) { :_test_gitlab_main_my_model_example_table } context 'creating partitions (mocked)' do subject(:sync_partitions) { described_class.new(model).sync_partitions } @@ -45,7 +45,7 @@ RSpec.describe Gitlab::Database::Partitioning::PartitionManager do sync_partitions end - context 'with eplicitly provided connection' do + context 'with explicitly provided connection' do let(:connection) { Ci::ApplicationRecord.connection } it 'uses the explicitly provided connection when any' do @@ -59,6 +59,14 @@ RSpec.describe Gitlab::Database::Partitioning::PartitionManager do end end + context 'when an ArgumentError occurs during partition management' do + it 'raises error' do + expect(partitioning_strategy).to receive(:missing_partitions).and_raise(ArgumentError) + + expect { sync_partitions }.to raise_error(ArgumentError) + end + end + context 'when an error occurs during partition management' do it 'does not raise an error' do expect(partitioning_strategy).to receive(:missing_partitions).and_raise('this should never happen (tm)') @@ -115,7 +123,7 @@ RSpec.describe Gitlab::Database::Partitioning::PartitionManager do let(:manager) { described_class.new(model) } let(:model) { double(partitioning_strategy: partitioning_strategy, table_name: table, connection: connection) } let(:connection) { ActiveRecord::Base.connection } - let(:table) { "foo" } + let(:table) { :_test_foo } let(:partitioning_strategy) do double(extra_partitions: extra_partitions, missing_partitions: [], after_adding_partitions: nil) end @@ -144,7 +152,7 @@ RSpec.describe Gitlab::Database::Partitioning::PartitionManager do end it 'logs an error if the partitions are not detachable' do - allow(Gitlab::Database::PostgresForeignKey).to receive(:by_referenced_table_identifier).with("public.foo") + allow(Gitlab::Database::PostgresForeignKey).to receive(:by_referenced_table_identifier).with("public._test_foo") .and_return([double(name: "fk_1", constrained_table_identifier: "public.constrainted_table_1")]) expect(Gitlab::AppLogger).to receive(:error).with( @@ -154,7 +162,7 @@ RSpec.describe Gitlab::Database::Partitioning::PartitionManager do exception_class: Gitlab::Database::Partitioning::PartitionManager::UnsafeToDetachPartitionError, exception_message: "Cannot detach foo1, it would block while checking foreign key fk_1 on public.constrainted_table_1", - table_name: "foo" + table_name: :_test_foo } ) @@ -230,23 +238,20 @@ RSpec.describe Gitlab::Database::Partitioning::PartitionManager do expect(pending_drop.drop_after).to eq(Time.current + described_class::RETAIN_DETACHED_PARTITIONS_FOR) end - # Postgres 11 does not support foreign keys to partitioned tables - if ApplicationRecord.database.version.to_f >= 12 - context 'when the model is the target of a foreign key' do - before do - connection.execute(<<~SQL) + context 'when the model is the target of a foreign key' do + before do + connection.execute(<<~SQL) create unique index idx_for_fk ON #{partitioned_table_name}(created_at); create table _test_gitlab_main_referencing_table ( id bigserial primary key not null, referencing_created_at timestamptz references #{partitioned_table_name}(created_at) ); - SQL - end + SQL + end - it 'does not detach partitions with a referenced foreign key' do - expect { subject }.not_to change { find_partitions(my_model.table_name).size } - end + it 'does not detach partitions with a referenced foreign key' do + expect { subject }.not_to change { find_partitions(my_model.table_name).size } end end end diff --git a/spec/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table_spec.rb b/spec/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table_spec.rb index 1885e84ac4c..fc279051800 100644 --- a/spec/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table_spec.rb +++ b/spec/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table_spec.rb @@ -54,6 +54,11 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::BackfillPartition allow(backfill_job).to receive(:sleep) end + after do + connection.drop_table source_table + connection.drop_table destination_table + end + let(:source_model) { Class.new(ActiveRecord::Base) } let(:destination_model) { Class.new(ActiveRecord::Base) } let(:timestamp) { Time.utc(2020, 1, 2).round } @@ -82,7 +87,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::BackfillPartition end it 'breaks the assigned batch into smaller batches' do - expect_next_instance_of(described_class::BulkCopy) do |bulk_copy| + expect_next_instance_of(Gitlab::Database::PartitioningMigrationHelpers::BulkCopy) do |bulk_copy| expect(bulk_copy).to receive(:copy_between).with(source1.id, source2.id) expect(bulk_copy).to receive(:copy_between).with(source3.id, source3.id) end diff --git a/spec/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers_spec.rb b/spec/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers_spec.rb index f0e34476cf2..d5f4afd7ba4 100644 --- a/spec/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers_spec.rb +++ b/spec/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::ForeignKeyHelpers do +RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::ForeignKeyHelpers, feature_category: :database do include Database::TableSchemaHelpers let(:migration) do @@ -16,15 +16,23 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::ForeignKeyHelpers let(:partition_schema) { 'gitlab_partitions_dynamic' } let(:partition1_name) { "#{partition_schema}.#{source_table_name}_202001" } let(:partition2_name) { "#{partition_schema}.#{source_table_name}_202002" } + let(:validate) { true } let(:options) do { column: column_name, name: foreign_key_name, on_delete: :cascade, - validate: true + on_update: nil, + primary_key: :id } end + let(:create_options) do + options + .except(:primary_key) + .merge!(reverse_lock_order: false, target_column: :id, validate: validate) + end + before do allow(migration).to receive(:puts) allow(migration).to receive(:transaction_open?).and_return(false) @@ -67,12 +75,11 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::ForeignKeyHelpers expect(migration).to receive(:concurrent_partitioned_foreign_key_name).and_return(foreign_key_name) - expect_add_concurrent_fk_and_call_original(partition1_name, target_table_name, **options) - expect_add_concurrent_fk_and_call_original(partition2_name, target_table_name, **options) + expect_add_concurrent_fk_and_call_original(partition1_name, target_table_name, **create_options) + expect_add_concurrent_fk_and_call_original(partition2_name, target_table_name, **create_options) - expect(migration).to receive(:with_lock_retries).ordered.and_yield - expect(migration).to receive(:add_foreign_key) - .with(source_table_name, target_table_name, **options) + expect(migration).to receive(:add_concurrent_foreign_key) + .with(source_table_name, target_table_name, allow_partitioned: true, **create_options) .ordered .and_call_original @@ -81,6 +88,39 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::ForeignKeyHelpers expect_foreign_key_to_exist(source_table_name, foreign_key_name) end + context 'with validate: false option' do + let(:validate) { false } + let(:options) do + { + column: column_name, + name: foreign_key_name, + on_delete: :cascade, + on_update: nil, + primary_key: :id + } + end + + it 'creates the foreign key only on partitions' do + expect(migration).to receive(:foreign_key_exists?) + .with(source_table_name, target_table_name, **options) + .and_return(false) + + expect(migration).to receive(:concurrent_partitioned_foreign_key_name).and_return(foreign_key_name) + + expect_add_concurrent_fk_and_call_original(partition1_name, target_table_name, **create_options) + expect_add_concurrent_fk_and_call_original(partition2_name, target_table_name, **create_options) + + expect(migration).not_to receive(:add_concurrent_foreign_key) + .with(source_table_name, target_table_name, **create_options) + + migration.add_concurrent_partitioned_foreign_key( + source_table_name, target_table_name, + column: column_name, validate: false) + + expect_foreign_key_not_to_exist(source_table_name, foreign_key_name) + end + end + def expect_add_concurrent_fk_and_call_original(source_table_name, target_table_name, options) expect(migration).to receive(:add_concurrent_foreign_key) .ordered @@ -100,8 +140,6 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::ForeignKeyHelpers .and_return(true) expect(migration).not_to receive(:add_concurrent_foreign_key) - expect(migration).not_to receive(:with_lock_retries) - expect(migration).not_to receive(:add_foreign_key) migration.add_concurrent_partitioned_foreign_key(source_table_name, target_table_name, column: column_name) @@ -110,30 +148,43 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::ForeignKeyHelpers end context 'when additional foreign key options are given' do - let(:options) do + let(:exits_options) do { column: column_name, name: '_my_fk_name', on_delete: :restrict, - validate: true + on_update: nil, + primary_key: :id } end + let(:create_options) do + exits_options + .except(:primary_key) + .merge!(reverse_lock_order: false, target_column: :id, validate: true) + end + it 'forwards them to the foreign key helper methods' do expect(migration).to receive(:foreign_key_exists?) - .with(source_table_name, target_table_name, **options) + .with(source_table_name, target_table_name, **exits_options) .and_return(false) expect(migration).not_to receive(:concurrent_partitioned_foreign_key_name) - expect_add_concurrent_fk(partition1_name, target_table_name, **options) - expect_add_concurrent_fk(partition2_name, target_table_name, **options) + expect_add_concurrent_fk(partition1_name, target_table_name, **create_options) + expect_add_concurrent_fk(partition2_name, target_table_name, **create_options) - expect(migration).to receive(:with_lock_retries).ordered.and_yield - expect(migration).to receive(:add_foreign_key).with(source_table_name, target_table_name, **options).ordered + expect(migration).to receive(:add_concurrent_foreign_key) + .with(source_table_name, target_table_name, allow_partitioned: true, **create_options) + .ordered - migration.add_concurrent_partitioned_foreign_key(source_table_name, target_table_name, - column: column_name, name: '_my_fk_name', on_delete: :restrict) + migration.add_concurrent_partitioned_foreign_key( + source_table_name, + target_table_name, + column: column_name, + name: '_my_fk_name', + on_delete: :restrict + ) end def expect_add_concurrent_fk(source_table_name, target_table_name, options) @@ -153,4 +204,39 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::ForeignKeyHelpers end end end + + describe '#validate_partitioned_foreign_key' do + context 'when run inside a transaction block' do + it 'raises an error' do + expect(migration).to receive(:transaction_open?).and_return(true) + + expect do + migration.validate_partitioned_foreign_key(source_table_name, column_name, name: '_my_fk_name') + end.to raise_error(/can not be run inside a transaction/) + end + end + + context 'when run outside a transaction block' do + before do + migration.add_concurrent_partitioned_foreign_key( + source_table_name, + target_table_name, + column: column_name, + name: foreign_key_name, + validate: false + ) + end + + it 'validates FK for each partition' do + expect(migration).to receive(:execute).with(/SET statement_timeout TO 0/).twice + expect(migration).to receive(:execute).with(/RESET statement_timeout/).twice + expect(migration).to receive(:execute) + .with(/ALTER TABLE #{partition1_name} VALIDATE CONSTRAINT #{foreign_key_name}/).ordered + expect(migration).to receive(:execute) + .with(/ALTER TABLE #{partition2_name} VALIDATE CONSTRAINT #{foreign_key_name}/).ordered + + migration.validate_partitioned_foreign_key(source_table_name, column_name, name: foreign_key_name) + end + end + end end diff --git a/spec/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers_spec.rb b/spec/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers_spec.rb index e76b1da3834..571c67db597 100644 --- a/spec/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers_spec.rb +++ b/spec/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers_spec.rb @@ -2,10 +2,11 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHelpers do +RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHelpers, feature_category: :database do include Database::PartitioningHelpers include Database::TriggerHelpers include Database::TableSchemaHelpers + include MigrationsHelpers let(:migration) do ActiveRecord::Migration.new.extend(described_class) @@ -14,9 +15,9 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe let_it_be(:connection) { ActiveRecord::Base.connection } let(:source_table) { :_test_original_table } - let(:partitioned_table) { '_test_migration_partitioned_table' } - let(:function_name) { '_test_migration_function_name' } - let(:trigger_name) { '_test_migration_trigger_name' } + let(:partitioned_table) { :_test_migration_partitioned_table } + let(:function_name) { :_test_migration_function_name } + let(:trigger_name) { :_test_migration_trigger_name } let(:partition_column) { 'created_at' } let(:min_date) { Date.new(2019, 12) } let(:max_date) { Date.new(2020, 3) } @@ -42,15 +43,15 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe end context 'list partitioning conversion helpers' do - shared_examples_for 'delegates to ConvertTableToFirstListPartition' do + shared_examples_for 'delegates to ConvertTable' do let(:extra_options) { {} } it 'throws an error if in a transaction' do allow(migration).to receive(:transaction_open?).and_return(true) expect { migrate }.to raise_error(/cannot be run inside a transaction/) end - it 'delegates to a method on ConvertTableToFirstListPartition' do - expect_next_instance_of(Gitlab::Database::Partitioning::ConvertTableToFirstListPartition, + it 'delegates to a method on List::ConvertTable' do + expect_next_instance_of(Gitlab::Database::Partitioning::List::ConvertTable, migration_context: migration, table_name: source_table, parent_table_name: partitioned_table, @@ -65,7 +66,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe end describe '#convert_table_to_first_list_partition' do - it_behaves_like 'delegates to ConvertTableToFirstListPartition' do + it_behaves_like 'delegates to ConvertTable' do let(:lock_tables) { [source_table] } let(:extra_options) { { lock_tables: lock_tables } } let(:expected_method) { :partition } @@ -80,7 +81,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe end describe '#revert_converting_table_to_first_list_partition' do - it_behaves_like 'delegates to ConvertTableToFirstListPartition' do + it_behaves_like 'delegates to ConvertTable' do let(:expected_method) { :revert_partitioning } let(:migrate) do migration.revert_converting_table_to_first_list_partition(table_name: source_table, @@ -92,19 +93,20 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe end describe '#prepare_constraint_for_list_partitioning' do - it_behaves_like 'delegates to ConvertTableToFirstListPartition' do + it_behaves_like 'delegates to ConvertTable' do let(:expected_method) { :prepare_for_partitioning } let(:migrate) do migration.prepare_constraint_for_list_partitioning(table_name: source_table, partitioning_column: partition_column, parent_table_name: partitioned_table, - initial_partitioning_value: min_date) + initial_partitioning_value: min_date, + async: false) end end end describe '#revert_preparing_constraint_for_list_partitioning' do - it_behaves_like 'delegates to ConvertTableToFirstListPartition' do + it_behaves_like 'delegates to ConvertTable' do let(:expected_method) { :revert_preparation_for_partitioning } let(:migrate) do migration.revert_preparing_constraint_for_list_partitioning(table_name: source_table, @@ -121,12 +123,8 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe let(:old_primary_key) { 'id' } let(:new_primary_key) { [old_primary_key, partition_column] } - before do - allow(migration).to receive(:queue_background_migration_jobs_by_range_at_intervals) - end - context 'when the table is not allowed' do - let(:source_table) { :this_table_is_not_allowed } + let(:source_table) { :_test_this_table_is_not_allowed } it 'raises an error' do expect(migration).to receive(:assert_table_is_allowed).with(source_table).and_call_original @@ -227,7 +225,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe end end - let(:non_int_table) { :another_example } + let(:non_int_table) { :_test_another_example } let(:old_primary_key) { 'identifier' } it 'does not change the primary key datatype' do @@ -422,7 +420,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe let(:migration_class) { 'Gitlab::Database::PartitioningMigrationHelpers::BackfillPartitionedTable' } context 'when the table is not allowed' do - let(:source_table) { :this_table_is_not_allowed } + let(:source_table) { :_test_this_table_is_not_allowed } it 'raises an error' do expect(migration).to receive(:assert_table_is_allowed).with(source_table).and_call_original @@ -462,7 +460,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe describe '#enqueue_partitioning_data_migration' do context 'when the table is not allowed' do - let(:source_table) { :this_table_is_not_allowed } + let(:source_table) { :_test_this_table_is_not_allowed } it 'raises an error' do expect(migration).to receive(:assert_table_is_allowed).with(source_table).and_call_original @@ -484,17 +482,15 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe end context 'when records exist in the source table' do - let(:migration_class) { '::Gitlab::Database::PartitioningMigrationHelpers::BackfillPartitionedTable' } + let(:migration_class) { described_class::MIGRATION } let(:sub_batch_size) { described_class::SUB_BATCH_SIZE } - let(:pause_seconds) { described_class::PAUSE_SECONDS } let!(:first_id) { source_model.create!(name: 'Bob', age: 20).id } let!(:second_id) { source_model.create!(name: 'Alice', age: 30).id } let!(:third_id) { source_model.create!(name: 'Sam', age: 40).id } before do stub_const("#{described_class.name}::BATCH_SIZE", 2) - - expect(migration).to receive(:queue_background_migration_jobs_by_range_at_intervals).and_call_original + stub_const("#{described_class.name}::SUB_BATCH_SIZE", 1) end it 'enqueues jobs to copy each batch of data' do @@ -503,13 +499,13 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe Sidekiq::Testing.fake! do migration.enqueue_partitioning_data_migration source_table - expect(BackgroundMigrationWorker.jobs.size).to eq(2) - - first_job_arguments = [first_id, second_id, source_table.to_s, partitioned_table, 'id'] - expect(BackgroundMigrationWorker.jobs[0]['args']).to eq([migration_class, first_job_arguments]) - - second_job_arguments = [third_id, third_id, source_table.to_s, partitioned_table, 'id'] - expect(BackgroundMigrationWorker.jobs[1]['args']).to eq([migration_class, second_job_arguments]) + expect(migration_class).to have_scheduled_batched_migration( + table_name: source_table, + column_name: :id, + job_arguments: [partitioned_table], + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE + ) end end end @@ -517,7 +513,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe describe '#cleanup_partitioning_data_migration' do context 'when the table is not allowed' do - let(:source_table) { :this_table_is_not_allowed } + let(:source_table) { :_test_this_table_is_not_allowed } it 'raises an error' do expect(migration).to receive(:assert_table_is_allowed).with(source_table).and_call_original @@ -528,18 +524,36 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe end end - context 'when tracking records exist in the background_migration_jobs table' do - let(:migration_class) { 'Gitlab::Database::PartitioningMigrationHelpers::BackfillPartitionedTable' } - let!(:job1) { create(:background_migration_job, class_name: migration_class, arguments: [1, 10, source_table]) } - let!(:job2) { create(:background_migration_job, class_name: migration_class, arguments: [11, 20, source_table]) } - let!(:job3) { create(:background_migration_job, class_name: migration_class, arguments: [1, 10, 'other_table']) } + context 'when tracking records exist in the batched_background_migrations table' do + let(:migration_class) { described_class::MIGRATION } + + before do + create( + :batched_background_migration, + job_class_name: migration_class, + table_name: source_table, + column_name: :id, + job_arguments: [partitioned_table] + ) + + create( + :batched_background_migration, + job_class_name: migration_class, + table_name: 'other_table', + column_name: :id, + job_arguments: ['other_table_partitioned'] + ) + end it 'deletes those pertaining to the given table' do expect { migration.cleanup_partitioning_data_migration(source_table) } - .to change { ::Gitlab::Database::BackgroundMigrationJob.count }.from(3).to(1) + .to change { ::Gitlab::Database::BackgroundMigration::BatchedMigration.count }.by(-1) - remaining_record = ::Gitlab::Database::BackgroundMigrationJob.first - expect(remaining_record).to have_attributes(class_name: migration_class, arguments: [1, 10, 'other_table']) + expect(::Gitlab::Database::BackgroundMigration::BatchedMigration.where(table_name: 'other_table').any?) + .to be_truthy + + expect(::Gitlab::Database::BackgroundMigration::BatchedMigration.where(table_name: source_table).any?) + .to be_falsy end end end @@ -577,10 +591,10 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe end describe '#finalize_backfilling_partitioned_table' do - let(:source_column) { 'id' } + let(:source_column) { :id } context 'when the table is not allowed' do - let(:source_table) { :this_table_is_not_allowed } + let(:source_table) { :_test_this_table_is_not_allowed } it 'raises an error' do expect(migration).to receive(:assert_table_is_allowed).with(source_table).and_call_original @@ -601,131 +615,28 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe end end - context 'finishing pending background migration jobs' do + context 'finishing pending batched background migration jobs' do let(:source_table_double) { double('table name') } let(:raw_arguments) { [1, 50_000, source_table_double, partitioned_table, source_column] } let(:background_job) { double('background job', args: ['background jobs', raw_arguments]) } - - before do - allow(migration).to receive(:table_exists?).with(partitioned_table).and_return(true) - allow(migration).to receive(:copy_missed_records) - allow(migration).to receive(:execute).with(/VACUUM/) - allow(migration).to receive(:execute).with(/^(RE)?SET/) - end - - it 'finishes remaining jobs for the correct table' do - expect_next_instance_of(described_class::JobArguments) do |job_arguments| - expect(job_arguments).to receive(:source_table_name).and_call_original - end - - expect(Gitlab::BackgroundMigration).to receive(:steal) - .with(described_class::MIGRATION_CLASS_NAME) - .and_yield(background_job) - - expect(source_table_double).to receive(:==).with(source_table.to_s) - - migration.finalize_backfilling_partitioned_table source_table - end - - it 'requires the migration helper to execute in DML mode' do - expect(Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas).to receive(:require_dml_mode!) - - expect(Gitlab::BackgroundMigration).to receive(:steal) - .with(described_class::MIGRATION_CLASS_NAME) - .and_yield(background_job) - - migration.finalize_backfilling_partitioned_table source_table - end - end - - context 'when there is missed data' do - let(:partitioned_model) { Class.new(ActiveRecord::Base) } - let(:timestamp) { Time.utc(2019, 12, 1, 12).round } - let!(:record1) { source_model.create!(name: 'Bob', age: 20, created_at: timestamp, updated_at: timestamp) } - let!(:record2) { source_model.create!(name: 'Alice', age: 30, created_at: timestamp, updated_at: timestamp) } - let!(:record3) { source_model.create!(name: 'Sam', age: 40, created_at: timestamp, updated_at: timestamp) } - let!(:record4) { source_model.create!(name: 'Sue', age: 50, created_at: timestamp, updated_at: timestamp) } - - let!(:pending_job1) do - create(:background_migration_job, - class_name: described_class::MIGRATION_CLASS_NAME, - arguments: [record1.id, record2.id, source_table, partitioned_table, source_column]) - end - - let!(:pending_job2) do - create(:background_migration_job, - class_name: described_class::MIGRATION_CLASS_NAME, - arguments: [record3.id, record3.id, source_table, partitioned_table, source_column]) - end - - let!(:succeeded_job) do - create(:background_migration_job, :succeeded, - class_name: described_class::MIGRATION_CLASS_NAME, - arguments: [record4.id, record4.id, source_table, partitioned_table, source_column]) + let(:bbm_arguments) do + { + job_class_name: described_class::MIGRATION, + table_name: source_table, + column_name: connection.primary_key(source_table), + job_arguments: [partitioned_table] + } end before do - partitioned_model.primary_key = :id - partitioned_model.table_name = partitioned_table - - allow(migration).to receive(:queue_background_migration_jobs_by_range_at_intervals) - - migration.partition_table_by_date source_table, partition_column, min_date: min_date, max_date: max_date - - allow(Gitlab::BackgroundMigration).to receive(:steal) + allow(migration).to receive(:table_exists?).with(partitioned_table).and_return(true) allow(migration).to receive(:execute).with(/VACUUM/) allow(migration).to receive(:execute).with(/^(RE)?SET/) end - it 'idempotently cleans up after failed background migrations' do - expect(partitioned_model.count).to eq(0) - - partitioned_model.insert(record2.attributes, unique_by: [:id, :created_at]) - - expect_next_instance_of(Gitlab::Database::PartitioningMigrationHelpers::BackfillPartitionedTable) do |backfill| - allow(backfill).to receive(:transaction_open?).and_return(false) - - expect(backfill).to receive(:perform) - .with(record1.id, record2.id, source_table, partitioned_table, source_column) - .and_call_original - - expect(backfill).to receive(:perform) - .with(record3.id, record3.id, source_table, partitioned_table, source_column) - .and_call_original - end - - migration.finalize_backfilling_partitioned_table source_table - - expect(partitioned_model.count).to eq(3) - - [record1, record2, record3].each do |original| - copy = partitioned_model.find(original.id) - expect(copy.attributes).to eq(original.attributes) - end - - expect(partitioned_model.find_by_id(record4.id)).to be_nil - - [pending_job1, pending_job2].each do |job| - expect(job.reload).to be_succeeded - end - end - - it 'raises an error if no job tracking records are marked as succeeded' do - expect_next_instance_of(Gitlab::Database::PartitioningMigrationHelpers::BackfillPartitionedTable) do |backfill| - allow(backfill).to receive(:transaction_open?).and_return(false) - - expect(backfill).to receive(:perform).and_return(0) - end - - expect do - migration.finalize_backfilling_partitioned_table source_table - end.to raise_error(/failed to update tracking record/) - end - - it 'vacuums the table after loading is complete' do - expect_next_instance_of(Gitlab::Database::PartitioningMigrationHelpers::BackfillPartitionedTable) do |backfill| - allow(backfill).to receive(:perform).and_return(1) - end + it 'ensures finishing of remaining jobs and vacuums the partitioned table' do + expect(migration).to receive(:ensure_batched_background_migration_is_finished) + .with(bbm_arguments) expect(Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas).to receive(:with_suppressed).and_yield expect(migration).to receive(:disable_statement_timeout).and_call_original diff --git a/spec/lib/gitlab/database/partitioning_spec.rb b/spec/lib/gitlab/database/partitioning_spec.rb index ae74ee60a4b..9df238a0024 100644 --- a/spec/lib/gitlab/database/partitioning_spec.rb +++ b/spec/lib/gitlab/database/partitioning_spec.rb @@ -2,11 +2,11 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::Partitioning do +RSpec.describe Gitlab::Database::Partitioning, feature_category: :database do include Database::PartitioningHelpers include Database::TableSchemaHelpers - let(:connection) { ApplicationRecord.connection } + let(:main_connection) { ApplicationRecord.connection } around do |example| previously_registered_models = described_class.registered_models.dup @@ -65,21 +65,26 @@ RSpec.describe Gitlab::Database::Partitioning do describe '.sync_partitions' do let(:ci_connection) { Ci::ApplicationRecord.connection } - let(:table_names) { %w[partitioning_test1 partitioning_test2] } + let(:table_names) { %w[_test_partitioning_test1 _test_partitioning_test2] } let(:models) do - table_names.map do |table_name| + [ Class.new(ApplicationRecord) do include PartitionedTable - self.table_name = table_name + self.table_name = :_test_partitioning_test1 partitioned_by :created_at, strategy: :monthly + end, + Class.new(Gitlab::Database::Partitioning::TableWithoutModel).tap do |klass| + klass.table_name = :_test_partitioning_test2 + klass.partitioned_by(:created_at, strategy: :monthly) + klass.limit_connection_names = %i[main] end - end + ] end before do table_names.each do |table_name| - connection.execute(<<~SQL) + execute_on_each_database(<<~SQL) CREATE TABLE #{table_name} ( id serial not null, created_at timestamptz not null, @@ -96,32 +101,12 @@ RSpec.describe Gitlab::Database::Partitioning do end context 'with multiple databases' do - before do - table_names.each do |table_name| - ci_connection.execute("DROP TABLE IF EXISTS #{table_name}") - - ci_connection.execute(<<~SQL) - CREATE TABLE #{table_name} ( - id serial not null, - created_at timestamptz not null, - PRIMARY KEY (id, created_at)) - PARTITION BY RANGE (created_at); - SQL - end - end - - after do - table_names.each do |table_name| - ci_connection.execute("DROP TABLE IF EXISTS #{table_name}") - end - end - it 'creates partitions in each database' do - skip_if_multiple_databases_not_setup(:ci) + skip_if_shared_database(:ci) expect { described_class.sync_partitions(models) } - .to change { find_partitions(table_names.first, conn: connection).size }.from(0) - .and change { find_partitions(table_names.last, conn: connection).size }.from(0) + .to change { find_partitions(table_names.first, conn: main_connection).size }.from(0) + .and change { find_partitions(table_names.last, conn: main_connection).size }.from(0) .and change { find_partitions(table_names.first, conn: ci_connection).size }.from(0) .and change { find_partitions(table_names.last, conn: ci_connection).size }.from(0) end @@ -150,16 +135,18 @@ RSpec.describe Gitlab::Database::Partitioning do Class.new(Ci::ApplicationRecord) do include PartitionedTable - self.table_name = 'partitioning_test3' + self.table_name = :_test_partitioning_test3 partitioned_by :created_at, strategy: :monthly end end before do - (table_names + ['partitioning_test3']).each do |table_name| - ci_connection.execute("DROP TABLE IF EXISTS #{table_name}") + skip_if_shared_database(:ci) + + (table_names + [:_test_partitioning_test3]).each do |table_name| + execute_on_each_database("DROP TABLE IF EXISTS #{table_name}") - ci_connection.execute(<<~SQL) + execute_on_each_database(<<~SQL) CREATE TABLE #{table_name} ( id serial not null, created_at timestamptz not null, @@ -170,20 +157,33 @@ RSpec.describe Gitlab::Database::Partitioning do end after do - (table_names + ['partitioning_test3']).each do |table_name| + (table_names + [:_test_partitioning_test3]).each do |table_name| ci_connection.execute("DROP TABLE IF EXISTS #{table_name}") end end it 'manages partitions for models for the given database', :aggregate_failures do - skip_if_multiple_databases_not_setup(:ci) - expect { described_class.sync_partitions([models.first, ci_model], only_on: 'ci') } .to change { find_partitions(ci_model.table_name, conn: ci_connection).size }.from(0) - expect(find_partitions(models.first.table_name).size).to eq(0) + expect(find_partitions(models.first.table_name, conn: main_connection).size).to eq(0) expect(find_partitions(models.first.table_name, conn: ci_connection).size).to eq(0) - expect(find_partitions(ci_model.table_name).size).to eq(0) + expect(find_partitions(ci_model.table_name, conn: main_connection).size).to eq(0) + end + end + + context 'when partition_manager_sync_partitions feature flag is disabled' do + before do + described_class.register_models(models) + stub_feature_flags(partition_manager_sync_partitions: false) + end + + it 'skips sync_partitions' do + expect(described_class::PartitionManager).not_to receive(:new) + expect(described_class).to receive(:sync_partitions) + .and_call_original + + described_class.sync_partitions(models) end end end @@ -228,7 +228,7 @@ RSpec.describe Gitlab::Database::Partitioning do end describe '.drop_detached_partitions' do - let(:table_names) { %w[detached_test_partition1 detached_test_partition2] } + let(:table_names) { %w[_test_detached_test_partition1 _test_detached_test_partition2] } before do table_names.each do |table_name| diff --git a/spec/lib/gitlab/database/pg_depend_spec.rb b/spec/lib/gitlab/database/pg_depend_spec.rb new file mode 100644 index 00000000000..547a2c84b76 --- /dev/null +++ b/spec/lib/gitlab/database/pg_depend_spec.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::PgDepend, type: :model, feature_category: :database do + let(:connection) { described_class.connection } + + describe '.from_pg_extension' do + subject { described_class.from_pg_extension('VIEW') } + + context 'when having views as dependency' do + before do + connection.execute('CREATE EXTENSION IF NOT EXISTS pg_stat_statements;') + end + + it 'returns pg_stat_statements', quarantine: 'https://gitlab.com/gitlab-org/gitlab/-/issues/410508' do + expect(subject.pluck('relname')).to eq(['pg_stat_statements']) + end + end + end +end diff --git a/spec/lib/gitlab/database/postgres_foreign_key_spec.rb b/spec/lib/gitlab/database/postgres_foreign_key_spec.rb index ae56f66737d..03343c134ae 100644 --- a/spec/lib/gitlab/database/postgres_foreign_key_spec.rb +++ b/spec/lib/gitlab/database/postgres_foreign_key_spec.rb @@ -70,13 +70,29 @@ RSpec.describe Gitlab::Database::PostgresForeignKey, type: :model, feature_categ end describe '#by_constrained_table_name' do - it 'finds the foreign keys for the constrained table' do - expected = described_class.where(name: %w[fk_constrained_to_referenced fk_constrained_to_other_referenced]).to_a + let(:expected) { described_class.where(name: %w[fk_constrained_to_referenced fk_constrained_to_other_referenced]).to_a } + it 'finds the foreign keys for the constrained table' do expect(described_class.by_constrained_table_name(table_name("constrained_table"))).to match_array(expected) end end + describe '#by_constrained_table_name_or_identifier' do + let(:expected) { described_class.where(name: %w[fk_constrained_to_referenced fk_constrained_to_other_referenced]).to_a } + + context 'when using table name' do + it 'finds the foreign keys for the constrained table' do + expect(described_class.by_constrained_table_name_or_identifier(table_name("constrained_table"))).to match_array(expected) + end + end + + context 'when using identifier' do + it 'finds the foreign keys for the constrained table' do + expect(described_class.by_constrained_table_name_or_identifier(schema_table_name('constrained_table'))).to match_array(expected) + end + end + end + describe '#by_name' do it 'finds foreign keys by name' do expect(described_class.by_name('fk_constrained_to_referenced').pluck(:name)).to contain_exactly('fk_constrained_to_referenced') @@ -187,10 +203,8 @@ RSpec.describe Gitlab::Database::PostgresForeignKey, type: :model, feature_categ end end - context 'when supporting foreign keys to inherited tables in postgres 12' do + context 'when supporting foreign keys on partitioned tables' do before do - skip('not supported before postgres 12') if ApplicationRecord.database.version.to_f < 12 - ApplicationRecord.connection.execute(<<~SQL) create table #{schema_table_name('parent')} ( id bigserial primary key not null @@ -232,6 +246,40 @@ RSpec.describe Gitlab::Database::PostgresForeignKey, type: :model, feature_categ end end + context 'with two tables both partitioned' do + before do + ApplicationRecord.connection.execute(<<~SQL) + create table #{table_name('parent')} ( + id bigserial primary key not null + ) partition by hash(id); + + create table #{table_name('child')} + partition of #{table_name('parent')} for values with (remainder 1, modulus 2); + + create table #{table_name('ref_parent')} ( + id bigserial primary key not null + ) partition by hash(id); + + create table #{table_name('ref_child_1')} + partition of #{table_name('ref_parent')} for values with (remainder 1, modulus 3); + + create table #{table_name('ref_child_2')} + partition of #{table_name('ref_parent')} for values with (remainder 2, modulus 3); + + alter table #{table_name('parent')} add constraint fk foreign key (id) references #{table_name('ref_parent')} (id); + SQL + end + + describe '#child_foreign_keys' do + it 'is the child foreign keys of the partitioned parent fk' do + fk = described_class.by_constrained_table_name(table_name('parent')).first + children = fk.child_foreign_keys + expect(children.count).to eq(1) + expect(children.first.constrained_table_name).to eq(table_name('child')) + end + end + end + def schema_table_name(name) "public.#{table_name(name)}" end diff --git a/spec/lib/gitlab/database/postgres_partition_spec.rb b/spec/lib/gitlab/database/postgres_partition_spec.rb index 14a4d405621..48dbdbc7757 100644 --- a/spec/lib/gitlab/database/postgres_partition_spec.rb +++ b/spec/lib/gitlab/database/postgres_partition_spec.rb @@ -2,7 +2,8 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::PostgresPartition, type: :model do +RSpec.describe Gitlab::Database::PostgresPartition, type: :model, feature_category: :database do + let(:current_schema) { ActiveRecord::Base.connection.select_value("SELECT current_schema()") } let(:schema) { 'gitlab_partitions_dynamic' } let(:name) { '_test_partition_01' } let(:identifier) { "#{schema}.#{name}" } @@ -56,9 +57,20 @@ RSpec.describe Gitlab::Database::PostgresPartition, type: :model do expect(partitions.pluck(:name)).to eq([name, second_name]) end + it 'returns the partitions if the parent table schema is included in the table name' do + partitions = described_class.for_parent_table("#{current_schema}._test_partitioned_table") + + expect(partitions.count).to eq(2) + expect(partitions.pluck(:name)).to eq([name, second_name]) + end + it 'does not return partitions for tables not in the current schema' do expect(described_class.for_parent_table('_test_other_table').count).to eq(0) end + + it 'does not return partitions for tables if the schema is not the current' do + expect(described_class.for_parent_table('foo_bar._test_partitioned_table').count).to eq(0) + end end describe '#parent_identifier' do diff --git a/spec/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics_spec.rb b/spec/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics_spec.rb index 3a92f35d585..6a0c4226db8 100644 --- a/spec/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics_spec.rb +++ b/spec/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics_spec.rb @@ -57,10 +57,8 @@ RSpec.describe Gitlab::Database::QueryAnalyzers::GitlabSchemasMetrics, query_ana "for query accessing gitlab_main and unknown schema" => { model: ApplicationRecord, sql: "SELECT 1 FROM projects LEFT JOIN not_in_schema ON not_in_schema.project_id=projects.id", - expectations: { - gitlab_schemas: "gitlab_main,undefined_not_in_schema", - db_config_name: "main" - } + expect_error: + /Could not find gitlab schema for table not_in_schema/ } } end @@ -74,10 +72,14 @@ RSpec.describe Gitlab::Database::QueryAnalyzers::GitlabSchemasMetrics, query_ana allow(::Ci::ApplicationRecord.load_balancer).to receive(:configuration) .and_return(Gitlab::Database::LoadBalancing::Configuration.for_model(::Ci::ApplicationRecord)) - expect(described_class.schemas_metrics).to receive(:increment) - .with(expectations).and_call_original + if expect_error + expect { process_sql(model, sql) }.to raise_error(expect_error) + else + expect(described_class.schemas_metrics).to receive(:increment) + .with(expectations).and_call_original - process_sql(model, sql) + process_sql(model, sql) + end end end end diff --git a/spec/lib/gitlab/database/query_analyzers/gitlab_schemas_validate_connection_spec.rb b/spec/lib/gitlab/database/query_analyzers/gitlab_schemas_validate_connection_spec.rb index d31be6cb883..e3ff5ab4779 100644 --- a/spec/lib/gitlab/database/query_analyzers/gitlab_schemas_validate_connection_spec.rb +++ b/spec/lib/gitlab/database/query_analyzers/gitlab_schemas_validate_connection_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::Database::QueryAnalyzers::GitlabSchemasValidateConnection, query_analyzers: false, - feature_category: :pods do + feature_category: :cell do let(:analyzer) { described_class } # We keep only the GitlabSchemasValidateConnection analyzer running @@ -28,19 +28,19 @@ RSpec.describe Gitlab::Database::QueryAnalyzers::GitlabSchemasValidateConnection model: ApplicationRecord, sql: "SELECT 1 FROM projects LEFT JOIN ci_builds ON ci_builds.project_id=projects.id", expect_error: /The query tried to access \["projects", "ci_builds"\]/, - setup: -> (_) { skip_if_multiple_databases_not_setup(:ci) } + setup: -> (_) { skip_if_shared_database(:ci) } }, "for query accessing gitlab_ci and gitlab_main the gitlab_schemas is always ordered" => { model: ApplicationRecord, sql: "SELECT 1 FROM ci_builds LEFT JOIN projects ON ci_builds.project_id=projects.id", expect_error: /The query tried to access \["ci_builds", "projects"\]/, - setup: -> (_) { skip_if_multiple_databases_not_setup(:ci) } + setup: -> (_) { skip_if_shared_database(:ci) } }, "for query accessing main table from CI database" => { model: Ci::ApplicationRecord, sql: "SELECT 1 FROM projects", expect_error: /The query tried to access \["projects"\]/, - setup: -> (_) { skip_if_multiple_databases_not_setup(:ci) } + setup: -> (_) { skip_if_shared_database(:ci) } }, "for query accessing CI database" => { model: Ci::ApplicationRecord, @@ -51,13 +51,14 @@ RSpec.describe Gitlab::Database::QueryAnalyzers::GitlabSchemasValidateConnection model: ::ApplicationRecord, sql: "SELECT 1 FROM ci_builds", expect_error: /The query tried to access \["ci_builds"\]/, - setup: -> (_) { skip_if_multiple_databases_not_setup(:ci) } + setup: -> (_) { skip_if_shared_database(:ci) } }, "for query accessing unknown gitlab_schema" => { model: ::ApplicationRecord, sql: "SELECT 1 FROM new_table", - expect_error: /The query tried to access \["new_table"\] \(of undefined_new_table\)/, - setup: -> (_) { skip_if_multiple_databases_not_setup(:ci) } + expect_error: + /Could not find gitlab schema for table new_table/, + setup: -> (_) { skip_if_shared_database(:ci) } } } end @@ -77,7 +78,7 @@ RSpec.describe Gitlab::Database::QueryAnalyzers::GitlabSchemasValidateConnection context "when analyzer is enabled for tests", :query_analyzers do before do - skip_if_multiple_databases_not_setup(:ci) + skip_if_shared_database(:ci) end it "throws an error when trying to access a table that belongs to the gitlab_main schema from the ci database" do diff --git a/spec/lib/gitlab/database/query_analyzers/prevent_cross_database_modification_spec.rb b/spec/lib/gitlab/database/query_analyzers/prevent_cross_database_modification_spec.rb index a4322689bf9..02bd6b51463 100644 --- a/spec/lib/gitlab/database/query_analyzers/prevent_cross_database_modification_spec.rb +++ b/spec/lib/gitlab/database/query_analyzers/prevent_cross_database_modification_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::Database::QueryAnalyzers::PreventCrossDatabaseModification, query_analyzers: false, - feature_category: :pods do + feature_category: :cell do let_it_be(:pipeline, refind: true) { create(:ci_pipeline) } let_it_be(:project, refind: true) { create(:project) } @@ -118,6 +118,18 @@ RSpec.describe Gitlab::Database::QueryAnalyzers::PreventCrossDatabaseModificatio end end + context 'when ci_pipelines are ignored for cross modification' do + it 'does not raise error' do + Project.transaction do + expect do + described_class.temporary_ignore_tables_in_transaction(%w[ci_pipelines], url: 'TODO') do + run_queries + end + end.not_to raise_error + end + end + end + context 'when data modification happens in nested transactions' do it 'raises error' do Project.transaction(requires_new: true) do @@ -209,27 +221,16 @@ RSpec.describe Gitlab::Database::QueryAnalyzers::PreventCrossDatabaseModificatio end end - context 'when some table with a defined schema and another table with undefined gitlab_schema is modified' do - it 'raises an error including including message about undefined schema' do - expect do - Project.transaction do - project.touch - project.connection.execute('UPDATE foo_bars_undefined_table SET a=1 WHERE id = -1') - end - end.to raise_error /Cross-database data modification.*The gitlab_schema was undefined/ - end - end - context 'when execution is rescued with StandardError' do it 'raises cross-database data modification exception' do expect do Project.transaction do project.touch - project.connection.execute('UPDATE foo_bars_undefined_table SET a=1 WHERE id = -1') + project.connection.execute('UPDATE ci_pipelines SET id=1 WHERE id = -1') end rescue StandardError # Ensures that standard rescue does not silence errors - end.to raise_error /Cross-database data modification.*The gitlab_schema was undefined/ + end.to raise_error /Cross-database data modification/ end end diff --git a/spec/lib/gitlab/database/reflection_spec.rb b/spec/lib/gitlab/database/reflection_spec.rb index 779bdbe50f0..641dd48be36 100644 --- a/spec/lib/gitlab/database/reflection_spec.rb +++ b/spec/lib/gitlab/database/reflection_spec.rb @@ -191,9 +191,15 @@ RSpec.describe Gitlab::Database::Reflection, feature_category: :database do expect(database.postgresql_minimum_supported_version?).to eq(false) end - it 'returns true when using PostgreSQL 12' do + it 'returns false when using PostgreSQL 12' do allow(database).to receive(:version).and_return('12') + expect(database.postgresql_minimum_supported_version?).to eq(false) + end + + it 'returns true when using PostgreSQL 13' do + allow(database).to receive(:version).and_return('13') + expect(database.postgresql_minimum_supported_version?).to eq(true) end end diff --git a/spec/lib/gitlab/database/reindexing_spec.rb b/spec/lib/gitlab/database/reindexing_spec.rb index a8af9bb5a38..4d0e58b0937 100644 --- a/spec/lib/gitlab/database/reindexing_spec.rb +++ b/spec/lib/gitlab/database/reindexing_spec.rb @@ -71,7 +71,7 @@ RSpec.describe Gitlab::Database::Reindexing, feature_category: :database, time_t context 'when async FK validation is enabled' do it 'executes FK validation for each database prior to any reindexing actions' do - expect(Gitlab::Database::AsyncForeignKeys).to receive(:validate_pending_entries!).ordered.exactly(databases_count).times + expect(Gitlab::Database::AsyncConstraints).to receive(:validate_pending_entries!).ordered.exactly(databases_count).times expect(described_class).to receive(:automatic_reindexing).ordered.exactly(databases_count).times described_class.invoke @@ -82,7 +82,7 @@ RSpec.describe Gitlab::Database::Reindexing, feature_category: :database, time_t it 'does not execute FK validation' do stub_feature_flags(database_async_foreign_key_validation: false) - expect(Gitlab::Database::AsyncForeignKeys).not_to receive(:validate_pending_entries!) + expect(Gitlab::Database::AsyncConstraints).not_to receive(:validate_pending_entries!) described_class.invoke end diff --git a/spec/lib/gitlab/database/schema_validation/adapters/column_database_adapter_spec.rb b/spec/lib/gitlab/database/schema_validation/adapters/column_database_adapter_spec.rb new file mode 100644 index 00000000000..d81f5f3dbec --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/adapters/column_database_adapter_spec.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::Adapters::ColumnDatabaseAdapter, feature_category: :database do + subject(:adapter) { described_class.new(db_result) } + + let(:column_name) { 'email' } + let(:column_default) { "'no-reply@gitlab.com'::character varying" } + let(:not_null) { true } + let(:partition_key) { false } + let(:db_result) do + { + 'table_name' => 'projects', + 'column_name' => column_name, + 'data_type' => 'character varying', + 'column_default' => column_default, + 'not_null' => not_null, + 'partition_key' => partition_key + } + end + + describe '#name' do + it { expect(adapter.name).to eq('email') } + end + + describe '#table_name' do + it { expect(adapter.table_name).to eq('projects') } + end + + describe '#data_type' do + it { expect(adapter.data_type).to eq('character varying') } + end + + describe '#default' do + context "when there's no default value in the column" do + let(:column_default) { nil } + + it { expect(adapter.default).to be_nil } + end + + context 'when the column name is id' do + let(:column_name) { 'id' } + + it { expect(adapter.default).to be_nil } + end + + context 'when the column default includes nextval' do + let(:column_default) { "nextval('my_seq'::regclass)" } + + it { expect(adapter.default).to be_nil } + end + + it { expect(adapter.default).to eq("DEFAULT 'no-reply@gitlab.com'::character varying") } + end + + describe '#nullable' do + context 'when column is not null' do + it { expect(adapter.nullable).to eq('NOT NULL') } + end + + context 'when column is nullable' do + let(:not_null) { false } + + it { expect(adapter.nullable).to be_nil } + end + end + + describe '#partition_key?' do + it { expect(adapter.partition_key?).to be(false) } + end +end diff --git a/spec/lib/gitlab/database/schema_validation/adapters/column_structure_sql_adapter_spec.rb b/spec/lib/gitlab/database/schema_validation/adapters/column_structure_sql_adapter_spec.rb new file mode 100644 index 00000000000..64b59e65be6 --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/adapters/column_structure_sql_adapter_spec.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::Adapters::ColumnStructureSqlAdapter, feature_category: :database do + subject(:adapter) { described_class.new(table_name, column_def, partition_stmt) } + + let(:table_name) { 'test_table' } + let(:file_path) { Rails.root.join('spec/fixtures/structure.sql') } + let(:table_stmts) { PgQuery.parse(File.read(file_path)).tree.stmts.filter_map { |s| s.stmt.create_stmt } } + let(:table) { table_stmts.find { |table| table.relation.relname == table_name } } + let(:partition_stmt) { table.partspec } + let(:column_stmts) { table.table_elts } + let(:column_def) { column_stmts.find { |col| col.column_def.colname == column_name }.column_def } + + where(:column_name, :data_type, :default_value, :nullable, :partition_key) do + [ + ['id', 'bigint', nil, 'NOT NULL', false], + ['integer_column', 'integer', nil, nil, false], + ['integer_with_default_column', 'integer', 'DEFAULT 1', nil, false], + ['smallint_with_default_column', 'smallint', 'DEFAULT 0', 'NOT NULL', false], + ['double_precision_with_default_column', 'double precision', 'DEFAULT 1.0', nil, false], + ['numeric_with_default_column', 'numeric', 'DEFAULT 1.0', 'NOT NULL', false], + ['boolean_with_default_colum', 'boolean', 'DEFAULT true', 'NOT NULL', false], + ['varying_with_default_column', 'character varying', "DEFAULT 'DEFAULT'::character varying", 'NOT NULL', false], + ['varying_with_limit_and_default_column', 'character varying(255)', "DEFAULT 'DEFAULT'::character varying", + nil, false], + ['text_with_default_column', 'text', "DEFAULT ''::text", 'NOT NULL', false], + ['array_with_default_column', 'character varying(255)[]', "DEFAULT '{one,two}'::character varying[]", + 'NOT NULL', false], + ['jsonb_with_default_column', 'jsonb', "DEFAULT '[]'::jsonb", 'NOT NULL', false], + ['timestamptz_with_default_column', 'timestamp(6) with time zone', "DEFAULT now()", nil, false], + ['timestamp_with_default_column', 'timestamp(6) without time zone', + "DEFAULT '2022-01-23 00:00:00+00'::timestamp without time zone", 'NOT NULL', false], + ['date_with_default_column', 'date', 'DEFAULT 2023-04-05', nil, false], + ['inet_with_default_column', 'inet', "DEFAULT '0.0.0.0'::inet", 'NOT NULL', false], + ['macaddr_with_default_column', 'macaddr', "DEFAULT '00-00-00-00-00-000'::macaddr", 'NOT NULL', false], + ['uuid_with_default_column', 'uuid', "DEFAULT '00000000-0000-0000-0000-000000000000'::uuid", 'NOT NULL', false], + ['partition_key', 'bigint', 'DEFAULT 1', 'NOT NULL', true], + ['created_at', 'timestamp with time zone', 'DEFAULT now()', 'NOT NULL', true] + ] + end + + with_them do + describe '#name' do + it { expect(adapter.name).to eq(column_name) } + end + + describe '#table_name' do + it { expect(adapter.table_name).to eq(table_name) } + end + + describe '#data_type' do + it { expect(adapter.data_type).to eq(data_type) } + end + + describe '#nullable' do + it { expect(adapter.nullable).to eq(nullable) } + end + + describe '#default' do + it { expect(adapter.default).to eq(default_value) } + end + + describe '#partition_key?' do + it { expect(adapter.partition_key?).to eq(partition_key) } + end + end + + context 'when the data type is not mapped' do + let(:column_name) { 'unmapped_column_type' } + let(:error_class) { Gitlab::Database::SchemaValidation::Adapters::UndefinedPGType } + + describe '#data_type' do + it { expect { adapter.data_type }.to raise_error(error_class) } + end + end +end diff --git a/spec/lib/gitlab/database/schema_validation/database_spec.rb b/spec/lib/gitlab/database/schema_validation/database_spec.rb index c0026f91b46..0b5f433b1c9 100644 --- a/spec/lib/gitlab/database/schema_validation/database_spec.rb +++ b/spec/lib/gitlab/database/schema_validation/database_spec.rb @@ -2,44 +2,92 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::SchemaValidation::Database, feature_category: :database do - let(:database_name) { 'main' } - let(:database_indexes) do - [['index', 'CREATE UNIQUE INDEX "index" ON public.achievements USING btree (namespace_id, lower(name))']] - end +RSpec.shared_examples 'database schema assertions for' do |fetch_by_name_method, exists_method, all_objects_method| + subject(:database) { described_class.new(connection) } - let(:query_result) { instance_double('ActiveRecord::Result', rows: database_indexes) } - let(:database_model) { Gitlab::Database.database_base_models[database_name] } + let(:database_model) { Gitlab::Database.database_base_models['main'] } let(:connection) { database_model.connection } - subject(:database) { described_class.new(connection) } - before do - allow(connection).to receive(:exec_query).and_return(query_result) + allow(connection).to receive(:select_rows).and_return(results) + allow(connection).to receive(:exec_query).and_return(results) end - describe '#fetch_index_by_name' do - context 'when index does not exist' do - it 'returns nil' do - index = database.fetch_index_by_name('non_existing_index') + describe "##{fetch_by_name_method}" do + it 'returns nil when schema object does not exists' do + expect(database.public_send(fetch_by_name_method, 'invalid-object-name')).to be_nil + end + + it 'returns the schema object by name' do + expect(database.public_send(fetch_by_name_method, valid_schema_object_name).name).to eq(valid_schema_object_name) + end + end + + describe "##{exists_method}" do + it 'returns true when schema object exists' do + expect(database.public_send(exists_method, valid_schema_object_name)).to be_truthy + end - expect(index).to be_nil - end + it 'returns false when schema object does not exists' do + expect(database.public_send(exists_method, 'invalid-object')).to be_falsey end + end - it 'returns index by name' do - index = database.fetch_index_by_name('index') + describe "##{all_objects_method}" do + it 'returns all the schema objects' do + schema_objects = database.public_send(all_objects_method) - expect(index.name).to eq('index') + expect(schema_objects).to all(be_a(schema_object)) + expect(schema_objects.map(&:name)).to eq([valid_schema_object_name]) end end +end + +RSpec.describe Gitlab::Database::SchemaValidation::Database, feature_category: :database do + context 'when having indexes' do + let(:schema_object) { Gitlab::Database::SchemaValidation::SchemaObjects::Index } + let(:valid_schema_object_name) { 'index' } + let(:results) do + [['index', 'CREATE UNIQUE INDEX "index" ON public.achievements USING btree (namespace_id, lower(name))']] + end - describe '#indexes' do - it 'returns indexes' do - indexes = database.indexes + include_examples 'database schema assertions for', 'fetch_index_by_name', 'index_exists?', 'indexes' + end - expect(indexes).to all(be_a(Gitlab::Database::SchemaValidation::Index)) - expect(indexes.map(&:name)).to eq(['index']) + context 'when having triggers' do + let(:schema_object) { Gitlab::Database::SchemaValidation::SchemaObjects::Trigger } + let(:valid_schema_object_name) { 'my_trigger' } + let(:results) do + [['my_trigger', 'CREATE TRIGGER my_trigger BEFORE INSERT ON todos FOR EACH ROW EXECUTE FUNCTION trigger()']] end + + include_examples 'database schema assertions for', 'fetch_trigger_by_name', 'trigger_exists?', 'triggers' + end + + context 'when having tables' do + let(:schema_object) { Gitlab::Database::SchemaValidation::SchemaObjects::Table } + let(:valid_schema_object_name) { 'my_table' } + let(:results) do + [ + { + 'table_name' => 'my_table', + 'column_name' => 'id', + 'not_null' => true, + 'data_type' => 'bigint', + 'partition_key' => false, + 'column_default' => "nextval('audit_events_id_seq'::regclass)" + }, + { + 'table_name' => 'my_table', + 'column_name' => 'details', + 'not_null' => false, + 'data_type' => 'text', + 'partition_key' => false, + 'column_default' => nil + } + ] + end + + include_examples 'database schema assertions for', 'fetch_table_by_name', 'table_exists?', 'tables' end end diff --git a/spec/lib/gitlab/database/schema_validation/inconsistency_spec.rb b/spec/lib/gitlab/database/schema_validation/inconsistency_spec.rb new file mode 100644 index 00000000000..a49ff8339a1 --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/inconsistency_spec.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::Inconsistency, feature_category: :database do + let(:validator) { Gitlab::Database::SchemaValidation::Validators::DifferentDefinitionIndexes } + + let(:database_statement) { 'CREATE INDEX index_name ON public.achievements USING btree (namespace_id)' } + let(:structure_sql_statement) { 'CREATE INDEX index_name ON public.achievements USING btree (id)' } + + let(:structure_stmt) { PgQuery.parse(structure_sql_statement).tree.stmts.first.stmt.index_stmt } + let(:database_stmt) { PgQuery.parse(database_statement).tree.stmts.first.stmt.index_stmt } + + let(:structure_sql_object) { Gitlab::Database::SchemaValidation::SchemaObjects::Index.new(structure_stmt) } + let(:database_object) { Gitlab::Database::SchemaValidation::SchemaObjects::Index.new(database_stmt) } + + subject(:inconsistency) { described_class.new(validator, structure_sql_object, database_object) } + + describe '#object_name' do + it 'returns the index name' do + expect(inconsistency.object_name).to eq('index_name') + end + end + + describe '#diff' do + it 'returns a diff between the structure.sql and the database' do + expect(inconsistency.diff).to be_a(Diffy::Diff) + expect(inconsistency.diff.string1).to eq("#{structure_sql_statement}\n") + expect(inconsistency.diff.string2).to eq("#{database_statement}\n") + end + end + + describe '#error_message' do + it 'returns the error message' do + stub_const "#{validator}::ERROR_MESSAGE", 'error message %s' + + expect(inconsistency.error_message).to eq('error message index_name') + end + end + + describe '#type' do + it 'returns the type of the validator' do + expect(inconsistency.type).to eq('different_definition_indexes') + end + end + + describe '#table_name' do + it 'returns the table name' do + expect(inconsistency.table_name).to eq('achievements') + end + end + + describe '#object_type' do + it 'returns the structure sql object type' do + expect(inconsistency.object_type).to eq('Index') + end + + context 'when the structure sql object is not available' do + subject(:inconsistency) { described_class.new(validator, nil, database_object) } + + it 'returns the database object type' do + expect(inconsistency.object_type).to eq('Index') + end + end + end + + describe '#structure_sql_statement' do + it 'returns structure sql statement' do + expect(inconsistency.structure_sql_statement).to eq("#{structure_sql_statement}\n") + end + end + + describe '#database_statement' do + it 'returns database statement' do + expect(inconsistency.database_statement).to eq("#{database_statement}\n") + end + end + + describe '#inspect' do + let(:expected_output) do + <<~MSG + ------------------------------------------------------ + The index_name index has a different statement between structure.sql and database + Diff: + \e[31m-CREATE INDEX index_name ON public.achievements USING btree (id)\e[0m + \e[32m+CREATE INDEX index_name ON public.achievements USING btree (namespace_id)\e[0m + + ------------------------------------------------------ + MSG + end + + it 'prints the inconsistency message' do + expect(inconsistency.inspect).to eql(expected_output) + end + end +end diff --git a/spec/lib/gitlab/database/schema_validation/index_spec.rb b/spec/lib/gitlab/database/schema_validation/index_spec.rb deleted file mode 100644 index 297211d79ed..00000000000 --- a/spec/lib/gitlab/database/schema_validation/index_spec.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true -require 'spec_helper' - -RSpec.describe Gitlab::Database::SchemaValidation::Index, feature_category: :database do - let(:index_statement) { 'CREATE INDEX index_name ON public.achievements USING btree (namespace_id)' } - - let(:stmt) { PgQuery.parse(index_statement).tree.stmts.first.stmt.index_stmt } - - let(:index) { described_class.new(stmt) } - - describe '#name' do - it 'returns index name' do - expect(index.name).to eq('index_name') - end - end - - describe '#statement' do - it 'returns index statement' do - expect(index.statement).to eq(index_statement) - end - end -end diff --git a/spec/lib/gitlab/database/schema_validation/indexes_spec.rb b/spec/lib/gitlab/database/schema_validation/indexes_spec.rb deleted file mode 100644 index 4351031a4b4..00000000000 --- a/spec/lib/gitlab/database/schema_validation/indexes_spec.rb +++ /dev/null @@ -1,56 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Database::SchemaValidation::Indexes, feature_category: :database do - let(:structure_file_path) { Rails.root.join('spec/fixtures/structure.sql') } - let(:database_indexes) do - [ - ['wrong_index', 'CREATE UNIQUE INDEX wrong_index ON public.table_name (column_name)'], - ['extra_index', 'CREATE INDEX extra_index ON public.table_name (column_name)'], - ['index', 'CREATE UNIQUE INDEX "index" ON public.achievements USING btree (namespace_id, lower(name))'] - ] - end - - let(:database_name) { 'main' } - - let(:database_model) { Gitlab::Database.database_base_models[database_name] } - - let(:connection) { database_model.connection } - - let(:query_result) { instance_double('ActiveRecord::Result', rows: database_indexes) } - - let(:database) { Gitlab::Database::SchemaValidation::Database.new(connection) } - let(:structure_file) { Gitlab::Database::SchemaValidation::StructureSql.new(structure_file_path) } - - subject(:schema_validation) { described_class.new(structure_file, database) } - - before do - allow(connection).to receive(:exec_query).and_return(query_result) - end - - describe '#missing_indexes' do - it 'returns missing indexes' do - missing_indexes = %w[ - missing_index - index_namespaces_public_groups_name_id - index_on_deploy_keys_id_and_type_and_public - index_users_on_public_email_excluding_null_and_empty - ] - - expect(schema_validation.missing_indexes).to match_array(missing_indexes) - end - end - - describe '#extra_indexes' do - it 'returns extra indexes' do - expect(schema_validation.extra_indexes).to match_array(['extra_index']) - end - end - - describe '#wrong_indexes' do - it 'returns wrong indexes' do - expect(schema_validation.wrong_indexes).to match_array(['wrong_index']) - end - end -end diff --git a/spec/lib/gitlab/database/schema_validation/runner_spec.rb b/spec/lib/gitlab/database/schema_validation/runner_spec.rb new file mode 100644 index 00000000000..f5d1c6ba31b --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/runner_spec.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::Runner, feature_category: :database do + let(:structure_file_path) { Rails.root.join('spec/fixtures/structure.sql') } + let(:connection) { ActiveRecord::Base.connection } + + let(:database) { Gitlab::Database::SchemaValidation::Database.new(connection) } + let(:structure_sql) { Gitlab::Database::SchemaValidation::StructureSql.new(structure_file_path, 'public') } + + describe '#execute' do + subject(:inconsistencies) { described_class.new(structure_sql, database).execute } + + it 'returns inconsistencies' do + expect(inconsistencies).not_to be_empty + end + + it 'execute all validators' do + all_validators = Gitlab::Database::SchemaValidation::Validators::BaseValidator.all_validators + + expect(all_validators).to all(receive(:new).with(structure_sql, database).and_call_original) + + inconsistencies + end + + context 'when validators are passed' do + subject(:inconsistencies) { described_class.new(structure_sql, database, validators: validators).execute } + + let(:class_name) { 'Gitlab::Database::SchemaValidation::Validators::ExtraIndexes' } + let(:inconsistency_class_name) { 'Gitlab::Database::SchemaValidation::Inconsistency' } + + let(:extra_indexes) { class_double(class_name) } + let(:instace_extra_index) { instance_double(class_name, execute: [inconsistency]) } + let(:inconsistency) { instance_double(inconsistency_class_name, object_name: 'test') } + + let(:validators) { [extra_indexes] } + + it 'only execute the validators passed' do + expect(extra_indexes).to receive(:new).with(structure_sql, database).and_return(instace_extra_index) + + Gitlab::Database::SchemaValidation::Validators::BaseValidator.all_validators.each do |validator| + expect(validator).not_to receive(:new).with(structure_sql, database) + end + + expect(inconsistencies.map(&:object_name)).to eql ['test'] + end + end + end +end diff --git a/spec/lib/gitlab/database/schema_validation/schema_inconsistency_spec.rb b/spec/lib/gitlab/database/schema_validation/schema_inconsistency_spec.rb new file mode 100644 index 00000000000..7d6a279def9 --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/schema_inconsistency_spec.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::SchemaInconsistency, type: :model, feature_category: :database do + it { is_expected.to be_a ApplicationRecord } + + describe 'associations' do + it { is_expected.to belong_to(:issue) } + end + + describe "Validations" do + it { is_expected.to validate_presence_of(:object_name) } + it { is_expected.to validate_presence_of(:valitador_name) } + it { is_expected.to validate_presence_of(:table_name) } + end +end diff --git a/spec/lib/gitlab/database/schema_validation/schema_objects/column_spec.rb b/spec/lib/gitlab/database/schema_validation/schema_objects/column_spec.rb new file mode 100644 index 00000000000..74bc5f43b50 --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/schema_objects/column_spec.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::SchemaObjects::Column, feature_category: :database do + subject(:column) { described_class.new(adapter) } + + let(:database_adapter) { 'Gitlab::Database::SchemaValidation::Adapters::ColumnDatabaseAdapter' } + let(:adapter) do + instance_double(database_adapter, name: 'id', table_name: 'projects', + data_type: 'bigint', default: nil, nullable: 'NOT NULL') + end + + describe '#name' do + it { expect(column.name).to eq('id') } + end + + describe '#table_name' do + it { expect(column.table_name).to eq('projects') } + end + + describe '#statement' do + it { expect(column.statement).to eq('id bigint NOT NULL') } + end +end diff --git a/spec/lib/gitlab/database/schema_validation/schema_objects/index_spec.rb b/spec/lib/gitlab/database/schema_validation/schema_objects/index_spec.rb new file mode 100644 index 00000000000..43d8fa38ec8 --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/schema_objects/index_spec.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::SchemaObjects::Index, feature_category: :database do + let(:statement) { 'CREATE INDEX index_name ON public.achievements USING btree (namespace_id)' } + let(:name) { 'index_name' } + let(:table_name) { 'achievements' } + + include_examples 'schema objects assertions for', 'index_stmt' +end diff --git a/spec/lib/gitlab/database/schema_validation/schema_objects/table_spec.rb b/spec/lib/gitlab/database/schema_validation/schema_objects/table_spec.rb new file mode 100644 index 00000000000..60ea9581517 --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/schema_objects/table_spec.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::SchemaObjects::Table, feature_category: :database do + subject(:table) { described_class.new(name, columns) } + + let(:name) { 'my_table' } + let(:column_class) { 'Gitlab::Database::SchemaValidation::SchemaObjects::Column' } + let(:columns) do + [ + instance_double(column_class, name: 'id', statement: 'id bigint NOT NULL', partition_key?: false), + instance_double(column_class, name: 'col', statement: 'col text', partition_key?: false), + instance_double(column_class, name: 'partition', statement: 'partition integer DEFAULT 1', partition_key?: true) + ] + end + + describe '#name' do + it { expect(table.name).to eq('my_table') } + end + + describe '#table_name' do + it { expect(table.table_name).to eq('my_table') } + end + + describe '#statement' do + it { expect(table.statement).to eq('CREATE TABLE my_table (id bigint NOT NULL, col text)') } + + it 'ignores the partition column' do + expect(table.statement).not_to include('partition integer DEFAULT 1') + end + end + + describe '#fetch_column_by_name' do + it { expect(table.fetch_column_by_name('col')).not_to be_nil } + + it { expect(table.fetch_column_by_name('invalid')).to be_nil } + end + + describe '#column_exists?' do + it { expect(table.column_exists?('col')).to eq(true) } + + it { expect(table.column_exists?('invalid')).to eq(false) } + end +end diff --git a/spec/lib/gitlab/database/schema_validation/schema_objects/trigger_spec.rb b/spec/lib/gitlab/database/schema_validation/schema_objects/trigger_spec.rb new file mode 100644 index 00000000000..3c2481dfae0 --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/schema_objects/trigger_spec.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::SchemaObjects::Trigger, feature_category: :database do + let(:statement) { 'CREATE TRIGGER my_trigger BEFORE INSERT ON todos FOR EACH ROW EXECUTE FUNCTION trigger()' } + let(:name) { 'my_trigger' } + let(:table_name) { 'todos' } + + include_examples 'schema objects assertions for', 'create_trig_stmt' +end diff --git a/spec/lib/gitlab/database/schema_validation/structure_sql_spec.rb b/spec/lib/gitlab/database/schema_validation/structure_sql_spec.rb new file mode 100644 index 00000000000..b0c056ff5db --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/structure_sql_spec.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.shared_examples 'structure sql schema assertions for' do |object_exists_method, all_objects_method| + subject(:structure_sql) { described_class.new(structure_file_path, schema_name) } + + let(:structure_file_path) { Rails.root.join('spec/fixtures/structure.sql') } + let(:schema_name) { 'public' } + + describe "##{object_exists_method}" do + it 'returns true when schema object exists' do + expect(structure_sql.public_send(object_exists_method, valid_schema_object_name)).to be_truthy + end + + it 'returns false when schema object does not exists' do + expect(structure_sql.public_send(object_exists_method, 'invalid-object-name')).to be_falsey + end + end + + describe "##{all_objects_method}" do + it 'returns all the schema objects' do + schema_objects = structure_sql.public_send(all_objects_method) + + expect(schema_objects).to all(be_a(schema_object)) + expect(schema_objects.map(&:name)).to eq(expected_objects) + end + end +end + +RSpec.describe Gitlab::Database::SchemaValidation::StructureSql, feature_category: :database do + let(:structure_file_path) { Rails.root.join('spec/fixtures/structure.sql') } + let(:schema_name) { 'public' } + + subject(:structure_sql) { described_class.new(structure_file_path, schema_name) } + + context 'when having indexes' do + let(:schema_object) { Gitlab::Database::SchemaValidation::SchemaObjects::Index } + let(:valid_schema_object_name) { 'index' } + let(:expected_objects) do + %w[missing_index wrong_index index index_namespaces_public_groups_name_id + index_on_deploy_keys_id_and_type_and_public index_users_on_public_email_excluding_null_and_empty] + end + + include_examples 'structure sql schema assertions for', 'index_exists?', 'indexes' + end + + context 'when having triggers' do + let(:schema_object) { Gitlab::Database::SchemaValidation::SchemaObjects::Trigger } + let(:valid_schema_object_name) { 'trigger' } + let(:expected_objects) { %w[trigger wrong_trigger missing_trigger_1 projects_loose_fk_trigger] } + + include_examples 'structure sql schema assertions for', 'trigger_exists?', 'triggers' + end + + context 'when having tables' do + let(:schema_object) { Gitlab::Database::SchemaValidation::SchemaObjects::Table } + let(:valid_schema_object_name) { 'test_table' } + let(:expected_objects) do + %w[test_table ci_project_mirrors wrong_table extra_table_columns missing_table missing_table_columns + operations_user_lists] + end + + include_examples 'structure sql schema assertions for', 'table_exists?', 'tables' + end +end diff --git a/spec/lib/gitlab/database/schema_validation/track_inconsistency_spec.rb b/spec/lib/gitlab/database/schema_validation/track_inconsistency_spec.rb new file mode 100644 index 00000000000..84db721fc2d --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/track_inconsistency_spec.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::TrackInconsistency, feature_category: :database do + describe '#execute' do + let(:validator) { Gitlab::Database::SchemaValidation::Validators::DifferentDefinitionIndexes } + + let(:database_statement) { 'CREATE INDEX index_name ON public.achievements USING btree (namespace_id)' } + let(:structure_sql_statement) { 'CREATE INDEX index_name ON public.achievements USING btree (id)' } + + let(:structure_stmt) { PgQuery.parse(structure_sql_statement).tree.stmts.first.stmt.index_stmt } + let(:database_stmt) { PgQuery.parse(database_statement).tree.stmts.first.stmt.index_stmt } + + let(:structure_sql_object) { Gitlab::Database::SchemaValidation::SchemaObjects::Index.new(structure_stmt) } + let(:database_object) { Gitlab::Database::SchemaValidation::SchemaObjects::Index.new(database_stmt) } + + let(:inconsistency) do + Gitlab::Database::SchemaValidation::Inconsistency.new(validator, structure_sql_object, database_object) + end + + let_it_be(:project) { create(:project) } + let_it_be(:user) { create(:user) } + + subject(:execute) { described_class.new(inconsistency, project, user).execute } + + before do + stub_spam_services + end + + context 'when is not GitLab.com' do + it 'does not create a schema inconsistency record' do + allow(Gitlab).to receive(:com?).and_return(false) + + expect { execute }.not_to change { Gitlab::Database::SchemaValidation::SchemaInconsistency.count } + end + end + + context 'when the issue creation fails' do + let(:issue_creation) { instance_double(Mutations::Issues::Create, resolve: { errors: 'error' }) } + + before do + allow(Mutations::Issues::Create).to receive(:new).and_return(issue_creation) + end + + it 'does not create a schema inconsistency record' do + allow(Gitlab).to receive(:com?).and_return(true) + + expect { execute }.not_to change { Gitlab::Database::SchemaValidation::SchemaInconsistency.count } + end + end + + context 'when a new inconsistency is found' do + before do + project.add_developer(user) + end + + it 'creates a new schema inconsistency record' do + allow(Gitlab).to receive(:com?).and_return(true) + + expect { execute }.to change { Gitlab::Database::SchemaValidation::SchemaInconsistency.count } + end + end + + context 'when the schema inconsistency already exists' do + before do + project.add_developer(user) + end + + let!(:schema_inconsistency) do + create(:schema_inconsistency, object_name: 'index_name', table_name: 'achievements', + valitador_name: 'different_definition_indexes') + end + + it 'does not create a schema inconsistency record' do + allow(Gitlab).to receive(:com?).and_return(true) + + expect { execute }.not_to change { Gitlab::Database::SchemaValidation::SchemaInconsistency.count } + end + end + end +end diff --git a/spec/lib/gitlab/database/schema_validation/validators/base_validator_spec.rb b/spec/lib/gitlab/database/schema_validation/validators/base_validator_spec.rb new file mode 100644 index 00000000000..036ad6424f0 --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/validators/base_validator_spec.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::Validators::BaseValidator, feature_category: :database do + describe '.all_validators' do + subject(:all_validators) { described_class.all_validators } + + it 'returns an array of all validators' do + expect(all_validators).to eq([ + Gitlab::Database::SchemaValidation::Validators::ExtraTables, + Gitlab::Database::SchemaValidation::Validators::ExtraTableColumns, + Gitlab::Database::SchemaValidation::Validators::ExtraIndexes, + Gitlab::Database::SchemaValidation::Validators::ExtraTriggers, + Gitlab::Database::SchemaValidation::Validators::MissingTables, + Gitlab::Database::SchemaValidation::Validators::MissingTableColumns, + Gitlab::Database::SchemaValidation::Validators::MissingIndexes, + Gitlab::Database::SchemaValidation::Validators::MissingTriggers, + Gitlab::Database::SchemaValidation::Validators::DifferentDefinitionTables, + Gitlab::Database::SchemaValidation::Validators::DifferentDefinitionIndexes, + Gitlab::Database::SchemaValidation::Validators::DifferentDefinitionTriggers + ]) + end + end + + describe '#execute' do + let(:structure_sql) { instance_double(Gitlab::Database::SchemaValidation::StructureSql) } + let(:database) { instance_double(Gitlab::Database::SchemaValidation::Database) } + + subject(:inconsistencies) { described_class.new(structure_sql, database).execute } + + it 'raises an exception' do + expect { inconsistencies }.to raise_error(NoMethodError) + end + end +end diff --git a/spec/lib/gitlab/database/schema_validation/validators/different_definition_indexes_spec.rb b/spec/lib/gitlab/database/schema_validation/validators/different_definition_indexes_spec.rb new file mode 100644 index 00000000000..b9744c86b80 --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/validators/different_definition_indexes_spec.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::Validators::DifferentDefinitionIndexes, + feature_category: :database do + include_examples 'index validators', described_class, ['wrong_index'] +end diff --git a/spec/lib/gitlab/database/schema_validation/validators/different_definition_tables_spec.rb b/spec/lib/gitlab/database/schema_validation/validators/different_definition_tables_spec.rb new file mode 100644 index 00000000000..746418b757e --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/validators/different_definition_tables_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::Validators::DifferentDefinitionTables, feature_category: :database do + include_examples 'table validators', described_class, ['wrong_table'] +end diff --git a/spec/lib/gitlab/database/schema_validation/validators/different_definition_triggers_spec.rb b/spec/lib/gitlab/database/schema_validation/validators/different_definition_triggers_spec.rb new file mode 100644 index 00000000000..4d065929708 --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/validators/different_definition_triggers_spec.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::Validators::DifferentDefinitionTriggers, + feature_category: :database do + include_examples 'trigger validators', described_class, ['wrong_trigger'] +end diff --git a/spec/lib/gitlab/database/schema_validation/validators/extra_indexes_spec.rb b/spec/lib/gitlab/database/schema_validation/validators/extra_indexes_spec.rb new file mode 100644 index 00000000000..842dbb42120 --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/validators/extra_indexes_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::Validators::ExtraIndexes, feature_category: :database do + include_examples 'index validators', described_class, ['extra_index'] +end diff --git a/spec/lib/gitlab/database/schema_validation/validators/extra_table_columns_spec.rb b/spec/lib/gitlab/database/schema_validation/validators/extra_table_columns_spec.rb new file mode 100644 index 00000000000..9d17a2fffa9 --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/validators/extra_table_columns_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::Validators::ExtraTableColumns, feature_category: :database do + include_examples 'table validators', described_class, ['extra_table_columns'] +end diff --git a/spec/lib/gitlab/database/schema_validation/validators/extra_tables_spec.rb b/spec/lib/gitlab/database/schema_validation/validators/extra_tables_spec.rb new file mode 100644 index 00000000000..edaf79e3c93 --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/validators/extra_tables_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::Validators::ExtraTables, feature_category: :database do + include_examples 'table validators', described_class, ['extra_table'] +end diff --git a/spec/lib/gitlab/database/schema_validation/validators/extra_triggers_spec.rb b/spec/lib/gitlab/database/schema_validation/validators/extra_triggers_spec.rb new file mode 100644 index 00000000000..d2e1c18a1ab --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/validators/extra_triggers_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::Validators::ExtraTriggers, feature_category: :database do + include_examples 'trigger validators', described_class, ['extra_trigger'] +end diff --git a/spec/lib/gitlab/database/schema_validation/validators/missing_indexes_spec.rb b/spec/lib/gitlab/database/schema_validation/validators/missing_indexes_spec.rb new file mode 100644 index 00000000000..c402c3a2fa7 --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/validators/missing_indexes_spec.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::Validators::MissingIndexes, feature_category: :database do + missing_indexes = %w[ + missing_index + index_namespaces_public_groups_name_id + index_on_deploy_keys_id_and_type_and_public + index_users_on_public_email_excluding_null_and_empty + ] + + include_examples 'index validators', described_class, missing_indexes +end diff --git a/spec/lib/gitlab/database/schema_validation/validators/missing_table_columns_spec.rb b/spec/lib/gitlab/database/schema_validation/validators/missing_table_columns_spec.rb new file mode 100644 index 00000000000..de2956b4dd9 --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/validators/missing_table_columns_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::Validators::MissingTableColumns, feature_category: :database do + include_examples 'table validators', described_class, ['missing_table_columns'] +end diff --git a/spec/lib/gitlab/database/schema_validation/validators/missing_tables_spec.rb b/spec/lib/gitlab/database/schema_validation/validators/missing_tables_spec.rb new file mode 100644 index 00000000000..7c80923e860 --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/validators/missing_tables_spec.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::Validators::MissingTables, feature_category: :database do + missing_tables = %w[ci_project_mirrors missing_table operations_user_lists test_table] + + include_examples 'table validators', described_class, missing_tables +end diff --git a/spec/lib/gitlab/database/schema_validation/validators/missing_triggers_spec.rb b/spec/lib/gitlab/database/schema_validation/validators/missing_triggers_spec.rb new file mode 100644 index 00000000000..87bc3ded808 --- /dev/null +++ b/spec/lib/gitlab/database/schema_validation/validators/missing_triggers_spec.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::SchemaValidation::Validators::MissingTriggers, feature_category: :database do + missing_triggers = %w[missing_trigger_1 projects_loose_fk_trigger] + + include_examples 'trigger validators', described_class, missing_triggers +end diff --git a/spec/lib/gitlab/database/tables_locker_spec.rb b/spec/lib/gitlab/database/tables_locker_spec.rb index d74f455eaad..aaafe27f7ca 100644 --- a/spec/lib/gitlab/database/tables_locker_spec.rb +++ b/spec/lib/gitlab/database/tables_locker_spec.rb @@ -2,20 +2,42 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::TablesLocker, :reestablished_active_record_base, :delete, :silence_stdout, - :suppress_gitlab_schemas_validate_connection, feature_category: :pods do - let(:detached_partition_table) { '_test_gitlab_main_part_20220101' } - let(:lock_writes_manager) do - instance_double(Gitlab::Database::LockWritesManager, lock_writes: nil, unlock_writes: nil) +RSpec.describe Gitlab::Database::TablesLocker, :suppress_gitlab_schemas_validate_connection, :silence_stdout, + feature_category: :cell do + let(:default_lock_writes_manager) do + instance_double( + Gitlab::Database::LockWritesManager, + lock_writes: { action: 'any action' }, + unlock_writes: { action: 'unlocked' } + ) end before do - allow(Gitlab::Database::LockWritesManager).to receive(:new).with(any_args).and_return(lock_writes_manager) + allow(Gitlab::Database::LockWritesManager).to receive(:new).with(any_args).and_return(default_lock_writes_manager) + # Limiting the scope of the tests to a subset of the database tables + allow(Gitlab::Database::GitlabSchema).to receive(:tables_to_schema).and_return({ + 'application_setttings' => :gitlab_main_clusterwide, + 'projects' => :gitlab_main, + 'security_findings' => :gitlab_main, + 'ci_builds' => :gitlab_ci, + 'ci_jobs' => :gitlab_ci, + 'loose_foreign_keys_deleted_records' => :gitlab_shared, + 'ar_internal_metadata' => :gitlab_internal + }) end before(:all) do + create_partition_sql = <<~SQL + CREATE TABLE IF NOT EXISTS #{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.security_findings_test_partition + PARTITION OF security_findings + FOR VALUES IN (0) + SQL + + ApplicationRecord.connection.execute(create_partition_sql) + Ci::ApplicationRecord.connection.execute(create_partition_sql) + create_detached_partition_sql = <<~SQL - CREATE TABLE IF NOT EXISTS gitlab_partitions_dynamic._test_gitlab_main_part_20220101 ( + CREATE TABLE IF NOT EXISTS #{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}._test_gitlab_main_part_202201 ( id bigserial primary key not null ) SQL @@ -29,35 +51,89 @@ RSpec.describe Gitlab::Database::TablesLocker, :reestablished_active_record_base drop_after: Time.current ) end + Gitlab::Database::SharedModel.using_connection(Ci::ApplicationRecord.connection) do + Postgresql::DetachedPartition.create!( + table_name: '_test_gitlab_main_part_20220101', + drop_after: Time.current + ) + end end - after(:all) do - drop_detached_partition_sql = <<~SQL - DROP TABLE IF EXISTS gitlab_partitions_dynamic._test_gitlab_main_part_20220101 - SQL + shared_examples "lock tables" do |gitlab_schema, database_name| + let(:connection) { Gitlab::Database.database_base_models[database_name].connection } + let(:tables_to_lock) do + Gitlab::Database::GitlabSchema + .tables_to_schema.filter_map { |table_name, schema| table_name if schema == gitlab_schema } + end - ApplicationRecord.connection.execute(drop_detached_partition_sql) - Ci::ApplicationRecord.connection.execute(drop_detached_partition_sql) + it "locks table in schema #{gitlab_schema} and database #{database_name}" do + expect(tables_to_lock).not_to be_empty - Gitlab::Database::SharedModel.using_connection(ApplicationRecord.connection) do - Postgresql::DetachedPartition.delete_all + tables_to_lock.each do |table_name| + lock_writes_manager = instance_double(Gitlab::Database::LockWritesManager, lock_writes: nil) + + expect(Gitlab::Database::LockWritesManager).to receive(:new).with( + table_name: table_name, + connection: connection, + database_name: database_name, + with_retries: true, + logger: anything, + dry_run: anything + ).once.and_return(lock_writes_manager) + expect(lock_writes_manager).to receive(:lock_writes).once + end + + subject + end + + it 'returns list of actions' do + expect(subject).to include({ action: 'any action' }) end end - shared_examples "lock tables" do |table_schema, database_name| - let(:table_name) do + shared_examples "unlock tables" do |gitlab_schema, database_name| + let(:connection) { Gitlab::Database.database_base_models[database_name].connection } + + let(:tables_to_unlock) do Gitlab::Database::GitlabSchema - .tables_to_schema.filter_map { |table_name, schema| table_name if schema == table_schema } - .first + .tables_to_schema.filter_map { |table_name, schema| table_name if schema == gitlab_schema } + end + + it "unlocks table in schema #{gitlab_schema} and database #{database_name}" do + expect(tables_to_unlock).not_to be_empty + + tables_to_unlock.each do |table_name| + lock_writes_manager = instance_double(Gitlab::Database::LockWritesManager, unlock_writes: nil) + + expect(Gitlab::Database::LockWritesManager).to receive(:new).with( + table_name: table_name, + connection: anything, + database_name: database_name, + with_retries: true, + logger: anything, + dry_run: anything + ).once.and_return(lock_writes_manager) + expect(lock_writes_manager).to receive(:unlock_writes) + end + + subject end - let(:database) { database_name } + it 'returns list of actions' do + expect(subject).to include({ action: 'unlocked' }) + end + end + + shared_examples "lock partitions" do |partition_identifier, database_name| + let(:connection) { Gitlab::Database.database_base_models[database_name].connection } + + it 'locks the partition' do + lock_writes_manager = instance_double(Gitlab::Database::LockWritesManager, lock_writes: nil) - it "locks table in schema #{table_schema} and database #{database_name}" do expect(Gitlab::Database::LockWritesManager).to receive(:new).with( - table_name: table_name, - connection: anything, - database_name: database, + table_name: partition_identifier, + connection: connection, + database_name: database_name, with_retries: true, logger: anything, dry_run: anything @@ -68,20 +144,16 @@ RSpec.describe Gitlab::Database::TablesLocker, :reestablished_active_record_base end end - shared_examples "unlock tables" do |table_schema, database_name| - let(:table_name) do - Gitlab::Database::GitlabSchema - .tables_to_schema.filter_map { |table_name, schema| table_name if schema == table_schema } - .first - end + shared_examples "unlock partitions" do |partition_identifier, database_name| + let(:connection) { Gitlab::Database.database_base_models[database_name].connection } - let(:database) { database_name } + it 'unlocks the partition' do + lock_writes_manager = instance_double(Gitlab::Database::LockWritesManager, unlock_writes: nil) - it "unlocks table in schema #{table_schema} and database #{database_name}" do expect(Gitlab::Database::LockWritesManager).to receive(:new).with( - table_name: table_name, - connection: anything, - database_name: database, + table_name: partition_identifier, + connection: connection, + database_name: database_name, with_retries: true, logger: anything, dry_run: anything @@ -94,31 +166,35 @@ RSpec.describe Gitlab::Database::TablesLocker, :reestablished_active_record_base context 'when running on single database' do before do - skip_if_multiple_databases_are_setup(:ci) + skip_if_database_exists(:ci) end describe '#lock_writes' do subject { described_class.new.lock_writes } - it 'does not call Gitlab::Database::LockWritesManager.lock_writes' do - expect(Gitlab::Database::LockWritesManager).to receive(:new).with(any_args).and_return(lock_writes_manager) - expect(lock_writes_manager).not_to receive(:lock_writes) + it 'does not lock any table' do + expect(Gitlab::Database::LockWritesManager).to receive(:new) + .with(any_args).and_return(default_lock_writes_manager) + expect(default_lock_writes_manager).not_to receive(:lock_writes) subject end - include_examples "unlock tables", :gitlab_main, 'main' - include_examples "unlock tables", :gitlab_ci, 'ci' - include_examples "unlock tables", :gitlab_shared, 'main' - include_examples "unlock tables", :gitlab_internal, 'main' + it_behaves_like 'unlock tables', :gitlab_main, 'main' + it_behaves_like 'unlock tables', :gitlab_ci, 'main' + it_behaves_like 'unlock tables', :gitlab_main_clusterwide, 'main' + it_behaves_like 'unlock tables', :gitlab_shared, 'main' + it_behaves_like 'unlock tables', :gitlab_internal, 'main' end describe '#unlock_writes' do subject { described_class.new.lock_writes } it 'does call Gitlab::Database::LockWritesManager.unlock_writes' do - expect(Gitlab::Database::LockWritesManager).to receive(:new).with(any_args).and_return(lock_writes_manager) - expect(lock_writes_manager).to receive(:unlock_writes) + expect(Gitlab::Database::LockWritesManager).to receive(:new) + .with(any_args).and_return(default_lock_writes_manager) + expect(default_lock_writes_manager).to receive(:unlock_writes) + expect(default_lock_writes_manager).not_to receive(:lock_writes) subject end @@ -127,49 +203,67 @@ RSpec.describe Gitlab::Database::TablesLocker, :reestablished_active_record_base context 'when running on multiple databases' do before do - skip_if_multiple_databases_not_setup(:ci) + skip_if_shared_database(:ci) end describe '#lock_writes' do subject { described_class.new.lock_writes } - include_examples "lock tables", :gitlab_ci, 'main' - include_examples "lock tables", :gitlab_main, 'ci' - - include_examples "unlock tables", :gitlab_main, 'main' - include_examples "unlock tables", :gitlab_ci, 'ci' - include_examples "unlock tables", :gitlab_shared, 'main' - include_examples "unlock tables", :gitlab_shared, 'ci' - include_examples "unlock tables", :gitlab_internal, 'main' - include_examples "unlock tables", :gitlab_internal, 'ci' + it_behaves_like 'lock tables', :gitlab_ci, 'main' + it_behaves_like 'lock tables', :gitlab_main, 'ci' + it_behaves_like 'lock tables', :gitlab_main_clusterwide, 'ci' + + it_behaves_like 'unlock tables', :gitlab_main_clusterwide, 'main' + it_behaves_like 'unlock tables', :gitlab_main, 'main' + it_behaves_like 'unlock tables', :gitlab_ci, 'ci' + it_behaves_like 'unlock tables', :gitlab_shared, 'main' + it_behaves_like 'unlock tables', :gitlab_shared, 'ci' + it_behaves_like 'unlock tables', :gitlab_internal, 'main' + it_behaves_like 'unlock tables', :gitlab_internal, 'ci' + + gitlab_main_partition = "#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.security_findings_test_partition" + it_behaves_like 'unlock partitions', gitlab_main_partition, 'main' + it_behaves_like 'lock partitions', gitlab_main_partition, 'ci' + + gitlab_main_detached_partition = "#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}._test_gitlab_main_part_20220101" + it_behaves_like 'unlock partitions', gitlab_main_detached_partition, 'main' + it_behaves_like 'lock partitions', gitlab_main_detached_partition, 'ci' end describe '#unlock_writes' do subject { described_class.new.unlock_writes } - include_examples "unlock tables", :gitlab_ci, 'main' - include_examples "unlock tables", :gitlab_main, 'ci' - include_examples "unlock tables", :gitlab_main, 'main' - include_examples "unlock tables", :gitlab_ci, 'ci' - include_examples "unlock tables", :gitlab_shared, 'main' - include_examples "unlock tables", :gitlab_shared, 'ci' - include_examples "unlock tables", :gitlab_internal, 'main' - include_examples "unlock tables", :gitlab_internal, 'ci' + it_behaves_like "unlock tables", :gitlab_ci, 'main' + it_behaves_like "unlock tables", :gitlab_main, 'ci' + it_behaves_like "unlock tables", :gitlab_main, 'main' + it_behaves_like "unlock tables", :gitlab_ci, 'ci' + it_behaves_like "unlock tables", :gitlab_shared, 'main' + it_behaves_like "unlock tables", :gitlab_shared, 'ci' + it_behaves_like "unlock tables", :gitlab_internal, 'main' + it_behaves_like "unlock tables", :gitlab_internal, 'ci' + + gitlab_main_partition = "#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.security_findings_test_partition" + it_behaves_like 'unlock partitions', gitlab_main_partition, 'main' + it_behaves_like 'unlock partitions', gitlab_main_partition, 'ci' + + gitlab_main_detached_partition = "#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}._test_gitlab_main_part_20220101" + it_behaves_like 'unlock partitions', gitlab_main_detached_partition, 'main' + it_behaves_like 'unlock partitions', gitlab_main_detached_partition, 'ci' end context 'when running in dry_run mode' do subject { described_class.new(dry_run: true).lock_writes } - it 'passes dry_run flag to LockManger' do + it 'passes dry_run flag to LockWritesManager' do expect(Gitlab::Database::LockWritesManager).to receive(:new).with( - table_name: 'users', + table_name: 'security_findings', connection: anything, database_name: 'ci', with_retries: true, logger: anything, dry_run: true - ).and_return(lock_writes_manager) - expect(lock_writes_manager).to receive(:lock_writes) + ).and_return(default_lock_writes_manager) + expect(default_lock_writes_manager).to receive(:lock_writes) subject end @@ -185,8 +279,9 @@ RSpec.describe Gitlab::Database::TablesLocker, :reestablished_active_record_base end it 'does not lock any tables if the ci database is shared with main database' do - expect(Gitlab::Database::LockWritesManager).to receive(:new).with(any_args).and_return(lock_writes_manager) - expect(lock_writes_manager).not_to receive(:lock_writes) + expect(Gitlab::Database::LockWritesManager).to receive(:new) + .with(any_args).and_return(default_lock_writes_manager) + expect(default_lock_writes_manager).not_to receive(:lock_writes) subject end @@ -220,7 +315,3 @@ RSpec.describe Gitlab::Database::TablesLocker, :reestablished_active_record_base end end end - -def number_of_triggers(connection) - connection.select_value("SELECT count(*) FROM information_schema.triggers") -end diff --git a/spec/lib/gitlab/database/tables_truncate_spec.rb b/spec/lib/gitlab/database/tables_truncate_spec.rb index 3bb2f4e982c..ef76c9b8da3 100644 --- a/spec/lib/gitlab/database/tables_truncate_spec.rb +++ b/spec/lib/gitlab/database/tables_truncate_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::Database::TablesTruncate, :reestablished_active_record_base, - :suppress_gitlab_schemas_validate_connection, feature_category: :pods do + :suppress_gitlab_schemas_validate_connection, feature_category: :cell do include MigrationsHelpers let(:min_batch_size) { 1 } @@ -48,7 +48,7 @@ RSpec.describe Gitlab::Database::TablesTruncate, :reestablished_active_record_ba end before do - skip_if_multiple_databases_not_setup(:ci) + skip_if_shared_database(:ci) # Creating some test tables on the main database main_tables_sql = <<~SQL @@ -79,8 +79,7 @@ RSpec.describe Gitlab::Database::TablesTruncate, :reestablished_active_record_ba ALTER TABLE _test_gitlab_hook_logs DETACH PARTITION gitlab_partitions_dynamic._test_gitlab_hook_logs_202201; SQL - main_connection.execute(main_tables_sql) - ci_connection.execute(main_tables_sql) + execute_on_each_database(main_tables_sql) ci_tables_sql = <<~SQL CREATE TABLE _test_gitlab_ci_items (id serial NOT NULL PRIMARY KEY); @@ -92,15 +91,13 @@ RSpec.describe Gitlab::Database::TablesTruncate, :reestablished_active_record_ba ); SQL - main_connection.execute(ci_tables_sql) - ci_connection.execute(ci_tables_sql) + execute_on_each_database(ci_tables_sql) internal_tables_sql = <<~SQL CREATE TABLE _test_gitlab_shared_items (id serial NOT NULL PRIMARY KEY); SQL - main_connection.execute(internal_tables_sql) - ci_connection.execute(internal_tables_sql) + execute_on_each_database(internal_tables_sql) # Filling the tables 5.times do |i| @@ -156,7 +153,9 @@ RSpec.describe Gitlab::Database::TablesTruncate, :reestablished_active_record_ba "_test_gitlab_ci_items" => :gitlab_ci, "_test_gitlab_ci_references" => :gitlab_ci, "_test_gitlab_shared_items" => :gitlab_shared, - "_test_gitlab_geo_items" => :gitlab_geo + "_test_gitlab_geo_items" => :gitlab_geo, + "detached_partitions" => :gitlab_shared, + "postgres_partitions" => :gitlab_shared } ) @@ -314,8 +313,7 @@ RSpec.describe Gitlab::Database::TablesTruncate, :reestablished_active_record_ba context 'when running with multiple shared databases' do before do skip_if_multiple_databases_not_setup(:ci) - ci_db_config = Ci::ApplicationRecord.connection_db_config - allow(::Gitlab::Database).to receive(:db_config_share_with).with(ci_db_config).and_return('main') + skip_if_database_exists(:ci) end it 'raises an error when truncating the main database that it is a single database setup' do diff --git a/spec/lib/gitlab/database/transaction_timeout_settings_spec.rb b/spec/lib/gitlab/database/transaction_timeout_settings_spec.rb index 5b68f9a3757..2725b22ca9d 100644 --- a/spec/lib/gitlab/database/transaction_timeout_settings_spec.rb +++ b/spec/lib/gitlab/database/transaction_timeout_settings_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Database::TransactionTimeoutSettings, feature_category: :pods do +RSpec.describe Gitlab::Database::TransactionTimeoutSettings, feature_category: :cell do let(:connection) { ActiveRecord::Base.connection } subject { described_class.new(connection) } diff --git a/spec/lib/gitlab/database/with_lock_retries_outside_transaction_spec.rb b/spec/lib/gitlab/database/with_lock_retries_outside_transaction_spec.rb index 9ccae754a92..82bba31193b 100644 --- a/spec/lib/gitlab/database/with_lock_retries_outside_transaction_spec.rb +++ b/spec/lib/gitlab/database/with_lock_retries_outside_transaction_spec.rb @@ -61,12 +61,12 @@ RSpec.describe Gitlab::Database::WithLockRetriesOutsideTransaction, feature_cate context 'lock_fiber' do it 'acquires lock successfully' do - check_exclusive_lock_query = """ + check_exclusive_lock_query = <<~QUERY SELECT 1 FROM pg_locks l JOIN pg_class t ON l.relation = t.oid WHERE t.relkind = 'r' AND l.mode = 'ExclusiveLock' AND t.relname = '#{Project.table_name}' - """ + QUERY expect(connection.execute(check_exclusive_lock_query).to_a).to be_present end diff --git a/spec/lib/gitlab/database/with_lock_retries_spec.rb b/spec/lib/gitlab/database/with_lock_retries_spec.rb index 7fe6362634b..7e0435c815b 100644 --- a/spec/lib/gitlab/database/with_lock_retries_spec.rb +++ b/spec/lib/gitlab/database/with_lock_retries_spec.rb @@ -61,12 +61,12 @@ RSpec.describe Gitlab::Database::WithLockRetries, feature_category: :database do context 'lock_fiber' do it 'acquires lock successfully' do - check_exclusive_lock_query = """ + check_exclusive_lock_query = <<~QUERY SELECT 1 FROM pg_locks l JOIN pg_class t ON l.relation = t.oid WHERE t.relkind = 'r' AND l.mode = 'ExclusiveLock' AND t.relname = '#{Project.table_name}' - """ + QUERY expect(connection.execute(check_exclusive_lock_query).to_a).to be_present end diff --git a/spec/lib/gitlab/database_importers/instance_administrators/create_group_spec.rb b/spec/lib/gitlab/database_importers/instance_administrators/create_group_spec.rb deleted file mode 100644 index 68c29bad287..00000000000 --- a/spec/lib/gitlab/database_importers/instance_administrators/create_group_spec.rb +++ /dev/null @@ -1,169 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::DatabaseImporters::InstanceAdministrators::CreateGroup do - describe '#execute' do - let(:result) { subject.execute } - - context 'without application_settings' do - it 'returns error' do - expect(subject).to receive(:log_error).and_call_original - expect(result).to eq( - status: :error, - message: 'No application_settings found', - last_step: :validate_application_settings - ) - - expect(Group.count).to eq(0) - end - end - - context 'without admin users' do - let(:application_setting) { Gitlab::CurrentSettings.current_application_settings } - - before do - allow(ApplicationSetting).to receive(:current_without_cache) { application_setting } - end - - it 'returns error' do - expect(subject).to receive(:log_error).and_call_original - expect(result).to eq( - status: :error, - message: 'No active admin user found', - last_step: :validate_admins - ) - - expect(Group.count).to eq(0) - end - end - - context( - 'with application settings and admin users', - :do_not_mock_admin_mode_setting, - :do_not_stub_snowplow_by_default - ) do - let(:group) { result[:group] } - let(:application_setting) { Gitlab::CurrentSettings.current_application_settings } - - let!(:user) { create(:user, :admin) } - - before do - allow(ApplicationSetting).to receive(:current_without_cache) { application_setting } - end - - it 'returns correct keys' do - expect(result.keys).to contain_exactly( - :status, :group - ) - end - - it "tracks successful install" do - expect(::Gitlab::Tracking).to receive(:event).with( - 'instance_administrators_group', 'group_created', namespace: group - ) - - subject.execute - end - - it 'creates group' do - expect(result[:status]).to eq(:success) - expect(group).to be_persisted - expect(group.name).to eq('GitLab Instance') - expect(group.path).to start_with('gitlab-instance') - expect(group.path.split('-').last.length).to eq(8) - expect(group.visibility_level).to eq(described_class::VISIBILITY_LEVEL) - end - - it 'adds all admins as maintainers' do - admin1 = create(:user, :admin) - admin2 = create(:user, :admin) - create(:user) - - expect(result[:status]).to eq(:success) - group.reset - expect(group.members.collect(&:user)).to contain_exactly(user, admin1, admin2) - expect(group.members.collect(&:access_level)).to contain_exactly( - Gitlab::Access::OWNER, - Gitlab::Access::MAINTAINER, - Gitlab::Access::MAINTAINER - ) - end - - it 'saves the group id' do - expect(result[:status]).to eq(:success) - expect(application_setting.instance_administrators_group_id).to eq(group.id) - end - - it 'returns error when saving group ID fails' do - allow(application_setting).to receive(:save) { false } - - expect(result).to eq( - status: :error, - message: 'Could not save group ID', - last_step: :save_group_id - ) - end - - context 'when group already exists' do - let(:existing_group) { create(:group) } - - before do - admin1 = create(:user, :admin) - admin2 = create(:user, :admin) - - existing_group.add_owner(user) - existing_group.add_members([admin1, admin2], Gitlab::Access::MAINTAINER) - - application_setting.instance_administrators_group_id = existing_group.id - end - - it 'returns success' do - expect(result).to eq( - status: :success, - group: existing_group - ) - - expect(Group.count).to eq(1) - end - end - - context 'when group cannot be created' do - let(:group) { build(:group) } - - before do - group.errors.add(:base, "Test error") - - expect_next_instance_of(::Groups::CreateService) do |group_create_service| - expect(group_create_service).to receive(:execute) - .and_return(group) - end - end - - it 'returns error' do - expect(subject).to receive(:log_error).and_call_original - expect(result).to eq( - status: :error, - message: 'Could not create group', - last_step: :create_group - ) - end - end - - context 'when user cannot be added to group' do - before do - subject.instance_variable_set(:@instance_admins, [user, build(:user, :admin)]) - end - - it 'returns error' do - expect(subject).to receive(:log_error).and_call_original - expect(result).to eq( - status: :error, - message: 'Could not add admins as members', - last_step: :add_group_members - ) - end - end - end - end -end diff --git a/spec/lib/gitlab/database_importers/self_monitoring/project/create_service_spec.rb b/spec/lib/gitlab/database_importers/self_monitoring/project/create_service_spec.rb deleted file mode 100644 index ad91320c6eb..00000000000 --- a/spec/lib/gitlab/database_importers/self_monitoring/project/create_service_spec.rb +++ /dev/null @@ -1,315 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::DatabaseImporters::SelfMonitoring::Project::CreateService do - describe '#execute' do - let(:result) { subject.execute } - - let(:prometheus_settings) do - { - enabled: true, - server_address: 'localhost:9090' - } - end - - before do - stub_config(prometheus: prometheus_settings) - end - - context 'without application_settings' do - it 'returns error' do - expect(subject).to receive(:log_error).and_call_original - expect(result).to eq( - status: :error, - message: 'No application_settings found', - last_step: :validate_application_settings - ) - - expect(Project.count).to eq(0) - expect(Group.count).to eq(0) - end - end - - context 'without admin users' do - let(:application_setting) { Gitlab::CurrentSettings.current_application_settings } - - before do - allow(ApplicationSetting).to receive(:current_without_cache) { application_setting } - end - - it 'returns error' do - expect(result).to eq( - status: :error, - message: 'No active admin user found', - last_step: :create_group - ) - - expect(Project.count).to eq(0) - expect(Group.count).to eq(0) - end - end - - context 'with application settings and admin users', :request_store do - let(:project) { result[:project] } - let(:group) { result[:group] } - let(:application_setting) { Gitlab::CurrentSettings.current_application_settings } - - let!(:user) { create(:user, :admin) } - - before do - stub_env('IN_MEMORY_APPLICATION_SETTINGS', 'false') - - application_setting.update!(allow_local_requests_from_web_hooks_and_services: true) - end - - shared_examples 'has prometheus integration' do |server_address| - it do - expect(result[:status]).to eq(:success) - - prometheus = project.prometheus_integration - expect(prometheus).not_to eq(nil) - expect(prometheus.api_url).to eq(server_address) - expect(prometheus.active).to eq(true) - expect(prometheus.manual_configuration).to eq(true) - end - end - - it_behaves_like 'has prometheus integration', 'http://localhost:9090' - - it 'is idempotent' do - result1 = subject.execute - expect(result1[:status]).to eq(:success) - - result2 = subject.execute - expect(result2[:status]).to eq(:success) - end - - it "tracks successful install" do - expect(::Gitlab::Tracking).to receive(:event).with("instance_administrators_group", "group_created", namespace: project.namespace) - expect(::Gitlab::Tracking).to receive(:event).with('self_monitoring', 'project_created', project: project, namespace: project.namespace) - - subject.execute - end - - it 'creates group' do - expect(result[:status]).to eq(:success) - expect(group).to be_persisted - end - - it 'creates project with internal visibility' do - expect(result[:status]).to eq(:success) - expect(project.visibility_level).to eq(Gitlab::VisibilityLevel::INTERNAL) - expect(project).to be_persisted - end - - it 'creates project with internal visibility even when internal visibility is restricted' do - application_setting.restricted_visibility_levels = [Gitlab::VisibilityLevel::INTERNAL] - - expect(result[:status]).to eq(:success) - expect(project.visibility_level).to eq(Gitlab::VisibilityLevel::INTERNAL) - expect(project).to be_persisted - end - - it 'creates project with correct name and description' do - path = 'administration/monitoring/gitlab_self_monitoring_project/index' - docs_path = Rails.application.routes.url_helpers.help_page_path(path) - - expect(result[:status]).to eq(:success) - expect(project.name).to eq(described_class::PROJECT_NAME) - expect(project.description).to eq( - 'This project is automatically generated and helps monitor this GitLab instance. ' \ - "[Learn more](#{docs_path})." - ) - expect(File).to exist("doc/#{path}.md") - end - - it 'creates project with group as owner' do - expect(result[:status]).to eq(:success) - expect(project.owner).to eq(group) - end - - it 'saves the project id' do - expect(result[:status]).to eq(:success) - expect(application_setting.reload.self_monitoring_project_id).to eq(project.id) - end - - it 'creates a Prometheus integration' do - expect(result[:status]).to eq(:success) - - integrations = result[:project].reload.integrations - - expect(integrations.count).to eq(1) - # Ensures Integrations::Prometheus#self_monitoring_project? is true - expect(integrations.first.allow_local_api_url?).to be_truthy - end - - it 'creates an environment for the project' do - expect(project.default_environment.name).to eq('production') - end - - context 'when the environment creation fails' do - let(:environment) { build(:environment, name: 'production') } - - it 'returns error' do - allow(Environment).to receive(:new).and_return(environment) - allow(environment).to receive(:save).and_return(false) - - expect(result).to eq( - status: :error, - message: 'Could not create environment', - last_step: :create_environment - ) - end - end - - it 'returns error when saving project ID fails' do - allow(subject.application_settings).to receive(:update).and_call_original - allow(subject.application_settings).to receive(:update) - .with(self_monitoring_project_id: anything) - .and_return(false) - - expect(result).to eq( - status: :error, - message: 'Could not save project ID', - last_step: :save_project_id - ) - end - - context 'when project already exists' do - let(:existing_group) { create(:group) } - let(:existing_project) { create(:project, namespace: existing_group) } - - before do - application_setting.update!( - instance_administrators_group_id: existing_group.id, self_monitoring_project_id: existing_project.id) - end - - it 'returns success' do - expect(result).to include(status: :success) - - expect(Project.count).to eq(1) - expect(Group.count).to eq(1) - end - end - - context 'when local requests from hooks and integrations are not allowed' do - before do - application_setting.update!(allow_local_requests_from_web_hooks_and_services: false) - end - - it_behaves_like 'has prometheus integration', 'http://localhost:9090' - end - - context 'with non default prometheus address' do - let(:server_address) { 'https://localhost:9090' } - - let(:prometheus_settings) do - { - enabled: true, - server_address: server_address - } - end - - it_behaves_like 'has prometheus integration', 'https://localhost:9090' - - context 'with :9090 symbol' do - let(:server_address) { :':9090' } - - it_behaves_like 'has prometheus integration', 'http://localhost:9090' - end - - context 'with 0.0.0.0:9090' do - let(:server_address) { '0.0.0.0:9090' } - - it_behaves_like 'has prometheus integration', 'http://localhost:9090' - end - end - - context 'when prometheus setting is not present in gitlab.yml' do - before do - allow(Gitlab.config).to receive(:prometheus).and_raise(Settingslogic::MissingSetting) - end - - it 'does not fail' do - expect(result).to include(status: :success) - expect(project.prometheus_integration).to be_nil - end - end - - context 'when prometheus setting is nil' do - before do - stub_config(prometheus: nil) - end - - it 'does not fail' do - expect(result).to include(status: :success) - expect(project.prometheus_integration).to be_nil - end - end - - context 'when prometheus setting is disabled in gitlab.yml' do - let(:prometheus_settings) do - { - enabled: false, - server_address: 'http://localhost:9090' - } - end - - it 'does not configure prometheus' do - expect(result).to include(status: :success) - expect(project.prometheus_integration).to be_nil - end - end - - context 'when prometheus server address is blank in gitlab.yml' do - let(:prometheus_settings) { { enabled: true, server_address: '' } } - - it 'does not configure prometheus' do - expect(result).to include(status: :success) - expect(project.prometheus_integration).to be_nil - end - end - - context 'when project cannot be created' do - let(:project) { build(:project) } - - before do - project.errors.add(:base, "Test error") - - expect_next_instance_of(::Projects::CreateService) do |project_create_service| - expect(project_create_service).to receive(:execute) - .and_return(project) - end - end - - it 'returns error' do - expect(subject).to receive(:log_error).and_call_original - expect(result).to eq( - status: :error, - message: 'Could not create project', - last_step: :create_project - ) - end - end - - context 'when prometheus manual configuration cannot be saved' do - let(:prometheus_settings) do - { - enabled: true, - server_address: 'httpinvalid://localhost:9090' - } - end - - it 'returns error' do - expect(subject).to receive(:log_error).and_call_original - expect(result).to eq( - status: :error, - message: 'Could not save prometheus manual configuration', - last_step: :add_prometheus_manual_configuration - ) - end - end - end - end -end diff --git a/spec/lib/gitlab/database_importers/self_monitoring/project/delete_service_spec.rb b/spec/lib/gitlab/database_importers/self_monitoring/project/delete_service_spec.rb deleted file mode 100644 index d878d46c883..00000000000 --- a/spec/lib/gitlab/database_importers/self_monitoring/project/delete_service_spec.rb +++ /dev/null @@ -1,53 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::DatabaseImporters::SelfMonitoring::Project::DeleteService do - describe '#execute' do - let!(:application_setting) { create(:application_setting) } - let(:result) { subject.execute } - - context 'when project does not exist' do - it 'returns error' do - expect(result).to eq( - status: :error, - message: 'Self-monitoring project does not exist', - last_step: :validate_self_monitoring_project_exists - ) - end - end - - context 'when self-monitoring project exists' do - let(:group) { create(:group) } - let(:project) { create(:project, namespace: group) } - - let(:application_setting) do - create( - :application_setting, - self_monitoring_project_id: project.id, - instance_administrators_group_id: group.id - ) - end - - it 'destroys project' do - subject.execute - - expect { project.reload }.to raise_error(ActiveRecord::RecordNotFound) - end - - it 'deletes project ID from application settings' do - subject.execute - - LooseForeignKeys::ProcessDeletedRecordsService.new(connection: Project.connection).execute - - expect(application_setting.reload.self_monitoring_project_id).to be_nil - end - - it 'does not delete group' do - subject.execute - - expect(application_setting.instance_administrators_group).to eq(group) - end - end - end -end diff --git a/spec/lib/gitlab/database_spec.rb b/spec/lib/gitlab/database_spec.rb index 26d6ff431ec..f2be888e6eb 100644 --- a/spec/lib/gitlab/database_spec.rb +++ b/spec/lib/gitlab/database_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Database do +RSpec.describe Gitlab::Database, feature_category: :database do before do stub_const('MigrationTest', Class.new { include Gitlab::Database }) end @@ -66,6 +66,48 @@ RSpec.describe Gitlab::Database do end end + describe '.has_database?' do + context 'three tier database config' do + it 'returns true for main' do + expect(described_class.has_database?(:main)).to eq(true) + end + + it 'returns false for shared database' do + skip_if_multiple_databases_not_setup(:ci) + skip_if_database_exists(:ci) + + expect(described_class.has_database?(:ci)).to eq(false) + end + + it 'returns false for non-existent' do + expect(described_class.has_database?(:nonexistent)).to eq(false) + end + end + end + + describe '.database_mode' do + context 'three tier database config' do + it 'returns single-database if ci is not configured' do + skip_if_multiple_databases_are_setup(:ci) + + expect(described_class.database_mode).to eq(::Gitlab::Database::MODE_SINGLE_DATABASE) + end + + it 'returns single-database-ci-connection if ci is shared with main database' do + skip_if_multiple_databases_not_setup(:ci) + skip_if_database_exists(:ci) + + expect(described_class.database_mode).to eq(::Gitlab::Database::MODE_SINGLE_DATABASE_CI_CONNECTION) + end + + it 'returns multiple-database if ci has its own database' do + skip_if_shared_database(:ci) + + expect(described_class.database_mode).to eq(::Gitlab::Database::MODE_MULTIPLE_DATABASES) + end + end + end + describe '.check_for_non_superuser' do subject { described_class.check_for_non_superuser } diff --git a/spec/lib/gitlab/diff/formatters/image_formatter_spec.rb b/spec/lib/gitlab/diff/formatters/image_formatter_spec.rb index 73c0d0dba88..1069666ac50 100644 --- a/spec/lib/gitlab/diff/formatters/image_formatter_spec.rb +++ b/spec/lib/gitlab/diff/formatters/image_formatter_spec.rb @@ -26,11 +26,13 @@ RSpec.describe Gitlab::Diff::Formatters::ImageFormatter do it { is_expected.to eq(subject) } [:width, :height, :x, :y].each do |attr| - let(:other_formatter) do - described_class.new(attrs.merge(attr => 9)) - end + context "with attribute:#{attr}" do + let(:other_formatter) do + described_class.new(attrs.merge(attr => 9)) + end - it { is_expected.not_to eq(other_formatter) } + it { is_expected.not_to eq(other_formatter) } + end end end end diff --git a/spec/lib/gitlab/diff/highlight_cache_spec.rb b/spec/lib/gitlab/diff/highlight_cache_spec.rb index 33e9360ee01..43e4f28b4df 100644 --- a/spec/lib/gitlab/diff/highlight_cache_spec.rb +++ b/spec/lib/gitlab/diff/highlight_cache_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Diff::HighlightCache, :clean_gitlab_redis_cache do +RSpec.describe Gitlab::Diff::HighlightCache, :clean_gitlab_redis_cache, feature_category: :source_code_management do let_it_be(:merge_request) { create(:merge_request_with_diffs) } let(:diff_hash) do @@ -282,17 +282,7 @@ RSpec.describe Gitlab::Diff::HighlightCache, :clean_gitlab_redis_cache do end it 'returns cache key' do - is_expected.to eq("highlighted-diff-files:#{cache.diffable.cache_key}:2:#{options_hash([cache.diff_options, true, true])}") - end - - context 'when the `use_marker_ranges` feature flag is disabled' do - before do - stub_feature_flags(use_marker_ranges: false) - end - - it 'returns the original version of the cache' do - is_expected.to eq("highlighted-diff-files:#{cache.diffable.cache_key}:2:#{options_hash([cache.diff_options, false, true])}") - end + is_expected.to eq("highlighted-diff-files:#{cache.diffable.cache_key}:2:#{options_hash([cache.diff_options, true])}") end context 'when the `diff_line_syntax_highlighting` feature flag is disabled' do @@ -301,7 +291,7 @@ RSpec.describe Gitlab::Diff::HighlightCache, :clean_gitlab_redis_cache do end it 'returns the original version of the cache' do - is_expected.to eq("highlighted-diff-files:#{cache.diffable.cache_key}:2:#{options_hash([cache.diff_options, true, false])}") + is_expected.to eq("highlighted-diff-files:#{cache.diffable.cache_key}:2:#{options_hash([cache.diff_options, false])}") end end end diff --git a/spec/lib/gitlab/diff/highlight_spec.rb b/spec/lib/gitlab/diff/highlight_spec.rb index c378ecb8134..233dddbdad7 100644 --- a/spec/lib/gitlab/diff/highlight_spec.rb +++ b/spec/lib/gitlab/diff/highlight_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Diff::Highlight do +RSpec.describe Gitlab::Diff::Highlight, feature_category: :source_code_management do include RepoHelpers let_it_be(:project) { create(:project, :repository) } @@ -15,7 +15,6 @@ RSpec.describe Gitlab::Diff::Highlight do let(:code) { '<h2 onmouseover="alert(2)">Test</h2>' } before do - allow(Gitlab::Diff::InlineDiff).to receive(:for_lines).and_return([]) allow_any_instance_of(Gitlab::Diff::Line).to receive(:text).and_return(code) end @@ -121,18 +120,6 @@ RSpec.describe Gitlab::Diff::Highlight do end end - context 'when `use_marker_ranges` feature flag is disabled' do - it 'returns the same result' do - with_feature_flag = described_class.new(diff_file, repository: project.repository).highlight - - stub_feature_flags(use_marker_ranges: false) - - without_feature_flag = described_class.new(diff_file, repository: project.repository).highlight - - expect(with_feature_flag.map(&:rich_text)).to eq(without_feature_flag.map(&:rich_text)) - end - end - context 'when no inline diffs' do it_behaves_like 'without inline diffs' end diff --git a/spec/lib/gitlab/discussions_diff/highlight_cache_spec.rb b/spec/lib/gitlab/discussions_diff/highlight_cache_spec.rb index 30981e4bd7d..0dc0f50b104 100644 --- a/spec/lib/gitlab/discussions_diff/highlight_cache_spec.rb +++ b/spec/lib/gitlab/discussions_diff/highlight_cache_spec.rb @@ -41,57 +41,81 @@ RSpec.describe Gitlab::DiscussionsDiff::HighlightCache, :clean_gitlab_redis_cach end describe '#read_multiple' do - it 'reads multiple keys and serializes content into Gitlab::Diff::Line objects' do - described_class.write_multiple(mapping) + shared_examples 'read multiple keys' do + it 'reads multiple keys and serializes content into Gitlab::Diff::Line objects' do + described_class.write_multiple(mapping) - found = described_class.read_multiple(mapping.keys) + found = described_class.read_multiple(mapping.keys) - expect(found.size).to eq(2) - expect(found.first.size).to eq(2) - expect(found.first).to all(be_a(Gitlab::Diff::Line)) - end + expect(found.size).to eq(2) + expect(found.first.size).to eq(2) + expect(found.first).to all(be_a(Gitlab::Diff::Line)) + end - it 'returns nil when cached key is not found' do - described_class.write_multiple(mapping) + it 'returns nil when cached key is not found' do + described_class.write_multiple(mapping) - found = described_class.read_multiple([2, 3]) + found = described_class.read_multiple([2, 3]) - expect(found.size).to eq(2) + expect(found.size).to eq(2) - expect(found.first).to eq(nil) - expect(found.second.size).to eq(2) - expect(found.second).to all(be_a(Gitlab::Diff::Line)) - end + expect(found.first).to eq(nil) + expect(found.second.size).to eq(2) + expect(found.second).to all(be_a(Gitlab::Diff::Line)) + end - it 'returns lines which rich_text are HTML-safe' do - described_class.write_multiple(mapping) + it 'returns lines which rich_text are HTML-safe' do + described_class.write_multiple(mapping) + + found = described_class.read_multiple(mapping.keys) + rich_texts = found.flatten.map(&:rich_text) + + expect(rich_texts).to all(be_html_safe) + end + end - found = described_class.read_multiple(mapping.keys) - rich_texts = found.flatten.map(&:rich_text) + context 'when feature flag is disabled' do + before do + stub_feature_flags(use_pipeline_over_multikey: false) + end - expect(rich_texts).to all(be_html_safe) + it_behaves_like 'read multiple keys' end + + it_behaves_like 'read multiple keys' end describe '#clear_multiple' do - it 'removes all named keys' do - described_class.write_multiple(mapping) + shared_examples 'delete multiple keys' do + it 'removes all named keys' do + described_class.write_multiple(mapping) - described_class.clear_multiple(mapping.keys) + described_class.clear_multiple(mapping.keys) - expect(described_class.read_multiple(mapping.keys)).to all(be_nil) - end + expect(described_class.read_multiple(mapping.keys)).to all(be_nil) + end - it 'only removed named keys' do - to_clear, to_leave = mapping.keys + it 'only removed named keys' do + to_clear, to_leave = mapping.keys - described_class.write_multiple(mapping) - described_class.clear_multiple([to_clear]) + described_class.write_multiple(mapping) + described_class.clear_multiple([to_clear]) - cleared, left = described_class.read_multiple([to_clear, to_leave]) + cleared, left = described_class.read_multiple([to_clear, to_leave]) - expect(cleared).to be_nil - expect(left).to all(be_a(Gitlab::Diff::Line)) + expect(cleared).to be_nil + expect(left).to all(be_a(Gitlab::Diff::Line)) + end end + + context 'when feature flag is disabled' do + before do + stub_feature_flags(use_pipeline_over_multikey: false) + end + + it_behaves_like 'delete multiple keys' + end + + it_behaves_like 'delete multiple keys' end end diff --git a/spec/lib/gitlab/doorkeeper_secret_storing/secret/pbkdf2_sha512_spec.rb b/spec/lib/gitlab/doorkeeper_secret_storing/secret/pbkdf2_sha512_spec.rb index df17d92bb0c..fb433923db5 100644 --- a/spec/lib/gitlab/doorkeeper_secret_storing/secret/pbkdf2_sha512_spec.rb +++ b/spec/lib/gitlab/doorkeeper_secret_storing/secret/pbkdf2_sha512_spec.rb @@ -10,16 +10,6 @@ RSpec.describe Gitlab::DoorkeeperSecretStoring::Secret::Pbkdf2Sha512 do expect(described_class.transform_secret(plaintext_secret)) .to eq("$pbkdf2-sha512$20000$$.c0G5XJVEew1TyeJk5TrkvB0VyOaTmDzPrsdNRED9vVeZlSyuG3G90F0ow23zUCiWKAVwmNnR/ceh.nJG3MdpQ") # rubocop:disable Layout/LineLength end - - context 'when hash_oauth_secrets is disabled' do - before do - stub_feature_flags(hash_oauth_secrets: false) - end - - it 'returns a plaintext secret' do - expect(described_class.transform_secret(plaintext_secret)).to eq(plaintext_secret) - end - end end describe 'STRETCHES' do @@ -36,7 +26,6 @@ RSpec.describe Gitlab::DoorkeeperSecretStoring::Secret::Pbkdf2Sha512 do describe '.secret_matches?' do it "match by hashing the input if the stored value is hashed" do - stub_feature_flags(hash_oauth_secrets: false) plain_secret = 'plain_secret' stored_value = '$pbkdf2-sha512$20000$$/BwQRdwSpL16xkQhstavh7nvA5avCP7.4n9LLKe9AupgJDeA7M5xOAvG3N3E5XbRyGWWBbbr.BsojPVWzd1Sqg' # rubocop:disable Layout/LineLength expect(described_class.secret_matches?(plain_secret, stored_value)).to be true diff --git a/spec/lib/gitlab/email/handler/create_issue_handler_spec.rb b/spec/lib/gitlab/email/handler/create_issue_handler_spec.rb index 8ff8de2379a..369d7e994d2 100644 --- a/spec/lib/gitlab/email/handler/create_issue_handler_spec.rb +++ b/spec/lib/gitlab/email/handler/create_issue_handler_spec.rb @@ -116,7 +116,7 @@ RSpec.describe Gitlab::Email::Handler::CreateIssueHandler do context "when the issue could not be saved" do before do allow_any_instance_of(Issue).to receive(:persisted?).and_return(false) - allow_any_instance_of(Issue).to receive(:ensure_metrics).and_return(nil) + allow_any_instance_of(Issue).to receive(:ensure_metrics!).and_return(nil) end it "raises an InvalidIssueError" do diff --git a/spec/lib/gitlab/email/handler/create_note_handler_spec.rb b/spec/lib/gitlab/email/handler/create_note_handler_spec.rb index f70645a8272..e3b0e90bff9 100644 --- a/spec/lib/gitlab/email/handler/create_note_handler_spec.rb +++ b/spec/lib/gitlab/email/handler/create_note_handler_spec.rb @@ -206,4 +206,26 @@ RSpec.describe Gitlab::Email::Handler::CreateNoteHandler do it_behaves_like 'a reply to existing comment' end + + context 'when note is authored from external author for service desk' do + before do + SentNotification.find_by(reply_key: mail_key).update!(recipient: User.support_bot) + end + + context 'when email contains text, quoted text and quick commands' do + let(:email_raw) { fixture_file('emails/commands_in_reply.eml') } + + it 'creates a discussion' do + expect { receiver.execute }.to change { noteable.notes.count }.by(1) + end + + it 'links external participant' do + receiver.execute + + new_note = noteable.notes.last + + expect(new_note.note_metadata.external_author).to eq('jake@adventuretime.ooo') + end + end + end end diff --git a/spec/lib/gitlab/email/hook/silent_mode_interceptor_spec.rb b/spec/lib/gitlab/email/hook/silent_mode_interceptor_spec.rb new file mode 100644 index 00000000000..cc371643bee --- /dev/null +++ b/spec/lib/gitlab/email/hook/silent_mode_interceptor_spec.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Email::Hook::SilentModeInterceptor, :mailer, feature_category: :geo_replication do + let_it_be(:user) { create(:user) } + + before do + Mail.register_interceptor(described_class) + end + + after do + Mail.unregister_interceptor(described_class) + end + + context 'when silent mode is enabled' do + it 'prevents mail delivery' do + stub_application_setting(silent_mode_enabled: true) + + deliver_mails(user) + + should_not_email_anyone + end + + it 'logs the suppression' do + stub_application_setting(silent_mode_enabled: true) + + expect(Gitlab::AppJsonLogger).to receive(:info).with( + message: 'SilentModeInterceptor prevented sending mail', + mail_subject: 'Two-factor authentication disabled', + silent_mode_enabled: true + ) + expect(Gitlab::AppJsonLogger).to receive(:info).with( + message: 'SilentModeInterceptor prevented sending mail', + mail_subject: 'Welcome to GitLab!', + silent_mode_enabled: true + ) + + deliver_mails(user) + end + end + + context 'when silent mode is disabled' do + it 'does not prevent mail delivery' do + stub_application_setting(silent_mode_enabled: false) + + deliver_mails(user) + + should_email(user, times: 2) + end + + it 'debug logs the no-op' do + stub_application_setting(silent_mode_enabled: false) + + expect(Gitlab::AppJsonLogger).to receive(:debug).with( + message: 'SilentModeInterceptor did nothing', + mail_subject: 'Two-factor authentication disabled', + silent_mode_enabled: false + ) + expect(Gitlab::AppJsonLogger).to receive(:debug).with( + message: 'SilentModeInterceptor did nothing', + mail_subject: 'Welcome to GitLab!', + silent_mode_enabled: false + ) + + deliver_mails(user) + end + end + + def deliver_mails(user) + Notify.disabled_two_factor_email(user).deliver_now + DeviseMailer.user_admin_approval(user).deliver_now + end +end diff --git a/spec/lib/gitlab/email/hook/validate_addresses_interceptor_spec.rb b/spec/lib/gitlab/email/hook/validate_addresses_interceptor_spec.rb deleted file mode 100644 index a3f0158db40..00000000000 --- a/spec/lib/gitlab/email/hook/validate_addresses_interceptor_spec.rb +++ /dev/null @@ -1,52 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Email::Hook::ValidateAddressesInterceptor do - describe 'UNSAFE_CHARACTERS' do - subject { described_class::UNSAFE_CHARACTERS } - - it { is_expected.to match('\\') } - it { is_expected.to match("\x00") } - it { is_expected.to match("\x01") } - it { is_expected.not_to match('') } - it { is_expected.not_to match('user@example.com') } - it { is_expected.not_to match('foo-123+bar_456@example.com') } - end - - describe '.delivering_email' do - let(:mail) do - ActionMailer::Base.mail(to: 'test@mail.com', from: 'info@mail.com', subject: 'title', body: 'hello') - end - - let(:unsafe_email) { "evil+\x01$HOME@example.com" } - - it 'sends emails to normal addresses' do - expect(Gitlab::AuthLogger).not_to receive(:info) - expect { mail.deliver_now }.to change(ActionMailer::Base.deliveries, :count) - end - - [:from, :to, :cc, :bcc].each do |header| - it "does not send emails if the #{header.inspect} header contains unsafe characters" do - mail[header] = unsafe_email - - expect(Gitlab::AuthLogger).to receive(:info).with( - message: 'Skipping email with unsafe characters in address', - address: unsafe_email, - subject: mail.subject - ) - - expect { mail.deliver_now }.not_to change(ActionMailer::Base.deliveries, :count) - end - end - - [:reply_to].each do |header| - it "sends emails if the #{header.inspect} header contains unsafe characters" do - mail[header] = unsafe_email - - expect(Gitlab::AuthLogger).not_to receive(:info) - expect { mail.deliver_now }.to change(ActionMailer::Base.deliveries, :count) - end - end - end -end diff --git a/spec/lib/gitlab/email/html_to_markdown_parser_spec.rb b/spec/lib/gitlab/email/html_to_markdown_parser_spec.rb index fe585d47d59..59c488739dc 100644 --- a/spec/lib/gitlab/email/html_to_markdown_parser_spec.rb +++ b/spec/lib/gitlab/email/html_to_markdown_parser_spec.rb @@ -1,17 +1,21 @@ # frozen_string_literal: true -require 'spec_helper' +require 'kramdown' +require 'html2text' +require 'fast_spec_helper' +require 'support/helpers/fixture_helpers' RSpec.describe Gitlab::Email::HtmlToMarkdownParser, feature_category: :service_desk do + include FixtureHelpers + subject { described_class.convert(html) } describe '.convert' do let(:html) { fixture_file("lib/gitlab/email/basic.html") } it 'parses html correctly' do - expect(subject) - .to eq( - <<-BODY.strip_heredoc.chomp + expect(subject).to eq( + <<~BODY.chomp Hello, World! This is some e-mail content. Even though it has whitespace and newlines, the e-mail converter will handle it correctly. *Even* mismatched tags. diff --git a/spec/lib/gitlab/incoming_email_spec.rb b/spec/lib/gitlab/email/incoming_email_spec.rb index acd6634058f..123b050aee7 100644 --- a/spec/lib/gitlab/incoming_email_spec.rb +++ b/spec/lib/gitlab/email/incoming_email_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::IncomingEmail do +RSpec.describe Gitlab::Email::IncomingEmail, feature_category: :service_desk do let(:setting_name) { :incoming_email } it_behaves_like 'common email methods' diff --git a/spec/lib/gitlab/email/message/build_ios_app_guide_spec.rb b/spec/lib/gitlab/email/message/build_ios_app_guide_spec.rb index 3089f955252..4b77b2f7192 100644 --- a/spec/lib/gitlab/email/message/build_ios_app_guide_spec.rb +++ b/spec/lib/gitlab/email/message/build_ios_app_guide_spec.rb @@ -2,13 +2,9 @@ require 'spec_helper' -RSpec.describe Gitlab::Email::Message::BuildIosAppGuide do +RSpec.describe Gitlab::Email::Message::BuildIosAppGuide, :saas do subject(:message) { described_class.new } - before do - allow(Gitlab).to receive(:com?) { true } - end - it 'contains the correct message', :aggregate_failures do expect(message.subject_line).to eq 'Get set up to build for iOS' expect(message.title).to eq "Building for iOS? We've got you covered." diff --git a/spec/lib/gitlab/email/message/in_product_marketing/helper_spec.rb b/spec/lib/gitlab/email/message/in_product_marketing/helper_spec.rb index 3c0d83d0f9e..a3c2d1b428e 100644 --- a/spec/lib/gitlab/email/message/in_product_marketing/helper_spec.rb +++ b/spec/lib/gitlab/email/message/in_product_marketing/helper_spec.rb @@ -27,11 +27,7 @@ RSpec.describe Gitlab::Email::Message::InProductMarketing::Helper do subject(:class_with_helper) { dummy_class_with_helper.new(format) } - context 'gitlab.com' do - before do - allow(Gitlab).to receive(:com?) { true } - end - + context 'for SaaS', :saas do context 'format is HTML' do it 'returns the correct HTML' do message = "If you no longer wish to receive marketing emails from us, " \ diff --git a/spec/lib/gitlab/email/message/repository_push_spec.rb b/spec/lib/gitlab/email/message/repository_push_spec.rb index f13d98ec9b9..bb68bca5dfa 100644 --- a/spec/lib/gitlab/email/message/repository_push_spec.rb +++ b/spec/lib/gitlab/email/message/repository_push_spec.rb @@ -45,7 +45,7 @@ RSpec.describe Gitlab::Email::Message::RepositoryPush do describe '#project_name_with_namespace' do subject { message.project_name_with_namespace } - it { is_expected.to eq "#{group.name} / #{project.path}" } + it { is_expected.to eq "#{group.name} / #{project.name}" } end describe '#author' do diff --git a/spec/lib/gitlab/email/receiver_spec.rb b/spec/lib/gitlab/email/receiver_spec.rb index 865e40d4ecb..e58da2478bf 100644 --- a/spec/lib/gitlab/email/receiver_spec.rb +++ b/spec/lib/gitlab/email/receiver_spec.rb @@ -11,9 +11,10 @@ RSpec.describe Gitlab::Email::Receiver do shared_examples 'successful receive' do let(:handler) { double(:handler, project: project, execute: true, metrics_event: nil, metrics_params: nil) } let(:client_id) { 'email/jake@example.com' } + let(:mail_key) { 'gitlabhq/gitlabhq+auth_token' } it 'correctly finds the mail key' do - expect(Gitlab::Email::Handler).to receive(:for).with(an_instance_of(Mail::Message), 'gitlabhq/gitlabhq+auth_token').and_return(handler) + expect(Gitlab::Email::Handler).to receive(:for).with(an_instance_of(Mail::Message), mail_key).and_return(handler) receiver.execute end @@ -92,6 +93,16 @@ RSpec.describe Gitlab::Email::Receiver do it_behaves_like 'successful receive' end + context 'when mail key is in the references header with a comma' do + let(:email_raw) { fixture_file('emails/valid_reply_with_references_in_comma.eml') } + let(:meta_key) { :references } + let(:meta_value) { ['"<reply-59d8df8370b7e95c5a49fbf86aeb2c93@localhost>,<issue_1@localhost>,<exchange@microsoft.com>"'] } + + it_behaves_like 'successful receive' do + let(:mail_key) { '59d8df8370b7e95c5a49fbf86aeb2c93' } + end + end + context 'when all other headers are missing' do let(:email_raw) { fixture_file('emails/missing_delivered_to_header.eml') } let(:meta_key) { :received_recipients } diff --git a/spec/lib/gitlab/email/reply_parser_spec.rb b/spec/lib/gitlab/email/reply_parser_spec.rb index e4c68dbba92..35065b74eff 100644 --- a/spec/lib/gitlab/email/reply_parser_spec.rb +++ b/spec/lib/gitlab/email/reply_parser_spec.rb @@ -3,7 +3,7 @@ require "spec_helper" # Inspired in great part by Discourse's Email::Receiver -RSpec.describe Gitlab::Email::ReplyParser do +RSpec.describe Gitlab::Email::ReplyParser, feature_category: :team_planning do describe '#execute' do def test_parse_body(mail_string, params = {}) described_class.new(Mail::Message.new(mail_string), **params).execute @@ -188,67 +188,36 @@ RSpec.describe Gitlab::Email::ReplyParser do ) end - context 'properly renders email reply from gmail web client' do - context 'when feature flag is enabled' do - it do - expect(test_parse_body(fixture_file("emails/html_only.eml"))) - .to eq( - <<-BODY.strip_heredoc.chomp - ### This is a reply from standard GMail in Google Chrome. - - The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. - - Here's some **bold** text, **strong** text and *italic* in Markdown. - - Here's a link http://example.com - - Here's an img ![Miro](http://img.png)<details> - <summary> - One</summary> - Some details</details> - - <details> - <summary> - Two</summary> - Some details</details> - - Test reply. - - First paragraph. - - Second paragraph. - BODY - ) - end - end - - context 'when feature flag is disabled' do - before do - stub_feature_flags(service_desk_html_to_text_email_handler: false) - end + context 'properly renders email reply from gmail web client', feature_category: :service_desk do + it do + expect(test_parse_body(fixture_file("emails/html_only.eml"))) + .to eq( + <<-BODY.strip_heredoc.chomp + ### This is a reply from standard GMail in Google Chrome. - it do - expect(test_parse_body(fixture_file("emails/html_only.eml"))) - .to eq( - <<-BODY.strip_heredoc.chomp - ### This is a reply from standard GMail in Google Chrome. + The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. - The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. + Here's some **bold** text, **strong** text and *italic* in Markdown. - Here's some **bold** text, strong text and italic in Markdown. + Here's a link http://example.com - Here's a link http://example.com + Here's an img ![Miro](http://img.png)<details> + <summary> + One</summary> + Some details</details> - Here's an img [Miro]One Some details Two Some details + <details> + <summary> + Two</summary> + Some details</details> - Test reply. + Test reply. - First paragraph. + First paragraph. - Second paragraph. - BODY - ) - end + Second paragraph. + BODY + ) end end diff --git a/spec/lib/gitlab/service_desk_email_spec.rb b/spec/lib/gitlab/email/service_desk_email_spec.rb index 69569c0f194..d59b8aa2cf7 100644 --- a/spec/lib/gitlab/service_desk_email_spec.rb +++ b/spec/lib/gitlab/email/service_desk_email_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::ServiceDeskEmail do +RSpec.describe Gitlab::Email::ServiceDeskEmail, feature_category: :service_desk do let(:setting_name) { :service_desk_email } it_behaves_like 'common email methods' diff --git a/spec/lib/gitlab/emoji_spec.rb b/spec/lib/gitlab/emoji_spec.rb index 0db3b5f3b11..44b2ec12246 100644 --- a/spec/lib/gitlab/emoji_spec.rb +++ b/spec/lib/gitlab/emoji_spec.rb @@ -3,23 +3,6 @@ require 'spec_helper' RSpec.describe Gitlab::Emoji do - describe '.emoji_image_tag' do - it 'returns emoji image tag' do - emoji_image = described_class.emoji_image_tag('emoji_one', 'src_url') - - expect(emoji_image).to eq("<img class=\"emoji\" src=\"src_url\" title=\":emoji_one:\" alt=\":emoji_one:\" height=\"20\" width=\"20\" align=\"absmiddle\" />") - end - - it 'escapes emoji image attrs to prevent XSS' do - xss_payload = "<script>alert(1)</script>" - escaped_xss_payload = html_escape(xss_payload) - - emoji_image = described_class.emoji_image_tag(xss_payload, 'http://aaa#' + xss_payload) - - expect(emoji_image).to eq("<img class=\"emoji\" src=\"http://aaa##{escaped_xss_payload}\" title=\":#{escaped_xss_payload}:\" alt=\":#{escaped_xss_payload}:\" height=\"20\" width=\"20\" align=\"absmiddle\" />") - end - end - describe '.gl_emoji_tag' do it 'returns gl emoji tag if emoji is found' do emoji = TanukiEmoji.find_by_alpha_code('small_airplane') diff --git a/spec/lib/gitlab/endpoint_attributes_spec.rb b/spec/lib/gitlab/endpoint_attributes_spec.rb index 53f5b302f05..a623070c3eb 100644 --- a/spec/lib/gitlab/endpoint_attributes_spec.rb +++ b/spec/lib/gitlab/endpoint_attributes_spec.rb @@ -1,11 +1,8 @@ # frozen_string_literal: true -require 'fast_spec_helper' -require_relative '../../support/matchers/be_request_urgency' -require_relative '../../../lib/gitlab/endpoint_attributes/config' -require_relative '../../../lib/gitlab/endpoint_attributes' +require 'spec_helper' -RSpec.describe Gitlab::EndpointAttributes do +RSpec.describe Gitlab::EndpointAttributes, feature_category: :api do let(:base_controller) do Class.new do include ::Gitlab::EndpointAttributes diff --git a/spec/lib/gitlab/error_tracking_spec.rb b/spec/lib/gitlab/error_tracking_spec.rb index 0f056ee9eac..79016335a40 100644 --- a/spec/lib/gitlab/error_tracking_spec.rb +++ b/spec/lib/gitlab/error_tracking_spec.rb @@ -5,7 +5,7 @@ require 'spec_helper' require 'raven/transports/dummy' require 'sentry/transport/dummy_transport' -RSpec.describe Gitlab::ErrorTracking do +RSpec.describe Gitlab::ErrorTracking, feature_category: :shared do let(:exception) { RuntimeError.new('boom') } let(:issue_url) { 'http://gitlab.com/gitlab-org/gitlab-foss/issues/1' } let(:extra) { { issue_url: issue_url, some_other_info: 'info' } } @@ -58,7 +58,7 @@ RSpec.describe Gitlab::ErrorTracking do stub_feature_flags(enable_new_sentry_integration: true) stub_sentry_settings - allow(described_class).to receive(:sentry_configurable?) { true } + allow(described_class).to receive(:sentry_configurable?).and_return(true) allow(Labkit::Correlation::CorrelationId).to receive(:current_id).and_return('cid') allow(I18n).to receive(:locale).and_return('en') @@ -82,7 +82,7 @@ RSpec.describe Gitlab::ErrorTracking do describe '.track_and_raise_for_dev_exception' do context 'when exceptions for dev should be raised' do before do - expect(described_class).to receive(:should_raise_for_dev?).and_return(true) + allow(described_class).to receive(:should_raise_for_dev?).and_return(true) end it 'raises the exception' do @@ -101,7 +101,7 @@ RSpec.describe Gitlab::ErrorTracking do context 'when exceptions for dev should not be raised' do before do - expect(described_class).to receive(:should_raise_for_dev?).and_return(false) + allow(described_class).to receive(:should_raise_for_dev?).and_return(false) end it 'logs the exception with all attributes passed' do @@ -219,7 +219,7 @@ RSpec.describe Gitlab::ErrorTracking do end end - context 'the exception implements :sentry_extra_data' do + context 'when the exception implements :sentry_extra_data' do let(:extra_info) { { event: 'explosion', size: :massive } } before do @@ -239,7 +239,7 @@ RSpec.describe Gitlab::ErrorTracking do end end - context 'the exception implements :sentry_extra_data, which returns nil' do + context 'when the exception implements :sentry_extra_data, which returns nil' do let(:extra) { { issue_url: issue_url } } before do @@ -260,7 +260,7 @@ RSpec.describe Gitlab::ErrorTracking do end end - context 'event processors' do + describe 'event processors' do subject(:track_exception) { described_class.track_exception(exception, extra) } before do @@ -269,7 +269,16 @@ RSpec.describe Gitlab::ErrorTracking do allow(Gitlab::ErrorTracking::Logger).to receive(:error) end - context 'custom GitLab context when using Raven.capture_exception directly' do + # This is a workaround for restoring Raven's user context below. + # Raven.user_context(&block) does not restore the user context correctly. + around do |example| + previous_user_context = Raven.context.user.dup + example.run + ensure + Raven.context.user = previous_user_context + end + + context 'with custom GitLab context when using Raven.capture_exception directly' do subject(:track_exception) { Raven.capture_exception(exception) } it 'merges a default set of tags into the existing tags' do @@ -289,7 +298,7 @@ RSpec.describe Gitlab::ErrorTracking do end end - context 'custom GitLab context when using Sentry.capture_exception directly' do + context 'with custom GitLab context when using Sentry.capture_exception directly' do subject(:track_exception) { Sentry.capture_exception(exception) } it 'merges a default set of tags into the existing tags' do @@ -401,15 +410,17 @@ RSpec.describe Gitlab::ErrorTracking do end ['Gitlab::SidekiqMiddleware::RetryError', 'SubclassRetryError'].each do |ex| - let(:exception) { ex.constantize.new } + context "with #{ex} exception" do + let(:exception) { ex.constantize.new } - it "does not report #{ex} exception to Sentry" do - expect(Gitlab::ErrorTracking::Logger).to receive(:error) + it "does not report exception to Sentry" do + expect(Gitlab::ErrorTracking::Logger).to receive(:error) - track_exception + track_exception - expect(Raven.client.transport.events).to eq([]) - expect(Sentry.get_current_client.transport.events).to eq([]) + expect(Raven.client.transport.events).to eq([]) + expect(Sentry.get_current_client.transport.events).to eq([]) + end end end end @@ -491,7 +502,7 @@ RSpec.describe Gitlab::ErrorTracking do end end - context 'Sentry performance monitoring' do + describe 'Sentry performance monitoring' do context 'when ENABLE_SENTRY_PERFORMANCE_MONITORING env is disabled' do before do stub_env('ENABLE_SENTRY_PERFORMANCE_MONITORING', false) diff --git a/spec/lib/gitlab/etag_caching/middleware_spec.rb b/spec/lib/gitlab/etag_caching/middleware_spec.rb index fa0b3d1c6dd..d25511843ff 100644 --- a/spec/lib/gitlab/etag_caching/middleware_spec.rb +++ b/spec/lib/gitlab/etag_caching/middleware_spec.rb @@ -145,8 +145,11 @@ RSpec.describe Gitlab::EtagCaching::Middleware, :clean_gitlab_redis_shared_state expect(payload[:headers].env['HTTP_IF_NONE_MATCH']).to eq('W/"123"') end - it 'log subscriber processes action' do - expect_any_instance_of(ActionController::LogSubscriber).to receive(:process_action) + it "publishes process_action.action_controller event to be picked up by lograge's subscriber" do + # Lograge unhooks the default Rails subscriber (ActionController::LogSubscriber) + # and replaces with its own (Lograge::LogSubscribers::ActionController). + # When `lograge.keep_original_rails_log = true`, ActionController::LogSubscriber is kept. + expect_any_instance_of(Lograge::LogSubscribers::ActionController).to receive(:process_action) .with(instance_of(ActiveSupport::Notifications::Event)) .and_call_original diff --git a/spec/lib/gitlab/exception_log_formatter_spec.rb b/spec/lib/gitlab/exception_log_formatter_spec.rb index 7dda56f0bf5..82166971603 100644 --- a/spec/lib/gitlab/exception_log_formatter_spec.rb +++ b/spec/lib/gitlab/exception_log_formatter_spec.rb @@ -45,6 +45,12 @@ RSpec.describe Gitlab::ExceptionLogFormatter do allow(exception).to receive(:cause).and_return(ActiveRecord::StatementInvalid.new(sql: 'SELECT "users".* FROM "users" WHERE "users"."id" = 1 AND "users"."foo" = $1')) end + it 'adds the cause_class to payload' do + described_class.format!(exception, payload) + + expect(payload['exception.cause_class']).to eq('ActiveRecord::StatementInvalid') + end + it 'adds the normalized SQL query to payload' do described_class.format!(exception, payload) diff --git a/spec/lib/gitlab/external_authorization/config_spec.rb b/spec/lib/gitlab/external_authorization/config_spec.rb index 4231b0d3747..f1daa9249f4 100644 --- a/spec/lib/gitlab/external_authorization/config_spec.rb +++ b/spec/lib/gitlab/external_authorization/config_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::ExternalAuthorization::Config, feature_category: :authentication_and_authorization do +RSpec.describe Gitlab::ExternalAuthorization::Config, feature_category: :system_access do it 'allows deploy tokens and keys when external authorization is disabled' do stub_application_setting(external_authorization_service_enabled: false) expect(described_class.allow_deploy_tokens_and_deploy_keys?).to be_eql(true) diff --git a/spec/lib/gitlab/favicon_spec.rb b/spec/lib/gitlab/favicon_spec.rb index 884425dab3b..033fa5d1b42 100644 --- a/spec/lib/gitlab/favicon_spec.rb +++ b/spec/lib/gitlab/favicon_spec.rb @@ -40,14 +40,22 @@ RSpec.describe Gitlab::Favicon, :request_store do end end - describe '.status_overlay' do - subject { described_class.status_overlay('favicon_status_created') } + describe '.ci_status_overlay' do + subject { described_class.ci_status_overlay('favicon_status_created') } it 'returns the overlay for the status' do expect(subject).to match_asset_path '/assets/ci_favicons/favicon_status_created.png' end end + describe '.mr_status_overlay' do + subject { described_class.mr_status_overlay('favicon_status_merged') } + + it 'returns the overlay for the status' do + expect(subject).to match_asset_path '/assets/mr_favicons/favicon_status_merged.png' + end + end + describe '.available_status_names' do subject { described_class.available_status_names } diff --git a/spec/lib/gitlab/file_finder_spec.rb b/spec/lib/gitlab/file_finder_spec.rb index 27750f10e87..8afaec3c381 100644 --- a/spec/lib/gitlab/file_finder_spec.rb +++ b/spec/lib/gitlab/file_finder_spec.rb @@ -13,124 +13,58 @@ RSpec.describe Gitlab::FileFinder, feature_category: :global_search do let(:expected_file_by_content) { 'CHANGELOG' } end - context 'when code_basic_search_files_by_regexp is enabled' do - before do - stub_feature_flags(code_basic_search_files_by_regexp: true) - end - - context 'with inclusive filters' do - it 'filters by filename' do - results = subject.find('files filename:wm.svg') - - expect(results.count).to eq(1) - end - - it 'filters by path' do - results = subject.find('white path:images') - - expect(results.count).to eq(2) - end - - it 'filters by extension' do - results = subject.find('files extension:md') - - expect(results.count).to eq(4) - end - end - - context 'with exclusive filters' do - it 'filters by filename' do - results = subject.find('files -filename:wm.svg') - - expect(results.count).to eq(26) - end - - it 'filters by path' do - results = subject.find('white -path:images') - - expect(results.count).to eq(5) - end - - it 'filters by extension' do - results = subject.find('files -extension:md') + context 'with inclusive filters' do + it 'filters by filename' do + results = subject.find('files filename:wm.svg') - expect(results.count).to eq(23) - end + expect(results.count).to eq(1) end - context 'with white space in the path' do - it 'filters by path correctly' do - results = subject.find('directory path:"with space/README.md"') + it 'filters by path' do + results = subject.find('white path:images') - expect(results.count).to eq(1) - end + expect(results.count).to eq(2) end - it 'does not cause N+1 query' do - expect(Gitlab::GitalyClient).to receive(:call).at_most(10).times.and_call_original + it 'filters by extension' do + results = subject.find('files extension:md') - subject.find(': filename:wm.svg') + expect(results.count).to eq(4) end end - context 'when code_basic_search_files_by_regexp is disabled' do - before do - stub_feature_flags(code_basic_search_files_by_regexp: false) - end - - context 'with inclusive filters' do - it 'filters by filename' do - results = subject.find('files filename:wm.svg') - - expect(results.count).to eq(1) - end - - it 'filters by path' do - results = subject.find('white path:images') - - expect(results.count).to eq(1) - end - - it 'filters by extension' do - results = subject.find('files extension:md') + context 'with exclusive filters' do + it 'filters by filename' do + results = subject.find('files -filename:wm.svg') - expect(results.count).to eq(4) - end + expect(results.count).to eq(26) end - context 'with exclusive filters' do - it 'filters by filename' do - results = subject.find('files -filename:wm.svg') + it 'filters by path' do + results = subject.find('white -path:images') - expect(results.count).to eq(26) - end - - it 'filters by path' do - results = subject.find('white -path:images') - - expect(results.count).to eq(4) - end + expect(results.count).to eq(5) + end - it 'filters by extension' do - results = subject.find('files -extension:md') + it 'filters by extension' do + results = subject.find('files -extension:md') - expect(results.count).to eq(23) - end + expect(results.count).to eq(23) end + end - context 'with white space in the path' do - it 'filters by path correctly' do - results = subject.find('directory path:"with space/README.md"') + context 'with white space in the path' do + it 'filters by path correctly' do + results = subject.find('directory path:"with space/README.md"') - expect(results.count).to eq(1) - end + expect(results.count).to eq(1) end + end - it 'does not cause N+1 query' do - expect(Gitlab::GitalyClient).to receive(:call).at_most(10).times.and_call_original + it 'does not cause N+1 query' do + expect(Gitlab::GitalyClient).to receive(:call).at_most(10).times.and_call_original - subject.find(': filename:wm.svg') - end + subject.find(': filename:wm.svg') end end end diff --git a/spec/lib/gitlab/fogbugz_import/project_creator_spec.rb b/spec/lib/gitlab/fogbugz_import/project_creator_spec.rb index 8be9f55dbb6..39dad1360a5 100644 --- a/spec/lib/gitlab/fogbugz_import/project_creator_spec.rb +++ b/spec/lib/gitlab/fogbugz_import/project_creator_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::FogbugzImport::ProjectCreator do +RSpec.describe Gitlab::FogbugzImport::ProjectCreator, feature_category: :importers do let(:user) { create(:user) } let(:repo) do instance_double(Gitlab::FogbugzImport::Repository, @@ -22,6 +22,10 @@ RSpec.describe Gitlab::FogbugzImport::ProjectCreator do project_creator.execute end + before do + stub_application_setting(import_sources: ['fogbugz']) + end + it 'creates project with private visibility level' do expect(subject.persisted?).to eq(true) expect(subject.visibility_level).to eq(Gitlab::VisibilityLevel::PRIVATE) diff --git a/spec/lib/gitlab/git/blame_mode_spec.rb b/spec/lib/gitlab/git/blame_mode_spec.rb new file mode 100644 index 00000000000..3496b763f92 --- /dev/null +++ b/spec/lib/gitlab/git/blame_mode_spec.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Git::BlameMode, feature_category: :source_code_management do + subject(:blame_mode) { described_class.new(project, params) } + + let_it_be(:project) { build(:project) } + let(:params) { {} } + + describe '#streaming?' do + subject { blame_mode.streaming? } + + it { is_expected.to be_falsey } + + context 'when streaming param is provided' do + let(:params) { { streaming: true } } + + it { is_expected.to be_truthy } + end + end + + describe '#pagination?' do + subject { blame_mode.pagination? } + + it { is_expected.to be_truthy } + + context 'when `streaming` params is enabled' do + let(:params) { { streaming: true } } + + it { is_expected.to be_falsey } + end + + context 'when `no_pagination` param is provided' do + let(:params) { { no_pagination: true } } + + it { is_expected.to be_falsey } + end + + context 'when `blame_page_pagination` is disabled' do + before do + stub_feature_flags(blame_page_pagination: false) + end + + it { is_expected.to be_falsey } + end + end + + describe '#full?' do + subject { blame_mode.full? } + + it { is_expected.to be_falsey } + + context 'when `blame_page_pagination` is disabled' do + before do + stub_feature_flags(blame_page_pagination: false) + end + + it { is_expected.to be_truthy } + end + end +end diff --git a/spec/lib/gitlab/git/blame_pagination_spec.rb b/spec/lib/gitlab/git/blame_pagination_spec.rb new file mode 100644 index 00000000000..1f3c0c0342e --- /dev/null +++ b/spec/lib/gitlab/git/blame_pagination_spec.rb @@ -0,0 +1,175 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Git::BlamePagination, feature_category: :source_code_management do + subject(:blame_pagination) { described_class.new(blob, blame_mode, params) } + + let_it_be(:project) { create(:project, :repository) } + let_it_be(:commit) { project.repository.commit } + let_it_be(:blob) { project.repository.blob_at('HEAD', 'README.md') } + + let(:blame_mode) do + instance_double( + 'Gitlab::Git::BlameMode', + 'streaming?' => streaming_mode, + 'full?' => full_mode + ) + end + + let(:params) { { page: page } } + let(:page) { 1 } + let(:streaming_mode) { false } + let(:full_mode) { false } + + using RSpec::Parameterized::TableSyntax + + describe '#page' do + subject { blame_pagination.page } + + where(:page, :expected_page) do + nil | 1 + 1 | 1 + 5 | 5 + -1 | 1 + 'a' | 1 + end + + with_them do + it { is_expected.to eq(expected_page) } + end + end + + describe '#per_page' do + subject { blame_pagination.per_page } + + it { is_expected.to eq(described_class::PAGINATION_PER_PAGE) } + + context 'when blame mode is streaming' do + let(:streaming_mode) { true } + + it { is_expected.to eq(described_class::STREAMING_PER_PAGE) } + end + end + + describe '#total_pages' do + subject { blame_pagination.total_pages } + + before do + stub_const("#{described_class.name}::PAGINATION_PER_PAGE", 2) + end + + it { is_expected.to eq(2) } + end + + describe '#total_extra_pages' do + subject { blame_pagination.total_extra_pages } + + before do + stub_const("#{described_class.name}::PAGINATION_PER_PAGE", 2) + end + + it { is_expected.to eq(1) } + end + + describe '#pagination' do + subject { blame_pagination.paginator } + + before do + stub_const("#{described_class.name}::PAGINATION_PER_PAGE", 2) + end + + it 'returns a pagination object' do + is_expected.to be_kind_of(Kaminari::PaginatableArray) + + expect(subject.current_page).to eq(1) + expect(subject.total_pages).to eq(2) + expect(subject.total_count).to eq(4) + end + + context 'when user disabled the pagination' do + let(:full_mode) { true } + + it { is_expected.to be_nil } + end + + context 'when user chose streaming' do + let(:streaming_mode) { true } + + it { is_expected.to be_nil } + end + + context 'when per_page is above the global max per page limit' do + before do + stub_const("#{described_class.name}::PAGINATION_PER_PAGE", 1000) + allow(blob).to receive_message_chain(:data, :lines, :count) { 500 } + end + + it 'returns a correct pagination object' do + is_expected.to be_kind_of(Kaminari::PaginatableArray) + + expect(subject.current_page).to eq(1) + expect(subject.total_pages).to eq(1) + expect(subject.total_count).to eq(500) + end + end + + describe 'Pagination attributes' do + where(:page, :current_page, :total_pages) do + 1 | 1 | 2 + 2 | 2 | 2 + 0 | 1 | 2 # Incorrect + end + + with_them do + it 'returns the correct pagination attributes' do + expect(subject.current_page).to eq(current_page) + expect(subject.total_pages).to eq(total_pages) + end + end + end + end + + describe '#blame_range' do + subject { blame_pagination.blame_range } + + before do + stub_const("#{described_class.name}::PAGINATION_PER_PAGE", 2) + end + + where(:page, :expected_range) do + 1 | (1..2) + 2 | (3..4) + 0 | (1..2) + end + + with_them do + it { is_expected.to eq(expected_range) } + end + + context 'when user disabled the pagination' do + let(:full_mode) { true } + + it { is_expected.to be_nil } + end + + context 'when streaming is enabled' do + let(:streaming_mode) { true } + + before do + stub_const("#{described_class.name}::STREAMING_FIRST_PAGE_SIZE", 1) + stub_const("#{described_class.name}::STREAMING_PER_PAGE", 1) + end + + where(:page, :expected_range) do + 1 | (1..1) + 2 | (2..2) + 0 | (1..1) + end + + with_them do + it { is_expected.to eq(expected_range) } + end + end + end +end diff --git a/spec/lib/gitlab/git/commit_spec.rb b/spec/lib/gitlab/git/commit_spec.rb index d873151421d..e5f8918f7bb 100644 --- a/spec/lib/gitlab/git/commit_spec.rb +++ b/spec/lib/gitlab/git/commit_spec.rb @@ -2,7 +2,7 @@ require "spec_helper" -RSpec.describe Gitlab::Git::Commit do +RSpec.describe Gitlab::Git::Commit, feature_category: :source_code_management do let(:repository) { create(:project, :repository).repository.raw } let(:commit) { described_class.find(repository, SeedRepo::Commit::ID) } @@ -61,10 +61,41 @@ RSpec.describe Gitlab::Git::Commit do context 'body_size greater than threshold' do let(:body_size) { described_class::MAX_COMMIT_MESSAGE_DISPLAY_SIZE + 1 } - it 'returns the suject plus a notice about message size' do + it 'returns the subject plus a notice about message size' do expect(commit.safe_message).to eq("My commit\n\n--commit message is too big") end end + + context "large commit message" do + let(:user) { create(:user) } + let(:sha) { create_commit_with_large_message } + let(:commit) { repository.commit(sha) } + + def create_commit_with_large_message + repository.commit_files( + user, + branch_name: 'HEAD', + message: "Repeat " * 10 * 1024, + actions: [] + ).newrev + end + + it 'returns a String' do + # When #message is called, its encoding is forced from + # ASCII-8BIT to UTF-8, and the method returns a + # string. Calling #message again may cause BatchLoader to + # return since the encoding has been modified to UTF-8, and + # the encoding helper will return the original object unmodified. + # + # To ensure #fetch_body_from_gitaly returns a String, invoke + # #to_s. In the test below, do a strict type check to ensure + # that a String is always returned. Note that the Rspec + # matcher be_instance_of(String) appears to evaluate the + # BatchLoader result, so we have to do a strict comparison + # here. + 2.times { expect(String === commit.message).to be true } + end + end end end @@ -660,7 +691,8 @@ RSpec.describe Gitlab::Git::Commit do id: SeedRepo::Commit::ID, message: "tree css fixes", parent_ids: ["874797c3a73b60d2187ed6e2fcabd289ff75171e"], - trailers: {} + trailers: {}, + referenced_by: [] } end end diff --git a/spec/lib/gitlab/git/diff_collection_spec.rb b/spec/lib/gitlab/git/diff_collection_spec.rb index 7fa5bd8a92b..5fa0447091c 100644 --- a/spec/lib/gitlab/git/diff_collection_spec.rb +++ b/spec/lib/gitlab/git/diff_collection_spec.rb @@ -777,6 +777,26 @@ RSpec.describe Gitlab::Git::DiffCollection do end end + describe '.limits' do + let(:options) { {} } + + subject { described_class.limits(options) } + + context 'when options do not include max_patch_bytes_for_file_extension' do + it 'sets max_patch_bytes_for_file_extension as empty' do + expect(subject[:max_patch_bytes_for_file_extension]).to eq({}) + end + end + + context 'when options include max_patch_bytes_for_file_extension' do + let(:options) { { max_patch_bytes_for_file_extension: { '.file' => 1 } } } + + it 'sets value for max_patch_bytes_for_file_extension' do + expect(subject[:max_patch_bytes_for_file_extension]).to eq({ '.file' => 1 }) + end + end + end + def fake_diff(line_length, line_count) { 'diff' => "#{'a' * line_length}\n" * line_count } end diff --git a/spec/lib/gitlab/git/repository_spec.rb b/spec/lib/gitlab/git/repository_spec.rb index a8423703716..06904849ef5 100644 --- a/spec/lib/gitlab/git/repository_spec.rb +++ b/spec/lib/gitlab/git/repository_spec.rb @@ -203,6 +203,25 @@ RSpec.describe Gitlab::Git::Repository, feature_category: :source_code_managemen expect(metadata['CommitId']).to eq(expected_commit_id) end end + + context 'when resolve_ambiguous_archives is disabled' do + before do + stub_feature_flags(resolve_ambiguous_archives: false) + end + + where(:ref, :expected_commit_id, :desc) do + 'refs/heads/branch-merged' | ref(:branch_merged_commit_id) | 'when tag looks like a branch (difference!)' + 'branch-merged' | ref(:branch_master_commit_id) | 'when tag has the same name as a branch' + ref(:branch_merged_commit_id) | ref(:branch_merged_commit_id) | 'when tag looks like a commit id' + 'v0.0.0' | ref(:branch_master_commit_id) | 'when tag looks like a normal tag' + end + + with_them do + it 'selects the correct commit' do + expect(metadata['CommitId']).to eq(expected_commit_id) + end + end + end end context 'when branch is ambiguous' do @@ -222,6 +241,25 @@ RSpec.describe Gitlab::Git::Repository, feature_category: :source_code_managemen expect(metadata['CommitId']).to eq(expected_commit_id) end end + + context 'when resolve_ambiguous_archives is disabled' do + before do + stub_feature_flags(resolve_ambiguous_archives: false) + end + + where(:ref, :expected_commit_id, :desc) do + 'refs/tags/v1.0.0' | ref(:tag_1_0_0_commit_id) | 'when branch looks like a tag (difference!)' + 'v1.0.0' | ref(:tag_1_0_0_commit_id) | 'when branch has the same name as a tag' + ref(:branch_merged_commit_id) | ref(:branch_merged_commit_id) | 'when branch looks like a commit id' + 'just-a-normal-branch' | ref(:branch_master_commit_id) | 'when branch looks like a normal branch' + end + + with_them do + it 'selects the correct commit' do + expect(metadata['CommitId']).to eq(expected_commit_id) + end + end + end end context 'when ref is HEAD' do @@ -1820,8 +1858,8 @@ RSpec.describe Gitlab::Git::Repository, feature_category: :source_code_managemen context 'when Gitaly returns Internal error' do before do - expect(repository.gitaly_ref_client) - .to receive(:find_tag) + expect(Gitlab::GitalyClient) + .to receive(:call) .and_raise(GRPC::Internal, "tag not found") end @@ -1830,8 +1868,8 @@ RSpec.describe Gitlab::Git::Repository, feature_category: :source_code_managemen context 'when Gitaly returns tag_not_found error' do before do - expect(repository.gitaly_ref_client) - .to receive(:find_tag) + expect(Gitlab::GitalyClient) + .to receive(:call) .and_raise(new_detailed_error(GRPC::Core::StatusCodes::NOT_FOUND, "tag was not found", Gitaly::FindTagError.new(tag_not_found: Gitaly::ReferenceNotFoundError.new))) @@ -1862,47 +1900,37 @@ RSpec.describe Gitlab::Git::Repository, feature_category: :source_code_managemen end describe '#license' do - where(from_gitaly: [true, false]) - with_them do - subject(:license) { repository.license(from_gitaly) } + subject(:license) { repository.license } - context 'when no license file can be found' do - let_it_be(:project) { create(:project, :repository) } - let(:repository) { project.repository.raw_repository } + context 'when no license file can be found' do + let_it_be(:project) { create(:project, :repository) } + let(:repository) { project.repository.raw_repository } - before do - project.repository.delete_file(project.owner, 'LICENSE', message: 'remove license', branch_name: 'master') - end - - it { is_expected.to be_nil } + before do + project.repository.delete_file(project.owner, 'LICENSE', message: 'remove license', branch_name: 'master') end - context 'when an mit license is found' do - it { is_expected.to have_attributes(key: 'mit') } - end + it { is_expected.to be_nil } + end - context 'when license is not recognized ' do - let_it_be(:project) { create(:project, :repository) } - let(:repository) { project.repository.raw_repository } + context 'when an mit license is found' do + it { is_expected.to have_attributes(key: 'mit') } + end - before do - project.repository.update_file( - project.owner, - 'LICENSE', - 'This software is licensed under the Dummy license.', - message: 'Update license', - branch_name: 'master') - end + context 'when license is not recognized ' do + let_it_be(:project) { create(:project, :repository) } + let(:repository) { project.repository.raw_repository } - it { is_expected.to have_attributes(key: 'other', nickname: 'LICENSE') } + before do + project.repository.update_file( + project.owner, + 'LICENSE', + 'This software is licensed under the Dummy license.', + message: 'Update license', + branch_name: 'master') end - end - it 'does not crash when license is invalid' do - expect(Licensee::License).to receive(:new) - .and_raise(Licensee::InvalidLicense) - - expect(repository.license(false)).to be_nil + it { is_expected.to have_attributes(key: 'other', nickname: 'LICENSE') } end end @@ -2424,107 +2452,6 @@ RSpec.describe Gitlab::Git::Repository, feature_category: :source_code_managemen end end - describe '#squash' do - let(:branch_name) { 'fix' } - let(:start_sha) { TestEnv::BRANCH_SHA['master'] } - let(:end_sha) { '12d65c8dd2b2676fa3ac47d955accc085a37a9c1' } - - subject do - opts = { - branch: branch_name, - start_sha: start_sha, - end_sha: end_sha, - author: user, - message: 'Squash commit message' - } - - repository.squash(user, opts) - end - - # Should be ported to gitaly-ruby rspec suite https://gitlab.com/gitlab-org/gitaly/issues/1234 - skip 'sparse checkout' do - let(:expected_files) { %w(files files/js files/js/application.js) } - - it 'checks out only the files in the diff' do - allow(repository).to receive(:with_worktree).and_wrap_original do |m, *args| - m.call(*args) do - worktree_path = args[0] - files_pattern = File.join(worktree_path, '**', '*') - expected = expected_files.map do |path| - File.expand_path(path, worktree_path) - end - - expect(Dir[files_pattern]).to eq(expected) - end - end - - subject - end - - context 'when the diff contains a rename' do - let(:end_sha) do - repository.commit_files( - user, - branch_name: repository.root_ref, - message: 'Move CHANGELOG to encoding/', - actions: [{ - action: :move, - previous_path: 'CHANGELOG', - file_path: 'encoding/CHANGELOG', - content: 'CHANGELOG' - }] - ).newrev - end - - after do - # Erase our commits so other tests get the original repo - repository.write_ref(repository.root_ref, TestEnv::BRANCH_SHA['master']) - end - - it 'does not include the renamed file in the sparse checkout' do - allow(repository).to receive(:with_worktree).and_wrap_original do |m, *args| - m.call(*args) do - worktree_path = args[0] - files_pattern = File.join(worktree_path, '**', '*') - - expect(Dir[files_pattern]).not_to include('CHANGELOG') - expect(Dir[files_pattern]).not_to include('encoding/CHANGELOG') - end - end - - subject - end - end - end - - # Should be ported to gitaly-ruby rspec suite https://gitlab.com/gitlab-org/gitaly/issues/1234 - skip 'with an ASCII-8BIT diff' do - let(:diff) { "diff --git a/README.md b/README.md\nindex faaf198..43c5edf 100644\n--- a/README.md\n+++ b/README.md\n@@ -1,4 +1,4 @@\n-testme\n+✓ testme\n ======\n \n Sample repo for testing gitlab features\n" } - - it 'applies a ASCII-8BIT diff' do - allow(repository).to receive(:run_git!).and_call_original - allow(repository).to receive(:run_git!).with(%W(diff --binary #{start_sha}...#{end_sha})).and_return(diff.force_encoding('ASCII-8BIT')) - - expect(subject).to match(/\h{40}/) - end - end - - # Should be ported to gitaly-ruby rspec suite https://gitlab.com/gitlab-org/gitaly/issues/1234 - skip 'with trailing whitespace in an invalid patch' do - let(:diff) { "diff --git a/README.md b/README.md\nindex faaf198..43c5edf 100644\n--- a/README.md\n+++ b/README.md\n@@ -1,4 +1,4 @@\n-testme\n+ \n ====== \n \n Sample repo for testing gitlab features\n" } - - it 'does not include whitespace warnings in the error' do - allow(repository).to receive(:run_git!).and_call_original - allow(repository).to receive(:run_git!).with(%W(diff --binary #{start_sha}...#{end_sha})).and_return(diff.force_encoding('ASCII-8BIT')) - - expect { subject }.to raise_error do |error| - expect(error).to be_a(described_class::GitError) - expect(error.message).not_to include('trailing whitespace') - end - end - end - end - def create_remote_branch(remote_name, branch_name, source_branch_name) source_branch = repository.find_branch(source_branch_name) repository.write_ref("refs/remotes/#{remote_name}/#{branch_name}", source_branch.dereferenced_target.sha) diff --git a/spec/lib/gitlab/git/wraps_gitaly_errors_spec.rb b/spec/lib/gitlab/git/wraps_gitaly_errors_spec.rb index e551dfaa1c5..c321d4bbdb9 100644 --- a/spec/lib/gitlab/git/wraps_gitaly_errors_spec.rb +++ b/spec/lib/gitlab/git/wraps_gitaly_errors_spec.rb @@ -2,24 +2,81 @@ require 'spec_helper' -RSpec.describe Gitlab::Git::WrapsGitalyErrors do +RSpec.describe Gitlab::Git::WrapsGitalyErrors, feature_category: :gitaly do subject(:wrapper) do klazz = Class.new { include Gitlab::Git::WrapsGitalyErrors } klazz.new end describe "#wrapped_gitaly_errors" do - mapping = { - GRPC::NotFound => Gitlab::Git::Repository::NoRepository, - GRPC::InvalidArgument => ArgumentError, - GRPC::DeadlineExceeded => Gitlab::Git::CommandTimedOut, - GRPC::BadStatus => Gitlab::Git::CommandError - } - - mapping.each do |grpc_error, error| - it "wraps #{grpc_error} in a #{error}" do - expect { wrapper.wrapped_gitaly_errors { raise grpc_error, 'wrapped' } } - .to raise_error(error) + where(:original_error, :wrapped_error) do + [ + [GRPC::NotFound, Gitlab::Git::Repository::NoRepository], + [GRPC::InvalidArgument, ArgumentError], + [GRPC::DeadlineExceeded, Gitlab::Git::CommandTimedOut], + [GRPC::BadStatus, Gitlab::Git::CommandError] + ] + end + + with_them do + it "wraps #{params[:original_error]} in a #{params[:wrapped_error]}" do + expect { wrapper.wrapped_gitaly_errors { raise original_error, 'wrapped' } } + .to raise_error(wrapped_error) + end + end + + context 'when wrap GRPC::ResourceExhausted' do + context 'with Gitaly::LimitError detail' do + let(:original_error) do + new_detailed_error( + GRPC::Core::StatusCodes::RESOURCE_EXHAUSTED, + 'resource exhausted', + Gitaly::LimitError.new( + error_message: "maximum time in concurrency queue reached", + retry_after: Google::Protobuf::Duration.new(seconds: 5, nanos: 1500) + ) + ) + end + + it "wraps in a Gitlab::Git::ResourceExhaustedError with error message" do + expect { wrapper.wrapped_gitaly_errors { raise original_error } }.to raise_error do |wrapped_error| + expect(wrapped_error).to be_a(Gitlab::Git::ResourceExhaustedError) + expect(wrapped_error.message).to eql( + "Upstream Gitaly has been exhausted: maximum time in concurrency queue reached. Try again later" + ) + expect(wrapped_error.headers).to eql({ 'Retry-After' => 5 }) + end + end + end + + context 'with Gitaly::LimitError detail without retry after' do + let(:original_error) do + new_detailed_error( + GRPC::Core::StatusCodes::RESOURCE_EXHAUSTED, + 'resource exhausted', + Gitaly::LimitError.new(error_message: "maximum time in concurrency queue reached") + ) + end + + it "wraps in a Gitlab::Git::ResourceExhaustedError with error message" do + expect { wrapper.wrapped_gitaly_errors { raise original_error } }.to raise_error do |wrapped_error| + expect(wrapped_error).to be_a(Gitlab::Git::ResourceExhaustedError) + expect(wrapped_error.message).to eql( + "Upstream Gitaly has been exhausted: maximum time in concurrency queue reached. Try again later" + ) + expect(wrapped_error.headers).to eql({}) + end + end + end + + context 'without Gitaly::LimitError detail' do + it("wraps in a Gitlab::Git::ResourceExhaustedError with default message") { + expect { wrapper.wrapped_gitaly_errors { raise GRPC::ResourceExhausted } }.to raise_error do |wrapped_error| + expect(wrapped_error).to be_a(Gitlab::Git::ResourceExhaustedError) + expect(wrapped_error.message).to eql("Upstream Gitaly has been exhausted. Try again later") + expect(wrapped_error.headers).to eql({}) + end + } end end diff --git a/spec/lib/gitlab/git_access_spec.rb b/spec/lib/gitlab/git_access_spec.rb index ea2c239df07..1b205aa5c85 100644 --- a/spec/lib/gitlab/git_access_spec.rb +++ b/spec/lib/gitlab/git_access_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::GitAccess, :aggregate_failures, feature_category: :authentication_and_authorization do +RSpec.describe Gitlab::GitAccess, :aggregate_failures, feature_category: :system_access do include TermsHelper include AdminModeHelper include ExternalAuthorizationServiceHelpers @@ -869,11 +869,13 @@ RSpec.describe Gitlab::GitAccess, :aggregate_failures, feature_category: :authen check = -> { push_changes(changes[action]) } if allowed - expect(&check).not_to raise_error, - -> { "expected #{action} to be allowed" } + expect(&check).not_to raise_error, -> do + "expected #{action} for #{role} to be allowed while #{who_can_action}" + end else - expect(&check).to raise_error(Gitlab::GitAccess::ForbiddenError), - -> { "expected #{action} to be disallowed" } + expect(&check).to raise_error(Gitlab::GitAccess::ForbiddenError), -> do + "expected #{action} for #{role} to be disallowed while #{who_can_action}" + end end end end @@ -886,12 +888,12 @@ RSpec.describe Gitlab::GitAccess, :aggregate_failures, feature_category: :authen any: true, push_new_branch: true, push_master: true, - push_protected_branch: true, + push_protected_branch: false, push_remove_protected_branch: false, push_tag: true, push_new_tag: true, - push_all: true, - merge_into_protected_branch: true + push_all: false, + merge_into_protected_branch: false }, admin_without_admin_mode: { @@ -957,19 +959,22 @@ RSpec.describe Gitlab::GitAccess, :aggregate_failures, feature_category: :authen [%w(feature exact), ['feat*', 'wildcard']].each do |protected_branch_name, protected_branch_type| context do - let(:protected_branch) { create(:protected_branch, :maintainers_can_push, name: protected_branch_name, project: project) } + let(:who_can_action) { :maintainers_can_push } + let(:protected_branch) { create(:protected_branch, who_can_action, name: protected_branch_name, project: project) } run_permission_checks(permissions_matrix) end context "when developers are allowed to push into the #{protected_branch_type} protected branch" do - let(:protected_branch) { create(:protected_branch, :developers_can_push, name: protected_branch_name, project: project) } + let(:who_can_action) { :developers_can_push } + let(:protected_branch) { create(:protected_branch, who_can_action, name: protected_branch_name, project: project) } run_permission_checks(permissions_matrix.deep_merge(developer: { push_protected_branch: true, push_all: true, merge_into_protected_branch: true })) end - context "developers are allowed to merge into the #{protected_branch_type} protected branch" do - let(:protected_branch) { create(:protected_branch, :developers_can_merge, name: protected_branch_name, project: project) } + context "when developers are allowed to merge into the #{protected_branch_type} protected branch" do + let(:who_can_action) { :developers_can_merge } + let(:protected_branch) { create(:protected_branch, who_can_action, name: protected_branch_name, project: project) } context "when a merge request exists for the given source/target branch" do context "when the merge request is in progress" do @@ -996,6 +1001,7 @@ RSpec.describe Gitlab::GitAccess, :aggregate_failures, feature_category: :authen end context "when developers are allowed to push and merge into the #{protected_branch_type} protected branch" do + let(:who_can_action) { :developers_can_push_and_merge } let(:protected_branch) { create(:protected_branch, :developers_can_merge, :developers_can_push, name: protected_branch_name, project: project) } run_permission_checks(permissions_matrix.deep_merge(developer: { push_protected_branch: true, push_all: true, merge_into_protected_branch: true })) diff --git a/spec/lib/gitlab/git_ref_validator_spec.rb b/spec/lib/gitlab/git_ref_validator_spec.rb index 03dd4e7b89b..1a79817130c 100644 --- a/spec/lib/gitlab/git_ref_validator_spec.rb +++ b/spec/lib/gitlab/git_ref_validator_spec.rb @@ -37,6 +37,11 @@ RSpec.describe Gitlab::GitRefValidator do it { expect(described_class.validate("\xA0\u0000\xB0")).to be false } it { expect(described_class.validate("")).to be false } it { expect(described_class.validate(nil)).to be false } + it { expect(described_class.validate('HEAD')).to be false } + + context 'when skip_head_ref_check is true' do + it { expect(described_class.validate('HEAD', skip_head_ref_check: true)).to be true } + end end describe '.validate_merge_request_branch' do diff --git a/spec/lib/gitlab/gitaly_client/commit_service_spec.rb b/spec/lib/gitlab/gitaly_client/commit_service_spec.rb index 252d20d9c3a..05205ab6d6a 100644 --- a/spec/lib/gitlab/gitaly_client/commit_service_spec.rb +++ b/spec/lib/gitlab/gitaly_client/commit_service_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::GitalyClient::CommitService do +RSpec.describe Gitlab::GitalyClient::CommitService, feature_category: :gitaly do let_it_be(:project) { create(:project, :repository) } let(:storage_name) { project.repository_storage } @@ -406,6 +406,18 @@ RSpec.describe Gitlab::GitalyClient::CommitService do end shared_examples 'a #list_all_commits message' do + let(:objects_exist_repo) do + # The object directory of the repository must not be set so that we + # don't use the quarantine directory. + repository.gitaly_repository.dup.tap do |repo| + repo.git_object_directory = '' + end + end + + let(:expected_object_exist_requests) do + [gitaly_request_with_params(repository: objects_exist_repo, revisions: gitaly_commits.map(&:id))] + end + it 'sends a list_all_commits message' do expected_repository = repository.gitaly_repository.dup expected_repository.git_alternate_object_directories = Google::Protobuf::RepeatedField.new(:string) @@ -415,24 +427,12 @@ RSpec.describe Gitlab::GitalyClient::CommitService do .with(gitaly_request_with_params(repository: expected_repository), kind_of(Hash)) .and_return([Gitaly::ListAllCommitsResponse.new(commits: gitaly_commits)]) - # The object directory of the repository must not be set so that we - # don't use the quarantine directory. - objects_exist_repo = repository.gitaly_repository.dup - objects_exist_repo.git_object_directory = "" - - # The first request contains the repository, the second request the - # commit IDs we want to check for existence. - objects_exist_request = [ - gitaly_request_with_params(repository: objects_exist_repo), - gitaly_request_with_params(revisions: gitaly_commits.map(&:id)) - ] - objects_exist_response = Gitaly::CheckObjectsExistResponse.new(revisions: revision_existence.map do |rev, exists| Gitaly::CheckObjectsExistResponse::RevisionExistence.new(name: rev, exists: exists) end) expect(service).to receive(:check_objects_exist) - .with(objects_exist_request, kind_of(Hash)) + .with(expected_object_exist_requests, kind_of(Hash)) .and_return([objects_exist_response]) end @@ -495,6 +495,20 @@ RSpec.describe Gitlab::GitalyClient::CommitService do it_behaves_like 'a #list_all_commits message' end + + context 'with more than 100 commits' do + let(:gitaly_commits) { build_list(:gitaly_commit, 101) } + let(:revision_existence) { gitaly_commits.to_h { |c| [c.id, false] } } + + it_behaves_like 'a #list_all_commits message' do + let(:expected_object_exist_requests) do + [ + gitaly_request_with_params(repository: objects_exist_repo, revisions: gitaly_commits[0...100].map(&:id)), + gitaly_request_with_params(revisions: gitaly_commits[100..].map(&:id)) + ] + end + end + end end context 'without hook environment' do @@ -588,9 +602,7 @@ RSpec.describe Gitlab::GitalyClient::CommitService do it 'returns expected results' do expect_next_instance_of(Gitaly::CommitService::Stub) do |service| - expect(service) - .to receive(:check_objects_exist) - .and_call_original + expect(service).to receive(:check_objects_exist).and_call_original end expect(client.object_existence_map(revisions.keys)).to eq(revisions) @@ -600,7 +612,11 @@ RSpec.describe Gitlab::GitalyClient::CommitService do context 'with empty request' do let(:revisions) { {} } - it_behaves_like 'a CheckObjectsExistRequest' + it 'doesnt call for Gitaly' do + expect(Gitaly::CommitService::Stub).not_to receive(:new) + + expect(client.object_existence_map(revisions.keys)).to eq(revisions) + end end context 'when revision exists' do diff --git a/spec/lib/gitlab/gitaly_client/ref_service_spec.rb b/spec/lib/gitlab/gitaly_client/ref_service_spec.rb index 09d8ea3cc0a..7bdfa8922d3 100644 --- a/spec/lib/gitlab/gitaly_client/ref_service_spec.rb +++ b/spec/lib/gitlab/gitaly_client/ref_service_spec.rb @@ -213,8 +213,13 @@ RSpec.describe Gitlab::GitalyClient::RefService, feature_category: :gitaly do client.local_branches(sort_by: 'name_asc') end - it 'raises an argument error if an invalid sort_by parameter is passed' do - expect { client.local_branches(sort_by: 'invalid_sort') }.to raise_error(ArgumentError) + it 'uses default sort by name' do + expect_any_instance_of(Gitaly::RefService::Stub) + .to receive(:find_local_branches) + .with(gitaly_request_with_params(sort_by: :NAME), kind_of(Hash)) + .and_return([]) + + client.local_branches(sort_by: 'invalid') end end @@ -270,6 +275,17 @@ RSpec.describe Gitlab::GitalyClient::RefService, feature_category: :gitaly do client.tags(sort_by: 'version_asc') end end + + context 'when sorting option is invalid' do + it 'uses default sort by name' do + expect_any_instance_of(Gitaly::RefService::Stub) + .to receive(:find_all_tags) + .with(gitaly_request_with_params(sort_by: nil), kind_of(Hash)) + .and_return([]) + + client.tags(sort_by: 'invalid') + end + end end context 'with pagination option' do diff --git a/spec/lib/gitlab/gitaly_client/repository_service_spec.rb b/spec/lib/gitlab/gitaly_client/repository_service_spec.rb index 434550186c1..f457ba06074 100644 --- a/spec/lib/gitlab/gitaly_client/repository_service_spec.rb +++ b/spec/lib/gitlab/gitaly_client/repository_service_spec.rb @@ -275,7 +275,8 @@ RSpec.describe Gitlab::GitalyClient::RepositoryService do it 'sends a create_repository message without arguments' do expect_any_instance_of(Gitaly::RepositoryService::Stub) .to receive(:create_repository) - .with(gitaly_request_with_path(storage_name, relative_path).and(gitaly_request_with_params(default_branch: '')), kind_of(Hash)) + .with(gitaly_request_with_path(storage_name, relative_path) + .and(gitaly_request_with_params(default_branch: '')), kind_of(Hash)) .and_return(double) client.create_repository @@ -284,11 +285,23 @@ RSpec.describe Gitlab::GitalyClient::RepositoryService do it 'sends a create_repository message with default branch' do expect_any_instance_of(Gitaly::RepositoryService::Stub) .to receive(:create_repository) - .with(gitaly_request_with_path(storage_name, relative_path).and(gitaly_request_with_params(default_branch: 'default-branch-name')), kind_of(Hash)) + .with(gitaly_request_with_path(storage_name, relative_path) + .and(gitaly_request_with_params(default_branch: 'default-branch-name')), kind_of(Hash)) .and_return(double) client.create_repository('default-branch-name') end + + it 'sends a create_repository message with default branch containing non ascii chars' do + expect_any_instance_of(Gitaly::RepositoryService::Stub) + .to receive(:create_repository) + .with(gitaly_request_with_path(storage_name, relative_path) + .and(gitaly_request_with_params( + default_branch: Gitlab::EncodingHelper.encode_binary('feature/新機能'))), kind_of(Hash) + ).and_return(double) + + client.create_repository('feature/新機能') + end end describe '#create_from_snapshot' do @@ -314,17 +327,31 @@ RSpec.describe Gitlab::GitalyClient::RepositoryService do end describe '#search_files_by_regexp' do - subject(:result) { client.search_files_by_regexp('master', '.*') } + subject(:result) { client.search_files_by_regexp(ref, '.*') } before do expect_any_instance_of(Gitaly::RepositoryService::Stub) .to receive(:search_files_by_name) - .with(gitaly_request_with_path(storage_name, relative_path), kind_of(Hash)) - .and_return([double(files: ['file1.txt']), double(files: ['file2.txt'])]) + .with(gitaly_request_with_path(storage_name, relative_path), kind_of(Hash)) + .and_return([double(files: ['file1.txt']), double(files: ['file2.txt'])]) end - it 'sends a search_files_by_name message and returns a flatten array' do - expect(result).to contain_exactly('file1.txt', 'file2.txt') + shared_examples 'a search for files by regexp' do + it 'sends a search_files_by_name message and returns a flatten array' do + expect(result).to contain_exactly('file1.txt', 'file2.txt') + end + end + + context 'with ASCII ref' do + let(:ref) { 'master' } + + it_behaves_like 'a search for files by regexp' + end + + context 'with non-ASCII ref' do + let(:ref) { 'ref-ñéüçæøß-val' } + + it_behaves_like 'a search for files by regexp' end end diff --git a/spec/lib/gitlab/gitaly_client/with_feature_flag_actors_spec.rb b/spec/lib/gitlab/gitaly_client/with_feature_flag_actors_spec.rb index 61945cc06b8..42153a9a3d8 100644 --- a/spec/lib/gitlab/gitaly_client/with_feature_flag_actors_spec.rb +++ b/spec/lib/gitlab/gitaly_client/with_feature_flag_actors_spec.rb @@ -131,23 +131,23 @@ RSpec.describe Gitlab::GitalyClient::WithFeatureFlagActors do end context 'when project design' do - let_it_be(:project) { create(:project, group: create(:group)) } - let(:issue) { create(:issue, project: project) } - let(:design) { create(:design, issue: issue) } + let_it_be(:design_repo) do + create(:design_management_repository, project: create(:project, group: create(:group))) + end - let(:expected_project) { project } - let(:expected_group) { project.group } + let(:expected_project) { design_repo.project } + let(:expected_group) { design_repo.project.group } it_behaves_like 'Gitaly feature flag actors are inferred from repository' do - let(:repository) { design.repository } + let(:repository) { design_repo.repository } end it_behaves_like 'Gitaly feature flag actors are inferred from repository' do - let(:repository) { design.repository.raw } + let(:repository) { design_repo.repository.raw } end it_behaves_like 'Gitaly feature flag actors are inferred from repository' do - let(:repository) { raw_repo_without_container(design.repository) } + let(:repository) { raw_repo_without_container(design_repo.repository) } end end end diff --git a/spec/lib/gitlab/github_import/bulk_importing_spec.rb b/spec/lib/gitlab/github_import/bulk_importing_spec.rb index 136ddb566aa..28fbd4d883f 100644 --- a/spec/lib/gitlab/github_import/bulk_importing_spec.rb +++ b/spec/lib/gitlab/github_import/bulk_importing_spec.rb @@ -13,6 +13,8 @@ RSpec.describe Gitlab::GithubImport::BulkImporting, feature_category: :importers :object_type end + private + def model Label end @@ -26,85 +28,153 @@ RSpec.describe Gitlab::GithubImport::BulkImporting, feature_category: :importers end describe '#build_database_rows' do - it 'returns an Array containing the rows to insert and validation errors if object invalid' do - object = double(:object, title: 'Foo') - - expect(importer) - .to receive(:build_attributes) - .with(object) - .and_return({ title: 'Foo' }) - - expect(Label) - .to receive(:new) - .with({ title: 'Foo' }) - .and_return(label) - - expect(importer) - .to receive(:already_imported?) - .with(object) - .and_return(false) - - expect(Gitlab::Import::Logger) - .to receive(:info) - .with( - import_type: :github, - project_id: 1, - importer: 'MyImporter', - message: '1 object_types fetched' - ) - - expect(Gitlab::GithubImport::ObjectCounter) - .to receive(:increment) - .with( - project, - :object_type, - :fetched, - value: 1 - ) - - enum = [[object, 1]].to_enum - - rows, errors = importer.build_database_rows(enum) + context 'without validation errors' do + let(:object) { double(:object, title: 'Foo') } + + it 'returns an array containing the rows to insert' do + expect(importer) + .to receive(:build_attributes) + .with(object) + .and_return({ title: 'Foo' }) + + expect(Label) + .to receive(:new) + .with({ title: 'Foo' }) + .and_return(label) + + expect(importer) + .to receive(:already_imported?) + .with(object) + .and_return(false) + + expect(Gitlab::Import::Logger) + .to receive(:info) + .with( + import_type: :github, + project_id: 1, + importer: 'MyImporter', + message: '1 object_types fetched' + ) + + expect(Gitlab::GithubImport::ObjectCounter) + .to receive(:increment) + .with( + project, + :object_type, + :fetched, + value: 1 + ) + + enum = [[object, 1]].to_enum + + rows, errors = importer.build_database_rows(enum) + + expect(rows).to match_array([{ title: 'Foo' }]) + expect(errors).to be_empty + end - expect(rows).to match_array([{ title: 'Foo' }]) - expect(errors).to be_empty + it 'does not import objects that have already been imported' do + expect(importer) + .not_to receive(:build_attributes) + + expect(importer) + .to receive(:already_imported?) + .with(object) + .and_return(true) + + expect(Gitlab::Import::Logger) + .to receive(:info) + .with( + import_type: :github, + project_id: 1, + importer: 'MyImporter', + message: '0 object_types fetched' + ) + + expect(Gitlab::GithubImport::ObjectCounter) + .to receive(:increment) + .with( + project, + :object_type, + :fetched, + value: 0 + ) + + enum = [[object, 1]].to_enum + + rows, errors = importer.build_database_rows(enum) + + expect(rows).to be_empty + expect(errors).to be_empty + end end - it 'does not import objects that have already been imported' do - object = double(:object, title: 'Foo') - - expect(importer) - .not_to receive(:build_attributes) + context 'with validation errors' do + let(:object) { double(:object, id: 12345, title: 'bug,bug') } - expect(importer) - .to receive(:already_imported?) - .with(object) - .and_return(true) + before do + allow(importer) + .to receive(:already_imported?) + .with(object) + .and_return(false) - expect(Gitlab::Import::Logger) - .to receive(:info) - .with( - import_type: :github, - project_id: 1, - importer: 'MyImporter', - message: '0 object_types fetched' - ) - - expect(Gitlab::GithubImport::ObjectCounter) - .to receive(:increment) - .with( - project, - :object_type, - :fetched, - value: 0 - ) + allow(importer) + .to receive(:build_attributes) + .with(object) + .and_return({ title: 'bug,bug' }) + end - enum = [[object, 1]].to_enum + context 'without implemented github_identifiers method' do + it 'raises NotImplementedError' do + enum = [[object, 1]].to_enum - rows, errors = importer.build_database_rows(enum) + expect { importer.build_database_rows(enum) }.to raise_error(NotImplementedError) + end + end - expect(rows).to be_empty - expect(errors).to be_empty + context 'with implemented github_identifiers method' do + it 'returns an array containing the validation errors and logs them' do + expect(importer) + .to receive(:github_identifiers) + .with(object) + .and_return( + { + id: object.id, + title: object.title, + object_type: importer.object_type + } + ) + + expect(Gitlab::Import::Logger) + .to receive(:error) + .with( + import_type: :github, + project_id: 1, + importer: 'MyImporter', + message: ['Title is invalid'], + github_identifiers: { id: 12345, title: 'bug,bug', object_type: :object_type } + ) + + expect(Gitlab::GithubImport::ObjectCounter) + .to receive(:increment) + .with( + project, + :object_type, + :fetched, + value: 0 + ) + + enum = [[object, 1]].to_enum + + rows, errors = importer.build_database_rows(enum) + + expect(rows).to be_empty + expect(errors).not_to be_empty + + expect(errors[0][:validation_errors].full_messages).to match_array(['Title is invalid']) + expect(errors[0][:github_identifiers]).to eq({ id: 12345, title: 'bug,bug', object_type: :object_type }) + end + end end end @@ -157,7 +227,8 @@ RSpec.describe Gitlab::GithubImport::BulkImporting, feature_category: :importers exception_message: 'Title invalid', correlation_id_value: 'cid', retry_count: nil, - created_at: Time.zone.now + created_at: Time.zone.now, + external_identifiers: { id: 123456 } }] end @@ -170,8 +241,23 @@ RSpec.describe Gitlab::GithubImport::BulkImporting, feature_category: :importers expect(import_failures).to receive(:insert_all).with(formatted_errors) expect(Labkit::Correlation::CorrelationId).to receive(:current_or_new_id).and_return('cid') - importer.bulk_insert_failures([error]) + importer.bulk_insert_failures([{ + validation_errors: error, + github_identifiers: { id: 123456 } + }]) end end end + + describe '#object_type' do + let(:importer_class) do + Class.new do + include Gitlab::GithubImport::BulkImporting + end + end + + it 'raises NotImplementedError' do + expect { importer.object_type }.to raise_error(NotImplementedError) + end + end end diff --git a/spec/lib/gitlab/github_import/client_spec.rb b/spec/lib/gitlab/github_import/client_spec.rb index e93d585bc3c..c9f7fd4f748 100644 --- a/spec/lib/gitlab/github_import/client_spec.rb +++ b/spec/lib/gitlab/github_import/client_spec.rb @@ -131,6 +131,16 @@ RSpec.describe Gitlab::GithubImport::Client, feature_category: :importers do end end + describe '#collaborators' do + it 'returns the collaborators' do + expect(client) + .to receive(:each_object) + .with(:collaborators, 'foo/bar') + + client.collaborators('foo/bar') + end + end + describe '#branch_protection' do it 'returns the protection details for the given branch' do expect(client.octokit) @@ -580,7 +590,10 @@ RSpec.describe Gitlab::GithubImport::Client, feature_category: :importers do end describe '#search_repos_by_name_graphql' do - let(:expected_query) { 'test in:name is:public,private user:user repo:repo1 repo:repo2 org:org1 org:org2' } + let(:expected_query) do + 'test in:name is:public,private fork:true user:user repo:repo1 repo:repo2 org:org1 org:org2' + end + let(:expected_graphql_params) { "type: REPOSITORY, query: \"#{expected_query}\"" } let(:expected_graphql) do <<-TEXT @@ -600,7 +613,8 @@ RSpec.describe Gitlab::GithubImport::Client, feature_category: :importers do endCursor hasNextPage hasPreviousPage - } + }, + repositoryCount } } TEXT @@ -616,7 +630,7 @@ RSpec.describe Gitlab::GithubImport::Client, feature_category: :importers do context 'when relation type option present' do context 'when relation type is owned' do - let(:expected_query) { 'test in:name is:public,private user:user' } + let(:expected_query) { 'test in:name is:public,private fork:true user:user' } it 'searches for repositories within the organization based on name' do expect(client.octokit).to receive(:post).with( @@ -628,7 +642,7 @@ RSpec.describe Gitlab::GithubImport::Client, feature_category: :importers do end context 'when relation type is organization' do - let(:expected_query) { 'test in:name is:public,private org:test-login' } + let(:expected_query) { 'test in:name is:public,private fork:true org:test-login' } it 'searches for repositories within the organization based on name' do expect(client.octokit).to receive(:post).with( @@ -642,7 +656,7 @@ RSpec.describe Gitlab::GithubImport::Client, feature_category: :importers do end context 'when relation type is collaborated' do - let(:expected_query) { 'test in:name is:public,private repo:repo1 repo:repo2' } + let(:expected_query) { 'test in:name is:public,private fork:true repo:repo1 repo:repo2' } it 'searches for collaborated repositories based on name' do expect(client.octokit).to receive(:post).with( @@ -707,44 +721,30 @@ RSpec.describe Gitlab::GithubImport::Client, feature_category: :importers do end end - describe '#search_repos_by_name' do - let(:expected_query) { 'test in:name is:public,private user:user repo:repo1 repo:repo2 org:org1 org:org2' } - - it 'searches for repositories based on name' do - expect(client.octokit).to receive(:search_repositories).with(expected_query, {}) + describe '#count_repos_by_relation_type_graphql' do + relation_types = { + 'owned' => ' in:name is:public,private fork:true user:user', + 'collaborated' => ' in:name is:public,private fork:true repo:repo1 repo:repo2', + 'organization' => 'org:org1 org:org2' + } - client.search_repos_by_name('test') - end + relation_types.each do |relation_type, expected_query| + expected_graphql_params = "type: REPOSITORY, query: \"#{expected_query}\"" + expected_graphql = + <<-TEXT + { + search(#{expected_graphql_params}) { + repositoryCount + } + } + TEXT - context 'when pagination options present' do - it 'searches for repositories via expected query' do - expect(client.octokit).to receive(:search_repositories).with( - expected_query, { page: 2, per_page: 25 } + it 'returns count by relation_type' do + expect(client.octokit).to receive(:post).with( + '/graphql', { query: expected_graphql }.to_json ) - client.search_repos_by_name('test', { page: 2, per_page: 25 }) - end - end - - context 'when Faraday error received from octokit', :aggregate_failures do - let(:error_class) { described_class::CLIENT_CONNECTION_ERROR } - let(:info_params) { { 'error.class': error_class } } - - it 'retries on error and succeeds' do - allow_retry(:search_repositories) - - expect(Gitlab::Import::Logger).to receive(:info).with(hash_including(info_params)).once - - expect(client.search_repos_by_name('test')).to eq({}) - end - - it 'retries and does not succeed' do - allow(client.octokit) - .to receive(:search_repositories) - .with(expected_query, {}) - .and_raise(error_class, 'execution expired') - - expect { client.search_repos_by_name('test') }.to raise_error(error_class, 'execution expired') + client.count_repos_by_relation_type_graphql(relation_type: relation_type) end end end diff --git a/spec/lib/gitlab/github_import/clients/proxy_spec.rb b/spec/lib/gitlab/github_import/clients/proxy_spec.rb index 0baff7bafcb..7b2a8fa9d74 100644 --- a/spec/lib/gitlab/github_import/clients/proxy_spec.rb +++ b/spec/lib/gitlab/github_import/clients/proxy_spec.rb @@ -8,6 +8,10 @@ RSpec.describe Gitlab::GithubImport::Clients::Proxy, :manage, feature_category: let(:access_token) { 'test_token' } let(:client_options) { { foo: :bar } } + it { expect(client).to delegate_method(:each_object).to(:client) } + it { expect(client).to delegate_method(:user).to(:client) } + it { expect(client).to delegate_method(:octokit).to(:client) } + describe '#repos' do let(:search_text) { 'search text' } let(:pagination_options) { { limit: 10 } } @@ -15,54 +19,32 @@ RSpec.describe Gitlab::GithubImport::Clients::Proxy, :manage, feature_category: context 'when remove_legacy_github_client FF is enabled' do let(:client_stub) { instance_double(Gitlab::GithubImport::Client) } - context 'with github_client_fetch_repos_via_graphql FF enabled' do - let(:client_response) do - { - data: { - search: { - nodes: [{ name: 'foo' }, { name: 'bar' }], - pageInfo: { startCursor: 'foo', endCursor: 'bar' } - } + let(:client_response) do + { + data: { + search: { + nodes: [{ name: 'foo' }, { name: 'bar' }], + pageInfo: { startCursor: 'foo', endCursor: 'bar' }, + repositoryCount: 2 } } - end - - it 'fetches repos with Gitlab::GithubImport::Client (GraphQL API)' do - expect(Gitlab::GithubImport::Client) - .to receive(:new).with(access_token).and_return(client_stub) - expect(client_stub) - .to receive(:search_repos_by_name_graphql) - .with(search_text, pagination_options).and_return(client_response) - - expect(client.repos(search_text, pagination_options)).to eq( - { - repos: [{ name: 'foo' }, { name: 'bar' }], - page_info: { startCursor: 'foo', endCursor: 'bar' } - } - ) - end + } end - context 'with github_client_fetch_repos_via_graphql FF disabled' do - let(:client_response) do - { items: [{ name: 'foo' }, { name: 'bar' }] } - end - - before do - stub_feature_flags(github_client_fetch_repos_via_graphql: false) - end - - it 'fetches repos with Gitlab::GithubImport::Client (REST API)' do - expect(Gitlab::GithubImport::Client) - .to receive(:new).with(access_token).and_return(client_stub) - expect(client_stub) - .to receive(:search_repos_by_name) - .with(search_text, pagination_options).and_return(client_response) + it 'fetches repos with Gitlab::GithubImport::Client (GraphQL API)' do + expect(Gitlab::GithubImport::Client) + .to receive(:new).with(access_token).and_return(client_stub) + expect(client_stub) + .to receive(:search_repos_by_name_graphql) + .with(search_text, pagination_options).and_return(client_response) - expect(client.repos(search_text, pagination_options)).to eq( - { repos: [{ name: 'foo' }, { name: 'bar' }] } - ) - end + expect(client.repos(search_text, pagination_options)).to eq( + { + repos: [{ name: 'foo' }, { name: 'bar' }], + page_info: { startCursor: 'foo', endCursor: 'bar' }, + count: 2 + } + ) end end @@ -99,4 +81,59 @@ RSpec.describe Gitlab::GithubImport::Clients::Proxy, :manage, feature_category: end end end + + describe '#count_by', :clean_gitlab_redis_cache do + context 'when remove_legacy_github_client FF is enabled' do + let(:client_stub) { instance_double(Gitlab::GithubImport::Client) } + let(:client_response) { { data: { search: { repositoryCount: 1 } } } } + + before do + stub_feature_flags(remove_legacy_github_client: true) + end + + context 'when value is cached' do + before do + Gitlab::Cache::Import::Caching.write('github-importer/provider-repo-count/owned/user_id', 3) + end + + it 'returns repository count from cache' do + expect(Gitlab::GithubImport::Client) + .to receive(:new).with(access_token).and_return(client_stub) + expect(client_stub) + .not_to receive(:count_repos_by_relation_type_graphql) + .with({ relation_type: 'owned' }) + expect(client.count_repos_by('owned', 'user_id')).to eq(3) + end + end + + context 'when value is not cached' do + it 'returns repository count' do + expect(Gitlab::GithubImport::Client) + .to receive(:new).with(access_token).and_return(client_stub) + expect(client_stub) + .to receive(:count_repos_by_relation_type_graphql) + .with({ relation_type: 'owned' }).and_return(client_response) + expect(Gitlab::Cache::Import::Caching) + .to receive(:write) + .with('github-importer/provider-repo-count/owned/user_id', 1, timeout: 5.minutes) + .and_call_original + expect(client.count_repos_by('owned', 'user_id')).to eq(1) + end + end + end + + context 'when remove_legacy_github_client FF is disabled' do + let(:client_stub) { instance_double(Gitlab::LegacyGithubImport::Client) } + + before do + stub_feature_flags(remove_legacy_github_client: false) + end + + it 'returns nil' do + expect(Gitlab::LegacyGithubImport::Client) + .to receive(:new).with(access_token, client_options).and_return(client_stub) + expect(client.count_repos_by('owned', 'user_id')).to be_nil + end + end + end end diff --git a/spec/lib/gitlab/github_import/importer/attachments/issues_importer_spec.rb b/spec/lib/gitlab/github_import/importer/attachments/issues_importer_spec.rb index 85bc67376d3..7890561bf2d 100644 --- a/spec/lib/gitlab/github_import/importer/attachments/issues_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/attachments/issues_importer_spec.rb @@ -17,6 +17,8 @@ RSpec.describe Gitlab::GithubImport::Importer::Attachments::IssuesImporter do let(:importer_attrs) { [instance_of(Gitlab::GithubImport::Representation::NoteText), project, client] } it 'imports each project issue attachments' do + expect(project.issues).to receive(:select).with(:id, :description, :iid).and_call_original + expect_next_instances_of( Gitlab::GithubImport::Importer::NoteAttachmentsImporter, 2, false, *importer_attrs ) do |note_attachments_importer| diff --git a/spec/lib/gitlab/github_import/importer/attachments/merge_requests_importer_spec.rb b/spec/lib/gitlab/github_import/importer/attachments/merge_requests_importer_spec.rb index e4718c2d17c..e5aa17dd81e 100644 --- a/spec/lib/gitlab/github_import/importer/attachments/merge_requests_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/attachments/merge_requests_importer_spec.rb @@ -17,6 +17,8 @@ RSpec.describe Gitlab::GithubImport::Importer::Attachments::MergeRequestsImporte let(:importer_attrs) { [instance_of(Gitlab::GithubImport::Representation::NoteText), project, client] } it 'imports each project merge request attachments' do + expect(project.merge_requests).to receive(:select).with(:id, :description, :iid).and_call_original + expect_next_instances_of( Gitlab::GithubImport::Importer::NoteAttachmentsImporter, 2, false, *importer_attrs ) do |note_attachments_importer| diff --git a/spec/lib/gitlab/github_import/importer/attachments/releases_importer_spec.rb b/spec/lib/gitlab/github_import/importer/attachments/releases_importer_spec.rb index b989345ae09..e1b009c3eeb 100644 --- a/spec/lib/gitlab/github_import/importer/attachments/releases_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/attachments/releases_importer_spec.rb @@ -17,6 +17,8 @@ RSpec.describe Gitlab::GithubImport::Importer::Attachments::ReleasesImporter do let(:importer_attrs) { [instance_of(Gitlab::GithubImport::Representation::NoteText), project, client] } it 'imports each project release' do + expect(project.releases).to receive(:select).with(:id, :description, :tag).and_call_original + expect(Gitlab::GithubImport::Importer::NoteAttachmentsImporter).to receive(:new) .with(*importer_attrs).twice.and_return(importer_stub) expect(importer_stub).to receive(:execute).twice diff --git a/spec/lib/gitlab/github_import/importer/collaborator_importer_spec.rb b/spec/lib/gitlab/github_import/importer/collaborator_importer_spec.rb new file mode 100644 index 00000000000..07c10fe57f0 --- /dev/null +++ b/spec/lib/gitlab/github_import/importer/collaborator_importer_spec.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::GithubImport::Importer::CollaboratorImporter, feature_category: :importers do + subject(:importer) { described_class.new(collaborator, project, client) } + + let_it_be(:group) { create(:group) } + let_it_be(:project) { create(:project, :repository, group: group) } + let_it_be(:user) { create(:user) } + + let(:client) { instance_double(Gitlab::GithubImport::Client) } + let(:github_user_id) { rand(1000) } + let(:collaborator) do + Gitlab::GithubImport::Representation::Collaborator.from_json_hash( + 'id' => github_user_id, + 'login' => user.username, + 'role_name' => github_role_name + ) + end + + let(:basic_member_attrs) do + { + source: project, + user_id: user.id, + member_namespace_id: project.project_namespace_id, + created_by_id: project.creator_id + }.stringify_keys + end + + describe '#execute' do + before do + allow_next_instance_of(Gitlab::GithubImport::UserFinder) do |finder| + allow(finder).to receive(:find).with(github_user_id, user.username).and_return(user.id) + end + end + + shared_examples 'role mapping' do |collaborator_role, member_access_level| + let(:github_role_name) { collaborator_role } + + it 'creates expected member' do + expect { importer.execute }.to change { project.members.count } + .from(0).to(1) + + expected_member_attrs = basic_member_attrs.merge(access_level: member_access_level) + expect(project.members.last).to have_attributes(expected_member_attrs) + end + end + + it_behaves_like 'role mapping', 'read', Gitlab::Access::GUEST + it_behaves_like 'role mapping', 'triage', Gitlab::Access::REPORTER + it_behaves_like 'role mapping', 'write', Gitlab::Access::DEVELOPER + it_behaves_like 'role mapping', 'maintain', Gitlab::Access::MAINTAINER + it_behaves_like 'role mapping', 'admin', Gitlab::Access::OWNER + + context 'when role name is unknown (custom role)' do + let(:github_role_name) { 'custom_role' } + + it 'raises expected error' do + expect { importer.execute }.to raise_exception( + ::Gitlab::GithubImport::ObjectImporter::NotRetriableError + ).with_message("Unknown GitHub role: #{github_role_name}") + end + end + + context 'when user has lower role in a project group' do + before do + create(:group_member, group: group, user: user, access_level: Gitlab::Access::DEVELOPER) + end + + it_behaves_like 'role mapping', 'maintain', Gitlab::Access::MAINTAINER + end + + context 'when user has higher role in a project group' do + let(:github_role_name) { 'write' } + + before do + create(:group_member, group: group, user: user, access_level: Gitlab::Access::MAINTAINER) + end + + it 'skips creating member for the project' do + expect { importer.execute }.not_to change { project.members.count } + end + end + end +end diff --git a/spec/lib/gitlab/github_import/importer/collaborators_importer_spec.rb b/spec/lib/gitlab/github_import/importer/collaborators_importer_spec.rb new file mode 100644 index 00000000000..dcb02f32a28 --- /dev/null +++ b/spec/lib/gitlab/github_import/importer/collaborators_importer_spec.rb @@ -0,0 +1,156 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::GithubImport::Importer::CollaboratorsImporter, feature_category: :importers do + subject(:importer) { described_class.new(project, client, parallel: parallel) } + + let(:parallel) { true } + let(:project) { instance_double(Project, id: 4, import_source: 'foo/bar', import_state: nil) } + let(:client) { instance_double(Gitlab::GithubImport::Client) } + + let(:github_collaborator) do + { + id: 100500, + login: 'bob', + role_name: 'maintainer' + } + end + + describe '#parallel?' do + context 'when parallel option is true' do + it { expect(importer).to be_parallel } + end + + context 'when parallel option is false' do + let(:parallel) { false } + + it { expect(importer).not_to be_parallel } + end + end + + describe '#execute' do + context 'when running in parallel mode' do + it 'imports collaborators in parallel' do + expect(importer).to receive(:parallel_import) + importer.execute + end + end + + context 'when running in sequential mode' do + let(:parallel) { false } + + it 'imports collaborators in sequence' do + expect(importer).to receive(:sequential_import) + importer.execute + end + end + end + + describe '#sequential_import' do + let(:parallel) { false } + + it 'imports each collaborator in sequence' do + collaborator_importer = instance_double(Gitlab::GithubImport::Importer::CollaboratorImporter) + + allow(importer) + .to receive(:each_object_to_import) + .and_yield(github_collaborator) + + expect(Gitlab::GithubImport::Importer::CollaboratorImporter) + .to receive(:new) + .with( + an_instance_of(Gitlab::GithubImport::Representation::Collaborator), + project, + client + ) + .and_return(collaborator_importer) + + expect(collaborator_importer).to receive(:execute) + + importer.sequential_import + end + end + + describe '#parallel_import', :clean_gitlab_redis_cache do + before do + allow(client).to receive(:collaborators).with(project.import_source, affiliation: 'direct') + .and_return([github_collaborator]) + allow(client).to receive(:collaborators).with(project.import_source, affiliation: 'outside') + .and_return([]) + end + + it 'imports each collaborator in parallel' do + expect(Gitlab::GithubImport::ImportCollaboratorWorker).to receive(:perform_in) + .with(1.second, project.id, an_instance_of(Hash), an_instance_of(String)) + + waiter = importer.parallel_import + + expect(waiter).to be_an_instance_of(Gitlab::JobWaiter) + expect(waiter.jobs_remaining).to eq(1) + end + + context 'when collaborator is already imported' do + before do + Gitlab::Cache::Import::Caching.set_add( + "github-importer/already-imported/#{project.id}/collaborators", + github_collaborator[:id] + ) + end + + it "doesn't run importer on it" do + expect(Gitlab::GithubImport::ImportCollaboratorWorker).not_to receive(:perform_in) + + waiter = importer.parallel_import + + expect(waiter).to be_an_instance_of(Gitlab::JobWaiter) + expect(waiter.jobs_remaining).to eq(0) + end + end + end + + describe '#each_object_to_import', :clean_gitlab_redis_cache do + let(:github_collaborator_2) { { id: 100501, login: 'alice', role_name: 'owner' } } + let(:github_collaborator_3) { { id: 100502, login: 'tom', role_name: 'guest' } } + + before do + allow(client).to receive(:collaborators).with(project.import_source, affiliation: 'direct') + .and_return([github_collaborator, github_collaborator_2, github_collaborator_3]) + allow(client).to receive(:collaborators).with(project.import_source, affiliation: 'outside') + .and_return([github_collaborator_3]) + allow(Gitlab::GithubImport::ObjectCounter).to receive(:increment) + .with(project, :collaborator, :fetched) + end + + it 'yields every direct collaborator who is not an outside collaborator to the supplied block' do + expect { |b| importer.each_object_to_import(&b) } + .to yield_successive_args(github_collaborator, github_collaborator_2) + + expect(Gitlab::GithubImport::ObjectCounter).to have_received(:increment).twice + end + + context 'when a collaborator has been already imported' do + before do + allow(importer).to receive(:already_imported?).and_return(true) + end + + it 'does not yield anything' do + expect(Gitlab::GithubImport::ObjectCounter) + .not_to receive(:increment) + + expect(importer) + .not_to receive(:mark_as_imported) + + expect { |b| importer.each_object_to_import(&b) } + .not_to yield_control + end + end + end + + describe '#id_for_already_imported_cache' do + it 'returns the ID of the given note' do + expect(importer.id_for_already_imported_cache(github_collaborator)) + .to eq(100500) + end + end +end diff --git a/spec/lib/gitlab/github_import/importer/label_links_importer_spec.rb b/spec/lib/gitlab/github_import/importer/label_links_importer_spec.rb index e005d8eda84..16816dfbcea 100644 --- a/spec/lib/gitlab/github_import/importer/label_links_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/label_links_importer_spec.rb @@ -44,6 +44,10 @@ RSpec.describe Gitlab::GithubImport::Importer::LabelLinksImporter do end it 'does not insert label links for non-existing labels' do + expect(importer) + .to receive(:find_target_id) + .and_return(4) + expect(importer.label_finder) .to receive(:id_for) .with('bug') @@ -55,6 +59,20 @@ RSpec.describe Gitlab::GithubImport::Importer::LabelLinksImporter do importer.create_labels end + + it 'does not insert label links for non-existing targets' do + expect(importer) + .to receive(:find_target_id) + .and_return(nil) + + expect(importer.label_finder) + .not_to receive(:id_for) + + expect(LabelLink) + .not_to receive(:bulk_insert!) + + importer.create_labels + end end describe '#find_target_id' do diff --git a/spec/lib/gitlab/github_import/importer/labels_importer_spec.rb b/spec/lib/gitlab/github_import/importer/labels_importer_spec.rb index 9e295ab215a..fc8d9cee066 100644 --- a/spec/lib/gitlab/github_import/importer/labels_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/labels_importer_spec.rb @@ -56,14 +56,14 @@ feature_category: :importers do project_id: project.id, importer: described_class.name, message: ['Title is invalid'], - github_identifier: 1 + github_identifiers: { title: 'bug,bug', object_type: :label } ) rows, errors = importer.build_labels expect(rows).to be_empty expect(errors.length).to eq(1) - expect(errors[0].full_messages).to match_array(['Title is invalid']) + expect(errors[0][:validation_errors].full_messages).to match_array(['Title is invalid']) end end diff --git a/spec/lib/gitlab/github_import/importer/milestones_importer_spec.rb b/spec/lib/gitlab/github_import/importer/milestones_importer_spec.rb index 47b9a41c364..cf44d510c80 100644 --- a/spec/lib/gitlab/github_import/importer/milestones_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/milestones_importer_spec.rb @@ -74,7 +74,7 @@ RSpec.describe Gitlab::GithubImport::Importer::MilestonesImporter, :clean_gitlab end it 'does not build milestones that are invalid' do - milestone = { id: 1, title: nil } + milestone = { id: 123456, title: nil, number: 2 } expect(importer) .to receive(:each_milestone) @@ -86,14 +86,14 @@ RSpec.describe Gitlab::GithubImport::Importer::MilestonesImporter, :clean_gitlab project_id: project.id, importer: described_class.name, message: ["Title can't be blank"], - github_identifier: 1 + github_identifiers: { iid: 2, object_type: :milestone, title: nil } ) rows, errors = importer.build_milestones expect(rows).to be_empty expect(errors.length).to eq(1) - expect(errors[0].full_messages).to match_array(["Title can't be blank"]) + expect(errors[0][:validation_errors].full_messages).to match_array(["Title can't be blank"]) end end diff --git a/spec/lib/gitlab/github_import/importer/note_attachments_importer_spec.rb b/spec/lib/gitlab/github_import/importer/note_attachments_importer_spec.rb index 7d4e3c3bcce..450ebe9a719 100644 --- a/spec/lib/gitlab/github_import/importer/note_attachments_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/note_attachments_importer_spec.rb @@ -2,10 +2,10 @@ require 'spec_helper' -RSpec.describe Gitlab::GithubImport::Importer::NoteAttachmentsImporter do +RSpec.describe Gitlab::GithubImport::Importer::NoteAttachmentsImporter, feature_category: :importers do subject(:importer) { described_class.new(note_text, project, client) } - let_it_be(:project) { create(:project) } + let_it_be(:project) { create(:project, import_source: 'nickname/public-test-repo') } let(:note_text) { Gitlab::GithubImport::Representation::NoteText.from_db_record(record) } let(:client) { instance_double('Gitlab::GithubImport::Client') } @@ -13,6 +13,8 @@ RSpec.describe Gitlab::GithubImport::Importer::NoteAttachmentsImporter do let(:doc_url) { 'https://github.com/nickname/public-test-repo/files/9020437/git-cheat-sheet.txt' } let(:image_url) { 'https://user-images.githubusercontent.com/6833842/0cf366b61ef2.jpeg' } let(:image_tag_url) { 'https://user-images.githubusercontent.com/6833842/0cf366b61ea5.jpeg' } + let(:project_blob_url) { 'https://github.com/nickname/public-test-repo/blob/main/example.md' } + let(:other_project_blob_url) { 'https://github.com/nickname/other-repo/blob/main/README.md' } let(:text) do <<-TEXT.split("\n").map(&:strip).join("\n") Some text... @@ -20,11 +22,14 @@ RSpec.describe Gitlab::GithubImport::Importer::NoteAttachmentsImporter do [special-doc](#{doc_url}) ![image.jpeg](#{image_url}) <img width=\"248\" alt=\"tag-image\" src="#{image_tag_url}"> + + [link to project blob file](#{project_blob_url}) + [link to other project blob file](#{other_project_blob_url}) TEXT end shared_examples 'updates record description' do - it do + it 'changes attachment links' do importer.execute record.reload @@ -32,6 +37,22 @@ RSpec.describe Gitlab::GithubImport::Importer::NoteAttachmentsImporter do expect(record.description).to include('![image.jpeg](/uploads/') expect(record.description).to include('<img width="248" alt="tag-image" src="/uploads') end + + it 'changes link to project blob files' do + importer.execute + + record.reload + expected_blob_link = "[link to project blob file](http://localhost/#{project.full_path}/-/blob/main/example.md)" + expect(record.description).not_to include("[link to project blob file](#{project_blob_url})") + expect(record.description).to include(expected_blob_link) + end + + it "doesn't change links to other projects" do + importer.execute + + record.reload + expect(record.description).to include("[link to other project blob file](#{other_project_blob_url})") + end end describe '#execute' do @@ -72,7 +93,7 @@ RSpec.describe Gitlab::GithubImport::Importer::NoteAttachmentsImporter do context 'when importing note attachments' do let(:record) { create(:note, project: project, note: text) } - it 'updates note text with new attachment urls' do + it 'changes note text with new attachment urls' do importer.execute record.reload @@ -80,6 +101,22 @@ RSpec.describe Gitlab::GithubImport::Importer::NoteAttachmentsImporter do expect(record.note).to include('![image.jpeg](/uploads/') expect(record.note).to include('<img width="248" alt="tag-image" src="/uploads') end + + it 'changes note links to project blob files' do + importer.execute + + record.reload + expected_blob_link = "[link to project blob file](http://localhost/#{project.full_path}/-/blob/main/example.md)" + expect(record.note).not_to include("[link to project blob file](#{project_blob_url})") + expect(record.note).to include(expected_blob_link) + end + + it "doesn't change note links to other projects" do + importer.execute + + record.reload + expect(record.note).to include("[link to other project blob file](#{other_project_blob_url})") + end end end end diff --git a/spec/lib/gitlab/github_import/importer/pull_requests_merged_by_importer_spec.rb b/spec/lib/gitlab/github_import/importer/pull_requests/all_merged_by_importer_spec.rb index b6c162aafa9..8e13b35eb6b 100644 --- a/spec/lib/gitlab/github_import/importer/pull_requests_merged_by_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/pull_requests/all_merged_by_importer_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::GithubImport::Importer::PullRequestsMergedByImporter do +RSpec.describe Gitlab::GithubImport::Importer::PullRequests::AllMergedByImporter, feature_category: :importers do let(:client) { double } let_it_be(:project) { create(:project, import_source: 'http://somegithub.com') } @@ -16,7 +16,11 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestsMergedByImporter do end describe '#importer_class' do - it { expect(subject.importer_class).to eq(Gitlab::GithubImport::Importer::PullRequestMergedByImporter) } + it { expect(subject.importer_class).to eq(Gitlab::GithubImport::Importer::PullRequests::MergedByImporter) } + end + + describe '#sidekiq_worker_class' do + it { expect(subject.sidekiq_worker_class).to eq(Gitlab::GithubImport::PullRequests::ImportMergedByWorker) } end describe '#collection_method' do @@ -24,7 +28,7 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestsMergedByImporter do end describe '#id_for_already_imported_cache' do - it { expect(subject.id_for_already_imported_cache(double(id: 1))).to eq(1) } + it { expect(subject.id_for_already_imported_cache(instance_double(MergeRequest, id: 1))).to eq(1) } end describe '#each_object_to_import', :clean_gitlab_redis_cache do @@ -44,7 +48,7 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestsMergedByImporter do expect { |b| subject.each_object_to_import(&b) } .to yield_with_args(pull_request) - subject.each_object_to_import {} + subject.each_object_to_import end it 'skips cached merge requests' do @@ -55,7 +59,7 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestsMergedByImporter do expect(client).not_to receive(:pull_request) - subject.each_object_to_import {} + subject.each_object_to_import end end end diff --git a/spec/lib/gitlab/github_import/importer/pull_request_merged_by_importer_spec.rb b/spec/lib/gitlab/github_import/importer/pull_requests/merged_by_importer_spec.rb index 01d706beea2..25381594632 100644 --- a/spec/lib/gitlab/github_import/importer/pull_request_merged_by_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/pull_requests/merged_by_importer_spec.rb @@ -2,12 +2,16 @@ require 'spec_helper' -RSpec.describe Gitlab::GithubImport::Importer::PullRequestMergedByImporter, :clean_gitlab_redis_cache do +RSpec.describe Gitlab::GithubImport::Importer::PullRequests::MergedByImporter, + :clean_gitlab_redis_cache, feature_category: :importers do let_it_be(:merge_request) { create(:merged_merge_request) } let(:project) { merge_request.project } - let(:merged_at) { Time.new(2017, 1, 1, 12, 00).utc } - let(:client_double) { double(user: { id: 999, login: 'merger', email: 'merger@email.com' } ) } + let(:merged_at) { Time.utc(2017, 1, 1, 12) } + let(:client_double) do + instance_double(Gitlab::GithubImport::Client, user: { id: 999, login: 'merger', email: 'merger@email.com' }) + end + let(:merger_user) { { id: 999, login: 'merger' } } let(:pull_request) do @@ -25,7 +29,7 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestMergedByImporter, :cle shared_examples 'adds a note referencing the merger user' do it 'adds a note referencing the merger user' do expect { subject.execute } - .to change(Note, :count).by(1) + .to change { Note.count }.by(1) .and not_change(merge_request, :updated_at) metrics = merge_request.metrics.reload @@ -68,7 +72,7 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestMergedByImporter, :cle it 'adds a note referencing the merger user' do expect { subject.execute } - .to change(Note, :count).by(1) + .to change { Note.count }.by(1) .and not_change(merge_request, :updated_at) metrics = merge_request.metrics.reload diff --git a/spec/lib/gitlab/github_import/importer/pull_request_review_importer_spec.rb b/spec/lib/gitlab/github_import/importer/pull_requests/review_importer_spec.rb index 3e62e8f473c..ba14ea603e0 100644 --- a/spec/lib/gitlab/github_import/importer/pull_request_review_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/pull_requests/review_importer_spec.rb @@ -2,14 +2,14 @@ require 'spec_helper' -RSpec.describe Gitlab::GithubImport::Importer::PullRequestReviewImporter, - :clean_gitlab_redis_cache, feature_category: :importers do +RSpec.describe Gitlab::GithubImport::Importer::PullRequests::ReviewImporter, + :clean_gitlab_redis_cache, feature_category: :importers do using RSpec::Parameterized::TableSyntax let_it_be(:merge_request) { create(:merge_request) } let(:project) { merge_request.project } - let(:submitted_at) { Time.new(2017, 1, 1, 12, 00).utc } + let(:submitted_at) { Time.new(2017, 1, 1, 12).utc } let(:client_double) do instance_double( 'Gitlab::GithubImport::Client', @@ -21,7 +21,7 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestReviewImporter, shared_examples 'imports a reviewer for the Merge Request' do it 'creates reviewer for the Merge Request' do - expect { subject.execute }.to change(MergeRequestReviewer, :count).by(1) + expect { subject.execute }.to change { MergeRequestReviewer.count }.by(1) expect(merge_request.reviewers).to contain_exactly(author) end @@ -35,7 +35,7 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestReviewImporter, end it 'does not change Merge Request reviewers' do - expect { subject.execute }.not_to change(MergeRequestReviewer, :count) + expect { subject.execute }.not_to change { MergeRequestReviewer.count } expect(merge_request.reviewers).to contain_exactly(author) end @@ -48,7 +48,7 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestReviewImporter, end it 'does not change Merge Request reviewers', :aggregate_failures do - expect { subject.execute }.not_to change(MergeRequestReviewer, :count) + expect { subject.execute }.not_to change { MergeRequestReviewer.count } expect(merge_request.reviewers).to contain_exactly(author) end @@ -57,7 +57,7 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestReviewImporter, shared_examples 'imports an approval for the Merge Request' do it 'creates an approval for the Merge Request' do - expect { subject.execute }.to change(Approval, :count).by(1) + expect { subject.execute }.to change { Approval.count }.by(1) expect(merge_request.approved_by_users.reload).to include(author) expect(merge_request.approvals.last.created_at).to eq(submitted_at) @@ -75,7 +75,7 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestReviewImporter, it_behaves_like 'imports a reviewer for the Merge Request' it 'creates a note for the review' do - expect { subject.execute }.to change(Note, :count).by(1) + expect { subject.execute }.to change { Note.count }.by(1) last_note = merge_request.notes.last expect(last_note.note).to eq('approved this merge request') @@ -91,8 +91,8 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestReviewImporter, it 'does not import second approve and note' do expect { subject.execute } - .to change(Note, :count).by(0) - .and change(Approval, :count).by(0) + .to change { Note.count }.by(0) + .and change { Approval.count }.by(0) end end end @@ -103,7 +103,7 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestReviewImporter, it_behaves_like 'imports a reviewer for the Merge Request' it 'does not create note for the review' do - expect { subject.execute }.not_to change(Note, :count) + expect { subject.execute }.not_to change { Note.count } end end @@ -113,7 +113,7 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestReviewImporter, it_behaves_like 'imports a reviewer for the Merge Request' it 'does not create a note for the review' do - expect { subject.execute }.not_to change(Note, :count) + expect { subject.execute }.not_to change { Note.count } end end end @@ -126,7 +126,7 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestReviewImporter, it_behaves_like 'imports a reviewer for the Merge Request' it 'creates a note for the review' do - expect { subject.execute }.to change(Note, :count).by(2) + expect { subject.execute }.to change { Note.count }.by(2) note = merge_request.notes.where(system: false).last expect(note.note).to eq("**Review:** Approved\n\nnote") @@ -146,7 +146,7 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestReviewImporter, it 'creates a note for the review' do expect { subject.execute } - .to change(Note, :count).by(1) + .to change { Note.count }.by(1) .and not_change(Approval, :count) last_note = merge_request.notes.last @@ -162,7 +162,7 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestReviewImporter, it 'creates a note for the review' do expect { subject.execute } - .to change(Note, :count).by(1) + .to change { Note.count }.by(1) .and not_change(Approval, :count) last_note = merge_request.notes.last @@ -182,7 +182,7 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestReviewImporter, it 'creates a note for the review with *Approved by by<author>*' do expect { subject.execute } - .to change(Note, :count).by(1) + .to change { Note.count }.by(1) last_note = merge_request.notes.last expect(last_note.note).to eq("*Created by: author*\n\n**Review:** Approved") @@ -195,7 +195,7 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestReviewImporter, let(:review) { create_review(type: 'COMMENTED', note: '') } it 'creates a note for the review with *Commented by<author>*' do - expect { subject.execute }.not_to change(Note, :count) + expect { subject.execute }.not_to change { Note.count } end end @@ -203,7 +203,7 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestReviewImporter, let(:review) { create_review(type: 'CHANGES_REQUESTED', note: '') } it 'creates a note for the review with *Changes requested by <author>*' do - expect { subject.execute }.not_to change(Note, :count) + expect { subject.execute }.not_to change { Note.count } end end end @@ -213,7 +213,7 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestReviewImporter, it 'creates a note for the review without the author information' do expect { subject.execute } - .to change(Note, :count).by(1) + .to change { Note.count }.by(1) last_note = merge_request.notes.last expect(last_note.note).to eq('**Review:** Approved') @@ -231,7 +231,7 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestReviewImporter, it 'creates a note for the review with the author username' do expect { subject.execute } - .to change(Note, :count).by(1) + .to change { Note.count }.by(1) last_note = merge_request.notes.last expect(last_note.note).to eq("*Created by: author*\n\n**Review:** Approved") expect(last_note.author).to eq(project.creator) @@ -243,7 +243,7 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestReviewImporter, let(:review) { create_review(type: 'APPROVED', note: '', submitted_at: nil) } it 'creates a note for the review without the author information' do - expect { subject.execute }.to change(Note, :count).by(1) + expect { subject.execute }.to change { Note.count }.by(1) last_note = merge_request.notes.last @@ -258,7 +258,7 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestReviewImporter, it 'creates a note for the review with *Approved by by<author>*' do expect { subject.execute } - .to change(Note, :count).by(1) + .to change { Note.count }.by(1) last_note = merge_request.notes.last @@ -273,7 +273,7 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestReviewImporter, it 'creates a note for the review with *Commented by<author>*' do expect { subject.execute } - .to change(Note, :count).by(1) + .to change { Note.count }.by(1) last_note = merge_request.notes.last @@ -288,7 +288,7 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestReviewImporter, it 'creates a note for the review with *Changes requested by <author>*' do expect { subject.execute } - .to change(Note, :count).by(1) + .to change { Note.count }.by(1) last_note = merge_request.notes.last diff --git a/spec/lib/gitlab/github_import/importer/pull_requests/review_requests_importer_spec.rb b/spec/lib/gitlab/github_import/importer/pull_requests/review_requests_importer_spec.rb index 536983fea06..9e9d6c6e9cd 100644 --- a/spec/lib/gitlab/github_import/importer/pull_requests/review_requests_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/pull_requests/review_requests_importer_spec.rb @@ -86,6 +86,7 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequests::ReviewRequestsImpor project.id, { merge_request_id: merge_request_1.id, + merge_request_iid: merge_request_1.iid, users: [ { id: 4, login: 'alice' }, { id: 5, login: 'bob' } @@ -97,6 +98,7 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequests::ReviewRequestsImpor project.id, { merge_request_id: merge_request_2.id, + merge_request_iid: merge_request_2.iid, users: [ { id: 4, login: 'alice' } ] diff --git a/spec/lib/gitlab/github_import/importer/pull_requests_reviews_importer_spec.rb b/spec/lib/gitlab/github_import/importer/pull_requests/reviews_importer_spec.rb index 5f9c73cbfff..4321997815a 100644 --- a/spec/lib/gitlab/github_import/importer/pull_requests_reviews_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/pull_requests/reviews_importer_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::GithubImport::Importer::PullRequestsReviewsImporter do +RSpec.describe Gitlab::GithubImport::Importer::PullRequests::ReviewsImporter, feature_category: :importers do let(:client) { double } let(:project) { create(:project, import_source: 'github/repo') } @@ -15,13 +15,21 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestsReviewsImporter do end describe '#importer_class' do - it { expect(subject.importer_class).to eq(Gitlab::GithubImport::Importer::PullRequestReviewImporter) } + it { expect(subject.importer_class).to eq(Gitlab::GithubImport::Importer::PullRequests::ReviewImporter) } + end + + describe '#sidekiq_worker_class' do + it { expect(subject.sidekiq_worker_class).to eq(Gitlab::GithubImport::PullRequests::ImportReviewWorker) } end describe '#collection_method' do it { expect(subject.collection_method).to eq(:pull_request_reviews) } end + describe '#object_type' do + it { expect(subject.object_type).to eq(:pull_request_review) } + end + describe '#id_for_already_imported_cache' do it { expect(subject.id_for_already_imported_cache({ id: 1 })).to eq(1) } end @@ -39,7 +47,7 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestsReviewsImporter do let(:review) { { id: 1 } } it 'fetches the pull requests reviews data' do - page = double(objects: [review], number: 1) + page = Struct.new(:objects, :number).new([review], 1) expect(client) .to receive(:each_page) @@ -50,9 +58,10 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestsReviewsImporter do expect { |b| subject.each_object_to_import(&b) } .to yield_with_args(review) - subject.each_object_to_import {} + subject.each_object_to_import expect(review[:merge_request_id]).to eq(merge_request.id) + expect(review[:merge_request_iid]).to eq(merge_request.iid) end it 'skips cached pages' do @@ -67,7 +76,7 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestsReviewsImporter do .exactly(:once) # ensure to be cached on the second call .with(:pull_request_reviews, 'github/repo', merge_request.iid, { page: 2 }) - subject.each_object_to_import {} + subject.each_object_to_import end it 'skips cached merge requests' do @@ -80,7 +89,7 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestsReviewsImporter do expect(client).not_to receive(:each_page) - subject.each_object_to_import {} + subject.each_object_to_import end end end diff --git a/spec/lib/gitlab/github_import/importer/releases_importer_spec.rb b/spec/lib/gitlab/github_import/importer/releases_importer_spec.rb index fe4d3e9d90b..a3d20af22c7 100644 --- a/spec/lib/gitlab/github_import/importer/releases_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/releases_importer_spec.rb @@ -18,6 +18,7 @@ RSpec.describe Gitlab::GithubImport::Importer::ReleasesImporter, feature_categor let(:github_release) do { + id: 123456, tag_name: '1.0', name: github_release_name, body: 'This is my release', @@ -144,7 +145,10 @@ RSpec.describe Gitlab::GithubImport::Importer::ReleasesImporter, feature_categor expect(releases).to be_empty expect(errors.length).to eq(1) - expect(errors[0].full_messages).to match_array(['Description is too long (maximum is 1000000 characters)']) + expect(errors[0][:validation_errors].full_messages).to match_array( + ['Description is too long (maximum is 1000000 characters)'] + ) + expect(errors[0][:github_identifiers]).to eq({ tag: '1.0', object_type: :release }) end end diff --git a/spec/lib/gitlab/github_import/logger_spec.rb b/spec/lib/gitlab/github_import/logger_spec.rb index 6fd0f5db93e..97806872746 100644 --- a/spec/lib/gitlab/github_import/logger_spec.rb +++ b/spec/lib/gitlab/github_import/logger_spec.rb @@ -5,37 +5,5 @@ require 'spec_helper' RSpec.describe Gitlab::GithubImport::Logger do subject(:logger) { described_class.new('/dev/null') } - let(:now) { Time.zone.now } - - describe '#format_message' do - before do - allow(Labkit::Correlation::CorrelationId).to receive(:current_id).and_return('new-correlation-id') - end - - it 'formats strings' do - output = subject.format_message('INFO', now, 'test', 'Hello world') - - expect(Gitlab::Json.parse(output)).to eq({ - 'severity' => 'INFO', - 'time' => now.utc.iso8601(3), - 'message' => 'Hello world', - 'correlation_id' => 'new-correlation-id', - 'feature_category' => 'importers', - 'import_type' => 'github' - }) - end - - it 'formats hashes' do - output = subject.format_message('INFO', now, 'test', { hello: 1 }) - - expect(Gitlab::Json.parse(output)).to eq({ - 'severity' => 'INFO', - 'time' => now.utc.iso8601(3), - 'hello' => 1, - 'correlation_id' => 'new-correlation-id', - 'feature_category' => 'importers', - 'import_type' => 'github' - }) - end - end + it_behaves_like 'a json logger', { 'feature_category' => 'importers', 'import_type' => 'github' } end diff --git a/spec/lib/gitlab/github_import/markdown/attachment_spec.rb b/spec/lib/gitlab/github_import/markdown/attachment_spec.rb index 588a3076f59..84b0886ebcc 100644 --- a/spec/lib/gitlab/github_import/markdown/attachment_spec.rb +++ b/spec/lib/gitlab/github_import/markdown/attachment_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::GithubImport::Markdown::Attachment do +RSpec.describe Gitlab::GithubImport::Markdown::Attachment, feature_category: :importers do let(:name) { FFaker::Lorem.word } let(:url) { FFaker::Internet.uri('https') } @@ -101,6 +101,62 @@ RSpec.describe Gitlab::GithubImport::Markdown::Attachment do end end + describe '#part_of_project_blob?' do + let(:attachment) { described_class.new('test', url) } + let(:import_source) { 'nickname/public-test-repo' } + + context 'when url is a part of project blob' do + let(:url) { "https://github.com/#{import_source}/blob/main/example.md" } + + it { expect(attachment.part_of_project_blob?(import_source)).to eq true } + end + + context 'when url is not a part of project blob' do + let(:url) { "https://github.com/#{import_source}/files/9020437/git-cheat-sheet.txt" } + + it { expect(attachment.part_of_project_blob?(import_source)).to eq false } + end + end + + describe '#doc_belongs_to_project?' do + let(:attachment) { described_class.new('test', url) } + let(:import_source) { 'nickname/public-test-repo' } + + context 'when url relates to this project' do + let(:url) { "https://github.com/#{import_source}/files/9020437/git-cheat-sheet.txt" } + + it { expect(attachment.doc_belongs_to_project?(import_source)).to eq true } + end + + context 'when url is not related to this project' do + let(:url) { 'https://github.com/nickname/other-repo/files/9020437/git-cheat-sheet.txt' } + + it { expect(attachment.doc_belongs_to_project?(import_source)).to eq false } + end + + context 'when url is a part of project blob' do + let(:url) { "https://github.com/#{import_source}/blob/main/example.md" } + + it { expect(attachment.doc_belongs_to_project?(import_source)).to eq false } + end + end + + describe '#media?' do + let(:attachment) { described_class.new('test', url) } + + context 'when it is a media link' do + let(:url) { 'https://user-images.githubusercontent.com/6833842/0cf366b61ef2.jpeg' } + + it { expect(attachment.media?).to eq true } + end + + context 'when it is not a media link' do + let(:url) { 'https://github.com/nickname/public-test-repo/files/9020437/git-cheat-sheet.txt' } + + it { expect(attachment.media?).to eq false } + end + end + describe '#inspect' do it 'returns attachment basic info' do attachment = described_class.new(name, url) diff --git a/spec/lib/gitlab/github_import/parallel_scheduling_spec.rb b/spec/lib/gitlab/github_import/parallel_scheduling_spec.rb index c351ead91eb..9de39a3ff7e 100644 --- a/spec/lib/gitlab/github_import/parallel_scheduling_spec.rb +++ b/spec/lib/gitlab/github_import/parallel_scheduling_spec.rb @@ -289,77 +289,52 @@ RSpec.describe Gitlab::GithubImport::ParallelScheduling, feature_category: :impo .and_return({ title: 'One' }, { title: 'Two' }, { title: 'Three' }) end - context 'with multiple objects' do - before do - stub_feature_flags(improved_spread_parallel_import: false) - - expect(importer).to receive(:each_object_to_import).and_yield(object).and_yield(object).and_yield(object) - end - - it 'imports data in parallel batches with delays' do - expect(worker_class).to receive(:bulk_perform_in) - .with(1.second, [ - [project.id, { title: 'One' }, an_instance_of(String)], - [project.id, { title: 'Two' }, an_instance_of(String)], - [project.id, { title: 'Three' }, an_instance_of(String)] - ], batch_size: batch_size, batch_delay: batch_delay) - - importer.parallel_import - end + it 'imports data in parallel with delays respecting parallel_import_batch definition and return job waiter' do + allow(::Gitlab::JobWaiter).to receive(:generate_key).and_return('waiter-key') + allow(importer).to receive(:parallel_import_batch).and_return({ size: 2, delay: 1.minute }) + + expect(importer).to receive(:each_object_to_import) + .and_yield(object).and_yield(object).and_yield(object) + expect(worker_class).to receive(:perform_in) + .with(1.second, project.id, { title: 'One' }, 'waiter-key').ordered + expect(worker_class).to receive(:perform_in) + .with(1.second, project.id, { title: 'Two' }, 'waiter-key').ordered + expect(worker_class).to receive(:perform_in) + .with(1.minute + 1.second, project.id, { title: 'Three' }, 'waiter-key').ordered + + job_waiter = importer.parallel_import + + expect(job_waiter.key).to eq('waiter-key') + expect(job_waiter.jobs_remaining).to eq(3) end - context 'when the feature flag `improved_spread_parallel_import` is enabled' do + context 'when job restarts due to API rate limit or Sidekiq interruption' do before do - stub_feature_flags(improved_spread_parallel_import: true) + cache_key = format(described_class::JOB_WAITER_CACHE_KEY, + project: project.id, collection: importer.collection_method) + Gitlab::Cache::Import::Caching.write(cache_key, 'waiter-key') + + cache_key = format(described_class::JOB_WAITER_REMAINING_CACHE_KEY, + project: project.id, collection: importer.collection_method) + Gitlab::Cache::Import::Caching.write(cache_key, 3) end - it 'imports data in parallel with delays respecting parallel_import_batch definition and return job waiter' do - allow(::Gitlab::JobWaiter).to receive(:generate_key).and_return('waiter-key') - allow(importer).to receive(:parallel_import_batch).and_return({ size: 2, delay: 1.minute }) + it "restores job waiter's key and jobs_remaining" do + allow(importer).to receive(:parallel_import_batch).and_return({ size: 1, delay: 1.minute }) + + expect(importer).to receive(:each_object_to_import).and_yield(object).and_yield(object).and_yield(object) - expect(importer).to receive(:each_object_to_import) - .and_yield(object).and_yield(object).and_yield(object) expect(worker_class).to receive(:perform_in) .with(1.second, project.id, { title: 'One' }, 'waiter-key').ordered expect(worker_class).to receive(:perform_in) - .with(1.second, project.id, { title: 'Two' }, 'waiter-key').ordered + .with(1.minute + 1.second, project.id, { title: 'Two' }, 'waiter-key').ordered expect(worker_class).to receive(:perform_in) - .with(1.minute + 1.second, project.id, { title: 'Three' }, 'waiter-key').ordered + .with(2.minutes + 1.second, project.id, { title: 'Three' }, 'waiter-key').ordered job_waiter = importer.parallel_import expect(job_waiter.key).to eq('waiter-key') - expect(job_waiter.jobs_remaining).to eq(3) - end - - context 'when job restarts due to API rate limit or Sidekiq interruption' do - before do - cache_key = format(described_class::JOB_WAITER_CACHE_KEY, - project: project.id, collection: importer.collection_method) - Gitlab::Cache::Import::Caching.write(cache_key, 'waiter-key') - - cache_key = format(described_class::JOB_WAITER_REMAINING_CACHE_KEY, - project: project.id, collection: importer.collection_method) - Gitlab::Cache::Import::Caching.write(cache_key, 3) - end - - it "restores job waiter's key and jobs_remaining" do - allow(importer).to receive(:parallel_import_batch).and_return({ size: 1, delay: 1.minute }) - - expect(importer).to receive(:each_object_to_import).and_yield(object).and_yield(object).and_yield(object) - - expect(worker_class).to receive(:perform_in) - .with(1.second, project.id, { title: 'One' }, 'waiter-key').ordered - expect(worker_class).to receive(:perform_in) - .with(1.minute + 1.second, project.id, { title: 'Two' }, 'waiter-key').ordered - expect(worker_class).to receive(:perform_in) - .with(2.minutes + 1.second, project.id, { title: 'Three' }, 'waiter-key').ordered - - job_waiter = importer.parallel_import - - expect(job_waiter.key).to eq('waiter-key') - expect(job_waiter.jobs_remaining).to eq(6) - end + expect(job_waiter.jobs_remaining).to eq(6) end end end diff --git a/spec/lib/gitlab/github_import/project_relation_type_spec.rb b/spec/lib/gitlab/github_import/project_relation_type_spec.rb new file mode 100644 index 00000000000..419cb6de121 --- /dev/null +++ b/spec/lib/gitlab/github_import/project_relation_type_spec.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::GithubImport::ProjectRelationType, :manage, feature_category: :importers do + subject(:project_relation_type) { described_class.new(client) } + + let(:octokit) { instance_double(Octokit::Client) } + let(:client) do + instance_double(Gitlab::GithubImport::Clients::Proxy, octokit: octokit, user: { login: 'nickname' }) + end + + describe '#for', :use_clean_rails_redis_caching do + before do + allow(client).to receive(:each_object).with(:organizations).and_yield({ login: 'great-org' }) + allow(octokit).to receive(:access_token).and_return('stub') + end + + context "when it's user owned repo" do + let(:import_source) { 'nickname/repo_name' } + + it { expect(project_relation_type.for(import_source)).to eq 'owned' } + end + + context "when it's organization repo" do + let(:import_source) { 'great-org/repo_name' } + + it { expect(project_relation_type.for(import_source)).to eq 'organization' } + end + + context "when it's user collaborated repo" do + let(:import_source) { 'some-another-namespace/repo_name' } + + it { expect(project_relation_type.for(import_source)).to eq 'collaborated' } + end + + context 'with cache' do + let(:import_source) { 'some-another-namespace/repo_name' } + + it 'calls client only once during 5 minutes timeframe', :request_store do + expect(project_relation_type.for(import_source)).to eq 'collaborated' + expect(project_relation_type.for('another/repo')).to eq 'collaborated' + + expect(client).to have_received(:each_object).once + expect(client).to have_received(:user).once + end + end + end +end diff --git a/spec/lib/gitlab/github_import/representation/collaborator_spec.rb b/spec/lib/gitlab/github_import/representation/collaborator_spec.rb new file mode 100644 index 00000000000..cc52c34ec74 --- /dev/null +++ b/spec/lib/gitlab/github_import/representation/collaborator_spec.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +RSpec.describe Gitlab::GithubImport::Representation::Collaborator, feature_category: :importers do + shared_examples 'a Collaborator' do + it 'returns an instance of Collaborator' do + expect(collaborator).to be_an_instance_of(described_class) + end + + context 'with Collaborator' do + it 'includes the user ID' do + expect(collaborator.id).to eq(42) + end + + it 'includes the username' do + expect(collaborator.login).to eq('alice') + end + + it 'includes the role' do + expect(collaborator.role_name).to eq('maintainer') + end + + describe '#github_identifiers' do + it 'returns a hash with needed identifiers' do + expect(collaborator.github_identifiers).to eq( + { + id: 42, + login: 'alice' + } + ) + end + end + end + end + + describe '.from_api_response' do + it_behaves_like 'a Collaborator' do + let(:response) { { id: 42, login: 'alice', role_name: 'maintainer' } } + let(:collaborator) { described_class.from_api_response(response) } + end + end + + describe '.from_json_hash' do + it_behaves_like 'a Collaborator' do + let(:hash) { { 'id' => 42, 'login' => 'alice', role_name: 'maintainer' } } + let(:collaborator) { described_class.from_json_hash(hash) } + end + end +end diff --git a/spec/lib/gitlab/github_import/representation/diff_note_spec.rb b/spec/lib/gitlab/github_import/representation/diff_note_spec.rb index 56fabe854f9..3e76b4ae698 100644 --- a/spec/lib/gitlab/github_import/representation/diff_note_spec.rb +++ b/spec/lib/gitlab/github_import/representation/diff_note_spec.rb @@ -131,7 +131,7 @@ RSpec.describe Gitlab::GithubImport::Representation::DiffNote, :clean_gitlab_red describe '#github_identifiers' do it 'returns a hash with needed identifiers' do expect(note.github_identifiers).to eq( - noteable_id: 42, + noteable_iid: 42, noteable_type: 'MergeRequest', note_id: 1 ) diff --git a/spec/lib/gitlab/github_import/representation/issue_event_spec.rb b/spec/lib/gitlab/github_import/representation/issue_event_spec.rb index 0dd281cb3b0..6620dee0fd0 100644 --- a/spec/lib/gitlab/github_import/representation/issue_event_spec.rb +++ b/spec/lib/gitlab/github_import/representation/issue_event_spec.rb @@ -156,7 +156,11 @@ RSpec.describe Gitlab::GithubImport::Representation::IssueEvent do describe '#github_identifiers' do it 'returns a hash with needed identifiers' do - expect(issue_event.github_identifiers).to eq({ id: 6501124486 }) + expect(issue_event.github_identifiers).to eq( + id: 6501124486, + issuable_iid: 2, + event: 'closed' + ) end end end diff --git a/spec/lib/gitlab/github_import/representation/issue_spec.rb b/spec/lib/gitlab/github_import/representation/issue_spec.rb index 263ef8b1708..39447da0fac 100644 --- a/spec/lib/gitlab/github_import/representation/issue_spec.rb +++ b/spec/lib/gitlab/github_import/representation/issue_spec.rb @@ -192,7 +192,8 @@ RSpec.describe Gitlab::GithubImport::Representation::Issue do it 'returns a hash with needed identifiers' do github_identifiers = { iid: 42, - issuable_type: 'MergeRequest' + issuable_type: 'MergeRequest', + title: 'Implement cool feature' } other_attributes = { pull_request: true, something_else: '_something_else_' } issue = described_class.new(github_identifiers.merge(other_attributes)) diff --git a/spec/lib/gitlab/github_import/representation/lfs_object_spec.rb b/spec/lib/gitlab/github_import/representation/lfs_object_spec.rb index 6663a7366a5..799a77afb0c 100644 --- a/spec/lib/gitlab/github_import/representation/lfs_object_spec.rb +++ b/spec/lib/gitlab/github_import/representation/lfs_object_spec.rb @@ -6,7 +6,8 @@ RSpec.describe Gitlab::GithubImport::Representation::LfsObject do describe '#github_identifiers' do it 'returns a hash with needed identifiers' do github_identifiers = { - oid: 42 + oid: 42, + size: 123456 } other_attributes = { something_else: '_something_else_' } lfs_object = described_class.new(github_identifiers.merge(other_attributes)) diff --git a/spec/lib/gitlab/github_import/representation/note_spec.rb b/spec/lib/gitlab/github_import/representation/note_spec.rb index 49126dbe9c5..5c2cea3653f 100644 --- a/spec/lib/gitlab/github_import/representation/note_spec.rb +++ b/spec/lib/gitlab/github_import/representation/note_spec.rb @@ -43,6 +43,16 @@ RSpec.describe Gitlab::GithubImport::Representation::Note do it 'includes the note ID' do expect(note.note_id).to eq(1) end + + describe '#github_identifiers' do + it 'returns a hash with needed identifiers' do + expect(note.github_identifiers).to eq( + noteable_iid: 42, + noteable_type: 'Issue', + note_id: 1 + ) + end + end end end @@ -103,18 +113,4 @@ RSpec.describe Gitlab::GithubImport::Representation::Note do expect(note.author).to be_nil end end - - describe '#github_identifiers' do - it 'returns a hash with needed identifiers' do - github_identifiers = { - noteable_id: 42, - noteable_type: 'Issue', - note_id: 1 - } - other_attributes = { something_else: '_something_else_' } - note = described_class.new(github_identifiers.merge(other_attributes)) - - expect(note.github_identifiers).to eq(github_identifiers) - end - end end diff --git a/spec/lib/gitlab/github_import/representation/note_text_spec.rb b/spec/lib/gitlab/github_import/representation/note_text_spec.rb index 8b57c9a0373..7aa458a1c33 100644 --- a/spec/lib/gitlab/github_import/representation/note_text_spec.rb +++ b/spec/lib/gitlab/github_import/representation/note_text_spec.rb @@ -22,35 +22,45 @@ RSpec.describe Gitlab::GithubImport::Representation::NoteText do end describe '.from_db_record' do + let(:representation) { described_class.from_db_record(record) } + context 'with Release' do - let(:record) { build_stubbed(:release, id: 42, description: 'Some text here..') } + let(:record) { build_stubbed(:release, id: 42, description: 'Some text here..', tag: 'v1.0') } + + it_behaves_like 'a Note text data', 'Release' - it_behaves_like 'a Note text data', 'Release' do - let(:representation) { described_class.from_db_record(record) } + it 'includes tag' do + expect(representation.tag).to eq 'v1.0' end end context 'with Issue' do - let(:record) { build_stubbed(:issue, id: 42, description: 'Some text here..') } + let(:record) { build_stubbed(:issue, id: 42, iid: 2, description: 'Some text here..') } + + it_behaves_like 'a Note text data', 'Issue' - it_behaves_like 'a Note text data', 'Issue' do - let(:representation) { described_class.from_db_record(record) } + it 'includes noteable iid' do + expect(representation.iid).to eq 2 end end context 'with MergeRequest' do - let(:record) { build_stubbed(:merge_request, id: 42, description: 'Some text here..') } + let(:record) { build_stubbed(:merge_request, id: 42, iid: 2, description: 'Some text here..') } - it_behaves_like 'a Note text data', 'MergeRequest' do - let(:representation) { described_class.from_db_record(record) } + it_behaves_like 'a Note text data', 'MergeRequest' + + it 'includes noteable iid' do + expect(representation.iid).to eq 2 end end context 'with Note' do - let(:record) { build_stubbed(:note, id: 42, note: 'Some text here..') } + let(:record) { build_stubbed(:note, id: 42, note: 'Some text here..', noteable_type: 'Issue') } + + it_behaves_like 'a Note text data', 'Note' - it_behaves_like 'a Note text data', 'Note' do - let(:representation) { described_class.from_db_record(record) } + it 'includes noteable type' do + expect(representation.noteable_type).to eq 'Issue' end end end @@ -61,7 +71,8 @@ RSpec.describe Gitlab::GithubImport::Representation::NoteText do { 'record_db_id' => 42, 'record_type' => 'Release', - 'text' => 'Some text here..' + 'text' => 'Some text here..', + 'tag' => 'v1.0' } end @@ -70,11 +81,76 @@ RSpec.describe Gitlab::GithubImport::Representation::NoteText do end describe '#github_identifiers' do - it 'returns a hash with needed identifiers' do - record_id = rand(100) - representation = described_class.new(record_db_id: record_id, text: 'text') + let(:iid) { nil } + let(:tag) { nil } + let(:noteable_type) { nil } + let(:hash) do + { + 'record_db_id' => 42, + 'record_type' => record_type, + 'text' => 'Some text here..', + 'iid' => iid, + 'tag' => tag, + 'noteable_type' => noteable_type + } + end + + subject { described_class.from_json_hash(hash) } + + context 'with Release' do + let(:record_type) { 'Release' } + let(:tag) { 'v1.0' } + + it 'returns a hash with needed identifiers' do + expect(subject.github_identifiers).to eq( + { + db_id: 42, + tag: 'v1.0' + } + ) + end + end + + context 'with Issue' do + let(:record_type) { 'Issue' } + let(:iid) { 2 } + + it 'returns a hash with needed identifiers' do + expect(subject.github_identifiers).to eq( + { + db_id: 42, + noteable_iid: 2 + } + ) + end + end - expect(representation.github_identifiers).to eq({ db_id: record_id }) + context 'with Merge Request' do + let(:record_type) { 'MergeRequest' } + let(:iid) { 3 } + + it 'returns a hash with needed identifiers' do + expect(subject.github_identifiers).to eq( + { + db_id: 42, + noteable_iid: 3 + } + ) + end + end + + context 'with Note' do + let(:record_type) { 'Note' } + let(:noteable_type) { 'MergeRequest' } + + it 'returns a hash with needed identifiers' do + expect(subject.github_identifiers).to eq( + { + db_id: 42, + noteable_type: 'MergeRequest' + } + ) + end end end end diff --git a/spec/lib/gitlab/github_import/representation/pull_request_review_spec.rb b/spec/lib/gitlab/github_import/representation/pull_request_review_spec.rb index 0203da9f4fb..8925f466e27 100644 --- a/spec/lib/gitlab/github_import/representation/pull_request_review_spec.rb +++ b/spec/lib/gitlab/github_import/representation/pull_request_review_spec.rb @@ -77,7 +77,7 @@ RSpec.describe Gitlab::GithubImport::Representation::PullRequestReview do it 'returns a hash with needed identifiers' do github_identifiers = { review_id: 999, - merge_request_id: 42 + merge_request_iid: 1 } other_attributes = { something_else: '_something_else_' } review = described_class.new(github_identifiers.merge(other_attributes)) diff --git a/spec/lib/gitlab/github_import/representation/pull_request_spec.rb b/spec/lib/gitlab/github_import/representation/pull_request_spec.rb index b8c1c67e07c..4b8e7401e9d 100644 --- a/spec/lib/gitlab/github_import/representation/pull_request_spec.rb +++ b/spec/lib/gitlab/github_import/representation/pull_request_spec.rb @@ -287,7 +287,8 @@ RSpec.describe Gitlab::GithubImport::Representation::PullRequest do describe '#github_identifiers' do it 'returns a hash with needed identifiers' do github_identifiers = { - iid: 1 + iid: 1, + title: 'My Pull Request' } other_attributes = { something_else: '_something_else_' } pr = described_class.new(github_identifiers.merge(other_attributes)) diff --git a/spec/lib/gitlab/github_import/representation/pull_requests/review_requests_spec.rb b/spec/lib/gitlab/github_import/representation/pull_requests/review_requests_spec.rb index 0393f692a69..0259fbedee3 100644 --- a/spec/lib/gitlab/github_import/representation/pull_requests/review_requests_spec.rb +++ b/spec/lib/gitlab/github_import/representation/pull_requests/review_requests_spec.rb @@ -46,4 +46,27 @@ RSpec.describe Gitlab::GithubImport::Representation::PullRequests::ReviewRequest let(:review_requests) { described_class.from_json_hash(response) } end end + + describe '#github_identifiers' do + it 'returns a hash with needed identifiers' do + review_requests = { + merge_request_iid: 2, + merge_request_id: merge_request_id, + users: [ + { id: 4, login: 'alice' }, + { id: 5, login: 'bob' } + ] + } + + github_identifiers = { + merge_request_iid: 2, + requested_reviewers: %w[alice bob] + } + + other_attributes = { merge_request_id: 123, something_else: '_something_else_' } + review_requests = described_class.new(review_requests.merge(other_attributes)) + + expect(review_requests.github_identifiers).to eq(github_identifiers) + end + end end diff --git a/spec/lib/gitlab/github_import/settings_spec.rb b/spec/lib/gitlab/github_import/settings_spec.rb index ad0c47e8e8a..43e096863b8 100644 --- a/spec/lib/gitlab/github_import/settings_spec.rb +++ b/spec/lib/gitlab/github_import/settings_spec.rb @@ -11,7 +11,8 @@ RSpec.describe Gitlab::GithubImport::Settings do { single_endpoint_issue_events_import: true, single_endpoint_notes_import: false, - attachments_import: false + attachments_import: false, + collaborators_import: false } end @@ -22,17 +23,26 @@ RSpec.describe Gitlab::GithubImport::Settings do { name: 'single_endpoint_issue_events_import', label: stages[:single_endpoint_issue_events_import][:label], + selected: false, details: stages[:single_endpoint_issue_events_import][:details] }, { name: 'single_endpoint_notes_import', label: stages[:single_endpoint_notes_import][:label], + selected: false, details: stages[:single_endpoint_notes_import][:details] }, { name: 'attachments_import', label: stages[:attachments_import][:label].strip, + selected: false, details: stages[:attachments_import][:details] + }, + { + name: 'collaborators_import', + label: stages[:collaborators_import][:label].strip, + selected: true, + details: stages[:collaborators_import][:details] } ] end @@ -48,6 +58,7 @@ RSpec.describe Gitlab::GithubImport::Settings do single_endpoint_issue_events_import: true, single_endpoint_notes_import: 'false', attachments_import: nil, + collaborators_import: false, foo: :bar }.stringify_keys end @@ -67,6 +78,7 @@ RSpec.describe Gitlab::GithubImport::Settings do expect(settings.enabled?(:single_endpoint_issue_events_import)).to eq true expect(settings.enabled?(:single_endpoint_notes_import)).to eq false expect(settings.enabled?(:attachments_import)).to eq false + expect(settings.enabled?(:collaborators_import)).to eq false end end @@ -77,6 +89,7 @@ RSpec.describe Gitlab::GithubImport::Settings do expect(settings.disabled?(:single_endpoint_issue_events_import)).to eq false expect(settings.disabled?(:single_endpoint_notes_import)).to eq true expect(settings.disabled?(:attachments_import)).to eq true + expect(settings.disabled?(:collaborators_import)).to eq true end end end diff --git a/spec/lib/gitlab/github_import/user_finder_spec.rb b/spec/lib/gitlab/github_import/user_finder_spec.rb index d77aaa0e846..b6e369cb35b 100644 --- a/spec/lib/gitlab/github_import/user_finder_spec.rb +++ b/spec/lib/gitlab/github_import/user_finder_spec.rb @@ -259,6 +259,41 @@ RSpec.describe Gitlab::GithubImport::UserFinder, :clean_gitlab_redis_cache do expect(finder.email_for_github_username('kittens')).to be_nil end + + context 'when a username does not exist on GitHub' do + context 'when github username inexistence is not cached' do + it 'caches github username inexistence' do + expect(client) + .to receive(:user) + .with('kittens') + .and_raise(::Octokit::NotFound) + + expect(Gitlab::Cache::Import::Caching) + .to receive(:write).with( + described_class::INEXISTENCE_OF_GITHUB_USERNAME_CACHE_KEY % 'kittens', true + ) + + expect(finder.email_for_github_username('kittens')).to be_nil + end + end + + context 'when github username inexistence is already cached' do + it 'does not make request to the client' do + expect(Gitlab::Cache::Import::Caching) + .to receive(:read).with(described_class::EMAIL_FOR_USERNAME_CACHE_KEY % 'kittens') + + expect(Gitlab::Cache::Import::Caching) + .to receive(:read).with( + described_class::INEXISTENCE_OF_GITHUB_USERNAME_CACHE_KEY % 'kittens' + ).and_return('true') + + expect(client) + .not_to receive(:user) + + expect(finder.email_for_github_username('kittens')).to be_nil + end + end + end end end diff --git a/spec/lib/gitlab/gitlab_import/client_spec.rb b/spec/lib/gitlab/gitlab_import/client_spec.rb deleted file mode 100644 index 7f57d5fbf1b..00000000000 --- a/spec/lib/gitlab/gitlab_import/client_spec.rb +++ /dev/null @@ -1,111 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::GitlabImport::Client do - include ImportSpecHelper - - let(:token) { '123456' } - let(:client) { described_class.new(token) } - - before do - stub_omniauth_provider('gitlab') - end - - it 'all OAuth2 client options are symbols' do - expect(client.client.options.keys).to all(be_kind_of(Symbol)) - end - - it 'uses membership and simple flags' do - stub_request('/api/v4/projects?membership=true&page=1&per_page=100&simple=true') - - expect_next_instance_of(OAuth2::Response) do |instance| - expect(instance).to receive(:parsed).and_return([]) - end - - expect(client.projects.to_a).to eq [] - end - - shared_examples 'pagination params' do - before do - allow_next_instance_of(OAuth2::Response) do |instance| - allow(instance).to receive(:parsed).and_return([]) - end - end - - it 'allows page_limit param' do - allow_next_instance_of(OAuth2::Response) do |instance| - allow(instance).to receive(:parsed).and_return(element_list) - end - - expect(client).to receive(:lazy_page_iterator).with(hash_including(page_limit: 2)).and_call_original - - client.send(method, *args, page_limit: 2, per_page: 1).to_a - end - - it 'allows per_page param' do - expect(client).to receive(:lazy_page_iterator).with(hash_including(per_page: 2)).and_call_original - - client.send(method, *args, per_page: 2).to_a - end - - it 'allows starting_page param' do - expect(client).to receive(:lazy_page_iterator).with(hash_including(starting_page: 3)).and_call_original - - client.send(method, *args, starting_page: 3).to_a - end - end - - describe '#projects' do - subject(:method) { :projects } - - let(:args) { [] } - let(:element_list) { build_list(:project, 2) } - - before do - stub_request('/api/v4/projects?membership=true&page=1&per_page=1&simple=true') - stub_request('/api/v4/projects?membership=true&page=2&per_page=1&simple=true') - stub_request('/api/v4/projects?membership=true&page=1&per_page=2&simple=true') - stub_request('/api/v4/projects?membership=true&page=3&per_page=100&simple=true') - end - - it_behaves_like 'pagination params' - end - - describe '#issues' do - subject(:method) { :issues } - - let(:args) { [1] } - let(:element_list) { build_list(:issue, 2) } - - before do - stub_request('/api/v4/projects/1/issues?page=1&per_page=1') - stub_request('/api/v4/projects/1/issues?page=2&per_page=1') - stub_request('/api/v4/projects/1/issues?page=1&per_page=2') - stub_request('/api/v4/projects/1/issues?page=3&per_page=100') - end - - it_behaves_like 'pagination params' - end - - describe '#issue_comments' do - subject(:method) { :issue_comments } - - let(:args) { [1, 1] } - let(:element_list) { build_list(:note_on_issue, 2) } - - before do - stub_request('/api/v4/projects/1/issues/1/notes?page=1&per_page=1') - stub_request('/api/v4/projects/1/issues/1/notes?page=2&per_page=1') - stub_request('/api/v4/projects/1/issues/1/notes?page=1&per_page=2') - stub_request('/api/v4/projects/1/issues/1/notes?page=3&per_page=100') - end - - it_behaves_like 'pagination params' - end - - def stub_request(path) - WebMock.stub_request(:get, "https://gitlab.com#{path}") - .to_return(status: 200) - end -end diff --git a/spec/lib/gitlab/gitlab_import/importer_spec.rb b/spec/lib/gitlab/gitlab_import/importer_spec.rb deleted file mode 100644 index 984c690add6..00000000000 --- a/spec/lib/gitlab/gitlab_import/importer_spec.rb +++ /dev/null @@ -1,57 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::GitlabImport::Importer do - include ImportSpecHelper - - describe '#execute' do - before do - stub_omniauth_provider('gitlab') - stub_request('issues', [ - { - 'id' => 2579857, - 'iid' => 3, - 'title' => 'Issue', - 'description' => 'Lorem ipsum', - 'state' => 'opened', - 'confidential' => true, - 'author' => { - 'id' => 283999, - 'name' => 'John Doe' - } - } - ].to_json) - stub_request('issues/3/notes', [].to_json) - end - - it 'persists issues' do - project = create(:project, import_source: 'asd/vim') - project.build_import_data(credentials: { password: 'password' }) - - subject = described_class.new(project) - subject.execute - - expected_attributes = { - iid: 3, - title: 'Issue', - description: "*Created by: John Doe*\n\nLorem ipsum", - state: 'opened', - confidential: true, - author_id: project.creator_id - } - - expect(project.issues.first).to have_attributes(expected_attributes) - end - - def stub_request(path, body) - url = "https://gitlab.com/api/v4/projects/asd%2Fvim/#{path}?page=1&per_page=100" - - WebMock.stub_request(:get, url) - .to_return( - headers: { 'Content-Type' => 'application/json' }, - body: body - ) - end - end -end diff --git a/spec/lib/gitlab/gitlab_import/project_creator_spec.rb b/spec/lib/gitlab/gitlab_import/project_creator_spec.rb deleted file mode 100644 index 53bf1db3438..00000000000 --- a/spec/lib/gitlab/gitlab_import/project_creator_spec.rb +++ /dev/null @@ -1,37 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::GitlabImport::ProjectCreator do - let(:user) { create(:user) } - let(:repo) do - { - name: 'vim', - path: 'vim', - visibility_level: Gitlab::VisibilityLevel::PRIVATE, - path_with_namespace: 'asd/vim', - http_url_to_repo: "https://gitlab.com/asd/vim.git", - owner: { name: "john" } - }.with_indifferent_access - end - - let(:namespace) { create(:group) } - let(:token) { "asdffg" } - let(:access_params) { { gitlab_access_token: token } } - - before do - namespace.add_owner(user) - end - - it 'creates project' do - expect_next_instance_of(Project) do |project| - expect(project).to receive(:add_import_job) - end - - project_creator = described_class.new(repo, namespace, user, access_params) - project = project_creator.execute - - expect(project.import_url).to eq("https://oauth2:asdffg@gitlab.com/asd/vim.git") - expect(project.visibility_level).to eq(Gitlab::VisibilityLevel::PRIVATE) - end -end diff --git a/spec/lib/gitlab/gl_repository/identifier_spec.rb b/spec/lib/gitlab/gl_repository/identifier_spec.rb index 0a8559dd800..dbdcafea6d6 100644 --- a/spec/lib/gitlab/gl_repository/identifier_spec.rb +++ b/spec/lib/gitlab/gl_repository/identifier_spec.rb @@ -68,10 +68,12 @@ RSpec.describe Gitlab::GlRepository::Identifier do end describe 'design' do + let(:design_repository_container) { project.design_repository.container } + it_behaves_like 'parsing gl_repository identifier' do let(:record_id) { project.id } - let(:identifier) { "design-#{project.id}" } - let(:expected_container) { project } + let(:identifier) { "design-#{design_repository_container.id}" } + let(:expected_container) { design_repository_container } let(:expected_type) { Gitlab::GlRepository::DESIGN } end end diff --git a/spec/lib/gitlab/gl_repository/repo_type_spec.rb b/spec/lib/gitlab/gl_repository/repo_type_spec.rb index 0ec94563cbb..2ac2fc1fd4b 100644 --- a/spec/lib/gitlab/gl_repository/repo_type_spec.rb +++ b/spec/lib/gitlab/gl_repository/repo_type_spec.rb @@ -12,6 +12,8 @@ RSpec.describe Gitlab::GlRepository::RepoType do let(:personal_snippet_path) { "snippets/#{personal_snippet.id}" } let(:project_snippet_path) { "#{project.full_path}/snippets/#{project_snippet.id}" } + let(:expected_repository_resolver) { expected_container } + describe Gitlab::GlRepository::PROJECT do it_behaves_like 'a repo type' do let(:expected_id) { project.id } @@ -133,11 +135,12 @@ RSpec.describe Gitlab::GlRepository::RepoType do describe Gitlab::GlRepository::DESIGN do it_behaves_like 'a repo type' do - let(:expected_identifier) { "design-#{project.id}" } - let(:expected_id) { project.id } + let(:expected_repository) { project.design_repository } + let(:expected_container) { project.design_management_repository } + let(:expected_id) { expected_container.id } + let(:expected_identifier) { "design-#{expected_id}" } let(:expected_suffix) { '.design' } - let(:expected_repository) { ::DesignManagement::Repository.new(project) } - let(:expected_container) { project } + let(:expected_repository_resolver) { project } end it 'uses the design access checker' do @@ -162,5 +165,17 @@ RSpec.describe Gitlab::GlRepository::RepoType do expect(described_class.valid?(project_snippet_path)).to be_falsey end end + + describe '.project_for' do + it 'returns a project' do + expect(described_class.project_for(project.design_repository.container)).to be_instance_of(Project) + end + end + + describe '.repository_for' do + it 'returns a DesignManagement::GitRepository when a project is passed' do + expect(described_class.repository_for(project)).to be_instance_of(DesignManagement::GitRepository) + end + end end end diff --git a/spec/lib/gitlab/gl_repository_spec.rb b/spec/lib/gitlab/gl_repository_spec.rb index 05914f92c01..7be01507a82 100644 --- a/spec/lib/gitlab/gl_repository_spec.rb +++ b/spec/lib/gitlab/gl_repository_spec.rb @@ -6,6 +6,7 @@ RSpec.describe ::Gitlab::GlRepository do describe '.parse' do let_it_be(:project) { create(:project, :repository) } let_it_be(:snippet) { create(:personal_snippet) } + let(:design_repository_container) { project.design_repository.container } it 'parses a project gl_repository' do expect(described_class.parse("project-#{project.id}")).to eq([project, project, Gitlab::GlRepository::PROJECT]) @@ -20,7 +21,13 @@ RSpec.describe ::Gitlab::GlRepository do end it 'parses a design gl_repository' do - expect(described_class.parse("design-#{project.id}")).to eq([project, project, Gitlab::GlRepository::DESIGN]) + expect(described_class.parse("design-#{design_repository_container.id}")).to eq( + [ + design_repository_container, + project, + Gitlab::GlRepository::DESIGN + ] + ) end it 'throws an argument error on an invalid gl_repository type' do diff --git a/spec/lib/gitlab/grape_logging/loggers/response_logger_spec.rb b/spec/lib/gitlab/grape_logging/loggers/response_logger_spec.rb index 94e880d979d..449096a6faf 100644 --- a/spec/lib/gitlab/grape_logging/loggers/response_logger_spec.rb +++ b/spec/lib/gitlab/grape_logging/loggers/response_logger_spec.rb @@ -27,5 +27,11 @@ RSpec.describe Gitlab::GrapeLogging::Loggers::ResponseLogger do it { expect(subject).to eq({}) } end + + context 'when response is a String' do + let(:response) { response1 } + + it { expect(subject).to eq({ response_bytes: response1.bytesize }) } + end end end diff --git a/spec/lib/gitlab/graphql/authorize/authorize_resource_spec.rb b/spec/lib/gitlab/graphql/authorize/authorize_resource_spec.rb index ac512e28e7b..1cd93d7b364 100644 --- a/spec/lib/gitlab/graphql/authorize/authorize_resource_spec.rb +++ b/spec/lib/gitlab/graphql/authorize/authorize_resource_spec.rb @@ -76,13 +76,17 @@ RSpec.describe Gitlab::Graphql::Authorize::AuthorizeResource do end end - context 'when the class does not define #find_object' do + describe '#find_object' do let(:fake_class) do Class.new { include Gitlab::Graphql::Authorize::AuthorizeResource } end - it 'raises a comprehensive error message' do - expect { fake_class.new.find_object }.to raise_error(/Implement #find_object in #{fake_class.name}/) + let(:id) { "id" } + let(:return_value) { "return value" } + + it 'calls GitlabSchema.find_by_gid' do + expect(GitlabSchema).to receive(:find_by_gid).with(id).and_return(return_value) + expect(fake_class.new.find_object(id: id)).to be return_value end end diff --git a/spec/lib/gitlab/graphql/deprecations/deprecation_spec.rb b/spec/lib/gitlab/graphql/deprecations/deprecation_spec.rb index 55650b0480e..172872fd7eb 100644 --- a/spec/lib/gitlab/graphql/deprecations/deprecation_spec.rb +++ b/spec/lib/gitlab/graphql/deprecations/deprecation_spec.rb @@ -55,7 +55,7 @@ RSpec.describe ::Gitlab::Graphql::Deprecations::Deprecation, feature_category: : it 'raises an error' do expect { parsed_deprecation }.to raise_error(ArgumentError, - '`alpha` and `deprecated` arguments cannot be passed at the same time' + '`experiment` and `deprecated` arguments cannot be passed at the same time' ) end end diff --git a/spec/lib/gitlab/graphql/known_operations_spec.rb b/spec/lib/gitlab/graphql/known_operations_spec.rb index 3ebfefbb43c..c7bc47e1e6a 100644 --- a/spec/lib/gitlab/graphql/known_operations_spec.rb +++ b/spec/lib/gitlab/graphql/known_operations_spec.rb @@ -2,7 +2,6 @@ require 'fast_spec_helper' require 'rspec-parameterized' -require "support/graphql/fake_query_type" RSpec.describe Gitlab::Graphql::KnownOperations do using RSpec::Parameterized::TableSyntax diff --git a/spec/lib/gitlab/graphql/loaders/lazy_relation_loader/registry_spec.rb b/spec/lib/gitlab/graphql/loaders/lazy_relation_loader/registry_spec.rb new file mode 100644 index 00000000000..265839d1236 --- /dev/null +++ b/spec/lib/gitlab/graphql/loaders/lazy_relation_loader/registry_spec.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Graphql::Loaders::LazyRelationLoader::Registry, feature_category: :vulnerability_management do + describe '#respond_to?' do + let(:relation) { Project.all } + let(:registry) { described_class.new(relation) } + + subject { registry.respond_to?(method_name) } + + context 'when the relation responds to given method' do + let(:method_name) { :sorted_by_updated_asc } + + it { is_expected.to be_truthy } + end + + context 'when the relation does not respond to given method' do + let(:method_name) { :this_method_does_not_exist } + + it { is_expected.to be_falsey } + end + end +end diff --git a/spec/lib/gitlab/graphql/loaders/lazy_relation_loader/relation_proxy_spec.rb b/spec/lib/gitlab/graphql/loaders/lazy_relation_loader/relation_proxy_spec.rb new file mode 100644 index 00000000000..f54fb6e77c5 --- /dev/null +++ b/spec/lib/gitlab/graphql/loaders/lazy_relation_loader/relation_proxy_spec.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Graphql::Loaders::LazyRelationLoader::RelationProxy, feature_category: :vulnerability_management do + describe '#respond_to?' do + let(:object) { double } + let(:registry) { instance_double(Gitlab::Graphql::Loaders::LazyRelationLoader::Registry) } + let(:relation_proxy) { described_class.new(object, registry) } + + subject { relation_proxy.respond_to?(:foo) } + + before do + allow(registry).to receive(:respond_to?).with(:foo, false).and_return(responds_to?) + end + + context 'when the registry responds to given method' do + let(:responds_to?) { true } + + it { is_expected.to be_truthy } + end + + context 'when the registry does not respond to given method' do + let(:responds_to?) { false } + + it { is_expected.to be_falsey } + end + end +end diff --git a/spec/lib/gitlab/graphql/loaders/lazy_relation_loader_spec.rb b/spec/lib/gitlab/graphql/loaders/lazy_relation_loader_spec.rb new file mode 100644 index 00000000000..e56cb68c6cb --- /dev/null +++ b/spec/lib/gitlab/graphql/loaders/lazy_relation_loader_spec.rb @@ -0,0 +1,123 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Graphql::Loaders::LazyRelationLoader, feature_category: :vulnerability_management do + let(:query_context) { {} } + let(:args) { {} } + + let_it_be(:project) { create(:project) } + + let(:loader) { loader_class.new(query_context, project, **args) } + + describe '#load' do + subject(:load_relation) { loader.load } + + context 'when the association is has many' do + let_it_be(:public_issue) { create(:issue, project: project) } + let_it_be(:confidential_issue) { create(:issue, :confidential, project: project) } + + let(:loader_class) do + Class.new(described_class) do + self.model = Project + self.association = :issues + + def relation(public_only: false) + relation = base_relation + relation = relation.public_only if public_only + + relation + end + end + end + + it { is_expected.to be_an_instance_of(described_class::RelationProxy) } + + describe '#relation' do + subject { load_relation.load } + + context 'without arguments' do + it { is_expected.to contain_exactly(public_issue, confidential_issue) } + end + + context 'with arguments' do + let(:args) { { public_only: true } } + + it { is_expected.to contain_exactly(public_issue) } + end + end + + describe 'using the same context for different records' do + let_it_be(:another_project) { create(:project) } + + let(:loader_for_another_project) { loader_class.new(query_context, another_project, **args) } + let(:records_for_another_project) { loader_for_another_project.load.load } + let(:records_for_project) { load_relation.load } + + before do + loader # register the original loader to query context + end + + it 'does not mix associated records' do + expect(records_for_another_project).to be_empty + expect(records_for_project).to contain_exactly(public_issue, confidential_issue) + end + + it 'does not cause N+1 queries' do + expect { records_for_another_project }.not_to exceed_query_limit(1) + end + end + + describe 'using Active Record querying methods' do + subject { load_relation.limit(1).load.count } + + it { is_expected.to be(1) } + end + + describe 'using Active Record finder methods' do + subject { load_relation.last(2) } + + it { is_expected.to contain_exactly(public_issue, confidential_issue) } + end + + describe 'calling a method that returns a non relation object' do + subject { load_relation.limit(1).limit_value } + + it { is_expected.to be(1) } + end + + describe 'calling a prohibited method' do + subject(:count) { load_relation.count } + + it 'raises a `PrematureQueryExecutionTriggered` error' do + expect { count }.to raise_error(described_class::Registry::PrematureQueryExecutionTriggered) + end + end + end + + context 'when the association is has one' do + let!(:project_setting) { create(:project_setting, project: project) } + let(:loader_class) do + Class.new(described_class) do + self.model = Project + self.association = :project_setting + end + end + + it { is_expected.to eq(project_setting) } + end + + context 'when the association is belongs to' do + let(:loader_class) do + Class.new(described_class) do + self.model = Project + self.association = :namespace + end + end + + it 'raises error' do + expect { load_relation }.to raise_error(RuntimeError) + end + end + end +end diff --git a/spec/lib/gitlab/graphql/subscriptions/action_cable_with_load_balancing_spec.rb b/spec/lib/gitlab/graphql/subscriptions/action_cable_with_load_balancing_spec.rb new file mode 100644 index 00000000000..8d8b879a90f --- /dev/null +++ b/spec/lib/gitlab/graphql/subscriptions/action_cable_with_load_balancing_spec.rb @@ -0,0 +1,138 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Graphql::Subscriptions::ActionCableWithLoadBalancing, feature_category: :shared do + let(:field) { Types::SubscriptionType.fields.each_value.first } + let(:event) { ::GraphQL::Subscriptions::Event.new(name: 'test-event', arguments: {}, field: field) } + let(:object) { build(:project, id: 1) } + let(:action_cable) { instance_double(::ActionCable::Server::Broadcasting) } + + subject(:subscriptions) { described_class.new(schema: GitlabSchema) } + + include_context 'when tracking WAL location reference' + + before do + allow(::ActionCable).to receive(:server).and_return(action_cable) + end + + context 'when triggering subscription' do + shared_examples_for 'injecting WAL locations' do + it 'injects correct WAL location into message' do + expect(action_cable).to receive(:broadcast) do |topic, payload| + expect(topic).to match(/^graphql-event/) + expect(Gitlab::Json.parse(payload)).to match({ + described_class::KEY_WAL_LOCATIONS => expected_locations, + described_class::KEY_PAYLOAD => { '__gid__' => 'Z2lkOi8vZ2l0bGFiL1Byb2plY3QvMQ' } + }) + end + + subscriptions.execute_all(event, object) + end + end + + context 'when database load balancing is disabled' do + let!(:expected_locations) { {} } + + before do + stub_load_balancing_disabled! + end + + it_behaves_like 'injecting WAL locations' + end + + context 'when database load balancing is enabled' do + before do + stub_load_balancing_enabled! + end + + context 'when write was not performed' do + before do + stub_no_writes_performed! + end + + context 'when replica hosts are available' do + let!(:expected_locations) { expect_tracked_locations_when_replicas_available.with_indifferent_access } + + it_behaves_like 'injecting WAL locations' + end + + context 'when no replica hosts are available' do + let!(:expected_locations) { expect_tracked_locations_when_no_replicas_available.with_indifferent_access } + + it_behaves_like 'injecting WAL locations' + end + end + + context 'when write was performed' do + let!(:expected_locations) { expect_tracked_locations_from_primary_only.with_indifferent_access } + + before do + stub_write_performed! + end + + it_behaves_like 'injecting WAL locations' + end + end + end + + context 'when handling event' do + def handle_event!(wal_locations: nil) + subscriptions.execute_update('sub:123', event, { + described_class::KEY_WAL_LOCATIONS => wal_locations || { + 'main' => current_location + }, + described_class::KEY_PAYLOAD => { '__gid__' => 'Z2lkOi8vZ2l0bGFiL1Byb2plY3QvMQ' } + }) + end + + before do + allow(action_cable).to receive(:broadcast) + end + + context 'when event payload is not wrapped' do + it 'does not attempt to unwrap it' do + expect(object).not_to receive(:[]).with(described_class::KEY_PAYLOAD) + + subscriptions.execute_update('sub:123', event, object) + end + end + + context 'when WAL locations are not present' do + it 'uses the primary' do + expect(::Gitlab::Database::LoadBalancing::Session.current).to receive(:use_primary!) + + handle_event!(wal_locations: {}) + end + end + + it 'strips out WAL location information before broadcasting payload' do + expect(action_cable).to receive(:broadcast) do |topic, payload| + expect(topic).to eq('graphql-subscription:sub:123') + expect(payload).to eq({ more: false }) + end + + handle_event! + end + + context 'when database replicas are in sync' do + it 'does not use the primary' do + stub_replica_available!(true) + + expect(::Gitlab::Database::LoadBalancing::Session.current).not_to receive(:use_primary!) + + handle_event! + end + end + + context 'when database replicas are not in sync' do + it 'uses the primary' do + stub_replica_available!(false) + + expect(::Gitlab::Database::LoadBalancing::Session.current).to receive(:use_primary!) + + handle_event! + end + end + end +end diff --git a/spec/lib/gitlab/graphql/tracers/metrics_tracer_spec.rb b/spec/lib/gitlab/graphql/tracers/metrics_tracer_spec.rb index 168f5aa529e..f0312293469 100644 --- a/spec/lib/gitlab/graphql/tracers/metrics_tracer_spec.rb +++ b/spec/lib/gitlab/graphql/tracers/metrics_tracer_spec.rb @@ -2,7 +2,6 @@ require 'spec_helper' require 'rspec-parameterized' -require "support/graphql/fake_query_type" RSpec.describe Gitlab::Graphql::Tracers::MetricsTracer do using RSpec::Parameterized::TableSyntax diff --git a/spec/lib/gitlab/graphql/tracers/timer_tracer_spec.rb b/spec/lib/gitlab/graphql/tracers/timer_tracer_spec.rb index 986120dcd95..e42883aafd8 100644 --- a/spec/lib/gitlab/graphql/tracers/timer_tracer_spec.rb +++ b/spec/lib/gitlab/graphql/tracers/timer_tracer_spec.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true require "fast_spec_helper" -require "support/graphql/fake_tracer" -require "support/graphql/fake_query_type" RSpec.describe Gitlab::Graphql::Tracers::TimerTracer do let(:expected_duration) { 5 } diff --git a/spec/lib/gitlab/harbor/client_spec.rb b/spec/lib/gitlab/harbor/client_spec.rb index 4e80b8b53e3..745e22191bd 100644 --- a/spec/lib/gitlab/harbor/client_spec.rb +++ b/spec/lib/gitlab/harbor/client_spec.rb @@ -265,18 +265,20 @@ RSpec.describe Gitlab::Harbor::Client do end end - describe '#ping' do + describe '#check_project_availability' do before do - stub_request(:get, "https://demo.goharbor.io/api/v2.0/ping") + stub_request(:head, "https://demo.goharbor.io/api/v2.0/projects?project_name=testproject") .with( headers: { + 'Accept': 'application/json', + 'Authorization': 'Basic aGFyYm9ydXNlcm5hbWU6aGFyYm9ycGFzc3dvcmQ=', 'Content-Type': 'application/json' }) - .to_return(status: 200, body: 'pong') + .to_return(status: 200, body: '', headers: {}) end - it "calls api/v2.0/ping successfully" do - expect(client.ping).to eq(success: true) + it "calls api/v2.0/projects successfully" do + expect(client.check_project_availability).to eq(success: true) end end diff --git a/spec/lib/gitlab/http_connection_adapter_spec.rb b/spec/lib/gitlab/http_connection_adapter_spec.rb index dbf0252da46..fac0c1a2a9f 100644 --- a/spec/lib/gitlab/http_connection_adapter_spec.rb +++ b/spec/lib/gitlab/http_connection_adapter_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::HTTPConnectionAdapter do +RSpec.describe Gitlab::HTTPConnectionAdapter, feature_category: :shared do include StubRequests let(:uri) { URI('https://example.org') } @@ -111,17 +111,39 @@ RSpec.describe Gitlab::HTTPConnectionAdapter do end end - context 'when http(s) environment variable is set' do + context 'when proxy is enabled' do before do - stub_env('https_proxy' => 'https://my.proxy') + stub_env('http_proxy', 'http://proxy.example.com') end - it 'sets up the connection' do - expect(connection).to be_a(Gitlab::NetHttpAdapter) - expect(connection.address).to eq('example.org') - expect(connection.hostname_override).to eq(nil) - expect(connection.addr_port).to eq('example.org') - expect(connection.port).to eq(443) + it 'proxy stays configured' do + expect(connection.proxy?).to be true + expect(connection.proxy_from_env?).to be true + expect(connection.proxy_address).to eq('proxy.example.com') + end + + context 'when no_proxy matches the request' do + before do + stub_env('no_proxy', 'example.org') + end + + it 'proxy is disabled' do + expect(connection.proxy?).to be false + expect(connection.proxy_from_env?).to be false + expect(connection.proxy_address).to be nil + end + end + + context 'when no_proxy does not match the request' do + before do + stub_env('no_proxy', 'example.com') + end + + it 'proxy stays configured' do + expect(connection.proxy?).to be true + expect(connection.proxy_from_env?).to be true + expect(connection.proxy_address).to eq('proxy.example.com') + end end end diff --git a/spec/lib/gitlab/i18n/pluralization_spec.rb b/spec/lib/gitlab/i18n/pluralization_spec.rb new file mode 100644 index 00000000000..857562d549c --- /dev/null +++ b/spec/lib/gitlab/i18n/pluralization_spec.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' +require 'rspec-parameterized' +require 'gettext_i18n_rails' + +RSpec.describe Gitlab::I18n::Pluralization, feature_category: :internationalization do + describe '.call' do + subject(:rule) { described_class.call(1) } + + context 'with available locales' do + around do |example| + Gitlab::I18n.with_locale(locale, &example) + end + + where(:locale) do + Gitlab::I18n.available_locales + end + + with_them do + it 'supports pluralization' do + expect(rule).not_to be_nil + end + end + + context 'with missing rules' do + let(:locale) { "pl_PL" } + + before do + stub_const("#{described_class}::MAP", described_class::MAP.except(locale)) + end + + it 'raises an ArgumentError' do + expect { rule }.to raise_error(ArgumentError, + /Missing pluralization rule for locale "#{locale}"/ + ) + end + end + end + end + + describe '.install_on' do + let(:mod) { Module.new } + + before do + described_class.install_on(mod) + end + + it 'adds pluralisation_rule method' do + expect(mod.pluralisation_rule).to eq(described_class) + end + end +end diff --git a/spec/lib/gitlab/i18n_spec.rb b/spec/lib/gitlab/i18n_spec.rb index b752d89bf0d..ee92831922d 100644 --- a/spec/lib/gitlab/i18n_spec.rb +++ b/spec/lib/gitlab/i18n_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::I18n do +RSpec.describe Gitlab::I18n, feature_category: :internationalization do let(:user) { create(:user, preferred_language: :es) } describe '.selectable_locales' do @@ -47,4 +47,19 @@ RSpec.describe Gitlab::I18n do expect(::I18n.locale).to eq(:en) end end + + describe '.pluralisation_rule' do + context 'when overridden' do + before do + # Internally, FastGettext sets + # Thread.current[:fast_gettext_pluralisation_rule]. + # Our patch patches `FastGettext.pluralisation_rule` instead. + FastGettext.pluralisation_rule = :something + end + + it 'returns custom definition regardless' do + expect(FastGettext.pluralisation_rule).to eq(Gitlab::I18n::Pluralization) + end + end + end end diff --git a/spec/lib/gitlab/import/errors_spec.rb b/spec/lib/gitlab/import/errors_spec.rb new file mode 100644 index 00000000000..f89cb36bbb4 --- /dev/null +++ b/spec/lib/gitlab/import/errors_spec.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Import::Errors, feature_category: :importers do + let_it_be(:project) { create(:project) } + + describe '.merge_nested_errors' do + it 'merges nested collection errors' do + issue = project.issues.new( + title: 'test', + notes: [ + Note.new( + award_emoji: [AwardEmoji.new(name: 'test')] + ) + ], + sentry_issue: SentryIssue.new + ) + + issue.validate + + expect(issue.errors.full_messages) + .to contain_exactly( + "Author can't be blank", + "Notes is invalid", + "Sentry issue sentry issue identifier can't be blank" + ) + + described_class.merge_nested_errors(issue) + + expect(issue.errors.full_messages) + .to contain_exactly( + "Notes is invalid", + "Author can't be blank", + "Sentry issue sentry issue identifier can't be blank", + "Award emoji is invalid", + "Note can't be blank", + "Project can't be blank", + "Noteable can't be blank", + "Author can't be blank", + "Project does not match noteable project", + "User can't be blank", + "Awardable can't be blank", + "Name is not a valid emoji name" + ) + end + end +end diff --git a/spec/lib/gitlab/import/logger_spec.rb b/spec/lib/gitlab/import/logger_spec.rb index 60978aaa25c..a85ba84108e 100644 --- a/spec/lib/gitlab/import/logger_spec.rb +++ b/spec/lib/gitlab/import/logger_spec.rb @@ -5,35 +5,5 @@ require 'spec_helper' RSpec.describe Gitlab::Import::Logger do subject { described_class.new('/dev/null') } - let(:now) { Time.zone.now } - - describe '#format_message' do - before do - allow(Labkit::Correlation::CorrelationId).to receive(:current_id).and_return('new-correlation-id') - end - - it 'formats strings' do - output = subject.format_message('INFO', now, 'test', 'Hello world') - - expect(Gitlab::Json.parse(output)).to eq({ - 'severity' => 'INFO', - 'time' => now.utc.iso8601(3), - 'message' => 'Hello world', - 'correlation_id' => 'new-correlation-id', - 'feature_category' => 'importers' - }) - end - - it 'formats hashes' do - output = subject.format_message('INFO', now, 'test', { hello: 1 }) - - expect(Gitlab::Json.parse(output)).to eq({ - 'severity' => 'INFO', - 'time' => now.utc.iso8601(3), - 'hello' => 1, - 'correlation_id' => 'new-correlation-id', - 'feature_category' => 'importers' - }) - end - end + it_behaves_like 'a json logger', { 'feature_category' => 'importers' } end diff --git a/spec/lib/gitlab/import/metrics_spec.rb b/spec/lib/gitlab/import/metrics_spec.rb index 9b8b58d00f3..9a7eb7b875e 100644 --- a/spec/lib/gitlab/import/metrics_spec.rb +++ b/spec/lib/gitlab/import/metrics_spec.rb @@ -11,7 +11,6 @@ RSpec.describe Gitlab::Import::Metrics, :aggregate_failures do subject { described_class.new(importer, project) } before do - allow(Gitlab::Metrics).to receive(:counter) { counter } allow(counter).to receive(:increment) allow(histogram).to receive(:observe) end @@ -42,6 +41,13 @@ RSpec.describe Gitlab::Import::Metrics, :aggregate_failures do context 'when project is not a github import' do it 'does not emit importer metrics' do expect(subject).not_to receive(:track_usage_event) + expect_no_snowplow_event( + category: 'Import::GithubService', + action: 'create', + label: 'github_import_project_state', + project: project, + import_type: 'github', state: 'failed' + ) subject.track_failed_import end @@ -50,39 +56,81 @@ RSpec.describe Gitlab::Import::Metrics, :aggregate_failures do context 'when project is a github import' do before do project.import_type = 'github' + allow(project).to receive(:import_status).and_return('failed') end it 'emits importer metrics' do expect(subject).to receive(:track_usage_event).with(:github_import_project_failure, project.id) subject.track_failed_import + + expect_snowplow_event( + category: 'Import::GithubService', + action: 'create', + label: 'github_import_project_state', + project: project, + import_type: 'github', state: 'failed' + ) end end end describe '#track_finished_import' do - before do - allow(Gitlab::Metrics).to receive(:histogram) { histogram } - end + context 'when project is a github import' do + before do + project.import_type = 'github' + allow(Gitlab::Metrics).to receive(:counter) { counter } + allow(Gitlab::Metrics).to receive(:histogram) { histogram } + allow(project).to receive(:beautified_import_status_name).and_return('completed') + end - it 'emits importer metrics' do - expect(Gitlab::Metrics).to receive(:counter).with( - :test_importer_imported_projects_total, - 'The number of imported projects' - ) + it 'emits importer metrics' do + expect(Gitlab::Metrics).to receive(:counter).with( + :test_importer_imported_projects_total, + 'The number of imported projects' + ) - expect(Gitlab::Metrics).to receive(:histogram).with( - :test_importer_total_duration_seconds, - 'Total time spent importing projects, in seconds', - {}, - described_class::IMPORT_DURATION_BUCKETS - ) + expect(Gitlab::Metrics).to receive(:histogram).with( + :test_importer_total_duration_seconds, + 'Total time spent importing projects, in seconds', + {}, + described_class::IMPORT_DURATION_BUCKETS + ) + + expect(counter).to receive(:increment) - expect(counter).to receive(:increment) + subject.track_finished_import - subject.track_finished_import + expect_snowplow_event( + category: 'Import::GithubService', + action: 'create', + label: 'github_import_project_state', + project: project, + import_type: 'github', state: 'completed' + ) + + expect(subject.duration).not_to be_nil + end - expect(subject.duration).not_to be_nil + context 'when import is partially completed' do + before do + allow(project).to receive(:beautified_import_status_name).and_return('partially completed') + end + + it 'emits snowplow metrics' do + expect(subject).to receive(:track_usage_event).with(:github_import_project_partially_completed, project.id) + + subject.track_finished_import + + expect_snowplow_event( + category: 'Import::GithubService', + action: 'create', + label: 'github_import_project_state', + project: project, + import_type: 'github', state: 'partially completed' + ) + end + end end context 'when project is not a github import' do @@ -91,7 +139,51 @@ RSpec.describe Gitlab::Import::Metrics, :aggregate_failures do subject.track_finished_import - expect(histogram).to have_received(:observe).with({ importer: :test_importer }, anything) + expect_no_snowplow_event( + category: 'Import::GithubService', + action: 'create', + label: 'github_import_project_state', + project: project, + import_type: 'github', state: 'completed' + ) + end + end + end + + describe '#track_cancelled_import' do + context 'when project is not a github import' do + it 'does not emit importer metrics' do + expect(subject).not_to receive(:track_usage_event) + expect_no_snowplow_event( + category: 'Import::GithubService', + action: 'create', + label: 'github_import_project_state', + project: project, + import_type: 'github', state: 'canceled' + ) + + subject.track_canceled_import + end + end + + context 'when project is a github import' do + before do + project.import_type = 'github' + allow(project).to receive(:import_status).and_return('canceled') + end + + it 'emits importer metrics' do + expect(subject).to receive(:track_usage_event).with(:github_import_project_cancelled, project.id) + + subject.track_canceled_import + + expect_snowplow_event( + category: 'Import::GithubService', + action: 'create', + label: 'github_import_project_state', + project: project, + import_type: 'github', state: 'canceled' + ) end end end diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index 0c2c3ffc664..34f9948b9dc 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -14,6 +14,7 @@ issues: - resource_milestone_events - resource_state_events - resource_iteration_events +- assignment_events - sent_notifications - sentry_issue - issuable_severity @@ -92,6 +93,25 @@ notes: - suggestions - diff_note_positions - review +- note_metadata +note_metadata: + - note + - email_participant +commit_notes: +- award_emoji +- noteable +- author +- updated_by +- last_edited_by +- resolved_by +- todos +- events +- system_note_metadata +- note_diff_file +- suggestions +- diff_note_positions +- review +- note_metadata label_links: - target - label @@ -166,6 +186,7 @@ merge_requests: - resource_milestone_events - resource_state_events - resource_iteration_events +- assignment_events - label_links - labels - last_edited_by @@ -202,7 +223,7 @@ merge_requests: - approver_groups - approved_by_users - draft_notes -- merge_train +- merge_train_car - blocks_as_blocker - blocks_as_blockee - blocking_merge_requests @@ -246,6 +267,11 @@ ci_pipelines: - statuses - statuses_order_id_desc - latest_statuses_ordered_by_stage +- latest_statuses +- all_jobs +- current_jobs +- all_processable_jobs +- current_processable_jobs - builds - bridges - processables @@ -283,6 +309,7 @@ ci_pipelines: - job_artifacts - vulnerabilities_finding_pipelines - vulnerability_findings +- vulnerability_state_transitions - pipeline_config - security_scans - security_findings @@ -293,7 +320,6 @@ ci_pipelines: - latest_builds_report_results - messages - pipeline_artifacts -- latest_statuses - dast_profile - dast_profiles_pipeline - dast_site_profile @@ -317,6 +343,7 @@ stages: - processables - builds - bridges +- generic_commit_statuses - latest_statuses - retried_statuses statuses: @@ -327,6 +354,92 @@ statuses: - auto_canceled_by - needs - ci_stage +builds: +- user +- auto_canceled_by +- ci_stage +- needs +- resource +- pipeline +- sourced_pipeline +- resource_group +- metadata +- runner +- trigger_request +- erased_by +- deployment +- pending_state +- queuing_entry +- runtime_metadata +- trace_chunks +- report_results +- namespace +- job_artifacts +- job_variables +- sourced_pipelines +- pages_deployments +- job_artifacts_archive +- job_artifacts_metadata +- job_artifacts_trace +- job_artifacts_junit +- job_artifacts_sast +- job_artifacts_dependency_scanning +- job_artifacts_container_scanning +- job_artifacts_dast +- job_artifacts_codequality +- job_artifacts_license_scanning +- job_artifacts_performance +- job_artifacts_metrics +- job_artifacts_metrics_referee +- job_artifacts_network_referee +- job_artifacts_lsif +- job_artifacts_dotenv +- job_artifacts_cobertura +- job_artifacts_terraform +- job_artifacts_accessibility +- job_artifacts_cluster_applications +- job_artifacts_secret_detection +- job_artifacts_requirements +- job_artifacts_coverage_fuzzing +- job_artifacts_browser_performance +- job_artifacts_load_performance +- job_artifacts_api_fuzzing +- job_artifacts_cluster_image_scanning +- job_artifacts_cyclonedx +- job_artifacts_requirements_v2 +- runner_manager +- runner_manager_build +- runner_session +- trace_metadata +- terraform_state_versions +- taggings +- base_tags +- tag_taggings +- tags +- security_scans +- dast_site_profiles_build +- dast_site_profile +- dast_scanner_profiles_build +- dast_scanner_profile +bridges: +- user +- pipeline +- auto_canceled_by +- ci_stage +- needs +- resource +- sourced_pipeline +- resource_group +- metadata +- trigger_request +- downstream_pipeline +- upstream_pipeline +generic_commit_statuses: +- user +- pipeline +- auto_canceled_by +- ci_stage +- needs variables: - project triggers: @@ -391,6 +504,7 @@ container_repositories: - project - name project: +- catalog_resource - external_status_checks - base_tags - project_topics @@ -399,7 +513,9 @@ project: - cluster - clusters - cluster_agents +- ci_access_project_authorizations - cluster_project +- workspaces - creator - cycle_analytics_stages - value_streams @@ -408,6 +524,7 @@ project: - project_namespace - management_clusters - boards +- application_setting - last_event - integrations - push_hooks_integrations @@ -432,6 +549,7 @@ project: - discord_integration - drone_ci_integration - emails_on_push_integration +- google_play_integration - pipelines_email_integration - mattermost_slash_commands_integration - shimo_integration @@ -466,12 +584,14 @@ project: - external_wiki_integration - mock_ci_integration - mock_monitoring_integration +- squash_tm_integration - forked_to_members - forked_from_project - forks - merge_requests - fork_merge_requests - issues +- work_items - labels - events - milestones @@ -527,6 +647,7 @@ project: - redirect_routes - statistics - container_repositories +- container_registry_data_repair_detail - uploads - file_uploads - import_state @@ -600,14 +721,15 @@ project: - project_registry - packages - package_files -- repository_files +- rpm_repository_files +- npm_metadata_caches - packages_cleanup_policy - alerting_setting - project_setting - webide_pipelines - reviews - incident_management_setting -- merge_trains +- merge_train_cars - designs - project_aliases - external_pull_requests @@ -618,6 +740,8 @@ project: - upstream_project_subscriptions - downstream_project_subscriptions - service_desk_setting +- service_desk_custom_email_verification +- service_desk_custom_email_credential - security_setting - import_failures - container_expiration_policy @@ -673,6 +797,7 @@ project: - sbom_occurrences - analytics_dashboards_configuration_project - analytics_dashboards_pointer +- design_management_repository award_emoji: - awardable - user @@ -759,6 +884,7 @@ incident_management_setting: - project merge_trains: - project +merge_train_cars: - merge_request boards: - group @@ -859,6 +985,8 @@ bulk_import_export: - group service_desk_setting: - file_template_project +service_desk_custom_email_verification: + - triggerer approvals: - user - merge_request @@ -890,3 +1018,22 @@ resource_iteration_events: iterations_cadence: - group - iterations +catalog_resource: + - project +approval_rules: + - users + - groups + - group_users + - security_orchestration_policy_configuration + - protected_branches + - approval_merge_request_rule_sources + - approval_merge_request_rules + - approval_project_rules_users + - approval_project_rules_protected_branches + - scan_result_policy_read +approval_project_rules_users: + - user + - approval_project_rule +approval_project_rules_protected_branches: + - protected_branch + - approval_project_rule diff --git a/spec/lib/gitlab/import_export/attribute_configuration_spec.rb b/spec/lib/gitlab/import_export/attribute_configuration_spec.rb index 572f809e43b..1d84cba3825 100644 --- a/spec/lib/gitlab/import_export/attribute_configuration_spec.rb +++ b/spec/lib/gitlab/import_export/attribute_configuration_spec.rb @@ -9,7 +9,7 @@ require 'spec_helper' # to be included as part of the export, or blacklist them using the import_export.yml configuration file. # Likewise, new models added to import_export.yml, will need to be added with their correspondent attributes # to this spec. -RSpec.describe 'Import/Export attribute configuration' do +RSpec.describe 'Import/Export attribute configuration', feature_category: :importers do include ConfigurationHelper let(:safe_attributes_file) { 'spec/lib/gitlab/import_export/safe_model_attributes.yml' } diff --git a/spec/lib/gitlab/import_export/attributes_finder_spec.rb b/spec/lib/gitlab/import_export/attributes_finder_spec.rb index 6536b895b2f..f12cbe4f82f 100644 --- a/spec/lib/gitlab/import_export/attributes_finder_spec.rb +++ b/spec/lib/gitlab/import_export/attributes_finder_spec.rb @@ -2,7 +2,7 @@ require 'fast_spec_helper' -RSpec.describe Gitlab::ImportExport::AttributesFinder do +RSpec.describe Gitlab::ImportExport::AttributesFinder, feature_category: :importers do describe '#find_root' do subject { described_class.new(config: config).find_root(model_key) } @@ -177,7 +177,8 @@ RSpec.describe Gitlab::ImportExport::AttributesFinder do end def setup_yaml(hash) - allow(YAML).to receive(:load_file).with(test_config).and_return(hash) + allow(YAML).to receive(:safe_load_file) + .with(test_config, aliases: true, permitted_classes: [Symbol]).and_return(hash) end end end @@ -207,6 +208,19 @@ RSpec.describe Gitlab::ImportExport::AttributesFinder do it { is_expected.to be_nil } end + + context 'when include_import_only_tree is true' do + subject { described_class.new(config: config).find_relations_tree(model_key, include_import_only_tree: true) } + + let(:config) do + { + tree: { project: { ci_pipelines: { stages: { builds: nil } } } }, + import_only_tree: { project: { ci_pipelines: { stages: { statuses: nil } } } } + } + end + + it { is_expected.to eq({ ci_pipelines: { stages: { builds: nil, statuses: nil } } }) } + end end describe '#find_excluded_keys' do diff --git a/spec/lib/gitlab/import_export/attributes_permitter_spec.rb b/spec/lib/gitlab/import_export/attributes_permitter_spec.rb index c748f966463..8089b40cae8 100644 --- a/spec/lib/gitlab/import_export/attributes_permitter_spec.rb +++ b/spec/lib/gitlab/import_export/attributes_permitter_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::ImportExport::AttributesPermitter do +RSpec.describe Gitlab::ImportExport::AttributesPermitter, feature_category: :importers do let(:yml_config) do <<-EOF tree: @@ -12,6 +12,15 @@ RSpec.describe Gitlab::ImportExport::AttributesPermitter do - milestones: - events: - :push_event_payload + - ci_pipelines: + - stages: + - :builds + + import_only_tree: + project: + - ci_pipelines: + - stages: + - :statuses included_attributes: labels: @@ -43,12 +52,16 @@ RSpec.describe Gitlab::ImportExport::AttributesPermitter do it 'builds permitted attributes hash' do expect(subject.permitted_attributes).to match( a_hash_including( - project: [:labels, :milestones], + project: [:labels, :milestones, :ci_pipelines], labels: [:priorities, :title, :description, :type], events: [:push_event_payload], milestones: [:events], priorities: [], - push_event_payload: [] + push_event_payload: [], + ci_pipelines: [:stages], + stages: [:builds, :statuses], + statuses: [], + builds: [] ) ) end @@ -129,6 +142,9 @@ RSpec.describe Gitlab::ImportExport::AttributesPermitter do :external_pull_request | true :external_pull_requests | true :statuses | true + :builds | true + :generic_commit_statuses | true + :bridges | true :ci_pipelines | true :stages | true :actions | true diff --git a/spec/lib/gitlab/import_export/base/relation_object_saver_spec.rb b/spec/lib/gitlab/import_export/base/relation_object_saver_spec.rb index a8b4b9a6f05..e42a1d0ff8b 100644 --- a/spec/lib/gitlab/import_export/base/relation_object_saver_spec.rb +++ b/spec/lib/gitlab/import_export/base/relation_object_saver_spec.rb @@ -82,24 +82,13 @@ RSpec.describe Gitlab::ImportExport::Base::RelationObjectSaver, feature_category it 'saves valid subrelations and logs invalid subrelation' do expect(relation_object.notes).to receive(:<<).twice.and_call_original expect(relation_object).to receive(:save).and_call_original - expect(Gitlab::Import::Logger) - .to receive(:info) - .with( - message: '[Project/Group Import] Invalid subrelation', - project_id: project.id, - relation_key: 'issues', - error_messages: "Project does not match noteable project" - ) saver.execute issue = project.issues.last - import_failure = project.import_failures.last expect(invalid_note.persisted?).to eq(false) expect(issue.notes.count).to eq(5) - expect(import_failure.source).to eq('RelationObjectSaver#save!') - expect(import_failure.exception_message).to eq('Project does not match noteable project') end context 'when invalid subrelation can still be persisted' do @@ -111,7 +100,6 @@ RSpec.describe Gitlab::ImportExport::Base::RelationObjectSaver, feature_category it 'saves the subrelation' do expect(approval_1.valid?).to eq(false) - expect(Gitlab::Import::Logger).not_to receive(:info) saver.execute @@ -128,24 +116,10 @@ RSpec.describe Gitlab::ImportExport::Base::RelationObjectSaver, feature_category let(:invalid_priority) { build(:label_priority, priority: -1) } let(:relation_object) { build(:group_label, group: importable, title: 'test', priorities: valid_priorities + [invalid_priority]) } - it 'logs invalid subrelation for a group' do - expect(Gitlab::Import::Logger) - .to receive(:info) - .with( - message: '[Project/Group Import] Invalid subrelation', - group_id: importable.id, - relation_key: 'labels', - error_messages: 'Priority must be greater than or equal to 0' - ) - + it 'saves relation without invalid subrelations' do saver.execute - label = importable.labels.last - import_failure = importable.import_failures.last - - expect(label.priorities.count).to eq(5) - expect(import_failure.source).to eq('RelationObjectSaver#save!') - expect(import_failure.exception_message).to eq('Priority must be greater than or equal to 0') + expect(importable.labels.last.priorities.count).to eq(5) end end end diff --git a/spec/lib/gitlab/import_export/command_line_util_spec.rb b/spec/lib/gitlab/import_export/command_line_util_spec.rb index f47f1ab58a8..91cfab1688a 100644 --- a/spec/lib/gitlab/import_export/command_line_util_spec.rb +++ b/spec/lib/gitlab/import_export/command_line_util_spec.rb @@ -2,13 +2,14 @@ require 'spec_helper' -RSpec.describe Gitlab::ImportExport::CommandLineUtil do +RSpec.describe Gitlab::ImportExport::CommandLineUtil, feature_category: :importers do include ExportFileHelper let(:path) { "#{Dir.tmpdir}/symlink_test" } let(:archive) { 'spec/fixtures/symlink_export.tar.gz' } let(:shared) { Gitlab::ImportExport::Shared.new(nil) } let(:tmpdir) { Dir.mktmpdir } + let(:archive_dir) { Dir.mktmpdir } subject do Class.new do @@ -25,20 +26,38 @@ RSpec.describe Gitlab::ImportExport::CommandLineUtil do before do FileUtils.mkdir_p(path) - subject.untar_zxf(archive: archive, dir: path) end after do FileUtils.rm_rf(path) + FileUtils.rm_rf(archive_dir) FileUtils.remove_entry(tmpdir) end - it 'has the right mask for project.json' do - expect(file_permissions("#{path}/project.json")).to eq(0755) # originally 777 - end - - it 'has the right mask for uploads' do - expect(file_permissions("#{path}/uploads")).to eq(0755) # originally 555 + shared_examples 'deletes symlinks' do |compression, decompression| + it 'deletes the symlinks', :aggregate_failures do + Dir.mkdir("#{tmpdir}/.git") + Dir.mkdir("#{tmpdir}/folder") + FileUtils.touch("#{tmpdir}/file.txt") + FileUtils.touch("#{tmpdir}/folder/file.txt") + FileUtils.touch("#{tmpdir}/.gitignore") + FileUtils.touch("#{tmpdir}/.git/config") + File.symlink('file.txt', "#{tmpdir}/.symlink") + File.symlink('file.txt', "#{tmpdir}/.git/.symlink") + File.symlink('file.txt', "#{tmpdir}/folder/.symlink") + archive = File.join(archive_dir, 'archive') + subject.public_send(compression, archive: archive, dir: tmpdir) + + subject.public_send(decompression, archive: archive, dir: archive_dir) + + expect(File.exist?("#{archive_dir}/file.txt")).to eq(true) + expect(File.exist?("#{archive_dir}/folder/file.txt")).to eq(true) + expect(File.exist?("#{archive_dir}/.gitignore")).to eq(true) + expect(File.exist?("#{archive_dir}/.git/config")).to eq(true) + expect(File.exist?("#{archive_dir}/.symlink")).to eq(false) + expect(File.exist?("#{archive_dir}/.git/.symlink")).to eq(false) + expect(File.exist?("#{archive_dir}/folder/.symlink")).to eq(false) + end end describe '#download_or_copy_upload' do @@ -228,12 +247,6 @@ RSpec.describe Gitlab::ImportExport::CommandLineUtil do end describe '#tar_cf' do - let(:archive_dir) { Dir.mktmpdir } - - after do - FileUtils.remove_entry(archive_dir) - end - it 'archives a folder without compression' do archive_file = File.join(archive_dir, 'archive.tar') @@ -256,12 +269,24 @@ RSpec.describe Gitlab::ImportExport::CommandLineUtil do end end - describe '#untar_xf' do - let(:archive_dir) { Dir.mktmpdir } + describe '#untar_zxf' do + it_behaves_like 'deletes symlinks', :tar_czf, :untar_zxf - after do - FileUtils.remove_entry(archive_dir) + it 'has the right mask for project.json' do + subject.untar_zxf(archive: archive, dir: path) + + expect(file_permissions("#{path}/project.json")).to eq(0755) # originally 777 + end + + it 'has the right mask for uploads' do + subject.untar_zxf(archive: archive, dir: path) + + expect(file_permissions("#{path}/uploads")).to eq(0755) # originally 555 end + end + + describe '#untar_xf' do + it_behaves_like 'deletes symlinks', :tar_cf, :untar_xf it 'extracts archive without decompression' do filename = 'archive.tar.gz' diff --git a/spec/lib/gitlab/import_export/config_spec.rb b/spec/lib/gitlab/import_export/config_spec.rb index 8f848af8bd3..2a52a0a2ff2 100644 --- a/spec/lib/gitlab/import_export/config_spec.rb +++ b/spec/lib/gitlab/import_export/config_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::ImportExport::Config do +RSpec.describe Gitlab::ImportExport::Config, feature_category: :importers do let(:yaml_file) { described_class.new } describe '#to_h' do @@ -21,7 +21,9 @@ RSpec.describe Gitlab::ImportExport::Config do end it 'parses default config' do - expected_keys = [:tree, :excluded_attributes, :included_attributes, :methods, :preloads, :export_reorders] + expected_keys = [ + :tree, :import_only_tree, :excluded_attributes, :included_attributes, :methods, :preloads, :export_reorders + ] expected_keys << :include_if_exportable if ee expect { subject }.not_to raise_error @@ -82,7 +84,7 @@ RSpec.describe Gitlab::ImportExport::Config do EOF end - let(:config_hash) { YAML.safe_load(config, [Symbol]) } + let(:config_hash) { YAML.safe_load(config, permitted_classes: [Symbol]) } before do allow_any_instance_of(described_class).to receive(:parse_yaml) do @@ -110,6 +112,7 @@ RSpec.describe Gitlab::ImportExport::Config do } } }, + import_only_tree: {}, included_attributes: { user: [:id] }, @@ -153,6 +156,7 @@ RSpec.describe Gitlab::ImportExport::Config do } } }, + import_only_tree: {}, included_attributes: { user: [:id, :name_ee] }, diff --git a/spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb b/spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb index f18d9e64f52..02419267f0e 100644 --- a/spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb +++ b/spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::ImportExport::FastHashSerializer, :with_license do +RSpec.describe Gitlab::ImportExport::FastHashSerializer, :with_license, feature_category: :importers do # FastHashSerializer#execute generates the hash which is not easily accessible # and includes `JSONBatchRelation` items which are serialized at this point. # Wrapping the result into JSON generating/parsing is for making @@ -125,13 +125,13 @@ RSpec.describe Gitlab::ImportExport::FastHashSerializer, :with_license do expect(subject.dig('ci_pipelines', 0, 'stages')).not_to be_empty end - it 'has pipeline statuses' do - expect(subject.dig('ci_pipelines', 0, 'stages', 0, 'statuses')).not_to be_empty + it 'has pipeline builds' do + expect(subject.dig('ci_pipelines', 0, 'stages', 0, 'builds')).not_to be_empty end it 'has pipeline builds' do builds_count = subject - .dig('ci_pipelines', 0, 'stages', 0, 'statuses') + .dig('ci_pipelines', 0, 'stages', 0, 'builds') .count { |hash| hash['type'] == 'Ci::Build' } expect(builds_count).to eq(1) @@ -141,8 +141,8 @@ RSpec.describe Gitlab::ImportExport::FastHashSerializer, :with_license do expect(subject['ci_pipelines']).not_to be_empty end - it 'has ci pipeline notes' do - expect(subject['ci_pipelines'].first['notes']).not_to be_empty + it 'has commit notes' do + expect(subject['commit_notes']).not_to be_empty end it 'has labels with no associations' do diff --git a/spec/lib/gitlab/import_export/fork_spec.rb b/spec/lib/gitlab/import_export/fork_spec.rb deleted file mode 100644 index 9d766eb3af1..00000000000 --- a/spec/lib/gitlab/import_export/fork_spec.rb +++ /dev/null @@ -1,59 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe 'forked project import' do - include ProjectForksHelper - - let(:user) { create(:user) } - let!(:project_with_repo) { create(:project, :repository, name: 'test-repo-restorer', path: 'test-repo-restorer') } - let!(:project) { create(:project, name: 'test-repo-restorer-no-repo', path: 'test-repo-restorer-no-repo') } - let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" } - let(:shared) { project.import_export_shared } - let(:forked_from_project) { create(:project, :repository) } - let(:forked_project) { fork_project(project_with_repo, nil, repository: true) } - let(:repo_saver) { Gitlab::ImportExport::RepoSaver.new(exportable: project_with_repo, shared: shared) } - let(:bundle_path) { File.join(shared.export_path, Gitlab::ImportExport.project_bundle_filename) } - - let(:repo_restorer) do - Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: bundle_path, shared: shared, importable: project) - end - - let!(:merge_request) do - create(:merge_request, source_project: forked_project, target_project: project_with_repo) - end - - let(:saver) do - Gitlab::ImportExport::Project::TreeSaver.new(project: project_with_repo, current_user: user, shared: shared) - end - - let(:restorer) do - Gitlab::ImportExport::Project::TreeRestorer.new(user: user, shared: shared, project: project) - end - - before do - stub_feature_flags(project_export_as_ndjson: false) - - allow_next_instance_of(Gitlab::ImportExport) do |instance| - allow(instance).to receive(:storage_path).and_return(export_path) - end - - saver.save # rubocop:disable Rails/SaveBang - repo_saver.save # rubocop:disable Rails/SaveBang - - repo_restorer.restore - restorer.restore - end - - after do - FileUtils.rm_rf(export_path) - project_with_repo.repository.remove - project.repository.remove - end - - it 'can access the MR', :sidekiq_might_not_need_inline do - project.merge_requests.first.fetch_ref! - - expect(project.repository.ref_exists?('refs/merge-requests/1/head')).to be_truthy - end -end diff --git a/spec/lib/gitlab/import_export/group/relation_tree_restorer_spec.rb b/spec/lib/gitlab/import_export/group/relation_tree_restorer_spec.rb index 5e84284a060..495cefa002a 100644 --- a/spec/lib/gitlab/import_export/group/relation_tree_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/group/relation_tree_restorer_spec.rb @@ -9,25 +9,31 @@ require 'spec_helper' -RSpec.describe Gitlab::ImportExport::Group::RelationTreeRestorer do +RSpec.describe Gitlab::ImportExport::Group::RelationTreeRestorer, feature_category: :importers do let(:group) { create(:group).tap { |g| g.add_owner(user) } } let(:importable) { create(:group, parent: group) } include_context 'relation tree restorer shared context' do - let(:importable_name) { nil } + let(:importable_name) { 'groups/4353' } end - let(:path) { 'spec/fixtures/lib/gitlab/import_export/group_exports/no_children/group.json' } + let(:path) { Rails.root.join('spec/fixtures/lib/gitlab/import_export/group_exports/no_children/tree') } let(:relation_reader) do - Gitlab::ImportExport::Json::LegacyReader::File.new( - path, - relation_names: reader.group_relation_names) + Gitlab::ImportExport::Json::NdjsonReader.new(path) end let(:reader) do Gitlab::ImportExport::Reader.new( shared: shared, - config: Gitlab::ImportExport::Config.new(config: Gitlab::ImportExport.legacy_group_config_file).to_h + config: Gitlab::ImportExport::Config.new(config: Gitlab::ImportExport.group_config_file).to_h + ) + end + + let(:members_mapper) do + Gitlab::ImportExport::MembersMapper.new( + exported_members: relation_reader.consume_relation(importable_name, 'members').map(&:first), + user: user, + importable: importable ) end @@ -41,7 +47,7 @@ RSpec.describe Gitlab::ImportExport::Group::RelationTreeRestorer do relation_factory: Gitlab::ImportExport::Group::RelationFactory, reader: reader, importable: importable, - importable_path: nil, + importable_path: importable_name, importable_attributes: attributes ) end @@ -60,4 +66,74 @@ RSpec.describe Gitlab::ImportExport::Group::RelationTreeRestorer do subject end + + describe 'relation object saving' do + before do + allow(shared.logger).to receive(:info).and_call_original + allow(relation_reader).to receive(:consume_relation).and_call_original + + allow(relation_reader) + .to receive(:consume_relation) + .with(importable_name, 'labels') + .and_return([[label, 0]]) + end + + context 'when relation object is new' do + context 'when relation object has invalid subrelations' do + let(:label) do + { + 'title' => 'test', + 'priorities' => [LabelPriority.new, LabelPriority.new], + 'type' => 'GroupLabel' + } + end + + it 'logs invalid subrelations' do + expect(shared.logger) + .to receive(:info) + .with( + message: '[Project/Group Import] Invalid subrelation', + group_id: importable.id, + relation_key: 'labels', + error_messages: "Project can't be blank, Priority can't be blank, and Priority is not a number" + ) + + subject + + label = importable.labels.first + failure = importable.import_failures.first + + expect(importable.import_failures.count).to eq(2) + expect(label.title).to eq('test') + expect(failure.exception_class).to eq('ActiveRecord::RecordInvalid') + expect(failure.source).to eq('RelationTreeRestorer#save_relation_object') + expect(failure.exception_message) + .to eq("Project can't be blank, Priority can't be blank, and Priority is not a number") + end + end + end + + context 'when relation object is persisted' do + context 'when relation object is invalid' do + let(:label) { create(:group_label, group: group, title: 'test') } + + it 'saves import failure with nested errors' do + label.priorities << [LabelPriority.new, LabelPriority.new] + + subject + + failure = importable.import_failures.first + + expect(importable.labels.count).to eq(0) + expect(importable.import_failures.count).to eq(1) + expect(failure.exception_class).to eq('ActiveRecord::RecordInvalid') + expect(failure.source).to eq('process_relation_item!') + expect(failure.exception_message) + .to eq("Validation failed: Priorities is invalid, Project can't be blank, Priority can't be blank, " \ + "Priority is not a number, Project can't be blank, Priority can't be blank, " \ + "Priority is not a number") + end + end + end + end end diff --git a/spec/lib/gitlab/import_export/group/tree_restorer_spec.rb b/spec/lib/gitlab/import_export/group/tree_restorer_spec.rb index aa30e24296e..a6afd0a36ec 100644 --- a/spec/lib/gitlab/import_export/group/tree_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/group/tree_restorer_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::ImportExport::Group::TreeRestorer, feature: :subgroups do +RSpec.describe Gitlab::ImportExport::Group::TreeRestorer, feature: :subgroups, feature_category: :importers do include ImportExport::CommonUtil shared_examples 'group restoration' do @@ -171,7 +171,7 @@ RSpec.describe Gitlab::ImportExport::Group::TreeRestorer, feature: :subgroups do allow(shared).to receive(:export_path).and_return(tmpdir) expect(group_tree_restorer.restore).to eq(false) - expect(shared.errors).to include('Incorrect JSON format') + expect(shared.errors).to include('Invalid file') end end end diff --git a/spec/lib/gitlab/import_export/import_export_equivalence_spec.rb b/spec/lib/gitlab/import_export/import_export_equivalence_spec.rb deleted file mode 100644 index 6c997dc1361..00000000000 --- a/spec/lib/gitlab/import_export/import_export_equivalence_spec.rb +++ /dev/null @@ -1,67 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -# Verifies that given an exported project meta-data tree, when importing this -# tree and then exporting it again, we should obtain the initial tree. -# -# This equivalence only works up to a certain extent, for instance we need -# to ignore: -# -# - row IDs and foreign key IDs -# - some timestamps -# - randomly generated fields like tokens -# -# as these are expected to change between import/export cycles. -RSpec.describe Gitlab::ImportExport, feature_category: :importers do - include ImportExport::CommonUtil - include ConfigurationHelper - include ImportExport::ProjectTreeExpectations - - let(:json_fixture) { 'complex' } - - before do - stub_feature_flags(project_export_as_ndjson: false) - end - - it 'yields the initial tree when importing and exporting it again' do - project = create(:project) - user = create(:user, :admin) - - # We first generate a test fixture dynamically from a seed-fixture, so as to - # account for any fields in the initial fixture that are missing and set to - # defaults during import (ideally we should have realistic test fixtures - # that "honestly" represent exports) - expect( - restore_then_save_project( - project, - user, - import_path: seed_fixture_path, - export_path: test_fixture_path) - ).to be true - # Import, then export again from the generated fixture. Any residual changes - # in the JSON will count towards comparison i.e. test failures. - expect( - restore_then_save_project( - project, - user, - import_path: test_fixture_path, - export_path: test_tmp_path) - ).to be true - - imported_json = Gitlab::Json.parse(File.read("#{test_fixture_path}/project.json")) - exported_json = Gitlab::Json.parse(File.read("#{test_tmp_path}/project.json")) - - assert_relations_match(imported_json, exported_json) - end - - private - - def seed_fixture_path - "#{fixtures_path}/#{json_fixture}" - end - - def test_fixture_path - "#{test_tmp_path}/#{json_fixture}" - end -end diff --git a/spec/lib/gitlab/import_export/import_failure_service_spec.rb b/spec/lib/gitlab/import_export/import_failure_service_spec.rb index 51f1fc9c6a2..30d16347828 100644 --- a/spec/lib/gitlab/import_export/import_failure_service_spec.rb +++ b/spec/lib/gitlab/import_export/import_failure_service_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::ImportExport::ImportFailureService do +RSpec.describe Gitlab::ImportExport::ImportFailureService, feature_category: :importers do let(:importable) { create(:project, :builds_enabled, :issues_disabled, name: 'project', path: 'project') } let(:label) { create(:label) } let(:subject) { described_class.new(importable) } diff --git a/spec/lib/gitlab/import_export/json/legacy_reader/file_spec.rb b/spec/lib/gitlab/import_export/json/legacy_reader/file_spec.rb deleted file mode 100644 index 793b3ebfb9e..00000000000 --- a/spec/lib/gitlab/import_export/json/legacy_reader/file_spec.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require_relative 'shared_example' - -RSpec.describe Gitlab::ImportExport::Json::LegacyReader::File do - it_behaves_like 'import/export json legacy reader' do - let(:valid_path) { 'spec/fixtures/lib/gitlab/import_export/light/project.json' } - let(:data) { valid_path } - let(:json_data) { Gitlab::Json.parse(File.read(valid_path)) } - end - - describe '#exist?' do - let(:legacy_reader) do - described_class.new(path, relation_names: []) - end - - subject { legacy_reader.exist? } - - context 'given valid path' do - let(:path) { 'spec/fixtures/lib/gitlab/import_export/light/project.json' } - - it { is_expected.to be true } - end - - context 'given invalid path' do - let(:path) { 'spec/non-existing-folder/do-not-create-this-file.json' } - - it { is_expected.to be false } - end - end -end diff --git a/spec/lib/gitlab/import_export/json/legacy_reader/hash_spec.rb b/spec/lib/gitlab/import_export/json/legacy_reader/hash_spec.rb deleted file mode 100644 index 57d66dc0f50..00000000000 --- a/spec/lib/gitlab/import_export/json/legacy_reader/hash_spec.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require_relative 'shared_example' - -RSpec.describe Gitlab::ImportExport::Json::LegacyReader::Hash do - it_behaves_like 'import/export json legacy reader' do - let(:path) { 'spec/fixtures/lib/gitlab/import_export/light/project.json' } - - # the hash is modified by the `LegacyReader` - # we need to deep-dup it - let(:json_data) { Gitlab::Json.parse(File.read(path)) } - let(:data) { Gitlab::Json.parse(File.read(path)) } - end - - describe '#exist?' do - let(:legacy_reader) do - described_class.new(tree_hash, relation_names: []) - end - - subject { legacy_reader.exist? } - - context 'tree_hash is nil' do - let(:tree_hash) { nil } - - it { is_expected.to be_falsey } - end - - context 'tree_hash presents' do - let(:tree_hash) { { "issues": [] } } - - it { is_expected.to be_truthy } - end - end -end diff --git a/spec/lib/gitlab/import_export/json/legacy_reader/shared_example.rb b/spec/lib/gitlab/import_export/json/legacy_reader/shared_example.rb deleted file mode 100644 index 3e9bd3fe741..00000000000 --- a/spec/lib/gitlab/import_export/json/legacy_reader/shared_example.rb +++ /dev/null @@ -1,102 +0,0 @@ -# frozen_string_literal: true - -RSpec.shared_examples 'import/export json legacy reader' do - let(:relation_names) { [] } - - let(:legacy_reader) do - described_class.new( - data, - relation_names: relation_names, - allowed_path: "project") - end - - describe '#consume_attributes' do - context 'when valid path is passed' do - subject { legacy_reader.consume_attributes("project") } - - context 'no excluded attributes' do - let(:relation_names) { [] } - - it 'returns the whole tree from parsed JSON' do - expect(subject).to eq(json_data) - end - end - - context 'some attributes are excluded' do - let(:relation_names) { %w[milestones labels] } - - it 'returns hash without excluded attributes and relations' do - expect(subject).not_to include('milestones', 'labels') - end - end - end - - context 'when invalid path is passed' do - it 'raises an exception' do - expect { legacy_reader.consume_attributes("invalid-path") } - .to raise_error(ArgumentError) - end - end - end - - describe '#consume_relation' do - context 'when valid path is passed' do - let(:key) { 'labels' } - - subject { legacy_reader.consume_relation("project", key) } - - context 'key has not been consumed' do - it 'returns an Enumerator' do - expect(subject).to be_an_instance_of(Enumerator) - end - - context 'value is nil' do - before do - expect(legacy_reader).to receive(:relations).and_return({ key => nil }) - end - - it 'yields nothing to the Enumerator' do - expect(subject.to_a).to eq([]) - end - end - - context 'value is an array' do - before do - expect(legacy_reader).to receive(:relations).and_return({ key => %w[label1 label2] }) - end - - it 'yields every relation value to the Enumerator' do - expect(subject.to_a).to eq([['label1', 0], ['label2', 1]]) - end - end - - context 'value is not array' do - before do - expect(legacy_reader).to receive(:relations).and_return({ key => 'non-array value' }) - end - - it 'yields the value with index 0 to the Enumerator' do - expect(subject.to_a).to eq([['non-array value', 0]]) - end - end - end - - context 'key has been consumed' do - before do - legacy_reader.consume_relation("project", key).first - end - - it 'yields nothing to the Enumerator' do - expect(subject.to_a).to eq([]) - end - end - end - - context 'when invalid path is passed' do - it 'raises an exception' do - expect { legacy_reader.consume_relation("invalid") } - .to raise_error(ArgumentError) - end - end - end -end diff --git a/spec/lib/gitlab/import_export/json/legacy_writer_spec.rb b/spec/lib/gitlab/import_export/json/legacy_writer_spec.rb deleted file mode 100644 index e8ecd98b1e1..00000000000 --- a/spec/lib/gitlab/import_export/json/legacy_writer_spec.rb +++ /dev/null @@ -1,101 +0,0 @@ -# frozen_string_literal: true - -require 'fast_spec_helper' - -RSpec.describe Gitlab::ImportExport::Json::LegacyWriter do - let(:path) { "#{Dir.tmpdir}/legacy_writer_spec/test.json" } - - subject do - described_class.new(path, allowed_path: "project") - end - - after do - FileUtils.rm_rf(path) - end - - describe "#write_attributes" do - it "writes correct json" do - expected_hash = { "key" => "value_1", "key_1" => "value_2" } - subject.write_attributes("project", expected_hash) - - expect(subject_json).to eq(expected_hash) - end - - context 'when invalid path is used' do - it 'raises an exception' do - expect { subject.write_attributes("invalid", { "key" => "value" }) } - .to raise_error(ArgumentError) - end - end - end - - describe "#write_relation" do - context "when key is already written" do - it "raises exception" do - subject.write_relation("project", "key", "old value") - - expect { subject.write_relation("project", "key", "new value") } - .to raise_exception("key 'key' already written") - end - end - - context "when key is not already written" do - context "when multiple key value pairs are stored" do - it "writes correct json" do - expected_hash = { "key" => "value_1", "key_1" => "value_2" } - expected_hash.each do |key, value| - subject.write_relation("project", key, value) - end - - expect(subject_json).to eq(expected_hash) - end - end - end - - context 'when invalid path is used' do - it 'raises an exception' do - expect { subject.write_relation("invalid", "key", "value") } - .to raise_error(ArgumentError) - end - end - end - - describe "#write_relation_array" do - context 'when array is used' do - it 'writes correct json' do - subject.write_relation_array("project", "key", ["value"]) - - expect(subject_json).to eq({ "key" => ["value"] }) - end - end - - context 'when enumerable is used' do - it 'writes correct json' do - values = %w(value1 value2) - - enumerator = Enumerator.new do |items| - values.each { |value| items << value } - end - - subject.write_relation_array("project", "key", enumerator) - - expect(subject_json).to eq({ "key" => values }) - end - end - - context "when key is already written" do - it "raises an exception" do - subject.write_relation_array("project", "key", %w(old_value)) - - expect { subject.write_relation_array("project", "key", %w(new_value)) } - .to raise_error(ArgumentError) - end - end - end - - def subject_json - subject.close - - ::JSON.parse(File.read(subject.path)) - end -end diff --git a/spec/lib/gitlab/import_export/json/ndjson_reader_spec.rb b/spec/lib/gitlab/import_export/json/ndjson_reader_spec.rb index 0ca4c4ccc87..98afe01c08b 100644 --- a/spec/lib/gitlab/import_export/json/ndjson_reader_spec.rb +++ b/spec/lib/gitlab/import_export/json/ndjson_reader_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::ImportExport::Json::NdjsonReader do +RSpec.describe Gitlab::ImportExport::Json::NdjsonReader, feature_category: :importers do include ImportExport::CommonUtil let(:fixture) { 'spec/fixtures/lib/gitlab/import_export/light/tree' } @@ -26,14 +26,6 @@ RSpec.describe Gitlab::ImportExport::Json::NdjsonReader do end end - describe '#legacy?' do - let(:dir_path) { fixture } - - subject { ndjson_reader.legacy? } - - it { is_expected.to be false } - end - describe '#consume_attributes' do let(:dir_path) { fixture } @@ -42,6 +34,20 @@ RSpec.describe Gitlab::ImportExport::Json::NdjsonReader do it 'returns the whole root tree from parsed JSON' do expect(subject).to eq(root_tree) end + + context 'when project.json is symlink' do + it 'raises error an error' do + Dir.mktmpdir do |tmpdir| + FileUtils.touch(File.join(tmpdir, 'passwd')) + File.symlink(File.join(tmpdir, 'passwd'), File.join(tmpdir, 'project.json')) + + ndjson_reader = described_class.new(tmpdir) + + expect { ndjson_reader.consume_attributes(importable_path) } + .to raise_error(Gitlab::ImportExport::Error, 'Invalid file') + end + end + end end describe '#consume_relation' do @@ -91,6 +97,22 @@ RSpec.describe Gitlab::ImportExport::Json::NdjsonReader do end end + context 'when relation file is a symlink' do + it 'yields nothing to the Enumerator' do + Dir.mktmpdir do |tmpdir| + Dir.mkdir(File.join(tmpdir, 'project')) + File.write(File.join(tmpdir, 'passwd'), "{}\n{}") + File.symlink(File.join(tmpdir, 'passwd'), File.join(tmpdir, 'project', 'issues.ndjson')) + + ndjson_reader = described_class.new(tmpdir) + + result = ndjson_reader.consume_relation(importable_path, 'issues') + + expect(result.to_a).to eq([]) + end + end + end + context 'relation file is empty' do let(:key) { 'empty' } diff --git a/spec/lib/gitlab/import_export/json/streaming_serializer_spec.rb b/spec/lib/gitlab/import_export/json/streaming_serializer_spec.rb index 103d3512e8b..f4c9189030b 100644 --- a/spec/lib/gitlab/import_export/json/streaming_serializer_spec.rb +++ b/spec/lib/gitlab/import_export/json/streaming_serializer_spec.rb @@ -28,7 +28,7 @@ RSpec.describe Gitlab::ImportExport::Json::StreamingSerializer, feature_category let(:exportable_path) { 'project' } let(:logger) { Gitlab::Export::Logger.build } - let(:json_writer) { instance_double('Gitlab::ImportExport::Json::LegacyWriter') } + let(:json_writer) { instance_double('Gitlab::ImportExport::Json::NdjsonWriter') } let(:hash) { { name: exportable.name, description: exportable.description }.stringify_keys } let(:include) { [] } let(:custom_orderer) { nil } diff --git a/spec/lib/gitlab/import_export/model_configuration_spec.rb b/spec/lib/gitlab/import_export/model_configuration_spec.rb index 4f01f470ce7..8e5fe96f3b4 100644 --- a/spec/lib/gitlab/import_export/model_configuration_spec.rb +++ b/spec/lib/gitlab/import_export/model_configuration_spec.rb @@ -5,11 +5,11 @@ require 'spec_helper' # Part of the test security suite for the Import/Export feature # Finds if a new model has been added that can potentially be part of the Import/Export # If it finds a new model, it will show a +failure_message+ with the options available. -RSpec.describe 'Import/Export model configuration' do +RSpec.describe 'Import/Export model configuration', feature_category: :importers do include ConfigurationHelper let(:all_models_yml) { 'spec/lib/gitlab/import_export/all_models.yml' } - let(:all_models_hash) { YAML.load_file(all_models_yml) } + let(:all_models_hash) { YAML.safe_load_file(all_models_yml, aliases: true) } let(:current_models) { setup_models } let(:model_names) { relation_names_for(:project) } diff --git a/spec/lib/gitlab/import_export/project/export_task_spec.rb b/spec/lib/gitlab/import_export/project/export_task_spec.rb index 3dd1e9257cc..95971d08175 100644 --- a/spec/lib/gitlab/import_export/project/export_task_spec.rb +++ b/spec/lib/gitlab/import_export/project/export_task_spec.rb @@ -10,14 +10,14 @@ RSpec.describe Gitlab::ImportExport::Project::ExportTask, :silence_stdout do let(:measurement_enabled) { false } let(:file_path) { 'spec/fixtures/gitlab/import_export/test_project_export.tar.gz' } let(:project) { create(:project, creator: user, namespace: user.namespace) } - let(:project_name) { project.name } + let(:project_path) { project.path } let(:rake_task) { described_class.new(task_params) } let(:task_params) do { username: username, namespace_path: namespace_path, - project_path: project_name, + project_path: project_path, file_path: file_path, measurement_enabled: measurement_enabled } @@ -48,10 +48,10 @@ RSpec.describe Gitlab::ImportExport::Project::ExportTask, :silence_stdout do end context 'when project is not found' do - let(:project_name) { 'invalid project name' } + let(:project_path) { 'invalid project path' } it 'logs an error' do - expect { subject }.to output(/Project with path: #{project_name} was not found. Please provide correct project path/).to_stdout + expect { subject }.to output(/Project with path: #{project_path} was not found. Please provide correct project path/).to_stdout end it 'returns false' do diff --git a/spec/lib/gitlab/import_export/project/exported_relations_merger_spec.rb b/spec/lib/gitlab/import_export/project/exported_relations_merger_spec.rb index d70e89c6856..f8018e75879 100644 --- a/spec/lib/gitlab/import_export/project/exported_relations_merger_spec.rb +++ b/spec/lib/gitlab/import_export/project/exported_relations_merger_spec.rb @@ -64,8 +64,8 @@ RSpec.describe Gitlab::ImportExport::Project::ExportedRelationsMerger do expect(result).to eq(false) expect(shared.errors).to match_array( [ - "undefined method `export_file' for nil:NilClass", - "undefined method `export_file' for nil:NilClass" + /^undefined method `export_file' for nil:NilClass/, + /^undefined method `export_file' for nil:NilClass/ ] ) end diff --git a/spec/lib/gitlab/import_export/project/import_task_spec.rb b/spec/lib/gitlab/import_export/project/import_task_spec.rb index c847224cb9b..693f1984ce8 100644 --- a/spec/lib/gitlab/import_export/project/import_task_spec.rb +++ b/spec/lib/gitlab/import_export/project/import_task_spec.rb @@ -2,7 +2,7 @@ require 'rake_helper' -RSpec.describe Gitlab::ImportExport::Project::ImportTask, :request_store, :silence_stdout do +RSpec.describe Gitlab::ImportExport::Project::ImportTask, :request_store, :silence_stdout, feature_category: :importers do let(:username) { 'root' } let(:namespace_path) { username } let!(:user) { create(:user, username: username) } diff --git a/spec/lib/gitlab/import_export/project/object_builder_spec.rb b/spec/lib/gitlab/import_export/project/object_builder_spec.rb index 189b798c2e8..5fa8590e8fd 100644 --- a/spec/lib/gitlab/import_export/project/object_builder_spec.rb +++ b/spec/lib/gitlab/import_export/project/object_builder_spec.rb @@ -86,13 +86,16 @@ RSpec.describe Gitlab::ImportExport::Project::ObjectBuilder do 'group' => group)).to eq(group_label) end - it 'creates a new label' do + it 'creates a new project label' do label = described_class.build(Label, 'title' => 'group label', 'project' => project, - 'group' => project.group) + 'group' => project.group, + 'group_id' => project.group.id) expect(label.persisted?).to be true + expect(label).to be_an_instance_of(ProjectLabel) + expect(label.group_id).to be_nil end end diff --git a/spec/lib/gitlab/import_export/project/relation_tree_restorer_spec.rb b/spec/lib/gitlab/import_export/project/relation_tree_restorer_spec.rb index 6053df8ba97..180a6b6ff0a 100644 --- a/spec/lib/gitlab/import_export/project/relation_tree_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/project/relation_tree_restorer_spec.rb @@ -50,58 +50,24 @@ RSpec.describe Gitlab::ImportExport::Project::RelationTreeRestorer, feature_cate expect(project.custom_attributes.count).to eq(2) expect(project.project_badges.count).to eq(2) expect(project.snippets.count).to eq(1) + expect(project.commit_notes.count).to eq(3) end end end - context 'with legacy reader' do - let(:path) { 'spec/fixtures/lib/gitlab/import_export/complex/project.json' } - let(:relation_reader) do - Gitlab::ImportExport::Json::LegacyReader::File.new( - path, - relation_names: reader.project_relation_names, - allowed_path: 'project' - ) - end - - let(:attributes) { relation_reader.consume_attributes('project') } - - it_behaves_like 'import project successfully' - - context 'with logging of relations creation' do - let_it_be(:group) { create(:group).tap { |g| g.add_maintainer(user) } } - let_it_be(:importable) do - create(:project, :builds_enabled, :issues_disabled, name: 'project', path: 'project', group: group) - end - - it 'logs top-level relation creation' do - expect(shared.logger) - .to receive(:info) - .with(hash_including(message: '[Project/Group Import] Created new object relation')) - .at_least(:once) - - subject - end - end - end - - context 'with ndjson reader' do + context 'when inside a group' do let(:path) { 'spec/fixtures/lib/gitlab/import_export/complex/tree' } let(:relation_reader) { Gitlab::ImportExport::Json::NdjsonReader.new(path) } - it_behaves_like 'import project successfully' - - context 'when inside a group' do - let_it_be(:group) do - create(:group, :disabled_and_unoverridable).tap { |g| g.add_maintainer(user) } - end - - before do - importable.update!(shared_runners_enabled: false, group: group) - end + let_it_be(:group) do + create(:group, :disabled_and_unoverridable).tap { |g| g.add_maintainer(user) } + end - it_behaves_like 'import project successfully' + before do + importable.update!(shared_runners_enabled: false, group: group) end + + it_behaves_like 'import project successfully' end context 'with invalid relations' do diff --git a/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb b/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb index 125d1736b9b..5aa16f9508d 100644 --- a/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb @@ -12,7 +12,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i let(:shared) { project.import_export_shared } - RSpec.shared_examples 'project tree restorer work properly' do |reader, ndjson_enabled| + RSpec.shared_examples 'project tree restorer work properly' do describe 'restore project tree' do before_all do # Using an admin for import, so we can check assignment of existing members @@ -27,10 +27,9 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i @shared = @project.import_export_shared stub_all_feature_flags - stub_feature_flags(project_import_ndjson: ndjson_enabled) setup_import_export_config('complex') - setup_reader(reader) + setup_reader allow_any_instance_of(Repository).to receive(:fetch_source_branch!).and_return(true) allow_any_instance_of(Gitlab::Git::Repository).to receive(:branch_exists?).and_return(false) @@ -295,6 +294,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i it 'has project labels' do expect(ProjectLabel.count).to eq(3) + expect(ProjectLabel.pluck(:group_id).compact).to be_empty end it 'has merge request approvals' do @@ -528,7 +528,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i it 'has the correct number of pipelines and statuses' do expect(@project.ci_pipelines.size).to eq(7) - @project.ci_pipelines.order(:id).zip([2, 0, 2, 2, 2, 2, 0]) + @project.ci_pipelines.order(:id).zip([2, 0, 2, 3, 2, 2, 0]) .each do |(pipeline, expected_status_size)| expect(pipeline.statuses.size).to eq(expected_status_size) end @@ -548,8 +548,16 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i expect(Ci::Stage.all).to all(have_attributes(pipeline_id: a_value > 0)) end - it 'restores statuses' do - expect(CommitStatus.all.count).to be 10 + it 'restores builds' do + expect(Ci::Build.all.count).to be 7 + end + + it 'restores bridges' do + expect(Ci::Bridge.all.count).to be 1 + end + + it 'restores generic commit statuses' do + expect(GenericCommitStatus.all.count).to be 1 end it 'correctly restores association between a stage and a job' do @@ -574,6 +582,10 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i expect(@project.import_failures.size).to eq 0 end end + + it 'restores commit notes' do + expect(@project.commit_notes.count).to eq(3) + end end end @@ -593,23 +605,15 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i end end - context 'project.json file access check' do + context 'when expect tree structure is not present in the export path' do let(:user) { create(:user) } - let!(:project) { create(:project, :builds_disabled, :issues_disabled, name: 'project', path: 'project') } - let(:project_tree_restorer) do - described_class.new(user: user, shared: shared, project: project) - end - - let(:restored_project_json) { project_tree_restorer.restore } + let_it_be(:project) { create(:project, :builds_disabled, :issues_disabled, name: 'project', path: 'project') } - it 'does not read a symlink' do - Dir.mktmpdir do |tmpdir| - setup_symlink(tmpdir, 'project.json') - allow(shared).to receive(:export_path).and_call_original + it 'fails to restore the project' do + result = described_class.new(user: user, shared: shared, project: project).restore - expect(project_tree_restorer.restore).to eq(false) - expect(shared.errors).to include('invalid import format') - end + expect(result).to eq(false) + expect(shared.errors).to include('invalid import format') end end @@ -622,7 +626,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i context 'with a simple project' do before do setup_import_export_config('light') - setup_reader(reader) + setup_reader expect(restored_project_json).to eq(true) end @@ -657,7 +661,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i context 'multiple pipelines reference the same external pull request' do before do setup_import_export_config('multi_pipeline_ref_one_external_pr') - setup_reader(reader) + setup_reader expect(restored_project_json).to eq(true) end @@ -685,7 +689,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i before do setup_import_export_config('light') - setup_reader(reader) + setup_reader expect(project).to receive(:merge_requests).and_call_original expect(project).to receive(:merge_requests).and_raise(exception) @@ -702,7 +706,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i before do setup_import_export_config('light') - setup_reader(reader) + setup_reader expect(project).to receive(:merge_requests).and_call_original expect(project).to receive(:merge_requests).and_raise(exception) @@ -734,7 +738,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i context 'when the project has overridden params in import data' do before do setup_import_export_config('light') - setup_reader(reader) + setup_reader end it 'handles string versions of visibility_level' do @@ -800,7 +804,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i before do setup_import_export_config('group') - setup_reader(reader) + setup_reader expect(restored_project_json).to eq(true) end @@ -836,7 +840,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i before do setup_import_export_config('light') - setup_reader(reader) + setup_reader end it 'imports labels' do @@ -872,7 +876,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i before do setup_import_export_config('milestone-iid') - setup_reader(reader) + setup_reader end it 'preserves the project milestone IID' do @@ -888,7 +892,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i context 'with external authorization classification labels' do before do setup_import_export_config('light') - setup_reader(reader) + setup_reader end it 'converts empty external classification authorization labels to nil' do @@ -915,76 +919,80 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i described_class.new(user: user, shared: shared, project: project) end - before do - allow_any_instance_of(Gitlab::ImportExport::Json::LegacyReader::File).to receive(:exist?).and_return(true) - allow_any_instance_of(Gitlab::ImportExport::Json::NdjsonReader).to receive(:exist?).and_return(false) - allow_any_instance_of(Gitlab::ImportExport::Json::LegacyReader::File).to receive(:tree_hash) { tree_hash } - end - - context 'no group visibility' do - let(:visibility) { Gitlab::VisibilityLevel::PRIVATE } + describe 'visibility level' do + before do + setup_import_export_config('light') - it 'uses the project visibility' do - expect(restorer.restore).to eq(true) - expect(restorer.project.visibility_level).to eq(visibility) + allow_next_instance_of(Gitlab::ImportExport::Json::NdjsonReader) do |relation_reader| + allow(relation_reader).to receive(:consume_attributes).and_return(tree_hash) + end end - end - context 'with restricted internal visibility' do - describe 'internal project' do - let(:visibility) { Gitlab::VisibilityLevel::INTERNAL } - - it 'uses private visibility' do - stub_application_setting(restricted_visibility_levels: [Gitlab::VisibilityLevel::INTERNAL]) + context 'no group visibility' do + let(:visibility) { Gitlab::VisibilityLevel::PRIVATE } + it 'uses the project visibility' do expect(restorer.restore).to eq(true) - expect(restorer.project.visibility_level).to eq(Gitlab::VisibilityLevel::PRIVATE) + expect(restorer.project.visibility_level).to eq(visibility) end end - end - context 'with group visibility' do - before do - group = create(:group, visibility_level: group_visibility) - group.add_members([user], GroupMember::MAINTAINER) - project.update!(group: group) - end + context 'with restricted internal visibility' do + describe 'internal project' do + let(:visibility) { Gitlab::VisibilityLevel::INTERNAL } - context 'private group visibility' do - let(:group_visibility) { Gitlab::VisibilityLevel::PRIVATE } - let(:visibility) { Gitlab::VisibilityLevel::PUBLIC } + it 'uses private visibility' do + stub_application_setting(restricted_visibility_levels: [Gitlab::VisibilityLevel::INTERNAL]) - it 'uses the group visibility' do - expect(restorer.restore).to eq(true) - expect(restorer.project.visibility_level).to eq(group_visibility) + expect(restorer.restore).to eq(true) + expect(restorer.project.visibility_level).to eq(Gitlab::VisibilityLevel::PRIVATE) + end end end - context 'public group visibility' do - let(:group_visibility) { Gitlab::VisibilityLevel::PUBLIC } - let(:visibility) { Gitlab::VisibilityLevel::PRIVATE } + context 'with group visibility' do + before do + group = create(:group, visibility_level: group_visibility) + group.add_members([user], GroupMember::MAINTAINER) + project.update!(group: group) + end - it 'uses the project visibility' do - expect(restorer.restore).to eq(true) - expect(restorer.project.visibility_level).to eq(visibility) + context 'private group visibility' do + let(:group_visibility) { Gitlab::VisibilityLevel::PRIVATE } + let(:visibility) { Gitlab::VisibilityLevel::PUBLIC } + + it 'uses the group visibility' do + expect(restorer.restore).to eq(true) + expect(restorer.project.visibility_level).to eq(group_visibility) + end end - end - context 'internal group visibility' do - let(:group_visibility) { Gitlab::VisibilityLevel::INTERNAL } - let(:visibility) { Gitlab::VisibilityLevel::PUBLIC } + context 'public group visibility' do + let(:group_visibility) { Gitlab::VisibilityLevel::PUBLIC } + let(:visibility) { Gitlab::VisibilityLevel::PRIVATE } - it 'uses the group visibility' do - expect(restorer.restore).to eq(true) - expect(restorer.project.visibility_level).to eq(group_visibility) + it 'uses the project visibility' do + expect(restorer.restore).to eq(true) + expect(restorer.project.visibility_level).to eq(visibility) + end end - context 'with restricted internal visibility' do - it 'sets private visibility' do - stub_application_setting(restricted_visibility_levels: [Gitlab::VisibilityLevel::INTERNAL]) + context 'internal group visibility' do + let(:group_visibility) { Gitlab::VisibilityLevel::INTERNAL } + let(:visibility) { Gitlab::VisibilityLevel::PUBLIC } + it 'uses the group visibility' do expect(restorer.restore).to eq(true) - expect(restorer.project.visibility_level).to eq(Gitlab::VisibilityLevel::PRIVATE) + expect(restorer.project.visibility_level).to eq(group_visibility) + end + + context 'with restricted internal visibility' do + it 'sets private visibility' do + stub_application_setting(restricted_visibility_levels: [Gitlab::VisibilityLevel::INTERNAL]) + + expect(restorer.restore).to eq(true) + expect(restorer.project.visibility_level).to eq(Gitlab::VisibilityLevel::PRIVATE) + end end end end @@ -995,24 +1003,35 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i let(:user2) { create(:user) } let(:project_members) do [ - { - "id" => 2, - "access_level" => 40, - "source_type" => "Project", - "notification_level" => 3, - "user" => { - "id" => user2.id, - "email" => user2.email, - "username" => 'test' - } - } + [ + { + "id" => 2, + "access_level" => 40, + "source_type" => "Project", + "notification_level" => 3, + "user" => { + "id" => user2.id, + "email" => user2.email, + "username" => 'test' + } + }, + 0 + ] ] end - let(:tree_hash) { { 'project_members' => project_members } } - before do project.add_maintainer(user) + + setup_import_export_config('light') + + allow_next_instance_of(Gitlab::ImportExport::Json::NdjsonReader) do |relation_reader| + allow(relation_reader).to receive(:consume_relation).and_call_original + + allow(relation_reader).to receive(:consume_relation) + .with('project', 'project_members') + .and_return(project_members) + end end it 'restores project members' do @@ -1032,7 +1051,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i before do setup_import_export_config('with_invalid_records') - setup_reader(reader) + setup_reader subject end @@ -1125,13 +1144,5 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i end end - context 'enable ndjson import' do - it_behaves_like 'project tree restorer work properly', :legacy_reader, true - - it_behaves_like 'project tree restorer work properly', :ndjson_reader, true - end - - context 'disable ndjson import' do - it_behaves_like 'project tree restorer work properly', :legacy_reader, false - end + it_behaves_like 'project tree restorer work properly' end diff --git a/spec/lib/gitlab/import_export/project/tree_saver_spec.rb b/spec/lib/gitlab/import_export/project/tree_saver_spec.rb index 74b6e039601..4166eba4e8e 100644 --- a/spec/lib/gitlab/import_export/project/tree_saver_spec.rb +++ b/spec/lib/gitlab/import_export/project/tree_saver_spec.rb @@ -2,35 +2,28 @@ require 'spec_helper' -RSpec.describe Gitlab::ImportExport::Project::TreeSaver, :with_license do +RSpec.describe Gitlab::ImportExport::Project::TreeSaver, :with_license, feature_category: :importers do let_it_be(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" } let_it_be(:exportable_path) { 'project' } let_it_be(:user) { create(:user) } let_it_be(:group) { create(:group) } let_it_be(:project) { setup_project } - shared_examples 'saves project tree successfully' do |ndjson_enabled| + shared_examples 'saves project tree successfully' do include ImportExport::CommonUtil - subject { get_json(full_path, exportable_path, relation_name, ndjson_enabled) } + subject { get_json(full_path, exportable_path, relation_name) } describe 'saves project tree attributes' do let_it_be(:shared) { project.import_export_shared } let(:relation_name) { :projects } - let_it_be(:full_path) do - if ndjson_enabled - File.join(shared.export_path, 'tree') - else - File.join(shared.export_path, Gitlab::ImportExport.project_filename) - end - end + let_it_be(:full_path) { File.join(shared.export_path, 'tree') } before_all do RSpec::Mocks.with_temporary_scope do stub_all_feature_flags - stub_feature_flags(project_export_as_ndjson: ndjson_enabled) project.add_maintainer(user) @@ -223,22 +216,31 @@ RSpec.describe Gitlab::ImportExport::Project::TreeSaver, :with_license do expect(subject.dig(0, 'stages')).not_to be_empty end - it 'has pipeline statuses' do - expect(subject.dig(0, 'stages', 0, 'statuses')).not_to be_empty + it 'has pipeline builds' do + count = subject.dig(0, 'stages', 0, 'builds').count + + expect(count).to eq(1) end - it 'has pipeline builds' do - builds_count = subject.dig(0, 'stages', 0, 'statuses') - .count { |hash| hash['type'] == 'Ci::Build' } + it 'has pipeline generic_commit_statuses' do + count = subject.dig(0, 'stages', 0, 'generic_commit_statuses').count - expect(builds_count).to eq(1) + expect(count).to eq(1) end - it 'has ci pipeline notes' do - expect(subject.first['notes']).not_to be_empty + it 'has pipeline bridges' do + count = subject.dig(0, 'stages', 0, 'bridges').count + + expect(count).to eq(1) end end + context 'with commit_notes' do + let(:relation_name) { :commit_notes } + + it { is_expected.not_to be_empty } + end + context 'with labels' do let(:relation_name) { :labels } @@ -291,13 +293,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeSaver, :with_license do let_it_be(:group) { create(:group) } let(:project) { setup_project } - let(:full_path) do - if ndjson_enabled - File.join(shared.export_path, 'tree') - else - File.join(shared.export_path, Gitlab::ImportExport.project_filename) - end - end + let(:full_path) { File.join(shared.export_path, 'tree') } let(:shared) { project.import_export_shared } let(:params) { {} } @@ -305,7 +301,6 @@ RSpec.describe Gitlab::ImportExport::Project::TreeSaver, :with_license do let(:project_tree_saver ) { described_class.new(project: project, current_user: user, shared: shared, params: params) } before do - stub_feature_flags(project_export_as_ndjson: ndjson_enabled) project.add_maintainer(user) FileUtils.rm_rf(export_path) @@ -416,13 +411,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeSaver, :with_license do end end - context 'with JSON' do - it_behaves_like "saves project tree successfully", false - end - - context 'with NDJSON' do - it_behaves_like "saves project tree successfully", true - end + it_behaves_like "saves project tree successfully" context 'when streaming has to retry', :aggregate_failures do let(:shared) { double('shared', export_path: exportable_path) } @@ -468,6 +457,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeSaver, :with_license do end end + # rubocop: disable Metrics/AbcSize def setup_project release = create(:release) @@ -496,6 +486,8 @@ RSpec.describe Gitlab::ImportExport::Project::TreeSaver, :with_license do ci_build = create(:ci_build, project: project, when: nil) ci_build.pipeline.update!(project: project) create(:commit_status, project: project, pipeline: ci_build.pipeline) + create(:generic_commit_status, pipeline: ci_build.pipeline, ci_stage: ci_build.ci_stage, project: project) + create(:ci_bridge, pipeline: ci_build.pipeline, ci_stage: ci_build.ci_stage, project: project) create(:milestone, project: project) discussion_note = create(:discussion_note, noteable: issue, project: project) @@ -528,4 +520,5 @@ RSpec.describe Gitlab::ImportExport::Project::TreeSaver, :with_license do project end + # rubocop: enable Metrics/AbcSize end diff --git a/spec/lib/gitlab/import_export/references_configuration_spec.rb b/spec/lib/gitlab/import_export/references_configuration_spec.rb index ad165790b77..84c5b564cb1 100644 --- a/spec/lib/gitlab/import_export/references_configuration_spec.rb +++ b/spec/lib/gitlab/import_export/references_configuration_spec.rb @@ -9,7 +9,7 @@ require 'spec_helper' # or to be blacklisted by using the import_export.yml configuration file. # Likewise, new models added to import_export.yml, will need to be added with their correspondent relations # to this spec. -RSpec.describe 'Import/Export Project configuration' do +RSpec.describe 'Import/Export Project configuration', feature_category: :importers do include ConfigurationHelper where(:relation_path, :relation_name) do diff --git a/spec/lib/gitlab/import_export/safe_model_attributes.yml b/spec/lib/gitlab/import_export/safe_model_attributes.yml index e14e929faf3..faf345e8f78 100644 --- a/spec/lib/gitlab/import_export/safe_model_attributes.yml +++ b/spec/lib/gitlab/import_export/safe_model_attributes.yml @@ -86,6 +86,12 @@ Note: - original_discussion_id - confidential - last_edited_at +- internal +Notes::NoteMetadata: +- note_id +- email_participant +- created_at +- updated_at LabelLink: - id - target_type @@ -347,7 +353,111 @@ Ci::Stage: - pipeline_id - created_at - updated_at -CommitStatus: +Ci::Build: +- id +- project_id +- status +- finished_at +- trace +- created_at +- updated_at +- started_at +- runner_id +- coverage +- commit_id +- commands +- job_id +- name +- deploy +- options +- allow_failure +- stage +- trigger_request_id +- stage_idx +- stage_id +- tag +- ref +- user_id +- type +- target_url +- description +- artifacts_file +- artifacts_file_store +- artifacts_metadata +- artifacts_metadata_store +- erased_by_id +- erased_at +- artifacts_expire_at +- environment +- artifacts_size +- when +- yaml_variables +- queued_at +- token +- lock_version +- coverage_regex +- auto_canceled_by_id +- retried +- protected +- failure_reason +- scheduled_at +- upstream_pipeline_id +- interruptible +- processed +- scheduling_type +Ci::Bridge: +- id +- project_id +- status +- finished_at +- trace +- created_at +- updated_at +- started_at +- runner_id +- coverage +- commit_id +- commands +- job_id +- name +- deploy +- options +- allow_failure +- stage +- trigger_request_id +- stage_idx +- stage_id +- tag +- ref +- user_id +- type +- target_url +- description +- artifacts_file +- artifacts_file_store +- artifacts_metadata +- artifacts_metadata_store +- erased_by_id +- erased_at +- artifacts_expire_at +- environment +- artifacts_size +- when +- yaml_variables +- queued_at +- token +- lock_version +- coverage_regex +- auto_canceled_by_id +- retried +- protected +- failure_reason +- scheduled_at +- upstream_pipeline_id +- interruptible +- processed +- scheduling_type +GenericCommitStatus: - id - project_id - status @@ -822,6 +932,11 @@ DesignManagement::Version: - created_at - sha - author_id +DesignManagement::Repository: +- id +- project_id +- created_at +- updated_at ZoomMeeting: - id - project_id @@ -955,3 +1070,21 @@ ResourceIterationEvent: - action Iterations::Cadence: - title +ApprovalProjectRule: + - approvals_required + - name + - rule_type + - scanners + - vulnerabilities_allowed + - severity_levels + - report_type + - vulnerability_states + - orchestration_policy_idx + - applies_to_all_protected_branches +ApprovalProjectRulesUser: + - user_id + - approval_project_rule_id +ApprovalProjectRulesProtectedBranch: + - protected_branch_id + - approval_project_rule_id + - branch_name diff --git a/spec/lib/gitlab/import_sources_spec.rb b/spec/lib/gitlab/import_sources_spec.rb index 393e0a9be10..f1ea5f3e85e 100644 --- a/spec/lib/gitlab/import_sources_spec.rb +++ b/spec/lib/gitlab/import_sources_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::ImportSources do +RSpec.describe Gitlab::ImportSources, feature_category: :importers do describe '.options' do it 'returns a hash' do expected = @@ -10,13 +10,11 @@ RSpec.describe Gitlab::ImportSources do 'GitHub' => 'github', 'Bitbucket Cloud' => 'bitbucket', 'Bitbucket Server' => 'bitbucket_server', - 'GitLab.com' => 'gitlab', 'FogBugz' => 'fogbugz', 'Repository by URL' => 'git', 'GitLab export' => 'gitlab_project', 'Gitea' => 'gitea', - 'Manifest file' => 'manifest', - 'Phabricator' => 'phabricator' + 'Manifest file' => 'manifest' } expect(described_class.options).to eq(expected) @@ -30,13 +28,11 @@ RSpec.describe Gitlab::ImportSources do github bitbucket bitbucket_server - gitlab fogbugz git gitlab_project gitea manifest - phabricator ) expect(described_class.values).to eq(expected) @@ -50,11 +46,9 @@ RSpec.describe Gitlab::ImportSources do github bitbucket bitbucket_server - gitlab fogbugz gitlab_project gitea - phabricator ) expect(described_class.importer_names).to eq(expected) @@ -66,13 +60,11 @@ RSpec.describe Gitlab::ImportSources do 'github' => Gitlab::GithubImport::ParallelImporter, 'bitbucket' => Gitlab::BitbucketImport::Importer, 'bitbucket_server' => Gitlab::BitbucketServerImport::Importer, - 'gitlab' => Gitlab::GitlabImport::Importer, 'fogbugz' => Gitlab::FogbugzImport::Importer, 'git' => nil, 'gitlab_project' => Gitlab::ImportExport::Importer, 'gitea' => Gitlab::LegacyGithubImport::Importer, - 'manifest' => nil, - 'phabricator' => Gitlab::PhabricatorImport::Importer + 'manifest' => nil } import_sources.each do |name, klass| @@ -87,13 +79,11 @@ RSpec.describe Gitlab::ImportSources do 'github' => 'GitHub', 'bitbucket' => 'Bitbucket Cloud', 'bitbucket_server' => 'Bitbucket Server', - 'gitlab' => 'GitLab.com', 'fogbugz' => 'FogBugz', 'git' => 'Repository by URL', 'gitlab_project' => 'GitLab export', 'gitea' => 'Gitea', - 'manifest' => 'Manifest file', - 'phabricator' => 'Phabricator' + 'manifest' => 'Manifest file' } import_sources.each do |name, title| @@ -104,7 +94,7 @@ RSpec.describe Gitlab::ImportSources do end describe 'imports_repository? checker' do - let(:allowed_importers) { %w[github gitlab_project bitbucket_server phabricator] } + let(:allowed_importers) { %w[github gitlab_project bitbucket_server] } it 'fails if any importer other than the allowed ones implements this method' do current_importers = described_class.values.select { |kind| described_class.importer(kind).try(:imports_repository?) } diff --git a/spec/lib/gitlab/instrumentation/redis_base_spec.rb b/spec/lib/gitlab/instrumentation/redis_base_spec.rb index 656e6ffba05..426997f6e86 100644 --- a/spec/lib/gitlab/instrumentation/redis_base_spec.rb +++ b/spec/lib/gitlab/instrumentation/redis_base_spec.rb @@ -210,4 +210,16 @@ RSpec.describe Gitlab::Instrumentation::RedisBase, :request_store do end end end + + describe '.log_exception' do + it 'logs exception with storage details' do + expect(::Gitlab::ErrorTracking).to receive(:log_exception) + .with( + an_instance_of(StandardError), + storage: instrumentation_class_a.storage_key + ) + + instrumentation_class_a.log_exception(StandardError.new) + end + end end diff --git a/spec/lib/gitlab/instrumentation/redis_interceptor_spec.rb b/spec/lib/gitlab/instrumentation/redis_interceptor_spec.rb index 187a6ff1739..be6586ca610 100644 --- a/spec/lib/gitlab/instrumentation/redis_interceptor_spec.rb +++ b/spec/lib/gitlab/instrumentation/redis_interceptor_spec.rb @@ -64,16 +64,34 @@ RSpec.describe Gitlab::Instrumentation::RedisInterceptor, :clean_gitlab_redis_sh end end - it 'counts exceptions' do - expect(instrumentation_class).to receive(:instance_count_exception) - .with(instance_of(Redis::CommandError)).and_call_original - expect(instrumentation_class).to receive(:instance_count_request).and_call_original + context 'when encountering exceptions' do + where(:case_name, :exception, :exception_counter) do + 'generic exception' | Redis::CommandError | :instance_count_exception + 'moved redirection' | Redis::CommandError.new("MOVED 123 127.0.0.1:6380") | :instance_count_cluster_redirection + 'ask redirection' | Redis::CommandError.new("ASK 123 127.0.0.1:6380") | :instance_count_cluster_redirection + end - expect do - Gitlab::Redis::SharedState.with do |redis| - redis.call(:auth, 'foo', 'bar') + with_them do + before do + Gitlab::Redis::SharedState.with do |redis| + # We need to go 1 layer deeper to stub _client as we monkey-patch Redis::Client + # with the interceptor. Stubbing `redis` will skip the instrumentation_class. + allow(redis._client).to receive(:process).and_raise(exception) + end end - end.to raise_exception(Redis::CommandError) + + it 'counts exception' do + expect(instrumentation_class).to receive(exception_counter) + .with(instance_of(Redis::CommandError)).and_call_original + expect(instrumentation_class).to receive(:log_exception) + .with(instance_of(Redis::CommandError)).and_call_original + expect(instrumentation_class).to receive(:instance_count_request).and_call_original + + expect do + Gitlab::Redis::SharedState.with { |redis| redis.call(:auth, 'foo', 'bar') } + end.to raise_exception(Redis::CommandError) + end + end end context 'in production environment' do @@ -174,6 +192,7 @@ RSpec.describe Gitlab::Instrumentation::RedisInterceptor, :clean_gitlab_redis_sh [['zadd', 'foobar', 1, 'a']] | ['bzpopmax', 'foobar', 0] [['xadd', 'mystream', 1, 'myfield', 'mydata']] | ['xread', 'block', 1, 'streams', 'mystream', '0-0'] [['xadd', 'foobar', 1, 'myfield', 'mydata'], ['xgroup', 'create', 'foobar', 'mygroup', 0]] | ['xreadgroup', 'group', 'mygroup', 'myconsumer', 'block', 1, 'streams', 'foobar', '0-0'] + [] | ['command'] end with_them do diff --git a/spec/lib/gitlab/internal_post_receive/response_spec.rb b/spec/lib/gitlab/internal_post_receive/response_spec.rb index 23ea5191486..2792cf49d06 100644 --- a/spec/lib/gitlab/internal_post_receive/response_spec.rb +++ b/spec/lib/gitlab/internal_post_receive/response_spec.rb @@ -76,7 +76,7 @@ RSpec.describe Gitlab::InternalPostReceive::Response do describe '#add_alert_message' do context 'when text is present' do - it 'adds a alert message' do + it 'adds an alert message' do subject.add_alert_message('hello') expect(subject.messages.first.message).to eq('hello') diff --git a/spec/lib/gitlab/issuable_sorter_spec.rb b/spec/lib/gitlab/issuable_sorter_spec.rb index b8d0c7b0609..0d9940bab6f 100644 --- a/spec/lib/gitlab/issuable_sorter_spec.rb +++ b/spec/lib/gitlab/issuable_sorter_spec.rb @@ -4,16 +4,42 @@ require 'spec_helper' RSpec.describe Gitlab::IssuableSorter do let(:namespace1) { build_stubbed(:namespace, id: 1) } - let(:project1) { build_stubbed(:project, id: 1, namespace: namespace1) } - - let(:project2) { build_stubbed(:project, id: 2, path: "a", namespace: project1.namespace) } - let(:project3) { build_stubbed(:project, id: 3, path: "b", namespace: project1.namespace) } - let(:namespace2) { build_stubbed(:namespace, id: 2, path: "a") } let(:namespace3) { build_stubbed(:namespace, id: 3, path: "b") } - let(:project4) { build_stubbed(:project, id: 4, path: "a", namespace: namespace2) } - let(:project5) { build_stubbed(:project, id: 5, path: "b", namespace: namespace2) } - let(:project6) { build_stubbed(:project, id: 6, path: "a", namespace: namespace3) } + + let(:project1) do + build_stubbed(:project, id: 1, namespace: namespace1, project_namespace: build_stubbed(:project_namespace)) + end + + let(:project2) do + build_stubbed( + :project, id: 2, path: "a", namespace: project1.namespace, project_namespace: build_stubbed(:project_namespace) + ) + end + + let(:project3) do + build_stubbed( + :project, id: 3, path: "b", namespace: project1.namespace, project_namespace: build_stubbed(:project_namespace) + ) + end + + let(:project4) do + build_stubbed( + :project, id: 4, path: "a", namespace: namespace2, project_namespace: build_stubbed(:project_namespace) + ) + end + + let(:project5) do + build_stubbed( + :project, id: 5, path: "b", namespace: namespace2, project_namespace: build_stubbed(:project_namespace) + ) + end + + let(:project6) do + build_stubbed( + :project, id: 6, path: "a", namespace: namespace3, project_namespace: build_stubbed(:project_namespace) + ) + end let(:unsorted) { [sorted[2], sorted[3], sorted[0], sorted[1]] } diff --git a/spec/lib/gitlab/jira_import/issues_importer_spec.rb b/spec/lib/gitlab/jira_import/issues_importer_spec.rb index 9f654bbcd15..36135c56dd9 100644 --- a/spec/lib/gitlab/jira_import/issues_importer_spec.rb +++ b/spec/lib/gitlab/jira_import/issues_importer_spec.rb @@ -44,7 +44,7 @@ RSpec.describe Gitlab::JiraImport::IssuesImporter do def mock_issue_serializer(count, raise_exception_on_even_mocks: false) serializer = instance_double(Gitlab::JiraImport::IssueSerializer, execute: { key: 'data' }) - allow(Issue).to receive(:with_project_iid_supply).and_return('issue_iid') + allow(Issue).to receive(:with_namespace_iid_supply).and_return('issue_iid') count.times do |i| if raise_exception_on_even_mocks && i.even? diff --git a/spec/lib/gitlab/json_logger_spec.rb b/spec/lib/gitlab/json_logger_spec.rb index 801de357ddc..87df20c066b 100644 --- a/spec/lib/gitlab/json_logger_spec.rb +++ b/spec/lib/gitlab/json_logger_spec.rb @@ -5,7 +5,7 @@ require 'spec_helper' RSpec.describe Gitlab::JsonLogger do subject { described_class.new('/dev/null') } - let(:now) { Time.now } + it_behaves_like 'a json logger', {} describe '#file_name' do let(:subclass) do @@ -26,31 +26,4 @@ RSpec.describe Gitlab::JsonLogger do expect(subclass.file_name).to eq('testlogger.log') end end - - describe '#format_message' do - before do - allow(Labkit::Correlation::CorrelationId).to receive(:current_id).and_return('new-correlation-id') - end - - it 'formats strings' do - output = subject.format_message('INFO', now, 'test', 'Hello world') - data = Gitlab::Json.parse(output) - - expect(data['severity']).to eq('INFO') - expect(data['time']).to eq(now.utc.iso8601(3)) - expect(data['message']).to eq('Hello world') - expect(data['correlation_id']).to eq('new-correlation-id') - end - - it 'formats hashes' do - output = subject.format_message('INFO', now, 'test', { hello: 1 }) - data = Gitlab::Json.parse(output) - - expect(data['severity']).to eq('INFO') - expect(data['time']).to eq(now.utc.iso8601(3)) - expect(data['hello']).to eq(1) - expect(data['message']).to be_nil - expect(data['correlation_id']).to eq('new-correlation-id') - end - end end diff --git a/spec/lib/gitlab/jwt_authenticatable_spec.rb b/spec/lib/gitlab/jwt_authenticatable_spec.rb index 92d5feceb75..9a06f9b91df 100644 --- a/spec/lib/gitlab/jwt_authenticatable_spec.rb +++ b/spec/lib/gitlab/jwt_authenticatable_spec.rb @@ -172,11 +172,17 @@ RSpec.describe Gitlab::JwtAuthenticatable do end it 'raises an error if iat is invalid' do - encoded_message = JWT.encode(payload.merge(iat: 'wrong'), test_class.secret, 'HS256') + encoded_message = JWT.encode(payload.merge(iat: Time.current.to_i + 1), test_class.secret, 'HS256') expect { test_class.decode_jwt(encoded_message, iat_after: true) }.to raise_error(JWT::DecodeError) end + it 'raises InvalidPayload exception if iat is a string' do + expect do + JWT.encode(payload.merge(iat: 'wrong'), test_class.secret, 'HS256') + end.to raise_error(JWT::InvalidPayload) + end + it 'raises an error if iat is absent' do encoded_message = JWT.encode(payload, test_class.secret, 'HS256') diff --git a/spec/lib/gitlab/kas/client_spec.rb b/spec/lib/gitlab/kas/client_spec.rb index 9a0fa6c4067..5668c265611 100644 --- a/spec/lib/gitlab/kas/client_spec.rb +++ b/spec/lib/gitlab/kas/client_spec.rb @@ -109,6 +109,35 @@ RSpec.describe Gitlab::Kas::Client do it { expect(subject).to eq(agent_configurations) } end + describe '#send_git_push_event' do + let(:stub) { instance_double(Gitlab::Agent::Notifications::Rpc::Notifications::Stub) } + let(:request) { instance_double(Gitlab::Agent::Notifications::Rpc::GitPushEventRequest) } + let(:project_param) { instance_double(Gitlab::Agent::Notifications::Rpc::Project) } + let(:response) { double(Gitlab::Agent::Notifications::Rpc::GitPushEventResponse) } + + subject { described_class.new.send_git_push_event(project: project) } + + before do + expect(Gitlab::Agent::Notifications::Rpc::Notifications::Stub).to receive(:new) + .with('example.kas.internal', :this_channel_is_insecure, timeout: described_class::TIMEOUT) + .and_return(stub) + + expect(Gitlab::Agent::Notifications::Rpc::Project).to receive(:new) + .with(id: project.id, full_path: project.full_path) + .and_return(project_param) + + expect(Gitlab::Agent::Notifications::Rpc::GitPushEventRequest).to receive(:new) + .with(project: project_param) + .and_return(request) + + expect(stub).to receive(:git_push_event) + .with(request, metadata: { 'authorization' => 'bearer test-token' }) + .and_return(response) + end + + it { expect(subject).to eq(response) } + end + describe 'with grpcs' do let(:stub) { instance_double(Gitlab::Agent::ConfigurationProject::Rpc::ConfigurationProject::Stub) } let(:credentials) { instance_double(GRPC::Core::ChannelCredentials) } diff --git a/spec/lib/gitlab/kas/user_access_spec.rb b/spec/lib/gitlab/kas/user_access_spec.rb new file mode 100644 index 00000000000..a8296d23a18 --- /dev/null +++ b/spec/lib/gitlab/kas/user_access_spec.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Kas::UserAccess, feature_category: :deployment_management do + describe '.enabled?' do + subject { described_class.enabled? } + + before do + allow(::Gitlab::Kas).to receive(:enabled?).and_return true + end + + it { is_expected.to be true } + + context 'when flag kas_user_access is disabled' do + before do + stub_feature_flags(kas_user_access: false) + end + + it { is_expected.to be false } + end + end + + describe '.enabled_for?' do + subject { described_class.enabled_for?(agent) } + + let(:agent) { build(:cluster_agent) } + + before do + allow(::Gitlab::Kas).to receive(:enabled?).and_return true + end + + it { is_expected.to be true } + + context 'when flag kas_user_access is disabled' do + before do + stub_feature_flags(kas_user_access: false) + end + + it { is_expected.to be false } + end + + context 'when flag kas_user_access_project is disabled' do + before do + stub_feature_flags(kas_user_access_project: false) + end + + it { is_expected.to be false } + end + end + + describe '.{encrypt,decrypt}_public_session_id' do + let(:data) { 'the data' } + let(:encrypted) { described_class.encrypt_public_session_id(data) } + let(:decrypted) { described_class.decrypt_public_session_id(encrypted) } + + it { expect(encrypted).not_to include data } + it { expect(decrypted).to eq data } + end + + describe '.cookie_data' do + subject(:cookie_data) { described_class.cookie_data(public_session_id) } + + let(:public_session_id) { 'the-public-session-id' } + let(:external_k8s_proxy_url) { 'https://example.com:1234' } + + before do + stub_config( + gitlab: { host: 'example.com', https: true }, + gitlab_kas: { external_k8s_proxy_url: external_k8s_proxy_url } + ) + end + + it 'is encrypted, secure, httponly', :aggregate_failures do + expect(cookie_data[:value]).not_to include public_session_id + expect(cookie_data).to include(httponly: true, secure: true, path: '/') + expect(cookie_data).not_to have_key(:domain) + end + + context 'when on non-root path' do + let(:external_k8s_proxy_url) { 'https://example.com/k8s-proxy' } + + it 'sets :path' do + expect(cookie_data).to include(httponly: true, secure: true, path: '/k8s-proxy') + end + end + + context 'when on subdomain' do + let(:external_k8s_proxy_url) { 'https://k8s-proxy.example.com' } + + it 'sets :domain' do + expect(cookie_data[:domain]).to eq "example.com" + end + end + end +end diff --git a/spec/lib/gitlab/kroki_spec.rb b/spec/lib/gitlab/kroki_spec.rb index 3d6ecf20377..6d8e6ecbf54 100644 --- a/spec/lib/gitlab/kroki_spec.rb +++ b/spec/lib/gitlab/kroki_spec.rb @@ -6,7 +6,7 @@ RSpec.describe Gitlab::Kroki do describe '.formats' do def default_formats - %w[bytefield c4plantuml ditaa erd graphviz nomnoml pikchr plantuml + %w[bytefield c4plantuml d2 dbml diagramsnet ditaa erd graphviz nomnoml pikchr plantuml structurizr svgbob umlet vega vegalite wavedrom].freeze end diff --git a/spec/lib/gitlab/kubernetes/config_map_spec.rb b/spec/lib/gitlab/kubernetes/config_map_spec.rb index 2d0d205ffb1..ebc2202921b 100644 --- a/spec/lib/gitlab/kubernetes/config_map_spec.rb +++ b/spec/lib/gitlab/kubernetes/config_map_spec.rb @@ -4,20 +4,23 @@ require 'spec_helper' RSpec.describe Gitlab::Kubernetes::ConfigMap do let(:kubeclient) { double('kubernetes client') } - let(:application) { create(:clusters_applications_prometheus) } - let(:config_map) { described_class.new(application.name, application.files) } + let(:name) { 'my-name' } + let(:files) { [] } + let(:config_map) { described_class.new(name, files) } let(:namespace) { Gitlab::Kubernetes::Helm::NAMESPACE } let(:metadata) do { - name: "values-content-configuration-#{application.name}", + name: "values-content-configuration-#{name}", namespace: namespace, - labels: { name: "values-content-configuration-#{application.name}" } + labels: { name: "values-content-configuration-#{name}" } } end describe '#generate' do - let(:resource) { ::Kubeclient::Resource.new(metadata: metadata, data: application.files) } + let(:resource) do + ::Kubeclient::Resource.new(metadata: metadata, data: files) + end subject { config_map.generate } @@ -28,7 +31,8 @@ RSpec.describe Gitlab::Kubernetes::ConfigMap do describe '#config_map_name' do it 'returns the config_map name' do - expect(config_map.config_map_name).to eq("values-content-configuration-#{application.name}") + expect(config_map.config_map_name) + .to eq("values-content-configuration-#{name}") end end end diff --git a/spec/lib/gitlab/kubernetes/helm/api_spec.rb b/spec/lib/gitlab/kubernetes/helm/api_spec.rb deleted file mode 100644 index e022f5bd912..00000000000 --- a/spec/lib/gitlab/kubernetes/helm/api_spec.rb +++ /dev/null @@ -1,269 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Kubernetes::Helm::API do - let(:client) { double('kubernetes client') } - let(:helm) { described_class.new(client) } - let(:gitlab_namespace) { Gitlab::Kubernetes::Helm::NAMESPACE } - let(:gitlab_namespace_labels) { Gitlab::Kubernetes::Helm::NAMESPACE_LABELS } - let(:namespace) { Gitlab::Kubernetes::Namespace.new(gitlab_namespace, client, labels: gitlab_namespace_labels) } - let(:application_name) { 'app-name' } - let(:rbac) { false } - let(:files) { {} } - - let(:command) do - Gitlab::Kubernetes::Helm::V2::InstallCommand.new( - name: application_name, - chart: 'chart-name', - rbac: rbac, - files: files - ) - end - - subject { helm } - - before do - allow(Gitlab::Kubernetes::Namespace).to( - receive(:new).with(gitlab_namespace, client, labels: gitlab_namespace_labels).and_return(namespace) - ) - allow(client).to receive(:create_config_map) - end - - describe '#initialize' do - it 'creates a namespace object' do - expect(Gitlab::Kubernetes::Namespace).to( - receive(:new).with(gitlab_namespace, client, labels: gitlab_namespace_labels) - ) - - subject - end - end - - describe '#uninstall' do - before do - allow(client).to receive(:create_pod).and_return(nil) - allow(client).to receive(:get_config_map).and_return(nil) - allow(client).to receive(:create_config_map).and_return(nil) - allow(client).to receive(:delete_pod).and_return(nil) - allow(namespace).to receive(:ensure_exists!).once - end - - it 'ensures the namespace exists before creating the POD' do - expect(namespace).to receive(:ensure_exists!).once.ordered - expect(client).to receive(:create_pod).once.ordered - - subject.uninstall(command) - end - - it 'removes an existing pod before installing' do - expect(client).to receive(:delete_pod).with('install-app-name', 'gitlab-managed-apps').once.ordered - expect(client).to receive(:create_pod).once.ordered - - subject.uninstall(command) - end - - context 'with a ConfigMap' do - let(:resource) { Gitlab::Kubernetes::ConfigMap.new(application_name, files).generate } - - it 'creates a ConfigMap on kubeclient' do - expect(client).to receive(:create_config_map).with(resource).once - - subject.install(command) - end - - context 'config map already exists' do - before do - expect(client).to receive(:get_config_map).with("values-content-configuration-#{application_name}", gitlab_namespace).and_return(resource) - end - - it 'updates the config map' do - expect(client).to receive(:update_config_map).with(resource).once - - subject.install(command) - end - end - end - end - - describe '#install' do - before do - allow(client).to receive(:create_pod).and_return(nil) - allow(client).to receive(:get_config_map).and_return(nil) - allow(client).to receive(:create_config_map).and_return(nil) - allow(client).to receive(:create_service_account).and_return(nil) - allow(client).to receive(:delete_pod).and_return(nil) - allow(namespace).to receive(:ensure_exists!).once - end - - it 'ensures the namespace exists before creating the POD' do - expect(namespace).to receive(:ensure_exists!).once.ordered - expect(client).to receive(:create_pod).once.ordered - - subject.install(command) - end - - it 'removes an existing pod before installing' do - expect(client).to receive(:delete_pod).with('install-app-name', 'gitlab-managed-apps').once.ordered - expect(client).to receive(:create_pod).once.ordered - - subject.install(command) - end - - context 'with a ConfigMap' do - let(:resource) { Gitlab::Kubernetes::ConfigMap.new(application_name, files).generate } - - it 'creates a ConfigMap on kubeclient' do - expect(client).to receive(:create_config_map).with(resource).once - - subject.install(command) - end - - context 'config map already exists' do - before do - expect(client).to receive(:get_config_map).with("values-content-configuration-#{application_name}", gitlab_namespace).and_return(resource) - end - - it 'updates the config map' do - expect(client).to receive(:update_config_map).with(resource).once - - subject.install(command) - end - end - end - - context 'without a service account' do - it 'does not create a service account on kubeclient' do - expect(client).not_to receive(:create_service_account) - expect(client).not_to receive(:update_cluster_role_binding) - - subject.install(command) - end - end - - context 'with a service account' do - let(:command) { Gitlab::Kubernetes::Helm::V2::InitCommand.new(name: application_name, files: files, rbac: rbac) } - - context 'rbac-enabled cluster' do - let(:rbac) { true } - - let(:service_account_resource) do - Kubeclient::Resource.new(metadata: { name: 'tiller', namespace: 'gitlab-managed-apps' }) - end - - let(:cluster_role_binding_resource) do - Kubeclient::Resource.new( - metadata: { name: 'tiller-admin' }, - roleRef: { apiGroup: 'rbac.authorization.k8s.io', kind: 'ClusterRole', name: 'cluster-admin' }, - subjects: [{ kind: 'ServiceAccount', name: 'tiller', namespace: 'gitlab-managed-apps' }] - ) - end - - context 'service account does not exist' do - before do - expect(client).to receive(:get_service_account).with('tiller', 'gitlab-managed-apps').and_raise(Kubeclient::ResourceNotFoundError.new(404, 'Not found', nil)) - end - - it 'creates a service account, followed the cluster role binding on kubeclient' do - expect(client).to receive(:create_service_account).with(service_account_resource).once.ordered - expect(client).to receive(:update_cluster_role_binding).with(cluster_role_binding_resource).once.ordered - - subject.install(command) - end - end - - context 'service account already exists' do - before do - expect(client).to receive(:get_service_account).with('tiller', 'gitlab-managed-apps').and_return(service_account_resource) - end - - it 'updates the service account, followed by creating the cluster role binding' do - expect(client).to receive(:update_service_account).with(service_account_resource).once.ordered - expect(client).to receive(:update_cluster_role_binding).with(cluster_role_binding_resource).once.ordered - - subject.install(command) - end - end - - context 'a non-404 error is thrown' do - before do - expect(client).to receive(:get_service_account).with('tiller', 'gitlab-managed-apps').and_raise(Kubeclient::HttpError.new(401, 'Unauthorized', nil)) - end - - it 'raises an error' do - expect { subject.install(command) }.to raise_error(Kubeclient::HttpError) - end - end - end - - context 'legacy abac cluster' do - it 'does not create a service account on kubeclient' do - expect(client).not_to receive(:create_service_account) - expect(client).not_to receive(:update_cluster_role_binding) - - subject.install(command) - end - end - end - end - - describe '#status' do - let(:phase) { Gitlab::Kubernetes::Pod::RUNNING } - let(:pod) { Kubeclient::Resource.new(status: { phase: phase }) } # partial representation - - it 'fetches POD phase from kubernetes cluster' do - expect(client).to receive(:get_pod).with(command.pod_name, gitlab_namespace).once.and_return(pod) - - expect(subject.status(command.pod_name)).to eq(phase) - end - end - - describe '#log' do - let(:log) { 'some output' } - let(:response) { RestClient::Response.new(log) } - - it 'fetches POD phase from kubernetes cluster' do - expect(client).to receive(:get_pod_log).with(command.pod_name, gitlab_namespace).once.and_return(response) - - expect(subject.log(command.pod_name)).to eq(log) - end - end - - describe '#delete_pod!' do - it 'deletes the POD from kubernetes cluster' do - expect(client).to receive(:delete_pod).with('install-app-name', 'gitlab-managed-apps').once - - subject.delete_pod!('install-app-name') - end - - context 'when the resource being deleted does not exist' do - it 'catches the error' do - expect(client).to receive(:delete_pod).with('install-app-name', 'gitlab-managed-apps') - .and_raise(Kubeclient::ResourceNotFoundError.new(404, 'Not found', nil)) - - subject.delete_pod!('install-app-name') - end - end - end - - describe '#get_config_map' do - before do - allow(namespace).to receive(:ensure_exists!).once - allow(client).to receive(:get_config_map).and_return(nil) - end - - it 'ensures the namespace exists before retrieving the config map' do - expect(namespace).to receive(:ensure_exists!).once - - subject.get_config_map('example-config-map-name') - end - - it 'gets the config map on kubeclient' do - expect(client).to receive(:get_config_map) - .with('example-config-map-name', namespace.name) - .once - - subject.get_config_map('example-config-map-name') - end - end -end diff --git a/spec/lib/gitlab/kubernetes/helm/pod_spec.rb b/spec/lib/gitlab/kubernetes/helm/pod_spec.rb deleted file mode 100644 index e3763977add..00000000000 --- a/spec/lib/gitlab/kubernetes/helm/pod_spec.rb +++ /dev/null @@ -1,89 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Kubernetes::Helm::Pod do - describe '#generate' do - using RSpec::Parameterized::TableSyntax - - where(:helm_major_version, :expected_helm_version, :expected_command_env) do - 2 | '2.17.0' | [:TILLER_NAMESPACE] - 3 | '3.2.4' | nil - end - - with_them do - let(:cluster) { create(:cluster, helm_major_version: helm_major_version) } - let(:app) { create(:clusters_applications_prometheus, cluster: cluster) } - let(:command) { app.install_command } - let(:namespace) { Gitlab::Kubernetes::Helm::NAMESPACE } - let(:service_account_name) { nil } - - subject { described_class.new(command, namespace, service_account_name: service_account_name) } - - context 'with a command' do - it 'generates a Kubeclient::Resource' do - expect(subject.generate).to be_a_kind_of(Kubeclient::Resource) - end - - it 'generates the appropriate metadata' do - metadata = subject.generate.metadata - expect(metadata.name).to eq("install-#{app.name}") - expect(metadata.namespace).to eq('gitlab-managed-apps') - expect(metadata.labels['gitlab.org/action']).to eq('install') - expect(metadata.labels['gitlab.org/application']).to eq(app.name) - end - - it 'generates a container spec' do - spec = subject.generate.spec - expect(spec.containers.count).to eq(1) - end - - it 'generates the appropriate specifications for the container' do - container = subject.generate.spec.containers.first - expect(container.name).to eq('helm') - expect(container.image).to eq("registry.gitlab.com/gitlab-org/cluster-integration/helm-install-image/releases/#{expected_helm_version}-kube-1.13.12-alpine-3.12") - expect(container.env.map(&:name)).to include(:HELM_VERSION, :COMMAND_SCRIPT, *expected_command_env) - expect(container.command).to match_array(["/bin/sh"]) - expect(container.args).to match_array(["-c", "$(COMMAND_SCRIPT)"]) - end - - it 'includes a never restart policy' do - spec = subject.generate.spec - expect(spec.restartPolicy).to eq('Never') - end - - it 'includes volumes for the container' do - container = subject.generate.spec.containers.first - expect(container.volumeMounts.first['name']).to eq('configuration-volume') - expect(container.volumeMounts.first['mountPath']).to eq("/data/helm/#{app.name}/config") - end - - it 'includes a volume inside the specification' do - spec = subject.generate.spec - expect(spec.volumes.first['name']).to eq('configuration-volume') - end - - it 'mounts configMap specification in the volume' do - volume = subject.generate.spec.volumes.first - expect(volume.configMap['name']).to eq("values-content-configuration-#{app.name}") - expect(volume.configMap['items'].first['key']).to eq(:'values.yaml') - expect(volume.configMap['items'].first['path']).to eq(:'values.yaml') - end - - it 'has no serviceAccountName' do - spec = subject.generate.spec - expect(spec.serviceAccountName).to be_nil - end - - context 'with a service_account_name' do - let(:service_account_name) { 'sa' } - - it 'uses the serviceAccountName provided' do - spec = subject.generate.spec - expect(spec.serviceAccountName).to eq(service_account_name) - end - end - end - end - end -end diff --git a/spec/lib/gitlab/kubernetes/helm/v2/base_command_spec.rb b/spec/lib/gitlab/kubernetes/helm/v2/base_command_spec.rb deleted file mode 100644 index 3d2b36b9094..00000000000 --- a/spec/lib/gitlab/kubernetes/helm/v2/base_command_spec.rb +++ /dev/null @@ -1,50 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Kubernetes::Helm::V2::BaseCommand do - subject(:base_command) do - test_class.new(rbac) - end - - let(:application) { create(:clusters_applications_helm) } - let(:rbac) { false } - - let(:test_class) do - Class.new(described_class) do - def initialize(rbac) - super( - name: 'test-class-name', - rbac: rbac, - files: { some: 'value' } - ) - end - end - end - - describe 'HELM_VERSION' do - subject { described_class::HELM_VERSION } - - it { is_expected.to match /^2\.\d+\.\d+$/ } - end - - describe '#env' do - subject { base_command.env } - - it { is_expected.to include(TILLER_NAMESPACE: 'gitlab-managed-apps') } - end - - it_behaves_like 'helm command generator' do - let(:commands) { '' } - end - - describe '#pod_name' do - subject { base_command.pod_name } - - it { is_expected.to eq('install-test-class-name') } - end - - it_behaves_like 'helm command' do - let(:command) { base_command } - end -end diff --git a/spec/lib/gitlab/kubernetes/helm/v2/certificate_spec.rb b/spec/lib/gitlab/kubernetes/helm/v2/certificate_spec.rb deleted file mode 100644 index 698b88c9fa1..00000000000 --- a/spec/lib/gitlab/kubernetes/helm/v2/certificate_spec.rb +++ /dev/null @@ -1,28 +0,0 @@ -# frozen_string_literal: true -require 'fast_spec_helper' - -RSpec.describe Gitlab::Kubernetes::Helm::V2::Certificate do - describe '.generate_root' do - subject { described_class.generate_root } - - it 'generates a root CA that expires a long way in the future' do - expect(subject.cert.not_after).to be > 999.years.from_now - end - end - - describe '#issue' do - subject { described_class.generate_root.issue } - - it 'generates a cert that expires soon' do - expect(subject.cert.not_after).to be < 60.minutes.from_now - end - - context 'passing in INFINITE_EXPIRY' do - subject { described_class.generate_root.issue(expires_in: described_class::INFINITE_EXPIRY) } - - it 'generates a cert that expires a long way in the future' do - expect(subject.cert.not_after).to be > 999.years.from_now - end - end - end -end diff --git a/spec/lib/gitlab/kubernetes/helm/v2/delete_command_spec.rb b/spec/lib/gitlab/kubernetes/helm/v2/delete_command_spec.rb deleted file mode 100644 index 4a3a41dba4a..00000000000 --- a/spec/lib/gitlab/kubernetes/helm/v2/delete_command_spec.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Kubernetes::Helm::V2::DeleteCommand do - subject(:delete_command) { described_class.new(name: app_name, rbac: rbac, files: files) } - - let(:app_name) { 'app-name' } - let(:rbac) { true } - let(:files) { {} } - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - export HELM_HOST="localhost:44134" - tiller -listen ${HELM_HOST} -alsologtostderr & - helm init --client-only - helm delete --purge app-name - EOS - end - end - - describe '#pod_name' do - subject { delete_command.pod_name } - - it { is_expected.to eq('uninstall-app-name') } - end - - it_behaves_like 'helm command' do - let(:command) { delete_command } - end - - describe '#delete_command' do - it 'deletes the release' do - expect(subject.delete_command).to eq('helm delete --purge app-name') - end - end -end diff --git a/spec/lib/gitlab/kubernetes/helm/v2/init_command_spec.rb b/spec/lib/gitlab/kubernetes/helm/v2/init_command_spec.rb deleted file mode 100644 index 8ae78ada15c..00000000000 --- a/spec/lib/gitlab/kubernetes/helm/v2/init_command_spec.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Kubernetes::Helm::V2::InitCommand do - subject(:init_command) { described_class.new(name: application.name, files: files, rbac: rbac) } - - let(:application) { create(:clusters_applications_helm) } - let(:rbac) { false } - let(:files) { {} } - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - helm init --tiller-tls --tiller-tls-verify --tls-ca-cert /data/helm/helm/config/ca.pem --tiller-tls-cert /data/helm/helm/config/cert.pem --tiller-tls-key /data/helm/helm/config/key.pem - EOS - end - end - - context 'on a rbac-enabled cluster' do - let(:rbac) { true } - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - helm init --tiller-tls --tiller-tls-verify --tls-ca-cert /data/helm/helm/config/ca.pem --tiller-tls-cert /data/helm/helm/config/cert.pem --tiller-tls-key /data/helm/helm/config/key.pem --service-account tiller - EOS - end - end - end - - it_behaves_like 'helm command' do - let(:command) { init_command } - end -end diff --git a/spec/lib/gitlab/kubernetes/helm/v2/install_command_spec.rb b/spec/lib/gitlab/kubernetes/helm/v2/install_command_spec.rb deleted file mode 100644 index 250d1a82e7a..00000000000 --- a/spec/lib/gitlab/kubernetes/helm/v2/install_command_spec.rb +++ /dev/null @@ -1,183 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Kubernetes::Helm::V2::InstallCommand do - subject(:install_command) do - described_class.new( - name: 'app-name', - chart: 'chart-name', - rbac: rbac, - files: files, - version: version, - repository: repository, - preinstall: preinstall, - postinstall: postinstall - ) - end - - let(:files) { { 'ca.pem': 'some file content' } } - let(:repository) { 'https://repository.example.com' } - let(:rbac) { false } - let(:version) { '1.2.3' } - let(:preinstall) { nil } - let(:postinstall) { nil } - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - export HELM_HOST="localhost:44134" - tiller -listen ${HELM_HOST} -alsologtostderr & - helm init --client-only - helm repo add app-name https://repository.example.com - helm repo update - #{helm_install_comand} - EOS - end - - let(:helm_install_comand) do - <<~EOS.squish - helm upgrade app-name chart-name - --install - --atomic - --cleanup-on-fail - --reset-values - --version 1.2.3 - --set rbac.create\\=false,rbac.enabled\\=false - --namespace gitlab-managed-apps - -f /data/helm/app-name/config/values.yaml - EOS - end - end - - context 'when rbac is true' do - let(:rbac) { true } - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - export HELM_HOST="localhost:44134" - tiller -listen ${HELM_HOST} -alsologtostderr & - helm init --client-only - helm repo add app-name https://repository.example.com - helm repo update - #{helm_install_command} - EOS - end - - let(:helm_install_command) do - <<~EOS.squish - helm upgrade app-name chart-name - --install - --atomic - --cleanup-on-fail - --reset-values - --version 1.2.3 - --set rbac.create\\=true,rbac.enabled\\=true - --namespace gitlab-managed-apps - -f /data/helm/app-name/config/values.yaml - EOS - end - end - end - - context 'when there is a pre-install script' do - let(:preinstall) { ['/bin/date', '/bin/true'] } - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - export HELM_HOST="localhost:44134" - tiller -listen ${HELM_HOST} -alsologtostderr & - helm init --client-only - helm repo add app-name https://repository.example.com - helm repo update - /bin/date - /bin/true - #{helm_install_command} - EOS - end - - let(:helm_install_command) do - <<~EOS.squish - helm upgrade app-name chart-name - --install - --atomic - --cleanup-on-fail - --reset-values - --version 1.2.3 - --set rbac.create\\=false,rbac.enabled\\=false - --namespace gitlab-managed-apps - -f /data/helm/app-name/config/values.yaml - EOS - end - end - end - - context 'when there is a post-install script' do - let(:postinstall) { ['/bin/date', "/bin/false\n"] } - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - export HELM_HOST="localhost:44134" - tiller -listen ${HELM_HOST} -alsologtostderr & - helm init --client-only - helm repo add app-name https://repository.example.com - helm repo update - #{helm_install_command} - /bin/date - /bin/false - EOS - end - - let(:helm_install_command) do - <<~EOS.squish - helm upgrade app-name chart-name - --install - --atomic - --cleanup-on-fail - --reset-values - --version 1.2.3 - --set rbac.create\\=false,rbac.enabled\\=false - --namespace gitlab-managed-apps - -f /data/helm/app-name/config/values.yaml - EOS - end - end - end - - context 'when there is no version' do - let(:version) { nil } - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - export HELM_HOST="localhost:44134" - tiller -listen ${HELM_HOST} -alsologtostderr & - helm init --client-only - helm repo add app-name https://repository.example.com - helm repo update - #{helm_install_command} - EOS - end - - let(:helm_install_command) do - <<~EOS.squish - helm upgrade app-name chart-name - --install - --atomic - --cleanup-on-fail - --reset-values - --set rbac.create\\=false,rbac.enabled\\=false - --namespace gitlab-managed-apps - -f /data/helm/app-name/config/values.yaml - EOS - end - end - end - - it_behaves_like 'helm command' do - let(:command) { install_command } - end -end diff --git a/spec/lib/gitlab/kubernetes/helm/v2/patch_command_spec.rb b/spec/lib/gitlab/kubernetes/helm/v2/patch_command_spec.rb deleted file mode 100644 index 98eb77d397c..00000000000 --- a/spec/lib/gitlab/kubernetes/helm/v2/patch_command_spec.rb +++ /dev/null @@ -1,87 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Kubernetes::Helm::V2::PatchCommand do - let(:files) { { 'ca.pem': 'some file content' } } - let(:repository) { 'https://repository.example.com' } - let(:rbac) { false } - let(:version) { '1.2.3' } - - subject(:patch_command) do - described_class.new( - name: 'app-name', - chart: 'chart-name', - rbac: rbac, - files: files, - version: version, - repository: repository - ) - end - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - export HELM_HOST="localhost:44134" - tiller -listen ${HELM_HOST} -alsologtostderr & - helm init --client-only - helm repo add app-name https://repository.example.com - helm repo update - #{helm_upgrade_comand} - EOS - end - - let(:helm_upgrade_comand) do - <<~EOS.squish - helm upgrade app-name chart-name - --reuse-values - --version 1.2.3 - --namespace gitlab-managed-apps - -f /data/helm/app-name/config/values.yaml - EOS - end - end - - context 'when rbac is true' do - let(:rbac) { true } - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - export HELM_HOST="localhost:44134" - tiller -listen ${HELM_HOST} -alsologtostderr & - helm init --client-only - helm repo add app-name https://repository.example.com - helm repo update - #{helm_upgrade_command} - EOS - end - - let(:helm_upgrade_command) do - <<~EOS.squish - helm upgrade app-name chart-name - --reuse-values - --version 1.2.3 - --namespace gitlab-managed-apps - -f /data/helm/app-name/config/values.yaml - EOS - end - end - end - - context 'when there is no version' do - let(:version) { nil } - - it { expect { patch_command }.to raise_error(ArgumentError, 'version is required') } - end - - describe '#pod_name' do - subject { patch_command.pod_name } - - it { is_expected.to eq 'install-app-name' } - end - - it_behaves_like 'helm command' do - let(:command) { patch_command } - end -end diff --git a/spec/lib/gitlab/kubernetes/helm/v2/reset_command_spec.rb b/spec/lib/gitlab/kubernetes/helm/v2/reset_command_spec.rb deleted file mode 100644 index 2a3a4cec2b0..00000000000 --- a/spec/lib/gitlab/kubernetes/helm/v2/reset_command_spec.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Kubernetes::Helm::V2::ResetCommand do - subject(:reset_command) { described_class.new(name: name, rbac: rbac, files: files) } - - let(:rbac) { true } - let(:name) { 'helm' } - let(:files) { {} } - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - export HELM_HOST="localhost:44134" - tiller -listen ${HELM_HOST} -alsologtostderr & - helm init --client-only - helm reset --force - EOS - end - end - - describe '#pod_name' do - subject { reset_command.pod_name } - - it { is_expected.to eq('uninstall-helm') } - end - - it_behaves_like 'helm command' do - let(:command) { reset_command } - end -end diff --git a/spec/lib/gitlab/kubernetes/helm/v3/base_command_spec.rb b/spec/lib/gitlab/kubernetes/helm/v3/base_command_spec.rb deleted file mode 100644 index ad5ff13b4c9..00000000000 --- a/spec/lib/gitlab/kubernetes/helm/v3/base_command_spec.rb +++ /dev/null @@ -1,44 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Kubernetes::Helm::V3::BaseCommand do - subject(:base_command) do - test_class.new(rbac) - end - - let(:application) { create(:clusters_applications_helm) } - let(:rbac) { false } - - let(:test_class) do - Class.new(described_class) do - def initialize(rbac) - super( - name: 'test-class-name', - rbac: rbac, - files: { some: 'value' } - ) - end - end - end - - describe 'HELM_VERSION' do - subject { described_class::HELM_VERSION } - - it { is_expected.to match /^3\.\d+\.\d+$/ } - end - - it_behaves_like 'helm command generator' do - let(:commands) { '' } - end - - describe '#pod_name' do - subject { base_command.pod_name } - - it { is_expected.to eq('install-test-class-name') } - end - - it_behaves_like 'helm command' do - let(:command) { base_command } - end -end diff --git a/spec/lib/gitlab/kubernetes/helm/v3/delete_command_spec.rb b/spec/lib/gitlab/kubernetes/helm/v3/delete_command_spec.rb deleted file mode 100644 index 63e7a8d2f25..00000000000 --- a/spec/lib/gitlab/kubernetes/helm/v3/delete_command_spec.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Kubernetes::Helm::V3::DeleteCommand do - subject(:delete_command) { described_class.new(name: app_name, rbac: rbac, files: files) } - - let(:app_name) { 'app-name' } - let(:rbac) { true } - let(:files) { {} } - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - helm uninstall app-name --namespace gitlab-managed-apps - EOS - end - end - - describe '#pod_name' do - subject { delete_command.pod_name } - - it { is_expected.to eq('uninstall-app-name') } - end - - it_behaves_like 'helm command' do - let(:command) { delete_command } - end - - describe '#delete_command' do - it 'deletes the release' do - expect(subject.delete_command).to eq('helm uninstall app-name --namespace gitlab-managed-apps') - end - end -end diff --git a/spec/lib/gitlab/kubernetes/helm/v3/install_command_spec.rb b/spec/lib/gitlab/kubernetes/helm/v3/install_command_spec.rb deleted file mode 100644 index 2bf1f713b3f..00000000000 --- a/spec/lib/gitlab/kubernetes/helm/v3/install_command_spec.rb +++ /dev/null @@ -1,168 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Kubernetes::Helm::V3::InstallCommand do - subject(:install_command) do - described_class.new( - name: 'app-name', - chart: 'chart-name', - rbac: rbac, - files: files, - version: version, - repository: repository, - preinstall: preinstall, - postinstall: postinstall - ) - end - - let(:files) { { 'ca.pem': 'some file content' } } - let(:repository) { 'https://repository.example.com' } - let(:rbac) { false } - let(:version) { '1.2.3' } - let(:preinstall) { nil } - let(:postinstall) { nil } - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - helm repo add app-name https://repository.example.com - helm repo update - #{helm_install_comand} - EOS - end - - let(:helm_install_comand) do - <<~EOS.squish - helm upgrade app-name chart-name - --install - --atomic - --cleanup-on-fail - --reset-values - --version 1.2.3 - --set rbac.create\\=false,rbac.enabled\\=false - --namespace gitlab-managed-apps - -f /data/helm/app-name/config/values.yaml - EOS - end - end - - context 'when rbac is true' do - let(:rbac) { true } - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - helm repo add app-name https://repository.example.com - helm repo update - #{helm_install_command} - EOS - end - - let(:helm_install_command) do - <<~EOS.squish - helm upgrade app-name chart-name - --install - --atomic - --cleanup-on-fail - --reset-values - --version 1.2.3 - --set rbac.create\\=true,rbac.enabled\\=true - --namespace gitlab-managed-apps - -f /data/helm/app-name/config/values.yaml - EOS - end - end - end - - context 'when there is a pre-install script' do - let(:preinstall) { ['/bin/date', '/bin/true'] } - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - helm repo add app-name https://repository.example.com - helm repo update - /bin/date - /bin/true - #{helm_install_command} - EOS - end - - let(:helm_install_command) do - <<~EOS.squish - helm upgrade app-name chart-name - --install - --atomic - --cleanup-on-fail - --reset-values - --version 1.2.3 - --set rbac.create\\=false,rbac.enabled\\=false - --namespace gitlab-managed-apps - -f /data/helm/app-name/config/values.yaml - EOS - end - end - end - - context 'when there is a post-install script' do - let(:postinstall) { ['/bin/date', "/bin/false\n"] } - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - helm repo add app-name https://repository.example.com - helm repo update - #{helm_install_command} - /bin/date - /bin/false - EOS - end - - let(:helm_install_command) do - <<~EOS.squish - helm upgrade app-name chart-name - --install - --atomic - --cleanup-on-fail - --reset-values - --version 1.2.3 - --set rbac.create\\=false,rbac.enabled\\=false - --namespace gitlab-managed-apps - -f /data/helm/app-name/config/values.yaml - EOS - end - end - end - - context 'when there is no version' do - let(:version) { nil } - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - helm repo add app-name https://repository.example.com - helm repo update - #{helm_install_command} - EOS - end - - let(:helm_install_command) do - <<~EOS.squish - helm upgrade app-name chart-name - --install - --atomic - --cleanup-on-fail - --reset-values - --set rbac.create\\=false,rbac.enabled\\=false - --namespace gitlab-managed-apps - -f /data/helm/app-name/config/values.yaml - EOS - end - end - end - - it_behaves_like 'helm command' do - let(:command) { install_command } - end -end diff --git a/spec/lib/gitlab/kubernetes/helm/v3/patch_command_spec.rb b/spec/lib/gitlab/kubernetes/helm/v3/patch_command_spec.rb deleted file mode 100644 index 2f22e0f2e77..00000000000 --- a/spec/lib/gitlab/kubernetes/helm/v3/patch_command_spec.rb +++ /dev/null @@ -1,81 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Kubernetes::Helm::V3::PatchCommand do - let(:files) { { 'ca.pem': 'some file content' } } - let(:repository) { 'https://repository.example.com' } - let(:rbac) { false } - let(:version) { '1.2.3' } - - subject(:patch_command) do - described_class.new( - name: 'app-name', - chart: 'chart-name', - rbac: rbac, - files: files, - version: version, - repository: repository - ) - end - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - helm repo add app-name https://repository.example.com - helm repo update - #{helm_upgrade_comand} - EOS - end - - let(:helm_upgrade_comand) do - <<~EOS.squish - helm upgrade app-name chart-name - --reuse-values - --version 1.2.3 - --namespace gitlab-managed-apps - -f /data/helm/app-name/config/values.yaml - EOS - end - end - - context 'when rbac is true' do - let(:rbac) { true } - - it_behaves_like 'helm command generator' do - let(:commands) do - <<~EOS - helm repo add app-name https://repository.example.com - helm repo update - #{helm_upgrade_command} - EOS - end - - let(:helm_upgrade_command) do - <<~EOS.squish - helm upgrade app-name chart-name - --reuse-values - --version 1.2.3 - --namespace gitlab-managed-apps - -f /data/helm/app-name/config/values.yaml - EOS - end - end - end - - context 'when there is no version' do - let(:version) { nil } - - it { expect { patch_command }.to raise_error(ArgumentError, 'version is required') } - end - - describe '#pod_name' do - subject { patch_command.pod_name } - - it { is_expected.to eq 'install-app-name' } - end - - it_behaves_like 'helm command' do - let(:command) { patch_command } - end -end diff --git a/spec/lib/gitlab/legacy_github_import/client_spec.rb b/spec/lib/gitlab/legacy_github_import/client_spec.rb index 08679b7e9f1..d0f63d11469 100644 --- a/spec/lib/gitlab/legacy_github_import/client_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/client_spec.rb @@ -4,7 +4,7 @@ require 'spec_helper' RSpec.describe Gitlab::LegacyGithubImport::Client do let(:token) { '123456' } - let(:github_provider) { Settingslogic.new('app_id' => 'asd123', 'app_secret' => 'asd123', 'name' => 'github', 'args' => { 'client_options' => {} }) } + let(:github_provider) { GitlabSettings::Options.build('app_id' => 'asd123', 'app_secret' => 'asd123', 'name' => 'github', 'args' => { 'client_options' => {} }) } let(:wait_for_rate_limit_reset) { true } subject(:client) { described_class.new(token, wait_for_rate_limit_reset: wait_for_rate_limit_reset) } @@ -17,7 +17,7 @@ RSpec.describe Gitlab::LegacyGithubImport::Client do expect(client.client.options.keys).to all(be_kind_of(Symbol)) end - it 'does not crash (e.g. Settingslogic::MissingSetting) when verify_ssl config is not present' do + it 'does not crash (e.g. GitlabSettings::MissingSetting) when verify_ssl config is not present' do expect { client.api }.not_to raise_error end diff --git a/spec/lib/gitlab/legacy_github_import/importer_spec.rb b/spec/lib/gitlab/legacy_github_import/importer_spec.rb index cd66b93eb8b..bb38f4b1bca 100644 --- a/spec/lib/gitlab/legacy_github_import/importer_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/importer_spec.rb @@ -2,7 +2,9 @@ require 'spec_helper' -RSpec.describe Gitlab::LegacyGithubImport::Importer do +RSpec.describe Gitlab::LegacyGithubImport::Importer, feature_category: :importers do + subject(:importer) { described_class.new(project) } + shared_examples 'Gitlab::LegacyGithubImport::Importer#execute' do let(:expected_not_called) { [] } @@ -11,8 +13,6 @@ RSpec.describe Gitlab::LegacyGithubImport::Importer do end it 'calls import methods' do - importer = described_class.new(project) - expected_called = [ :import_labels, :import_milestones, :import_pull_requests, :import_issues, :import_wiki, :import_releases, :handle_errors, @@ -51,11 +51,13 @@ RSpec.describe Gitlab::LegacyGithubImport::Importer do allow_any_instance_of(Octokit::Client).to receive(:labels).and_return([label1, label2]) allow_any_instance_of(Octokit::Client).to receive(:milestones).and_return([milestone, milestone]) allow_any_instance_of(Octokit::Client).to receive(:issues).and_return([issue1, issue2]) - allow_any_instance_of(Octokit::Client).to receive(:pull_requests).and_return([pull_request, pull_request]) + allow_any_instance_of(Octokit::Client).to receive(:pull_requests).and_return([pull_request, pull_request_missing_source_branch]) allow_any_instance_of(Octokit::Client).to receive(:issues_comments).and_raise(Octokit::NotFound) allow_any_instance_of(Octokit::Client).to receive(:pull_requests_comments).and_return([]) allow_any_instance_of(Octokit::Client).to receive(:last_response).and_return(double(rels: { next: nil })) allow_any_instance_of(Octokit::Client).to receive(:releases).and_return([release1, release2]) + + allow(importer).to receive(:restore_source_branch).and_raise(StandardError, 'Some error') end let(:label1) do @@ -153,8 +155,6 @@ RSpec.describe Gitlab::LegacyGithubImport::Importer do } end - subject { described_class.new(project) } - it 'returns true' do expect(subject.execute).to eq true end @@ -163,18 +163,19 @@ RSpec.describe Gitlab::LegacyGithubImport::Importer do expect { subject.execute }.not_to raise_error end - it 'stores error messages' do + it 'stores error messages', :unlimited_max_formatted_output_length do error = { message: 'The remote data could not be fully imported.', errors: [ { type: :label, url: "#{api_root}/repos/octocat/Hello-World/labels/bug", errors: "Validation failed: Title can't be blank, Title is invalid" }, + { type: :pull_request, url: "#{api_root}/repos/octocat/Hello-World/pulls/1347", errors: 'Some error' }, { type: :issue, url: "#{api_root}/repos/octocat/Hello-World/issues/1348", errors: "Validation failed: Title can't be blank" }, { type: :issues_comments, errors: 'Octokit::NotFound' }, { type: :wiki, errors: "Gitlab::Git::CommandError" } ] } - described_class.new(project).execute + importer.execute expect(project.import_state.last_error).to eq error.to_json end @@ -182,8 +183,6 @@ RSpec.describe Gitlab::LegacyGithubImport::Importer do shared_examples 'Gitlab::LegacyGithubImport unit-testing' do describe '#clean_up_restored_branches' do - subject { described_class.new(project) } - before do allow(gh_pull_request).to receive(:source_branch_exists?).at_least(:once) { false } allow(gh_pull_request).to receive(:target_branch_exists?).at_least(:once) { false } @@ -240,6 +239,16 @@ RSpec.describe Gitlab::LegacyGithubImport::Importer do } end + let(:pull_request_missing_source_branch) do + pull_request.merge( + head: { + ref: 'missing', + repo: repository, + sha: RepoHelpers.another_sample_commit + } + ) + end + let(:closed_pull_request) do { number: 1347, @@ -264,8 +273,6 @@ RSpec.describe Gitlab::LegacyGithubImport::Importer do let(:api_root) { 'https://try.gitea.io/api/v1' } let(:repo_root) { 'https://try.gitea.io' } - subject { described_class.new(project) } - before do project.update!(import_type: 'gitea', import_url: "#{repo_root}/foo/group/project.git") end diff --git a/spec/lib/gitlab/legacy_github_import/project_creator_spec.rb b/spec/lib/gitlab/legacy_github_import/project_creator_spec.rb index 17ecd183ac9..15624a0558e 100644 --- a/spec/lib/gitlab/legacy_github_import/project_creator_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/project_creator_spec.rb @@ -20,9 +20,11 @@ RSpec.describe Gitlab::LegacyGithubImport::ProjectCreator do before do namespace.add_owner(user) - expect_next_instance_of(Project) do |project| + allow_next_instance_of(Project) do |project| allow(project).to receive(:add_import_job) end + + stub_application_setting(import_sources: ['github']) end describe '#execute' do diff --git a/spec/lib/gitlab/legacy_github_import/user_formatter_spec.rb b/spec/lib/gitlab/legacy_github_import/user_formatter_spec.rb index bc127f74e84..0844ab7eccc 100644 --- a/spec/lib/gitlab/legacy_github_import/user_formatter_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/user_formatter_spec.rb @@ -5,14 +5,15 @@ require 'spec_helper' RSpec.describe Gitlab::LegacyGithubImport::UserFormatter do let(:client) { double } let(:octocat) { { id: 123456, login: 'octocat', email: 'octocat@example.com' } } + let(:gitea_ghost) { { id: -1, login: 'Ghost', email: '' } } - subject(:user) { described_class.new(client, octocat) } + describe '#gitlab_id' do + subject(:user) { described_class.new(client, octocat) } - before do - allow(client).to receive(:user).and_return(octocat) - end + before do + allow(client).to receive(:user).and_return(octocat) + end - describe '#gitlab_id' do context 'when GitHub user is a GitLab user' do it 'return GitLab user id when user associated their account with GitHub' do gl_user = create(:omniauth_user, extern_uid: octocat[:id], provider: 'github') @@ -51,4 +52,16 @@ RSpec.describe Gitlab::LegacyGithubImport::UserFormatter do expect(user.gitlab_id).to be_nil end end + + describe '.email' do + subject(:user) { described_class.new(client, gitea_ghost) } + + before do + allow(client).to receive(:user).and_return(gitea_ghost) + end + + it 'assigns a dummy email address when user is a Ghost gitea user' do + expect(subject.send(:email)).to eq described_class::GITEA_GHOST_EMAIL + end + end end diff --git a/spec/lib/gitlab/loggable_spec.rb b/spec/lib/gitlab/loggable_spec.rb new file mode 100644 index 00000000000..8238e47014b --- /dev/null +++ b/spec/lib/gitlab/loggable_spec.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Loggable, feature_category: :logging do + subject(:klass_instance) do + Class.new do + include Gitlab::Loggable + + def self.name + 'MyTestClass' + end + end.new + end + + describe '#build_structured_payload' do + it 'adds class and returns formatted json' do + expected = { + 'class' => 'MyTestClass', + 'message' => 'test' + } + + expect(klass_instance.build_structured_payload(message: 'test')).to eq(expected) + end + + it 'appends additional params and returns formatted json' do + expected = { + 'class' => 'MyTestClass', + 'message' => 'test', + 'extra_param' => 1 + } + + expect(klass_instance.build_structured_payload(message: 'test', extra_param: 1)).to eq(expected) + end + + it 'does not raise an error in loggers when passed non-symbols' do + expected = { + 'class' => 'MyTestClass', + 'message' => 'test', + '["hello", "thing"]' => :world + } + + payload = klass_instance.build_structured_payload(message: 'test', %w[hello thing] => :world) + expect(payload).to eq(expected) + expect { Gitlab::Export::Logger.info(payload) }.not_to raise_error + end + + it 'handles anonymous classes' do + anonymous_klass_instance = Class.new { include Gitlab::Loggable }.new + + expected = { + 'class' => '<Anonymous>', + 'message' => 'test' + } + + expect(anonymous_klass_instance.build_structured_payload(message: 'test')).to eq(expected) + end + + it 'handles duplicate keys' do + expected = { + 'class' => 'MyTestClass', + 'message' => 'test2' + } + + expect(klass_instance.build_structured_payload(message: 'test', 'message' => 'test2')).to eq(expected) + end + end +end diff --git a/spec/lib/gitlab/manifest_import/project_creator_spec.rb b/spec/lib/gitlab/manifest_import/project_creator_spec.rb index 0ab5b277552..2d878e5496e 100644 --- a/spec/lib/gitlab/manifest_import/project_creator_spec.rb +++ b/spec/lib/gitlab/manifest_import/project_creator_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::ManifestImport::ProjectCreator do +RSpec.describe Gitlab::ManifestImport::ProjectCreator, feature_category: :importers do let(:group) { create(:group) } let(:user) { create(:user) } let(:repository) do @@ -14,6 +14,8 @@ RSpec.describe Gitlab::ManifestImport::ProjectCreator do before do group.add_owner(user) + + stub_application_setting(import_sources: ['manifest']) end subject { described_class.new(repository, group, user) } diff --git a/spec/lib/gitlab/memory/watchdog/monitor/rss_memory_limit_spec.rb b/spec/lib/gitlab/memory/watchdog/monitor/rss_memory_limit_spec.rb index 4780b1eba53..67d185fd2f1 100644 --- a/spec/lib/gitlab/memory/watchdog/monitor/rss_memory_limit_spec.rb +++ b/spec/lib/gitlab/memory/watchdog/monitor/rss_memory_limit_spec.rb @@ -1,9 +1,10 @@ # frozen_string_literal: true require 'fast_spec_helper' +require 'prometheus/client' require 'support/shared_examples/lib/gitlab/memory/watchdog/monitor_result_shared_examples' -RSpec.describe Gitlab::Memory::Watchdog::Monitor::RssMemoryLimit do +RSpec.describe Gitlab::Memory::Watchdog::Monitor::RssMemoryLimit, feature_category: :application_performance do let(:max_rss_limit_gauge) { instance_double(::Prometheus::Client::Gauge) } let(:memory_limit_bytes) { 2_097_152_000 } let(:worker_memory_bytes) { 1_048_576_000 } diff --git a/spec/lib/gitlab/metrics/boot_time_tracker_spec.rb b/spec/lib/gitlab/metrics/boot_time_tracker_spec.rb index 8a17fa8dd2e..3175c0a6b32 100644 --- a/spec/lib/gitlab/metrics/boot_time_tracker_spec.rb +++ b/spec/lib/gitlab/metrics/boot_time_tracker_spec.rb @@ -1,8 +1,8 @@ # frozen_string_literal: true -require 'fast_spec_helper' +require 'spec_helper' -RSpec.describe Gitlab::Metrics::BootTimeTracker do +RSpec.describe Gitlab::Metrics::BootTimeTracker, feature_category: :metrics do let(:logger) { double('logger') } let(:gauge) { double('gauge') } diff --git a/spec/lib/gitlab/metrics/dashboard/finder_spec.rb b/spec/lib/gitlab/metrics/dashboard/finder_spec.rb index f922eff2980..d3cb9760052 100644 --- a/spec/lib/gitlab/metrics/dashboard/finder_spec.rb +++ b/spec/lib/gitlab/metrics/dashboard/finder_spec.rb @@ -44,12 +44,6 @@ RSpec.describe Gitlab::Metrics::Dashboard::Finder, :use_clean_rails_memory_store it_behaves_like 'valid dashboard service response' end - context 'when the self-monitoring dashboard is specified' do - let(:dashboard_path) { self_monitoring_dashboard_path } - - it_behaves_like 'valid dashboard service response' - end - context 'when no dashboard is specified' do let(:service_call) { described_class.find(project, user, environment: environment) } @@ -180,36 +174,5 @@ RSpec.describe Gitlab::Metrics::Dashboard::Finder, :use_clean_rails_memory_store expect(all_dashboard_paths).to eq([project_dashboard2, k8s_pod_health_dashboard, project_dashboard1, system_dashboard]) end end - - context 'when the project is self-monitoring' do - let(:self_monitoring_dashboard) do - { - path: self_monitoring_dashboard_path, - display_name: 'Overview', - default: true, - system_dashboard: true, - out_of_the_box_dashboard: true - } - end - - let(:dashboard_path) { '.gitlab/dashboards/test.yml' } - let(:project) { project_with_dashboard(dashboard_path) } - - before do - stub_application_setting(self_monitoring_project_id: project.id) - end - - it 'includes self-monitoring and project dashboards' do - project_dashboard = { - path: dashboard_path, - display_name: 'test.yml', - default: false, - system_dashboard: false, - out_of_the_box_dashboard: false - } - - expect(all_dashboard_paths).to eq([self_monitoring_dashboard, project_dashboard]) - end - end end end diff --git a/spec/lib/gitlab/metrics/dashboard/service_selector_spec.rb b/spec/lib/gitlab/metrics/dashboard/service_selector_spec.rb index b41b51f53c3..343596af5cf 100644 --- a/spec/lib/gitlab/metrics/dashboard/service_selector_spec.rb +++ b/spec/lib/gitlab/metrics/dashboard/service_selector_spec.rb @@ -30,12 +30,6 @@ RSpec.describe Gitlab::Metrics::Dashboard::ServiceSelector do end end - context 'when the path is for the self-monitoring dashboard' do - let(:arguments) { { dashboard_path: self_monitoring_dashboard_path } } - - it { is_expected.to be Metrics::Dashboard::SelfMonitoringDashboardService } - end - context 'when the embedded flag is provided' do let(:arguments) { { embedded: true } } diff --git a/spec/lib/gitlab/metrics/dashboard/stages/grafana_formatter_spec.rb b/spec/lib/gitlab/metrics/dashboard/stages/grafana_formatter_spec.rb index 8a236f72a60..3cfdfafb0c5 100644 --- a/spec/lib/gitlab/metrics/dashboard/stages/grafana_formatter_spec.rb +++ b/spec/lib/gitlab/metrics/dashboard/stages/grafana_formatter_spec.rb @@ -5,8 +5,8 @@ require 'spec_helper' RSpec.describe Gitlab::Metrics::Dashboard::Stages::GrafanaFormatter do include GrafanaApiHelpers - let_it_be(:namespace) { create(:namespace, name: 'foo') } - let_it_be(:project) { create(:project, namespace: namespace, name: 'bar') } + let_it_be(:namespace) { create(:namespace, path: 'foo') } + let_it_be(:project) { create(:project, namespace: namespace, path: 'bar') } describe '#transform!' do let(:grafana_dashboard) { Gitlab::Json.parse(fixture_file('grafana/simplified_dashboard_response.json'), symbolize_names: true) } diff --git a/spec/lib/gitlab/metrics/sidekiq_slis_spec.rb b/spec/lib/gitlab/metrics/sidekiq_slis_spec.rb new file mode 100644 index 00000000000..eef9a9c79e6 --- /dev/null +++ b/spec/lib/gitlab/metrics/sidekiq_slis_spec.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Metrics::SidekiqSlis, feature_category: :error_budgets do + using RSpec::Parameterized::TableSyntax + + describe ".initialize_slis!" do + let(:possible_labels) do + [ + { + worker: "Projects::RecordTargetPlatformsWorker", + feature_category: "projects", + urgency: "low" + } + ] + end + + it "initializes the apdex and error rate SLIs" do + expect(Gitlab::Metrics::Sli::Apdex).to receive(:initialize_sli).with(:sidekiq_execution, possible_labels) + expect(Gitlab::Metrics::Sli::ErrorRate).to receive(:initialize_sli).with(:sidekiq_execution, possible_labels) + + described_class.initialize_slis!(possible_labels) + end + end + + describe ".record_execution_apdex" do + where(:urgency, :duration, :success) do + "high" | 5 | true + "high" | 11 | false + "low" | 295 | true + "low" | 400 | false + "throttled" | 295 | true + "throttled" | 400 | false + "not_found" | 295 | true + "not_found" | 400 | false + end + + with_them do + it "increments the apdex SLI with success based on urgency requirement" do + labels = { urgency: urgency } + expect(Gitlab::Metrics::Sli::Apdex[:sidekiq_execution]).to receive(:increment).with( + labels: labels, + success: success + ) + + described_class.record_execution_apdex(labels, duration) + end + end + end + + describe ".record_execution_error" do + it "increments the error rate SLI with the given labels and error" do + labels = { urgency: :throttled } + error = StandardError.new("something went wrong") + + expect(Gitlab::Metrics::Sli::ErrorRate[:sidekiq_execution]).to receive(:increment).with( + labels: labels, + error: error + ) + + described_class.record_execution_error(labels, error) + end + end +end diff --git a/spec/lib/gitlab/metrics/subscribers/action_cable_spec.rb b/spec/lib/gitlab/metrics/subscribers/action_cable_spec.rb index 08437920e0c..54868bb6ca4 100644 --- a/spec/lib/gitlab/metrics/subscribers/action_cable_spec.rb +++ b/spec/lib/gitlab/metrics/subscribers/action_cable_spec.rb @@ -2,22 +2,25 @@ require 'spec_helper' -RSpec.describe Gitlab::Metrics::Subscribers::ActionCable, :request_store do +RSpec.describe Gitlab::Metrics::Subscribers::ActionCable, :request_store, feature_category: :application_performance do let(:subscriber) { described_class.new } let(:counter) { double(:counter) } - let(:data) { { 'result' => { 'data' => { 'event' => 'updated' } } } } + let(:transmitted_bytes_counter) { double(:counter) } let(:channel_class) { 'IssuesChannel' } - let(:event) do - double( - :event, - name: name, - payload: payload - ) + let(:event) { double(:event, name: name, payload: payload) } + + before do + allow(::Gitlab::Metrics).to receive(:counter).with( + :action_cable_single_client_transmissions_total, /transmit/ + ).and_return(counter) + allow(::Gitlab::Metrics).to receive(:counter).with( + :action_cable_transmitted_bytes_total, /transmit/ + ).and_return(transmitted_bytes_counter) end describe '#transmit' do let(:name) { 'transmit.action_cable' } - let(:via) { 'streamed from issues:Z2lkOi8vZs2l0bGFiL0lzc3VlLzQ0Ng' } + let(:via) { nil } let(:payload) do { channel_class: channel_class, @@ -26,25 +29,71 @@ RSpec.describe Gitlab::Metrics::Subscribers::ActionCable, :request_store do } end - it 'tracks the transmit event' do - allow(::Gitlab::Metrics).to receive(:counter).with( - :action_cable_single_client_transmissions_total, /transmit/ - ).and_return(counter) + let(:message_size) { ::Gitlab::Json.generate(data).bytesize } - expect(counter).to receive(:increment) + context 'for transmissions initiated by Channel instance' do + let(:data) { {} } + let(:expected_labels) do + { + channel: channel_class, + broadcasting: nil, + caller: 'channel' + } + end - subscriber.transmit(event) + it 'tracks the event with "caller" set to "channel"' do + expect(counter).to receive(:increment).with(expected_labels) + expect(transmitted_bytes_counter).to receive(:increment).with(expected_labels, message_size) + + subscriber.transmit(event) + end end - it 'tracks size of payload as JSON' do - allow(::Gitlab::Metrics).to receive(:histogram).with( - :action_cable_transmitted_bytes, /transmit/ - ).and_return(counter) - message_size = ::Gitlab::Json.generate(data).bytesize + context 'for transmissions initiated by GraphQL event subscriber' do + let(:via) { 'streamed from graphql-subscription:09ae595a-45c4-4ae0-b765-4e503203211d' } + let(:data) { { result: { 'data' => { 'issuableEpicUpdated' => '<GQL query result>' } } } } + let(:expected_labels) do + { + channel: channel_class, + broadcasting: 'graphql-event:issuableEpicUpdated', + caller: 'graphql-subscription' + } + end + + it 'tracks the event with correct "caller" and "broadcasting"' do + expect(counter).to receive(:increment).with(expected_labels) + expect(transmitted_bytes_counter).to receive(:increment).with(expected_labels, message_size) - expect(counter).to receive(:observe).with({ channel: channel_class, operation: 'event' }, message_size) + subscriber.transmit(event) + end - subscriber.transmit(event) + it 'is indifferent to keys being symbols or strings in result payload' do + expect(counter).to receive(:increment).with(expected_labels) + expect(transmitted_bytes_counter).to receive(:increment).with(expected_labels, message_size) + + event.payload[:data].deep_stringify_keys! + + subscriber.transmit(event) + end + end + + context 'when transmission is coming from unknown source' do + let(:via) { 'streamed from something else' } + let(:data) { {} } + let(:expected_labels) do + { + channel: channel_class, + broadcasting: nil, + caller: 'unknown' + } + end + + it 'tracks the event with "caller" set to "unknown"' do + expect(counter).to receive(:increment).with(expected_labels) + expect(transmitted_bytes_counter).to receive(:increment).with(expected_labels, message_size) + + subscriber.transmit(event) + end end end @@ -55,7 +104,6 @@ RSpec.describe Gitlab::Metrics::Subscribers::ActionCable, :request_store do { event: :updated } end - let(:broadcasting) { 'issues:Z2lkOi8vZ2l0bGFiL0lzc3VlLzQ0Ng' } let(:payload) do { broadcasting: broadcasting, @@ -64,14 +112,40 @@ RSpec.describe Gitlab::Metrics::Subscribers::ActionCable, :request_store do } end - it 'tracks the broadcast event' do + before do allow(::Gitlab::Metrics).to receive(:counter).with( :action_cable_broadcasts_total, /broadcast/ ).and_return(counter) + end - expect(counter).to receive(:increment) + context 'when broadcast is for a GraphQL event' do + let(:broadcasting) { 'graphql-event::issuableEpicUpdated:issuableId:Z2lkOi8vZ2l0bGFiL0lzc3VlLzM' } + + it 'tracks the event with broadcasting set to event topic' do + expect(counter).to receive(:increment).with({ broadcasting: 'graphql-event:issuableEpicUpdated' }) + + subscriber.broadcast(event) + end + end + + context 'when broadcast is for a GraphQL channel subscription' do + let(:broadcasting) { 'graphql-subscription:09ae595a-45c4-4ae0-b765-4e503203211d' } + + it 'strips out subscription ID from broadcasting' do + expect(counter).to receive(:increment).with({ broadcasting: 'graphql-subscription' }) + + subscriber.broadcast(event) + end + end + + context 'when broadcast is something else' do + let(:broadcasting) { 'unknown-topic' } + + it 'tracks the event as "unknown"' do + expect(counter).to receive(:increment).with({ broadcasting: 'unknown' }) - subscriber.broadcast(event) + subscriber.broadcast(event) + end end end diff --git a/spec/lib/gitlab/metrics/subscribers/active_record_spec.rb b/spec/lib/gitlab/metrics/subscribers/active_record_spec.rb index 7ce5cbec18d..afb029a96cb 100644 --- a/spec/lib/gitlab/metrics/subscribers/active_record_spec.rb +++ b/spec/lib/gitlab/metrics/subscribers/active_record_spec.rb @@ -226,7 +226,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::ActiveRecord do # Emulate Marginalia pre-pending comments def sql(query, comments: true) - if comments && !%w[BEGIN COMMIT].include?(query) + if comments "/*application:web,controller:badges,action:pipeline,correlation_id:01EYN39K9VMJC56Z7808N7RSRH*/ #{query}" else query @@ -244,8 +244,9 @@ RSpec.describe Gitlab::Metrics::Subscribers::ActiveRecord do 'SQL' | 'UPDATE users SET admin = true WHERE id = 10' | true | true | false 'CACHE' | 'SELECT * FROM users WHERE id = 10' | true | false | true 'SCHEMA' | "SELECT attr.attname FROM pg_attribute attr INNER JOIN pg_constraint cons ON attr.attrelid = cons.conrelid AND attr.attnum = any(cons.conkey) WHERE cons.contype = 'p' AND cons.conrelid = '\"projects\"'::regclass" | false | false | false - nil | 'BEGIN' | false | false | false - nil | 'COMMIT' | false | false | false + 'TRANSACTION' | 'BEGIN' | false | false | false + 'TRANSACTION' | 'COMMIT' | false | false | false + 'TRANSACTION' | 'ROLLBACK' | false | false | false end with_them do @@ -291,7 +292,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::ActiveRecord do # Emulate Marginalia pre-pending comments def sql(query, comments: true) - if comments && !%w[BEGIN COMMIT].include?(query) + if comments "/*application:web,controller:badges,action:pipeline,correlation_id:01EYN39K9VMJC56Z7808N7RSRH*/ #{query}" else query @@ -313,8 +314,9 @@ RSpec.describe Gitlab::Metrics::Subscribers::ActiveRecord do 'CACHE' | 'SELECT pg_last_wal_replay_lsn()::text AS location' | true | false | true | true 'CACHE' | 'SELECT * FROM users WHERE id = 10' | true | false | true | false 'SCHEMA' | "SELECT attr.attname FROM pg_attribute attr INNER JOIN pg_constraint cons ON attr.attrelid = cons.conrelid AND attr.attnum = any(cons.conkey) WHERE cons.contype = 'p' AND cons.conrelid = '\"projects\"'::regclass" | false | false | false | false - nil | 'BEGIN' | false | false | false | false - nil | 'COMMIT' | false | false | false | false + 'TRANSACTION' | 'BEGIN' | false | false | false | false + 'TRANSACTION' | 'COMMIT' | false | false | false | false + 'TRANSACTION' | 'ROLLBACK' | false | false | false | false end with_them do diff --git a/spec/lib/gitlab/metrics/subscribers/external_http_spec.rb b/spec/lib/gitlab/metrics/subscribers/external_http_spec.rb index e489ac97b9c..18a5d2c2c3f 100644 --- a/spec/lib/gitlab/metrics/subscribers/external_http_spec.rb +++ b/spec/lib/gitlab/metrics/subscribers/external_http_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Metrics::Subscribers::ExternalHttp, :request_store do +RSpec.describe Gitlab::Metrics::Subscribers::ExternalHttp, :request_store, feature_category: :logging do let(:transaction) { Gitlab::Metrics::WebTransaction.new({}) } let(:subscriber) { described_class.new } @@ -15,7 +15,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::ExternalHttp, :request_store do :event, payload: { method: 'POST', code: "200", duration: 0.321, - scheme: 'https', host: 'gitlab.com', port: 80, path: '/api/v4/projects', + scheme: 'https', host: 'gitlab.com', port: 443, path: '/api/v4/projects', query: 'current=true' }, time: Time.current @@ -95,6 +95,47 @@ RSpec.describe Gitlab::Metrics::Subscribers::ExternalHttp, :request_store do expect(described_class.payload).to eql(external_http_count: 7, external_http_duration_s: 1.2) end end + + context 'with multiple requests' do + let(:slow_requests) do + [ + { + method: 'POST', + host: 'gitlab.com', + port: 80, + path: '/api/v4/projects/2/issues', + duration_s: 5.3 + }, + { + method: 'POST', + host: 'gitlab.com', + port: 443, + path: '/api/v4/projects', + duration_s: 0.321 + } + ] + end + + before do + stub_const("#{described_class}::MAX_SLOW_REQUESTS", 2) + stub_const("#{described_class}::THRESHOLD_SLOW_REQUEST_S", 0.01) + + subscriber.request(event_1) + subscriber.request(event_2) + subscriber.request(event_3) + end + + it 'returns a payload containing a limited set of slow requests' do + expect(described_class.payload).to eq( + external_http_count: 3, + external_http_duration_s: 5.741, + external_http_slow_requests: slow_requests + ) + expect(described_class.top_slowest_requests).to eq(slow_requests) + + expect(Gitlab::SafeRequestStore[:external_http_slow_requests].length).to eq(3) + end + end end describe '#request' do @@ -153,7 +194,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::ExternalHttp, :request_store do expect(Gitlab::SafeRequestStore[:external_http_detail_store][0]).to match a_hash_including( start: be_like_time(Time.current), method: 'POST', code: "200", duration: 0.321, - scheme: 'https', host: 'gitlab.com', port: 80, path: '/api/v4/projects', + scheme: 'https', host: 'gitlab.com', port: 443, path: '/api/v4/projects', query: 'current=true', exception_object: nil, backtrace: be_a(Array) ) diff --git a/spec/lib/gitlab/metrics/subscribers/load_balancing_spec.rb b/spec/lib/gitlab/metrics/subscribers/load_balancing_spec.rb index b401b7cc996..c2c3bb29b16 100644 --- a/spec/lib/gitlab/metrics/subscribers/load_balancing_spec.rb +++ b/spec/lib/gitlab/metrics/subscribers/load_balancing_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Metrics::Subscribers::LoadBalancing, :request_store, feature_category: :pods do +RSpec.describe Gitlab::Metrics::Subscribers::LoadBalancing, :request_store, feature_category: :cell do let(:subscriber) { described_class.new } describe '#caught_up_replica_pick' do diff --git a/spec/lib/gitlab/metrics/subscribers/rack_attack_spec.rb b/spec/lib/gitlab/metrics/subscribers/rack_attack_spec.rb index 9f939d0d7d6..13965bf1244 100644 --- a/spec/lib/gitlab/metrics/subscribers/rack_attack_spec.rb +++ b/spec/lib/gitlab/metrics/subscribers/rack_attack_spec.rb @@ -32,33 +32,6 @@ RSpec.describe Gitlab::Metrics::Subscribers::RackAttack, :request_store do end end - describe '#redis' do - it 'accumulates per-request RackAttack cache usage' do - freeze_time do - subscriber.redis( - ActiveSupport::Notifications::Event.new( - 'redis.rack_attack', Time.current, Time.current + 1.second, '1', { operation: 'fetch' } - ) - ) - subscriber.redis( - ActiveSupport::Notifications::Event.new( - 'redis.rack_attack', Time.current, Time.current + 2.seconds, '1', { operation: 'write' } - ) - ) - subscriber.redis( - ActiveSupport::Notifications::Event.new( - 'redis.rack_attack', Time.current, Time.current + 3.seconds, '1', { operation: 'read' } - ) - ) - end - - expect(Gitlab::SafeRequestStore[:rack_attack_instrumentation]).to eql( - rack_attack_redis_count: 3, - rack_attack_redis_duration_s: 6.0 - ) - end - end - shared_examples 'log into auth logger' do context 'when matched throttle does not require user information' do let(:event) do diff --git a/spec/lib/gitlab/metrics/subscribers/rails_cache_spec.rb b/spec/lib/gitlab/metrics/subscribers/rails_cache_spec.rb index 59bfe2042fa..2d4c6d1cc56 100644 --- a/spec/lib/gitlab/metrics/subscribers/rails_cache_spec.rb +++ b/spec/lib/gitlab/metrics/subscribers/rails_cache_spec.rb @@ -6,13 +6,14 @@ RSpec.describe Gitlab::Metrics::Subscribers::RailsCache do let(:env) { {} } let(:transaction) { Gitlab::Metrics::WebTransaction.new(env) } let(:subscriber) { described_class.new } - - let(:event) { double(:event, duration: 15.2, payload: { key: %w[a b c] }) } + let(:store) { 'Gitlab::CustomStore' } + let(:store_label) { 'CustomStore' } + let(:event) { double(:event, duration: 15.2, payload: { key: %w[a b c], store: store }) } describe '#cache_read' do it 'increments the cache_read duration' do expect(subscriber).to receive(:observe) - .with(:read, event.duration) + .with(:read, event) subscriber.cache_read(event) end @@ -27,7 +28,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::RailsCache do let(:event) { double(:event, duration: 15.2, payload: { hit: true }) } context 'when super operation is fetch' do - let(:event) { double(:event, duration: 15.2, payload: { hit: true, super_operation: :fetch }) } + let(:event) { double(:event, duration: 15.2, payload: { hit: true, super_operation: :fetch, store: store }) } it 'does not increment cache read miss total' do expect(transaction).not_to receive(:increment) @@ -39,7 +40,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::RailsCache do end context 'with miss event' do - let(:event) { double(:event, duration: 15.2, payload: { hit: false }) } + let(:event) { double(:event, duration: 15.2, payload: { hit: false, store: store }) } it 'increments the cache_read_miss total' do expect(transaction).to receive(:increment) @@ -51,7 +52,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::RailsCache do end context 'when super operation is fetch' do - let(:event) { double(:event, duration: 15.2, payload: { hit: false, super_operation: :fetch }) } + let(:event) { double(:event, duration: 15.2, payload: { hit: false, super_operation: :fetch, store: store }) } it 'does not increment cache read miss total' do expect(transaction).not_to receive(:increment) @@ -75,7 +76,9 @@ RSpec.describe Gitlab::Metrics::Subscribers::RailsCache do it 'observes multi-key count' do expect(transaction).to receive(:observe) - .with(:gitlab_cache_read_multikey_count, event.payload[:key].size) + .with(:gitlab_cache_read_multikey_count, + event.payload[:key].size, + { store: store_label }) subject end @@ -92,7 +95,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::RailsCache do it 'observes read_multi duration' do expect(subscriber).to receive(:observe) - .with(:read_multi, event.duration) + .with(:read_multi, event) subject end @@ -101,7 +104,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::RailsCache do describe '#cache_write' do it 'observes write duration' do expect(subscriber).to receive(:observe) - .with(:write, event.duration) + .with(:write, event) subscriber.cache_write(event) end @@ -110,7 +113,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::RailsCache do describe '#cache_delete' do it 'observes delete duration' do expect(subscriber).to receive(:observe) - .with(:delete, event.duration) + .with(:delete, event) subscriber.cache_delete(event) end @@ -119,7 +122,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::RailsCache do describe '#cache_exist?' do it 'observes the exists duration' do expect(subscriber).to receive(:observe) - .with(:exists, event.duration) + .with(:exists, event) subscriber.cache_exist?(event) end @@ -179,7 +182,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::RailsCache do it 'returns' do expect(transaction).not_to receive(:increment) - subscriber.observe(:foo, 15.2) + subscriber.observe(:foo, event) end end @@ -192,17 +195,17 @@ RSpec.describe Gitlab::Metrics::Subscribers::RailsCache do it 'observes cache metric' do expect(subscriber.send(:metric_cache_operation_duration_seconds)) .to receive(:observe) - .with({ operation: :delete }, event.duration / 1000.0) + .with({ operation: :delete, store: store_label }, event.duration / 1000.0) - subscriber.observe(:delete, event.duration) + subscriber.observe(:delete, event) end it 'increments the operations total' do expect(transaction) .to receive(:increment) - .with(:gitlab_cache_operations_total, 1, { operation: :delete }) + .with(:gitlab_cache_operations_total, 1, { operation: :delete, store: store_label }) - subscriber.observe(:delete, event.duration) + subscriber.observe(:delete, event) end end end diff --git a/spec/lib/gitlab/middleware/compressed_json_spec.rb b/spec/lib/gitlab/middleware/compressed_json_spec.rb index 1444e6a9881..5978b2422e0 100644 --- a/spec/lib/gitlab/middleware/compressed_json_spec.rb +++ b/spec/lib/gitlab/middleware/compressed_json_spec.rb @@ -49,21 +49,21 @@ RSpec.describe Gitlab::Middleware::CompressedJson do end end - shared_examples 'handles non integer project ID' do - context 'with a URL-encoded project ID' do - let_it_be(:project_id) { 'gitlab-org%2fgitlab' } + shared_examples 'handles non integer ID' do + context 'with a URL-encoded ID' do + let(:id) { 'gitlab-org%2fgitlab' } it_behaves_like 'decompress middleware' end - context 'with a non URL-encoded project ID' do - let_it_be(:project_id) { '1/repository/files/api/v4' } + context 'with a non URL-encoded ID' do + let(:id) { '1/repository/files/api/v4' } it_behaves_like 'passes input' end - context 'with a blank project ID' do - let_it_be(:project_id) { '' } + context 'with a blank ID' do + let(:id) { '' } it_behaves_like 'passes input' end @@ -116,44 +116,82 @@ RSpec.describe Gitlab::Middleware::CompressedJson do end context 'with project level endpoint' do - let_it_be(:project_id) { 1 } + let(:id) { 1 } context 'with npm advisory bulk url' do - let(:path) { "/api/v4/projects/#{project_id}/packages/npm/-/npm/v1/security/advisories/bulk" } + let(:path) { "/api/v4/projects/#{id}/packages/npm/-/npm/v1/security/advisories/bulk" } it_behaves_like 'decompress middleware' include_context 'with relative url' do - let(:path) { "#{relative_url_root}/api/v4/projects/#{project_id}/packages/npm/-/npm/v1/security/advisories/bulk" } # rubocop disable Layout/LineLength + let(:path) { "#{relative_url_root}/api/v4/projects/#{id}/packages/npm/-/npm/v1/security/advisories/bulk" } # rubocop disable Layout/LineLength it_behaves_like 'decompress middleware' end - it_behaves_like 'handles non integer project ID' + it_behaves_like 'handles non integer ID' end context 'with npm quick audit url' do - let(:path) { "/api/v4/projects/#{project_id}/packages/npm/-/npm/v1/security/audits/quick" } + let(:path) { "/api/v4/projects/#{id}/packages/npm/-/npm/v1/security/audits/quick" } it_behaves_like 'decompress middleware' include_context 'with relative url' do - let(:path) { "#{relative_url_root}/api/v4/projects/#{project_id}/packages/npm/-/npm/v1/security/audits/quick" } # rubocop disable Layout/LineLength + let(:path) { "#{relative_url_root}/api/v4/projects/#{id}/packages/npm/-/npm/v1/security/audits/quick" } # rubocop disable Layout/LineLength it_behaves_like 'decompress middleware' end - it_behaves_like 'handles non integer project ID' + it_behaves_like 'handles non integer ID' end end end + context 'with group level endpoint' do + let(:id) { 1 } + + context 'with npm advisory bulk url' do + let(:path) { "/api/v4/groups/#{id}/-/packages/npm/-/npm/v1/security/advisories/bulk" } + + it_behaves_like 'decompress middleware' + + include_context 'with relative url' do + let(:path) { "#{relative_url_root}/api/v4/groups/#{id}/-/packages/npm/-/npm/v1/security/advisories/bulk" } # rubocop disable Layout/LineLength + + it_behaves_like 'decompress middleware' + end + + it_behaves_like 'handles non integer ID' + end + + context 'with npm quick audit url' do + let(:path) { "/api/v4/groups/#{id}/-/packages/npm/-/npm/v1/security/audits/quick" } + + it_behaves_like 'decompress middleware' + + include_context 'with relative url' do + let(:path) { "#{relative_url_root}/api/v4/groups/#{id}/-/packages/npm/-/npm/v1/security/audits/quick" } # rubocop disable Layout/LineLength + + it_behaves_like 'decompress middleware' + end + + it_behaves_like 'handles non integer ID' + end + end + context 'with some other route' do let(:path) { '/api/projects/123' } it_behaves_like 'passes input' end + context 'with the wrong project path' do + let(:path) { '/api/v4/projects/123/-/packages/npm/-/npm/v1/security/advisories/bulk' } + + it_behaves_like 'passes input' + end + context 'payload is too large' do let(:body_limit) { Gitlab::Middleware::CompressedJson::MAXIMUM_BODY_SIZE } let(:decompressed_input) { 'a' * (body_limit + 100) } diff --git a/spec/lib/gitlab/middleware/go_spec.rb b/spec/lib/gitlab/middleware/go_spec.rb index aaa274e252d..83d4d3fb612 100644 --- a/spec/lib/gitlab/middleware/go_spec.rb +++ b/spec/lib/gitlab/middleware/go_spec.rb @@ -138,7 +138,7 @@ RSpec.describe Gitlab::Middleware::Go, feature_category: :source_code_management context 'with a blacklisted ip' do it 'returns forbidden' do - expect(Gitlab::Auth).to receive(:find_for_git_client).and_raise(Gitlab::Auth::IpBlacklisted) + expect(Gitlab::Auth).to receive(:find_for_git_client).and_raise(Gitlab::Auth::IpBlocked) response = go expect(response[0]).to eq(403) diff --git a/spec/lib/gitlab/middleware/multipart_spec.rb b/spec/lib/gitlab/middleware/multipart_spec.rb index 294a5ee82ed..509a4bb921b 100644 --- a/spec/lib/gitlab/middleware/multipart_spec.rb +++ b/spec/lib/gitlab/middleware/multipart_spec.rb @@ -175,7 +175,7 @@ RSpec.describe Gitlab::Middleware::Multipart do end it 'raises an error' do - expect { subject }.to raise_error(JWT::VerificationError, 'Signature verification raised') + expect { subject }.to raise_error(JWT::VerificationError, 'Signature verification failed') end end @@ -191,7 +191,7 @@ RSpec.describe Gitlab::Middleware::Multipart do end it 'raises an error' do - expect { subject }.to raise_error(JWT::VerificationError, 'Signature verification raised') + expect { subject }.to raise_error(JWT::VerificationError, 'Signature verification failed') end end end diff --git a/spec/lib/gitlab/middleware/request_context_spec.rb b/spec/lib/gitlab/middleware/request_context_spec.rb index 6d5b581feaa..cd21209bcee 100644 --- a/spec/lib/gitlab/middleware/request_context_spec.rb +++ b/spec/lib/gitlab/middleware/request_context_spec.rb @@ -1,10 +1,10 @@ # frozen_string_literal: true -require 'fast_spec_helper' +require 'spec_helper' require 'rack' require 'request_store' require_relative '../../../support/helpers/next_instance_of' -RSpec.describe Gitlab::Middleware::RequestContext do +RSpec.describe Gitlab::Middleware::RequestContext, feature_category: :application_instrumentation do include NextInstanceOf let(:app) { -> (env) {} } @@ -55,6 +55,10 @@ RSpec.describe Gitlab::Middleware::RequestContext do it 'sets the `request_start_time`' do expect { subject }.to change { instance.request_start_time }.from(nil).to(Float) end + + it 'sets the `spam_params`' do + expect { subject }.to change { instance.spam_params }.from(nil).to(::Spam::SpamParams) + end end end end diff --git a/spec/lib/gitlab/monitor/demo_projects_spec.rb b/spec/lib/gitlab/monitor/demo_projects_spec.rb index 262c78eb62e..6b0f855e38d 100644 --- a/spec/lib/gitlab/monitor/demo_projects_spec.rb +++ b/spec/lib/gitlab/monitor/demo_projects_spec.rb @@ -6,15 +6,13 @@ RSpec.describe Gitlab::Monitor::DemoProjects do describe '#primary_keys' do subject { described_class.primary_keys } - it 'fetches primary_keys when on gitlab.com' do - allow(Gitlab).to receive(:com?).and_return(true) + it 'fetches primary_keys when on SaaS', :saas do allow(Gitlab).to receive(:staging?).and_return(false) expect(subject).to eq(Gitlab::Monitor::DemoProjects::DOT_COM_IDS) end - it 'fetches primary_keys when on staging' do - allow(Gitlab).to receive(:com?).and_return(true) + it 'fetches primary_keys when on staging', :saas do allow(Gitlab).to receive(:staging?).and_return(true) expect(subject).to eq(Gitlab::Monitor::DemoProjects::STAGING_IDS) diff --git a/spec/lib/gitlab/multi_collection_paginator_spec.rb b/spec/lib/gitlab/multi_collection_paginator_spec.rb index 080b3382684..25baa8913bf 100644 --- a/spec/lib/gitlab/multi_collection_paginator_spec.rb +++ b/spec/lib/gitlab/multi_collection_paginator_spec.rb @@ -5,6 +5,13 @@ require 'spec_helper' RSpec.describe Gitlab::MultiCollectionPaginator do subject(:paginator) { described_class.new(Project.all.order(:id), Group.all.order(:id), per_page: 3) } + it 'raises an error for invalid page size' do + expect { described_class.new(Project.all.order(:id), Group.all.order(:id), per_page: 0) } + .to raise_error(ArgumentError) + expect { described_class.new(Project.all.order(:id), Group.all.order(:id), per_page: -1) } + .to raise_error(ArgumentError) + end + it 'combines both collections' do project = create(:project) group = create(:group) diff --git a/spec/lib/gitlab/nav/top_nav_menu_item_spec.rb b/spec/lib/gitlab/nav/top_nav_menu_item_spec.rb index 6632a8106ca..1d3452a004a 100644 --- a/spec/lib/gitlab/nav/top_nav_menu_item_spec.rb +++ b/spec/lib/gitlab/nav/top_nav_menu_item_spec.rb @@ -14,7 +14,8 @@ RSpec.describe ::Gitlab::Nav::TopNavMenuItem, feature_category: :navigation do view: 'view', css_class: 'css_class', data: {}, - emoji: 'smile' + partial: 'groups/some_view_partial_file', + component: '_some_component_used_as_a_trigger_for_frontend_dropdown_item_render_' } expect(described_class.build(**item)).to eq(item.merge(type: :item)) diff --git a/spec/lib/gitlab/net_http_adapter_spec.rb b/spec/lib/gitlab/net_http_adapter_spec.rb index fdaf35be31e..cfb90578a4b 100644 --- a/spec/lib/gitlab/net_http_adapter_spec.rb +++ b/spec/lib/gitlab/net_http_adapter_spec.rb @@ -1,8 +1,9 @@ # frozen_string_literal: true require 'fast_spec_helper' +require 'net/http' -RSpec.describe Gitlab::NetHttpAdapter do +RSpec.describe Gitlab::NetHttpAdapter, feature_category: :api do describe '#connect' do let(:url) { 'https://example.org' } let(:net_http_adapter) { described_class.new(url) } diff --git a/spec/lib/gitlab/observability_spec.rb b/spec/lib/gitlab/observability_spec.rb index 8068d2f8ec9..5082d193197 100644 --- a/spec/lib/gitlab/observability_spec.rb +++ b/spec/lib/gitlab/observability_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Observability do +RSpec.describe Gitlab::Observability, feature_category: :error_tracking do describe '.observability_url' do let(:gitlab_url) { 'https://example.com' } @@ -31,29 +31,189 @@ RSpec.describe Gitlab::Observability do end end - describe '.observability_enabled?' do - let_it_be(:group) { build(:user) } - let_it_be(:user) { build(:group) } + describe '.build_full_url' do + let_it_be(:group) { build_stubbed(:group, id: 123) } + let(:observability_url) { described_class.observability_url } + + it 'returns the full observability url for the given params' do + url = described_class.build_full_url(group, '/foo?bar=baz', '/') + expect(url).to eq("https://observe.gitlab.com/-/123/foo?bar=baz") + end + + it 'handles missing / from observability_path' do + url = described_class.build_full_url(group, 'foo?bar=baz', '/') + expect(url).to eq("https://observe.gitlab.com/-/123/foo?bar=baz") + end + + it 'sanitises observability_path' do + url = described_class.build_full_url(group, "/test?groupId=<script>alert('attack!')</script>", '/') + expect(url).to eq("https://observe.gitlab.com/-/123/test?groupId=alert('attack!')") + end + + context 'when observability_path is missing' do + it 'builds the url with the fallback_path' do + url = described_class.build_full_url(group, nil, '/fallback') + expect(url).to eq("https://observe.gitlab.com/-/123/fallback") + end + + it 'defaults to / if fallback_path is also missing' do + url = described_class.build_full_url(group, nil, nil) + expect(url).to eq("https://observe.gitlab.com/-/123/") + end + end + end + + describe '.embeddable_url' do + before do + stub_config_setting(url: "https://www.gitlab.com") + # Can't use build/build_stubbed as we want the routes to be generated as well + create(:group, path: 'test-path', id: 123) + end + + context 'when URL is valid' do + where(:input, :expected) do + [ + [ + "https://www.gitlab.com/groups/test-path/-/observability/explore?observability_path=%2Fexplore%3FgroupId%3D14485840%26left%3D%255B%2522now-1h%2522,%2522now%2522,%2522new-sentry.gitlab.net%2522,%257B%257D%255D", + "https://observe.gitlab.com/-/123/explore?groupId=14485840&left=%5B%22now-1h%22,%22now%22,%22new-sentry.gitlab.net%22,%7B%7D%5D" + ], + [ + "https://www.gitlab.com/groups/test-path/-/observability/explore?observability_path=/goto/foo", + "https://observe.gitlab.com/-/123/goto/foo" + ] + ] + end + + with_them do + it 'returns an embeddable observability url' do + expect(described_class.embeddable_url(input)).to eq(expected) + end + end + end + + context 'when URL is invalid' do + where(:input) do + [ + # direct links to observe.gitlab.com + "https://observe.gitlab.com/-/123/explore", + 'https://observe.gitlab.com/v1/auth/start', + + # invalid GitLab URL + "not a link", + "https://foo.bar/groups/test-path/-/observability/explore?observability_path=/explore", + "http://www.gitlab.com/groups/test-path/-/observability/explore?observability_path=/explore", + "https://www.gitlab.com:123/groups/test-path/-/observability/explore?observability_path=/explore", + "https://www.gitlab.com@example.com/groups/test-path/-/observability/explore?observability_path=/explore", + "https://www.gitlab.com/groups/test-path/-/observability/explore?observability_path=@example.com", + + # invalid group/controller/actions + "https://www.gitlab.com/groups/INVALID_GROUP/-/observability/explore?observability_path=/explore", + "https://www.gitlab.com/groups/test-path/-/INVALID_CONTROLLER/explore?observability_path=/explore", + "https://www.gitlab.com/groups/test-path/-/observability/INVALID_ACTION?observability_path=/explore", + + # invalid observablity path + "https://www.gitlab.com/groups/test-path/-/observability/explore", + "https://www.gitlab.com/groups/test-path/-/observability/explore?missing_observability_path=/explore", + "https://www.gitlab.com/groups/test-path/-/observability/explore?observability_path=/not_embeddable", + "https://www.gitlab.com/groups/test-path/-/observability/explore?observability_path=/datasources", + "https://www.gitlab.com/groups/test-path/-/observability/explore?observability_path=not a valid path" + ] + end + + with_them do + it 'returns nil' do + expect(described_class.embeddable_url(input)).to be_nil + end + end + + it 'returns nil if the path detection throws an error' do + test_url = "https://www.gitlab.com/groups/test-path/-/observability/explore" + allow(Rails.application.routes).to receive(:recognize_path).with(test_url) { + raise ActionController::RoutingError, 'test' + } + expect(described_class.embeddable_url(test_url)).to be_nil + end + + it 'returns nil if parsing observaboility path throws an error' do + observability_path = 'some-path' + test_url = "https://www.gitlab.com/groups/test-path/-/observability/explore?observability_path=#{observability_path}" + + allow(URI).to receive(:parse).and_call_original + allow(URI).to receive(:parse).with(observability_path) { + raise URI::InvalidURIError, 'test' + } + + expect(described_class.embeddable_url(test_url)).to be_nil + end + end + end + + describe '.allowed_for_action?' do + let(:group) { build_stubbed(:group) } + let(:user) { build_stubbed(:user) } + + before do + allow(described_class).to receive(:allowed?).and_call_original + end + + it 'returns false if action is nil' do + expect(described_class.allowed_for_action?(user, group, nil)).to eq(false) + end + + describe 'allowed? calls' do + using RSpec::Parameterized::TableSyntax + + where(:action, :permission) do + :foo | :admin_observability + :explore | :read_observability + :datasources | :admin_observability + :manage | :admin_observability + :dashboards | :read_observability + end + + with_them do + it "calls allowed? with #{params[:permission]} when actions is #{params[:action]}" do + described_class.allowed_for_action?(user, group, action) + expect(described_class).to have_received(:allowed?).with(user, group, permission) + end + end + end + end + + describe '.allowed?' do + let(:user) { build_stubbed(:user) } + let(:group) { build_stubbed(:group) } + let(:test_permission) { :read_observability } + + before do + allow(Ability).to receive(:allowed?).and_return(false) + end subject do - described_class.observability_enabled?(user, group) + described_class.allowed?(user, group, test_permission) end - it 'checks if read_observability ability is allowed for the given user and group' do + it 'checks if ability is allowed for the given user and group' do allow(Ability).to receive(:allowed?).and_return(true) subject - expect(Ability).to have_received(:allowed?).with(user, :read_observability, group) + expect(Ability).to have_received(:allowed?).with(user, test_permission, group) end - it 'returns true if the read_observability ability is allowed' do + it 'checks for admin_observability if permission is missing' do + described_class.allowed?(user, group) + + expect(Ability).to have_received(:allowed?).with(user, :admin_observability, group) + end + + it 'returns true if the ability is allowed' do allow(Ability).to receive(:allowed?).and_return(true) expect(subject).to eq(true) end - it 'returns false if the read_observability ability is not allowed' do + it 'returns false if the ability is not allowed' do allow(Ability).to receive(:allowed?).and_return(false) expect(subject).to eq(false) @@ -64,5 +224,13 @@ RSpec.describe Gitlab::Observability do expect(subject).to eq(false) end + + it 'returns false if group is missing' do + expect(described_class.allowed?(user, nil, :read_observability)).to eq(false) + end + + it 'returns false if user is missing' do + expect(described_class.allowed?(nil, group, :read_observability)).to eq(false) + end end end diff --git a/spec/lib/gitlab/octokit/middleware_spec.rb b/spec/lib/gitlab/octokit/middleware_spec.rb index 5555990b113..f7063f2c4f2 100644 --- a/spec/lib/gitlab/octokit/middleware_spec.rb +++ b/spec/lib/gitlab/octokit/middleware_spec.rb @@ -6,7 +6,7 @@ RSpec.describe Gitlab::Octokit::Middleware, feature_category: :importers do let(:app) { double(:app) } let(:middleware) { described_class.new(app) } - shared_examples 'Public URL' do + shared_examples 'Allowed URL' do it 'does not raise an error' do expect(app).to receive(:call).with(env) @@ -14,7 +14,7 @@ RSpec.describe Gitlab::Octokit::Middleware, feature_category: :importers do end end - shared_examples 'Local URL' do + shared_examples 'Blocked URL' do it 'raises an error' do expect { middleware.call(env) }.to raise_error(Gitlab::UrlBlocker::BlockedUrlError) end @@ -24,7 +24,24 @@ RSpec.describe Gitlab::Octokit::Middleware, feature_category: :importers do context 'when the URL is a public URL' do let(:env) { { url: 'https://public-url.com' } } - it_behaves_like 'Public URL' + it_behaves_like 'Allowed URL' + + context 'with failed address check' do + before do + stub_env('RSPEC_ALLOW_INVALID_URLS', 'false') + allow(Addrinfo).to receive(:getaddrinfo).and_raise(SocketError) + end + + it_behaves_like 'Blocked URL' + + context 'with disabled dns rebinding check' do + before do + stub_application_setting(dns_rebinding_protection_enabled: false) + end + + it_behaves_like 'Allowed URL' + end + end end context 'when the URL is a localhost address' do @@ -35,7 +52,7 @@ RSpec.describe Gitlab::Octokit::Middleware, feature_category: :importers do stub_application_setting(allow_local_requests_from_web_hooks_and_services: false) end - it_behaves_like 'Local URL' + it_behaves_like 'Blocked URL' end context 'when localhost requests are allowed' do @@ -43,7 +60,7 @@ RSpec.describe Gitlab::Octokit::Middleware, feature_category: :importers do stub_application_setting(allow_local_requests_from_web_hooks_and_services: true) end - it_behaves_like 'Public URL' + it_behaves_like 'Allowed URL' end end @@ -55,7 +72,7 @@ RSpec.describe Gitlab::Octokit::Middleware, feature_category: :importers do stub_application_setting(allow_local_requests_from_web_hooks_and_services: false) end - it_behaves_like 'Local URL' + it_behaves_like 'Blocked URL' end context 'when local network requests are allowed' do @@ -63,7 +80,7 @@ RSpec.describe Gitlab::Octokit::Middleware, feature_category: :importers do stub_application_setting(allow_local_requests_from_web_hooks_and_services: true) end - it_behaves_like 'Public URL' + it_behaves_like 'Allowed URL' end end diff --git a/spec/lib/gitlab/omniauth_initializer_spec.rb b/spec/lib/gitlab/omniauth_initializer_spec.rb index daef280dbaa..112fdb183ab 100644 --- a/spec/lib/gitlab/omniauth_initializer_spec.rb +++ b/spec/lib/gitlab/omniauth_initializer_spec.rb @@ -216,14 +216,6 @@ RSpec.describe Gitlab::OmniauthInitializer do expect { subject.execute([hash_config]) }.to raise_error(NameError) end - it 'configures on_single_sign_out proc for cas3' do - cas3_config = { 'name' => 'cas3', 'args' => {} } - - expect(devise_config).to receive(:omniauth).with(:cas3, { on_single_sign_out: an_instance_of(Proc) }) - - subject.execute([cas3_config]) - end - it 'configures defaults for google_oauth2' do google_config = { 'name' => 'google_oauth2', diff --git a/spec/lib/gitlab/optimistic_locking_spec.rb b/spec/lib/gitlab/optimistic_locking_spec.rb index 1d669573b74..34f197b5ddb 100644 --- a/spec/lib/gitlab/optimistic_locking_spec.rb +++ b/spec/lib/gitlab/optimistic_locking_spec.rb @@ -16,6 +16,19 @@ RSpec.describe Gitlab::OptimisticLocking do describe '#retry_lock' do let(:name) { 'optimistic_locking_spec' } + it 'does not change current_scope', :aggregate_failures do + instance = Class.new { include Gitlab::OptimisticLocking }.new + relation = pipeline.cancelable_statuses + + expected_scope = Ci::Build.current_scope&.to_sql + + instance.send(:retry_lock, relation, name: :test) do + expect(Ci::Build.current_scope&.to_sql).to eq(expected_scope) + end + + expect(Ci::Build.current_scope&.to_sql).to eq(expected_scope) + end + context 'when state changed successfully without retries' do subject do described_class.retry_lock(pipeline, name: name) do |lock_subject| diff --git a/spec/lib/gitlab/other_markup_spec.rb b/spec/lib/gitlab/other_markup_spec.rb index 6b4b0e8fda6..74e2c5e26c1 100644 --- a/spec/lib/gitlab/other_markup_spec.rb +++ b/spec/lib/gitlab/other_markup_spec.rb @@ -35,8 +35,8 @@ RSpec.describe Gitlab::OtherMarkup do end it 'times out' do - # expect twice because of timeout in SyntaxHighlightFilter - expect(Gitlab::RenderTimeout).to receive(:timeout).twice.and_call_original + # expect 3 times because of timeout in SyntaxHighlightFilter and BlockquoteFenceFilter + expect(Gitlab::RenderTimeout).to receive(:timeout).exactly(3).times.and_call_original expect(Gitlab::ErrorTracking).to receive(:track_exception).with( instance_of(Timeout::Error), project_id: context[:project].id, file_name: file_name, diff --git a/spec/lib/gitlab/pages/random_domain_spec.rb b/spec/lib/gitlab/pages/random_domain_spec.rb new file mode 100644 index 00000000000..978412bb72c --- /dev/null +++ b/spec/lib/gitlab/pages/random_domain_spec.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Pages::RandomDomain, feature_category: :pages do + let(:namespace_path) { 'namespace' } + + subject(:generator) do + described_class.new(project_path: project_path, namespace_path: namespace_path) + end + + RSpec.shared_examples 'random domain' do |domain| + it do + expect(SecureRandom) + .to receive(:hex) + .and_wrap_original do |_, size, _| + ('h' * size) + end + + generated = generator.generate + + expect(generated).to eq(domain) + expect(generated.length).to eq(63) + end + end + + context 'when project path is less than 48 chars' do + let(:project_path) { 'p' } + + it_behaves_like 'random domain', 'p-namespace-hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh' + end + + context 'when project path is close to 48 chars' do + let(:project_path) { 'p' * 45 } + + it_behaves_like 'random domain', 'ppppppppppppppppppppppppppppppppppppppppppppp-na-hhhhhhhhhhhhhh' + end + + context 'when project path is larger than 48 chars' do + let(:project_path) { 'p' * 49 } + + it_behaves_like 'random domain', 'pppppppppppppppppppppppppppppppppppppppppppppppp-hhhhhhhhhhhhhh' + end +end diff --git a/spec/lib/gitlab/pages/virtual_host_finder_spec.rb b/spec/lib/gitlab/pages/virtual_host_finder_spec.rb new file mode 100644 index 00000000000..4b584a45503 --- /dev/null +++ b/spec/lib/gitlab/pages/virtual_host_finder_spec.rb @@ -0,0 +1,214 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Pages::VirtualHostFinder, feature_category: :pages do + let_it_be(:project) { create(:project) } + + before_all do + project.update_pages_deployment!(create(:pages_deployment, project: project)) + end + + it 'returns nil when host is empty' do + expect(described_class.new(nil).execute).to be_nil + expect(described_class.new('').execute).to be_nil + end + + context 'when host is a pages custom domain host' do + let_it_be(:pages_domain) { create(:pages_domain, project: project) } + + subject(:virtual_domain) { described_class.new(pages_domain.domain).execute } + + context 'when there are no pages deployed for the project' do + before_all do + project.mark_pages_as_not_deployed + end + + it 'returns nil' do + expect(virtual_domain).to be_nil + end + end + + context 'when there are pages deployed for the project' do + before_all do + project.mark_pages_as_deployed + end + + it 'returns the virual domain when there are pages deployed for the project' do + expect(virtual_domain).to be_an_instance_of(Pages::VirtualDomain) + expect(virtual_domain.cache_key).to match(/pages_domain_for_domain_#{pages_domain.id}_/) + expect(virtual_domain.lookup_paths.length).to eq(1) + expect(virtual_domain.lookup_paths.first.project_id).to eq(project.id) + end + + context 'when :cache_pages_domain_api is disabled' do + before do + stub_feature_flags(cache_pages_domain_api: false) + end + + it 'returns the virual domain when there are pages deployed for the project' do + expect(virtual_domain).to be_an_instance_of(Pages::VirtualDomain) + expect(virtual_domain.cache_key).to be_nil + expect(virtual_domain.lookup_paths.length).to eq(1) + expect(virtual_domain.lookup_paths.first.project_id).to eq(project.id) + end + end + end + end + + context 'when host is a namespace domain' do + context 'when there are no pages deployed for the project' do + before_all do + project.mark_pages_as_not_deployed + end + + it 'returns no result if the provided host is not subdomain of the Pages host' do + virtual_domain = described_class.new("#{project.namespace.path}.something.io").execute + + expect(virtual_domain).to eq(nil) + end + + it 'returns the virual domain with no lookup_paths' do + virtual_domain = described_class.new("#{project.namespace.path}.#{Settings.pages.host}").execute + + expect(virtual_domain).to be_an_instance_of(Pages::VirtualDomain) + expect(virtual_domain.cache_key).to match(/pages_domain_for_namespace_#{project.namespace.id}_/) + expect(virtual_domain.lookup_paths.length).to eq(0) + end + + context 'when :cache_pages_domain_api is disabled' do + before do + stub_feature_flags(cache_pages_domain_api: false) + end + + it 'returns the virual domain with no lookup_paths' do + virtual_domain = described_class.new("#{project.namespace.path}.#{Settings.pages.host}".downcase).execute + + expect(virtual_domain).to be_an_instance_of(Pages::VirtualDomain) + expect(virtual_domain.cache_key).to be_nil + expect(virtual_domain.lookup_paths.length).to eq(0) + end + end + end + + context 'when there are pages deployed for the project' do + before_all do + project.mark_pages_as_deployed + project.namespace.update!(path: 'topNAMEspace') + end + + it 'returns no result if the provided host is not subdomain of the Pages host' do + virtual_domain = described_class.new("#{project.namespace.path}.something.io").execute + + expect(virtual_domain).to eq(nil) + end + + it 'returns the virual domain when there are pages deployed for the project' do + virtual_domain = described_class.new("#{project.namespace.path}.#{Settings.pages.host}").execute + + expect(virtual_domain).to be_an_instance_of(Pages::VirtualDomain) + expect(virtual_domain.cache_key).to match(/pages_domain_for_namespace_#{project.namespace.id}_/) + expect(virtual_domain.lookup_paths.length).to eq(1) + expect(virtual_domain.lookup_paths.first.project_id).to eq(project.id) + end + + it 'finds domain with case-insensitive' do + virtual_domain = described_class.new("#{project.namespace.path}.#{Settings.pages.host.upcase}").execute + + expect(virtual_domain).to be_an_instance_of(Pages::VirtualDomain) + expect(virtual_domain.cache_key).to match(/pages_domain_for_namespace_#{project.namespace.id}_/) + expect(virtual_domain.lookup_paths.length).to eq(1) + expect(virtual_domain.lookup_paths.first.project_id).to eq(project.id) + end + + context 'when :cache_pages_domain_api is disabled' do + before_all do + stub_feature_flags(cache_pages_domain_api: false) + end + + it 'returns the virual domain when there are pages deployed for the project' do + virtual_domain = described_class.new("#{project.namespace.path}.#{Settings.pages.host}").execute + + expect(virtual_domain).to be_an_instance_of(Pages::VirtualDomain) + expect(virtual_domain.cache_key).to be_nil + expect(virtual_domain.lookup_paths.length).to eq(1) + expect(virtual_domain.lookup_paths.first.project_id).to eq(project.id) + end + end + end + end + + context 'when host is a unique domain' do + before_all do + project.project_setting.update!(pages_unique_domain: 'unique-domain') + end + + subject(:virtual_domain) { described_class.new("unique-domain.#{Settings.pages.host.upcase}").execute } + + context 'when pages unique domain is enabled' do + before_all do + project.project_setting.update!(pages_unique_domain_enabled: true) + end + + context 'when there are no pages deployed for the project' do + before_all do + project.mark_pages_as_not_deployed + end + + it 'returns nil' do + expect(virtual_domain).to be_nil + end + end + + context 'when there are pages deployed for the project' do + before_all do + project.mark_pages_as_deployed + end + + it 'returns the virual domain when there are pages deployed for the project' do + expect(virtual_domain).to be_an_instance_of(Pages::VirtualDomain) + expect(virtual_domain.lookup_paths.length).to eq(1) + expect(virtual_domain.lookup_paths.first.project_id).to eq(project.id) + end + + context 'when :cache_pages_domain_api is disabled' do + before do + stub_feature_flags(cache_pages_domain_api: false) + end + + it 'returns the virual domain when there are pages deployed for the project' do + expect(virtual_domain).to be_an_instance_of(Pages::VirtualDomain) + expect(virtual_domain.lookup_paths.length).to eq(1) + expect(virtual_domain.lookup_paths.first.project_id).to eq(project.id) + end + end + end + end + + context 'when pages unique domain is disabled' do + before_all do + project.project_setting.update!(pages_unique_domain_enabled: false) + end + + context 'when there are no pages deployed for the project' do + before_all do + project.mark_pages_as_not_deployed + end + + it 'returns nil' do + expect(virtual_domain).to be_nil + end + end + + context 'when there are pages deployed for the project' do + before_all do + project.mark_pages_as_deployed + end + + it 'returns nil' do + expect(virtual_domain).to be_nil + end + end + end + end +end diff --git a/spec/lib/gitlab/patch/draw_route_spec.rb b/spec/lib/gitlab/patch/draw_route_spec.rb index 4d1c7bf9fcf..d983f6f15bb 100644 --- a/spec/lib/gitlab/patch/draw_route_spec.rb +++ b/spec/lib/gitlab/patch/draw_route_spec.rb @@ -20,8 +20,10 @@ RSpec.describe Gitlab::Patch::DrawRoute do it 'evaluates CE only route' do subject.draw(:help) + route_file_path = subject.route_path('config/routes/help.rb') + expect(subject).to have_received(:instance_eval) - .with(File.read(subject.route_path('config/routes/help.rb'))) + .with(File.read(route_file_path), route_file_path) .once expect(subject).to have_received(:instance_eval) diff --git a/spec/lib/gitlab/patch/node_loader_spec.rb b/spec/lib/gitlab/patch/node_loader_spec.rb new file mode 100644 index 00000000000..000083fc6d0 --- /dev/null +++ b/spec/lib/gitlab/patch/node_loader_spec.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Patch::NodeLoader, feature_category: :redis do + using RSpec::Parameterized::TableSyntax + + describe '#fetch_node_info' do + let(:redis) { double(:redis) } # rubocop:disable RSpec/VerifiedDoubles + + # rubocop:disable Naming/InclusiveLanguage + where(:case_name, :args, :value) do + [ + [ + 'when only ip address is present', + "07c37df 127.0.0.1:30004@31004 slave e7d1eec 0 1426238317239 4 connected +67ed2db 127.0.0.1:30002@31002 master - 0 1426238316232 2 connected 5461-10922 +292f8b3 127.0.0.1:30003@31003 master - 0 1426238318243 3 connected 10923-16383 +6ec2392 127.0.0.1:30005@31005 slave 67ed2db 0 1426238316232 5 connected +824fe11 127.0.0.1:30006@31006 slave 292f8b3 0 1426238317741 6 connected +e7d1eec 127.0.0.1:30001@31001 myself,master - 0 0 1 connected 0-5460", + { + '127.0.0.1:30004' => 'slave', '127.0.0.1:30002' => 'master', '127.0.0.1:30003' => 'master', + '127.0.0.1:30005' => 'slave', '127.0.0.1:30006' => 'slave', '127.0.0.1:30001' => 'master' + } + ], + [ + 'when hostname is present', + "07c37df 127.0.0.1:30004@31004,host1 slave e7d1eec 0 1426238317239 4 connected +67ed2db 127.0.0.1:30002@31002,host2 master - 0 1426238316232 2 connected 5461-10922 +292f8b3 127.0.0.1:30003@31003,host3 master - 0 1426238318243 3 connected 10923-16383 +6ec2392 127.0.0.1:30005@31005,host4 slave 67ed2db 0 1426238316232 5 connected +824fe11 127.0.0.1:30006@31006,host5 slave 292f8b3 0 1426238317741 6 connected +e7d1eec 127.0.0.1:30001@31001,host6 myself,master - 0 0 1 connected 0-5460", + { + 'host1:30004' => 'slave', 'host2:30002' => 'master', 'host3:30003' => 'master', + 'host4:30005' => 'slave', 'host5:30006' => 'slave', 'host6:30001' => 'master' + } + ], + [ + 'when auxiliary fields are present', + "07c37df 127.0.0.1:30004@31004,,shard-id=69bc slave e7d1eec 0 1426238317239 4 connected +67ed2db 127.0.0.1:30002@31002,,shard-id=114f master - 0 1426238316232 2 connected 5461-10922 +292f8b3 127.0.0.1:30003@31003,,shard-id=fdb3 master - 0 1426238318243 3 connected 10923-16383 +6ec2392 127.0.0.1:30005@31005,,shard-id=114f slave 67ed2db 0 1426238316232 5 connected +824fe11 127.0.0.1:30006@31006,,shard-id=fdb3 slave 292f8b3 0 1426238317741 6 connected +e7d1eec 127.0.0.1:30001@31001,,shard-id=69bc myself,master - 0 0 1 connected 0-5460", + { + '127.0.0.1:30004' => 'slave', '127.0.0.1:30002' => 'master', '127.0.0.1:30003' => 'master', + '127.0.0.1:30005' => 'slave', '127.0.0.1:30006' => 'slave', '127.0.0.1:30001' => 'master' + } + ], + [ + 'when hostname and auxiliary fields are present', + "07c37df 127.0.0.1:30004@31004,host1,shard-id=69bc slave e7d1eec 0 1426238317239 4 connected +67ed2db 127.0.0.1:30002@31002,host2,shard-id=114f master - 0 1426238316232 2 connected 5461-10922 +292f8b3 127.0.0.1:30003@31003,host3,shard-id=fdb3 master - 0 1426238318243 3 connected 10923-16383 +6ec2392 127.0.0.1:30005@31005,host4,shard-id=114f slave 67ed2db 0 1426238316232 5 connected +824fe11 127.0.0.1:30006@31006,host5,shard-id=fdb3 slave 292f8b3 0 1426238317741 6 connected +e7d1eec 127.0.0.1:30001@31001,host6,shard-id=69bc myself,master - 0 0 1 connected 0-5460", + { + 'host1:30004' => 'slave', 'host2:30002' => 'master', 'host3:30003' => 'master', + 'host4:30005' => 'slave', 'host5:30006' => 'slave', 'host6:30001' => 'master' + } + ] + ] + end + # rubocop:enable Naming/InclusiveLanguage + + with_them do + before do + allow(redis).to receive(:call).with([:cluster, :nodes]).and_return(args) + end + + it do + expect(Redis::Cluster::NodeLoader.load_flags([redis])).to eq(value) + end + end + end +end diff --git a/spec/lib/gitlab/path_regex_spec.rb b/spec/lib/gitlab/path_regex_spec.rb index 0a647befb50..718b20c59ed 100644 --- a/spec/lib/gitlab/path_regex_spec.rb +++ b/spec/lib/gitlab/path_regex_spec.rb @@ -177,7 +177,12 @@ RSpec.describe Gitlab::PathRegex do missing_words: missing_words, additional_words: additional_words) end - expect(described_class::TOP_LEVEL_ROUTES) + # We have to account for routes that are added by gems into the RAILS_ENV=test only. + test_only_top_level_routes = [ + '_system_test_entrypoint' # added by the view_component gem + ] + + expect(described_class::TOP_LEVEL_ROUTES + test_only_top_level_routes) .to contain_exactly(*top_level_words), failure_block end diff --git a/spec/lib/gitlab/phabricator_import/cache/map_spec.rb b/spec/lib/gitlab/phabricator_import/cache/map_spec.rb deleted file mode 100644 index 157b3ca56c9..00000000000 --- a/spec/lib/gitlab/phabricator_import/cache/map_spec.rb +++ /dev/null @@ -1,85 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::PhabricatorImport::Cache::Map, :clean_gitlab_redis_cache do - let_it_be(:project) { create(:project) } - - let(:redis) { Gitlab::Redis::Cache } - - subject(:map) { described_class.new(project) } - - describe '#get_gitlab_model' do - it 'returns nil if there was nothing cached for the phabricator id' do - expect(map.get_gitlab_model('does not exist')).to be_nil - end - - it 'returns the object if it was set in redis' do - issue = create(:issue, project: project) - set_in_redis('exists', issue) - - expect(map.get_gitlab_model('exists')).to eq(issue) - end - - it 'extends the TTL for the cache key' do - set_in_redis('extend', create(:issue, project: project)) do |redis| - redis.expire(cache_key('extend'), 10.seconds.to_i) - end - - map.get_gitlab_model('extend') - - ttl = redis.with { |redis| redis.ttl(cache_key('extend')) } - - expect(ttl).to be > 10.seconds - end - - it 'sets the object in redis once if a block was given and nothing was cached' do - issue = create(:issue, project: project) - - expect(map.get_gitlab_model('does not exist') { issue }).to eq(issue) - - expect { |b| map.get_gitlab_model('does not exist', &b) } - .not_to yield_control - end - - it 'does not cache `nil` objects' do - expect(map).not_to receive(:set_gitlab_model) - - map.get_gitlab_model('does not exist') { nil } - end - end - - describe '#set_gitlab_model' do - around do |example| - freeze_time { example.run } - end - - it 'sets the class and id in redis with a ttl' do - issue = create(:issue, project: project) - - map.set_gitlab_model(issue, 'it is set') - - set_data, ttl = redis.with do |redis| - redis.pipelined do |p| - p.mapped_hmget(cache_key('it is set'), :classname, :database_id) - p.ttl(cache_key('it is set')) - end - end - - expect(set_data).to eq({ classname: 'Issue', database_id: issue.id.to_s }) - expect(ttl).to be_within(1.second).of(Gitlab::Import::StuckImportJob::IMPORT_JOBS_EXPIRATION) - end - end - - def set_in_redis(key, object) - redis.with do |redis| - redis.mapped_hmset(cache_key(key), - { classname: object.class, database_id: object.id }) - yield(redis) if block_given? - end - end - - def cache_key(phabricator_id) - subject.__send__(:cache_key_for_phabricator_id, phabricator_id) - end -end diff --git a/spec/lib/gitlab/phabricator_import/conduit/client_spec.rb b/spec/lib/gitlab/phabricator_import/conduit/client_spec.rb deleted file mode 100644 index dad349f3255..00000000000 --- a/spec/lib/gitlab/phabricator_import/conduit/client_spec.rb +++ /dev/null @@ -1,59 +0,0 @@ -# frozen_string_literal: true -require 'spec_helper' - -RSpec.describe Gitlab::PhabricatorImport::Conduit::Client do - let(:client) do - described_class.new('https://see-ya-later.phabricator', 'api-token') - end - - describe '#get' do - it 'performs and parses a request' do - params = { some: 'extra', values: %w[are passed] } - stub_valid_request(params) - - response = client.get('test', params: params) - - expect(response).to be_a(Gitlab::PhabricatorImport::Conduit::Response) - expect(response).to be_success - end - - it 'wraps request errors in an `ApiError`' do - stub_timeout - - expect { client.get('test') }.to raise_error(Gitlab::PhabricatorImport::Conduit::ApiError) - end - - it 'raises response error' do - stub_error_response - - expect { client.get('test') } - .to raise_error(Gitlab::PhabricatorImport::Conduit::ResponseError, /has the wrong length/) - end - end - - def stub_valid_request(params = {}) - WebMock.stub_request( - :get, 'https://see-ya-later.phabricator/api/test' - ).with( - body: CGI.unescape(params.reverse_merge('api.token' => 'api-token').to_query) - ).and_return( - status: 200, - body: fixture_file('phabricator_responses/maniphest.search.json') - ) - end - - def stub_timeout - WebMock.stub_request( - :get, 'https://see-ya-later.phabricator/api/test' - ).to_timeout - end - - def stub_error_response - WebMock.stub_request( - :get, 'https://see-ya-later.phabricator/api/test' - ).and_return( - status: 200, - body: fixture_file('phabricator_responses/auth_failed.json') - ) - end -end diff --git a/spec/lib/gitlab/phabricator_import/conduit/maniphest_spec.rb b/spec/lib/gitlab/phabricator_import/conduit/maniphest_spec.rb deleted file mode 100644 index e655a39a28d..00000000000 --- a/spec/lib/gitlab/phabricator_import/conduit/maniphest_spec.rb +++ /dev/null @@ -1,39 +0,0 @@ -# frozen_string_literal: true -require 'spec_helper' - -RSpec.describe Gitlab::PhabricatorImport::Conduit::Maniphest do - let(:maniphest) do - described_class.new(phabricator_url: 'https://see-ya-later.phabricator', api_token: 'api-token') - end - - describe '#tasks' do - let(:fake_client) { double('Phabricator client') } - - before do - allow(maniphest).to receive(:client).and_return(fake_client) - end - - it 'calls the api with the correct params' do - expected_params = { - after: '123', - attachments: { - projects: 1, subscribers: 1, columns: 1 - } - } - - expect(fake_client).to receive(:get).with('maniphest.search', - params: expected_params) - - maniphest.tasks(after: '123') - end - - it 'returns a parsed response' do - response = Gitlab::PhabricatorImport::Conduit::Response - .new(fixture_file('phabricator_responses/maniphest.search.json')) - - allow(fake_client).to receive(:get).and_return(response) - - expect(maniphest.tasks).to be_a(Gitlab::PhabricatorImport::Conduit::TasksResponse) - end - end -end diff --git a/spec/lib/gitlab/phabricator_import/conduit/response_spec.rb b/spec/lib/gitlab/phabricator_import/conduit/response_spec.rb deleted file mode 100644 index a444e7fdf47..00000000000 --- a/spec/lib/gitlab/phabricator_import/conduit/response_spec.rb +++ /dev/null @@ -1,79 +0,0 @@ -# frozen_string_literal: true -require 'spec_helper' - -RSpec.describe Gitlab::PhabricatorImport::Conduit::Response do - let(:response) { described_class.new(Gitlab::Json.parse(fixture_file('phabricator_responses/maniphest.search.json'))) } - let(:error_response) { described_class.new(Gitlab::Json.parse(fixture_file('phabricator_responses/auth_failed.json'))) } - - describe '.parse!' do - it 'raises a ResponseError if the http response was not successfull' do - fake_response = double(:http_response, success?: false, status: 401) - - expect { described_class.parse!(fake_response) } - .to raise_error(Gitlab::PhabricatorImport::Conduit::ResponseError, /responded with 401/) - end - - it 'raises a ResponseError if the response contained a Phabricator error' do - fake_response = double(:http_response, - success?: true, - status: 200, - body: fixture_file('phabricator_responses/auth_failed.json')) - - expect { described_class.parse!(fake_response) } - .to raise_error(Gitlab::PhabricatorImport::Conduit::ResponseError, /ERR-INVALID-AUTH: API token/) - end - - it 'raises a ResponseError if JSON parsing failed' do - fake_response = double(:http_response, - success?: true, - status: 200, - body: 'This is no JSON') - - expect { described_class.parse!(fake_response) } - .to raise_error(Gitlab::PhabricatorImport::Conduit::ResponseError, /unexpected character/) - end - - it 'returns a parsed response for valid input' do - fake_response = double(:http_response, - success?: true, - status: 200, - body: fixture_file('phabricator_responses/maniphest.search.json')) - - expect(described_class.parse!(fake_response)).to be_a(described_class) - end - end - - describe '#success?' do - it { expect(response).to be_success } - it { expect(error_response).not_to be_success } - end - - describe '#error_code' do - it { expect(error_response.error_code).to eq('ERR-INVALID-AUTH') } - it { expect(response.error_code).to be_nil } - end - - describe '#error_info' do - it 'returns the correct error info' do - expected_message = 'API token "api-token" has the wrong length. API tokens should be 32 characters long.' - - expect(error_response.error_info).to eq(expected_message) - end - - it { expect(response.error_info).to be_nil } - end - - describe '#data' do - it { expect(error_response.data).to be_nil } - it { expect(response.data).to be_an(Array) } - end - - describe '#pagination' do - it { expect(error_response.pagination).to be_nil } - - it 'builds the pagination correctly' do - expect(response.pagination).to be_a(Gitlab::PhabricatorImport::Conduit::Pagination) - expect(response.pagination.next_page).to eq('284') - end - end -end diff --git a/spec/lib/gitlab/phabricator_import/conduit/tasks_response_spec.rb b/spec/lib/gitlab/phabricator_import/conduit/tasks_response_spec.rb deleted file mode 100644 index 4e56dead5c0..00000000000 --- a/spec/lib/gitlab/phabricator_import/conduit/tasks_response_spec.rb +++ /dev/null @@ -1,27 +0,0 @@ -# frozen_string_literal: true -require 'spec_helper' - -RSpec.describe Gitlab::PhabricatorImport::Conduit::TasksResponse do - let(:conduit_response) do - Gitlab::PhabricatorImport::Conduit::Response - .new(Gitlab::Json.parse(fixture_file('phabricator_responses/maniphest.search.json'))) - end - - subject(:response) { described_class.new(conduit_response) } - - describe '#pagination' do - it 'delegates to the conduit reponse' do - expect(response.pagination).to eq(conduit_response.pagination) - end - end - - describe '#tasks' do - it 'builds the correct tasks representation' do - tasks = response.tasks - - titles = tasks.map(&:issue_attributes).map { |attrs| attrs[:title] } - - expect(titles).to contain_exactly('Things are slow', 'Things are broken') - end - end -end diff --git a/spec/lib/gitlab/phabricator_import/conduit/user_spec.rb b/spec/lib/gitlab/phabricator_import/conduit/user_spec.rb deleted file mode 100644 index d38421c9405..00000000000 --- a/spec/lib/gitlab/phabricator_import/conduit/user_spec.rb +++ /dev/null @@ -1,49 +0,0 @@ -# frozen_string_literal: true -require 'spec_helper' - -RSpec.describe Gitlab::PhabricatorImport::Conduit::User do - let(:user_client) do - described_class.new(phabricator_url: 'https://see-ya-later.phabricator', api_token: 'api-token') - end - - describe '#users' do - let(:fake_client) { double('Phabricator client') } - - before do - allow(user_client).to receive(:client).and_return(fake_client) - end - - it 'calls the api with the correct params' do - expected_params = { - constraints: { phids: %w[phid-1 phid-2] } - } - - expect(fake_client).to receive(:get).with('user.search', - params: expected_params) - - user_client.users(%w[phid-1 phid-2]) - end - - it 'returns an array of parsed responses' do - response = Gitlab::PhabricatorImport::Conduit::Response - .new(fixture_file('phabricator_responses/user.search.json')) - - allow(fake_client).to receive(:get).and_return(response) - - expect(user_client.users(%w[some phids])).to match_array([an_instance_of(Gitlab::PhabricatorImport::Conduit::UsersResponse)]) - end - - it 'performs multiple requests if more phids than the maximum page size are passed' do - stub_const('Gitlab::PhabricatorImport::Conduit::User::MAX_PAGE_SIZE', 1) - first_params = { constraints: { phids: ['phid-1'] } } - second_params = { constraints: { phids: ['phid-2'] } } - - expect(fake_client).to receive(:get).with('user.search', - params: first_params).once - expect(fake_client).to receive(:get).with('user.search', - params: second_params).once - - user_client.users(%w[phid-1 phid-2]) - end - end -end diff --git a/spec/lib/gitlab/phabricator_import/conduit/users_response_spec.rb b/spec/lib/gitlab/phabricator_import/conduit/users_response_spec.rb deleted file mode 100644 index ebbb2c0598c..00000000000 --- a/spec/lib/gitlab/phabricator_import/conduit/users_response_spec.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true -require 'spec_helper' - -RSpec.describe Gitlab::PhabricatorImport::Conduit::UsersResponse do - let(:conduit_response) do - Gitlab::PhabricatorImport::Conduit::Response - .new(Gitlab::Json.parse(fixture_file('phabricator_responses/user.search.json'))) - end - - subject(:response) { described_class.new(conduit_response) } - - describe '#users' do - it 'builds the correct users representation' do - tasks = response.users - - usernames = tasks.map(&:username) - - expect(usernames).to contain_exactly('jane', 'john') - end - end -end diff --git a/spec/lib/gitlab/phabricator_import/importer_spec.rb b/spec/lib/gitlab/phabricator_import/importer_spec.rb deleted file mode 100644 index e78024c35c1..00000000000 --- a/spec/lib/gitlab/phabricator_import/importer_spec.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::PhabricatorImport::Importer do - it { expect(described_class).to be_async } - - it "acts like it's importing repositories" do - expect(described_class).to be_imports_repository - end - - describe '#execute' do - let(:project) { create(:project, :import_scheduled) } - - subject(:importer) { described_class.new(project) } - - it 'sets a custom jid that will be kept up to date' do - expect { importer.execute }.to change { project.import_state.reload.jid } - end - - it 'starts importing tasks' do - expect(Gitlab::PhabricatorImport::ImportTasksWorker).to receive(:schedule).with(project.id) - - importer.execute - end - - it 'marks the import as failed when something goes wrong' do - allow(importer).to receive(:schedule_first_tasks_page).and_raise('Stuff is broken') - - importer.execute - - expect(project.import_state).to be_failed - end - end -end diff --git a/spec/lib/gitlab/phabricator_import/issues/importer_spec.rb b/spec/lib/gitlab/phabricator_import/issues/importer_spec.rb deleted file mode 100644 index 63ba575aea3..00000000000 --- a/spec/lib/gitlab/phabricator_import/issues/importer_spec.rb +++ /dev/null @@ -1,61 +0,0 @@ -# frozen_string_literal: true -require 'spec_helper' - -RSpec.describe Gitlab::PhabricatorImport::Issues::Importer do - let(:project) { create(:project) } - - let(:response) do - Gitlab::PhabricatorImport::Conduit::TasksResponse.new( - Gitlab::PhabricatorImport::Conduit::Response - .new(Gitlab::Json.parse(fixture_file('phabricator_responses/maniphest.search.json'))) - ) - end - - subject(:importer) { described_class.new(project, nil) } - - before do - client = instance_double(Gitlab::PhabricatorImport::Conduit::Maniphest) - allow(client).to receive(:tasks).and_return(response) - allow(importer).to receive(:client).and_return(client) - end - - describe '#execute' do - it 'imports each task in the response' do - response.tasks.each do |task| - task_importer = instance_double(Gitlab::PhabricatorImport::Issues::TaskImporter) - - expect(task_importer).to receive(:execute) - expect(Gitlab::PhabricatorImport::Issues::TaskImporter) - .to receive(:new).with(project, task) - .and_return(task_importer) - end - - importer.execute - end - - context 'stubbed task import' do - before do - # Stub out the actual importing so we don't perform aditional requests - expect_next_instance_of(Gitlab::PhabricatorImport::Issues::TaskImporter) do |task_importer| - allow(task_importer).to receive(:execute) - end.at_least(1) - end - - it 'schedules the next batch if there is one' do - expect(Gitlab::PhabricatorImport::ImportTasksWorker) - .to receive(:schedule).with(project.id, response.pagination.next_page) - - importer.execute - end - - it 'does not reschedule when there is no next page' do - allow(response.pagination).to receive(:has_next_page?).and_return(false) - - expect(Gitlab::PhabricatorImport::ImportTasksWorker) - .not_to receive(:schedule) - - importer.execute - end - end - end -end diff --git a/spec/lib/gitlab/phabricator_import/issues/task_importer_spec.rb b/spec/lib/gitlab/phabricator_import/issues/task_importer_spec.rb deleted file mode 100644 index 0539bacba44..00000000000 --- a/spec/lib/gitlab/phabricator_import/issues/task_importer_spec.rb +++ /dev/null @@ -1,83 +0,0 @@ -# frozen_string_literal: true -require 'spec_helper' - -RSpec.describe Gitlab::PhabricatorImport::Issues::TaskImporter do - let_it_be(:project) { create(:project) } - - let(:task) do - Gitlab::PhabricatorImport::Representation::Task.new( - { - 'phid' => 'the-phid', - 'fields' => { - 'name' => 'Title', - 'description' => { - 'raw' => '# This is markdown\n it can contain more text.' - }, - 'authorPHID' => 'PHID-USER-456', - 'ownerPHID' => 'PHID-USER-123', - 'dateCreated' => '1518688921', - 'dateClosed' => '1518789995' - } - } - ) - end - - subject(:importer) { described_class.new(project, task) } - - describe '#execute' do - let(:fake_user_finder) { instance_double(Gitlab::PhabricatorImport::UserFinder) } - - before do - allow(fake_user_finder).to receive(:find) - allow(importer).to receive(:user_finder).and_return(fake_user_finder) - end - - it 'creates the issue with the expected attributes' do - issue = importer.execute - - expect(issue.project).to eq(project) - expect(issue).to be_persisted - expect(issue.author).to eq(User.ghost) - expect(issue.title).to eq('Title') - expect(issue.description).to eq('# This is markdown\n it can contain more text.') - expect(issue).to be_closed - expect(issue.created_at).to eq(Time.at(1518688921)) - expect(issue.closed_at).to eq(Time.at(1518789995)) - end - - it 'does not recreate the issue when called multiple times' do - expect { importer.execute } - .to change { project.issues.reload.size }.from(0).to(1) - expect { importer.execute } - .not_to change { project.issues.reload.size } - end - - it 'does not trigger a save when the object did not change' do - existing_issue = create(:issue, - task.issue_attributes.merge(author: User.ghost)) - allow(importer).to receive(:issue).and_return(existing_issue) - - expect(existing_issue).not_to receive(:save!) - - importer.execute - end - - it 'links the author if the author can be found' do - author = create(:user) - expect(fake_user_finder).to receive(:find).with('PHID-USER-456').and_return(author) - - issue = importer.execute - - expect(issue.author).to eq(author) - end - - it 'links an assignee if the user can be found' do - assignee = create(:user) - expect(fake_user_finder).to receive(:find).with('PHID-USER-123').and_return(assignee) - - issue = importer.execute - - expect(issue.assignees).to include(assignee) - end - end -end diff --git a/spec/lib/gitlab/phabricator_import/project_creator_spec.rb b/spec/lib/gitlab/phabricator_import/project_creator_spec.rb deleted file mode 100644 index 016aa0abe4d..00000000000 --- a/spec/lib/gitlab/phabricator_import/project_creator_spec.rb +++ /dev/null @@ -1,59 +0,0 @@ -# frozen_string_literal: true -require 'spec_helper' - -RSpec.describe Gitlab::PhabricatorImport::ProjectCreator do - let(:user) { create(:user) } - let(:params) do - { path: 'new-phab-import', - phabricator_server_url: 'http://phab.example.com', - api_token: 'the-token' } - end - - subject(:creator) { described_class.new(user, params) } - - describe '#execute' do - it 'creates a project correctly and schedule an import', :sidekiq_might_not_need_inline do - expect_next_instance_of(Gitlab::PhabricatorImport::Importer) do |importer| - expect(importer).to receive(:execute) - end - - project = creator.execute - - expect(project).to be_persisted - expect(project).to be_import - expect(project.import_type).to eq('phabricator') - expect(project.import_data.credentials).to match(a_hash_including(api_token: 'the-token')) - expect(project.import_data.data).to match(a_hash_including('phabricator_url' => 'http://phab.example.com')) - expect(project.import_url).to eq(Project::UNKNOWN_IMPORT_URL) - expect(project.namespace).to eq(user.namespace) - end - - context 'when import params are missing' do - let(:params) do - { path: 'new-phab-import', - phabricator_server_url: 'http://phab.example.com', - api_token: '' } - end - - it 'returns nil' do - expect(creator.execute).to be_nil - end - end - - context 'when import params are invalid' do - let(:params) do - { path: 'new-phab-import', - namespace_id: '-1', - phabricator_server_url: 'http://phab.example.com', - api_token: 'the-token' } - end - - it 'returns an unpersisted project' do - project = creator.execute - - expect(project).not_to be_persisted - expect(project).not_to be_valid - end - end - end -end diff --git a/spec/lib/gitlab/phabricator_import/representation/task_spec.rb b/spec/lib/gitlab/phabricator_import/representation/task_spec.rb deleted file mode 100644 index 2b8570e4aff..00000000000 --- a/spec/lib/gitlab/phabricator_import/representation/task_spec.rb +++ /dev/null @@ -1,49 +0,0 @@ -# frozen_string_literal: true - -require 'fast_spec_helper' - -RSpec.describe Gitlab::PhabricatorImport::Representation::Task do - subject(:task) do - described_class.new( - { - 'phid' => 'the-phid', - 'fields' => { - 'name' => 'Title'.ljust(257, '.'), # A string padded to 257 chars - 'authorPHID' => 'a phid', - 'ownerPHID' => 'another user phid', - 'description' => { - 'raw' => '# This is markdown\n it can contain more text.' - }, - 'dateCreated' => '1518688921', - 'dateClosed' => '1518789995' - } - } - ) - end - - describe '#issue_attributes' do - it 'contains the expected values' do - expected_attributes = { - title: 'Title'.ljust(255, '.'), - description: '# This is markdown\n it can contain more text.', - state: :closed, - created_at: Time.at(1518688921), - closed_at: Time.at(1518789995) - } - - expect(task.issue_attributes).to eq(expected_attributes) - end - end - - describe '#author_phid' do - it 'returns the correct field' do - expect(task.author_phid).to eq('a phid') - end - end - - describe '#owner_phid' do - it 'returns the correct field' do - expect(task.owner_phid).to eq('another user phid') - end - end -end diff --git a/spec/lib/gitlab/phabricator_import/representation/user_spec.rb b/spec/lib/gitlab/phabricator_import/representation/user_spec.rb deleted file mode 100644 index 6df26b905cc..00000000000 --- a/spec/lib/gitlab/phabricator_import/representation/user_spec.rb +++ /dev/null @@ -1,28 +0,0 @@ -# frozen_string_literal: true - -require 'fast_spec_helper' - -RSpec.describe Gitlab::PhabricatorImport::Representation::User do - subject(:user) do - described_class.new( - { - 'phid' => 'the-phid', - 'fields' => { - 'username' => 'the-username' - } - } - ) - end - - describe '#phabricator_id' do - it 'returns the phabricator id' do - expect(user.phabricator_id).to eq('the-phid') - end - end - - describe '#username' do - it 'returns the username' do - expect(user.username).to eq('the-username') - end - end -end diff --git a/spec/lib/gitlab/phabricator_import/user_finder_spec.rb b/spec/lib/gitlab/phabricator_import/user_finder_spec.rb deleted file mode 100644 index 2ec2571b7fe..00000000000 --- a/spec/lib/gitlab/phabricator_import/user_finder_spec.rb +++ /dev/null @@ -1,93 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::PhabricatorImport::UserFinder, :clean_gitlab_redis_cache do - let(:project) { create(:project, namespace: create(:group)) } - - subject(:finder) { described_class.new(project, %w[first-phid second-phid]) } - - before do - project.namespace.add_developer(existing_user) - end - - describe '#find' do - let!(:existing_user) { create(:user, username: 'existing-user') } - let(:cache) { Gitlab::PhabricatorImport::Cache::Map.new(project) } - - before do - allow(finder).to receive(:object_map).and_return(cache) - end - - context 'for a cached phid' do - before do - cache.set_gitlab_model(existing_user, 'first-phid') - end - - it 'returns the existing user' do - expect(finder.find('first-phid')).to eq(existing_user) - end - - it 'does not perform a find using the API' do - expect(finder).not_to receive(:find_user_for_phid) - - finder.find('first-phid') - end - - it 'excludes the phid from the request if one needs to be made' do - client = instance_double(Gitlab::PhabricatorImport::Conduit::User) - allow(finder).to receive(:client).and_return(client) - - expect(client).to receive(:users).with(['second-phid']).and_return([]) - - finder.find('first-phid') - finder.find('second-phid') - end - end - - context 'when the phid is not cached' do - let(:response) do - [ - instance_double( - Gitlab::PhabricatorImport::Conduit::UsersResponse, - users: [instance_double(Gitlab::PhabricatorImport::Representation::User, phabricator_id: 'second-phid', username: 'existing-user')] - ), - instance_double( - Gitlab::PhabricatorImport::Conduit::UsersResponse, - users: [instance_double(Gitlab::PhabricatorImport::Representation::User, phabricator_id: 'first-phid', username: 'other-user')] - ) - ] - end - - let(:client) do - client = instance_double(Gitlab::PhabricatorImport::Conduit::User) - allow(client).to receive(:users).and_return(response) - - client - end - - before do - allow(finder).to receive(:client).and_return(client) - end - - it 'loads the users from the API once' do - expect(client).to receive(:users).and_return(response).once - - expect(finder.find('second-phid')).to eq(existing_user) - expect(finder.find('first-phid')).to be_nil - end - - it 'adds found users to the cache' do - expect { finder.find('second-phid') } - .to change { cache.get_gitlab_model('second-phid') } - .from(nil).to(existing_user) - end - - it 'only returns users that are members of the project' do - create(:user, username: 'other-user') - - expect(finder.find('first-phid')).to eq(nil) - end - end - end -end diff --git a/spec/lib/gitlab/phabricator_import/worker_state_spec.rb b/spec/lib/gitlab/phabricator_import/worker_state_spec.rb deleted file mode 100644 index 4a07e28440f..00000000000 --- a/spec/lib/gitlab/phabricator_import/worker_state_spec.rb +++ /dev/null @@ -1,49 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::PhabricatorImport::WorkerState, :clean_gitlab_redis_shared_state do - subject(:state) { described_class.new('weird-project-id') } - - let(:key) { 'phabricator-import/jobs/project-weird-project-id/job-count' } - - describe '#add_job' do - it 'increments the counter for jobs' do - set_value(3) - - expect { state.add_job }.to change { get_value }.from('3').to('4') - end - end - - describe '#remove_job' do - it 'decrements the counter for jobs' do - set_value(3) - - expect { state.remove_job }.to change { get_value }.from('3').to('2') - end - end - - describe '#running_count' do - it 'reads the value' do - set_value(9) - - expect(state.running_count).to eq(9) - end - - it 'returns 0 when nothing was set' do - expect(state.running_count).to eq(0) - end - end - - def set_value(value) - redis.with { |r| r.set(key, value) } - end - - def get_value - redis.with { |r| r.get(key) } - end - - def redis - Gitlab::Redis::SharedState - end -end diff --git a/spec/lib/gitlab/project_authorizations_spec.rb b/spec/lib/gitlab/project_authorizations_spec.rb index 640cf9be453..b076bb65fb5 100644 --- a/spec/lib/gitlab/project_authorizations_spec.rb +++ b/spec/lib/gitlab/project_authorizations_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::ProjectAuthorizations do +RSpec.describe Gitlab::ProjectAuthorizations, feature_category: :system_access do def map_access_levels(rows) rows.each_with_object({}) do |row, hash| hash[row.project_id] = row.access_level @@ -13,408 +13,421 @@ RSpec.describe Gitlab::ProjectAuthorizations do described_class.new(user).calculate end - context 'user added to group and project' do - let(:group) { create(:group) } - let!(:other_project) { create(:project) } - let!(:group_project) { create(:project, namespace: group) } - let!(:owned_project) { create(:project) } - let(:user) { owned_project.namespace.owner } + # Inline this shared example while cleaning up feature flag linear_project_authorization + RSpec.shared_examples 'project authorizations' do + context 'user added to group and project' do + let(:group) { create(:group) } + let!(:other_project) { create(:project) } + let!(:group_project) { create(:project, namespace: group) } + let!(:owned_project) { create(:project) } + let(:user) { owned_project.namespace.owner } - before do - other_project.add_reporter(user) - group.add_developer(user) - end + before do + other_project.add_reporter(user) + group.add_developer(user) + end - it 'returns the correct number of authorizations' do - expect(authorizations.length).to eq(3) - end + it 'returns the correct number of authorizations' do + expect(authorizations.length).to eq(3) + end - it 'includes the correct projects' do - expect(authorizations.pluck(:project_id)) - .to include(owned_project.id, other_project.id, group_project.id) - end + it 'includes the correct projects' do + expect(authorizations.pluck(:project_id)) + .to include(owned_project.id, other_project.id, group_project.id) + end - it 'includes the correct access levels' do - mapping = map_access_levels(authorizations) + it 'includes the correct access levels' do + mapping = map_access_levels(authorizations) - expect(mapping[owned_project.id]).to eq(Gitlab::Access::OWNER) - expect(mapping[other_project.id]).to eq(Gitlab::Access::REPORTER) - expect(mapping[group_project.id]).to eq(Gitlab::Access::DEVELOPER) + expect(mapping[owned_project.id]).to eq(Gitlab::Access::OWNER) + expect(mapping[other_project.id]).to eq(Gitlab::Access::REPORTER) + expect(mapping[group_project.id]).to eq(Gitlab::Access::DEVELOPER) + end end - end - context 'unapproved access request' do - let_it_be(:group) { create(:group) } - let_it_be(:user) { create(:user) } + context 'unapproved access request' do + let_it_be(:group) { create(:group) } + let_it_be(:user) { create(:user) } - subject(:mapping) { map_access_levels(authorizations) } + subject(:mapping) { map_access_levels(authorizations) } - context 'group membership' do - let!(:group_project) { create(:project, namespace: group) } + context 'group membership' do + let!(:group_project) { create(:project, namespace: group) } - before do - create(:group_member, :developer, :access_request, user: user, group: group) - end + before do + create(:group_member, :developer, :access_request, user: user, group: group) + end - it 'does not create authorization' do - expect(mapping[group_project.id]).to be_nil + it 'does not create authorization' do + expect(mapping[group_project.id]).to be_nil + end end - end - context 'inherited group membership' do - let!(:sub_group) { create(:group, parent: group) } - let!(:sub_group_project) { create(:project, namespace: sub_group) } + context 'inherited group membership' do + let!(:sub_group) { create(:group, parent: group) } + let!(:sub_group_project) { create(:project, namespace: sub_group) } - before do - create(:group_member, :developer, :access_request, user: user, group: group) - end + before do + create(:group_member, :developer, :access_request, user: user, group: group) + end - it 'does not create authorization' do - expect(mapping[sub_group_project.id]).to be_nil + it 'does not create authorization' do + expect(mapping[sub_group_project.id]).to be_nil + end end - end - context 'project membership' do - let!(:group_project) { create(:project, namespace: group) } + context 'project membership' do + let!(:group_project) { create(:project, namespace: group) } - before do - create(:project_member, :developer, :access_request, user: user, project: group_project) - end + before do + create(:project_member, :developer, :access_request, user: user, project: group_project) + end - it 'does not create authorization' do - expect(mapping[group_project.id]).to be_nil + it 'does not create authorization' do + expect(mapping[group_project.id]).to be_nil + end end - end - context 'shared group' do - let!(:shared_group) { create(:group) } - let!(:shared_group_project) { create(:project, namespace: shared_group) } + context 'shared group' do + let!(:shared_group) { create(:group) } + let!(:shared_group_project) { create(:project, namespace: shared_group) } - before do - create(:group_group_link, shared_group: shared_group, shared_with_group: group) - create(:group_member, :developer, :access_request, user: user, group: group) - end + before do + create(:group_group_link, shared_group: shared_group, shared_with_group: group) + create(:group_member, :developer, :access_request, user: user, group: group) + end - it 'does not create authorization' do - expect(mapping[shared_group_project.id]).to be_nil + it 'does not create authorization' do + expect(mapping[shared_group_project.id]).to be_nil + end end - end - context 'shared project' do - let!(:another_group) { create(:group) } - let!(:shared_project) { create(:project, namespace: another_group) } + context 'shared project' do + let!(:another_group) { create(:group) } + let!(:shared_project) { create(:project, namespace: another_group) } - before do - create(:project_group_link, group: group, project: shared_project) - create(:group_member, :developer, :access_request, user: user, group: group) - end + before do + create(:project_group_link, group: group, project: shared_project) + create(:group_member, :developer, :access_request, user: user, group: group) + end - it 'does not create authorization' do - expect(mapping[shared_project.id]).to be_nil + it 'does not create authorization' do + expect(mapping[shared_project.id]).to be_nil + end end end - end - context 'user with minimal access to group' do - let_it_be(:group) { create(:group) } - let_it_be(:user) { create(:user) } + context 'user with minimal access to group' do + let_it_be(:group) { create(:group) } + let_it_be(:user) { create(:user) } - subject(:mapping) { map_access_levels(authorizations) } + subject(:mapping) { map_access_levels(authorizations) } - context 'group membership' do - let!(:group_project) { create(:project, namespace: group) } + context 'group membership' do + let!(:group_project) { create(:project, namespace: group) } - before do - create(:group_member, :minimal_access, user: user, source: group) - end + before do + create(:group_member, :minimal_access, user: user, source: group) + end - it 'does not create authorization' do - expect(mapping[group_project.id]).to be_nil + it 'does not create authorization' do + expect(mapping[group_project.id]).to be_nil + end end - end - context 'inherited group membership' do - let!(:sub_group) { create(:group, parent: group) } - let!(:sub_group_project) { create(:project, namespace: sub_group) } + context 'inherited group membership' do + let!(:sub_group) { create(:group, parent: group) } + let!(:sub_group_project) { create(:project, namespace: sub_group) } - before do - create(:group_member, :minimal_access, user: user, source: group) - end + before do + create(:group_member, :minimal_access, user: user, source: group) + end - it 'does not create authorization' do - expect(mapping[sub_group_project.id]).to be_nil + it 'does not create authorization' do + expect(mapping[sub_group_project.id]).to be_nil + end end - end - context 'shared group' do - let!(:shared_group) { create(:group) } - let!(:shared_group_project) { create(:project, namespace: shared_group) } + context 'shared group' do + let!(:shared_group) { create(:group) } + let!(:shared_group_project) { create(:project, namespace: shared_group) } - before do - create(:group_group_link, shared_group: shared_group, shared_with_group: group) - create(:group_member, :minimal_access, user: user, source: group) - end + before do + create(:group_group_link, shared_group: shared_group, shared_with_group: group) + create(:group_member, :minimal_access, user: user, source: group) + end - it 'does not create authorization' do - expect(mapping[shared_group_project.id]).to be_nil + it 'does not create authorization' do + expect(mapping[shared_group_project.id]).to be_nil + end end - end - context 'shared project' do - let!(:another_group) { create(:group) } - let!(:shared_project) { create(:project, namespace: another_group) } + context 'shared project' do + let!(:another_group) { create(:group) } + let!(:shared_project) { create(:project, namespace: another_group) } - before do - create(:project_group_link, group: group, project: shared_project) - create(:group_member, :minimal_access, user: user, source: group) - end + before do + create(:project_group_link, group: group, project: shared_project) + create(:group_member, :minimal_access, user: user, source: group) + end - it 'does not create authorization' do - expect(mapping[shared_project.id]).to be_nil + it 'does not create authorization' do + expect(mapping[shared_project.id]).to be_nil + end end end - end - context 'with nested groups' do - let(:group) { create(:group) } - let!(:nested_group) { create(:group, parent: group) } - let!(:nested_project) { create(:project, namespace: nested_group) } - let(:user) { create(:user) } + context 'with nested groups' do + let(:group) { create(:group) } + let!(:nested_group) { create(:group, parent: group) } + let!(:nested_project) { create(:project, namespace: nested_group) } + let(:user) { create(:user) } - before do - group.add_developer(user) - end + before do + group.add_developer(user) + end - it 'includes nested groups' do - expect(authorizations.pluck(:project_id)).to include(nested_project.id) - end + it 'includes nested groups' do + expect(authorizations.pluck(:project_id)).to include(nested_project.id) + end - it 'inherits access levels when the user is not a member of a nested group' do - mapping = map_access_levels(authorizations) + it 'inherits access levels when the user is not a member of a nested group' do + mapping = map_access_levels(authorizations) - expect(mapping[nested_project.id]).to eq(Gitlab::Access::DEVELOPER) - end + expect(mapping[nested_project.id]).to eq(Gitlab::Access::DEVELOPER) + end - it 'uses the greatest access level when a user is a member of a nested group' do - nested_group.add_maintainer(user) + it 'uses the greatest access level when a user is a member of a nested group' do + nested_group.add_maintainer(user) - mapping = map_access_levels(authorizations) + mapping = map_access_levels(authorizations) - expect(mapping[nested_project.id]).to eq(Gitlab::Access::MAINTAINER) + expect(mapping[nested_project.id]).to eq(Gitlab::Access::MAINTAINER) + end end - end - context 'with shared projects' do - let_it_be(:shared_with_group) { create(:group) } - let_it_be(:user) { create(:user) } - let_it_be(:project) { create(:project, group: create(:group)) } + context 'with shared projects' do + let_it_be(:shared_with_group) { create(:group) } + let_it_be(:user) { create(:user) } + let_it_be(:project) { create(:project, group: create(:group)) } - let(:mapping) { map_access_levels(authorizations) } + let(:mapping) { map_access_levels(authorizations) } - before do - create(:project_group_link, :developer, project: project, group: shared_with_group) - shared_with_group.add_maintainer(user) - end - - it 'creates proper authorizations' do - expect(mapping[project.id]).to eq(Gitlab::Access::DEVELOPER) - end - - context 'even when the `lock_memberships_to_ldap` setting has been turned ON' do before do - stub_application_setting(lock_memberships_to_ldap: true) + create(:project_group_link, :developer, project: project, group: shared_with_group) + shared_with_group.add_maintainer(user) end it 'creates proper authorizations' do expect(mapping[project.id]).to eq(Gitlab::Access::DEVELOPER) end - end - context 'when the group containing the project has forbidden group shares for any of its projects' do - before do - project.namespace.update!(share_with_group_lock: true) + context 'even when the `lock_memberships_to_ldap` setting has been turned ON' do + before do + stub_application_setting(lock_memberships_to_ldap: true) + end + + it 'creates proper authorizations' do + expect(mapping[project.id]).to eq(Gitlab::Access::DEVELOPER) + end end - it 'does not create authorizations' do - expect(mapping[project.id]).to be_nil + context 'when the group containing the project has forbidden group shares for any of its projects' do + before do + project.namespace.update!(share_with_group_lock: true) + end + + it 'does not create authorizations' do + expect(mapping[project.id]).to be_nil + end end end - end - context 'with shared groups' do - let(:parent_group_user) { create(:user) } - let(:group_user) { create(:user) } - let(:child_group_user) { create(:user) } + context 'with shared groups' do + let(:parent_group_user) { create(:user) } + let(:group_user) { create(:user) } + let(:child_group_user) { create(:user) } - let_it_be(:group_parent) { create(:group, :private) } - let_it_be(:group) { create(:group, :private, parent: group_parent) } - let_it_be(:group_child) { create(:group, :private, parent: group) } + let_it_be(:group_parent) { create(:group, :private) } + let_it_be(:group) { create(:group, :private, parent: group_parent) } + let_it_be(:group_child) { create(:group, :private, parent: group) } - let_it_be(:shared_group_parent) { create(:group, :private) } - let_it_be(:shared_group) { create(:group, :private, parent: shared_group_parent) } - let_it_be(:shared_group_child) { create(:group, :private, parent: shared_group) } + let_it_be(:shared_group_parent) { create(:group, :private) } + let_it_be(:shared_group) { create(:group, :private, parent: shared_group_parent) } + let_it_be(:shared_group_child) { create(:group, :private, parent: shared_group) } - let_it_be(:project_parent) { create(:project, group: shared_group_parent) } - let_it_be(:project) { create(:project, group: shared_group) } - let_it_be(:project_child) { create(:project, group: shared_group_child) } + let_it_be(:project_parent) { create(:project, group: shared_group_parent) } + let_it_be(:project) { create(:project, group: shared_group) } + let_it_be(:project_child) { create(:project, group: shared_group_child) } - before do - group_parent.add_owner(parent_group_user) - group.add_owner(group_user) - group_child.add_owner(child_group_user) + before do + group_parent.add_owner(parent_group_user) + group.add_owner(group_user) + group_child.add_owner(child_group_user) - create(:group_group_link, shared_group: shared_group, shared_with_group: group) - end + create(:group_group_link, shared_group: shared_group, shared_with_group: group) + end - context 'group user' do - let(:user) { group_user } + context 'group user' do + let(:user) { group_user } - it 'creates proper authorizations' do - mapping = map_access_levels(authorizations) + it 'creates proper authorizations' do + mapping = map_access_levels(authorizations) - expect(mapping[project_parent.id]).to be_nil - expect(mapping[project.id]).to eq(Gitlab::Access::DEVELOPER) - expect(mapping[project_child.id]).to eq(Gitlab::Access::DEVELOPER) + expect(mapping[project_parent.id]).to be_nil + expect(mapping[project.id]).to eq(Gitlab::Access::DEVELOPER) + expect(mapping[project_child.id]).to eq(Gitlab::Access::DEVELOPER) + end end - end - context 'with lower group access level than max access level for share' do - let(:user) { create(:user) } + context 'with lower group access level than max access level for share' do + let(:user) { create(:user) } - it 'creates proper authorizations' do - group.add_reporter(user) + it 'creates proper authorizations' do + group.add_reporter(user) - mapping = map_access_levels(authorizations) + mapping = map_access_levels(authorizations) - expect(mapping[project_parent.id]).to be_nil - expect(mapping[project.id]).to eq(Gitlab::Access::REPORTER) - expect(mapping[project_child.id]).to eq(Gitlab::Access::REPORTER) + expect(mapping[project_parent.id]).to be_nil + expect(mapping[project.id]).to eq(Gitlab::Access::REPORTER) + expect(mapping[project_child.id]).to eq(Gitlab::Access::REPORTER) + end end - end - context 'parent group user' do - let(:user) { parent_group_user } + context 'parent group user' do + let(:user) { parent_group_user } - it 'creates proper authorizations' do - mapping = map_access_levels(authorizations) + it 'creates proper authorizations' do + mapping = map_access_levels(authorizations) - expect(mapping[project_parent.id]).to be_nil - expect(mapping[project.id]).to be_nil - expect(mapping[project_child.id]).to be_nil + expect(mapping[project_parent.id]).to be_nil + expect(mapping[project.id]).to be_nil + expect(mapping[project_child.id]).to be_nil + end end - end - context 'child group user' do - let(:user) { child_group_user } + context 'child group user' do + let(:user) { child_group_user } - it 'creates proper authorizations' do - mapping = map_access_levels(authorizations) + it 'creates proper authorizations' do + mapping = map_access_levels(authorizations) - expect(mapping[project_parent.id]).to be_nil - expect(mapping[project.id]).to be_nil - expect(mapping[project_child.id]).to be_nil + expect(mapping[project_parent.id]).to be_nil + expect(mapping[project.id]).to be_nil + expect(mapping[project_child.id]).to be_nil + end end - end - context 'user without accepted access request' do - let!(:user) { create(:user) } + context 'user without accepted access request' do + let!(:user) { create(:user) } - it 'does not have access to group and its projects' do - create(:group_member, :developer, :access_request, user: user, group: group) + it 'does not have access to group and its projects' do + create(:group_member, :developer, :access_request, user: user, group: group) - mapping = map_access_levels(authorizations) + mapping = map_access_levels(authorizations) - expect(mapping[project_parent.id]).to be_nil - expect(mapping[project.id]).to be_nil - expect(mapping[project_child.id]).to be_nil + expect(mapping[project_parent.id]).to be_nil + expect(mapping[project.id]).to be_nil + expect(mapping[project_child.id]).to be_nil + end end - end - context 'unrelated project owner' do - let(:common_id) { non_existing_record_id } - let!(:group) { create(:group, id: common_id) } - let!(:unrelated_project) { create(:project, id: common_id) } - let(:user) { unrelated_project.first_owner } + context 'unrelated project owner' do + let(:common_id) { non_existing_record_id } + let!(:group) { create(:group, id: common_id) } + let!(:unrelated_project) { create(:project, id: common_id) } + let(:user) { unrelated_project.first_owner } - it 'does not have access to group and its projects' do - mapping = map_access_levels(authorizations) + it 'does not have access to group and its projects' do + mapping = map_access_levels(authorizations) - expect(mapping[project_parent.id]).to be_nil - expect(mapping[project.id]).to be_nil - expect(mapping[project_child.id]).to be_nil + expect(mapping[project_parent.id]).to be_nil + expect(mapping[project.id]).to be_nil + expect(mapping[project_child.id]).to be_nil + end end end - end - context 'with pending memberships' do - let_it_be(:group) { create(:group) } - let_it_be(:user) { create(:user) } + context 'with pending memberships' do + let_it_be(:group) { create(:group) } + let_it_be(:user) { create(:user) } - subject(:mapping) { map_access_levels(authorizations) } + subject(:mapping) { map_access_levels(authorizations) } - context 'group membership' do - let!(:group_project) { create(:project, namespace: group) } + context 'group membership' do + let!(:group_project) { create(:project, namespace: group) } - before do - create(:group_member, :developer, :awaiting, user: user, group: group) - end + before do + create(:group_member, :developer, :awaiting, user: user, group: group) + end - it 'does not create authorization' do - expect(mapping[group_project.id]).to be_nil + it 'does not create authorization' do + expect(mapping[group_project.id]).to be_nil + end end - end - context 'inherited group membership' do - let!(:sub_group) { create(:group, parent: group) } - let!(:sub_group_project) { create(:project, namespace: sub_group) } + context 'inherited group membership' do + let!(:sub_group) { create(:group, parent: group) } + let!(:sub_group_project) { create(:project, namespace: sub_group) } - before do - create(:group_member, :developer, :awaiting, user: user, group: group) - end + before do + create(:group_member, :developer, :awaiting, user: user, group: group) + end - it 'does not create authorization' do - expect(mapping[sub_group_project.id]).to be_nil + it 'does not create authorization' do + expect(mapping[sub_group_project.id]).to be_nil + end end - end - context 'project membership' do - let!(:group_project) { create(:project, namespace: group) } + context 'project membership' do + let!(:group_project) { create(:project, namespace: group) } - before do - create(:project_member, :developer, :awaiting, user: user, project: group_project) - end + before do + create(:project_member, :developer, :awaiting, user: user, project: group_project) + end - it 'does not create authorization' do - expect(mapping[group_project.id]).to be_nil + it 'does not create authorization' do + expect(mapping[group_project.id]).to be_nil + end end - end - context 'shared group' do - let!(:shared_group) { create(:group) } - let!(:shared_group_project) { create(:project, namespace: shared_group) } + context 'shared group' do + let!(:shared_group) { create(:group) } + let!(:shared_group_project) { create(:project, namespace: shared_group) } - before do - create(:group_group_link, shared_group: shared_group, shared_with_group: group) - create(:group_member, :developer, :awaiting, user: user, group: group) - end + before do + create(:group_group_link, shared_group: shared_group, shared_with_group: group) + create(:group_member, :developer, :awaiting, user: user, group: group) + end - it 'does not create authorization' do - expect(mapping[shared_group_project.id]).to be_nil + it 'does not create authorization' do + expect(mapping[shared_group_project.id]).to be_nil + end end - end - context 'shared project' do - let!(:another_group) { create(:group) } - let!(:shared_project) { create(:project, namespace: another_group) } + context 'shared project' do + let!(:another_group) { create(:group) } + let!(:shared_project) { create(:project, namespace: another_group) } - before do - create(:project_group_link, group: group, project: shared_project) - create(:group_member, :developer, :awaiting, user: user, group: group) - end + before do + create(:project_group_link, group: group, project: shared_project) + create(:group_member, :developer, :awaiting, user: user, group: group) + end - it 'does not create authorization' do - expect(mapping[shared_project.id]).to be_nil + it 'does not create authorization' do + expect(mapping[shared_project.id]).to be_nil + end end end end + + context 'when feature_flag linear_project_authorization_is disabled' do + before do + stub_feature_flags(linear_project_authorization: false) + end + + it_behaves_like 'project authorizations' + end + + it_behaves_like 'project authorizations' end diff --git a/spec/lib/gitlab/prometheus/internal_spec.rb b/spec/lib/gitlab/prometheus/internal_spec.rb index b08b8813470..ff5da301347 100644 --- a/spec/lib/gitlab/prometheus/internal_spec.rb +++ b/spec/lib/gitlab/prometheus/internal_spec.rb @@ -81,7 +81,7 @@ RSpec.describe Gitlab::Prometheus::Internal do context 'when prometheus setting is not present in gitlab.yml' do before do - allow(Gitlab.config).to receive(:prometheus).and_raise(Settingslogic::MissingSetting) + allow(Gitlab.config).to receive(:prometheus).and_raise(GitlabSettings::MissingSetting) end it 'does not fail' do @@ -97,7 +97,7 @@ RSpec.describe Gitlab::Prometheus::Internal do context 'when prometheus setting is not present in gitlab.yml' do before do - allow(Gitlab.config).to receive(:prometheus).and_raise(Settingslogic::MissingSetting) + allow(Gitlab.config).to receive(:prometheus).and_raise(GitlabSettings::MissingSetting) end it 'does not fail' do diff --git a/spec/lib/gitlab/prometheus/queries/knative_invocation_query_spec.rb b/spec/lib/gitlab/prometheus/queries/knative_invocation_query_spec.rb deleted file mode 100644 index ff48b9ada90..00000000000 --- a/spec/lib/gitlab/prometheus/queries/knative_invocation_query_spec.rb +++ /dev/null @@ -1,31 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Prometheus::Queries::KnativeInvocationQuery do - include PrometheusHelpers - - let(:project) { create(:project) } - let(:serverless_func) { ::Serverless::Function.new(project, 'test-name', 'test-ns') } - let(:client) { double('prometheus_client') } - - subject { described_class.new(client) } - - context 'verify queries' do - before do - create(:prometheus_metric, - :common, - identifier: :system_metrics_knative_function_invocation_count, - query: 'sum(ceil(rate(istio_requests_total{destination_service_namespace="%{kube_namespace}", destination_service=~"%{function_name}.*"}[1m])*60))') - end - - it 'has the query, but no data' do - expect(client).to receive(:query_range).with( - 'sum(ceil(rate(istio_requests_total{destination_service_namespace="test-ns", destination_service=~"test-name.*"}[1m])*60))', - hash_including(:start_time, :end_time) - ) - - subject.query(serverless_func.id) - end - end -end diff --git a/spec/lib/gitlab/quick_actions/extractor_spec.rb b/spec/lib/gitlab/quick_actions/extractor_spec.rb index e2f289041ce..f91e8d2a7ef 100644 --- a/spec/lib/gitlab/quick_actions/extractor_spec.rb +++ b/spec/lib/gitlab/quick_actions/extractor_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::QuickActions::Extractor do +RSpec.describe Gitlab::QuickActions::Extractor, feature_category: :team_planning do let(:definitions) do Class.new do include Gitlab::QuickActions::Dsl @@ -19,7 +19,8 @@ RSpec.describe Gitlab::QuickActions::Extractor do end.command_definitions end - let(:extractor) { described_class.new(definitions) } + let(:extractor) { described_class.new(definitions, keep_actions: keep_actions) } + let(:keep_actions) { false } shared_examples 'command with no argument' do it 'extracts command' do @@ -176,6 +177,31 @@ RSpec.describe Gitlab::QuickActions::Extractor do end end + describe 'command with keep_actions' do + let(:keep_actions) { true } + + context 'at the start of content' do + it_behaves_like 'command with a single argument' do + let(:original_msg) { "/assign @joe\nworld" } + let(:final_msg) { "\n/assign @joe\n\nworld" } + end + end + + context 'in the middle of content' do + it_behaves_like 'command with a single argument' do + let(:original_msg) { "hello\n/assign @joe\nworld" } + let(:final_msg) { "hello\n\n/assign @joe\n\nworld" } + end + end + + context 'at the end of content' do + it_behaves_like 'command with a single argument' do + let(:original_msg) { "hello\n/assign @joe" } + let(:final_msg) { "hello\n\n/assign @joe" } + end + end + end + it 'extracts command with multiple arguments and various prefixes' do msg = %(hello\n/power @user.name %9.10 ~"bar baz.2"\nworld) msg, commands = extractor.extract_commands(msg) @@ -244,10 +270,19 @@ RSpec.describe Gitlab::QuickActions::Extractor do msg = %(hello\nworld\n/reopen\n/substitution wow this is a thing.) msg, commands = extractor.extract_commands(msg) - expect(commands).to eq [['reopen'], ['substitution', 'wow this is a thing.']] + expect(commands).to match_array [['reopen'], ['substitution', 'wow this is a thing.']] expect(msg).to eq "hello\nworld\nfoo" end + it 'extracts and performs substitution commands with keep_actions' do + extractor = described_class.new(definitions, keep_actions: true) + msg = %(hello\nworld\n/reopen\n/substitution wow this is a thing.) + msg, commands = extractor.extract_commands(msg) + + expect(commands).to match_array [['reopen'], ['substitution', 'wow this is a thing.']] + expect(msg).to eq "hello\nworld\n\n/reopen\n\nfoo" + end + it 'extracts multiple commands' do msg = %(hello\n/power @user.name %9.10 ~"bar baz.2" label\nworld\n/reopen) msg, commands = extractor.extract_commands(msg) diff --git a/spec/lib/gitlab/rack_attack/instrumented_cache_store_spec.rb b/spec/lib/gitlab/rack_attack/instrumented_cache_store_spec.rb deleted file mode 100644 index 8151519ddec..00000000000 --- a/spec/lib/gitlab/rack_attack/instrumented_cache_store_spec.rb +++ /dev/null @@ -1,89 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::RackAttack::InstrumentedCacheStore do - using RSpec::Parameterized::TableSyntax - - let(:store) { ::ActiveSupport::Cache::NullStore.new } - - subject { described_class.new(upstream_store: store) } - - where(:operation, :params, :test_proc) do - :fetch | [:key] | ->(s) { s.fetch(:key) } - :read | [:key] | ->(s) { s.read(:key) } - :read_multi | [:key_1, :key_2, :key_3] | ->(s) { s.read_multi(:key_1, :key_2, :key_3) } - :write_multi | [{ key_1: 1, key_2: 2, key_3: 3 }] | ->(s) { s.write_multi(key_1: 1, key_2: 2, key_3: 3) } - :fetch_multi | [:key_1, :key_2, :key_3] | ->(s) { s.fetch_multi(:key_1, :key_2, :key_3) {} } - :write | [:key, :value, { option_1: 1 }] | ->(s) { s.write(:key, :value, option_1: 1) } - :delete | [:key] | ->(s) { s.delete(:key) } - :exist? | [:key, { option_1: 1 }] | ->(s) { s.exist?(:key, option_1: 1) } - :delete_matched | [/^key$/, { option_1: 1 }] | ->(s) { s.delete_matched(/^key$/, option_1: 1 ) } - :increment | [:key, 1] | ->(s) { s.increment(:key, 1) } - :decrement | [:key, 1] | ->(s) { s.decrement(:key, 1) } - :cleanup | [] | ->(s) { s.cleanup } - :clear | [] | ->(s) { s.clear } - end - - with_them do - it 'publishes a notification' do - event = nil - - begin - subscriber = ActiveSupport::Notifications.subscribe("redis.rack_attack") do |*args| - event = ActiveSupport::Notifications::Event.new(*args) - end - - test_proc.call(subject) - ensure - ActiveSupport::Notifications.unsubscribe(subscriber) if subscriber - end - - expect(event).not_to be_nil - expect(event.name).to eq("redis.rack_attack") - expect(event.duration).to be_a(Float).and(be > 0.0) - expect(event.payload[:operation]).to eql(operation) - end - - it 'publishes a notification even if the cache store returns an error' do - allow(store).to receive(operation).and_raise('Something went wrong') - - event = nil - exception = nil - - begin - subscriber = ActiveSupport::Notifications.subscribe("redis.rack_attack") do |*args| - event = ActiveSupport::Notifications::Event.new(*args) - end - - begin - test_proc.call(subject) - rescue StandardError => e - exception = e - end - ensure - ActiveSupport::Notifications.unsubscribe(subscriber) if subscriber - end - - expect(event).not_to be_nil - expect(event.name).to eq("redis.rack_attack") - expect(event.duration).to be_a(Float).and(be > 0.0) - expect(event.payload[:operation]).to eql(operation) - - expect(exception).not_to be_nil - expect(exception.message).to eql('Something went wrong') - end - - it 'delegates to the upstream store' do - allow(store).to receive(operation).and_call_original - - if params.empty? - expect(store).to receive(operation).with(no_args) - else - expect(store).to receive(operation).with(*params) - end - - test_proc.call(subject) - end - end -end diff --git a/spec/lib/gitlab/rack_attack/request_spec.rb b/spec/lib/gitlab/rack_attack/request_spec.rb index 5345205e15b..ae0abfd0bc5 100644 --- a/spec/lib/gitlab/rack_attack/request_spec.rb +++ b/spec/lib/gitlab/rack_attack/request_spec.rb @@ -259,7 +259,7 @@ RSpec.describe Gitlab::RackAttack::Request do other_token = SecureRandom.base64(ActionController::RequestForgeryProtection::AUTHENTICITY_TOKEN_LENGTH) where(:session, :env, :expected) do - {} | {} | false # rubocop:disable Lint/BinaryOperatorWithIdenticalOperands + {} | {} | false {} | { 'HTTP_X_CSRF_TOKEN' => valid_token } | false { _csrf_token: valid_token } | { 'HTTP_X_CSRF_TOKEN' => other_token } | false { _csrf_token: valid_token } | { 'HTTP_X_CSRF_TOKEN' => valid_token } | true diff --git a/spec/lib/gitlab/rack_attack/store_spec.rb b/spec/lib/gitlab/rack_attack/store_spec.rb new file mode 100644 index 00000000000..19b3f239d91 --- /dev/null +++ b/spec/lib/gitlab/rack_attack/store_spec.rb @@ -0,0 +1,113 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::RackAttack::Store, :clean_gitlab_redis_rate_limiting, feature_category: :scalability do + let(:store) { described_class.new } + let(:key) { 'foobar' } + let(:namespaced_key) { "cache:gitlab:#{key}" } + + def with_redis(&block) + Gitlab::Redis::RateLimiting.with(&block) + end + + describe '#increment' do + it 'increments without expiry' do + 5.times do |i| + expect(store.increment(key, 1)).to eq(i + 1) + + with_redis do |redis| + expect(redis.get(namespaced_key).to_i).to eq(i + 1) + expect(redis.ttl(namespaced_key)).to eq(-1) + end + end + end + + it 'rejects amounts other than 1' do + expect { store.increment(key, 2) }.to raise_exception(described_class::InvalidAmount) + end + + context 'with expiry' do + it 'increments and sets expiry' do + 5.times do |i| + expect(store.increment(key, 1, expires_in: 456)).to eq(i + 1) + + with_redis do |redis| + expect(redis.get(namespaced_key).to_i).to eq(i + 1) + expect(redis.ttl(namespaced_key)).to be_within(10).of(456) + end + end + end + end + end + + describe '#read' do + subject { store.read(key) } + + it 'reads the namespaced key' do + with_redis { |r| r.set(namespaced_key, '123') } + + expect(subject).to eq('123') + end + end + + describe '#write' do + subject { store.write(key, '123', options) } + + let(:options) { {} } + + it 'sets the key' do + subject + + with_redis do |redis| + expect(redis.get(namespaced_key)).to eq('123') + expect(redis.ttl(namespaced_key)).to eq(-1) + end + end + + context 'with expiry' do + let(:options) { { expires_in: 456 } } + + it 'sets the key with expiry' do + subject + + with_redis do |redis| + expect(redis.get(namespaced_key)).to eq('123') + expect(redis.ttl(namespaced_key)).to be_within(10).of(456) + end + end + end + end + + describe '#delete' do + subject { store.delete(key) } + + it { expect(subject).to eq(0) } + + context 'when the key exists' do + before do + with_redis { |r| r.set(namespaced_key, '123') } + end + + it { expect(subject).to eq(1) } + end + end + + describe '#with' do + subject { store.send(:with, &:ping) } + + it { expect(subject).to eq('PONG') } + + context 'when redis is unavailable' do + before do + broken_redis = Redis.new( + url: 'redis://127.0.0.0:0', + instrumentation_class: Gitlab::Redis::RateLimiting.instrumentation_class + ) + allow(Gitlab::Redis::RateLimiting).to receive(:with).and_yield(broken_redis) + end + + it { expect(subject).to eq(nil) } + end + end +end diff --git a/spec/lib/gitlab/reactive_cache_set_cache_spec.rb b/spec/lib/gitlab/reactive_cache_set_cache_spec.rb index 207ac1c0eaa..a78d15134fa 100644 --- a/spec/lib/gitlab/reactive_cache_set_cache_spec.rb +++ b/spec/lib/gitlab/reactive_cache_set_cache_spec.rb @@ -46,17 +46,29 @@ RSpec.describe Gitlab::ReactiveCacheSetCache, :clean_gitlab_redis_cache do end describe '#clear_cache!', :use_clean_rails_redis_caching do - it 'deletes the cached items' do - # Cached key and value - Rails.cache.write('test_item', 'test_value') - # Add key to set - cache.write(cache_prefix, 'test_item') + shared_examples 'clears cache' do + it 'deletes the cached items' do + # Cached key and value + Rails.cache.write('test_item', 'test_value') + # Add key to set + cache.write(cache_prefix, 'test_item') - expect(cache.read(cache_prefix)).to contain_exactly('test_item') - cache.clear_cache!(cache_prefix) + expect(cache.read(cache_prefix)).to contain_exactly('test_item') + cache.clear_cache!(cache_prefix) + + expect(cache.read(cache_prefix)).to be_empty + end + end - expect(cache.read(cache_prefix)).to be_empty + context 'when featuer flag disabled' do + before do + stub_feature_flags(use_pipeline_over_multikey: false) + end + + it_behaves_like 'clears cache' end + + it_behaves_like 'clears cache' end describe '#include?' do diff --git a/spec/lib/gitlab/redis/cache_spec.rb b/spec/lib/gitlab/redis/cache_spec.rb index 64615c4d9ad..b7b4ba0eb2f 100644 --- a/spec/lib/gitlab/redis/cache_spec.rb +++ b/spec/lib/gitlab/redis/cache_spec.rb @@ -4,18 +4,9 @@ require 'spec_helper' RSpec.describe Gitlab::Redis::Cache do let(:instance_specific_config_file) { "config/redis.cache.yml" } - let(:environment_config_file_name) { "GITLAB_REDIS_CACHE_CONFIG_FILE" } include_examples "redis_shared_examples" - describe '#raw_config_hash' do - it 'has a legacy default URL' do - expect(subject).to receive(:fetch_config) { false } - - expect(subject.send(:raw_config_hash)).to eq(url: 'redis://localhost:6380' ) - end - end - describe '.active_support_config' do it 'has a default ttl of 8 hours' do expect(described_class.active_support_config[:expires_in]).to eq(8.hours) @@ -26,22 +17,5 @@ RSpec.describe Gitlab::Redis::Cache do expect(described_class.active_support_config[:expires_in]).to eq(1.day) end - - context 'when encountering an error' do - let(:cache) { ActiveSupport::Cache::RedisCacheStore.new(**described_class.active_support_config) } - - subject { cache.read('x') } - - before do - described_class.with do |redis| - allow(redis).to receive(:get).and_raise(::Redis::CommandError) - end - end - - it 'logs error' do - expect(::Gitlab::ErrorTracking).to receive(:log_exception) - subject - end - end end end diff --git a/spec/lib/gitlab/redis/cluster_rate_limiting_spec.rb b/spec/lib/gitlab/redis/cluster_rate_limiting_spec.rb deleted file mode 100644 index 3eba3233f08..00000000000 --- a/spec/lib/gitlab/redis/cluster_rate_limiting_spec.rb +++ /dev/null @@ -1,7 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Redis::ClusterRateLimiting, feature_category: :redis do - include_examples "redis_new_instance_shared_examples", 'cluster_rate_limiting', Gitlab::Redis::Cache -end diff --git a/spec/lib/gitlab/redis/db_load_balancing_spec.rb b/spec/lib/gitlab/redis/db_load_balancing_spec.rb index d633413ddec..d3d3ced62a9 100644 --- a/spec/lib/gitlab/redis/db_load_balancing_spec.rb +++ b/spec/lib/gitlab/redis/db_load_balancing_spec.rb @@ -41,12 +41,4 @@ RSpec.describe Gitlab::Redis::DbLoadBalancing, feature_category: :scalability do it_behaves_like 'multi store feature flags', :use_primary_and_secondary_stores_for_db_load_balancing, :use_primary_store_as_default_for_db_load_balancing end - - describe '#raw_config_hash' do - it 'has a legacy default URL' do - expect(subject).to receive(:fetch_config).and_return(false) - - expect(subject.send(:raw_config_hash)).to eq(url: 'redis://localhost:6382') - end - end end diff --git a/spec/lib/gitlab/redis/feature_flag_spec.rb b/spec/lib/gitlab/redis/feature_flag_spec.rb new file mode 100644 index 00000000000..49d15ea1b4a --- /dev/null +++ b/spec/lib/gitlab/redis/feature_flag_spec.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Redis::FeatureFlag, feature_category: :redis do + include_examples "redis_new_instance_shared_examples", 'feature_flag', Gitlab::Redis::Cache + + describe '.cache_store' do + it 'has a default ttl of 1 hour' do + expect(described_class.cache_store.options[:expires_in]).to eq(1.hour) + end + end +end diff --git a/spec/lib/gitlab/redis/multi_store_spec.rb b/spec/lib/gitlab/redis/multi_store_spec.rb index 423a7e80ead..e45c29a9dd2 100644 --- a/spec/lib/gitlab/redis/multi_store_spec.rb +++ b/spec/lib/gitlab/redis/multi_store_spec.rb @@ -210,47 +210,6 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do end end - RSpec.shared_examples_for 'fallback read from the non-default store' do - let(:counter) { Gitlab::Metrics::NullMetric.instance } - - before do - allow(Gitlab::Metrics).to receive(:counter).and_return(counter) - end - - it 'fallback and execute on secondary instance' do - expect(multi_store.fallback_store).to receive(name).with(*expected_args).and_call_original - - subject - end - - it 'logs the ReadFromPrimaryError' do - expect(Gitlab::ErrorTracking).to receive(:log_exception).with( - an_instance_of(Gitlab::Redis::MultiStore::ReadFromPrimaryError), - hash_including(command_name: name, instance_name: instance_name) - ) - - subject - end - - it 'increment read fallback count metrics' do - expect(counter).to receive(:increment).with(command: name, instance_name: instance_name) - - subject - end - - include_examples 'reads correct value' - - context 'when fallback read from the secondary instance raises an exception' do - before do - allow(multi_store.fallback_store).to receive(name).with(*expected_args).and_raise(StandardError) - end - - it 'fails with exception' do - expect { subject }.to raise_error(StandardError) - end - end - end - RSpec.shared_examples_for 'secondary store' do it 'execute on the secondary instance' do expect(secondary_store).to receive(name).with(*expected_args).and_call_original @@ -283,31 +242,21 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do subject end - unless params[:block] - it 'does not execute on the secondary store' do - expect(secondary_store).not_to receive(name) - - subject - end - end - include_examples 'reads correct value' end - context 'when reading from primary instance is raising an exception' do + context 'when reading from default instance is raising an exception' do before do allow(multi_store.default_store).to receive(name).with(*expected_args).and_raise(StandardError) allow(Gitlab::ErrorTracking).to receive(:log_exception) end - it 'logs the exception' do + it 'logs the exception and re-raises the error' do expect(Gitlab::ErrorTracking).to receive(:log_exception).with(an_instance_of(StandardError), hash_including(:multi_store_error_message, instance_name: instance_name, command_name: name)) - subject + expect { subject }.to raise_error(an_instance_of(StandardError)) end - - include_examples 'fallback read from the non-default store' end context 'when reading from empty default instance' do @@ -316,7 +265,9 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do multi_store.default_store.flushdb end - include_examples 'fallback read from the non-default store' + it 'does not call the fallback store' do + expect(multi_store.fallback_store).not_to receive(name) + end end context 'when the command is executed within pipelined block' do @@ -346,16 +297,16 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do end context 'when block is provided' do - it 'both stores yields to the block' do + it 'only default store yields to the block' do expect(primary_store).to receive(name).and_yield(value) - expect(secondary_store).to receive(name).and_yield(value) + expect(secondary_store).not_to receive(name).and_yield(value) subject end - it 'both stores to execute' do + it 'only default store to execute' do expect(primary_store).to receive(name).with(*expected_args).and_call_original - expect(secondary_store).to receive(name).with(*expected_args).and_call_original + expect(secondary_store).not_to receive(name).with(*expected_args).and_call_original subject end @@ -382,7 +333,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do stub_feature_flags(use_primary_store_as_default_for_test_store: false) end - it 'executes only on secondary redis store', :aggregate_errors do + it 'executes only on secondary redis store', :aggregate_failures do expect(secondary_store).to receive(name).with(*expected_args).and_call_original expect(primary_store).not_to receive(name).with(*expected_args).and_call_original @@ -391,7 +342,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do end context 'when using primary store as default' do - it 'executes only on primary redis store', :aggregate_errors do + it 'executes only on primary redis store', :aggregate_failures do expect(primary_store).to receive(name).with(*expected_args).and_call_original expect(secondary_store).not_to receive(name).with(*expected_args).and_call_original @@ -421,27 +372,19 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do subject do multi_store.mget(values) do |v| multi_store.sadd(skey, v) - multi_store.scard(skey) - end - end - - RSpec.shared_examples_for 'primary instance executes block' do - it 'ensures primary instance is executing the block' do - expect(primary_store).to receive(:send).with(:mget, values).and_call_original - expect(primary_store).to receive(:send).with(:sadd, skey, %w[1 2 3]).and_call_original - expect(primary_store).to receive(:send).with(:scard, skey).and_call_original - - expect(secondary_store).to receive(:send).with(:mget, values).and_call_original - expect(secondary_store).to receive(:send).with(:sadd, skey, %w[10 20 30]).and_call_original - expect(secondary_store).to receive(:send).with(:scard, skey).and_call_original - - subject end end context 'when using both stores' do context 'when primary instance is default store' do - it_behaves_like 'primary instance executes block' + it 'ensures primary instance is executing the block' do + expect(primary_store).to receive(:send).with(:mget, values).and_call_original + expect(primary_store).to receive(:send).with(:sadd, skey, %w[1 2 3]).and_call_original + + expect(secondary_store).not_to receive(:send) + + subject + end end context 'when secondary instance is default store' do @@ -449,8 +392,14 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do stub_feature_flags(use_primary_store_as_default_for_test_store: false) end - # multistore read still favours the primary store - it_behaves_like 'primary instance executes block' + it 'ensures secondary instance is executing the block' do + expect(primary_store).not_to receive(:send) + + expect(secondary_store).to receive(:send).with(:mget, values).and_call_original + expect(secondary_store).to receive(:send).with(:sadd, skey, %w[10 20 30]).and_call_original + + subject + end end end @@ -465,7 +414,6 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do expect(primary_store).to receive(:send).with(:mget, values).and_call_original expect(primary_store).to receive(:send).with(:sadd, skey, %w[1 2 3]).and_call_original - expect(primary_store).to receive(:send).with(:scard, skey).and_call_original subject end @@ -479,7 +427,6 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do it 'ensures only secondary instance is executing the block' do expect(secondary_store).to receive(:send).with(:mget, values).and_call_original expect(secondary_store).to receive(:send).with(:sadd, skey, %w[10 20 30]).and_call_original - expect(secondary_store).to receive(:send).with(:scard, skey).and_call_original expect(primary_store).not_to receive(:send) @@ -490,7 +437,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do end RSpec.shared_examples_for 'verify that store contains values' do |store| - it "#{store} redis store contains correct values", :aggregate_errors do + it "#{store} redis store contains correct values", :aggregate_failures do subject redis_store = multi_store.send(store) @@ -583,7 +530,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do end context 'when executing on primary instance is successful' do - it 'executes on both primary and secondary redis store', :aggregate_errors do + it 'executes on both primary and secondary redis store', :aggregate_failures do expect(primary_store).to receive(name).with(*expected_args).and_call_original expect(secondary_store).to receive(name).with(*expected_args).and_call_original @@ -604,7 +551,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do stub_feature_flags(use_primary_store_as_default_for_test_store: false) end - it 'executes only on secondary redis store', :aggregate_errors do + it 'executes only on secondary redis store', :aggregate_failures do expect(secondary_store).to receive(name).with(*expected_args).and_call_original expect(primary_store).not_to receive(name).with(*expected_args).and_call_original @@ -613,7 +560,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do end context 'when using primary store as default' do - it 'executes only on primary redis store', :aggregate_errors do + it 'executes only on primary redis store', :aggregate_failures do expect(primary_store).to receive(name).with(*expected_args).and_call_original expect(secondary_store).not_to receive(name).with(*expected_args).and_call_original @@ -628,7 +575,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do allow(Gitlab::ErrorTracking).to receive(:log_exception) end - it 'logs the exception and execute on secondary instance', :aggregate_errors do + it 'logs the exception and execute on secondary instance', :aggregate_failures do expect(Gitlab::ErrorTracking).to receive(:log_exception).with(an_instance_of(StandardError), hash_including(:multi_store_error_message, command_name: name, instance_name: instance_name)) expect(secondary_store).to receive(name).with(*expected_args).and_call_original @@ -646,7 +593,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do end end - it 'is executed only 1 time on each instance', :aggregate_errors do + it 'is executed only 1 time on each instance', :aggregate_failures do expect(primary_store).to receive(:pipelined).and_call_original expect_next_instance_of(Redis::PipelinedConnection) do |pipeline| expect(pipeline).to receive(name).with(*expected_args).once.and_call_original @@ -668,120 +615,6 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do end # rubocop:enable RSpec/MultipleMemoizedHelpers - context 'with ENUMERATOR_COMMANDS redis commands' do - let_it_be(:hkey) { "redis:hash" } - let_it_be(:skey) { "redis:set" } - let_it_be(:zkey) { "redis:sortedset" } - let_it_be(:rvalue) { "value1" } - let_it_be(:scan_kwargs) { { match: 'redis:hash' } } - - where(:case_name, :name, :args, :kwargs) do - 'execute :scan_each command' | :scan_each | nil | ref(:scan_kwargs) - 'execute :sscan_each command' | :sscan_each | ref(:skey) | {} - 'execute :hscan_each command' | :hscan_each | ref(:hkey) | {} - 'execute :zscan_each command' | :zscan_each | ref(:zkey) | {} - end - - before(:all) do - primary_store.hset(hkey, rvalue, 1) - primary_store.sadd?(skey, rvalue) - primary_store.zadd(zkey, 1, rvalue) - - secondary_store.hset(hkey, rvalue, 1) - secondary_store.sadd?(skey, rvalue) - secondary_store.zadd(zkey, 1, rvalue) - end - - RSpec.shared_examples_for 'enumerator commands execution' do |both_stores, default_primary| - context 'without block passed in' do - subject do - multi_store.send(name, *args, **kwargs) - end - - it 'returns an enumerator' do - expect(subject).to be_instance_of(Enumerator) - end - end - - context 'with block passed in' do - subject do - multi_store.send(name, *args, **kwargs) { |key| multi_store.incr(rvalue) } - end - - it 'returns nil' do - expect(subject).to eq(nil) - end - - it 'runs block on correct Redis instance' do - if both_stores - expect(primary_store).to receive(name).with(*expected_args).and_call_original - expect(secondary_store).to receive(name).with(*expected_args).and_call_original - - expect(primary_store).to receive(:incr).with(rvalue) - expect(secondary_store).to receive(:incr).with(rvalue) - elsif default_primary - expect(primary_store).to receive(name).with(*expected_args).and_call_original - expect(primary_store).to receive(:incr).with(rvalue) - - expect(secondary_store).not_to receive(name) - expect(secondary_store).not_to receive(:incr).with(rvalue) - else - expect(secondary_store).to receive(name).with(*expected_args).and_call_original - expect(secondary_store).to receive(:incr).with(rvalue) - - expect(primary_store).not_to receive(name) - expect(primary_store).not_to receive(:incr).with(rvalue) - end - - subject - end - end - end - - with_them do - describe name.to_s do - let(:expected_args) { kwargs.present? ? [*args, { **kwargs }] : Array(args) } - - before do - allow(primary_store).to receive(name).and_call_original - allow(secondary_store).to receive(name).and_call_original - end - - context 'when only using 1 store' do - before do - stub_feature_flags(use_primary_and_secondary_stores_for_test_store: false) - end - - context 'when using secondary store as default' do - before do - stub_feature_flags(use_primary_store_as_default_for_test_store: false) - end - - it_behaves_like 'enumerator commands execution', false, false - end - - context 'when using primary store as default' do - it_behaves_like 'enumerator commands execution', false, true - end - end - - context 'when using both stores' do - context 'when using secondary store as default' do - before do - stub_feature_flags(use_primary_store_as_default_for_test_store: false) - end - - it_behaves_like 'enumerator commands execution', true, false - end - - context 'when using primary store as default' do - it_behaves_like 'enumerator commands execution', true, true - end - end - end - end - end - RSpec.shared_examples_for 'pipelined command' do |name| let_it_be(:key1) { "redis:{1}:key_a" } let_it_be(:value1) { "redis_value1" } @@ -812,7 +645,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do end context 'when executing on primary instance is successful' do - it 'executes on both primary and secondary redis store', :aggregate_errors do + it 'executes on both primary and secondary redis store', :aggregate_failures do expect(primary_store).to receive(name).and_call_original expect(secondary_store).to receive(name).and_call_original @@ -829,7 +662,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do allow(Gitlab::ErrorTracking).to receive(:log_exception) end - it 'logs the exception and execute on secondary instance', :aggregate_errors do + it 'logs the exception and execute on secondary instance', :aggregate_failures do expect(Gitlab::ErrorTracking).to receive(:log_exception).with(an_instance_of(StandardError), hash_including(:multi_store_error_message, command_name: name)) expect(secondary_store).to receive(name).and_call_original @@ -927,7 +760,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do stub_feature_flags(use_primary_store_as_default_for_test_store: false) end - it 'executes on secondary store', :aggregate_errors do + it 'executes on secondary store', :aggregate_failures do expect(primary_store).not_to receive(:send).and_call_original expect(secondary_store).to receive(:send).and_call_original @@ -936,7 +769,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do end context 'when using primary store as default' do - it 'executes on primary store', :aggregate_errors do + it 'executes on primary store', :aggregate_failures do expect(secondary_store).not_to receive(:send).and_call_original expect(primary_store).to receive(:send).and_call_original @@ -1097,7 +930,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do subject end - it 'fallback and executes only on the secondary store', :aggregate_errors do + it 'fallback and executes only on the secondary store', :aggregate_failures do expect(primary_store).to receive(:command).and_call_original expect(secondary_store).not_to receive(:command) @@ -1122,7 +955,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do end context 'with feature flag :use_primary_store_as_default_for_test_store is enabled' do - it 'fallback and executes only on the secondary store', :aggregate_errors do + it 'fallback and executes only on the secondary store', :aggregate_failures do expect(primary_store).to receive(:command).and_call_original expect(secondary_store).not_to receive(:command) @@ -1135,7 +968,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do stub_feature_flags(use_primary_store_as_default_for_test_store: false) end - it 'fallback and executes only on the secondary store', :aggregate_errors do + it 'fallback and executes only on the secondary store', :aggregate_failures do expect(secondary_store).to receive(:command).and_call_original expect(primary_store).not_to receive(:command) @@ -1148,7 +981,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do multi_store.pipelined(&:command) end - it 'is executed only 1 time on each instance', :aggregate_errors do + it 'is executed only 1 time on each instance', :aggregate_failures do expect(primary_store).to receive(:pipelined).once.and_call_original expect(secondary_store).to receive(:pipelined).once.and_call_original diff --git a/spec/lib/gitlab/redis/queues_spec.rb b/spec/lib/gitlab/redis/queues_spec.rb index a0f73a654e7..62b30431f6f 100644 --- a/spec/lib/gitlab/redis/queues_spec.rb +++ b/spec/lib/gitlab/redis/queues_spec.rb @@ -4,7 +4,6 @@ require 'spec_helper' RSpec.describe Gitlab::Redis::Queues do let(:instance_specific_config_file) { "config/redis.queues.yml" } - let(:environment_config_file_name) { "GITLAB_REDIS_QUEUES_CONFIG_FILE" } include_examples "redis_shared_examples" @@ -13,14 +12,6 @@ RSpec.describe Gitlab::Redis::Queues do expect(subject).to receive(:fetch_config) { config } end - context 'when the config url is blank' do - let(:config) { nil } - - it 'has a legacy default URL' do - expect(subject.send(:raw_config_hash)).to eq(url: 'redis://localhost:6381' ) - end - end - context 'when the config url is present' do let(:config) { { url: 'redis://localhost:1111' } } diff --git a/spec/lib/gitlab/redis/rate_limiting_spec.rb b/spec/lib/gitlab/redis/rate_limiting_spec.rb index d82228426f0..0bea7f8bcb2 100644 --- a/spec/lib/gitlab/redis/rate_limiting_spec.rb +++ b/spec/lib/gitlab/redis/rate_limiting_spec.rb @@ -6,19 +6,8 @@ RSpec.describe Gitlab::Redis::RateLimiting do include_examples "redis_new_instance_shared_examples", 'rate_limiting', Gitlab::Redis::Cache describe '.cache_store' do - context 'when encountering an error' do - subject { described_class.cache_store.read('x') } - - before do - described_class.with do |redis| - allow(redis).to receive(:get).and_raise(::Redis::CommandError) - end - end - - it 'logs error' do - expect(::Gitlab::ErrorTracking).to receive(:log_exception) - subject - end + it 'uses the CACHE_NAMESPACE namespace' do + expect(described_class.cache_store.options[:namespace]).to eq(Gitlab::Redis::Cache::CACHE_NAMESPACE) end end end diff --git a/spec/lib/gitlab/redis/repository_cache_spec.rb b/spec/lib/gitlab/redis/repository_cache_spec.rb index 2c167a6eb62..bc48ee208c1 100644 --- a/spec/lib/gitlab/redis/repository_cache_spec.rb +++ b/spec/lib/gitlab/redis/repository_cache_spec.rb @@ -5,32 +5,9 @@ require 'spec_helper' RSpec.describe Gitlab::Redis::RepositoryCache, feature_category: :scalability do include_examples "redis_new_instance_shared_examples", 'repository_cache', Gitlab::Redis::Cache - describe '#raw_config_hash' do - it 'has a legacy default URL' do - expect(subject).to receive(:fetch_config).and_return(false) - - expect(subject.send(:raw_config_hash)).to eq(url: 'redis://localhost:6380') - end - end - describe '.cache_store' do it 'has a default ttl of 8 hours' do expect(described_class.cache_store.options[:expires_in]).to eq(8.hours) end - - context 'when encountering an error' do - subject { described_class.cache_store.read('x') } - - before do - described_class.with do |redis| - allow(redis).to receive(:get).and_raise(::Redis::CommandError) - end - end - - it 'logs error' do - expect(::Gitlab::ErrorTracking).to receive(:log_exception) - subject - end - end end end diff --git a/spec/lib/gitlab/redis/shared_state_spec.rb b/spec/lib/gitlab/redis/shared_state_spec.rb index d240abfbf5b..a5247903d50 100644 --- a/spec/lib/gitlab/redis/shared_state_spec.rb +++ b/spec/lib/gitlab/redis/shared_state_spec.rb @@ -4,15 +4,6 @@ require 'spec_helper' RSpec.describe Gitlab::Redis::SharedState do let(:instance_specific_config_file) { "config/redis.shared_state.yml" } - let(:environment_config_file_name) { "GITLAB_REDIS_SHARED_STATE_CONFIG_FILE" } include_examples "redis_shared_examples" - - describe '#raw_config_hash' do - it 'has a legacy default URL' do - expect(subject).to receive(:fetch_config) { false } - - expect(subject.send(:raw_config_hash)).to eq(url: 'redis://localhost:6382' ) - end - end end diff --git a/spec/lib/gitlab/redis/sidekiq_status_spec.rb b/spec/lib/gitlab/redis/sidekiq_status_spec.rb index bbfec13e6c8..45578030ca8 100644 --- a/spec/lib/gitlab/redis/sidekiq_status_spec.rb +++ b/spec/lib/gitlab/redis/sidekiq_status_spec.rb @@ -7,7 +7,6 @@ RSpec.describe Gitlab::Redis::SidekiqStatus do # to move away from `Sidekiq.redis` for sidekiq status data. Thus, we use the # same store configuration as the former. let(:instance_specific_config_file) { "config/redis.shared_state.yml" } - let(:environment_config_file_name) { "GITLAB_REDIS_SHARED_STATE_CONFIG_FILE" } include_examples "redis_shared_examples" @@ -49,14 +48,6 @@ RSpec.describe Gitlab::Redis::SidekiqStatus do :use_primary_store_as_default_for_sidekiq_status end - describe '#raw_config_hash' do - it 'has a legacy default URL' do - expect(subject).to receive(:fetch_config) { false } - - expect(subject.send(:raw_config_hash)).to eq(url: 'redis://localhost:6382') - end - end - describe '#store_name' do it 'returns the name of the SharedState store' do expect(described_class.store_name).to eq('SharedState') diff --git a/spec/lib/gitlab/reference_extractor_spec.rb b/spec/lib/gitlab/reference_extractor_spec.rb index 4d608c07736..62fcb4821fc 100644 --- a/spec/lib/gitlab/reference_extractor_spec.rb +++ b/spec/lib/gitlab/reference_extractor_spec.rb @@ -193,7 +193,7 @@ RSpec.describe Gitlab::ReferenceExtractor do end context 'with an external issue tracker' do - let(:project) { create(:project, :with_jira_integration) } + let_it_be(:project) { create(:project, :with_jira_integration) } let(:issue) { create(:issue, project: project) } context 'when GitLab issues are enabled' do @@ -301,7 +301,7 @@ RSpec.describe Gitlab::ReferenceExtractor do describe 'referables prefixes' do def prefixes - described_class::REFERABLES.each_with_object({}) do |referable, result| + described_class.referrables.each_with_object({}) do |referable, result| class_name = referable.to_s.camelize klass = class_name.constantize if Object.const_defined?(class_name) @@ -314,7 +314,7 @@ RSpec.describe Gitlab::ReferenceExtractor do end it 'returns all supported prefixes' do - expect(prefixes.keys.uniq).to match_array(%w(@ # ~ % ! $ & [vulnerability: *iteration:)) + expect(prefixes.keys.uniq).to include(*%w(@ # ~ % ! $ & [vulnerability:)) end it 'does not allow one prefix for multiple referables if not allowed specificly' do diff --git a/spec/lib/gitlab/regex_spec.rb b/spec/lib/gitlab/regex_spec.rb index bc0f9e22d50..5e58282ff92 100644 --- a/spec/lib/gitlab/regex_spec.rb +++ b/spec/lib/gitlab/regex_spec.rb @@ -79,10 +79,10 @@ RSpec.describe Gitlab::Regex, feature_category: :tooling do it { is_expected - .to eq("cannot start with a non-alphanumeric character except for periods or underscores, " \ - "can contain only alphanumeric characters, forward slashes, periods, and underscores, " \ - "cannot end with a period or forward slash, and has a relative path structure " \ - "with no http protocol chars or leading or trailing forward slashes") + .to eq("must have a relative path structure with no HTTP " \ + "protocol characters, or leading or trailing forward slashes. Path segments must not start or " \ + "end with a special character, and must not contain consecutive special characters." + ) } end @@ -101,22 +101,37 @@ RSpec.describe Gitlab::Regex, feature_category: :tooling do it { is_expected.not_to match('good_for+you') } it { is_expected.not_to match('source/') } it { is_expected.not_to match('.source/full./path') } + it { is_expected.not_to match('.source/.full/.path') } + it { is_expected.not_to match('_source') } + it { is_expected.not_to match('.source') } it { is_expected.to match('source') } - it { is_expected.to match('.source') } - it { is_expected.to match('_source') } it { is_expected.to match('source/full') } it { is_expected.to match('source/full/path') } - it { is_expected.to match('.source/.full/.path') } + it { is_expected.to match('sou_rce/fu-ll/pa.th') } it { is_expected.to match('domain_namespace') } it { is_expected.to match('gitlab-migration-test') } + it { is_expected.to match('1-project-path') } + it { is_expected.to match('e-project-path') } it { is_expected.to match('') } # it is possible to pass an empty string for destination_namespace in bulk_import POST request end + describe '.bulk_import_source_full_path_regex_message' do + subject { described_class.bulk_import_source_full_path_regex_message } + + it { + is_expected + .to eq( + "must have a relative path structure with no HTTP " \ + "protocol characters, or leading or trailing forward slashes. Path segments must not start or " \ + "end with a special character, and must not contain consecutive special characters." + ) + } + end + describe '.bulk_import_source_full_path_regex' do subject { described_class.bulk_import_source_full_path_regex } - it { is_expected.not_to match('?gitlab') } it { is_expected.not_to match("Users's something") } it { is_expected.not_to match('/source') } it { is_expected.not_to match('http:') } @@ -124,20 +139,32 @@ RSpec.describe Gitlab::Regex, feature_category: :tooling do it { is_expected.not_to match('example.com/?stuff=true') } it { is_expected.not_to match('example.com:5000/?stuff=true') } it { is_expected.not_to match('http://gitlab.example/gitlab-org/manage/import/gitlab-migration-test') } - it { is_expected.not_to match('_good_for_me!') } - it { is_expected.not_to match('good_for+you') } it { is_expected.not_to match('source/') } - it { is_expected.not_to match('.source/full./path') } it { is_expected.not_to match('') } + it { is_expected.not_to match('.source/full./path') } + it { is_expected.not_to match('?gitlab') } + it { is_expected.not_to match('_good_for_me!') } + it { is_expected.not_to match('group/@*%_my_other-project-----') } + it { is_expected.not_to match('_foog-for-me!') } + it { is_expected.not_to match('.source/full/path.') } + it { is_expected.to match('good_for+you') } it { is_expected.to match('source') } it { is_expected.to match('.source') } it { is_expected.to match('_source') } it { is_expected.to match('source/full') } it { is_expected.to match('source/full/path') } - it { is_expected.to match('.source/.full/.path') } it { is_expected.to match('domain_namespace') } it { is_expected.to match('gitlab-migration-test') } + it { is_expected.to match('source/full/path-') } + it { is_expected.to match('.source/full/path') } + it { is_expected.to match('.source/.full/.path') } + it { is_expected.to match('source/full/.path') } + it { is_expected.to match('source/full/..path') } + it { is_expected.to match('source/full/---1path') } + it { is_expected.to match('source/full/-___path') } + it { is_expected.to match('source/full/path---') } + it { is_expected.to match('group/__my_other-project-----') } end describe '.group_path_regex' do @@ -710,6 +737,7 @@ RSpec.describe Gitlab::Regex, feature_category: :tooling do it { is_expected.to match('libsample0_1.2.3~alpha2_amd64.deb') } it { is_expected.to match('sample-dev_1.2.3~binary_amd64.deb') } it { is_expected.to match('sample-udeb_1.2.3~alpha2_amd64.udeb') } + it { is_expected.to match('sample-ddeb_1.2.3~alpha2_amd64.ddeb') } it { is_expected.not_to match('sample_1.2.3~alpha2_amd64.buildinfo') } it { is_expected.not_to match('sample_1.2.3~alpha2_amd64.changes') } @@ -1015,6 +1043,34 @@ RSpec.describe Gitlab::Regex, feature_category: :tooling do it { is_expected.not_to match('/api/v4/groups/1234/packages/debian/pool/compon/a/pkg/file.name') } end + describe 'Packages::MAVEN_SNAPSHOT_DYNAMIC_PARTS' do + subject { described_class::Packages::MAVEN_SNAPSHOT_DYNAMIC_PARTS } + + it { is_expected.to match('test-2.11-20230303.163304-1.jar') } + it { is_expected.to match('test-2.11-20230303.163304-1-javadoc.jar') } + it { is_expected.to match('test-2.11-20230303.163304-1-sources.jar') } + it { is_expected.to match('test-2.11-20230303.163304-1-20230303.163304-1.jar') } + it { is_expected.to match('test-2.11-20230303.163304-1-20230303.163304-1-javadoc.jar') } + it { is_expected.to match('test-2.11-20230303.163304-1-20230303.163304-1-sources.jar') } + it { is_expected.to match("#{'a' * 500}-20230303.163304-1-sources.jar") } + it { is_expected.to match("test-2.11-20230303.163304-1-#{'a' * 500}.jar") } + it { is_expected.to match("#{'a' * 500}-20230303.163304-1-#{'a' * 500}.jar") } + + it { is_expected.not_to match('') } + it { is_expected.not_to match(nil) } + it { is_expected.not_to match('test') } + it { is_expected.not_to match('1.2.3') } + it { is_expected.not_to match('1.2.3-javadoc.jar') } + it { is_expected.not_to match('-202303039.163304-1.jar') } + it { is_expected.not_to match('test-2.11-202303039.163304-1.jar') } + it { is_expected.not_to match('test-2.11-20230303.16330-1.jar') } + it { is_expected.not_to match('test-2.11-202303039.163304.jar') } + it { is_expected.not_to match('test-2.11-202303039.163304-.jar') } + it { is_expected.not_to match("#{'a' * 2000}-20230303.163304-1-sources.jar") } + it { is_expected.not_to match("test-2.11-20230303.163304-1-#{'a' * 2000}.jar") } + it { is_expected.not_to match("#{'a' * 2000}-20230303.163304-1-#{'a' * 2000}.jar") } + end + describe '.composer_package_version_regex' do subject { described_class.composer_package_version_regex } @@ -1133,10 +1189,21 @@ RSpec.describe Gitlab::Regex, feature_category: :tooling do MARKDOWN end - it { is_expected.to match(%(<section>\nsomething\n</section>)) } - it { is_expected.not_to match(%(must start in first column <section>\nsomething\n</section>)) } - it { is_expected.not_to match(%(<section>must be multi-line</section>)) } - it { expect(subject.match(markdown)[:html]).to eq expected } + describe 'normal regular expression' do + it { is_expected.to match(%(<section>\nsomething\n</section>)) } + it { is_expected.not_to match(%(must start in first column <section>\nsomething\n</section>)) } + it { is_expected.not_to match(%(<section>must be multi-line</section>)) } + it { expect(subject.match(markdown)[:html]).to eq expected } + end + + describe 'untrusted regular expression' do + subject { Gitlab::UntrustedRegexp.new(described_class::MARKDOWN_HTML_BLOCK_REGEX_UNTRUSTED, multiline: true) } + + it { is_expected.to match(%(<section>\nsomething\n</section>)) } + it { is_expected.not_to match(%(must start in first column <section>\nsomething\n</section>)) } + it { is_expected.not_to match(%(<section>must be multi-line</section>)) } + it { expect(subject.match(markdown)[:html]).to eq expected } + end end context 'HTML comment lines' do diff --git a/spec/lib/gitlab/repository_set_cache_spec.rb b/spec/lib/gitlab/repository_set_cache_spec.rb index c93fd884347..65a50b68c44 100644 --- a/spec/lib/gitlab/repository_set_cache_spec.rb +++ b/spec/lib/gitlab/repository_set_cache_spec.rb @@ -72,48 +72,60 @@ RSpec.describe Gitlab::RepositorySetCache, :clean_gitlab_redis_cache do end describe '#expire' do - subject { cache.expire(*keys) } + shared_examples 'expires varying amount of keys' do + subject { cache.expire(*keys) } - before do - cache.write(:foo, ['value']) - cache.write(:bar, ['value2']) - end + before do + cache.write(:foo, ['value']) + cache.write(:bar, ['value2']) + end - it 'actually wrote the values' do - expect(cache.read(:foo)).to contain_exactly('value') - expect(cache.read(:bar)).to contain_exactly('value2') - end + it 'actually wrote the values' do + expect(cache.read(:foo)).to contain_exactly('value') + expect(cache.read(:bar)).to contain_exactly('value2') + end - context 'single key' do - let(:keys) { %w(foo) } + context 'single key' do + let(:keys) { %w(foo) } - it { is_expected.to eq(1) } + it { is_expected.to eq(1) } - it 'deletes the given key from the cache' do - subject + it 'deletes the given key from the cache' do + subject - expect(cache.read(:foo)).to be_empty + expect(cache.read(:foo)).to be_empty + end end - end - context 'multiple keys' do - let(:keys) { %w(foo bar) } + context 'multiple keys' do + let(:keys) { %w(foo bar) } - it { is_expected.to eq(2) } + it { is_expected.to eq(2) } - it 'deletes the given keys from the cache' do - subject + it 'deletes the given keys from the cache' do + subject - expect(cache.read(:foo)).to be_empty - expect(cache.read(:bar)).to be_empty + expect(cache.read(:foo)).to be_empty + expect(cache.read(:bar)).to be_empty + end + end + + context 'no keys' do + let(:keys) { [] } + + it { is_expected.to eq(0) } end end - context 'no keys' do - let(:keys) { [] } + context 'when feature flag is disabled' do + before do + stub_feature_flags(use_pipeline_over_multikey: false) + end - it { is_expected.to eq(0) } + it_behaves_like 'expires varying amount of keys' end + + it_behaves_like 'expires varying amount of keys' end describe '#exist?' do diff --git a/spec/lib/gitlab/request_context_spec.rb b/spec/lib/gitlab/request_context_spec.rb index b9acfa4a841..44664be7d39 100644 --- a/spec/lib/gitlab/request_context_spec.rb +++ b/spec/lib/gitlab/request_context_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::RequestContext, :request_store do +RSpec.describe Gitlab::RequestContext, :request_store, feature_category: :application_instrumentation do subject { described_class.instance } before do @@ -11,6 +11,44 @@ RSpec.describe Gitlab::RequestContext, :request_store do it { is_expected.to have_attributes(client_ip: nil, start_thread_cpu_time: nil, request_start_time: nil) } + describe '.start_request_context' do + let(:request) { ActionDispatch::Request.new({ 'REMOTE_ADDR' => '1.2.3.4' }) } + let(:start_request_context) { described_class.start_request_context(request: request) } + + before do + allow(Gitlab::Metrics::System).to receive(:real_time).and_return(123) + end + + it 'sets the client IP' do + expect { start_request_context }.to change { subject.client_ip }.from(nil).to('1.2.3.4') + end + + it 'sets the spam params' do + expect { start_request_context }.to change { subject.spam_params }.from(nil).to(::Spam::SpamParams) + end + + it 'sets the request start time' do + expect { start_request_context }.to change { subject.request_start_time }.from(nil).to(123) + end + end + + describe '.start_thread_context' do + let(:start_thread_context) { described_class.start_thread_context } + + before do + allow(Gitlab::Metrics::System).to receive(:thread_cpu_time).and_return(123) + allow(Gitlab::Memory::Instrumentation).to receive(:start_thread_memory_allocations).and_return(456) + end + + it 'sets the thread cpu time' do + expect { start_thread_context }.to change { subject.start_thread_cpu_time }.from(nil).to(123) + end + + it 'sets the thread memory allocations' do + expect { start_thread_context }.to change { subject.thread_memory_allocations }.from(nil).to(456) + end + end + describe '#request_deadline' do let(:request_start_time) { 1575982156.206008 } diff --git a/spec/lib/gitlab/resource_events/assignment_event_recorder_spec.rb b/spec/lib/gitlab/resource_events/assignment_event_recorder_spec.rb new file mode 100644 index 00000000000..b15f95dbd9c --- /dev/null +++ b/spec/lib/gitlab/resource_events/assignment_event_recorder_spec.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::ResourceEvents::AssignmentEventRecorder, feature_category: :value_stream_management do + using RSpec::Parameterized::TableSyntax + + let_it_be(:user1) { create(:user) } + let_it_be(:user2) { create(:user) } + let_it_be(:user3) { create(:user) } + + let_it_be_with_refind(:issue_with_two_assignees) { create(:issue, assignees: [user1, user2]) } + let_it_be_with_refind(:mr_with_no_assignees) { create(:merge_request) } + let_it_be_with_refind(:mr_with_one_assignee) { create(:merge_request, assignee: [user3]) } + + let(:parent_records) do + { + issue_with_two_assignees: issue_with_two_assignees, + mr_with_no_assignees: mr_with_no_assignees, + mr_with_one_assignee: mr_with_one_assignee + } + end + + let(:user_records) do + { + user1: user1, + user2: user2, + user3: user3 + } + end + + where(:parent, :new_assignees, :assignee_history) do + :issue_with_two_assignees | [:user1, :user2, :user3] | [[:user3, :add]] + :issue_with_two_assignees | [:user1, :user3] | [[:user2, :remove], [:user3, :add]] + :issue_with_two_assignees | [:user1] | [[:user2, :remove]] + :issue_with_two_assignees | [] | [[:user1, :remove], [:user2, :remove]] + :mr_with_no_assignees | [:user1] | [[:user1, :add]] + :mr_with_no_assignees | [] | [] + :mr_with_one_assignee | [:user3] | [] + :mr_with_one_assignee | [:user1] | [[:user3, :remove], [:user1, :add]] + end + + with_them do + it 'records the assignment history corrently' do + parent_record = parent_records[parent] + old_assignees = parent_record.assignees.to_a + parent_record.assignees = new_assignees.map { |user_variable_name| user_records[user_variable_name] } + + described_class.new(parent: parent_record, old_assignees: old_assignees).record + + expected_records = assignee_history.map do |user_variable_name, action| + have_attributes({ + user_id: user_records[user_variable_name].id, + action: action.to_s + }) + end + + expect(parent_record.assignment_events).to match_array(expected_records) + end + end + + context 'when batching' do + it 'invokes multiple insert queries' do + stub_const('Gitlab::ResourceEvents::AssignmentEventRecorder::BATCH_SIZE', 1) + + expect(ResourceEvents::MergeRequestAssignmentEvent).to receive(:insert_all).twice + + described_class.new(parent: mr_with_one_assignee, old_assignees: [user1]).record # 1 assignment, 1 unassignment + end + end + + context 'when duplicated old assignees were given' do + it 'deduplicates the records' do + expect do + described_class.new(parent: mr_with_one_assignee, old_assignees: [user3, user2, user2]).record + end.to change { ResourceEvents::MergeRequestAssignmentEvent.count }.by(1) + end + end + + context 'when the record_issue_and_mr_assignee_events FF is off' do + before do + stub_feature_flags(record_issue_and_mr_assignee_events: false) + end + + it 'does nothing' do + expect do + described_class.new(parent: mr_with_one_assignee, old_assignees: [user2, user3]).record + end.not_to change { mr_with_one_assignee.assignment_events.count } + end + end +end diff --git a/spec/lib/gitlab/runtime_spec.rb b/spec/lib/gitlab/runtime_spec.rb index 181a911c667..fa0fad65520 100644 --- a/spec/lib/gitlab/runtime_spec.rb +++ b/spec/lib/gitlab/runtime_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Runtime do +RSpec.describe Gitlab::Runtime, feature_category: :application_performance do shared_examples "valid runtime" do |runtime, max_threads| it "identifies itself" do expect(subject.identify).to eq(runtime) @@ -39,9 +39,21 @@ RSpec.describe Gitlab::Runtime do end end + context 'with Puma' do + before do + stub_const('::Puma::Server', double) + end + + describe '.puma?' do + it 'returns true' do + expect(subject.puma?).to be true + end + end + end + context "on multiple matches" do before do - stub_const('::Puma', double) + stub_const('::Puma::Server', double) stub_const('::Rails::Console', double) end @@ -64,6 +76,7 @@ RSpec.describe Gitlab::Runtime do before do stub_const('::Puma', puma_type) + allow(described_class).to receive(:puma?).and_return(true) end it_behaves_like "valid runtime", :puma, 1 + Gitlab::ActionCable::Config.worker_pool_size @@ -75,6 +88,7 @@ RSpec.describe Gitlab::Runtime do before do stub_const('::Puma', puma_type) + allow(described_class).to receive(:puma?).and_return(true) allow(puma_type).to receive_message_chain(:cli_config, :options).and_return(max_threads: 2, workers: max_workers) end diff --git a/spec/lib/gitlab/safe_device_detector_spec.rb b/spec/lib/gitlab/safe_device_detector_spec.rb index c37dc1e1c7e..56ba084c435 100644 --- a/spec/lib/gitlab/safe_device_detector_spec.rb +++ b/spec/lib/gitlab/safe_device_detector_spec.rb @@ -4,7 +4,7 @@ require 'fast_spec_helper' require 'device_detector' require_relative '../../../lib/gitlab/safe_device_detector' -RSpec.describe Gitlab::SafeDeviceDetector, feature_category: :authentication_and_authorization do +RSpec.describe Gitlab::SafeDeviceDetector, feature_category: :system_access do it 'retains the behavior for normal user agents' do chrome_user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 \ (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36" diff --git a/spec/lib/gitlab/sanitizers/exception_message_spec.rb b/spec/lib/gitlab/sanitizers/exception_message_spec.rb index 8b54b353235..c2c4a5de32d 100644 --- a/spec/lib/gitlab/sanitizers/exception_message_spec.rb +++ b/spec/lib/gitlab/sanitizers/exception_message_spec.rb @@ -1,9 +1,10 @@ # frozen_string_literal: true require 'fast_spec_helper' +require 'addressable' require 'rspec-parameterized' -RSpec.describe Gitlab::Sanitizers::ExceptionMessage do +RSpec.describe Gitlab::Sanitizers::ExceptionMessage, feature_category: :compliance_management do describe '.clean' do let(:exception_name) { exception.class.name } let(:exception_message) { exception.message } diff --git a/spec/lib/gitlab/seeders/ci/runner/runner_fleet_seeder_spec.rb b/spec/lib/gitlab/seeders/ci/runner/runner_fleet_seeder_spec.rb index fe52b586d49..4597cc6b315 100644 --- a/spec/lib/gitlab/seeders/ci/runner/runner_fleet_seeder_spec.rb +++ b/spec/lib/gitlab/seeders/ci/runner/runner_fleet_seeder_spec.rb @@ -67,5 +67,29 @@ RSpec.describe ::Gitlab::Seeders::Ci::Runner::RunnerFleetSeeder, feature_categor expect(::Ci::Build.where(runner_id: project[:runner_ids])).to be_empty end end + + context 'when number of group runners exceeds plan limit' do + before do + create(:plan_limits, :default_plan, ci_registered_group_runners: 1) + end + + it { is_expected.to be_nil } + + it 'does not change runner count' do + expect { seed }.not_to change { Ci::Runner.count } + end + end + + context 'when number of project runners exceeds plan limit' do + before do + create(:plan_limits, :default_plan, ci_registered_project_runners: 1) + end + + it { is_expected.to be_nil } + + it 'does not change runner count' do + expect { seed }.not_to change { Ci::Runner.count } + end + end end end diff --git a/spec/lib/gitlab/seeders/ci/variables_group_seeder_spec.rb b/spec/lib/gitlab/seeders/ci/variables_group_seeder_spec.rb new file mode 100644 index 00000000000..52898cb17a5 --- /dev/null +++ b/spec/lib/gitlab/seeders/ci/variables_group_seeder_spec.rb @@ -0,0 +1,92 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe ::Gitlab::Seeders::Ci::VariablesGroupSeeder, feature_category: :secrets_management do + let_it_be(:group) { create(:group) } + + let(:seeder) { described_class.new(name: group.name) } + + let(:custom_seeder) do + described_class.new( + name: group.name, + seed_count: 2, + environment_scope: 'staging', + prefix: 'STAGING_' + ) + end + + let(:unique_env_seeder) do + described_class.new( + name: group.name, + seed_count: 2, + environment_scope: 'unique' + ) + end + + let(:invalid_group_name_seeder) do + described_class.new( + name: 'nonexistent_group', + seed_count: 1 + ) + end + + describe '#seed' do + it 'creates group-level CI variables with default values' do + expect { seeder.seed }.to change { + group.variables.count + }.by(Gitlab::Seeders::Ci::VariablesGroupSeeder::DEFAULT_SEED_COUNT) + + ci_variable = group.reload.variables.last + + expect(ci_variable.key.include?('GROUP_VAR_')).to eq true + expect(ci_variable.environment_scope).to eq '*' + end + + it 'creates group-level CI variables with custom arguments' do + expect { custom_seeder.seed }.to change { + group.variables.count + }.by(2) + + ci_variable = group.reload.variables.last + + expect(ci_variable.key.include?('STAGING_')).to eq true + expect(ci_variable.environment_scope).to eq 'staging' + end + + it 'creates group-level CI variables with unique environment scopes' do + unique_env_seeder.seed + + ci_variable_first_env = group.reload.variables.first.environment_scope + ci_variable_last_env = group.reload.variables.last.environment_scope + + expect(ci_variable_first_env).not_to eq ci_variable_last_env + end + + it 'skips seeding when group name is invalid' do + expect { invalid_group_name_seeder.seed }.to change { + group.variables.count + }.by(0) + end + + it 'skips CI variable creation if CI variable already exists' do + group.variables.create!( + environment_scope: '*', + key: "GROUP_VAR_#{group.variables.maximum(:id).to_i}", + value: SecureRandom.hex(32) + ) + + # first id is assigned randomly, so we're creating a new variable + # based on that id that is sure to be skipped during seed + group.variables.create!( + environment_scope: '*', + key: "GROUP_VAR_#{group.variables.maximum(:id).to_i + 2}", + value: SecureRandom.hex(32) + ) + + expect { seeder.seed }.to change { + group.variables.count + }.by(Gitlab::Seeders::Ci::VariablesGroupSeeder::DEFAULT_SEED_COUNT - 1) + end + end +end diff --git a/spec/lib/gitlab/seeders/ci/variables_instance_seeder_spec.rb b/spec/lib/gitlab/seeders/ci/variables_instance_seeder_spec.rb new file mode 100644 index 00000000000..5b6d2471edd --- /dev/null +++ b/spec/lib/gitlab/seeders/ci/variables_instance_seeder_spec.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe ::Gitlab::Seeders::Ci::VariablesInstanceSeeder, feature_category: :secrets_management do + let(:seeder) { described_class.new } + + let(:custom_seeder) do + described_class.new( + seed_count: 2, + prefix: 'STAGING_' + ) + end + + describe '#seed' do + it 'creates instance-level CI variables with default values' do + expect { seeder.seed }.to change { + Ci::InstanceVariable.all.count + }.by(Gitlab::Seeders::Ci::VariablesInstanceSeeder::DEFAULT_SEED_COUNT) + + ci_variable = Ci::InstanceVariable.last + + expect(ci_variable.key.include?('INSTANCE_VAR_')).to eq true + end + + it 'creates instance-level CI variables with custom arguments' do + expect { custom_seeder.seed }.to change { + Ci::InstanceVariable.all.count + }.by(2) + + ci_variable = Ci::InstanceVariable.last + + expect(ci_variable.key.include?('STAGING_')).to eq true + end + + it 'skips CI variable creation if CI variable already exists' do + ::Ci::InstanceVariable.new( + key: "INSTANCE_VAR_#{::Ci::InstanceVariable.maximum(:id).to_i}", + value: SecureRandom.hex(32) + ).save! + + # first id is assigned randomly, so we're creating a new variable + # based on that id that is sure to be skipped during seed + ::Ci::InstanceVariable.new( + key: "INSTANCE_VAR_#{::Ci::InstanceVariable.maximum(:id).to_i + 2}", + value: SecureRandom.hex(32) + ).save! + + expect { seeder.seed }.to change { + Ci::InstanceVariable.all.count + }.by(Gitlab::Seeders::Ci::VariablesInstanceSeeder::DEFAULT_SEED_COUNT - 1) + end + end +end diff --git a/spec/lib/gitlab/seeders/ci/variables_project_seeder_spec.rb b/spec/lib/gitlab/seeders/ci/variables_project_seeder_spec.rb new file mode 100644 index 00000000000..45b6a0a51fd --- /dev/null +++ b/spec/lib/gitlab/seeders/ci/variables_project_seeder_spec.rb @@ -0,0 +1,92 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe ::Gitlab::Seeders::Ci::VariablesProjectSeeder, feature_category: :secrets_management do + let_it_be(:project) { create(:project) } + + let(:seeder) { described_class.new(project_path: project.full_path) } + + let(:custom_seeder) do + described_class.new( + project_path: project.full_path, + seed_count: 2, + environment_scope: 'staging', + prefix: 'STAGING_' + ) + end + + let(:unique_env_seeder) do + described_class.new( + project_path: project.full_path, + seed_count: 2, + environment_scope: 'unique' + ) + end + + let(:invalid_project_path_seeder) do + described_class.new( + project_path: 'invalid_path', + seed_count: 1 + ) + end + + describe '#seed' do + it 'creates project-level CI variables with default values' do + expect { seeder.seed }.to change { + project.variables.count + }.by(Gitlab::Seeders::Ci::VariablesProjectSeeder::DEFAULT_SEED_COUNT) + + ci_variable = project.reload.variables.last + + expect(ci_variable.key.include?('VAR_')).to eq true + expect(ci_variable.environment_scope).to eq '*' + end + + it 'creates project-level CI variables with custom arguments' do + expect { custom_seeder.seed }.to change { + project.variables.count + }.by(2) + + ci_variable = project.reload.variables.last + + expect(ci_variable.key.include?('STAGING_')).to eq true + expect(ci_variable.environment_scope).to eq 'staging' + end + + it 'creates project-level CI variables with unique environment scopes' do + unique_env_seeder.seed + + ci_variable_first_env = project.reload.variables.first.environment_scope + ci_variable_last_env = project.reload.variables.last.environment_scope + + expect(ci_variable_first_env).not_to eq ci_variable_last_env + end + + it 'skips seeding when project path is invalid' do + expect { invalid_project_path_seeder.seed }.to change { + project.variables.count + }.by(0) + end + + it 'skips CI variable creation if CI variable already exists' do + project.variables.create!( + environment_scope: '*', + key: "VAR_#{project.variables.maximum(:id).to_i}", + value: SecureRandom.hex(32) + ) + + # first id is assigned randomly, so we're creating a new variable + # based on that id that is sure to be skipped during seed + project.variables.create!( + environment_scope: '*', + key: "VAR_#{project.variables.maximum(:id).to_i + 2}", + value: SecureRandom.hex(32) + ) + + expect { seeder.seed }.to change { + project.variables.count + }.by(Gitlab::Seeders::Ci::VariablesProjectSeeder::DEFAULT_SEED_COUNT - 1) + end + end +end diff --git a/spec/lib/gitlab/seeders/project_environment_seeder_spec.rb b/spec/lib/gitlab/seeders/project_environment_seeder_spec.rb new file mode 100644 index 00000000000..8401d189373 --- /dev/null +++ b/spec/lib/gitlab/seeders/project_environment_seeder_spec.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe ::Gitlab::Seeders::ProjectEnvironmentSeeder, feature_category: :secrets_management do + let_it_be(:project) { create(:project) } + + let(:seeder) { described_class.new(project_path: project.full_path) } + let(:custom_seeder) do + described_class.new(project_path: project.full_path, seed_count: 2, prefix: 'staging_') + end + + let(:invalid_project_path_seeder) do + described_class.new(project_path: 'invalid_path', seed_count: 1) + end + + describe '#seed' do + it 'creates environments for the project' do + expect { seeder.seed }.to change { + project.environments.count + }.by(Gitlab::Seeders::ProjectEnvironmentSeeder::DEFAULT_SEED_COUNT) + end + + it 'creates environments with custom arguments' do + expect { custom_seeder.seed }.to change { + project.environments.count + }.by(2) + + env = project.environments.last + + expect(env.name.include?('staging_')).to eq true + end + + it 'skips seeding when project path is invalid' do + expect { invalid_project_path_seeder.seed }.to change { + project.environments.count + }.by(0) + end + + it 'skips environment creation if environment already exists' do + project.environments.create!(name: "ENV_#{project.environments.maximum(:id).to_i}") + + # first id is assigned randomly, so we're creating a new variable + # based on that id that is sure to be skipped during seed + project.environments.create!(name: "ENV_#{project.environments.maximum(:id).to_i + 2}") + + expect { seeder.seed }.to change { + project.environments.count + }.by(Gitlab::Seeders::ProjectEnvironmentSeeder::DEFAULT_SEED_COUNT - 1) + end + end +end diff --git a/spec/lib/gitlab/serverless/service_spec.rb b/spec/lib/gitlab/serverless/service_spec.rb deleted file mode 100644 index 3400be5b48e..00000000000 --- a/spec/lib/gitlab/serverless/service_spec.rb +++ /dev/null @@ -1,136 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Serverless::Service do - let(:cluster) { create(:cluster) } - let(:environment) { create(:environment) } - let(:attributes) do - { - 'apiVersion' => 'serving.knative.dev/v1alpha1', - 'kind' => 'Service', - 'metadata' => { - 'creationTimestamp' => '2019-10-22T21:19:13Z', - 'name' => 'kubetest', - 'namespace' => 'project1-1-environment1' - }, - 'spec' => { - 'runLatest' => { - 'configuration' => { - 'build' => { - 'template' => { - 'name' => 'some-image' - } - } - } - } - }, - 'environment_scope' => '*', - 'cluster' => cluster, - 'environment' => environment, - 'podcount' => 0 - } - end - - it 'exposes methods extracting data from the attributes hash' do - service = Gitlab::Serverless::Service.new(attributes) - - expect(service.name).to eq('kubetest') - expect(service.namespace).to eq('project1-1-environment1') - expect(service.environment_scope).to eq('*') - expect(service.podcount).to eq(0) - expect(service.created_at).to eq(DateTime.parse('2019-10-22T21:19:13Z')) - expect(service.image).to eq('some-image') - expect(service.cluster).to eq(cluster) - expect(service.environment).to eq(environment) - end - - it 'returns nil for missing attributes' do - service = Gitlab::Serverless::Service.new({}) - - [:name, :namespace, :environment_scope, :cluster, :podcount, :created_at, :image, :description, :url, :environment].each do |method| - expect(service.send(method)).to be_nil - end - end - - describe '#description' do - it 'extracts the description in knative 7 format if available' do - attributes = { - 'spec' => { - 'template' => { - 'metadata' => { - 'annotations' => { - 'Description' => 'some description' - } - } - } - } - } - service = Gitlab::Serverless::Service.new(attributes) - - expect(service.description).to eq('some description') - end - - it 'extracts the description in knative 5/6 format if 7 is not available' do - attributes = { - 'spec' => { - 'runLatest' => { - 'configuration' => { - 'revisionTemplate' => { - 'metadata' => { - 'annotations' => { - 'Description' => 'some description' - } - } - } - } - } - } - } - service = Gitlab::Serverless::Service.new(attributes) - - expect(service.description).to eq('some description') - end - end - - describe '#url' do - let(:serverless_domain) { instance_double(::Serverless::Domain, uri: URI('https://proxy.example.com')) } - - it 'returns proxy URL if cluster has serverless domain' do - # cluster = create(:cluster) - knative = create(:clusters_applications_knative, :installed, cluster: cluster) - create(:serverless_domain_cluster, clusters_applications_knative_id: knative.id) - service = Gitlab::Serverless::Service.new(attributes.merge('cluster' => cluster)) - - expect(::Serverless::Domain).to receive(:new).with( - function_name: service.name, - serverless_domain_cluster: service.cluster.serverless_domain, - environment: service.environment - ).and_return(serverless_domain) - - expect(service.url).to eq('https://proxy.example.com') - end - - it 'returns the URL from the knative 6/7 format' do - attributes = { - 'status' => { - 'url' => 'https://example.com' - } - } - service = Gitlab::Serverless::Service.new(attributes) - - expect(service.url).to eq('https://example.com') - end - - it 'returns the URL from the knative 5 format' do - attributes = { - 'status' => { - 'domain' => 'example.com' - } - } - service = Gitlab::Serverless::Service.new(attributes) - - expect(service.url).to eq('http://example.com') - end - end -end diff --git a/spec/lib/gitlab/service_desk_spec.rb b/spec/lib/gitlab/service_desk_spec.rb index f554840ec78..d6725f37d39 100644 --- a/spec/lib/gitlab/service_desk_spec.rb +++ b/spec/lib/gitlab/service_desk_spec.rb @@ -4,8 +4,8 @@ require 'spec_helper' RSpec.describe Gitlab::ServiceDesk do before do - allow(Gitlab::IncomingEmail).to receive(:enabled?).and_return(true) - allow(Gitlab::IncomingEmail).to receive(:supports_wildcard?).and_return(true) + allow(Gitlab::Email::IncomingEmail).to receive(:enabled?).and_return(true) + allow(Gitlab::Email::IncomingEmail).to receive(:supports_wildcard?).and_return(true) end describe 'enabled?' do @@ -39,7 +39,7 @@ RSpec.describe Gitlab::ServiceDesk do context 'when incoming emails are disabled' do before do - allow(Gitlab::IncomingEmail).to receive(:enabled?).and_return(false) + allow(Gitlab::Email::IncomingEmail).to receive(:enabled?).and_return(false) end it { is_expected.to be_falsy } @@ -47,7 +47,7 @@ RSpec.describe Gitlab::ServiceDesk do context 'when email key is not supported' do before do - allow(Gitlab::IncomingEmail).to receive(:supports_wildcard?).and_return(false) + allow(Gitlab::Email::IncomingEmail).to receive(:supports_wildcard?).and_return(false) end it { is_expected.to be_falsy } diff --git a/spec/lib/gitlab/sidekiq_config/worker_router_spec.rb b/spec/lib/gitlab/sidekiq_config/worker_router_spec.rb index 4a8dbe69d36..ea9d77bcfa4 100644 --- a/spec/lib/gitlab/sidekiq_config/worker_router_spec.rb +++ b/spec/lib/gitlab/sidekiq_config/worker_router_spec.rb @@ -21,7 +21,6 @@ RSpec.describe Gitlab::SidekiqConfig::WorkerRouter do create_worker('PostReceive', :git) | 'git:post_receive' create_worker('PipelineHooksWorker', :pipeline_hooks) | 'pipeline_hooks:pipeline_hooks' create_worker('Gitlab::JiraImport::AdvanceStageWorker') | 'jira_import_advance_stage' - create_worker('Gitlab::PhabricatorImport::ImportTasksWorker', :importer) | 'importer:phabricator_import_import_tasks' end with_them do @@ -127,6 +126,7 @@ RSpec.describe Gitlab::SidekiqConfig::WorkerRouter do describe '.global' do before do described_class.remove_instance_variable(:@global_worker_router) if described_class.instance_variable_defined?(:@global_worker_router) + stub_config(sidekiq: { routing_rules: routing_rules }) end after do @@ -137,10 +137,6 @@ RSpec.describe Gitlab::SidekiqConfig::WorkerRouter do include_context 'router examples setup' with_them do - before do - stub_config(sidekiq: { routing_rules: routing_rules }) - end - it 'routes the worker to the correct queue' do expect(described_class.global.route(worker)).to eql(expected_queue) end @@ -158,10 +154,6 @@ RSpec.describe Gitlab::SidekiqConfig::WorkerRouter do end end - before do - stub_config(sidekiq: { routing_rules: routing_rules }) - end - context 'invalid routing rules format' do let(:routing_rules) { ['feature_category=a'] } @@ -184,6 +176,26 @@ RSpec.describe Gitlab::SidekiqConfig::WorkerRouter do end end end + + context 'when routing rules is missing `*` as the last rule' do + let(:routing_rules) { [['resource_boundary=cpu', 'cpu']] } + + it 'logs a warning' do + expect(Gitlab::AppLogger).to receive(:warn).with(a_string_matching('sidekiq.routing_rules config is missing')) + + described_class.global + end + end + + context 'when routing rules has a `*` rule as the last rule' do + let(:routing_rules) { [['resource_boundary=cpu', 'cpu'], ['*', 'default']] } + + it 'does not log any warning' do + expect(Gitlab::AppLogger).not_to receive(:warn) + + described_class.global + end + end end describe '#route' do diff --git a/spec/lib/gitlab/sidekiq_config_spec.rb b/spec/lib/gitlab/sidekiq_config_spec.rb index 5f72a3feba7..00b1666106f 100644 --- a/spec/lib/gitlab/sidekiq_config_spec.rb +++ b/spec/lib/gitlab/sidekiq_config_spec.rb @@ -17,6 +17,27 @@ RSpec.describe Gitlab::SidekiqConfig do end end + describe '.cron_jobs' do + it 'renames job_class to class and removes incomplete jobs' do + expect(Gitlab) + .to receive(:config) + .twice + .and_return(GitlabSettings::Options.build( + load_dynamic_cron_schedules!: true, + cron_jobs: { + job: { cron: '0 * * * *', job_class: 'SomeWorker' }, + incomplete_job: { cron: '0 * * * *' } + })) + + expect(Gitlab::AppLogger) + .to receive(:error) + .with("Invalid cron_jobs config key: 'incomplete_job'. Check your gitlab config file.") + + expect(described_class.cron_jobs) + .to eq('job' => { 'class' => 'SomeWorker', 'cron' => '0 * * * *' }) + end + end + describe '.worker_queues' do it 'includes all queues' do queues = described_class.worker_queues diff --git a/spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb b/spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb deleted file mode 100644 index 6f46a5aea3b..00000000000 --- a/spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb +++ /dev/null @@ -1,562 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::SidekiqDaemon::MemoryKiller do - let(:memory_killer) { described_class.new } - let(:sidekiq_daemon_monitor) { instance_double(Gitlab::SidekiqDaemon::Monitor) } - let(:running_jobs) { {} } - let(:pid) { 12345 } - let(:worker) do - Class.new do - def self.name - 'DummyWorker' - end - end - end - - before do - stub_const('DummyWorker', worker) - allow(Sidekiq.logger).to receive(:info) - allow(Sidekiq.logger).to receive(:warn) - allow(Gitlab::SidekiqDaemon::Monitor).to receive(:instance).and_return(sidekiq_daemon_monitor) - allow(sidekiq_daemon_monitor).to receive(:jobs).and_return(running_jobs) - allow(memory_killer).to receive(:pid).and_return(pid) - - # make sleep no-op - allow(memory_killer).to receive(:sleep) {} - end - - describe '#run_thread' do - subject { memory_killer.send(:run_thread) } - - before do - # let enabled? return 3 times: true, true, false - allow(memory_killer).to receive(:enabled?).and_return(true, true, false) - end - - context 'when structured logging is used' do - it 'logs start message once' do - expect(Sidekiq.logger).to receive(:info).once - .with( - class: described_class.to_s, - action: 'start', - pid: pid, - message: 'Starting Gitlab::SidekiqDaemon::MemoryKiller Daemon') - - subject - end - - it 'logs StandardError message twice' do - expect(Sidekiq.logger).to receive(:warn).twice - .with( - class: described_class.to_s, - pid: pid, - message: "Exception from run_thread: My Exception") - - expect(memory_killer).to receive(:rss_within_range?) - .twice - .and_raise(StandardError, 'My Exception') - - expect { subject }.not_to raise_exception - end - - it 'logs exception message once and raise exception and log stop message' do - expect(Sidekiq.logger).to receive(:warn).once - .with( - class: described_class.to_s, - pid: pid, - message: "Exception from run_thread: My Exception") - - expect(memory_killer).to receive(:rss_within_range?) - .once - .and_raise(Exception, 'My Exception') - - expect(memory_killer).to receive(:sleep).with(Gitlab::SidekiqDaemon::MemoryKiller::CHECK_INTERVAL_SECONDS) - expect(Sidekiq.logger).to receive(:warn).once - .with( - class: described_class.to_s, - action: 'stop', - pid: pid, - message: 'Stopping Gitlab::SidekiqDaemon::MemoryKiller Daemon') - - expect { subject }.to raise_exception(Exception, 'My Exception') - end - - it 'logs stop message once' do - expect(Sidekiq.logger).to receive(:warn).once - .with( - class: described_class.to_s, - action: 'stop', - pid: pid, - message: 'Stopping Gitlab::SidekiqDaemon::MemoryKiller Daemon') - - subject - end - end - - it 'not invoke restart_sidekiq when rss in range' do - expect(memory_killer).to receive(:rss_within_range?) - .twice - .and_return(true) - - expect(memory_killer).not_to receive(:restart_sidekiq) - - subject - end - - it 'invoke restart_sidekiq when rss not in range' do - expect(memory_killer).to receive(:rss_within_range?) - .at_least(:once) - .and_return(false) - - expect(memory_killer).to receive(:restart_sidekiq) - .at_least(:once) - - subject - end - end - - describe '#stop_working' do - subject { memory_killer.send(:stop_working) } - - it 'changes enable? to false' do - expect { subject }.to change { memory_killer.send(:enabled?) } - .from(true).to(false) - end - end - - describe '#rss_within_range?' do - let(:shutdown_timeout_seconds) { 7 } - let(:check_interval_seconds) { 2 } - let(:grace_balloon_seconds) { 5 } - - subject { memory_killer.send(:rss_within_range?) } - - before do - stub_const("#{described_class}::SHUTDOWN_TIMEOUT_SECONDS", shutdown_timeout_seconds) - stub_const("#{described_class}::CHECK_INTERVAL_SECONDS", check_interval_seconds) - stub_const("#{described_class}::GRACE_BALLOON_SECONDS", grace_balloon_seconds) - allow(Process).to receive(:getpgrp).and_return(pid) - allow(Sidekiq).to receive(:[]).with(:timeout).and_return(9) - end - - it 'return true when everything is within limit', :aggregate_failures do - expect(memory_killer).to receive(:get_rss_kb).and_return(100) - expect(memory_killer).to receive(:get_soft_limit_rss_kb).and_return(200) - expect(memory_killer).to receive(:get_hard_limit_rss_kb).and_return(300) - expect(memory_killer).to receive(:get_memory_total_kb).and_return(3072) - - expect(memory_killer).to receive(:refresh_state) - .with(:running) - .and_call_original - - expect(Gitlab::Metrics::System).to receive(:monotonic_time).and_call_original - expect(memory_killer).not_to receive(:log_rss_out_of_range) - - expect(subject).to be true - end - - it 'return false when rss exceeds hard_limit_rss', :aggregate_failures do - expect(memory_killer).to receive(:get_rss_kb).at_least(:once).and_return(400) - expect(memory_killer).to receive(:get_soft_limit_rss_kb).at_least(:once).and_return(200) - expect(memory_killer).to receive(:get_hard_limit_rss_kb).at_least(:once).and_return(300) - expect(memory_killer).to receive(:get_memory_total_kb).at_least(:once).and_return(3072) - - expect(memory_killer).to receive(:refresh_state) - .with(:running) - .and_call_original - - expect(memory_killer).to receive(:refresh_state) - .with(:above_soft_limit) - .and_call_original - - expect(Gitlab::Metrics::System).to receive(:monotonic_time).and_call_original - - expect(memory_killer).to receive(:out_of_range_description).with(400, 300, 200, true) - - expect(subject).to be false - end - - it 'return false when rss exceed hard_limit_rss after a while', :aggregate_failures do - expect(memory_killer).to receive(:get_rss_kb).and_return(250, 400, 400) - expect(memory_killer).to receive(:get_soft_limit_rss_kb).at_least(:once).and_return(200) - expect(memory_killer).to receive(:get_hard_limit_rss_kb).at_least(:once).and_return(300) - expect(memory_killer).to receive(:get_memory_total_kb).at_least(:once).and_return(3072) - - expect(memory_killer).to receive(:refresh_state) - .with(:running) - .and_call_original - - expect(memory_killer).to receive(:refresh_state) - .at_least(:once) - .with(:above_soft_limit) - .and_call_original - - expect(Gitlab::Metrics::System).to receive(:monotonic_time).twice.and_call_original - expect(memory_killer).to receive(:sleep).with(check_interval_seconds) - expect(memory_killer).to receive(:out_of_range_description).with(400, 300, 200, false) - expect(memory_killer).to receive(:out_of_range_description).with(400, 300, 200, true) - - expect(subject).to be false - end - - it 'return true when rss below soft_limit_rss after a while within GRACE_BALLOON_SECONDS', :aggregate_failures do - expect(memory_killer).to receive(:get_rss_kb).and_return(250, 100) - expect(memory_killer).to receive(:get_soft_limit_rss_kb).and_return(200, 200) - expect(memory_killer).to receive(:get_hard_limit_rss_kb).and_return(300, 300) - expect(memory_killer).to receive(:get_memory_total_kb).and_return(3072, 3072) - - expect(memory_killer).to receive(:refresh_state) - .with(:running) - .and_call_original - - expect(memory_killer).to receive(:refresh_state) - .with(:above_soft_limit) - .and_call_original - - expect(Gitlab::Metrics::System).to receive(:monotonic_time).twice.and_call_original - expect(memory_killer).to receive(:sleep).with(check_interval_seconds) - - expect(memory_killer).to receive(:out_of_range_description).with(100, 300, 200, false) - - expect(subject).to be true - end - - context 'when exceeds GRACE_BALLOON_SECONDS' do - let(:grace_balloon_seconds) { 0 } - - it 'return false when rss exceed soft_limit_rss', :aggregate_failures do - allow(memory_killer).to receive(:get_rss_kb).and_return(250) - allow(memory_killer).to receive(:get_soft_limit_rss_kb).and_return(200) - allow(memory_killer).to receive(:get_hard_limit_rss_kb).and_return(300) - allow(memory_killer).to receive(:get_memory_total_kb).and_return(3072) - - expect(memory_killer).to receive(:refresh_state) - .with(:running) - .and_call_original - - expect(memory_killer).to receive(:refresh_state) - .with(:above_soft_limit) - .and_call_original - - expect(memory_killer).to receive(:out_of_range_description).with(250, 300, 200, true) - - expect(subject).to be false - end - end - end - - describe '#restart_sidekiq' do - let(:shutdown_timeout_seconds) { 7 } - - subject { memory_killer.send(:restart_sidekiq) } - - context 'when sidekiq_memory_killer_read_only_mode is enabled' do - before do - stub_feature_flags(sidekiq_memory_killer_read_only_mode: true) - end - - it 'does not send signal' do - expect(memory_killer).not_to receive(:refresh_state) - expect(memory_killer).not_to receive(:signal_and_wait) - - subject - end - end - - context 'when sidekiq_memory_killer_read_only_mode is disabled' do - before do - stub_const("#{described_class}::SHUTDOWN_TIMEOUT_SECONDS", shutdown_timeout_seconds) - stub_feature_flags(sidekiq_memory_killer_read_only_mode: false) - allow(Sidekiq).to receive(:[]).with(:timeout).and_return(9) - allow(memory_killer).to receive(:get_rss_kb).and_return(100) - allow(memory_killer).to receive(:get_soft_limit_rss_kb).and_return(200) - allow(memory_killer).to receive(:get_hard_limit_rss_kb).and_return(300) - allow(memory_killer).to receive(:get_memory_total_kb).and_return(3072) - end - - it 'send signal' do - expect(memory_killer).to receive(:refresh_state) - .with(:stop_fetching_new_jobs) - .ordered - .and_call_original - expect(memory_killer).to receive(:signal_and_wait) - .with(shutdown_timeout_seconds, 'SIGTSTP', 'stop fetching new jobs') - .ordered - - expect(memory_killer).to receive(:refresh_state) - .with(:shutting_down) - .ordered - .and_call_original - expect(memory_killer).to receive(:signal_and_wait) - .with(11, 'SIGTERM', 'gracefully shut down') - .ordered - - expect(memory_killer).to receive(:refresh_state) - .with(:killing_sidekiq) - .ordered - .and_call_original - expect(memory_killer).to receive(:signal_pgroup) - .with('SIGKILL', 'die') - .ordered - - subject - end - end - end - - describe '#signal_and_wait' do - let(:time) { 0.1 } - let(:signal) { 'my-signal' } - let(:explanation) { 'my-explanation' } - let(:check_interval_seconds) { 0.1 } - - subject { memory_killer.send(:signal_and_wait, time, signal, explanation) } - - before do - stub_const("#{described_class}::CHECK_INTERVAL_SECONDS", check_interval_seconds) - end - - it 'send signal and wait till deadline' do - expect(Process).to receive(:kill) - .with(signal, pid) - .ordered - - expect(Gitlab::Metrics::System).to receive(:monotonic_time) - .and_call_original - .at_least(3) - - expect(memory_killer).to receive(:enabled?).and_return(true).at_least(:twice) - expect(memory_killer).to receive(:sleep).at_least(:once).and_call_original - - subject - end - end - - describe '#signal_pgroup' do - let(:signal) { 'my-signal' } - let(:explanation) { 'my-explanation' } - - subject { memory_killer.send(:signal_pgroup, signal, explanation) } - - it 'send signal to this process if it is not group leader' do - expect(Process).to receive(:getpgrp).and_return(pid + 1) - - expect(Sidekiq.logger).to receive(:warn).once - .with( - class: described_class.to_s, - signal: signal, - pid: pid, - message: "sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})") - expect(Process).to receive(:kill).with(signal, pid).ordered - - subject - end - - it 'send signal to whole process group as group leader' do - expect(Process).to receive(:getpgrp).and_return(pid) - - expect(Sidekiq.logger).to receive(:warn).once - .with( - class: described_class.to_s, - signal: signal, - pid: pid, - message: "sending Sidekiq worker PGRP-#{pid} #{signal} (#{explanation})") - expect(Process).to receive(:kill).with(signal, 0).ordered - - subject - end - end - - describe '#log_rss_out_of_range' do - let(:current_rss) { 100 } - let(:soft_limit_rss) { 200 } - let(:hard_limit_rss) { 300 } - let(:memory_total) { 3072 } - let(:jid) { 1 } - let(:reason) { 'rss out of range reason description' } - let(:queue) { 'default' } - - let(:metrics) { memory_killer.instance_variable_get(:@metrics) } - let(:running_jobs) { { jid => { worker_class: DummyWorker } } } - - before do - allow(memory_killer).to receive(:get_rss_kb).and_return(*current_rss) - allow(memory_killer).to receive(:get_soft_limit_rss_kb).and_return(soft_limit_rss) - allow(memory_killer).to receive(:get_hard_limit_rss_kb).and_return(hard_limit_rss) - allow(memory_killer).to receive(:get_memory_total_kb).and_return(memory_total) - - memory_killer.send(:refresh_state, :running) - end - - subject { memory_killer.send(:log_rss_out_of_range) } - - it 'invoke sidekiq logger warn' do - expect(memory_killer).to receive(:out_of_range_description).with(current_rss, hard_limit_rss, soft_limit_rss, true).and_return(reason) - expect(Sidekiq.logger).to receive(:warn) - .with( - class: described_class.to_s, - pid: pid, - message: 'Sidekiq worker RSS out of range', - current_rss: current_rss, - hard_limit_rss: hard_limit_rss, - soft_limit_rss: soft_limit_rss, - reason: reason, - running_jobs: [jid: jid, worker_class: 'DummyWorker'], - memory_total_kb: memory_total) - - expect(metrics[:sidekiq_memory_killer_running_jobs]).to receive(:increment) - .with({ worker_class: "DummyWorker", deadline_exceeded: true }) - - subject - end - end - - describe '#out_of_range_description' do - let(:hard_limit) { 300 } - let(:soft_limit) { 200 } - let(:grace_balloon_seconds) { 12 } - let(:deadline_exceeded) { true } - - subject { memory_killer.send(:out_of_range_description, rss, hard_limit, soft_limit, deadline_exceeded) } - - context 'when rss > hard_limit' do - let(:rss) { 400 } - - it 'tells reason' do - expect(subject).to eq("current_rss(#{rss}) > hard_limit_rss(#{hard_limit})") - end - end - - context 'when rss <= hard_limit' do - let(:rss) { 300 } - - context 'deadline exceeded' do - let(:deadline_exceeded) { true } - - it 'tells reason' do - stub_const("#{described_class}::GRACE_BALLOON_SECONDS", grace_balloon_seconds) - expect(subject).to eq("current_rss(#{rss}) > soft_limit_rss(#{soft_limit}) longer than GRACE_BALLOON_SECONDS(#{grace_balloon_seconds})") - end - end - - context 'deadline not exceeded' do - let(:deadline_exceeded) { false } - - it 'tells reason' do - expect(subject).to eq("current_rss(#{rss}) > soft_limit_rss(#{soft_limit})") - end - end - end - end - - describe '#rss_increase_by_jobs' do - let(:running_jobs) { { 'job1' => { worker_class: "Job1" }, 'job2' => { worker_class: "Job2" } } } - - subject { memory_killer.send(:rss_increase_by_jobs) } - - before do - allow(memory_killer).to receive(:rss_increase_by_job).and_return(11, 22) - end - - it 'adds up individual rss_increase_by_job' do - expect(subject).to eq(33) - end - - context 'when there is no running job' do - let(:running_jobs) { {} } - - it 'return 0 if no job' do - expect(subject).to eq(0) - end - end - end - - describe '#rss_increase_by_job' do - let(:worker_class) { Chaos::SleepWorker } - let(:job) { { worker_class: worker_class, started_at: 321 } } - let(:max_memory_kb) { 100000 } - - subject { memory_killer.send(:rss_increase_by_job, job) } - - before do - stub_const("#{described_class}::DEFAULT_MAX_MEMORY_GROWTH_KB", max_memory_kb) - end - - it 'return 0 if memory_growth_kb return 0' do - expect(memory_killer).to receive(:get_job_options).with(job, 'memory_killer_memory_growth_kb', 0).and_return(0) - expect(memory_killer).to receive(:get_job_options).with(job, 'memory_killer_max_memory_growth_kb', max_memory_kb).and_return(0) - - expect(Time).not_to receive(:now) - expect(subject).to eq(0) - end - - it 'return time factored growth value when it does not exceed max growth limit for whilited job' do - expect(memory_killer).to receive(:get_job_options).with(job, 'memory_killer_memory_growth_kb', 0).and_return(10) - expect(memory_killer).to receive(:get_job_options).with(job, 'memory_killer_max_memory_growth_kb', max_memory_kb).and_return(100) - - expect(Gitlab::Metrics::System).to receive(:monotonic_time).and_return(323) - expect(subject).to eq(20) - end - - it 'return max growth limit when time factored growth value exceed max growth limit for whilited job' do - expect(memory_killer).to receive(:get_job_options).with(job, 'memory_killer_memory_growth_kb', 0).and_return(10) - expect(memory_killer).to receive(:get_job_options).with(job, 'memory_killer_max_memory_growth_kb', max_memory_kb).and_return(100) - - expect(Gitlab::Metrics::System).to receive(:monotonic_time).and_return(332) - expect(subject).to eq(100) - end - end - - describe '#get_job_options' do - let(:worker_class) { Chaos::SleepWorker } - let(:job) { { worker_class: worker_class, started_at: 321 } } - let(:key) { 'my-key' } - let(:default) { 'my-default' } - - subject { memory_killer.send(:get_job_options, job, key, default) } - - it 'return default if key is not defined' do - expect(worker_class).to receive(:sidekiq_options).and_return({ "retry" => 5 }) - - expect(subject).to eq(default) - end - - it 'return default if get StandardError when retrieve sidekiq_options' do - expect(worker_class).to receive(:sidekiq_options).and_raise(StandardError) - - expect(subject).to eq(default) - end - - it 'return right value if sidekiq_options has the key' do - expect(worker_class).to receive(:sidekiq_options).and_return({ key => 10 }) - - expect(subject).to eq(10) - end - end - - describe '#refresh_state' do - let(:metrics) { memory_killer.instance_variable_get(:@metrics) } - - subject { memory_killer.send(:refresh_state, :shutting_down) } - - it 'calls gitlab metrics gauge set methods' do - expect(memory_killer).to receive(:get_rss_kb) { 1010 } - expect(memory_killer).to receive(:get_soft_limit_rss_kb) { 1020 } - expect(memory_killer).to receive(:get_hard_limit_rss_kb) { 1040 } - expect(memory_killer).to receive(:get_memory_total_kb) { 3072 } - - expect(metrics[:sidekiq_memory_killer_phase]).to receive(:set) - .with({}, described_class::PHASE[:shutting_down]) - expect(metrics[:sidekiq_current_rss]).to receive(:set) - .with({}, 1010) - expect(metrics[:sidekiq_memory_killer_soft_limit_rss]).to receive(:set) - .with({}, 1020) - expect(metrics[:sidekiq_memory_killer_hard_limit_rss]).to receive(:set) - .with({}, 1040) - - subject - end - end -end diff --git a/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb b/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb index e3d9549a3c0..4b589dc43af 100644 --- a/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb +++ b/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb @@ -309,7 +309,7 @@ RSpec.describe Gitlab::SidekiqLogging::StructuredLogger do end shared_examples 'performs database queries' do - it 'logs the database time', :aggregate_errors do + it 'logs the database time', :aggregate_failures do expect(logger).to receive(:info).with(expected_start_payload).ordered expect(logger).to receive(:info).with(expected_end_payload_with_db).ordered @@ -318,7 +318,7 @@ RSpec.describe Gitlab::SidekiqLogging::StructuredLogger do end end - it 'prevents database time from leaking to the next job', :aggregate_errors do + it 'prevents database time from leaking to the next job', :aggregate_failures do expect(logger).to receive(:info).with(expected_start_payload).ordered expect(logger).to receive(:info).with(expected_end_payload_with_db).ordered expect(logger).to receive(:info).with(expected_start_payload).ordered diff --git a/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job_spec.rb b/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job_spec.rb index 6a515a2b8a5..a46275d90b6 100644 --- a/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job_spec.rb @@ -2,7 +2,8 @@ require 'spec_helper' -RSpec.describe Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob, :clean_gitlab_redis_queues, :clean_gitlab_redis_shared_state do +RSpec.describe Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob, :clean_gitlab_redis_queues, :clean_gitlab_redis_shared_state, + feature_category: :shared do using RSpec::Parameterized::TableSyntax subject(:duplicate_job) do @@ -79,10 +80,10 @@ RSpec.describe Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob, :clean_gi context 'with Redis cookies' do def with_redis(&block) - Sidekiq.redis(&block) + Gitlab::Redis::Queues.with(&block) end - let(:cookie_key) { "#{idempotency_key}:cookie:v2" } + let(:cookie_key) { "#{Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE}:#{idempotency_key}:cookie:v2" } let(:cookie) { get_redis_msgpack(cookie_key) } describe '#check!' do diff --git a/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/server_spec.rb b/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/server_spec.rb index 1b01793d80d..f65f7a645ea 100644 --- a/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/server_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/server_spec.rb @@ -40,10 +40,10 @@ RSpec.describe Gitlab::SidekiqMiddleware::DuplicateJobs::Server, :clean_gitlab_r describe '#call' do it 'removes the stored job from redis before execution' do bare_job = { 'class' => 'TestDeduplicationWorker', 'args' => ['hello'] } - job_definition = Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob.new(bare_job.dup, 'test_deduplication') + job_definition = Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob.new(bare_job.dup, 'default') expect(Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob) - .to receive(:new).with(a_hash_including(bare_job), 'test_deduplication') + .to receive(:new).with(a_hash_including(bare_job), 'default') .and_return(job_definition).twice # once in client middleware expect(job_definition).to receive(:delete!).ordered.and_call_original @@ -59,10 +59,10 @@ RSpec.describe Gitlab::SidekiqMiddleware::DuplicateJobs::Server, :clean_gitlab_r it 'removes the stored job from redis after execution' do bare_job = { 'class' => 'TestDeduplicationWorker', 'args' => ['hello'] } - job_definition = Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob.new(bare_job.dup, 'test_deduplication') + job_definition = Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob.new(bare_job.dup, 'default') expect(Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob) - .to receive(:new).with(a_hash_including(bare_job), 'test_deduplication') + .to receive(:new).with(a_hash_including(bare_job), 'default') .and_return(job_definition).twice # once in client middleware expect(TestDeduplicationWorker).to receive(:work).ordered.and_call_original diff --git a/spec/lib/gitlab/sidekiq_middleware/server_metrics_spec.rb b/spec/lib/gitlab/sidekiq_middleware/server_metrics_spec.rb index f7cee6beb58..965ca612b3f 100644 --- a/spec/lib/gitlab/sidekiq_middleware/server_metrics_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware/server_metrics_spec.rb @@ -59,6 +59,45 @@ RSpec.describe Gitlab::SidekiqMiddleware::ServerMetrics do described_class.initialize_process_metrics end + context 'when sidekiq_execution_application_slis FF is turned on' do + it 'initializes sidekiq SLIs for the workers in the current Sidekiq process' do + allow(Gitlab::SidekiqConfig) + .to receive(:current_worker_queue_mappings) + .and_return('MergeWorker' => 'merge', 'Ci::BuildFinishedWorker' => 'default') + + allow(completion_seconds_metric).to receive(:get) + + expect(Gitlab::Metrics::SidekiqSlis) + .to receive(:initialize_slis!).with([ + { + worker: 'MergeWorker', + urgency: 'high', + feature_category: 'source_code_management' + }, + { + worker: 'Ci::BuildFinishedWorker', + urgency: 'high', + feature_category: 'continuous_integration' + } + ]) + + described_class.initialize_process_metrics + end + end + + context 'when sidekiq_execution_application_slis FF is turned off' do + before do + stub_feature_flags(sidekiq_execution_application_slis: false) + end + + it 'does not initialize sidekiq SLIs' do + expect(Gitlab::Metrics::SidekiqSlis) + .not_to receive(:initialize_slis!) + + described_class.initialize_process_metrics + end + end + context 'when the sidekiq_job_completion_metric_initialize feature flag is disabled' do before do stub_feature_flags(sidekiq_job_completion_metric_initialize: false) @@ -79,6 +118,17 @@ RSpec.describe Gitlab::SidekiqMiddleware::ServerMetrics do described_class.initialize_process_metrics end + + it 'does not initializes sidekiq SLIs' do + allow(Gitlab::SidekiqConfig) + .to receive(:current_worker_queue_mappings) + .and_return('MergeWorker' => 'merge', 'Ci::BuildFinishedWorker' => 'default') + + expect(Gitlab::Metrics::SidekiqSlis) + .not_to receive(:initialize_slis!) + + described_class.initialize_process_metrics + end end end @@ -110,6 +160,12 @@ RSpec.describe Gitlab::SidekiqMiddleware::ServerMetrics do expect(redis_requests_total).to receive(:increment).with(labels_with_job_status, redis_calls) expect(elasticsearch_requests_total).to receive(:increment).with(labels_with_job_status, elasticsearch_calls) expect(sidekiq_mem_total_bytes).to receive(:set).with(labels_with_job_status, mem_total_bytes) + expect(Gitlab::Metrics::SidekiqSlis).to receive(:record_execution_apdex).with(labels.slice(:worker, + :feature_category, + :urgency), monotonic_time_duration) + expect(Gitlab::Metrics::SidekiqSlis).to receive(:record_execution_error).with(labels.slice(:worker, + :feature_category, + :urgency), false) subject.call(worker, job, :test) { nil } end @@ -159,6 +215,16 @@ RSpec.describe Gitlab::SidekiqMiddleware::ServerMetrics do expect { subject.call(worker, job, :test) { raise StandardError, "Failed" } }.to raise_error(StandardError, "Failed") end + + it 'records sidekiq SLI error but does not record sidekiq SLI apdex' do + expect(failed_total_metric).to receive(:increment) + expect(Gitlab::Metrics::SidekiqSlis).not_to receive(:record_execution_apdex) + expect(Gitlab::Metrics::SidekiqSlis).to receive(:record_execution_error).with(labels.slice(:worker, + :feature_category, + :urgency), true) + + expect { subject.call(worker, job, :test) { raise StandardError, "Failed" } }.to raise_error(StandardError, "Failed") + end end context 'when job is retried' do @@ -180,6 +246,19 @@ RSpec.describe Gitlab::SidekiqMiddleware::ServerMetrics do subject.call(worker, job, :test) { nil } end end + + context 'when sidekiq_execution_application_slis FF is turned off' do + before do + stub_feature_flags(sidekiq_execution_application_slis: false) + end + + it 'does not call record_execution_apdex nor record_execution_error' do + expect(Gitlab::Metrics::SidekiqSlis).not_to receive(:record_execution_apdex) + expect(Gitlab::Metrics::SidekiqSlis).not_to receive(:record_execution_error) + + subject.call(worker, job, :test) { nil } + end + end end end @@ -331,7 +410,7 @@ RSpec.describe Gitlab::SidekiqMiddleware::ServerMetrics do include_context 'server metrics call' context 'when a worker has a feature category' do - let(:worker_category) { 'authentication_and_authorization' } + let(:worker_category) { 'system_access' } it 'uses that category for metrics' do expect(completion_seconds_metric).to receive(:observe).with(a_hash_including(feature_category: worker_category), anything) diff --git a/spec/lib/gitlab/sidekiq_middleware/worker_context/client_spec.rb b/spec/lib/gitlab/sidekiq_middleware/worker_context/client_spec.rb index 1b6cd7ac5fb..4fbc64a45d6 100644 --- a/spec/lib/gitlab/sidekiq_middleware/worker_context/client_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware/worker_context/client_spec.rb @@ -123,7 +123,7 @@ RSpec.describe Gitlab::SidekiqMiddleware::WorkerContext::Client do context 'when the feature category is already set in the surrounding block' do it 'takes the feature category from the worker, not the caller' do - Gitlab::ApplicationContext.with_context(feature_category: 'authentication_and_authorization') do + Gitlab::ApplicationContext.with_context(feature_category: 'system_access') do TestWithContextWorker.bulk_perform_async_with_contexts( %w(job1 job2), arguments_proc: -> (name) { [name, 1, 2, 3] }, @@ -139,7 +139,7 @@ RSpec.describe Gitlab::SidekiqMiddleware::WorkerContext::Client do end it 'takes the feature category from the caller if the worker is not owned' do - Gitlab::ApplicationContext.with_context(feature_category: 'authentication_and_authorization') do + Gitlab::ApplicationContext.with_context(feature_category: 'system_access') do TestNotOwnedWithContextWorker.bulk_perform_async_with_contexts( %w(job1 job2), arguments_proc: -> (name) { [name, 1, 2, 3] }, @@ -150,8 +150,8 @@ RSpec.describe Gitlab::SidekiqMiddleware::WorkerContext::Client do job1 = TestNotOwnedWithContextWorker.job_for_args(['job1', 1, 2, 3]) job2 = TestNotOwnedWithContextWorker.job_for_args(['job2', 1, 2, 3]) - expect(job1['meta.feature_category']).to eq('authentication_and_authorization') - expect(job2['meta.feature_category']).to eq('authentication_and_authorization') + expect(job1['meta.feature_category']).to eq('system_access') + expect(job2['meta.feature_category']).to eq('system_access') end end end diff --git a/spec/lib/gitlab/sidekiq_middleware/worker_context/server_spec.rb b/spec/lib/gitlab/sidekiq_middleware/worker_context/server_spec.rb index 2deab3064eb..eb077a0371c 100644 --- a/spec/lib/gitlab/sidekiq_middleware/worker_context/server_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware/worker_context/server_spec.rb @@ -69,7 +69,7 @@ RSpec.describe Gitlab::SidekiqMiddleware::WorkerContext::Server do context 'feature category' do it 'takes the feature category from the worker' do - Gitlab::ApplicationContext.with_context(feature_category: 'authentication_and_authorization') do + Gitlab::ApplicationContext.with_context(feature_category: 'system_access') do TestWorker.perform_async('identifier', 1) end @@ -78,11 +78,11 @@ RSpec.describe Gitlab::SidekiqMiddleware::WorkerContext::Server do context 'when the worker is not owned' do it 'takes the feature category from the surrounding context' do - Gitlab::ApplicationContext.with_context(feature_category: 'authentication_and_authorization') do + Gitlab::ApplicationContext.with_context(feature_category: 'system_access') do NotOwnedWorker.perform_async('identifier', 1) end - expect(NotOwnedWorker.contexts['identifier']).to include('meta.feature_category' => 'authentication_and_authorization') + expect(NotOwnedWorker.contexts['identifier']).to include('meta.feature_category' => 'system_access') end end end diff --git a/spec/lib/gitlab/sidekiq_migrate_jobs_spec.rb b/spec/lib/gitlab/sidekiq_migrate_jobs_spec.rb index 9ed2a0642fc..c66e36c5621 100644 --- a/spec/lib/gitlab/sidekiq_migrate_jobs_spec.rb +++ b/spec/lib/gitlab/sidekiq_migrate_jobs_spec.rb @@ -54,7 +54,7 @@ RSpec.describe Gitlab::SidekiqMigrateJobs, :clean_gitlab_redis_queues do expect(migrator.migrate_set(set_name)).to eq(scanned: 3, migrated: 0) expect(set_after.length).to eq(3) - expect(set_after.map(&:first)).to all(include('queue' => 'authorized_projects', + expect(set_after.map(&:first)).to all(include('queue' => 'default', 'class' => 'AuthorizedProjectsWorker')) end end @@ -73,7 +73,7 @@ RSpec.describe Gitlab::SidekiqMigrateJobs, :clean_gitlab_redis_queues do if item['class'] == 'AuthorizedProjectsWorker' expect(item).to include('queue' => 'new_queue', 'args' => [i]) else - expect(item).to include('queue' => 'post_receive', 'args' => [i]) + expect(item).to include('queue' => 'default', 'args' => [i]) end expect(score).to be_within(schedule_jitter).of(i.succ.hours.from_now.to_i) @@ -134,7 +134,7 @@ RSpec.describe Gitlab::SidekiqMigrateJobs, :clean_gitlab_redis_queues do expect(migrator.migrate_set(set_name)).to eq(scanned: 4, migrated: 0) expect(set_after.length).to eq(3) - expect(set_after.map(&:first)).to all(include('queue' => 'authorized_projects')) + expect(set_after.map(&:first)).to all(include('queue' => 'default')) end end @@ -157,7 +157,7 @@ RSpec.describe Gitlab::SidekiqMigrateJobs, :clean_gitlab_redis_queues do expect(migrator.migrate_set(set_name)).to eq(scanned: 4, migrated: 1) expect(set_after.group_by { |job| job.first['queue'] }.transform_values(&:count)) - .to eq('authorized_projects' => 6, 'new_queue' => 1) + .to eq('default' => 6, 'new_queue' => 1) end it 'iterates through the entire set of jobs' do diff --git a/spec/lib/gitlab/sidekiq_queue_spec.rb b/spec/lib/gitlab/sidekiq_queue_spec.rb index 5e91282612e..93632848788 100644 --- a/spec/lib/gitlab/sidekiq_queue_spec.rb +++ b/spec/lib/gitlab/sidekiq_queue_spec.rb @@ -4,15 +4,15 @@ require 'spec_helper' RSpec.describe Gitlab::SidekiqQueue, :clean_gitlab_redis_queues do around do |example| - Sidekiq::Queue.new('default').clear + Sidekiq::Queue.new('foobar').clear Sidekiq::Testing.disable!(&example) - Sidekiq::Queue.new('default').clear + Sidekiq::Queue.new('foobar').clear end def add_job(args, user:, klass: 'AuthorizedProjectsWorker') Sidekiq::Client.push( 'class' => klass, - 'queue' => 'default', + 'queue' => 'foobar', 'args' => args, 'meta.user' => user.username ) @@ -20,7 +20,7 @@ RSpec.describe Gitlab::SidekiqQueue, :clean_gitlab_redis_queues do describe '#drop_jobs!' do shared_examples 'queue processing' do - let(:sidekiq_queue) { described_class.new('default') } + let(:sidekiq_queue) { described_class.new('foobar') } let_it_be(:sidekiq_queue_user) { create(:user) } before do @@ -80,7 +80,7 @@ RSpec.describe Gitlab::SidekiqQueue, :clean_gitlab_redis_queues do it 'raises NoMetadataError' do add_job([1], user: create(:user)) - expect { described_class.new('default').drop_jobs!({ username: 'sidekiq_queue_user' }, timeout: 1) } + expect { described_class.new('foobar').drop_jobs!({ username: 'sidekiq_queue_user' }, timeout: 1) } .to raise_error(described_class::NoMetadataError) end end diff --git a/spec/lib/gitlab/slash_commands/global_slack_handler_spec.rb b/spec/lib/gitlab/slash_commands/global_slack_handler_spec.rb new file mode 100644 index 00000000000..4a58d65fc4a --- /dev/null +++ b/spec/lib/gitlab/slash_commands/global_slack_handler_spec.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::SlashCommands::GlobalSlackHandler, feature_category: :integrations do + include AfterNextHelpers + + let_it_be(:project) { create(:project) } + let_it_be(:user) { create(:user) } + + let_it_be_with_reload(:slack_integration) do + create(:gitlab_slack_application_integration, project: project).slack_integration + end + + let(:chat_name) { instance_double('ChatName', user: user) } + let(:verification_token) { '123' } + + before do + stub_application_setting(slack_app_verification_token: verification_token) + end + + def handler(params) + described_class.new(params) + end + + def handler_with_valid_token(params) + handler(params.merge(token: verification_token)) + end + + it 'does not serve a request if token is invalid' do + result = handler(token: '123456', text: 'help').trigger + + expect(result).to be_falsey + end + + context 'with valid token' do + context 'with incident declare command' do + it 'calls command handler with no project alias' do + expect_next(Gitlab::SlashCommands::Command).to receive(:execute) + expect_next(ChatNames::FindUserService).to receive(:execute).and_return(chat_name) + + handler_with_valid_token( + text: "incident declare", + team_id: slack_integration.team_id + ).trigger + end + end + + it 'calls command handler if project alias is valid' do + expect_next(Gitlab::SlashCommands::Command).to receive(:execute) + expect_next(ChatNames::FindUserService).to receive(:execute).and_return(chat_name) + + slack_integration.update!(alias: project.full_path) + + handler_with_valid_token( + text: "#{project.full_path} issue new title", + team_id: slack_integration.team_id + ).trigger + end + + it 'returns error if project alias not found' do + expect_next(Gitlab::SlashCommands::Command).not_to receive(:execute) + expect_next(Gitlab::SlashCommands::Presenters::Error).to receive(:message) + + handler_with_valid_token( + text: "fake/fake issue new title", + team_id: slack_integration.team_id + ).trigger + end + + it 'returns authorization request' do + expect_next(ChatNames::AuthorizeUserService).to receive(:execute) + expect_next(Gitlab::SlashCommands::Presenters::Access).to receive(:authorize) + + slack_integration.update!(alias: project.full_path) + + handler_with_valid_token( + text: "#{project.full_path} issue new title", + team_id: slack_integration.team_id + ).trigger + end + + it 'calls help presenter' do + expect_next(Gitlab::SlashCommands::ApplicationHelp).to receive(:execute) + + handler_with_valid_token( + text: "help" + ).trigger + end + end +end diff --git a/spec/lib/gitlab/slug/environment_spec.rb b/spec/lib/gitlab/slug/environment_spec.rb index e8f0fba27b2..8e23ad118d4 100644 --- a/spec/lib/gitlab/slug/environment_spec.rb +++ b/spec/lib/gitlab/slug/environment_spec.rb @@ -1,38 +1,41 @@ # frozen_string_literal: true require 'fast_spec_helper' +require 'rspec-parameterized' -RSpec.describe Gitlab::Slug::Environment do +RSpec.describe Gitlab::Slug::Environment, feature_category: :environment_management do describe '#generate' do - { - "staging-12345678901234567" => "staging-123456789-q517sa", - "9-staging-123456789012345" => "env-9-staging-123-q517sa", - "staging-1234567890123456" => "staging-1234567890123456", - "staging-1234567890123456-" => "staging-123456789-q517sa", - "production" => "production", - "PRODUCTION" => "production-q517sa", - "review/1-foo" => "review-1-foo-q517sa", - "1-foo" => "env-1-foo-q517sa", - "1/foo" => "env-1-foo-q517sa", - "foo-" => "foo", - "foo--bar" => "foo-bar-q517sa", - "foo**bar" => "foo-bar-q517sa", - "*-foo" => "env-foo-q517sa", - "staging-12345678-" => "staging-12345678", - "staging-12345678-01234567" => "staging-12345678-q517sa", - "" => "env-q517sa", - nil => "env-q517sa" - }.each do |name, matcher| - before do - # ('a' * 64).to_i(16).to_s(36).last(6) gives 'q517sa' - allow(Digest::SHA2).to receive(:hexdigest).with(name).and_return('a' * 64) - end + using RSpec::Parameterized::TableSyntax - it "returns a slug matching #{matcher}, given #{name}" do - slug = described_class.new(name).generate + subject { described_class.new(name).generate } - expect(slug).to match(/\A#{matcher}\z/) - end + before do + # ('a' * 64).to_i(16).to_s(36).last(6) gives 'q517sa' + allow(Digest::SHA2).to receive(:hexdigest).with(name.to_s).and_return('a' * 64) + end + + where(:name, :slug) do + "staging-12345678901234567" | "staging-123456789-q517sa" + "9-staging-123456789012345" | "env-9-staging-123-q517sa" + "staging-1234567890123456" | "staging-1234567890123456" + "staging-1234567890123456-" | "staging-123456789-q517sa" + "production" | "production" + "PRODUCTION" | "production-q517sa" + "review/1-foo" | "review-1-foo-q517sa" + "1-foo" | "env-1-foo-q517sa" + "1/foo" | "env-1-foo-q517sa" + "foo-" | "foo" + "foo--bar" | "foo-bar-q517sa" + "foo**bar" | "foo-bar-q517sa" + "*-foo" | "env-foo-q517sa" + "staging-12345678-" | "staging-12345678" + "staging-12345678-01234567" | "staging-12345678-q517sa" + "" | "env-q517sa" + nil | "env-q517sa" + end + + with_them do + it { is_expected.to eq(slug) } end end end diff --git a/spec/lib/gitlab/slug/path_spec.rb b/spec/lib/gitlab/slug/path_spec.rb index 9a7067e40a2..bbc2a05713d 100644 --- a/spec/lib/gitlab/slug/path_spec.rb +++ b/spec/lib/gitlab/slug/path_spec.rb @@ -2,7 +2,7 @@ require 'fast_spec_helper' -RSpec.describe Gitlab::Slug::Path, feature_category: :not_owned do +RSpec.describe Gitlab::Slug::Path, feature_category: :shared do describe '#generate' do { 'name': 'name', diff --git a/spec/lib/gitlab/source_spec.rb b/spec/lib/gitlab/source_spec.rb new file mode 100644 index 00000000000..0b2515baf2b --- /dev/null +++ b/spec/lib/gitlab/source_spec.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Source, feature_category: :shared do + include StubVersion + + describe '.ref' do + subject(:ref) { described_class.ref } + + context 'when not on a pre-release' do + before do + stub_version('15.0.0-ee', 'a123a123') + end + + it { is_expected.to eq('v15.0.0-ee') } + end + + context 'when on a pre-release' do + before do + stub_version('15.0.0-pre', 'a123a123') + end + + it { is_expected.to eq('a123a123') } + end + end + + describe '.release_url' do + subject(:release_url) { described_class.release_url } + + def release_path + Gitlab::Utils.append_path( + described_class.send(:host_url), + "#{described_class.send(:group)}/#{described_class.send(:project)}") + end + + context 'when not on a pre-release' do + before do + stub_version('15.0.0-ee', 'a123a123') + end + + it 'returns a tag url' do + expect(release_url).to eq("#{release_path}/-/tags/v15.0.0-ee") + end + end + + context 'when on a pre-release' do + before do + stub_version('15.0.0-pre', 'a123a123') + end + + it 'returns a commit url' do + expect(release_url).to eq("#{release_path}/-/commits/a123a123") + end + end + end +end diff --git a/spec/lib/gitlab/spamcheck/client_spec.rb b/spec/lib/gitlab/spamcheck/client_spec.rb index 2fe978125c4..ba07da51fb4 100644 --- a/spec/lib/gitlab/spamcheck/client_spec.rb +++ b/spec/lib/gitlab/spamcheck/client_spec.rb @@ -2,19 +2,14 @@ require 'spec_helper' -RSpec.describe Gitlab::Spamcheck::Client do +RSpec.describe Gitlab::Spamcheck::Client, feature_category: :instance_resiliency do include_context 'includes Spam constants' let(:endpoint) { 'grpc://grpc.test.url' } let_it_be(:user) { create(:user, organization: 'GitLab') } let(:verdict_value) { ::Spamcheck::SpamVerdict::Verdict::ALLOW } - let(:error_value) { "" } - - let(:attribs_value) do - extra_attributes = Google::Protobuf::Map.new(:string, :string) - extra_attributes["monitorMode"] = "false" - extra_attributes - end + let(:verdict_score) { 0.01 } + let(:verdict_evaluated) { true } let_it_be(:issue) { create(:issue, description: 'Test issue description') } let_it_be(:snippet) { create(:personal_snippet, :public, description: 'Test issue description') } @@ -22,8 +17,8 @@ RSpec.describe Gitlab::Spamcheck::Client do let(:response) do verdict = ::Spamcheck::SpamVerdict.new verdict.verdict = verdict_value - verdict.error = error_value - verdict.extra_attributes = attribs_value + verdict.evaluated = verdict_evaluated + verdict.score = verdict_score verdict end @@ -67,19 +62,19 @@ RSpec.describe Gitlab::Spamcheck::Client do using RSpec::Parameterized::TableSyntax - where(:verdict, :expected) do - ::Spamcheck::SpamVerdict::Verdict::ALLOW | Spam::SpamConstants::ALLOW - ::Spamcheck::SpamVerdict::Verdict::CONDITIONAL_ALLOW | Spam::SpamConstants::CONDITIONAL_ALLOW - ::Spamcheck::SpamVerdict::Verdict::DISALLOW | Spam::SpamConstants::DISALLOW - ::Spamcheck::SpamVerdict::Verdict::BLOCK | Spam::SpamConstants::BLOCK_USER - ::Spamcheck::SpamVerdict::Verdict::NOOP | Spam::SpamConstants::NOOP + where(:verdict_value, :expected, :verdict_evaluated, :verdict_score) do + ::Spamcheck::SpamVerdict::Verdict::ALLOW | Spam::SpamConstants::ALLOW | true | 0.01 + ::Spamcheck::SpamVerdict::Verdict::CONDITIONAL_ALLOW | Spam::SpamConstants::CONDITIONAL_ALLOW | true | 0.50 + ::Spamcheck::SpamVerdict::Verdict::DISALLOW | Spam::SpamConstants::DISALLOW | true | 0.75 + ::Spamcheck::SpamVerdict::Verdict::BLOCK | Spam::SpamConstants::BLOCK_USER | true | 0.99 + ::Spamcheck::SpamVerdict::Verdict::NOOP | Spam::SpamConstants::NOOP | false | 0.0 end with_them do - let(:verdict_value) { verdict } - - it "returns expected spam constant" do - expect(subject).to eq([expected, { "monitorMode" => "false" }, ""]) + it "returns expected spam result", :aggregate_failures do + expect(subject.verdict).to eq(expected) + expect(subject.evaluated?).to eq(verdict_evaluated) + expect(subject.score).to be_within(0.000001).of(verdict_score) end end @@ -106,6 +101,19 @@ RSpec.describe Gitlab::Spamcheck::Client do end describe "#build_protobuf", :aggregate_failures do + let_it_be(:generic_spammable) { Object } + let_it_be(:generic_created_at) { issue.created_at } + let_it_be(:generic_updated_at) { issue.updated_at } + + before do + allow(generic_spammable).to receive_messages( + spammable_text: 'generic spam', + created_at: generic_created_at, + updated_at: generic_updated_at, + project: nil + ) + end + it 'builds the expected issue protobuf object' do cxt = { action: :create } issue_pb, _ = described_class.new.send(:build_protobuf, @@ -132,21 +140,37 @@ RSpec.describe Gitlab::Spamcheck::Client do expect(snippet_pb.updated_at).to eq timestamp_to_protobuf_timestamp(snippet.updated_at) expect(snippet_pb.action).to be ::Spamcheck::Action.lookup(::Spamcheck::Action::CREATE) expect(snippet_pb.user.username).to eq user.username - expect(snippet_pb.user.username).to eq user.username expect(snippet_pb.files.first.path).to eq 'first.rb' expect(snippet_pb.files.last.path).to eq 'second.rb' end + + it 'builds the expected generic protobuf object' do + cxt = { action: :create } + generic_pb, _ = described_class.new.send(:build_protobuf, spammable: generic_spammable, user: user, context: cxt, extra_features: {}) + + expect(generic_pb.text).to eq 'generic spam' + expect(generic_pb.created_at).to eq timestamp_to_protobuf_timestamp(generic_created_at) + expect(generic_pb.updated_at).to eq timestamp_to_protobuf_timestamp(generic_updated_at) + expect(generic_pb.action).to be ::Spamcheck::Action.lookup(::Spamcheck::Action::CREATE) + expect(generic_pb.user.username).to eq user.username + end end describe '#build_user_protobuf', :aggregate_failures do + before do + allow(user).to receive(:account_age_in_days).and_return(10) + end + it 'builds the expected protobuf object' do user_pb = described_class.new.send(:build_user_protobuf, user) expect(user_pb.username).to eq user.username + expect(user_pb.id).to eq user.id expect(user_pb.org).to eq user.organization expect(user_pb.created_at).to eq timestamp_to_protobuf_timestamp(user.created_at) expect(user_pb.emails.count).to be 1 expect(user_pb.emails.first.email).to eq user.email expect(user_pb.emails.first.verified).to eq user.confirmed? + expect(user_pb.abuse_metadata[:account_age]).to eq 10 end context 'when user has multiple email addresses' do @@ -176,15 +200,14 @@ RSpec.describe Gitlab::Spamcheck::Client do end describe "#get_spammable_mappings", :aggregate_failures do - it 'is an expected spammable' do + it 'is a defined spammable' do protobuf_class, _ = described_class.new.send(:get_spammable_mappings, issue) expect(protobuf_class).to eq ::Spamcheck::Issue end - it 'is an unexpected spammable' do - expect { described_class.new.send(:get_spammable_mappings, 'spam') }.to raise_error( - ArgumentError, 'Not a spammable type: String' - ) + it 'is a generic spammable' do + protobuf_class, _ = described_class.new.send(:get_spammable_mappings, Object) + expect(protobuf_class).to eq ::Spamcheck::Generic end end diff --git a/spec/lib/gitlab/spamcheck/result_spec.rb b/spec/lib/gitlab/spamcheck/result_spec.rb new file mode 100644 index 00000000000..69bd61da8bf --- /dev/null +++ b/spec/lib/gitlab/spamcheck/result_spec.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Spamcheck::Result, feature_category: :instance_resiliency do + include_context 'includes Spam constants' + + describe "#initialize", :aggregate_failures do + using RSpec::Parameterized::TableSyntax + + subject { described_class.new(response) } + + where(:verdict_value, :expected, :verdict_evaluated, :verdict_score) do + ::Spamcheck::SpamVerdict::Verdict::ALLOW | Spam::SpamConstants::ALLOW | true | 0.01 + ::Spamcheck::SpamVerdict::Verdict::CONDITIONAL_ALLOW | Spam::SpamConstants::CONDITIONAL_ALLOW | true | 0.50 + ::Spamcheck::SpamVerdict::Verdict::DISALLOW | Spam::SpamConstants::DISALLOW | true | 0.75 + ::Spamcheck::SpamVerdict::Verdict::BLOCK | Spam::SpamConstants::BLOCK_USER | true | 0.99 + ::Spamcheck::SpamVerdict::Verdict::NOOP | Spam::SpamConstants::NOOP | false | 0.0 + end + + with_them do + let(:response) do + verdict = ::Spamcheck::SpamVerdict.new + verdict.verdict = verdict_value + verdict.evaluated = verdict_evaluated + verdict.score = verdict_score + verdict + end + + it "returns expected verdict" do + expect(subject.verdict).to eq(expected) + end + + it "returns expected evaluated?" do + expect(subject.evaluated?).to eq(verdict_evaluated) + end + + it "returns expected score" do + expect(subject.score).to be_within(0.000001).of(verdict_score) + end + end + end +end diff --git a/spec/lib/gitlab/subscription_portal_spec.rb b/spec/lib/gitlab/subscription_portal_spec.rb index f93eb6f96cc..96d3e855843 100644 --- a/spec/lib/gitlab/subscription_portal_spec.rb +++ b/spec/lib/gitlab/subscription_portal_spec.rb @@ -12,61 +12,10 @@ RSpec.describe ::Gitlab::SubscriptionPortal do stub_env('CUSTOMER_PORTAL_URL', env_value) end - describe '.default_subscriptions_url' do - where(:test, :development, :result) do - false | false | prod_customers_url - false | true | staging_customers_url - true | false | staging_customers_url - end - - before do - allow(Rails).to receive_message_chain(:env, :test?).and_return(test) - allow(Rails).to receive_message_chain(:env, :development?).and_return(development) - end - - with_them do - subject { described_class.default_subscriptions_url } - - it { is_expected.to eq(result) } - end - end - - describe '.subscriptions_url' do - subject { described_class.subscriptions_url } - - context 'when CUSTOMER_PORTAL_URL ENV is unset' do - it { is_expected.to eq(staging_customers_url) } - end - - context 'when CUSTOMER_PORTAL_URL ENV is set' do - let(:env_value) { 'https://customers.example.com' } - - it { is_expected.to eq(env_value) } - end - end - - describe '.subscriptions_comparison_url' do - subject { described_class.subscriptions_comparison_url } - - link_match = %r{\Ahttps://about\.gitlab\.((cn/pricing/saas)|(com/pricing/gitlab-com))/feature-comparison\z} - - it { is_expected.to match(link_match) } - end - describe 'class methods' do where(:method_name, :result) do - :default_subscriptions_url | staging_customers_url - :payment_form_url | "#{staging_customers_url}/payment_forms/cc_validation" :payment_validation_form_id | 'payment_method_validation' - :registration_validation_form_url | "#{staging_customers_url}/payment_forms/cc_registration_validation" :registration_validation_form_id | 'cc_registration_validation' - :subscriptions_graphql_url | "#{staging_customers_url}/graphql" - :subscriptions_more_minutes_url | "#{staging_customers_url}/buy_pipeline_minutes" - :subscriptions_more_storage_url | "#{staging_customers_url}/buy_storage" - :subscriptions_manage_url | "#{staging_customers_url}/subscriptions" - :subscriptions_instance_review_url | "#{staging_customers_url}/instance_review" - :subscriptions_gitlab_plans_url | "#{staging_customers_url}/gitlab_plans" - :edit_account_url | "#{staging_customers_url}/customers/edit" end with_them do @@ -76,40 +25,6 @@ RSpec.describe ::Gitlab::SubscriptionPortal do end end - describe '.add_extra_seats_url' do - subject { described_class.add_extra_seats_url(group_id) } - - let(:group_id) { 153 } - - it do - url = "#{staging_customers_url}/gitlab/namespaces/#{group_id}/extra_seats" - is_expected.to eq(url) - end - end - - describe '.upgrade_subscription_url' do - subject { described_class.upgrade_subscription_url(group_id, plan_id) } - - let(:group_id) { 153 } - let(:plan_id) { 5 } - - it do - url = "#{staging_customers_url}/gitlab/namespaces/#{group_id}/upgrade/#{plan_id}" - is_expected.to eq(url) - end - end - - describe '.renew_subscription_url' do - subject { described_class.renew_subscription_url(group_id) } - - let(:group_id) { 153 } - - it do - url = "#{staging_customers_url}/gitlab/namespaces/#{group_id}/renew" - is_expected.to eq(url) - end - end - describe 'constants' do where(:constant_name, :result) do 'REGISTRATION_VALIDATION_FORM_ID' | 'cc_registration_validation' diff --git a/spec/lib/gitlab/template/finders/global_template_finder_spec.rb b/spec/lib/gitlab/template/finders/global_template_finder_spec.rb index 38ec28c2b9a..c1dfee3cccb 100644 --- a/spec/lib/gitlab/template/finders/global_template_finder_spec.rb +++ b/spec/lib/gitlab/template/finders/global_template_finder_spec.rb @@ -16,10 +16,12 @@ RSpec.describe Gitlab::Template::Finders::GlobalTemplateFinder do end subject(:finder) do - described_class.new(base_dir, '', - { 'General' => '', 'Bar' => 'Bar' }, - include_categories_for_file, - excluded_patterns: excluded_patterns) + described_class.new( + base_dir, '', + { 'General' => '', 'Bar' => 'Bar' }, + include_categories_for_file, + excluded_patterns: excluded_patterns + ) end let(:excluded_patterns) { [] } diff --git a/spec/lib/gitlab/timeless_spec.rb b/spec/lib/gitlab/timeless_spec.rb new file mode 100644 index 00000000000..d806349d326 --- /dev/null +++ b/spec/lib/gitlab/timeless_spec.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Timeless, feature_category: :shared do + let(:model) { build(:user) } + + it 'disables record_timestamps temporarily' do + expect(model.record_timestamps).to eq(true) + + Gitlab::Timeless.timeless(model) do |m| + expect(m.record_timestamps).to eq(false) + expect(model.record_timestamps).to eq(false) + end + + expect(model.record_timestamps).to eq(true) + end + + it 'does not record created_at' do + Gitlab::Timeless.timeless(model) do + model.save!(username: "#{model.username}-a") + end + + expect(model.created_at).to be(nil) + end + + it 'does not record updated_at' do + model.save! + previous = model.updated_at + + Gitlab::Timeless.timeless(model) do + model.update!(username: "#{model.username}-a") + end + + expect(model.updated_at).to eq(previous) + end +end diff --git a/spec/lib/gitlab/tracking/destinations/database_events_snowplow_spec.rb b/spec/lib/gitlab/tracking/destinations/database_events_snowplow_spec.rb new file mode 100644 index 00000000000..78a869b535a --- /dev/null +++ b/spec/lib/gitlab/tracking/destinations/database_events_snowplow_spec.rb @@ -0,0 +1,136 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Tracking::Destinations::DatabaseEventsSnowplow, :do_not_stub_snowplow_by_default, feature_category: :application_instrumentation do + let(:emitter) { SnowplowTracker::Emitter.new(endpoint: 'localhost', options: { buffer_size: 1 }) } + + let(:tracker) do + SnowplowTracker::Tracker + .new( + emitters: [emitter], + subject: SnowplowTracker::Subject.new, + namespace: 'namespace', + app_id: 'app_id' + ) + end + + before do + stub_application_setting(snowplow_app_id: '_abc123_') + end + + around do |example| + freeze_time { example.run } + end + + context 'when snowplow is enabled' do + before do + allow(SnowplowTracker::AsyncEmitter) + .to receive(:new) + .with(endpoint: endpoint, + options: + { + protocol: 'https', + on_success: subject.method(:increment_successful_events_emissions), + on_failure: subject.method(:failure_callback) + } + ).and_return(emitter) + + allow(SnowplowTracker::Tracker) + .to receive(:new) + .with( + emitters: [emitter], + subject: an_instance_of(SnowplowTracker::Subject), + namespace: described_class::SNOWPLOW_NAMESPACE, + app_id: '_abc123_' + ).and_return(tracker) + end + + describe '#event' do + let(:endpoint) { 'localhost:9091' } + let(:event_params) do + { + category: 'category', + action: 'action', + label: 'label', + property: 'property', + value: 1.5, + context: nil, + tstamp: (Time.now.to_f * 1000).to_i + } + end + + context 'when on gitlab.com environment' do + let(:endpoint) { 'db-snowplow.trx.gitlab.net' } + + it 'sends event to tracker' do + allow(Gitlab).to receive(:com?).and_return(true) + allow(tracker).to receive(:track_struct_event).and_call_original + + subject.event('category', 'action', label: 'label', property: 'property', value: 1.5) + + expect(tracker).to have_received(:track_struct_event).with(event_params) + end + end + + it 'sends event to tracker' do + allow(tracker).to receive(:track_struct_event).and_call_original + + subject.event('category', 'action', label: 'label', property: 'property', value: 1.5) + + expect(tracker).to have_received(:track_struct_event).with(event_params) + end + + it 'increase total snowplow events counter' do + counter = double + + expect(counter).to receive(:increment) + expect(Gitlab::Metrics).to receive(:counter) + .with(:gitlab_db_events_snowplow_events_total, 'Number of Snowplow events') + .and_return(counter) + + subject.event('category', 'action', label: 'label', property: 'property', value: 1.5) + end + end + end + + context 'for callbacks' do + describe 'on success' do + it 'increase gitlab_successful_snowplow_events_total counter' do + counter = double + + expect(counter).to receive(:increment).with({}, 2) + expect(Gitlab::Metrics).to receive(:counter) + .with( + :gitlab_db_events_snowplow_successful_events_total, + 'Number of successful Snowplow events emissions').and_return(counter) + + subject.method(:increment_successful_events_emissions).call(2) + end + end + + describe 'on failure' do + it 'increase gitlab_failed_snowplow_events_total counter and logs failures', :aggregate_failures do + counter = double + error_message = "Issue database_event_update failed to be reported to collector at localhost:9091" + failures = [{ "e" => "se", + "se_ca" => "Issue", + "se_la" => "issues", + "se_ac" => "database_event_update" }] + allow(Gitlab::Metrics).to receive(:counter) + .with( + :gitlab_db_events_snowplow_successful_events_total, + 'Number of successful Snowplow events emissions').and_call_original + + expect(Gitlab::AppLogger).to receive(:error).with(error_message) + expect(counter).to receive(:increment).with({}, 1) + expect(Gitlab::Metrics).to receive(:counter) + .with( + :gitlab_db_events_snowplow_failed_events_total, + 'Number of failed Snowplow events emissions').and_return(counter) + + subject.method(:failure_callback).call(2, failures) + end + end + end +end diff --git a/spec/lib/gitlab/tracking/destinations/snowplow_micro_spec.rb b/spec/lib/gitlab/tracking/destinations/snowplow_micro_spec.rb index 48092a33da3..ea3c030541f 100644 --- a/spec/lib/gitlab/tracking/destinations/snowplow_micro_spec.rb +++ b/spec/lib/gitlab/tracking/destinations/snowplow_micro_spec.rb @@ -45,7 +45,7 @@ RSpec.describe Gitlab::Tracking::Destinations::SnowplowMicro do context 'when snowplow_micro config is not set' do before do - allow(Gitlab.config).to receive(:snowplow_micro).and_raise(Settingslogic::MissingSetting) + allow(Gitlab.config).to receive(:snowplow_micro).and_raise(GitlabSettings::MissingSetting) end it 'returns localhost hostname' do diff --git a/spec/lib/gitlab/tracking/event_definition_spec.rb b/spec/lib/gitlab/tracking/event_definition_spec.rb index c8e616b092b..b27aaa35695 100644 --- a/spec/lib/gitlab/tracking/event_definition_spec.rb +++ b/spec/lib/gitlab/tracking/event_definition_spec.rb @@ -12,7 +12,6 @@ RSpec.describe Gitlab::Tracking::EventDefinition do property_description: 'The string "issue_id"', value_description: 'ID of the issue', extra_properties: { confidential: false }, - product_category: 'collection', product_stage: 'growth', product_section: 'dev', product_group: 'group::product analytics', @@ -47,7 +46,6 @@ RSpec.describe Gitlab::Tracking::EventDefinition do :property_description | 1 :value_description | 1 :extra_properties | 'smth' - :product_category | 1 :product_stage | 1 :product_section | nil :product_group | nil diff --git a/spec/lib/gitlab/tracking/standard_context_spec.rb b/spec/lib/gitlab/tracking/standard_context_spec.rb index cfb83bc0528..e1ae362e797 100644 --- a/spec/lib/gitlab/tracking/standard_context_spec.rb +++ b/spec/lib/gitlab/tracking/standard_context_spec.rb @@ -70,7 +70,9 @@ RSpec.describe Gitlab::Tracking::StandardContext do end context 'when namespace is available' do - subject { described_class.new(namespace: create(:namespace)) } + let(:namespace) { create(:namespace) } + + subject { described_class.new(namespace_id: namespace.id, plan_name: namespace.actual_plan_name) } it 'contains plan name' do expect(snowplow_context.to_json.dig(:data, :plan)).to eq(Plan::DEFAULT) @@ -93,7 +95,7 @@ RSpec.describe Gitlab::Tracking::StandardContext do end context 'with incorrect argument type' do - subject { described_class.new(project: create(:group)) } + subject { described_class.new(project_id: create(:group)) } it 'does call `track_and_raise_for_dev_exception`' do expect(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception) diff --git a/spec/lib/gitlab/tracking_spec.rb b/spec/lib/gitlab/tracking_spec.rb index e79bb2ef129..a353a3a512c 100644 --- a/spec/lib/gitlab/tracking_spec.rb +++ b/spec/lib/gitlab/tracking_spec.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true require 'spec_helper' -RSpec.describe Gitlab::Tracking do +RSpec.describe Gitlab::Tracking, feature_category: :application_instrumentation do include StubENV before do @@ -102,12 +102,28 @@ RSpec.describe Gitlab::Tracking do end end - describe '.event' do + context 'event tracking' do let(:namespace) { create(:namespace) } - shared_examples 'delegates to destination' do |klass| + shared_examples 'rescued error raised by destination class' do + it 'rescues error' do + error = StandardError.new("something went wrong") + allow_any_instance_of(destination_class).to receive(:event).and_raise(error) + + expect(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception) + .with( + error, + snowplow_category: category, + snowplow_action: action + ) + + expect { tracking_method }.not_to raise_error + end + end + + shared_examples 'delegates to destination' do |klass, method| before do - allow_any_instance_of(Gitlab::Tracking::Destinations::Snowplow).to receive(:event) + allow_any_instance_of(klass).to receive(:event) end it "delegates to #{klass} destination" do @@ -118,8 +134,8 @@ RSpec.describe Gitlab::Tracking do expect(Gitlab::Tracking::StandardContext) .to receive(:new) - .with(project: project, user: user, namespace: namespace, extra_key_1: 'extra value 1', extra_key_2: 'extra value 2') - .and_call_original + .with(project_id: project.id, user_id: user.id, namespace_id: namespace.id, plan_name: namespace.actual_plan_name, extra_key_1: 'extra value 1', extra_key_2: 'extra value 2') + .and_call_original expect_any_instance_of(klass).to receive(:event) do |_, category, action, args| expect(category).to eq('category') @@ -132,7 +148,7 @@ RSpec.describe Gitlab::Tracking do expect(args[:context].last).to eq(other_context) end - described_class.event('category', 'action', + described_class.method(method).call('category', 'action', label: 'label', property: 'property', value: 1.5, @@ -141,44 +157,95 @@ RSpec.describe Gitlab::Tracking do user: user, namespace: namespace, extra_key_1: 'extra value 1', - extra_key_2: 'extra value 2') + extra_key_2: 'extra value 2' + ) end end - context 'when the action is not passed in as a string' do - it 'allows symbols' do - expect(Gitlab::ErrorTracking).not_to receive(:track_and_raise_for_dev_exception) + describe '.database_event' do + context 'when the action is not passed in as a string' do + it 'allows symbols' do + expect(Gitlab::ErrorTracking).not_to receive(:track_and_raise_for_dev_exception) - described_class.event('category', :some_action) - end + described_class.database_event('category', :some_action) + end + + it 'allows nil' do + expect(Gitlab::ErrorTracking).not_to receive(:track_and_raise_for_dev_exception) + + described_class.database_event('category', nil) + end - it 'allows nil' do - expect(Gitlab::ErrorTracking).not_to receive(:track_and_raise_for_dev_exception) + it 'allows integers' do + expect(Gitlab::ErrorTracking).not_to receive(:track_and_raise_for_dev_exception) - described_class.event('category', nil) + described_class.database_event('category', 1) + end end - it 'allows integers' do - expect(Gitlab::ErrorTracking).not_to receive(:track_and_raise_for_dev_exception) + it_behaves_like 'rescued error raised by destination class' do + let(:category) { 'Issue' } + let(:action) { 'created' } + let(:destination_class) { Gitlab::Tracking::Destinations::DatabaseEventsSnowplow } - described_class.event('category', 1) + subject(:tracking_method) { described_class.database_event(category, action) } end + + it_behaves_like 'delegates to destination', Gitlab::Tracking::Destinations::DatabaseEventsSnowplow, :database_event end - context 'when destination is Snowplow' do - before do - allow(Rails.env).to receive(:development?).and_return(true) + describe '.event' do + context 'when the action is not passed in as a string' do + it 'allows symbols' do + expect(Gitlab::ErrorTracking).not_to receive(:track_and_raise_for_dev_exception) + + described_class.event('category', :some_action) + end + + it 'allows nil' do + expect(Gitlab::ErrorTracking).not_to receive(:track_and_raise_for_dev_exception) + + described_class.event('category', nil) + end + + it 'allows integers' do + expect(Gitlab::ErrorTracking).not_to receive(:track_and_raise_for_dev_exception) + + described_class.event('category', 1) + end end - it_behaves_like 'delegates to destination', Gitlab::Tracking::Destinations::Snowplow - end + context 'when destination is Snowplow' do + before do + allow(Rails.env).to receive(:development?).and_return(true) + end - context 'when destination is SnowplowMicro' do - before do - allow(Rails.env).to receive(:development?).and_return(true) + it_behaves_like 'rescued error raised by destination class' do + let(:category) { 'category' } + let(:action) { 'action' } + let(:destination_class) { Gitlab::Tracking::Destinations::Snowplow } + + subject(:tracking_method) { described_class.event(category, action) } + end + + it_behaves_like 'delegates to destination', Gitlab::Tracking::Destinations::Snowplow, :event end - it_behaves_like 'delegates to destination', Gitlab::Tracking::Destinations::SnowplowMicro + context 'when destination is SnowplowMicro' do + before do + allow(Rails.env).to receive(:development?).and_return(true) + end + + it_behaves_like 'rescued error raised by destination class' do + let(:category) { 'category' } + let(:action) { 'action' } + let(:destination_class) { Gitlab::Tracking::Destinations::Snowplow } + + subject(:tracking_method) { described_class.event(category, action) } + end + + it_behaves_like 'delegates to destination', Gitlab::Tracking::Destinations::SnowplowMicro, :event + end end end @@ -245,7 +312,7 @@ RSpec.describe Gitlab::Tracking do end it 'returns false when snowplow_micro is not configured' do - allow(Gitlab.config).to receive(:snowplow_micro).and_raise(Settingslogic::MissingSetting) + allow(Gitlab.config).to receive(:snowplow_micro).and_raise(GitlabSettings::MissingSetting) expect(described_class).not_to be_snowplow_micro_enabled end diff --git a/spec/lib/gitlab/untrusted_regexp_spec.rb b/spec/lib/gitlab/untrusted_regexp_spec.rb index 66675b20107..8c3669d6773 100644 --- a/spec/lib/gitlab/untrusted_regexp_spec.rb +++ b/spec/lib/gitlab/untrusted_regexp_spec.rb @@ -3,7 +3,11 @@ require 'fast_spec_helper' require 'support/shared_examples/lib/gitlab/malicious_regexp_shared_examples' -RSpec.describe Gitlab::UntrustedRegexp do +RSpec.describe Gitlab::UntrustedRegexp, feature_category: :shared do + def create_regex(regex_str, multiline: false) + described_class.new(regex_str, multiline: multiline).freeze + end + describe '#initialize' do subject { described_class.new(pattern) } @@ -16,15 +20,48 @@ RSpec.describe Gitlab::UntrustedRegexp do describe '#replace_all' do it 'replaces all instances of the match in a string' do - result = described_class.new('foo').replace_all('foo bar foo', 'oof') + result = create_regex('foo').replace_all('foo bar foo', 'oof') expect(result).to eq('oof bar oof') end end + describe '#replace_gsub' do + let(:regex_str) { '(?P<scheme>(ftp))' } + let(:regex) { create_regex(regex_str, multiline: true) } + + def result(regex, text) + regex.replace_gsub(text) do |match| + if match[:scheme] + "http|#{match[:scheme]}|rss" + else + match.to_s + end + end + end + + it 'replaces all instances of the match in a string' do + text = 'Use only https instead of ftp' + + expect(result(regex, text)).to eq('Use only https instead of http|ftp|rss') + end + + it 'replaces nothing when no match' do + text = 'Use only https instead of gopher' + + expect(result(regex, text)).to eq(text) + end + + it 'handles empty text' do + text = '' + + expect(result(regex, text)).to eq('') + end + end + describe '#replace' do it 'replaces the first instance of the match in a string' do - result = described_class.new('foo').replace('foo bar foo', 'oof') + result = create_regex('foo').replace('foo bar foo', 'oof') expect(result).to eq('oof bar foo') end @@ -32,19 +69,19 @@ RSpec.describe Gitlab::UntrustedRegexp do describe '#===' do it 'returns true for a match' do - result = described_class.new('foo') === 'a foo here' + result = create_regex('foo') === 'a foo here' expect(result).to be_truthy end it 'returns false for no match' do - result = described_class.new('foo') === 'a bar here' + result = create_regex('foo') === 'a bar here' expect(result).to be_falsy end it 'can handle regular expressions in multiline mode' do - regexp = described_class.new('^\d', multiline: true) + regexp = create_regex('^\d', multiline: true) result = regexp === "Header\n\n1. Content" @@ -53,7 +90,7 @@ RSpec.describe Gitlab::UntrustedRegexp do end describe '#match?' do - subject { described_class.new(regexp).match?(text) } + subject { create_regex(regexp).match?(text) } context 'malicious regexp' do let(:text) { malicious_text } @@ -82,7 +119,7 @@ RSpec.describe Gitlab::UntrustedRegexp do end describe '#scan' do - subject { described_class.new(regexp).scan(text) } + subject { create_regex(regexp).scan(text) } context 'malicious regexp' do let(:text) { malicious_text } @@ -138,7 +175,7 @@ RSpec.describe Gitlab::UntrustedRegexp do end describe '#extract_named_group' do - let(:re) { described_class.new('(?P<name>\w+) (?P<age>\d+)|(?P<name_only>\w+)') } + let(:re) { create_regex('(?P<name>\w+) (?P<age>\d+)|(?P<name_only>\w+)') } let(:text) { 'Bob 40' } it 'returns values for both named groups' do @@ -172,7 +209,7 @@ RSpec.describe Gitlab::UntrustedRegexp do describe '#match' do context 'when there are matches' do it 'returns a match object' do - result = described_class.new('(?P<number>\d+)').match('hello 10') + result = create_regex('(?P<number>\d+)').match('hello 10') expect(result[:number]).to eq('10') end @@ -180,7 +217,7 @@ RSpec.describe Gitlab::UntrustedRegexp do context 'when there are no matches' do it 'returns nil' do - result = described_class.new('(?P<number>\d+)').match('hello') + result = create_regex('(?P<number>\d+)').match('hello') expect(result).to be_nil end diff --git a/spec/lib/gitlab/url_blocker_spec.rb b/spec/lib/gitlab/url_blocker_spec.rb index 05f7af7606d..cfd40fb93b5 100644 --- a/spec/lib/gitlab/url_blocker_spec.rb +++ b/spec/lib/gitlab/url_blocker_spec.rb @@ -2,13 +2,18 @@ require 'spec_helper' -RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do +RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only, feature_category: :shared do include StubRequests let(:schemes) { %w[http https] } + # This test ensures backward compatibliity for the validate! method. + # We shoud refactor all callers of validate! to handle a Result object: + # https://gitlab.com/gitlab-org/gitlab/-/issues/410890 describe '#validate!' do - subject { described_class.validate!(import_url, schemes: schemes) } + let(:options) { { schemes: schemes } } + + subject { described_class.validate!(import_url, **options) } shared_examples 'validates URI and hostname' do it 'runs the url validations' do @@ -19,13 +24,113 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do end end + context 'when the URL hostname is a domain' do + context 'when domain can be resolved' do + let(:import_url) { 'https://example.org' } + + before do + stub_dns(import_url, ip_address: '93.184.216.34') + end + + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { 'https://93.184.216.34' } + let(:expected_hostname) { 'example.org' } + let(:expected_use_proxy) { false } + end + end + end + end + + describe '#validate_url_with_proxy!' do + let(:options) { { schemes: schemes } } + + subject { described_class.validate_url_with_proxy!(import_url, **options) } + + shared_examples 'validates URI and hostname' do + it 'runs the url validations' do + expect(subject.uri).to eq(Addressable::URI.parse(expected_uri)) + expect(subject.hostname).to eq(expected_hostname) + expect(subject.use_proxy).to eq(expected_use_proxy) + end + end + + shared_context 'when instance configured to deny all requests' do + before do + allow(Gitlab::CurrentSettings).to receive(:current_application_settings?).and_return(true) + stub_application_setting(deny_all_requests_except_allowed: true) + end + end + + shared_examples 'a URI denied by `deny_all_requests_except_allowed`' do + context 'when instance setting is enabled' do + include_context 'when instance configured to deny all requests' + + it 'blocks the request' do + expect { subject }.to raise_error(described_class::BlockedUrlError) + end + end + + context 'when instance setting is not enabled' do + it 'does not block the request' do + expect { subject }.not_to raise_error + end + end + + context 'when passed as an argument' do + let(:options) { super().merge(deny_all_requests_except_allowed: arg_value) } + + context 'when argument is a proc that evaluates to true' do + let(:arg_value) { proc { true } } + + it 'blocks the request' do + expect { subject }.to raise_error(described_class::BlockedUrlError) + end + end + + context 'when argument is a proc that evaluates to false' do + let(:arg_value) { proc { false } } + + it 'does not block the request' do + expect { subject }.not_to raise_error + end + end + + context 'when argument is true' do + let(:arg_value) { true } + + it 'blocks the request' do + expect { subject }.to raise_error(described_class::BlockedUrlError) + end + end + + context 'when argument is false' do + let(:arg_value) { false } + + it 'does not block the request' do + expect { subject }.not_to raise_error + end + end + end + end + + shared_examples 'a URI exempt from `deny_all_requests_except_allowed`' do + include_context 'when instance configured to deny all requests' + + it 'does not block the request' do + expect { subject }.not_to raise_error + end + end + context 'when URI is nil' do let(:import_url) { nil } it_behaves_like 'validates URI and hostname' do let(:expected_uri) { nil } let(:expected_hostname) { nil } + let(:expected_use_proxy) { true } end + + it_behaves_like 'a URI exempt from `deny_all_requests_except_allowed`' end context 'when URI is internal' do @@ -38,7 +143,10 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do it_behaves_like 'validates URI and hostname' do let(:expected_uri) { 'http://127.0.0.1' } let(:expected_hostname) { 'localhost' } + let(:expected_use_proxy) { false } end + + it_behaves_like 'a URI exempt from `deny_all_requests_except_allowed`' end context 'when URI is for a local object storage' do @@ -61,7 +169,7 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do end context 'when allow_object_storage is true' do - subject { described_class.validate!(import_url, allow_object_storage: true, schemes: schemes) } + let(:options) { { allow_object_storage: true, schemes: schemes } } context 'with a local domain name' do let(:host) { 'http://review-minio-svc.svc:9000' } @@ -73,7 +181,10 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do it_behaves_like 'validates URI and hostname' do let(:expected_uri) { 'http://127.0.0.1:9000/external-diffs/merge_request_diffs/mr-1/diff-1' } let(:expected_hostname) { 'review-minio-svc.svc' } + let(:expected_use_proxy) { false } end + + it_behaves_like 'a URI exempt from `deny_all_requests_except_allowed`' end context 'with an IP address' do @@ -82,15 +193,18 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do it_behaves_like 'validates URI and hostname' do let(:expected_uri) { 'http://127.0.0.1:9000/external-diffs/merge_request_diffs/mr-1/diff-1' } let(:expected_hostname) { nil } + let(:expected_use_proxy) { false } end + + it_behaves_like 'a URI exempt from `deny_all_requests_except_allowed`' end context 'when LFS object storage is enabled' do let(:lfs_config) do { 'enabled' => lfs_enabled, - # This nesting of Settingslogic is necessary to trigger the bug - 'object_store' => Settingslogic.new({ 'enabled' => true }) + # This nesting of settings is necessary to trigger the bug + 'object_store' => GitlabSettings::Options.build({ 'enabled' => true }) } end @@ -98,16 +212,15 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do { 'gitlab' => Gitlab.config.gitlab, 'repositories' => { 'storages' => { 'default' => 'test' } }, - 'lfs' => Settingslogic.new(lfs_config) + 'lfs' => GitlabSettings::Options.build(lfs_config) } end let(:host) { 'http://127.0.0.1:9000' } - let(:settings) { Settingslogic.new(config) } + let(:settings) { GitlabSettings::Options.build(config) } before do allow(Gitlab).to receive(:config).and_return(settings) - # Triggers Settingslogic bug: https://gitlab.com/gitlab-org/gitlab/-/issues/286873 settings.repositories.storages.default end @@ -163,21 +276,52 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do it_behaves_like 'validates URI and hostname' do let(:expected_uri) { 'https://93.184.216.34' } let(:expected_hostname) { 'example.org' } + let(:expected_use_proxy) { false } end + + it_behaves_like 'a URI denied by `deny_all_requests_except_allowed`' end context 'when domain cannot be resolved' do let(:import_url) { 'http://foobar.x' } - it 'raises an error' do + before do stub_env('RSPEC_ALLOW_INVALID_URLS', 'false') + end + it 'raises an error' do expect { subject }.to raise_error(described_class::BlockedUrlError) end + + context 'with HTTP_PROXY' do + let(:import_url) { 'http://foobar.x' } + + before do + stub_env('http_proxy', 'http://proxy.example.com') + end + + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { import_url } + let(:expected_hostname) { nil } + let(:expected_use_proxy) { true } + end + + context 'with no_proxy' do + before do + stub_env('no_proxy', 'foobar.x') + end + + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { import_url } + let(:expected_hostname) { nil } + let(:expected_use_proxy) { false } + end + end + end end context 'when domain is too long' do - let(:import_url) { 'https://example' + 'a' * 1024 + '.com' } + let(:import_url) { "https://example#{'a' * 1024}.com" } it 'raises an error' do expect { subject }.to raise_error(described_class::BlockedUrlError) @@ -191,8 +335,11 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do it_behaves_like 'validates URI and hostname' do let(:expected_uri) { import_url } let(:expected_hostname) { nil } + let(:expected_use_proxy) { false } end + it_behaves_like 'a URI denied by `deny_all_requests_except_allowed`' + context 'when the address is invalid' do let(:import_url) { 'http://1.1.1.1.1' } @@ -204,7 +351,7 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do end end - context 'DNS rebinding protection with IP allowed' do + context 'when DNS rebinding protection with IP allowed' do let(:import_url) { 'http://a.192.168.0.120.3times.127.0.0.1.1time.repeat.rebind.network:9121/scrape?target=unix:///var/opt/gitlab/redis/redis.socket&check-keys=*' } before do @@ -216,11 +363,38 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do it_behaves_like 'validates URI and hostname' do let(:expected_uri) { 'http://192.168.0.120:9121/scrape?target=unix:///var/opt/gitlab/redis/redis.socket&check-keys=*' } let(:expected_hostname) { 'a.192.168.0.120.3times.127.0.0.1.1time.repeat.rebind.network' } + let(:expected_use_proxy) { false } + end + + it_behaves_like 'a URI exempt from `deny_all_requests_except_allowed`' + + context 'with HTTP_PROXY' do + before do + stub_env('http_proxy', 'http://proxy.example.com') + end + + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { import_url } + let(:expected_hostname) { nil } + let(:expected_use_proxy) { true } + end + + context 'when domain is in no_proxy env' do + before do + stub_env('no_proxy', 'a.192.168.0.120.3times.127.0.0.1.1time.repeat.rebind.network') + end + + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { 'http://192.168.0.120:9121/scrape?target=unix:///var/opt/gitlab/redis/redis.socket&check-keys=*' } + let(:expected_hostname) { 'a.192.168.0.120.3times.127.0.0.1.1time.repeat.rebind.network' } + let(:expected_use_proxy) { false } + end + end end end - context 'disabled DNS rebinding protection' do - subject { described_class.validate!(import_url, dns_rebind_protection: false, schemes: schemes) } + context 'with disabled DNS rebinding protection' do + let(:options) { { dns_rebind_protection: false, schemes: schemes } } context 'when URI is internal' do let(:import_url) { 'http://localhost' } @@ -228,7 +402,10 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do it_behaves_like 'validates URI and hostname' do let(:expected_uri) { import_url } let(:expected_hostname) { nil } + let(:expected_use_proxy) { false } end + + it_behaves_like 'a URI exempt from `deny_all_requests_except_allowed`' end context 'when the URL hostname is a domain' do @@ -242,7 +419,10 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do it_behaves_like 'validates URI and hostname' do let(:expected_uri) { import_url } let(:expected_hostname) { nil } + let(:expected_use_proxy) { false } end + + it_behaves_like 'a URI denied by `deny_all_requests_except_allowed`' end context 'when domain cannot be resolved' do @@ -251,7 +431,10 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do it_behaves_like 'validates URI and hostname' do let(:expected_uri) { import_url } let(:expected_hostname) { nil } + let(:expected_use_proxy) { false } end + + it_behaves_like 'a URI denied by `deny_all_requests_except_allowed`' end end @@ -261,15 +444,21 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do it_behaves_like 'validates URI and hostname' do let(:expected_uri) { import_url } let(:expected_hostname) { nil } + let(:expected_use_proxy) { false } end + it_behaves_like 'a URI denied by `deny_all_requests_except_allowed`' + context 'when it is invalid' do let(:import_url) { 'http://1.1.1.1.1' } it_behaves_like 'validates URI and hostname' do let(:expected_uri) { import_url } let(:expected_hostname) { nil } + let(:expected_use_proxy) { false } end + + it_behaves_like 'a URI denied by `deny_all_requests_except_allowed`' end end end @@ -390,7 +579,7 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do expect(described_class.blocked_url?('https://gitlab.com/foo/foo.git', schemes: schemes)).to be false end - context 'when allow_local_network is' do + describe 'allow_local_network' do let(:shared_address_space_ips) { ['100.64.0.0', '100.64.127.127', '100.64.255.255'] } let(:local_ips) do @@ -471,11 +660,11 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do end end - context 'true (default)' do + context 'when true (default)' do it_behaves_like 'allows local requests', { allow_localhost: true, allow_local_network: true, schemes: %w[http https] } end - context 'false' do + context 'when false' do it 'blocks urls from private networks' do local_ips.each do |ip| stub_domain_resolv(fake_domain, ip) do @@ -628,14 +817,14 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do end end - context 'when dns_rebinding_setting is' do - context 'enabled' do + describe 'dns_rebinding_setting' do + context 'when enabled' do let(:dns_rebind_value) { true } it_behaves_like 'allowlists the domain' end - context 'disabled' do + context 'when disabled' do let(:dns_rebind_value) { false } it_behaves_like 'allowlists the domain' @@ -675,8 +864,8 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do end end - context 'when enforce_user is' do - context 'false (default)' do + describe 'enforce_user' do + context 'when false (default)' do it 'does not block urls with a non-alphanumeric username' do expect(described_class).not_to be_blocked_url('ssh://-oProxyCommand=whoami@example.com/a', schemes: ['ssh']) @@ -688,7 +877,7 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do end end - context 'true' do + context 'when true' do it 'blocks urls with a non-alphanumeric username' do aggregate_failures do expect(described_class).to be_blocked_url('ssh://-oProxyCommand=whoami@example.com/a', enforce_user: true, schemes: ['ssh']) @@ -756,7 +945,7 @@ RSpec.describe Gitlab::UrlBlocker, :stub_invalid_dns_only do end end - def stub_domain_resolv(domain, ip, port = 80, &block) + def stub_domain_resolv(domain, ip, port = 80) address = instance_double(Addrinfo, ip_address: ip, ipv4_private?: true, diff --git a/spec/lib/gitlab/url_blockers/ip_allowlist_entry_spec.rb b/spec/lib/gitlab/url_blockers/ip_allowlist_entry_spec.rb index 8dcb402dfb2..c56e5ce4e7a 100644 --- a/spec/lib/gitlab/url_blockers/ip_allowlist_entry_spec.rb +++ b/spec/lib/gitlab/url_blockers/ip_allowlist_entry_spec.rb @@ -2,7 +2,7 @@ require 'fast_spec_helper' -RSpec.describe Gitlab::UrlBlockers::IpAllowlistEntry do +RSpec.describe Gitlab::UrlBlockers::IpAllowlistEntry, feature_category: :integrations do let(:ipv4) { IPAddr.new('192.168.1.1') } describe '#initialize' do @@ -65,11 +65,31 @@ RSpec.describe Gitlab::UrlBlockers::IpAllowlistEntry do end it 'matches IPv6 within IPv6 range' do - ipv6_range = IPAddr.new('fd84:6d02:f6d8:c89e::/124') + ipv6_range = IPAddr.new('::ffff:192.168.1.0/8') ip_allowlist_entry = described_class.new(ipv6_range) expect(ip_allowlist_entry).to be_match(ipv6_range.to_range.last.to_s, 8080) expect(ip_allowlist_entry).not_to be_match('fd84:6d02:f6d8:f::f', 8080) end + + it 'matches IPv4 to IPv6 mapped addresses in allow list' do + ipv6_range = IPAddr.new('::ffff:192.168.1.1') + ip_allowlist_entry = described_class.new(ipv6_range) + + expect(ip_allowlist_entry).to be_match(ipv4, 8080) + expect(ip_allowlist_entry).to be_match(ipv6_range.to_range.last.to_s, 8080) + expect(ip_allowlist_entry).not_to be_match('::ffff:192.168.1.0', 8080) + expect(ip_allowlist_entry).not_to be_match('::ffff:169.254.168.101', 8080) + end + + it 'matches IPv4 to IPv6 mapped addresses in requested IP' do + ipv4_range = IPAddr.new('192.168.1.1/24') + ip_allowlist_entry = described_class.new(ipv4_range) + + expect(ip_allowlist_entry).to be_match(ipv4, 8080) + expect(ip_allowlist_entry).to be_match('::ffff:192.168.1.0', 8080) + expect(ip_allowlist_entry).to be_match('::ffff:192.168.1.1', 8080) + expect(ip_allowlist_entry).not_to be_match('::ffff:169.254.170.100/8', 8080) + end end end diff --git a/spec/lib/gitlab/url_builder_spec.rb b/spec/lib/gitlab/url_builder_spec.rb index 2e9a444bd24..73627d3e6ff 100644 --- a/spec/lib/gitlab/url_builder_spec.rb +++ b/spec/lib/gitlab/url_builder_spec.rb @@ -22,8 +22,8 @@ RSpec.describe Gitlab::UrlBuilder do :group_board | ->(board) { "/groups/#{board.group.full_path}/-/boards/#{board.id}" } :commit | ->(commit) { "/#{commit.project.full_path}/-/commit/#{commit.id}" } :issue | ->(issue) { "/#{issue.project.full_path}/-/issues/#{issue.iid}" } - [:issue, :task] | ->(issue) { "/#{issue.project.full_path}/-/work_items/#{issue.iid}?iid_path=true" } - :work_item | ->(work_item) { "/#{work_item.project.full_path}/-/work_items/#{work_item.iid}?iid_path=true" } + [:issue, :task] | ->(issue) { "/#{issue.project.full_path}/-/work_items/#{issue.iid}" } + :work_item | ->(work_item) { "/#{work_item.project.full_path}/-/work_items/#{work_item.iid}" } :merge_request | ->(merge_request) { "/#{merge_request.project.full_path}/-/merge_requests/#{merge_request.iid}" } :project_milestone | ->(milestone) { "/#{milestone.project.full_path}/-/milestones/#{milestone.iid}" } :project_snippet | ->(snippet) { "/#{snippet.project.full_path}/-/snippets/#{snippet.id}" } @@ -227,27 +227,5 @@ RSpec.describe Gitlab::UrlBuilder do expect(subject.build(object, only_path: true)).to eq("/#{project.full_path}") end end - - context 'when use_iid_in_work_items_path feature flag is disabled' do - before do - stub_feature_flags(use_iid_in_work_items_path: false) - end - - context 'when a task issue is passed' do - it 'returns a path using the work item\'s ID and no query params' do - task = create(:issue, :task) - - expect(subject.build(task, only_path: true)).to eq("/#{task.project.full_path}/-/work_items/#{task.id}") - end - end - - context 'when a work item is passed' do - it 'returns a path using the work item\'s ID and no query params' do - work_item = create(:work_item) - - expect(subject.build(work_item, only_path: true)).to eq("/#{work_item.project.full_path}/-/work_items/#{work_item.id}") - end - end - end end end diff --git a/spec/lib/gitlab/usage/metric_definition_spec.rb b/spec/lib/gitlab/usage/metric_definition_spec.rb index 4b835d11975..c336a4850d2 100644 --- a/spec/lib/gitlab/usage/metric_definition_spec.rb +++ b/spec/lib/gitlab/usage/metric_definition_spec.rb @@ -7,7 +7,6 @@ RSpec.describe Gitlab::Usage::MetricDefinition do { description: 'GitLab instance unique identifier', value_type: 'string', - product_category: 'collection', product_stage: 'growth', product_section: 'devops', status: 'active', @@ -263,7 +262,6 @@ RSpec.describe Gitlab::Usage::MetricDefinition do { description: 'Test metric definition', value_type: 'string', - product_category: 'collection', product_stage: 'growth', product_section: 'devops', status: 'active', diff --git a/spec/lib/gitlab/usage/metric_spec.rb b/spec/lib/gitlab/usage/metric_spec.rb index 8e0fce37e46..d0ea4e7aa16 100644 --- a/spec/lib/gitlab/usage/metric_spec.rb +++ b/spec/lib/gitlab/usage/metric_spec.rb @@ -13,7 +13,6 @@ RSpec.describe Gitlab::Usage::Metric do product_section: "dev", product_stage: "plan", product_group: "plan", - product_category: "issue_tracking", value_type: "number", status: "active", time_frame: "all", diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric_spec.rb index ce15d44b1e1..317929f77e6 100644 --- a/spec/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric_spec.rb +++ b/spec/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Usage::Metrics::Instrumentations::CountBulkImportsEntitiesMetric do +RSpec.describe Gitlab::Usage::Metrics::Instrumentations::CountBulkImportsEntitiesMetric, feature_category: :importers do let_it_be(:user) { create(:user) } let_it_be(:bulk_import_projects) do create_list(:bulk_import_entity, 2, source_type: 'project_entity', created_at: 3.weeks.ago, status: 2) @@ -163,4 +163,121 @@ RSpec.describe Gitlab::Usage::Metrics::Instrumentations::CountBulkImportsEntitie options: { status: 2, source_type: 'project_entity' } end end + + context 'with has_failures: true' do + before(:all) do + create_list(:bulk_import_entity, 3, :project_entity, :finished, created_at: 3.weeks.ago, has_failures: true) + create_list(:bulk_import_entity, 2, :project_entity, :finished, created_at: 2.months.ago, has_failures: true) + create_list(:bulk_import_entity, 3, :group_entity, :finished, created_at: 3.weeks.ago, has_failures: true) + create_list(:bulk_import_entity, 2, :group_entity, :finished, created_at: 2.months.ago, has_failures: true) + end + + context 'with all time frame' do + context 'with project entity' do + let(:expected_value) { 5 } + let(:expected_query) do + "SELECT COUNT(\"bulk_import_entities\".\"id\") FROM \"bulk_import_entities\" " \ + "WHERE \"bulk_import_entities\".\"source_type\" = 1 AND \"bulk_import_entities\".\"status\" = 2 " \ + "AND \"bulk_import_entities\".\"has_failures\" = TRUE" + end + + it_behaves_like 'a correct instrumented metric value and query', + time_frame: 'all', + options: { status: 2, source_type: 'project_entity', has_failures: true } + end + + context 'with group entity' do + let(:expected_value) { 5 } + let(:expected_query) do + "SELECT COUNT(\"bulk_import_entities\".\"id\") FROM \"bulk_import_entities\" " \ + "WHERE \"bulk_import_entities\".\"source_type\" = 0 AND \"bulk_import_entities\".\"status\" = 2 " \ + "AND \"bulk_import_entities\".\"has_failures\" = TRUE" + end + + it_behaves_like 'a correct instrumented metric value and query', + time_frame: 'all', + options: { status: 2, source_type: 'group_entity', has_failures: true } + end + end + + context 'for 28d time frame' do + let(:expected_value) { 3 } + let(:start) { 30.days.ago.to_s(:db) } + let(:finish) { 2.days.ago.to_s(:db) } + let(:expected_query) do + "SELECT COUNT(\"bulk_import_entities\".\"id\") FROM \"bulk_import_entities\" " \ + "WHERE \"bulk_import_entities\".\"created_at\" BETWEEN '#{start}' AND '#{finish}' " \ + "AND \"bulk_import_entities\".\"source_type\" = 1 AND \"bulk_import_entities\".\"status\" = 2 " \ + "AND \"bulk_import_entities\".\"has_failures\" = TRUE" + end + + it_behaves_like 'a correct instrumented metric value and query', + time_frame: '28d', + options: { status: 2, source_type: 'project_entity', has_failures: true } + end + end + + context 'with has_failures: false' do + context 'with all time frame' do + context 'with project entity' do + let(:expected_value) { 3 } + let(:expected_query) do + "SELECT COUNT(\"bulk_import_entities\".\"id\") FROM \"bulk_import_entities\" " \ + "WHERE \"bulk_import_entities\".\"source_type\" = 1 AND \"bulk_import_entities\".\"status\" = 2 " \ + "AND \"bulk_import_entities\".\"has_failures\" = FALSE" + end + + it_behaves_like 'a correct instrumented metric value and query', + time_frame: 'all', + options: { status: 2, source_type: 'project_entity', has_failures: false } + end + + context 'with group entity' do + let(:expected_value) { 2 } + let(:expected_query) do + "SELECT COUNT(\"bulk_import_entities\".\"id\") FROM \"bulk_import_entities\" " \ + "WHERE \"bulk_import_entities\".\"source_type\" = 0 AND \"bulk_import_entities\".\"status\" = 2 " \ + "AND \"bulk_import_entities\".\"has_failures\" = FALSE" + end + + it_behaves_like 'a correct instrumented metric value and query', + time_frame: 'all', + options: { status: 2, source_type: 'group_entity', has_failures: false } + end + end + + context 'for 28d time frame' do + context 'with project entity' do + let(:expected_value) { 2 } + let(:start) { 30.days.ago.to_s(:db) } + let(:finish) { 2.days.ago.to_s(:db) } + let(:expected_query) do + "SELECT COUNT(\"bulk_import_entities\".\"id\") FROM \"bulk_import_entities\" " \ + "WHERE \"bulk_import_entities\".\"created_at\" BETWEEN '#{start}' AND '#{finish}' " \ + "AND \"bulk_import_entities\".\"source_type\" = 1 AND \"bulk_import_entities\".\"status\" = 2 " \ + "AND \"bulk_import_entities\".\"has_failures\" = FALSE" + end + + it_behaves_like 'a correct instrumented metric value and query', + time_frame: '28d', + options: { status: 2, source_type: 'project_entity', has_failures: false } + end + + context 'with group entity' do + let(:expected_value) { 2 } + let(:start) { 30.days.ago.to_s(:db) } + let(:finish) { 2.days.ago.to_s(:db) } + let(:expected_query) do + "SELECT COUNT(\"bulk_import_entities\".\"id\") FROM \"bulk_import_entities\" " \ + "WHERE \"bulk_import_entities\".\"created_at\" BETWEEN '#{start}' AND '#{finish}' " \ + "AND \"bulk_import_entities\".\"source_type\" = 0 AND \"bulk_import_entities\".\"status\" = 2 " \ + "AND \"bulk_import_entities\".\"has_failures\" = FALSE" + end + + it_behaves_like 'a correct instrumented metric value and query', + time_frame: '28d', + options: { status: 2, source_type: 'group_entity', has_failures: false } + end + end + end end diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/count_ci_internal_pipelines_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/count_ci_internal_pipelines_metric_spec.rb index afd8fccd56c..77c49d448d7 100644 --- a/spec/lib/gitlab/usage/metrics/instrumentations/count_ci_internal_pipelines_metric_spec.rb +++ b/spec/lib/gitlab/usage/metrics/instrumentations/count_ci_internal_pipelines_metric_spec.rb @@ -4,25 +4,23 @@ require 'spec_helper' RSpec.describe Gitlab::Usage::Metrics::Instrumentations::CountCiInternalPipelinesMetric, feature_category: :service_ping do - let_it_be(:ci_pipeline_1) { create(:ci_pipeline, source: :external) } - let_it_be(:ci_pipeline_2) { create(:ci_pipeline, source: :push) } - - let(:expected_value) { 1 } - let(:expected_query) do - 'SELECT COUNT("ci_pipelines"."id") FROM "ci_pipelines" ' \ - 'WHERE ("ci_pipelines"."source" IN (1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 12, 13, 14, 15) ' \ - 'OR "ci_pipelines"."source" IS NULL)' - end + let_it_be(:ci_pipeline_1) { create(:ci_pipeline, source: :external, created_at: 3.days.ago) } + let_it_be(:ci_pipeline_2) { create(:ci_pipeline, source: :push, created_at: 3.days.ago) } + let_it_be(:old_pipeline) { create(:ci_pipeline, source: :push, created_at: 2.months.ago) } + let_it_be(:expected_value) { 2 } it_behaves_like 'a correct instrumented metric value', { time_frame: 'all', data_source: 'database' } - context 'on Gitlab.com' do - before do - allow(Gitlab).to receive(:com?).and_return(true) - end + context 'for monthly counts' do + let_it_be(:expected_value) { 1 } + + it_behaves_like 'a correct instrumented metric value', { time_frame: '28d', data_source: 'database' } + end - let(:expected_value) { -1 } + context 'on SaaS', :saas do + let_it_be(:expected_value) { -1 } it_behaves_like 'a correct instrumented metric value', { time_frame: 'all', data_source: 'database' } + it_behaves_like 'a correct instrumented metric value', { time_frame: '28d', data_source: 'database' } end end diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_group_type_active_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_group_type_active_metric_spec.rb new file mode 100644 index 00000000000..33605783671 --- /dev/null +++ b/spec/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_group_type_active_metric_spec.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Usage::Metrics::Instrumentations::CountCiRunnersGroupTypeActiveMetric, feature_category: :runner do + let_it_be(:group) { create(:group) } + let(:expected_value) { 1 } + + before do + create(:ci_runner, + :group, + groups: [group] + ) + end + + it_behaves_like 'a correct instrumented metric value', { time_frame: 'all', data_source: 'database' } +end diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_group_type_active_online_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_group_type_active_online_metric_spec.rb new file mode 100644 index 00000000000..24d6ea6f1e9 --- /dev/null +++ b/spec/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_group_type_active_online_metric_spec.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Usage::Metrics::Instrumentations::CountCiRunnersGroupTypeActiveOnlineMetric, feature_category: :runner do + let(:group) { create(:group) } + let(:expected_value) { 1 } + + before do + create(:ci_runner, + :group, + groups: [group], + contacted_at: 1.second.ago + ) + end + + it_behaves_like 'a correct instrumented metric value', { time_frame: 'all', data_source: 'database' } +end diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_instance_type_active_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_instance_type_active_metric_spec.rb new file mode 100644 index 00000000000..ae4829cceef --- /dev/null +++ b/spec/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_instance_type_active_metric_spec.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Usage::Metrics::Instrumentations::CountCiRunnersInstanceTypeActiveMetric, feature_category: :runner do + let(:expected_value) { 1 } + + before do + create(:ci_runner) + end + + it_behaves_like 'a correct instrumented metric value', { time_frame: 'all', data_source: 'database' } +end diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_instance_type_active_online_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_instance_type_active_online_metric_spec.rb new file mode 100644 index 00000000000..b1b9a5a6cea --- /dev/null +++ b/spec/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_instance_type_active_online_metric_spec.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Usage::Metrics::Instrumentations::CountCiRunnersInstanceTypeActiveOnlineMetric, feature_category: :runner do + let(:expected_value) { 1 } + + before do + create(:ci_runner, contacted_at: 1.second.ago) + end + + it_behaves_like 'a correct instrumented metric value', { time_frame: 'all', data_source: 'database' } +end diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_metric_spec.rb new file mode 100644 index 00000000000..6a3a8e6dd58 --- /dev/null +++ b/spec/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_metric_spec.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Usage::Metrics::Instrumentations::CountCiRunnersMetric, feature_category: :runner do + let(:expected_value) { 1 } + + before do + create(:ci_runner) + end + + it_behaves_like 'a correct instrumented metric value', { time_frame: 'all', data_source: 'database' } +end diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_project_type_active_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_project_type_active_metric_spec.rb new file mode 100644 index 00000000000..eeb699c1377 --- /dev/null +++ b/spec/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_project_type_active_metric_spec.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Usage::Metrics::Instrumentations::CountCiRunnersProjectTypeActiveMetric, feature_category: :runner do + let(:project) { build(:project) } + let(:expected_value) { 1 } + + before do + create(:ci_runner, + :project, + projects: [project] + ) + end + + it_behaves_like 'a correct instrumented metric value', { time_frame: 'all', data_source: 'database' } +end diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_project_type_active_online_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_project_type_active_online_metric_spec.rb new file mode 100644 index 00000000000..c3ed752ae04 --- /dev/null +++ b/spec/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_project_type_active_online_metric_spec.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Usage::Metrics::Instrumentations::CountCiRunnersProjectTypeActiveOnlineMetric, feature_category: :runner do + let(:project) { build(:project) } + let(:expected_value) { 1 } + + before do + create(:ci_runner, + :project, + projects: [project], + contacted_at: 1.second.ago + ) + end + + it_behaves_like 'a correct instrumented metric value', { time_frame: 'all', data_source: 'database' } +end diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/count_issues_created_manually_from_alerts_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/count_issues_created_manually_from_alerts_metric_spec.rb index 86f54c48666..65e514bf345 100644 --- a/spec/lib/gitlab/usage/metrics/instrumentations/count_issues_created_manually_from_alerts_metric_spec.rb +++ b/spec/lib/gitlab/usage/metrics/instrumentations/count_issues_created_manually_from_alerts_metric_spec.rb @@ -16,11 +16,7 @@ feature_category: :service_ping do it_behaves_like 'a correct instrumented metric value', { time_frame: 'all', data_source: 'database' } - context 'on Gitlab.com' do - before do - allow(Gitlab).to receive(:com?).and_return(true) - end - + context 'on SaaS', :saas do let(:expected_value) { -1 } it_behaves_like 'a correct instrumented metric value', { time_frame: 'all', data_source: 'database' } diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/database_mode_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/database_mode_spec.rb new file mode 100644 index 00000000000..a6128b4df1f --- /dev/null +++ b/spec/lib/gitlab/usage/metrics/instrumentations/database_mode_spec.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Usage::Metrics::Instrumentations::DatabaseMode, feature_category: :cell do + let(:expected_value) { Gitlab::Database.database_mode } + + it_behaves_like 'a correct instrumented metric value', { time_frame: 'none' } +end diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/edition_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/edition_metric_spec.rb new file mode 100644 index 00000000000..2e23b9f5a15 --- /dev/null +++ b/spec/lib/gitlab/usage/metrics/instrumentations/edition_metric_spec.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Usage::Metrics::Instrumentations::EditionMetric, feature_category: :service_ping do + before do + allow(Gitlab).to receive(:ee?).and_return(false) + end + + let(:expected_value) { 'CE' } + + it_behaves_like 'a correct instrumented metric value', { time_frame: 'all' } +end diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/gitlab_dedicated_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/gitlab_dedicated_metric_spec.rb new file mode 100644 index 00000000000..a35022ec2c4 --- /dev/null +++ b/spec/lib/gitlab/usage/metrics/instrumentations/gitlab_dedicated_metric_spec.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Usage::Metrics::Instrumentations::GitlabDedicatedMetric, feature_category: :service_ping do + let(:expected_value) { Gitlab::CurrentSettings.gitlab_dedicated_instance } + + it_behaves_like 'a correct instrumented metric value', { time_frame: 'none' } +end diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/incoming_email_encrypted_secrets_enabled_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/incoming_email_encrypted_secrets_enabled_metric_spec.rb index ed35b2c8cde..b1b193c8d04 100644 --- a/spec/lib/gitlab/usage/metrics/instrumentations/incoming_email_encrypted_secrets_enabled_metric_spec.rb +++ b/spec/lib/gitlab/usage/metrics/instrumentations/incoming_email_encrypted_secrets_enabled_metric_spec.rb @@ -5,6 +5,6 @@ require 'spec_helper' RSpec.describe Gitlab::Usage::Metrics::Instrumentations::IncomingEmailEncryptedSecretsEnabledMetric, feature_category: :service_ping do it_behaves_like 'a correct instrumented metric value', { time_frame: 'none', data_source: 'ruby' } do - let(:expected_value) { ::Gitlab::IncomingEmail.encrypted_secrets.active? } + let(:expected_value) { ::Gitlab::Email::IncomingEmail.encrypted_secrets.active? } end end diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/index_inconsistencies_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/index_inconsistencies_metric_spec.rb new file mode 100644 index 00000000000..92a576d1a9f --- /dev/null +++ b/spec/lib/gitlab/usage/metrics/instrumentations/index_inconsistencies_metric_spec.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Usage::Metrics::Instrumentations::IndexInconsistenciesMetric, feature_category: :database do + it_behaves_like 'a correct instrumented metric value', { time_frame: 'all' } do + let(:expected_value) do + [ + { inconsistency_type: 'wrong_indexes', object_name: 'index_name_1' }, + { inconsistency_type: 'missing_indexes', object_name: 'index_name_2' }, + { inconsistency_type: 'extra_indexes', object_name: 'index_name_3' } + ] + end + + let(:runner) { instance_double(Gitlab::Database::SchemaValidation::Runner, execute: inconsistencies) } + let(:inconsistency_class) { Gitlab::Database::SchemaValidation::Inconsistency } + + let(:inconsistencies) do + [ + instance_double(inconsistency_class, object_name: 'index_name_1', type: 'wrong_indexes'), + instance_double(inconsistency_class, object_name: 'index_name_2', type: 'missing_indexes'), + instance_double(inconsistency_class, object_name: 'index_name_3', type: 'extra_indexes') + ] + end + + before do + allow(Gitlab::Database::SchemaValidation::Runner).to receive(:new).and_return(runner) + end + end +end diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/installation_creation_date_approximation_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/installation_creation_date_approximation_metric_spec.rb new file mode 100644 index 00000000000..11e1139e542 --- /dev/null +++ b/spec/lib/gitlab/usage/metrics/instrumentations/installation_creation_date_approximation_metric_spec.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Usage::Metrics::Instrumentations::InstallationCreationDateApproximationMetric, + feature_category: :service_ping do + let_it_be(:application_setting) { create(:application_setting) } + + context 'with a root user' do + let_it_be(:root) { create(:user, id: 1, created_at: DateTime.current - 2.days) } + let_it_be(:expected_value) { root.reload.created_at } # reloading to get the timestamp from the database + + it_behaves_like 'a correct instrumented metric value', { time_frame: 'all', data_source: 'database' } + end + + context 'without a root user' do + let_it_be(:another_user) { create(:user, id: 2, created_at: DateTime.current + 2.days) } + let_it_be(:expected_value) { application_setting.reload.created_at } + + it_behaves_like 'a correct instrumented metric value', { time_frame: 'all', data_source: 'database' } + end +end diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/installation_creation_date_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/installation_creation_date_metric_spec.rb new file mode 100644 index 00000000000..ff6be56c13f --- /dev/null +++ b/spec/lib/gitlab/usage/metrics/instrumentations/installation_creation_date_metric_spec.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Usage::Metrics::Instrumentations::InstallationCreationDateMetric, + feature_category: :service_ping do + context 'with a root user' do + let_it_be(:root) { create(:user, id: 1) } + let_it_be(:expected_value) { root.reload.created_at } # reloading to get the timestamp from the database + + it_behaves_like 'a correct instrumented metric value', { time_frame: 'all', data_source: 'database' } + end + + context 'without a root user' do + let_it_be(:another_user) { create(:user, id: 2) } + let_it_be(:expected_value) { nil } + + it_behaves_like 'a correct instrumented metric value', { time_frame: 'all', data_source: 'database' } + end +end diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/installation_type_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/installation_type_metric_spec.rb new file mode 100644 index 00000000000..7b59536e7d2 --- /dev/null +++ b/spec/lib/gitlab/usage/metrics/instrumentations/installation_type_metric_spec.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Usage::Metrics::Instrumentations::InstallationTypeMetric, feature_category: :service_ping do + context 'when Rails.env is production' do + before do + allow(Rails).to receive_message_chain(:env, :production?).and_return(true) + end + + let(:expected_value) { Gitlab::INSTALLATION_TYPE } + + it_behaves_like 'a correct instrumented metric value', { time_frame: 'all' } + end + + context 'with Rails.env is not production' do + let(:expected_value) { 'gitlab-development-kit' } + + it_behaves_like 'a correct instrumented metric value', { time_frame: 'all' } + end +end diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/service_desk_email_encrypted_secrets_enabled_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/service_desk_email_encrypted_secrets_enabled_metric_spec.rb index d602eae3159..ea239e53d01 100644 --- a/spec/lib/gitlab/usage/metrics/instrumentations/service_desk_email_encrypted_secrets_enabled_metric_spec.rb +++ b/spec/lib/gitlab/usage/metrics/instrumentations/service_desk_email_encrypted_secrets_enabled_metric_spec.rb @@ -5,6 +5,6 @@ require 'spec_helper' RSpec.describe Gitlab::Usage::Metrics::Instrumentations::ServiceDeskEmailEncryptedSecretsEnabledMetric, feature_category: :service_ping do it_behaves_like 'a correct instrumented metric value', { time_frame: 'none', data_source: 'ruby' } do - let(:expected_value) { ::Gitlab::ServiceDeskEmail.encrypted_secrets.active? } + let(:expected_value) { ::Gitlab::Email::ServiceDeskEmail.encrypted_secrets.active? } end end diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/version_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/version_metric_spec.rb new file mode 100644 index 00000000000..1f93a9632d0 --- /dev/null +++ b/spec/lib/gitlab/usage/metrics/instrumentations/version_metric_spec.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Usage::Metrics::Instrumentations::VersionMetric, feature_category: :service_ping do + let(:expected_value) { Gitlab::VERSION } + + it_behaves_like 'a correct instrumented metric value', { time_frame: 'all', data_source: 'database' } +end diff --git a/spec/lib/gitlab/usage/metrics/names_suggestions/generator_spec.rb b/spec/lib/gitlab/usage/metrics/names_suggestions/generator_spec.rb index 4f647c2700a..271e9595703 100644 --- a/spec/lib/gitlab/usage/metrics/names_suggestions/generator_spec.rb +++ b/spec/lib/gitlab/usage/metrics/names_suggestions/generator_spec.rb @@ -75,7 +75,7 @@ RSpec.describe Gitlab::Usage::Metrics::NamesSuggestions::Generator, feature_cate end end - context 'for redis metrics' do + context 'for redis metrics', quarantine: 'https://gitlab.com/gitlab-org/gitlab/-/issues/399421' do it_behaves_like 'name suggestion' do let(:key_path) { 'usage_activity_by_stage_monthly.create.merge_requests_users' } let(:name_suggestion) { /<please fill metric name, suggested format is: {subject}_{verb}{ing|ed}_{object} eg: users_creating_epics or merge_requests_viewed_in_single_file_mode>/ } diff --git a/spec/lib/gitlab/usage/service_ping_report_spec.rb b/spec/lib/gitlab/usage/service_ping_report_spec.rb index 730c05b7dcb..f1ce48468fe 100644 --- a/spec/lib/gitlab/usage/service_ping_report_spec.rb +++ b/spec/lib/gitlab/usage/service_ping_report_spec.rb @@ -72,25 +72,34 @@ RSpec.describe Gitlab::Usage::ServicePingReport, :use_clean_rails_memory_store_c context 'when using cached' do let(:new_usage_data) { { 'uuid' => '1112' } } + let(:instrumented_payload) { { 'instrumented' => { 'metric' => 1 } } } + let(:full_payload) { usage_data.merge(instrumented_payload) } + let(:new_full_payload) { new_usage_data.merge(instrumented_payload) } + + before do + allow_next_instance_of(Gitlab::Usage::ServicePing::InstrumentedPayload) do |instance| + allow(instance).to receive(:build).and_return(instrumented_payload) + end + end context 'for cached: true' do it 'caches the values' do allow(Gitlab::UsageData).to receive(:data).and_return(usage_data, new_usage_data) - expect(described_class.for(output: :all_metrics_values)).to eq(usage_data) - expect(described_class.for(output: :all_metrics_values, cached: true)).to eq(usage_data) + expect(described_class.for(output: :all_metrics_values)).to eq(full_payload) + expect(described_class.for(output: :all_metrics_values, cached: true)).to eq(full_payload) - expect(Rails.cache.fetch('usage_data')).to eq(usage_data) + expect(Rails.cache.fetch('usage_data')).to eq(full_payload) end it 'writes to cache and returns fresh data' do allow(Gitlab::UsageData).to receive(:data).and_return(usage_data, new_usage_data) - expect(described_class.for(output: :all_metrics_values)).to eq(usage_data) - expect(described_class.for(output: :all_metrics_values)).to eq(new_usage_data) - expect(described_class.for(output: :all_metrics_values, cached: true)).to eq(new_usage_data) + expect(described_class.for(output: :all_metrics_values)).to eq(full_payload) + expect(described_class.for(output: :all_metrics_values)).to eq(new_full_payload) + expect(described_class.for(output: :all_metrics_values, cached: true)).to eq(new_full_payload) - expect(Rails.cache.fetch('usage_data')).to eq(new_usage_data) + expect(Rails.cache.fetch('usage_data')).to eq(new_full_payload) end end @@ -98,10 +107,10 @@ RSpec.describe Gitlab::Usage::ServicePingReport, :use_clean_rails_memory_store_c it 'returns fresh data' do allow(Gitlab::UsageData).to receive(:data).and_return(usage_data, new_usage_data) - expect(described_class.for(output: :all_metrics_values)).to eq(usage_data) - expect(described_class.for(output: :all_metrics_values)).to eq(new_usage_data) + expect(described_class.for(output: :all_metrics_values)).to eq(full_payload) + expect(described_class.for(output: :all_metrics_values)).to eq(new_full_payload) - expect(Rails.cache.fetch('usage_data')).to eq(new_usage_data) + expect(Rails.cache.fetch('usage_data')).to eq(new_full_payload) end end end diff --git a/spec/lib/gitlab/usage_data_counters/code_review_events_spec.rb b/spec/lib/gitlab/usage_data_counters/code_review_events_spec.rb index 63a1da490ed..8da86e4fae5 100644 --- a/spec/lib/gitlab/usage_data_counters/code_review_events_spec.rb +++ b/spec/lib/gitlab/usage_data_counters/code_review_events_spec.rb @@ -6,17 +6,23 @@ require 'spec_helper' # NOTE: ONLY user related metrics to be added to the aggregates - otherwise add it to the exception list RSpec.describe 'Code review events' do it 'the aggregated metrics contain all the code review metrics' do - code_review_events = Gitlab::UsageDataCounters::HLLRedisCounter.events_for_category("code_review") + mr_related_events = %w[i_code_review_create_mr i_code_review_mr_diffs i_code_review_mr_with_invalid_approvers i_code_review_mr_single_file_diffs i_code_review_total_suggestions_applied i_code_review_total_suggestions_added i_code_review_create_note_in_ipynb_diff i_code_review_create_note_in_ipynb_diff_mr i_code_review_create_note_in_ipynb_diff_commit i_code_review_merge_request_widget_license_compliance_warning] + + all_code_review_events = Gitlab::Usage::MetricDefinition.all.flat_map do |definition| + next [] unless definition.attributes[:key_path].include?('.code_review.') && + definition.attributes[:status] == 'active' && + definition.attributes[:instrumentation_class] != 'AggregatedMetric' + + definition.attributes.dig(:options, :events) + end.uniq.compact + code_review_aggregated_events = Gitlab::Usage::MetricDefinition.all.flat_map do |definition| next [] unless code_review_aggregated_metric?(definition.attributes) definition.attributes.dig(:options, :events) end.uniq - exceptions = %w[i_code_review_create_mr i_code_review_mr_diffs i_code_review_mr_with_invalid_approvers i_code_review_mr_single_file_diffs i_code_review_total_suggestions_applied i_code_review_total_suggestions_added i_code_review_create_note_in_ipynb_diff i_code_review_create_note_in_ipynb_diff_mr i_code_review_create_note_in_ipynb_diff_commit] - code_review_aggregated_events += exceptions - - expect(code_review_events - code_review_aggregated_events).to be_empty + expect(all_code_review_events - (code_review_aggregated_events + mr_related_events)).to be_empty end def code_review_aggregated_metric?(attributes) diff --git a/spec/lib/gitlab/usage_data_counters/container_registry_event_counter_spec.rb b/spec/lib/gitlab/usage_data_counters/container_registry_event_counter_spec.rb new file mode 100644 index 00000000000..052735db96b --- /dev/null +++ b/spec/lib/gitlab/usage_data_counters/container_registry_event_counter_spec.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::UsageDataCounters::ContainerRegistryEventCounter, :clean_gitlab_redis_shared_state, + feature_category: :container_registry do + described_class::KNOWN_EVENTS.each do |event| + it_behaves_like 'a redis usage counter', 'ContainerRegistryEvent', event + it_behaves_like 'a redis usage counter with totals', :container_registry_events, "#{event}": 5 + end +end diff --git a/spec/lib/gitlab/usage_data_counters/editor_unique_counter_spec.rb b/spec/lib/gitlab/usage_data_counters/editor_unique_counter_spec.rb index f8a4603c1f8..19236cdbba0 100644 --- a/spec/lib/gitlab/usage_data_counters/editor_unique_counter_spec.rb +++ b/spec/lib/gitlab/usage_data_counters/editor_unique_counter_spec.rb @@ -18,19 +18,35 @@ RSpec.describe Gitlab::UsageDataCounters::EditorUniqueCounter, :clean_gitlab_red aggregate_failures do expect(track_action(author: user1, project: project)).to be_truthy expect(track_action(author: user2, project: project)).to be_truthy - expect(track_action(author: user3, time: time - 3.days, project: project)).to be_truthy + expect(track_action(author: user3, time: time.end_of_week - 3.days, project: project)).to be_truthy - expect(count_unique(date_from: time, date_to: Date.today)).to eq(2) - expect(count_unique(date_from: time - 5.days, date_to: Date.tomorrow)).to eq(3) + expect(count_unique(date_from: time.beginning_of_week, date_to: 1.week.from_now)).to eq(3) end end + it 'track snowplow event' do + track_action(author: user1, project: project) + + expect_snowplow_event( + category: described_class.name, + action: 'ide_edit', + label: 'usage_activity_by_stage_monthly.create.action_monthly_active_users_ide_edit', + namespace: project.namespace, + property: event_name, + project: project, + user: user1, + context: [Gitlab::Tracking::ServicePingContext.new(data_source: :redis_hll, event: event_name).to_h] + ) + end + it 'does not track edit actions if author is not present' do expect(track_action(author: nil, project: project)).to be_nil end end context 'for web IDE edit actions' do + let(:event_name) { described_class::EDIT_BY_WEB_IDE } + it_behaves_like 'tracks and counts action' do def track_action(params) described_class.track_web_ide_edit_action(**params) @@ -43,6 +59,8 @@ RSpec.describe Gitlab::UsageDataCounters::EditorUniqueCounter, :clean_gitlab_red end context 'for SFE edit actions' do + let(:event_name) { described_class::EDIT_BY_SFE } + it_behaves_like 'tracks and counts action' do def track_action(params) described_class.track_sfe_edit_action(**params) @@ -55,6 +73,8 @@ RSpec.describe Gitlab::UsageDataCounters::EditorUniqueCounter, :clean_gitlab_red end context 'for snippet editor edit actions' do + let(:event_name) { described_class::EDIT_BY_SNIPPET_EDITOR } + it_behaves_like 'tracks and counts action' do def track_action(params) described_class.track_snippet_editor_edit_action(**params) diff --git a/spec/lib/gitlab/usage_data_counters/gitlab_cli_activity_unique_counter_spec.rb b/spec/lib/gitlab/usage_data_counters/gitlab_cli_activity_unique_counter_spec.rb index d6eb67e5c35..9cbac835a6f 100644 --- a/spec/lib/gitlab/usage_data_counters/gitlab_cli_activity_unique_counter_spec.rb +++ b/spec/lib/gitlab/usage_data_counters/gitlab_cli_activity_unique_counter_spec.rb @@ -7,9 +7,18 @@ RSpec.describe Gitlab::UsageDataCounters::GitLabCliActivityUniqueCounter, :clean let(:user2) { build(:user, id: 2) } let(:time) { Time.current } let(:action) { described_class::GITLAB_CLI_API_REQUEST_ACTION } - let(:user_agent) { { user_agent: 'GLab - GitLab CLI' } } context 'when tracking a gitlab cli request' do - it_behaves_like 'a request from an extension' + context 'with the old UserAgent' do + let(:user_agent) { { user_agent: 'GLab - GitLab CLI' } } + + it_behaves_like 'a request from an extension' + end + + context 'with the current UserAgent' do + let(:user_agent) { { user_agent: 'glab/v1.25.3-27-g7ec258fb (built 2023-02-16), darwin' } } + + it_behaves_like 'a request from an extension' + end end end diff --git a/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb b/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb index f955fd265e5..2bf4c8bfca9 100644 --- a/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb +++ b/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb @@ -23,36 +23,69 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s described_class.clear_memoization(:known_events) end - describe '.categories' do - it 'gets CE unique category names' do - expect(described_class.categories).to include( - 'analytics', - 'ci_templates', - 'ci_users', - 'code_review', - 'deploy_token_packages', - 'ecosystem', - 'environments', - 'error_tracking', - 'geo', - 'ide_edit', - 'importer', - 'incident_management_alerts', - 'incident_management', - 'issues_edit', - 'kubernetes_agent', - 'manage', - 'pipeline_authoring', - 'quickactions', - 'search', - 'secure', - 'snippets', - 'source_code', - 'terraform', - 'testing', - 'user_packages', - 'work_items' - ) + describe '.track_event' do + # ToDo: remove during https://gitlab.com/groups/gitlab-org/-/epics/9542 cleanup + describe 'daily to weekly key migration precautions' do + let(:event_name) { 'example_event' } + let(:known_events) do + [ + { name: event_name, aggregation: 'daily' } + ].map(&:with_indifferent_access) + end + + let(:start_date) { (Date.current - 1.week).beginning_of_week } + let(:end_date) { Date.current } + + let(:daily_event) { known_events.first } + let(:daily_key) { described_class.send(:redis_key, daily_event, start_date) } + let(:weekly_key) do + weekly_event = known_events.first.merge(aggregation: 'weekly') + described_class.send(:redis_key, weekly_event, start_date) + end + + before do + allow(described_class).to receive(:known_events).and_return(known_events) + end + + shared_examples 'writes daily events to daily and weekly keys' do + it :aggregate_failures do + expect(Gitlab::Redis::HLL).to receive(:add).with(expiry: 29.days, key: daily_key, value: 1).and_call_original + expect(Gitlab::Redis::HLL).to receive(:add).with(expiry: 6.weeks, key: weekly_key, value: 1).and_call_original + + described_class.track_event(event_name, values: 1, time: start_date) + end + end + + context 'when revert_daily_hll_events_to_weekly_aggregation FF is disabled' do + before do + stub_feature_flags(revert_daily_hll_events_to_weekly_aggregation: false) + end + + it_behaves_like 'writes daily events to daily and weekly keys' + + it 'aggregates weekly for daily keys', :aggregate_failures do + expect(Gitlab::Redis::HLL).to receive(:count).with(keys: [weekly_key]).and_call_original + expect(Gitlab::Redis::HLL).not_to receive(:count).with(keys: [daily_key]).and_call_original + + described_class.unique_events(event_names: [event_name], start_date: start_date, end_date: end_date) + end + end + + context 'when revert_daily_hll_events_to_weekly_aggregation FF is enabled' do + before do + stub_feature_flags(revert_daily_hll_events_to_weekly_aggregation: true) + end + + # we want to write events no matter of the feature state + it_behaves_like 'writes daily events to daily and weekly keys' + + it 'aggregates daily for daily keys', :aggregate_failures do + expect(Gitlab::Redis::HLL).to receive(:count).with(keys: [daily_key]).and_call_original + expect(Gitlab::Redis::HLL).not_to receive(:count).with(keys: [weekly_key]).and_call_original + + described_class.unique_events(event_names: [event_name], start_date: start_date, end_date: start_date) + end + end end end @@ -62,8 +95,6 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s let(:ce_event) do { "name" => "ce_event", - "redis_slot" => "analytics", - "category" => "analytics", "aggregation" => "weekly" } end @@ -84,8 +115,6 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s end describe 'known_events' do - let(:feature) { 'test_hll_redis_counter_ff_check' } - let(:weekly_event) { 'g_analytics_contribution' } let(:daily_event) { 'g_analytics_search' } let(:analytics_slot_event) { 'g_analytics_contribution' } @@ -105,13 +134,13 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s let(:known_events) do [ - { name: weekly_event, redis_slot: "analytics", category: analytics_category, aggregation: "weekly", feature_flag: feature }, - { name: daily_event, redis_slot: "analytics", category: analytics_category, aggregation: "daily" }, - { name: category_productivity_event, redis_slot: "analytics", category: productivity_category, aggregation: "weekly" }, - { name: compliance_slot_event, redis_slot: "compliance", category: compliance_category, aggregation: "weekly" }, - { name: no_slot, category: global_category, aggregation: "daily" }, - { name: different_aggregation, category: global_category, aggregation: "monthly" }, - { name: context_event, category: other_category, aggregation: 'weekly' } + { name: weekly_event, aggregation: "weekly" }, + { name: daily_event, aggregation: "daily" }, + { name: category_productivity_event, aggregation: "weekly" }, + { name: compliance_slot_event, aggregation: "weekly" }, + { name: no_slot, aggregation: "daily" }, + { name: different_aggregation, aggregation: "monthly" }, + { name: context_event, aggregation: 'weekly' } ].map(&:with_indifferent_access) end @@ -121,12 +150,6 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s allow(described_class).to receive(:known_events).and_return(known_events) end - describe '.events_for_category' do - it 'gets the event names for given category' do - expect(described_class.events_for_category(:analytics)).to contain_exactly(weekly_event, daily_event) - end - end - describe '.track_event' do context 'with redis_hll_tracking' do it 'tracks the event when feature enabled' do @@ -146,32 +169,6 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s end end - context 'with event feature flag set' do - it 'tracks the event when feature enabled' do - stub_feature_flags(feature => true) - - expect(Gitlab::Redis::HLL).to receive(:add) - - described_class.track_event(weekly_event, values: 1) - end - - it 'does not track the event with feature flag disabled' do - stub_feature_flags(feature => false) - - expect(Gitlab::Redis::HLL).not_to receive(:add) - - described_class.track_event(weekly_event, values: 1) - end - end - - context 'with no event feature flag set' do - it 'tracks the event' do - expect(Gitlab::Redis::HLL).to receive(:add) - - described_class.track_event(daily_event, values: 1) - end - end - context 'when usage_ping is disabled' do it 'does not track the event' do allow(::ServicePing::ServicePingSettings).to receive(:enabled?).and_return(false) @@ -195,7 +192,7 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s it 'tracks events with multiple values' do values = [entity1, entity2] - expect(Gitlab::Redis::HLL).to receive(:add).with(key: /g_{analytics}_contribution/, value: values, + expect(Gitlab::Redis::HLL).to receive(:add).with(key: /g_analytics_contribution/, value: values, expiry: described_class::DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH) described_class.track_event(:g_analytics_contribution, values: values) @@ -237,7 +234,7 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s described_class.track_event("g_compliance_dashboard", values: entity1) Gitlab::Redis::SharedState.with do |redis| - keys = redis.scan_each(match: "g_{compliance}_dashboard-*").to_a + keys = redis.scan_each(match: "{#{described_class::REDIS_SLOT}}_g_compliance_dashboard-*").to_a expect(keys).not_to be_empty keys.each do |key| @@ -252,7 +249,7 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s described_class.track_event("no_slot", values: entity1) Gitlab::Redis::SharedState.with do |redis| - keys = redis.scan_each(match: "*-{no_slot}").to_a + keys = redis.scan_each(match: "*_no_slot").to_a expect(keys).not_to be_empty keys.each do |key| @@ -276,7 +273,7 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s it 'tracks events with multiple values' do values = [entity1, entity2] - expect(Gitlab::Redis::HLL).to receive(:add).with(key: /g_{analytics}_contribution/, + expect(Gitlab::Redis::HLL).to receive(:add).with(key: /g_analytics_contribution/, value: values, expiry: described_class::DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH) @@ -340,18 +337,6 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s expect(described_class.unique_events(event_names: [weekly_event], start_date: Date.current, end_date: 4.weeks.ago)).to eq(-1) end - it 'raise error if metrics are not in the same slot' do - expect do - described_class.unique_events(event_names: [compliance_slot_event, analytics_slot_event], start_date: 4.weeks.ago, end_date: Date.current) - end.to raise_error(Gitlab::UsageDataCounters::HLLRedisCounter::SlotMismatch) - end - - it 'raise error if metrics are not in the same category' do - expect do - described_class.unique_events(event_names: [category_analytics_event, category_productivity_event], start_date: 4.weeks.ago, end_date: Date.current) - end.to raise_error(Gitlab::UsageDataCounters::HLLRedisCounter::CategoryMismatch) - end - it "raise error if metrics don't have same aggregation" do expect do described_class.unique_events(event_names: [daily_event, weekly_event], start_date: 4.weeks.ago, end_date: Date.current) @@ -398,6 +383,10 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s let(:weekly_event) { 'i_search_total' } let(:redis_event) { described_class.send(:event_for, weekly_event) } + let(:week_one) { "{#{described_class::REDIS_SLOT}}_i_search_total-2020-52" } + let(:week_two) { "{#{described_class::REDIS_SLOT}}_i_search_total-2020-53" } + let(:week_three) { "{#{described_class::REDIS_SLOT}}_i_search_total-2021-01" } + let(:week_four) { "{#{described_class::REDIS_SLOT}}_i_search_total-2021-02" } subject(:weekly_redis_keys) { described_class.send(:weekly_redis_keys, events: [redis_event], start_date: DateTime.parse(start_date), end_date: DateTime.parse(end_date)) } @@ -406,13 +395,13 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s '2020-12-21' | '2020-12-20' | [] '2020-12-21' | '2020-11-21' | [] '2021-01-01' | '2020-12-28' | [] - '2020-12-21' | '2020-12-28' | ['i_{search}_total-2020-52'] - '2020-12-21' | '2021-01-01' | ['i_{search}_total-2020-52'] - '2020-12-27' | '2021-01-01' | ['i_{search}_total-2020-52'] - '2020-12-26' | '2021-01-04' | ['i_{search}_total-2020-52', 'i_{search}_total-2020-53'] - '2020-12-26' | '2021-01-11' | ['i_{search}_total-2020-52', 'i_{search}_total-2020-53', 'i_{search}_total-2021-01'] - '2020-12-26' | '2021-01-17' | ['i_{search}_total-2020-52', 'i_{search}_total-2020-53', 'i_{search}_total-2021-01'] - '2020-12-26' | '2021-01-18' | ['i_{search}_total-2020-52', 'i_{search}_total-2020-53', 'i_{search}_total-2021-01', 'i_{search}_total-2021-02'] + '2020-12-21' | '2020-12-28' | lazy { [week_one] } + '2020-12-21' | '2021-01-01' | lazy { [week_one] } + '2020-12-27' | '2021-01-01' | lazy { [week_one] } + '2020-12-26' | '2021-01-04' | lazy { [week_one, week_two] } + '2020-12-26' | '2021-01-11' | lazy { [week_one, week_two, week_three] } + '2020-12-26' | '2021-01-17' | lazy { [week_one, week_two, week_three] } + '2020-12-26' | '2021-01-18' | lazy { [week_one, week_two, week_three, week_four] } end with_them do @@ -435,9 +424,9 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s let(:known_events) do [ - { name: 'event_name_1', redis_slot: 'event', category: 'category1', aggregation: "weekly" }, - { name: 'event_name_2', redis_slot: 'event', category: 'category1', aggregation: "weekly" }, - { name: 'event_name_3', redis_slot: 'event', category: 'category1', aggregation: "weekly" } + { name: 'event_name_1', aggregation: "weekly" }, + { name: 'event_name_2', aggregation: "weekly" }, + { name: 'event_name_3', aggregation: "weekly" } ].map(&:with_indifferent_access) end @@ -476,11 +465,11 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s let(:time_range) { { start_date: 7.days.ago, end_date: DateTime.current } } let(:known_events) do [ - { name: 'event1_slot', redis_slot: "slot", category: 'category1', aggregation: "weekly" }, - { name: 'event2_slot', redis_slot: "slot", category: 'category2', aggregation: "weekly" }, - { name: 'event3_slot', redis_slot: "slot", category: 'category3', aggregation: "weekly" }, - { name: 'event5_slot', redis_slot: "slot", category: 'category4', aggregation: "daily" }, - { name: 'event4', category: 'category2', aggregation: "weekly" } + { name: 'event1_slot', aggregation: "weekly" }, + { name: 'event2_slot', aggregation: "weekly" }, + { name: 'event3_slot', aggregation: "weekly" }, + { name: 'event5_slot', aggregation: "daily" }, + { name: 'event4', aggregation: "weekly" } ].map(&:with_indifferent_access) end @@ -505,16 +494,11 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s described_class.track_event('event4', values: entity2, time: 2.days.ago) end - it 'calculates union of given events', :aggregate_failure do + it 'calculates union of given events', :aggregate_failures do expect(described_class.calculate_events_union(**time_range.merge(event_names: %w[event4]))).to eq 2 expect(described_class.calculate_events_union(**time_range.merge(event_names: %w[event1_slot event2_slot event3_slot]))).to eq 3 end - it 'validates and raise exception if events has mismatched slot or aggregation', :aggregate_failure do - expect { described_class.calculate_events_union(**time_range.merge(event_names: %w[event1_slot event4])) }.to raise_error described_class::SlotMismatch - expect { described_class.calculate_events_union(**time_range.merge(event_names: %w[event5_slot event3_slot])) }.to raise_error described_class::AggregationMismatch - end - it 'returns 0 if there are no keys for given events' do expect(Gitlab::Redis::HLL).not_to receive(:count) expect(described_class.calculate_events_union(event_names: %w[event1_slot event2_slot event3_slot], start_date: Date.current, end_date: 4.weeks.ago)).to eq(-1) diff --git a/spec/lib/gitlab/usage_data_counters/issue_activity_unique_counter_spec.rb b/spec/lib/gitlab/usage_data_counters/issue_activity_unique_counter_spec.rb index 33e0d446fca..ba83d979cad 100644 --- a/spec/lib/gitlab/usage_data_counters/issue_activity_unique_counter_spec.rb +++ b/spec/lib/gitlab/usage_data_counters/issue_activity_unique_counter_spec.rb @@ -6,16 +6,17 @@ RSpec.describe Gitlab::UsageDataCounters::IssueActivityUniqueCounter, :clean_git let_it_be(:user1) { build(:user, id: 1) } let_it_be(:user2) { build(:user, id: 2) } let_it_be(:user3) { build(:user, id: 3) } - let_it_be(:project) { build(:project) } + let_it_be(:project) { create(:project) } let_it_be(:category) { Gitlab::UsageDataCounters::IssueActivityUniqueCounter::ISSUE_CATEGORY } let_it_be(:event_action) { Gitlab::UsageDataCounters::IssueActivityUniqueCounter::ISSUE_ACTION } let_it_be(:event_label) { Gitlab::UsageDataCounters::IssueActivityUniqueCounter::ISSUE_LABEL } + let(:original_params) { nil } let(:event_property) { action } let(:time) { Time.zone.now } context 'for Issue title edit actions' do - it_behaves_like 'daily tracked issuable snowplow and service ping events with project' do + it_behaves_like 'tracked issuable snowplow and service ping events with project' do let(:action) { described_class::ISSUE_TITLE_CHANGED } def track_action(params) @@ -25,7 +26,7 @@ RSpec.describe Gitlab::UsageDataCounters::IssueActivityUniqueCounter, :clean_git end context 'for Issue description edit actions' do - it_behaves_like 'daily tracked issuable snowplow and service ping events with project' do + it_behaves_like 'tracked issuable snowplow and service ping events with project' do let(:action) { described_class::ISSUE_DESCRIPTION_CHANGED } def track_action(params) @@ -35,7 +36,7 @@ RSpec.describe Gitlab::UsageDataCounters::IssueActivityUniqueCounter, :clean_git end context 'for Issue assignee edit actions' do - it_behaves_like 'daily tracked issuable snowplow and service ping events with project' do + it_behaves_like 'tracked issuable snowplow and service ping events with project' do let(:action) { described_class::ISSUE_ASSIGNEE_CHANGED } def track_action(params) @@ -45,7 +46,7 @@ RSpec.describe Gitlab::UsageDataCounters::IssueActivityUniqueCounter, :clean_git end context 'for Issue make confidential actions' do - it_behaves_like 'daily tracked issuable snowplow and service ping events with project' do + it_behaves_like 'tracked issuable snowplow and service ping events with project' do let(:action) { described_class::ISSUE_MADE_CONFIDENTIAL } def track_action(params) @@ -55,7 +56,7 @@ RSpec.describe Gitlab::UsageDataCounters::IssueActivityUniqueCounter, :clean_git end context 'for Issue make visible actions' do - it_behaves_like 'daily tracked issuable snowplow and service ping events with project' do + it_behaves_like 'tracked issuable snowplow and service ping events with project' do let(:action) { described_class::ISSUE_MADE_VISIBLE } def track_action(params) @@ -65,8 +66,9 @@ RSpec.describe Gitlab::UsageDataCounters::IssueActivityUniqueCounter, :clean_git end context 'for Issue created actions' do - it_behaves_like 'daily tracked issuable snowplow and service ping events with project' do + it_behaves_like 'tracked issuable snowplow and service ping events with project' do let(:action) { described_class::ISSUE_CREATED } + let(:original_params) { { namespace: project.project_namespace.reload } } def track_action(params) described_class.track_issue_created_action(**params) @@ -75,7 +77,7 @@ RSpec.describe Gitlab::UsageDataCounters::IssueActivityUniqueCounter, :clean_git end context 'for Issue closed actions' do - it_behaves_like 'daily tracked issuable snowplow and service ping events with project' do + it_behaves_like 'tracked issuable snowplow and service ping events with project' do let(:action) { described_class::ISSUE_CLOSED } def track_action(params) @@ -85,7 +87,7 @@ RSpec.describe Gitlab::UsageDataCounters::IssueActivityUniqueCounter, :clean_git end context 'for Issue reopened actions' do - it_behaves_like 'daily tracked issuable snowplow and service ping events with project' do + it_behaves_like 'tracked issuable snowplow and service ping events with project' do let(:action) { described_class::ISSUE_REOPENED } def track_action(params) @@ -95,7 +97,7 @@ RSpec.describe Gitlab::UsageDataCounters::IssueActivityUniqueCounter, :clean_git end context 'for Issue label changed actions' do - it_behaves_like 'daily tracked issuable snowplow and service ping events with project' do + it_behaves_like 'tracked issuable snowplow and service ping events with project' do let(:action) { described_class::ISSUE_LABEL_CHANGED } def track_action(params) @@ -105,7 +107,7 @@ RSpec.describe Gitlab::UsageDataCounters::IssueActivityUniqueCounter, :clean_git end context 'for Issue label milestone actions' do - it_behaves_like 'daily tracked issuable snowplow and service ping events with project' do + it_behaves_like 'tracked issuable snowplow and service ping events with project' do let(:action) { described_class::ISSUE_MILESTONE_CHANGED } def track_action(params) @@ -115,7 +117,7 @@ RSpec.describe Gitlab::UsageDataCounters::IssueActivityUniqueCounter, :clean_git end context 'for Issue cross-referenced actions' do - it_behaves_like 'daily tracked issuable snowplow and service ping events with project' do + it_behaves_like 'tracked issuable snowplow and service ping events with project' do let(:action) { described_class::ISSUE_CROSS_REFERENCED } def track_action(params) @@ -125,7 +127,7 @@ RSpec.describe Gitlab::UsageDataCounters::IssueActivityUniqueCounter, :clean_git end context 'for Issue moved actions' do - it_behaves_like 'daily tracked issuable snowplow and service ping events with project' do + it_behaves_like 'tracked issuable snowplow and service ping events with project' do let(:action) { described_class::ISSUE_MOVED } def track_action(params) @@ -135,7 +137,7 @@ RSpec.describe Gitlab::UsageDataCounters::IssueActivityUniqueCounter, :clean_git end context 'for Issue cloned actions' do - it_behaves_like 'daily tracked issuable snowplow and service ping events with project' do + it_behaves_like 'tracked issuable snowplow and service ping events with project' do let_it_be(:action) { described_class::ISSUE_CLONED } def track_action(params) @@ -145,7 +147,7 @@ RSpec.describe Gitlab::UsageDataCounters::IssueActivityUniqueCounter, :clean_git end context 'for Issue relate actions' do - it_behaves_like 'daily tracked issuable snowplow and service ping events with project' do + it_behaves_like 'tracked issuable snowplow and service ping events with project' do let(:action) { described_class::ISSUE_RELATED } def track_action(params) @@ -155,7 +157,7 @@ RSpec.describe Gitlab::UsageDataCounters::IssueActivityUniqueCounter, :clean_git end context 'for Issue unrelate actions' do - it_behaves_like 'daily tracked issuable snowplow and service ping events with project' do + it_behaves_like 'tracked issuable snowplow and service ping events with project' do let(:action) { described_class::ISSUE_UNRELATED } def track_action(params) @@ -165,7 +167,7 @@ RSpec.describe Gitlab::UsageDataCounters::IssueActivityUniqueCounter, :clean_git end context 'for Issue marked as duplicate actions' do - it_behaves_like 'daily tracked issuable snowplow and service ping events with project' do + it_behaves_like 'tracked issuable snowplow and service ping events with project' do let(:action) { described_class::ISSUE_MARKED_AS_DUPLICATE } def track_action(params) @@ -175,7 +177,7 @@ RSpec.describe Gitlab::UsageDataCounters::IssueActivityUniqueCounter, :clean_git end context 'for Issue locked actions' do - it_behaves_like 'daily tracked issuable snowplow and service ping events with project' do + it_behaves_like 'tracked issuable snowplow and service ping events with project' do let(:action) { described_class::ISSUE_LOCKED } def track_action(params) @@ -185,7 +187,7 @@ RSpec.describe Gitlab::UsageDataCounters::IssueActivityUniqueCounter, :clean_git end context 'for Issue unlocked actions' do - it_behaves_like 'daily tracked issuable snowplow and service ping events with project' do + it_behaves_like 'tracked issuable snowplow and service ping events with project' do let(:action) { described_class::ISSUE_UNLOCKED } def track_action(params) @@ -195,7 +197,7 @@ RSpec.describe Gitlab::UsageDataCounters::IssueActivityUniqueCounter, :clean_git end context 'for Issue designs added actions' do - it_behaves_like 'daily tracked issuable snowplow and service ping events with project' do + it_behaves_like 'tracked issuable snowplow and service ping events with project' do let(:action) { described_class::ISSUE_DESIGNS_ADDED } def track_action(params) @@ -205,7 +207,7 @@ RSpec.describe Gitlab::UsageDataCounters::IssueActivityUniqueCounter, :clean_git end context 'for Issue designs modified actions' do - it_behaves_like 'daily tracked issuable snowplow and service ping events with project' do + it_behaves_like 'tracked issuable snowplow and service ping events with project' do let(:action) { described_class::ISSUE_DESIGNS_MODIFIED } def track_action(params) @@ -215,7 +217,7 @@ RSpec.describe Gitlab::UsageDataCounters::IssueActivityUniqueCounter, :clean_git end context 'for Issue designs removed actions' do - it_behaves_like 'daily tracked issuable snowplow and service ping events with project' do + it_behaves_like 'tracked issuable snowplow and service ping events with project' do let(:action) { described_class::ISSUE_DESIGNS_REMOVED } def track_action(params) @@ -225,7 +227,7 @@ RSpec.describe Gitlab::UsageDataCounters::IssueActivityUniqueCounter, :clean_git end context 'for Issue due date changed actions' do - it_behaves_like 'daily tracked issuable snowplow and service ping events with project' do + it_behaves_like 'tracked issuable snowplow and service ping events with project' do let(:action) { described_class::ISSUE_DUE_DATE_CHANGED } def track_action(params) @@ -235,7 +237,7 @@ RSpec.describe Gitlab::UsageDataCounters::IssueActivityUniqueCounter, :clean_git end context 'for Issue time estimate changed actions' do - it_behaves_like 'daily tracked issuable snowplow and service ping events with project' do + it_behaves_like 'tracked issuable snowplow and service ping events with project' do let(:action) { described_class::ISSUE_TIME_ESTIMATE_CHANGED } def track_action(params) @@ -245,7 +247,7 @@ RSpec.describe Gitlab::UsageDataCounters::IssueActivityUniqueCounter, :clean_git end context 'for Issue time spent changed actions' do - it_behaves_like 'daily tracked issuable snowplow and service ping events with project' do + it_behaves_like 'tracked issuable snowplow and service ping events with project' do let(:action) { described_class::ISSUE_TIME_SPENT_CHANGED } def track_action(params) @@ -255,7 +257,7 @@ RSpec.describe Gitlab::UsageDataCounters::IssueActivityUniqueCounter, :clean_git end context 'for Issue comment added actions', :snowplow do - it_behaves_like 'daily tracked issuable snowplow and service ping events with project' do + it_behaves_like 'tracked issuable snowplow and service ping events with project' do let(:action) { described_class::ISSUE_COMMENT_ADDED } def track_action(params) @@ -265,7 +267,7 @@ RSpec.describe Gitlab::UsageDataCounters::IssueActivityUniqueCounter, :clean_git end context 'for Issue comment edited actions', :snowplow do - it_behaves_like 'daily tracked issuable snowplow and service ping events with project' do + it_behaves_like 'tracked issuable snowplow and service ping events with project' do let(:action) { described_class::ISSUE_COMMENT_EDITED } def track_action(params) @@ -275,7 +277,7 @@ RSpec.describe Gitlab::UsageDataCounters::IssueActivityUniqueCounter, :clean_git end context 'for Issue comment removed actions', :snowplow do - it_behaves_like 'daily tracked issuable snowplow and service ping events with project' do + it_behaves_like 'tracked issuable snowplow and service ping events with project' do let(:action) { described_class::ISSUE_COMMENT_REMOVED } def track_action(params) @@ -285,7 +287,7 @@ RSpec.describe Gitlab::UsageDataCounters::IssueActivityUniqueCounter, :clean_git end context 'for Issue design comment removed actions' do - it_behaves_like 'daily tracked issuable snowplow and service ping events with project' do + it_behaves_like 'tracked issuable snowplow and service ping events with project' do let(:action) { described_class::ISSUE_DESIGN_COMMENT_REMOVED } def track_action(params) @@ -294,23 +296,24 @@ RSpec.describe Gitlab::UsageDataCounters::IssueActivityUniqueCounter, :clean_git end end - it 'can return the count of actions per user deduplicated', :aggregate_failures do - described_class.track_issue_title_changed_action(author: user1, project: project) - described_class.track_issue_description_changed_action(author: user1, project: project) - described_class.track_issue_assignee_changed_action(author: user1, project: project) + it 'can return the count of actions per user deduplicated' do + travel_to(Date.today.beginning_of_week) do # because events aggregated by week we need to emit events in the same week + described_class.track_issue_title_changed_action(author: user1, project: project) + described_class.track_issue_description_changed_action(author: user1, project: project) + described_class.track_issue_assignee_changed_action(author: user1, project: project) + end - travel_to(2.days.ago) do + travel_to(Date.today.beginning_of_week + 2.days) do described_class.track_issue_title_changed_action(author: user2, project: project) described_class.track_issue_title_changed_action(author: user3, project: project) described_class.track_issue_description_changed_action(author: user3, project: project) described_class.track_issue_assignee_changed_action(author: user3, project: project) end - events = Gitlab::UsageDataCounters::HLLRedisCounter.events_for_category(described_class::ISSUE_CATEGORY) - today_count = Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: events, start_date: time, end_date: time) - week_count = Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: events, start_date: time - 5.days, end_date: 1.day.since(time)) + events = [described_class::ISSUE_TITLE_CHANGED, described_class::ISSUE_DESCRIPTION_CHANGED, described_class::ISSUE_ASSIGNEE_CHANGED] + week_count = Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: events, start_date: time.beginning_of_week, + end_date: time + 1.week) - expect(today_count).to eq(1) expect(week_count).to eq(3) end end diff --git a/spec/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter_spec.rb b/spec/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter_spec.rb index 42aa84c2c3e..e41da6d9ea2 100644 --- a/spec/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter_spec.rb +++ b/spec/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter_spec.rb @@ -69,7 +69,6 @@ RSpec.describe Gitlab::UsageDataCounters::MergeRequestActivityUniqueCounter, :cl let(:project) { target_project } let(:namespace) { project.namespace.reload } let(:user) { project.creator } - let(:feature_flag_name) { :route_hll_to_snowplow_phase2 } let(:label) { 'redis_hll_counters.code_review.i_code_review_user_create_mr_monthly' } let(:property) { described_class::MR_USER_CREATE_ACTION } end @@ -118,7 +117,6 @@ RSpec.describe Gitlab::UsageDataCounters::MergeRequestActivityUniqueCounter, :cl let(:project) { target_project } let(:namespace) { project.namespace.reload } let(:user) { project.creator } - let(:feature_flag_name) { :route_hll_to_snowplow_phase2 } let(:label) { 'redis_hll_counters.code_review.i_code_review_user_approve_mr_monthly' } let(:property) { described_class::MR_APPROVE_ACTION } end diff --git a/spec/lib/gitlab/usage_data_counters/track_unique_events_spec.rb b/spec/lib/gitlab/usage_data_counters/track_unique_events_spec.rb deleted file mode 100644 index d1144dd0bc5..00000000000 --- a/spec/lib/gitlab/usage_data_counters/track_unique_events_spec.rb +++ /dev/null @@ -1,72 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::UsageDataCounters::TrackUniqueEvents, :clean_gitlab_redis_shared_state do - subject(:track_unique_events) { described_class } - - let(:time) { Time.zone.now } - - def track_event(params) - track_unique_events.track_event(**params) - end - - def count_unique(params) - track_unique_events.count_unique_events(**params) - end - - context 'tracking an event' do - context 'when tracking successfully' do - context 'when the application setting is enabled' do - context 'when the target and the action is valid' do - before do - stub_application_setting(usage_ping_enabled: true) - end - - it 'tracks and counts the events as expected' do - project = Event::TARGET_TYPES[:project] - design = Event::TARGET_TYPES[:design] - wiki = Event::TARGET_TYPES[:wiki] - - expect(track_event(event_action: :pushed, event_target: project, author_id: 1)).to be_truthy - expect(track_event(event_action: :pushed, event_target: project, author_id: 1)).to be_truthy - expect(track_event(event_action: :pushed, event_target: project, author_id: 2)).to be_truthy - expect(track_event(event_action: :pushed, event_target: project, author_id: 3)).to be_truthy - expect(track_event(event_action: :pushed, event_target: project, author_id: 4, time: time - 3.days)).to be_truthy - - expect(track_event(event_action: :destroyed, event_target: design, author_id: 3)).to be_truthy - expect(track_event(event_action: :created, event_target: design, author_id: 4)).to be_truthy - expect(track_event(event_action: :updated, event_target: design, author_id: 5)).to be_truthy - - expect(track_event(event_action: :destroyed, event_target: wiki, author_id: 5)).to be_truthy - expect(track_event(event_action: :created, event_target: wiki, author_id: 3)).to be_truthy - expect(track_event(event_action: :updated, event_target: wiki, author_id: 4)).to be_truthy - - expect(count_unique(event_action: described_class::PUSH_ACTION, date_from: time, date_to: Date.today)).to eq(3) - expect(count_unique(event_action: described_class::PUSH_ACTION, date_from: time - 5.days, date_to: Date.tomorrow)).to eq(4) - expect(count_unique(event_action: described_class::DESIGN_ACTION, date_from: time - 5.days, date_to: Date.today)).to eq(3) - expect(count_unique(event_action: described_class::WIKI_ACTION, date_from: time - 5.days, date_to: Date.today)).to eq(3) - expect(count_unique(event_action: described_class::PUSH_ACTION, date_from: time - 5.days, date_to: time - 2.days)).to eq(1) - end - end - end - end - - context 'when tracking unsuccessfully' do - using RSpec::Parameterized::TableSyntax - - where(:target, :action) do - Project | :invalid_action - :invalid_target | :pushed - Project | :created - end - - with_them do - it 'returns the expected values' do - expect(track_event(event_action: action, event_target: target, author_id: 2)).to be_nil - expect(count_unique(event_action: described_class::PUSH_ACTION, date_from: time, date_to: Date.today)).to eq(0) - end - end - end - end -end diff --git a/spec/lib/gitlab/usage_data_metrics_spec.rb b/spec/lib/gitlab/usage_data_metrics_spec.rb index 6391b003096..1f52819fd9e 100644 --- a/spec/lib/gitlab/usage_data_metrics_spec.rb +++ b/spec/lib/gitlab/usage_data_metrics_spec.rb @@ -85,16 +85,4 @@ RSpec.describe Gitlab::UsageDataMetrics, :with_license, feature_category: :servi end end end - - describe '.suggested_names' do - subject { described_class.suggested_names } - - let(:suggested_names) do - ::Gitlab::Usage::Metric.all.map(&:with_suggested_name).reduce({}, :deep_merge) - end - - it 'includes Service Ping suggested names' do - expect(subject).to match_array(suggested_names) - end - end end diff --git a/spec/lib/gitlab/usage_data_spec.rb b/spec/lib/gitlab/usage_data_spec.rb index 5325ef5b5dd..4544cb2eb26 100644 --- a/spec/lib/gitlab/usage_data_spec.rb +++ b/spec/lib/gitlab/usage_data_spec.rb @@ -29,10 +29,8 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures, feature_category: :servic .to include(:configure, :create, :manage, :monitor, :plan, :release, :verify) expect(subject[:usage_activity_by_stage_monthly]) .to include(:configure, :create, :manage, :monitor, :plan, :release, :verify) - expect(subject[:usage_activity_by_stage][:create]) - .not_to include(:merge_requests_users) expect(subject[:usage_activity_by_stage_monthly][:create]) - .to include(:merge_requests_users) + .to include(:snippets) end it 'clears memoized values' do @@ -265,7 +263,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures, feature_category: :servic for_defined_days_back do user = create(:user) - %w(gitlab_project gitlab github bitbucket bitbucket_server gitea git manifest fogbugz phabricator).each do |type| + %w(gitlab_project github bitbucket bitbucket_server gitea git manifest fogbugz).each do |type| create(:project, import_type: type, creator_id: user.id) end @@ -294,16 +292,14 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures, feature_category: :servic git: 2, gitea: 2, github: 2, - gitlab: 2, gitlab_migration: 2, gitlab_project: 2, manifest: 2, - total: 18 + total: 16 }, issue_imports: { jira: 2, fogbugz: 2, - phabricator: 2, csv: 2 }, group_imports: { @@ -323,16 +319,14 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures, feature_category: :servic git: 1, gitea: 1, github: 1, - gitlab: 1, gitlab_migration: 1, gitlab_project: 1, manifest: 1, - total: 9 + total: 8 }, issue_imports: { jira: 1, fogbugz: 1, - phabricator: 1, csv: 1 }, group_imports: { @@ -529,8 +523,6 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures, feature_category: :servic expect(count_data[:projects_prometheus_active]).to eq(1) expect(count_data[:projects_jenkins_active]).to eq(1) expect(count_data[:projects_jira_active]).to eq(4) - expect(count_data[:projects_jira_server_active]).to eq(2) - expect(count_data[:projects_jira_cloud_active]).to eq(2) expect(count_data[:jira_imports_projects_count]).to eq(2) expect(count_data[:jira_imports_total_imported_count]).to eq(3) expect(count_data[:jira_imports_total_imported_issues_count]).to eq(13) @@ -614,14 +606,6 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures, feature_category: :servic it 'raises an error' do expect { subject }.to raise_error(ActiveRecord::StatementInvalid) end - - context 'when metric calls find_in_batches' do - let(:metric_method) { :find_in_batches } - - it 'raises an error for jira_usage' do - expect { described_class.jira_usage }.to raise_error(ActiveRecord::StatementInvalid) - end - end end context 'with should_raise_for_dev? false' do @@ -630,14 +614,6 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures, feature_category: :servic it 'does not raise an error' do expect { subject }.not_to raise_error end - - context 'when metric calls find_in_batches' do - let(:metric_method) { :find_in_batches } - - it 'does not raise an error for jira_usage' do - expect { described_class.jira_usage }.not_to raise_error - end - end end end @@ -663,8 +639,6 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures, feature_category: :servic create(:alert_management_alert, project: project, created_at: n.days.ago) end - stub_application_setting(self_monitoring_project: project) - for_defined_days_back do create(:product_analytics_event, project: project, se_category: 'epics', se_action: 'promote') end @@ -687,37 +661,10 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures, feature_category: :servic end end - describe '.runners_usage' do - before do - project = build(:project) - create_list(:ci_runner, 2, :instance_type, :online) - create(:ci_runner, :group, :online) - create(:ci_runner, :group, :inactive) - create_list(:ci_runner, 3, :project_type, :online, projects: [project]) - end - - subject { described_class.runners_usage } - - it 'gathers runner usage counts correctly' do - expect(subject[:ci_runners]).to eq(7) - expect(subject[:ci_runners_instance_type_active]).to eq(2) - expect(subject[:ci_runners_group_type_active]).to eq(1) - expect(subject[:ci_runners_project_type_active]).to eq(3) - - expect(subject[:ci_runners_instance_type_active_online]).to eq(2) - expect(subject[:ci_runners_group_type_active_online]).to eq(1) - expect(subject[:ci_runners_project_type_active_online]).to eq(3) - end - end - describe '.license_usage_data' do subject { described_class.license_usage_data } it 'gathers license data' do - expect(subject[:uuid]).to eq(Gitlab::CurrentSettings.uuid) - expect(subject[:version]).to eq(Gitlab::VERSION) - expect(subject[:installation_type]).to eq('gitlab-development-kit') - expect(subject[:active_user_count]).to eq(User.active.size) expect(subject[:recorded_at]).to be_a(Time) end end @@ -733,7 +680,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures, feature_category: :servic expect(subject[:ldap_enabled]).to eq(Gitlab.config.ldap.enabled) expect(subject[:gravatar_enabled]).to eq(Gitlab::CurrentSettings.gravatar_enabled?) expect(subject[:omniauth_enabled]).to eq(Gitlab::Auth.omniauth_enabled?) - expect(subject[:reply_by_email_enabled]).to eq(Gitlab::IncomingEmail.enabled?) + expect(subject[:reply_by_email_enabled]).to eq(Gitlab::Email::IncomingEmail.enabled?) expect(subject[:container_registry_enabled]).to eq(Gitlab.config.registry.enabled) expect(subject[:dependency_proxy_enabled]).to eq(Gitlab.config.dependency_proxy.enabled) expect(subject[:gitlab_shared_runners_enabled]).to eq(Gitlab.config.gitlab_ci.shared_runners_enabled) @@ -1039,28 +986,6 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures, feature_category: :servic end end - describe '.merge_requests_users', :clean_gitlab_redis_shared_state do - let(:time_period) { { created_at: 2.days.ago..time } } - let(:time) { Time.current } - - before do - counter = Gitlab::UsageDataCounters::TrackUniqueEvents - merge_request = Event::TARGET_TYPES[:merge_request] - design = Event::TARGET_TYPES[:design] - - counter.track_event(event_action: :commented, event_target: merge_request, author_id: 1, time: time) - counter.track_event(event_action: :opened, event_target: merge_request, author_id: 1, time: time) - counter.track_event(event_action: :merged, event_target: merge_request, author_id: 2, time: time) - counter.track_event(event_action: :closed, event_target: merge_request, author_id: 3, time: time) - counter.track_event(event_action: :opened, event_target: merge_request, author_id: 4, time: time - 3.days) - counter.track_event(event_action: :created, event_target: design, author_id: 5, time: time) - end - - it 'returns the distinct count of users using merge requests (via events table) within the specified time period' do - expect(described_class.merge_requests_users(time_period)).to eq(3) - end - end - def for_defined_days_back(days: [31, 3]) days.each do |n| travel_to(n.days.ago) do @@ -1069,42 +994,6 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures, feature_category: :servic end end - describe '#action_monthly_active_users', :clean_gitlab_redis_shared_state do - let(:time_period) { { created_at: 2.days.ago..time } } - let(:time) { Time.zone.now } - let(:user1) { build(:user, id: 1) } - let(:user2) { build(:user, id: 2) } - let(:user3) { build(:user, id: 3) } - let(:user4) { build(:user, id: 4) } - let(:project) { build(:project) } - - before do - counter = Gitlab::UsageDataCounters::EditorUniqueCounter - - counter.track_web_ide_edit_action(author: user1, project: project) - counter.track_web_ide_edit_action(author: user1, project: project) - counter.track_sfe_edit_action(author: user1, project: project) - counter.track_snippet_editor_edit_action(author: user1, project: project) - counter.track_snippet_editor_edit_action(author: user1, time: time - 3.days, project: project) - - counter.track_web_ide_edit_action(author: user2, project: project) - counter.track_sfe_edit_action(author: user2, project: project) - - counter.track_web_ide_edit_action(author: user3, time: time - 3.days, project: project) - counter.track_snippet_editor_edit_action(author: user3, project: project) - end - - it 'returns the distinct count of user actions within the specified time period' do - expect(described_class.action_monthly_active_users(time_period)).to eq( - { - action_monthly_active_users_web_ide_edit: 2, - action_monthly_active_users_sfe_edit: 2, - action_monthly_active_users_snippet_editor_edit: 2 - } - ) - end - end - describe '.service_desk_counts' do subject { described_class.send(:service_desk_counts) } @@ -1125,7 +1014,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures, feature_category: :servic expect(result.duration).to be_an(Float) end - it 'records error and returns nil', :aggregated_errors do + it 'records error and returns nil', :aggregate_failures do allow(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception) result = described_class.with_metadata { raise } diff --git a/spec/lib/gitlab/user_access_spec.rb b/spec/lib/gitlab/user_access_spec.rb index 1ae45d41f2d..7d09330d185 100644 --- a/spec/lib/gitlab/user_access_spec.rb +++ b/spec/lib/gitlab/user_access_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::UserAccess do +RSpec.describe Gitlab::UserAccess, feature_category: :system_access do include ProjectForksHelper let(:access) { described_class.new(user, container: project) } @@ -85,10 +85,10 @@ RSpec.describe Gitlab::UserAccess do let(:not_existing_branch) { create :protected_branch, :developers_can_merge, project: project } context 'when admin mode is enabled', :enable_admin_mode do - it 'returns true for admins' do + it 'returns false for admins' do user.update!(admin: true) - expect(access.can_push_to_branch?(branch.name)).to be_truthy + expect(access.can_push_to_branch?(branch.name)).to be_falsey end end diff --git a/spec/lib/gitlab/utils/email_spec.rb b/spec/lib/gitlab/utils/email_spec.rb index d7a881d8655..c81c2558f70 100644 --- a/spec/lib/gitlab/utils/email_spec.rb +++ b/spec/lib/gitlab/utils/email_spec.rb @@ -8,13 +8,20 @@ RSpec.describe Gitlab::Utils::Email, feature_category: :service_desk do describe '.obfuscated_email' do where(:input, :output) do - 'alex@gitlab.com' | 'al**@g*****.com' - 'alex@gl.co.uk' | 'al**@g****.uk' - 'a@b.c' | 'a@b.c' - 'q@example.com' | 'q@e******.com' - 'q@w.' | 'q@w.' - 'a@b' | 'a@b' - 'no mail' | 'no mail' + 'alex@gitlab.com' | 'al**@g*****.com' + 'alex@gl.co.uk' | 'al**@g****.uk' + 'a@b.c' | 'aa@b.c' + 'qqwweerrttyy@example.com' | 'qq**********@e******.com' + 'getsuperfancysupport@paywhatyouwant.accounting' | 'ge******************@p*************.accounting' + 'q@example.com' | 'qq@e******.com' + 'q@w.' | 'qq@w.' + 'a@b' | 'aa@b' + 'trun"@"e@example.com' | 'tr******@e******.com' + '@' | '@' + 'n' | 'n' + 'no mail' | 'n******' + 'truncated@exa' | 'tr*******@exa' + '' | '' end with_them do @@ -29,9 +36,14 @@ RSpec.describe Gitlab::Utils::Email, feature_category: :service_desk do 'qqwweerrttyy@example.com' | 'qq*****@e*****.c**' 'getsuperfancysupport@paywhatyouwant.accounting' | 'ge*****@p*****.a**' 'q@example.com' | 'qq*****@e*****.c**' - 'q@w.' | 'q@w.' - 'a@b' | 'a@b' - 'no mail' | 'no mail' + 'q@w.' | 'qq*****@w*****.' + 'a@b' | 'aa*****@b**' + 'trun"@"e@example.com' | 'tr*****@e*****.c**' + '@' | '@' + 'no mail' | 'n**' + 'n' | 'n**' + 'truncated@exa' | 'tr*****@e**' + '' | '' end with_them do diff --git a/spec/lib/gitlab/utils/error_message_spec.rb b/spec/lib/gitlab/utils/error_message_spec.rb new file mode 100644 index 00000000000..a6de2520c5e --- /dev/null +++ b/spec/lib/gitlab/utils/error_message_spec.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +RSpec.describe Gitlab::Utils::ErrorMessage, feature_category: :error_tracking do + let(:klass) do + Class.new do + include Gitlab::Utils::ErrorMessage + end + end + + let(:message) { 'Something went wrong' } + + subject(:object) { klass.new } + + describe '#to_user_facing' do + it 'returns a user-facing error message with the UF prefix' do + expect(described_class.to_user_facing(message)).to eq("UF #{message}") + end + end + + describe '#prefixed_error_message' do + it 'returns a message with the given prefix' do + prefix = 'ERROR' + expect(described_class.prefixed_error_message(message, prefix)).to eq("#{prefix} #{message}") + end + end +end diff --git a/spec/lib/gitlab/utils/measuring_spec.rb b/spec/lib/gitlab/utils/measuring_spec.rb index 5dad79b1c5f..4d2791f771f 100644 --- a/spec/lib/gitlab/utils/measuring_spec.rb +++ b/spec/lib/gitlab/utils/measuring_spec.rb @@ -17,7 +17,7 @@ RSpec.describe Gitlab::Utils::Measuring do measurement.with_measuring { result } end - it 'measures and logs data', :aggregate_failure do + it 'measures and logs data', :aggregate_failures do expect(measurement).to receive(:with_measure_time).and_call_original expect(measurement).to receive(:with_count_queries).and_call_original expect(measurement).to receive(:with_gc_stats).and_call_original diff --git a/spec/lib/gitlab/utils/nokogiri_spec.rb b/spec/lib/gitlab/utils/nokogiri_spec.rb index 7b4c63f9168..10f34ca706c 100644 --- a/spec/lib/gitlab/utils/nokogiri_spec.rb +++ b/spec/lib/gitlab/utils/nokogiri_spec.rb @@ -17,8 +17,8 @@ RSpec.describe Gitlab::Utils::Nokogiri do '.js-render-metrics' | "descendant-or-self::*[contains(concat(' ',normalize-space(@class),' '),' js-render-metrics ')]" 'h1, h2, h3, h4, h5, h6' | "descendant-or-self::h1|descendant-or-self::h2|descendant-or-self::h3|descendant-or-self::h4|descendant-or-self::h5|descendant-or-self::h6" 'pre.code.language-math' | "descendant-or-self::pre[contains(concat(' ',normalize-space(@class),' '),' code ') and contains(concat(' ',normalize-space(@class),' '),' language-math ')]" - 'pre > code[lang="plantuml"]' | "descendant-or-self::pre/code[@lang=\"plantuml\"]" - 'pre[lang="mermaid"] > code' | "descendant-or-self::pre[@lang=\"mermaid\"]/code" + 'pre > code[data-canonical-lang="plantuml"]' | "descendant-or-self::pre/code[@data-canonical-lang=\"plantuml\"]" + 'pre[data-canonical-lang="mermaid"] > code' | "descendant-or-self::pre[@data-canonical-lang=\"mermaid\"]/code" 'pre.language-suggestion' | "descendant-or-self::pre[contains(concat(' ',normalize-space(@class),' '),' language-suggestion ')]" 'pre.language-suggestion > code' | "descendant-or-self::pre[contains(concat(' ',normalize-space(@class),' '),' language-suggestion ')]/code" 'a.gfm[data-reference-type="user"]' | "descendant-or-self::a[contains(concat(' ',normalize-space(@class),' '),' gfm ') and @data-reference-type=\"user\"]" diff --git a/spec/lib/gitlab/utils/strong_memoize_spec.rb b/spec/lib/gitlab/utils/strong_memoize_spec.rb index 71f2502b91c..ea8083e7d7f 100644 --- a/spec/lib/gitlab/utils/strong_memoize_spec.rb +++ b/spec/lib/gitlab/utils/strong_memoize_spec.rb @@ -3,12 +3,15 @@ require 'fast_spec_helper' require 'rspec-benchmark' require 'rspec-parameterized' +require 'active_support/testing/time_helpers' RSpec.configure do |config| config.include RSpec::Benchmark::Matchers end -RSpec.describe Gitlab::Utils::StrongMemoize, feature_category: :not_owned do +RSpec.describe Gitlab::Utils::StrongMemoize, feature_category: :shared do + include ActiveSupport::Testing::TimeHelpers + let(:klass) do strong_memoize_class = described_class @@ -30,6 +33,13 @@ RSpec.describe Gitlab::Utils::StrongMemoize, feature_category: :not_owned do end end + def method_name_with_expiration + strong_memoize_with_expiration(:method_name_with_expiration, 1) do + trace << value + value + end + end + def method_name_attr trace << value value @@ -142,6 +152,43 @@ RSpec.describe Gitlab::Utils::StrongMemoize, feature_category: :not_owned do end end + describe '#strong_memoize_with_expiration' do + [nil, false, true, 'value', 0, [0]].each do |value| + context "with value #{value}" do + let(:value) { value } + let(:method_name) { :method_name_with_expiration } + + it_behaves_like 'caching the value' + + it 'raises exception for invalid type as key' do + expect { object.strong_memoize_with_expiration(10, 1) { 20 } }.to raise_error /Invalid type of '10'/ + end + + it 'raises exception for invalid characters in key' do + expect { object.strong_memoize_with_expiration(:enabled?, 1) { 20 } } + .to raise_error /is not allowed as an instance variable name/ + end + end + end + + context 'value memoization test' do + let(:value) { 'value' } + + it 'caches the value for specified number of seconds' do + object.method_name_with_expiration + object.method_name_with_expiration + + expect(object.trace.count).to eq(1) + + travel_to(Time.current + 2.seconds) do + object.method_name_with_expiration + + expect(object.trace.count).to eq(2) + end + end + end + end + describe '#strong_memoize_with' do [nil, false, true, 'value', 0, [0]].each do |value| context "with value #{value}" do @@ -215,19 +262,21 @@ RSpec.describe Gitlab::Utils::StrongMemoize, feature_category: :not_owned do describe '.strong_memoize_attr' do [nil, false, true, 'value', 0, [0]].each do |value| - let(:value) { value } + context "with value '#{value}'" do + let(:value) { value } - context "memoized after method definition with value #{value}" do - let(:method_name) { :method_name_attr } + context 'memoized after method definition' do + let(:method_name) { :method_name_attr } - it_behaves_like 'caching the value' + it_behaves_like 'caching the value' - it 'calls the existing .method_added' do - expect(klass.method_added_list).to include(:method_name_attr) - end + it 'calls the existing .method_added' do + expect(klass.method_added_list).to include(:method_name_attr) + end - it 'retains method arity' do - expect(klass.instance_method(method_name).arity).to eq(0) + it 'retains method arity' do + expect(klass.instance_method(method_name).arity).to eq(0) + end end end end diff --git a/spec/lib/gitlab/utils/uniquify_spec.rb b/spec/lib/gitlab/utils/uniquify_spec.rb new file mode 100644 index 00000000000..df02fbe8c82 --- /dev/null +++ b/spec/lib/gitlab/utils/uniquify_spec.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +RSpec.describe Gitlab::Utils::Uniquify, feature_category: :shared do + subject(:uniquify) { described_class.new } + + describe "#string" do + it 'returns the given string if it does not exist' do + result = uniquify.string('test_string') { |_s| false } + + expect(result).to eq('test_string') + end + + it 'returns the given string with a counter attached if the string exists' do + result = uniquify.string('test_string') { |s| s == 'test_string' } + + expect(result).to eq('test_string1') + end + + it 'increments the counter for each candidate string that also exists' do + result = uniquify.string('test_string') { |s| s == 'test_string' || s == 'test_string1' } + + expect(result).to eq('test_string2') + end + + it 'allows to pass an initial value for the counter' do + start_counting_from = 2 + uniquify = described_class.new(start_counting_from) + + result = uniquify.string('test_string') { |s| s == 'test_string' } + + expect(result).to eq('test_string2') + end + + it 'allows passing in a base function that defines the location of the counter' do + result = uniquify.string(->(counter) { "test_#{counter}_string" }) do |s| + s == 'test__string' + end + + expect(result).to eq('test_1_string') + end + end +end diff --git a/spec/lib/gitlab/utils/usage_data_spec.rb b/spec/lib/gitlab/utils/usage_data_spec.rb index 2925ceef256..586ee04a835 100644 --- a/spec/lib/gitlab/utils/usage_data_spec.rb +++ b/spec/lib/gitlab/utils/usage_data_spec.rb @@ -487,12 +487,12 @@ RSpec.describe Gitlab::Utils::UsageData do end context 'when Redis HLL raises any error' do - subject { described_class.redis_usage_data { raise Gitlab::UsageDataCounters::HLLRedisCounter::CategoryMismatch } } + subject { described_class.redis_usage_data { raise Gitlab::UsageDataCounters::HLLRedisCounter::EventError } } let(:fallback) { 15 } let(:failing_class) { nil } - it_behaves_like 'failing hardening method', Gitlab::UsageDataCounters::HLLRedisCounter::CategoryMismatch + it_behaves_like 'failing hardening method', Gitlab::UsageDataCounters::HLLRedisCounter::EventError end it 'returns the evaluated block when given' do diff --git a/spec/lib/gitlab/utils/username_and_email_generator_spec.rb b/spec/lib/gitlab/utils/username_and_email_generator_spec.rb new file mode 100644 index 00000000000..45df8f08055 --- /dev/null +++ b/spec/lib/gitlab/utils/username_and_email_generator_spec.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Utils::UsernameAndEmailGenerator, feature_category: :system_access do + let(:username_prefix) { 'username_prefix' } + let(:email_domain) { 'example.com' } + + subject { described_class.new(username_prefix: username_prefix, email_domain: email_domain) } + + describe 'email domain' do + it 'defaults to `Gitlab.config.gitlab.host`' do + expect(described_class.new(username_prefix: username_prefix).email).to end_with("@#{Gitlab.config.gitlab.host}") + end + + context 'when specified' do + it 'uses the specified email domain' do + expect(subject.email).to end_with("@#{email_domain}") + end + end + end + + include_examples 'username and email pair is generated by Gitlab::Utils::UsernameAndEmailGenerator' +end diff --git a/spec/lib/gitlab/workhorse_spec.rb b/spec/lib/gitlab/workhorse_spec.rb index 3c7542ea5f9..a1c2f7d667f 100644 --- a/spec/lib/gitlab/workhorse_spec.rb +++ b/spec/lib/gitlab/workhorse_spec.rb @@ -458,18 +458,42 @@ RSpec.describe Gitlab::Workhorse do describe '.send_url' do let(:url) { 'http://example.com' } - subject { described_class.send_url(url) } - it 'sets the header correctly' do - key, command, params = decode_workhorse_header(subject) + key, command, params = decode_workhorse_header( + described_class.send_url(url) + ) expect(key).to eq("Gitlab-Workhorse-Send-Data") expect(command).to eq("send-url") expect(params).to eq({ 'URL' => url, - 'AllowRedirects' => false + 'AllowRedirects' => false, + 'Body' => '', + 'Method' => 'GET' }.deep_stringify_keys) end + + context 'when body, headers and method are specified' do + let(:body) { 'body' } + let(:headers) { { Authorization: ['Bearer token'] } } + let(:method) { 'POST' } + + it 'sets the header correctly' do + key, command, params = decode_workhorse_header( + described_class.send_url(url, body: body, headers: headers, method: method) + ) + + expect(key).to eq("Gitlab-Workhorse-Send-Data") + expect(command).to eq("send-url") + expect(params).to eq({ + 'URL' => url, + 'AllowRedirects' => false, + 'Body' => body, + 'Header' => headers, + 'Method' => method + }.deep_stringify_keys) + end + end end describe '.send_scaled_image' do |